graphframes - Details for Job 12

Details for Job 12

Status: SUCCEEDED
Completed Stages: 6

Enable zooming

digraph G { subgraph clusterstage_26 { label="Stage 26"; 51 [labelType="html" label="PythonRDD [51] RDD at PythonRDD.scala:53"]; 130 [labelType="html" label="SQLExecutionRDD [130] rdd at GraphFrame.scala:182"]; subgraph cluster87 { label="parallelize"; 50 [labelType="html" label="ParallelCollectionRDD [50] readRDDFromFile at PythonRDD.scala:262"]; } subgraph cluster88 { label="mapPartitions"; 52 [labelType="html" label="MapPartitionsRDD [52] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster89 { label="map"; 53 [labelType="html" label="MapPartitionsRDD [53] map at SerDeUtil.scala:137"]; } subgraph cluster90 { label="mapPartitions"; 54 [labelType="html" label="MapPartitionsRDD [54] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster301 { label="WholeStageCodegen (1)"; 128 [labelType="html" label="MapPartitionsRDD [128] rdd at GraphFrame.scala:182"]; } subgraph cluster300 { label="DeserializeToObject"; 129 [labelType="html" label="MapPartitionsRDD [129] rdd at GraphFrame.scala:182"]; } subgraph cluster304 { label="mapPartitions"; 131 [labelType="html" label="MapPartitionsRDD [131] rdd at GraphFrame.scala:182"]; } subgraph cluster305 { label="map"; 132 [labelType="html" label="MapPartitionsRDD [132] map at GraphFrame.scala:183"]; } subgraph cluster306 { label="mapPartitionsWithIndex"; 133 [labelType="html" label="EdgeRDD, EdgeRDD [133] mapPartitionsWithIndex at EdgeRDD.scala:105"]; } subgraph cluster308 { label="mapPartitions"; 137 [labelType="html" label="VertexRDD.createRoutingTables - vid2pid (aggregation) [137] mapPartitions at VertexRDD.scala:356"]; } } 50->51; 51->52; 52->53; 53->54; 54->128; 128->129; 129->130; 130->131; 131->132; 132->133; 133->137; }

digraph G { subgraph clusterstage_27 { label="Stage 27"; 32 [labelType="html" label="PythonRDD [32] RDD at PythonRDD.scala:53"]; 125 [labelType="html" label="SQLExecutionRDD [125] rdd at GraphFrame.scala:180"]; subgraph cluster60 { label="parallelize"; 31 [labelType="html" label="ParallelCollectionRDD [31] readRDDFromFile at PythonRDD.scala:262"]; } subgraph cluster61 { label="mapPartitions"; 33 [labelType="html" label="MapPartitionsRDD [33] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster62 { label="map"; 34 [labelType="html" label="MapPartitionsRDD [34] map at SerDeUtil.scala:137"]; } subgraph cluster63 { label="mapPartitions"; 35 [labelType="html" label="MapPartitionsRDD [35] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster295 { label="WholeStageCodegen (1)"; 123 [labelType="html" label="MapPartitionsRDD [123] rdd at GraphFrame.scala:180"]; } subgraph cluster294 { label="DeserializeToObject"; 124 [labelType="html" label="MapPartitionsRDD [124] rdd at GraphFrame.scala:180"]; } subgraph cluster298 { label="mapPartitions"; 126 [labelType="html" label="MapPartitionsRDD [126] rdd at GraphFrame.scala:180"]; } subgraph cluster299 { label="map"; 127 [labelType="html" label="MapPartitionsRDD [127] map at GraphFrame.scala:181"]; } } 31->32; 32->33; 33->34; 34->35; 35->123; 123->124; 124->125; 125->126; 126->127; }

digraph G { subgraph clusterstage_28 { label="Stage 28"; subgraph cluster307 { label="partitionBy"; 136 [labelType="html" label="ShuffledRDD [136] partitionBy at VertexRDD.scala:319"]; } subgraph cluster309 { label="partitionBy"; 138 [labelType="html" label="ShuffledRDD [138] partitionBy at VertexRDD.scala:361"]; } subgraph cluster310 { label="mapPartitions"; 139 [labelType="html" label="MapPartitionsRDD [139] mapPartitions at VertexRDD.scala:361"]; } subgraph cluster311 { label="zipPartitions"; 140 [labelType="html" label="VertexRDD, VertexRDD [140] [Cached] zipPartitions at VertexRDD.scala:322"]; } subgraph cluster313 { label="mapPartitions"; 145 [labelType="html" label="VertexRDD [145] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster316 { label="mapPartitions"; 151 [labelType="html" label="VertexRDD [151] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster318 { label="mapPartitions"; 155 [labelType="html" label="VertexRDD [155] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster323 { label="mapPartitions"; 163 [labelType="html" label="ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) [163] mapPartitions at VertexRDDImpl.scala:247"]; } } 138->139; 136->140; 139->140; 140->145; 145->151; 151->155; 155->163; }

127,136

137,138

140

145

151

155

digraph G { subgraph clusterstage_29 { label="Stage 29"; subgraph cluster307 { label="partitionBy"; 136 [labelType="html" label="ShuffledRDD [136] partitionBy at VertexRDD.scala:319"]; } subgraph cluster309 { label="partitionBy"; 138 [labelType="html" label="ShuffledRDD [138] partitionBy at VertexRDD.scala:361"]; } subgraph cluster310 { label="mapPartitions"; 139 [labelType="html" label="MapPartitionsRDD [139] mapPartitions at VertexRDD.scala:361"]; } subgraph cluster311 { label="zipPartitions"; 140 [labelType="html" label="VertexRDD, VertexRDD [140] [Cached] zipPartitions at VertexRDD.scala:322"]; } subgraph cluster313 { label="mapPartitions"; 145 [labelType="html" label="VertexRDD [145] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster316 { label="mapPartitions"; 151 [labelType="html" label="VertexRDD [151] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster318 { label="mapPartitions"; 155 [labelType="html" label="VertexRDD [155] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster319 { label="zipPartitions"; 157 [labelType="html" label="VertexRDD [157] zipPartitions at VertexRDDImpl.scala:144"]; } subgraph cluster320 { label="mapPartitions"; 159 [labelType="html" label="ReplicatedVertexView.updateVertices - shippedVerts false false (broadcast) [159] mapPartitions at VertexRDDImpl.scala:247"]; } } 138->139; 136->140; 139->140; 140->145; 145->151; 151->155; 151->157; 155->157; 157->159; }

127,136

137,138

140

145

151

155

digraph G { subgraph clusterstage_30 { label="Stage 30"; 51 [labelType="html" label="PythonRDD [51] RDD at PythonRDD.scala:53"]; 130 [labelType="html" label="SQLExecutionRDD [130] rdd at GraphFrame.scala:182"]; subgraph cluster87 { label="parallelize"; 50 [labelType="html" label="ParallelCollectionRDD [50] readRDDFromFile at PythonRDD.scala:262"]; } subgraph cluster88 { label="mapPartitions"; 52 [labelType="html" label="MapPartitionsRDD [52] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster89 { label="map"; 53 [labelType="html" label="MapPartitionsRDD [53] map at SerDeUtil.scala:137"]; } subgraph cluster90 { label="mapPartitions"; 54 [labelType="html" label="MapPartitionsRDD [54] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster301 { label="WholeStageCodegen (1)"; 128 [labelType="html" label="MapPartitionsRDD [128] rdd at GraphFrame.scala:182"]; } subgraph cluster300 { label="DeserializeToObject"; 129 [labelType="html" label="MapPartitionsRDD [129] rdd at GraphFrame.scala:182"]; } subgraph cluster304 { label="mapPartitions"; 131 [labelType="html" label="MapPartitionsRDD [131] rdd at GraphFrame.scala:182"]; } subgraph cluster305 { label="map"; 132 [labelType="html" label="MapPartitionsRDD [132] map at GraphFrame.scala:183"]; } subgraph cluster306 { label="mapPartitionsWithIndex"; 133 [labelType="html" label="EdgeRDD, EdgeRDD [133] mapPartitionsWithIndex at EdgeRDD.scala:105"]; } subgraph cluster312 { label="mapPartitions"; 143 [labelType="html" label="EdgeRDD [143] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster314 { label="mapPartitions"; 147 [labelType="html" label="EdgeRDD [147] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster315 { label="mapPartitions"; 149 [labelType="html" label="EdgeRDD [149] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster317 { label="mapPartitions"; 153 [labelType="html" label="EdgeRDD [153] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster321 { label="partitionBy"; 160 [labelType="html" label="ShuffledRDD [160] partitionBy at ReplicatedVertexView.scala:110"]; } subgraph cluster322 { label="zipPartitions"; 161 [labelType="html" label="EdgeRDD [161] [Cached] zipPartitions at ReplicatedVertexView.scala:112"]; } subgraph cluster324 { label="partitionBy"; 164 [labelType="html" label="ShuffledRDD [164] partitionBy at ReplicatedVertexView.scala:69"]; } subgraph cluster325 { label="zipPartitions"; 165 [labelType="html" label="EdgeRDD [165] zipPartitions at ReplicatedVertexView.scala:70"]; } subgraph cluster326 { label="mapPartitions"; 167 [labelType="html" label="GraphImpl.aggregateMessages - preAgg [167] mapPartitions at GraphImpl.scala:208"]; } } 50->51; 51->52; 52->53; 53->54; 54->128; 128->129; 129->130; 130->131; 131->132; 132->133; 133->143; 143->147; 147->149; 149->153; 153->161; 160->161; 161->165; 164->165; 165->167; }

159,160

163,164

143

147

153

161

digraph G { subgraph clusterstage_31 { label="Stage 31"; subgraph cluster307 { label="partitionBy"; 136 [labelType="html" label="ShuffledRDD [136] partitionBy at VertexRDD.scala:319"]; } subgraph cluster309 { label="partitionBy"; 138 [labelType="html" label="ShuffledRDD [138] partitionBy at VertexRDD.scala:361"]; } subgraph cluster310 { label="mapPartitions"; 139 [labelType="html" label="MapPartitionsRDD [139] mapPartitions at VertexRDD.scala:361"]; } subgraph cluster311 { label="zipPartitions"; 140 [labelType="html" label="VertexRDD, VertexRDD [140] [Cached] zipPartitions at VertexRDD.scala:322"]; } subgraph cluster313 { label="mapPartitions"; 145 [labelType="html" label="VertexRDD [145] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster316 { label="mapPartitions"; 151 [labelType="html" label="VertexRDD [151] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster318 { label="mapPartitions"; 155 [labelType="html" label="VertexRDD [155] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster327 { label="partitionBy"; 168 [labelType="html" label="ShuffledRDD [168] partitionBy at VertexRDDImpl.scala:214"]; } subgraph cluster328 { label="zipPartitions"; 169 [labelType="html" label="VertexRDD [169] [Cached] zipPartitions at VertexRDDImpl.scala:215"]; } subgraph cluster329 { label="map"; 171 [labelType="html" label="MapPartitionsRDD [171] map at VertexRDDImpl.scala:90"]; } } 138->139; 136->140; 139->140; 140->145; 145->151; 151->155; 155->169; 168->169; 169->171; }

127,136

137,138

167,168

140

145

151

155

169

Completed Stages (6)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
31	fold at VertexRDDImpl.scala:90 +details org.apache.spark.rdd.RDD.fold(RDD.scala:1151) org.apache.spark.graphx.impl.VertexRDDImpl.count(VertexRDDImpl.scala:90) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:140) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:28:08	36 ms	16/16	112.6 KiB	31.6 KiB
30	mapPartitions at GraphImpl.scala:208 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.GraphImpl.aggregateMessagesWithActiveSet(GraphImpl.scala:208) org.apache.spark.graphx.GraphXUtils$.mapReduceTriplets(GraphXUtils.scala:69) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:136) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:28:07	0.5 s	16/16		174.2 KiB	31.6 KiB
29	mapPartitions at VertexRDDImpl.scala:247 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.VertexRDDImpl.shipVertexAttributes(VertexRDDImpl.scala:247) org.apache.spark.graphx.impl.ReplicatedVertexView.updateVertices(ReplicatedVertexView.scala:107) org.apache.spark.graphx.impl.GraphImpl.mapVertices(GraphImpl.scala:133) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:130) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:28:07	0.1 s	16/16	121.5 KiB		71.7 KiB
28	mapPartitions at VertexRDDImpl.scala:247 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.VertexRDDImpl.shipVertexAttributes(VertexRDDImpl.scala:247) org.apache.spark.graphx.impl.ReplicatedVertexView.upgrade(ReplicatedVertexView.scala:66) org.apache.spark.graphx.impl.GraphImpl.aggregateMessagesWithActiveSet(GraphImpl.scala:198) org.apache.spark.graphx.GraphXUtils$.mapReduceTriplets(GraphXUtils.scala:69) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:136) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:28:07	0.2 s	16/16		248.3 KiB	102.5 KiB
27	map at GraphFrame.scala:181 +details org.apache.spark.rdd.RDD.map(RDD.scala:395) org.graphframes.GraphFrame.toGraphX(GraphFrame.scala:181) org.graphframes.GraphFrame.cachedGraphX$lzycompute(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedGraphX(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedTopologyGraphX$lzycompute(GraphFrame.scala:592) org.graphframes.GraphFrame.cachedTopologyGraphX(GraphFrame.scala:591) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:71) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:28:06	0.1 s	16/16			217.5 KiB
26	mapPartitions at VertexRDD.scala:356 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.VertexRDD$.createRoutingTables(VertexRDD.scala:356) org.apache.spark.graphx.VertexRDD$.apply(VertexRDD.scala:321) org.apache.spark.graphx.VertexRDD$.apply(VertexRDD.scala:299) org.apache.spark.graphx.impl.GraphImpl$.apply(GraphImpl.scala:315) org.apache.spark.graphx.Graph$.apply(Graph.scala:531) org.graphframes.GraphFrame.toGraphX(GraphFrame.scala:184) org.graphframes.GraphFrame.cachedGraphX$lzycompute(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedGraphX(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedTopologyGraphX$lzycompute(GraphFrame.scala:592) org.graphframes.GraphFrame.cachedTopologyGraphX(GraphFrame.scala:591) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:71) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282)	2024/12/27 14:28:06	0.9 s	16/16			30.8 KiB

Page: