graphframes - Details for Job 18

Details for Job 18

Status: SUCCEEDED
Completed Stages: 6

Enable zooming

digraph G { subgraph clusterstage_20 { label="Stage 20"; 84 [labelType="html" label="SQLExecutionRDD [84] rdd at GraphFrame.scala:180"]; subgraph cluster204 { label="Scan csv "; 80 [labelType="html" label="FileScanRDD [80] rdd at GraphFrame.scala:180"]; 81 [labelType="html" label="MapPartitionsRDD [81] rdd at GraphFrame.scala:180"]; } subgraph cluster201 { label="WholeStageCodegen (1)"; 82 [labelType="html" label="MapPartitionsRDD [82] rdd at GraphFrame.scala:180"]; } subgraph cluster200 { label="DeserializeToObject"; 83 [labelType="html" label="MapPartitionsRDD [83] rdd at GraphFrame.scala:180"]; } subgraph cluster205 { label="mapPartitions"; 85 [labelType="html" label="MapPartitionsRDD [85] rdd at GraphFrame.scala:180"]; } subgraph cluster206 { label="map"; 86 [labelType="html" label="MapPartitionsRDD [86] map at GraphFrame.scala:181"]; } } 80->81; 81->82; 82->83; 83->84; 84->85; 85->86; }

digraph G { subgraph clusterstage_21 { label="Stage 21"; 91 [labelType="html" label="SQLExecutionRDD [91] rdd at GraphFrame.scala:182"]; subgraph cluster211 { label="Scan csv "; 87 [labelType="html" label="FileScanRDD [87] rdd at GraphFrame.scala:182"]; 88 [labelType="html" label="MapPartitionsRDD [88] rdd at GraphFrame.scala:182"]; } subgraph cluster208 { label="WholeStageCodegen (1)"; 89 [labelType="html" label="MapPartitionsRDD [89] rdd at GraphFrame.scala:182"]; } subgraph cluster207 { label="DeserializeToObject"; 90 [labelType="html" label="MapPartitionsRDD [90] rdd at GraphFrame.scala:182"]; } subgraph cluster212 { label="mapPartitions"; 92 [labelType="html" label="MapPartitionsRDD [92] rdd at GraphFrame.scala:182"]; } subgraph cluster213 { label="map"; 93 [labelType="html" label="MapPartitionsRDD [93] map at GraphFrame.scala:183"]; } subgraph cluster214 { label="mapPartitionsWithIndex"; 94 [labelType="html" label="EdgeRDD, EdgeRDD [94] mapPartitionsWithIndex at EdgeRDD.scala:105"]; } subgraph cluster216 { label="mapPartitions"; 98 [labelType="html" label="VertexRDD.createRoutingTables - vid2pid (aggregation) [98] mapPartitions at VertexRDD.scala:356"]; } } 87->88; 88->89; 89->90; 90->91; 91->92; 92->93; 93->94; 94->98; }

digraph G { subgraph clusterstage_22 { label="Stage 22"; subgraph cluster215 { label="partitionBy"; 97 [labelType="html" label="ShuffledRDD [97] partitionBy at VertexRDD.scala:319"]; } subgraph cluster217 { label="partitionBy"; 99 [labelType="html" label="ShuffledRDD [99] partitionBy at VertexRDD.scala:361"]; } subgraph cluster218 { label="mapPartitions"; 100 [labelType="html" label="MapPartitionsRDD [100] mapPartitions at VertexRDD.scala:361"]; } subgraph cluster219 { label="zipPartitions"; 101 [labelType="html" label="VertexRDD, VertexRDD [101] [Cached] zipPartitions at VertexRDD.scala:322"]; } subgraph cluster221 { label="mapPartitions"; 106 [labelType="html" label="VertexRDD [106] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster224 { label="mapPartitions"; 112 [labelType="html" label="VertexRDD [112] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster226 { label="mapPartitions"; 116 [labelType="html" label="VertexRDD [116] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster231 { label="mapPartitions"; 124 [labelType="html" label="ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) [124] mapPartitions at VertexRDDImpl.scala:247"]; } } 99->100; 97->101; 100->101; 101->106; 106->112; 112->116; 116->124; }

86,97

98,99

101

106

112

116

digraph G { subgraph clusterstage_23 { label="Stage 23"; subgraph cluster215 { label="partitionBy"; 97 [labelType="html" label="ShuffledRDD [97] partitionBy at VertexRDD.scala:319"]; } subgraph cluster217 { label="partitionBy"; 99 [labelType="html" label="ShuffledRDD [99] partitionBy at VertexRDD.scala:361"]; } subgraph cluster218 { label="mapPartitions"; 100 [labelType="html" label="MapPartitionsRDD [100] mapPartitions at VertexRDD.scala:361"]; } subgraph cluster219 { label="zipPartitions"; 101 [labelType="html" label="VertexRDD, VertexRDD [101] [Cached] zipPartitions at VertexRDD.scala:322"]; } subgraph cluster221 { label="mapPartitions"; 106 [labelType="html" label="VertexRDD [106] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster224 { label="mapPartitions"; 112 [labelType="html" label="VertexRDD [112] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster226 { label="mapPartitions"; 116 [labelType="html" label="VertexRDD [116] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster227 { label="zipPartitions"; 118 [labelType="html" label="VertexRDD [118] zipPartitions at VertexRDDImpl.scala:144"]; } subgraph cluster228 { label="mapPartitions"; 120 [labelType="html" label="ReplicatedVertexView.updateVertices - shippedVerts false false (broadcast) [120] mapPartitions at VertexRDDImpl.scala:247"]; } } 99->100; 97->101; 100->101; 101->106; 106->112; 112->116; 112->118; 116->118; 118->120; }

86,97

98,99

101

106

112

116

digraph G { subgraph clusterstage_24 { label="Stage 24"; 91 [labelType="html" label="SQLExecutionRDD [91] rdd at GraphFrame.scala:182"]; subgraph cluster211 { label="Scan csv "; 87 [labelType="html" label="FileScanRDD [87] rdd at GraphFrame.scala:182"]; 88 [labelType="html" label="MapPartitionsRDD [88] rdd at GraphFrame.scala:182"]; } subgraph cluster208 { label="WholeStageCodegen (1)"; 89 [labelType="html" label="MapPartitionsRDD [89] rdd at GraphFrame.scala:182"]; } subgraph cluster207 { label="DeserializeToObject"; 90 [labelType="html" label="MapPartitionsRDD [90] rdd at GraphFrame.scala:182"]; } subgraph cluster212 { label="mapPartitions"; 92 [labelType="html" label="MapPartitionsRDD [92] rdd at GraphFrame.scala:182"]; } subgraph cluster213 { label="map"; 93 [labelType="html" label="MapPartitionsRDD [93] map at GraphFrame.scala:183"]; } subgraph cluster214 { label="mapPartitionsWithIndex"; 94 [labelType="html" label="EdgeRDD, EdgeRDD [94] mapPartitionsWithIndex at EdgeRDD.scala:105"]; } subgraph cluster220 { label="mapPartitions"; 104 [labelType="html" label="EdgeRDD [104] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster222 { label="mapPartitions"; 108 [labelType="html" label="EdgeRDD [108] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster223 { label="mapPartitions"; 110 [labelType="html" label="EdgeRDD [110] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster225 { label="mapPartitions"; 114 [labelType="html" label="EdgeRDD [114] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster229 { label="partitionBy"; 121 [labelType="html" label="ShuffledRDD [121] partitionBy at ReplicatedVertexView.scala:110"]; } subgraph cluster230 { label="zipPartitions"; 122 [labelType="html" label="EdgeRDD [122] [Cached] zipPartitions at ReplicatedVertexView.scala:112"]; } subgraph cluster232 { label="partitionBy"; 125 [labelType="html" label="ShuffledRDD [125] partitionBy at ReplicatedVertexView.scala:69"]; } subgraph cluster233 { label="zipPartitions"; 126 [labelType="html" label="EdgeRDD [126] zipPartitions at ReplicatedVertexView.scala:70"]; } subgraph cluster234 { label="mapPartitions"; 128 [labelType="html" label="GraphImpl.aggregateMessages - preAgg [128] mapPartitions at GraphImpl.scala:208"]; } } 87->88; 88->89; 89->90; 90->91; 91->92; 92->93; 93->94; 94->104; 104->108; 108->110; 110->114; 114->122; 121->122; 122->126; 125->126; 126->128; }

120,121

124,125

104

108

114

122

digraph G { subgraph clusterstage_25 { label="Stage 25"; subgraph cluster215 { label="partitionBy"; 97 [labelType="html" label="ShuffledRDD [97] partitionBy at VertexRDD.scala:319"]; } subgraph cluster217 { label="partitionBy"; 99 [labelType="html" label="ShuffledRDD [99] partitionBy at VertexRDD.scala:361"]; } subgraph cluster218 { label="mapPartitions"; 100 [labelType="html" label="MapPartitionsRDD [100] mapPartitions at VertexRDD.scala:361"]; } subgraph cluster219 { label="zipPartitions"; 101 [labelType="html" label="VertexRDD, VertexRDD [101] [Cached] zipPartitions at VertexRDD.scala:322"]; } subgraph cluster221 { label="mapPartitions"; 106 [labelType="html" label="VertexRDD [106] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster224 { label="mapPartitions"; 112 [labelType="html" label="VertexRDD [112] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster226 { label="mapPartitions"; 116 [labelType="html" label="VertexRDD [116] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster235 { label="partitionBy"; 129 [labelType="html" label="ShuffledRDD [129] partitionBy at VertexRDDImpl.scala:214"]; } subgraph cluster236 { label="zipPartitions"; 130 [labelType="html" label="VertexRDD [130] [Cached] zipPartitions at VertexRDDImpl.scala:215"]; } subgraph cluster237 { label="map"; 132 [labelType="html" label="MapPartitionsRDD [132] map at VertexRDDImpl.scala:90"]; } } 99->100; 97->101; 100->101; 101->106; 106->112; 112->116; 116->130; 129->130; 130->132; }

86,97

98,99

128,129

101

106

112

116

130

Completed Stages (6)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
25	fold at VertexRDDImpl.scala:90 +details org.apache.spark.rdd.RDD.fold(RDD.scala:1151) org.apache.spark.graphx.impl.VertexRDDImpl.count(VertexRDDImpl.scala:90) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:140) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/20 23:05:39	47 ms	1/1	43.4 KiB	1611.0 B
24	mapPartitions at GraphImpl.scala:208 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.GraphImpl.aggregateMessagesWithActiveSet(GraphImpl.scala:208) org.apache.spark.graphx.GraphXUtils$.mapReduceTriplets(GraphXUtils.scala:69) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:136) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/20 23:05:36	3 s	3/3	47.0 MiB	5.9 KiB	1611.0 B
23	mapPartitions at VertexRDDImpl.scala:247 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.VertexRDDImpl.shipVertexAttributes(VertexRDDImpl.scala:247) org.apache.spark.graphx.impl.ReplicatedVertexView.updateVertices(ReplicatedVertexView.scala:107) org.apache.spark.graphx.impl.GraphImpl.mapVertices(GraphImpl.scala:133) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:130) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/20 23:05:36	48 ms	1/1	56.3 KiB		860.0 B
22	mapPartitions at VertexRDDImpl.scala:247 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.VertexRDDImpl.shipVertexAttributes(VertexRDDImpl.scala:247) org.apache.spark.graphx.impl.ReplicatedVertexView.upgrade(ReplicatedVertexView.scala:66) org.apache.spark.graphx.impl.GraphImpl.aggregateMessagesWithActiveSet(GraphImpl.scala:198) org.apache.spark.graphx.GraphXUtils$.mapReduceTriplets(GraphXUtils.scala:69) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:136) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/20 23:05:36	0.3 s	1/1		10.4 KiB	5.1 KiB
21	mapPartitions at VertexRDD.scala:356 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.VertexRDD$.createRoutingTables(VertexRDD.scala:356) org.apache.spark.graphx.VertexRDD$.apply(VertexRDD.scala:321) org.apache.spark.graphx.VertexRDD$.apply(VertexRDD.scala:299) org.apache.spark.graphx.impl.GraphImpl$.apply(GraphImpl.scala:315) org.apache.spark.graphx.Graph$.apply(Graph.scala:531) org.graphframes.GraphFrame.toGraphX(GraphFrame.scala:184) org.graphframes.GraphFrame.cachedGraphX$lzycompute(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedGraphX(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedTopologyGraphX$lzycompute(GraphFrame.scala:592) org.graphframes.GraphFrame.cachedTopologyGraphX(GraphFrame.scala:591) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:71) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282)	2024/12/20 23:05:34	2 s	3/3	47.0 MiB		3.3 KiB
20	map at GraphFrame.scala:181 +details org.apache.spark.rdd.RDD.map(RDD.scala:395) org.graphframes.GraphFrame.toGraphX(GraphFrame.scala:181) org.graphframes.GraphFrame.cachedGraphX$lzycompute(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedGraphX(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedTopologyGraphX$lzycompute(GraphFrame.scala:592) org.graphframes.GraphFrame.cachedTopologyGraphX(GraphFrame.scala:591) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:71) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/20 23:05:34	0.3 s	1/1	10.5 KiB		7.1 KiB

Page: