graphframes - Details for Job 19

Details for Job 19

Status: SUCCEEDED
Completed Stages: 4
Skipped Stages: 5

Enable zooming

digraph G { subgraph clusterstage_26 { label="Stage 26 (skipped)"; 91 [labelType="html" label="SQLExecutionRDD [91] rdd at GraphFrame.scala:182"]; subgraph cluster211 { label="Scan csv "; 87 [labelType="html" label="FileScanRDD [87] rdd at GraphFrame.scala:182"]; 88 [labelType="html" label="MapPartitionsRDD [88] rdd at GraphFrame.scala:182"]; } subgraph cluster208 { label="WholeStageCodegen (1)"; 89 [labelType="html" label="MapPartitionsRDD [89] rdd at GraphFrame.scala:182"]; } subgraph cluster207 { label="DeserializeToObject"; 90 [labelType="html" label="MapPartitionsRDD [90] rdd at GraphFrame.scala:182"]; } subgraph cluster212 { label="mapPartitions"; 92 [labelType="html" label="MapPartitionsRDD [92] rdd at GraphFrame.scala:182"]; } subgraph cluster213 { label="map"; 93 [labelType="html" label="MapPartitionsRDD [93] map at GraphFrame.scala:183"]; } subgraph cluster214 { label="mapPartitionsWithIndex"; 94 [labelType="html" label="EdgeRDD, EdgeRDD [94] mapPartitionsWithIndex at EdgeRDD.scala:105"]; } subgraph cluster216 { label="mapPartitions"; 98 [labelType="html" label="VertexRDD.createRoutingTables - vid2pid (aggregation) [98] mapPartitions at VertexRDD.scala:356"]; } } 87->88; 88->89; 89->90; 90->91; 91->92; 92->93; 93->94; 94->98; }

digraph G { subgraph clusterstage_27 { label="Stage 27 (skipped)"; 84 [labelType="html" label="SQLExecutionRDD [84] rdd at GraphFrame.scala:180"]; subgraph cluster204 { label="Scan csv "; 80 [labelType="html" label="FileScanRDD [80] rdd at GraphFrame.scala:180"]; 81 [labelType="html" label="MapPartitionsRDD [81] rdd at GraphFrame.scala:180"]; } subgraph cluster201 { label="WholeStageCodegen (1)"; 82 [labelType="html" label="MapPartitionsRDD [82] rdd at GraphFrame.scala:180"]; } subgraph cluster200 { label="DeserializeToObject"; 83 [labelType="html" label="MapPartitionsRDD [83] rdd at GraphFrame.scala:180"]; } subgraph cluster205 { label="mapPartitions"; 85 [labelType="html" label="MapPartitionsRDD [85] rdd at GraphFrame.scala:180"]; } subgraph cluster206 { label="map"; 86 [labelType="html" label="MapPartitionsRDD [86] map at GraphFrame.scala:181"]; } } 80->81; 81->82; 82->83; 83->84; 84->85; 85->86; }

digraph G { subgraph clusterstage_28 { label="Stage 28 (skipped)"; subgraph cluster215 { label="partitionBy"; 97 [labelType="html" label="ShuffledRDD [97] partitionBy at VertexRDD.scala:319"]; } subgraph cluster217 { label="partitionBy"; 99 [labelType="html" label="ShuffledRDD [99] partitionBy at VertexRDD.scala:361"]; } subgraph cluster218 { label="mapPartitions"; 100 [labelType="html" label="MapPartitionsRDD [100] mapPartitions at VertexRDD.scala:361"]; } subgraph cluster219 { label="zipPartitions"; 101 [labelType="html" label="VertexRDD, VertexRDD [101] [Cached] zipPartitions at VertexRDD.scala:322"]; } subgraph cluster221 { label="mapPartitions"; 106 [labelType="html" label="VertexRDD [106] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster224 { label="mapPartitions"; 112 [labelType="html" label="VertexRDD [112] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster226 { label="mapPartitions"; 116 [labelType="html" label="VertexRDD [116] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster227 { label="zipPartitions"; 118 [labelType="html" label="VertexRDD [118] zipPartitions at VertexRDDImpl.scala:144"]; } subgraph cluster228 { label="mapPartitions"; 120 [labelType="html" label="ReplicatedVertexView.updateVertices - shippedVerts false false (broadcast) [120] mapPartitions at VertexRDDImpl.scala:247"]; } } 99->100; 97->101; 100->101; 101->106; 106->112; 112->116; 112->118; 116->118; 118->120; }

86,97

98,99

101

106

112

116

digraph G { subgraph clusterstage_29 { label="Stage 29 (skipped)"; subgraph cluster215 { label="partitionBy"; 97 [labelType="html" label="ShuffledRDD [97] partitionBy at VertexRDD.scala:319"]; } subgraph cluster217 { label="partitionBy"; 99 [labelType="html" label="ShuffledRDD [99] partitionBy at VertexRDD.scala:361"]; } subgraph cluster218 { label="mapPartitions"; 100 [labelType="html" label="MapPartitionsRDD [100] mapPartitions at VertexRDD.scala:361"]; } subgraph cluster219 { label="zipPartitions"; 101 [labelType="html" label="VertexRDD, VertexRDD [101] [Cached] zipPartitions at VertexRDD.scala:322"]; } subgraph cluster221 { label="mapPartitions"; 106 [labelType="html" label="VertexRDD [106] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster224 { label="mapPartitions"; 112 [labelType="html" label="VertexRDD [112] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster226 { label="mapPartitions"; 116 [labelType="html" label="VertexRDD [116] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster231 { label="mapPartitions"; 124 [labelType="html" label="ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) [124] mapPartitions at VertexRDDImpl.scala:247"]; } } 99->100; 97->101; 100->101; 101->106; 106->112; 112->116; 116->124; }

86,97

98,99

101

106

112

116

digraph G { subgraph clusterstage_30 { label="Stage 30 (skipped)"; 91 [labelType="html" label="SQLExecutionRDD [91] rdd at GraphFrame.scala:182"]; subgraph cluster211 { label="Scan csv "; 87 [labelType="html" label="FileScanRDD [87] rdd at GraphFrame.scala:182"]; 88 [labelType="html" label="MapPartitionsRDD [88] rdd at GraphFrame.scala:182"]; } subgraph cluster208 { label="WholeStageCodegen (1)"; 89 [labelType="html" label="MapPartitionsRDD [89] rdd at GraphFrame.scala:182"]; } subgraph cluster207 { label="DeserializeToObject"; 90 [labelType="html" label="MapPartitionsRDD [90] rdd at GraphFrame.scala:182"]; } subgraph cluster212 { label="mapPartitions"; 92 [labelType="html" label="MapPartitionsRDD [92] rdd at GraphFrame.scala:182"]; } subgraph cluster213 { label="map"; 93 [labelType="html" label="MapPartitionsRDD [93] map at GraphFrame.scala:183"]; } subgraph cluster214 { label="mapPartitionsWithIndex"; 94 [labelType="html" label="EdgeRDD, EdgeRDD [94] mapPartitionsWithIndex at EdgeRDD.scala:105"]; } subgraph cluster220 { label="mapPartitions"; 104 [labelType="html" label="EdgeRDD [104] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster222 { label="mapPartitions"; 108 [labelType="html" label="EdgeRDD [108] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster223 { label="mapPartitions"; 110 [labelType="html" label="EdgeRDD [110] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster225 { label="mapPartitions"; 114 [labelType="html" label="EdgeRDD [114] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster229 { label="partitionBy"; 121 [labelType="html" label="ShuffledRDD [121] partitionBy at ReplicatedVertexView.scala:110"]; } subgraph cluster230 { label="zipPartitions"; 122 [labelType="html" label="EdgeRDD [122] [Cached] zipPartitions at ReplicatedVertexView.scala:112"]; } subgraph cluster232 { label="partitionBy"; 125 [labelType="html" label="ShuffledRDD [125] partitionBy at ReplicatedVertexView.scala:69"]; } subgraph cluster233 { label="zipPartitions"; 126 [labelType="html" label="EdgeRDD [126] zipPartitions at ReplicatedVertexView.scala:70"]; } subgraph cluster234 { label="mapPartitions"; 128 [labelType="html" label="GraphImpl.aggregateMessages - preAgg [128] mapPartitions at GraphImpl.scala:208"]; } } 87->88; 88->89; 89->90; 90->91; 91->92; 92->93; 93->94; 94->104; 104->108; 108->110; 110->114; 114->122; 121->122; 122->126; 125->126; 126->128; }

120,121

124,125

104

108

114

122

digraph G { subgraph clusterstage_31 { label="Stage 31"; subgraph cluster215 { label="partitionBy"; 97 [labelType="html" label="ShuffledRDD [97] partitionBy at VertexRDD.scala:319"]; } subgraph cluster217 { label="partitionBy"; 99 [labelType="html" label="ShuffledRDD [99] partitionBy at VertexRDD.scala:361"]; } subgraph cluster218 { label="mapPartitions"; 100 [labelType="html" label="MapPartitionsRDD [100] mapPartitions at VertexRDD.scala:361"]; } subgraph cluster219 { label="zipPartitions"; 101 [labelType="html" label="VertexRDD, VertexRDD [101] [Cached] zipPartitions at VertexRDD.scala:322"]; } subgraph cluster221 { label="mapPartitions"; 106 [labelType="html" label="VertexRDD [106] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster224 { label="mapPartitions"; 112 [labelType="html" label="VertexRDD [112] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster226 { label="mapPartitions"; 116 [labelType="html" label="VertexRDD [116] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster235 { label="partitionBy"; 129 [labelType="html" label="ShuffledRDD [129] partitionBy at VertexRDDImpl.scala:214"]; } subgraph cluster236 { label="zipPartitions"; 130 [labelType="html" label="VertexRDD [130] [Cached] zipPartitions at VertexRDDImpl.scala:215"]; } subgraph cluster286 { label="mapPartitions"; 141 [labelType="html" label="ReplicatedVertexView.withActiveSet - shippedActives (broadcast) [141] mapPartitions at VertexRDDImpl.scala:251"]; } } 99->100; 97->101; 100->101; 101->106; 106->112; 112->116; 116->130; 129->130; 130->141; }

86,97

98,99

128,129

101

106

112

116

130

digraph G { subgraph clusterstage_32 { label="Stage 32"; subgraph cluster215 { label="partitionBy"; 97 [labelType="html" label="ShuffledRDD [97] partitionBy at VertexRDD.scala:319"]; } subgraph cluster217 { label="partitionBy"; 99 [labelType="html" label="ShuffledRDD [99] partitionBy at VertexRDD.scala:361"]; } subgraph cluster218 { label="mapPartitions"; 100 [labelType="html" label="MapPartitionsRDD [100] mapPartitions at VertexRDD.scala:361"]; } subgraph cluster219 { label="zipPartitions"; 101 [labelType="html" label="VertexRDD, VertexRDD [101] [Cached] zipPartitions at VertexRDD.scala:322"]; } subgraph cluster221 { label="mapPartitions"; 106 [labelType="html" label="VertexRDD [106] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster224 { label="mapPartitions"; 112 [labelType="html" label="VertexRDD [112] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster226 { label="mapPartitions"; 116 [labelType="html" label="VertexRDD [116] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster235 { label="partitionBy"; 129 [labelType="html" label="ShuffledRDD [129] partitionBy at VertexRDDImpl.scala:214"]; } subgraph cluster236 { label="zipPartitions"; 130 [labelType="html" label="VertexRDD [130] [Cached] zipPartitions at VertexRDDImpl.scala:215"]; } subgraph cluster281 { label="zipPartitions"; 133 [labelType="html" label="VertexRDD [133] [Cached] zipPartitions at VertexRDDImpl.scala:156"]; } subgraph cluster282 { label="zipPartitions"; 135 [labelType="html" label="VertexRDD [135] zipPartitions at VertexRDDImpl.scala:144"]; } subgraph cluster283 { label="mapPartitions"; 137 [labelType="html" label="ReplicatedVertexView.updateVertices - shippedVerts true true (broadcast) [137] mapPartitions at VertexRDDImpl.scala:247"]; } } 99->100; 97->101; 100->101; 101->106; 106->112; 112->116; 116->130; 129->130; 116->133; 130->133; 116->135; 133->135; 135->137; }

86,97

98,99

128,129

101

106

112

116

130

133

digraph G { subgraph clusterstage_33 { label="Stage 33"; 91 [labelType="html" label="SQLExecutionRDD [91] rdd at GraphFrame.scala:182"]; subgraph cluster211 { label="Scan csv "; 87 [labelType="html" label="FileScanRDD [87] rdd at GraphFrame.scala:182"]; 88 [labelType="html" label="MapPartitionsRDD [88] rdd at GraphFrame.scala:182"]; } subgraph cluster208 { label="WholeStageCodegen (1)"; 89 [labelType="html" label="MapPartitionsRDD [89] rdd at GraphFrame.scala:182"]; } subgraph cluster207 { label="DeserializeToObject"; 90 [labelType="html" label="MapPartitionsRDD [90] rdd at GraphFrame.scala:182"]; } subgraph cluster212 { label="mapPartitions"; 92 [labelType="html" label="MapPartitionsRDD [92] rdd at GraphFrame.scala:182"]; } subgraph cluster213 { label="map"; 93 [labelType="html" label="MapPartitionsRDD [93] map at GraphFrame.scala:183"]; } subgraph cluster214 { label="mapPartitionsWithIndex"; 94 [labelType="html" label="EdgeRDD, EdgeRDD [94] mapPartitionsWithIndex at EdgeRDD.scala:105"]; } subgraph cluster220 { label="mapPartitions"; 104 [labelType="html" label="EdgeRDD [104] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster222 { label="mapPartitions"; 108 [labelType="html" label="EdgeRDD [108] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster223 { label="mapPartitions"; 110 [labelType="html" label="EdgeRDD [110] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster225 { label="mapPartitions"; 114 [labelType="html" label="EdgeRDD [114] [Cached] mapPartitions at EdgeRDDImpl.scala:119"]; } subgraph cluster229 { label="partitionBy"; 121 [labelType="html" label="ShuffledRDD [121] partitionBy at ReplicatedVertexView.scala:110"]; } subgraph cluster230 { label="zipPartitions"; 122 [labelType="html" label="EdgeRDD [122] [Cached] zipPartitions at ReplicatedVertexView.scala:112"]; } subgraph cluster232 { label="partitionBy"; 125 [labelType="html" label="ShuffledRDD [125] partitionBy at ReplicatedVertexView.scala:69"]; } subgraph cluster233 { label="zipPartitions"; 126 [labelType="html" label="EdgeRDD [126] zipPartitions at ReplicatedVertexView.scala:70"]; } subgraph cluster284 { label="partitionBy"; 138 [labelType="html" label="ShuffledRDD [138] partitionBy at ReplicatedVertexView.scala:110"]; } subgraph cluster285 { label="zipPartitions"; 139 [labelType="html" label="EdgeRDD [139] [Cached] zipPartitions at ReplicatedVertexView.scala:112"]; } subgraph cluster287 { label="partitionBy"; 142 [labelType="html" label="ShuffledRDD [142] partitionBy at ReplicatedVertexView.scala:90"]; } subgraph cluster288 { label="zipPartitions"; 143 [labelType="html" label="EdgeRDD [143] zipPartitions at ReplicatedVertexView.scala:92"]; } subgraph cluster289 { label="mapPartitions"; 145 [labelType="html" label="GraphImpl.aggregateMessages - preAgg [145] mapPartitions at GraphImpl.scala:208"]; } } 87->88; 88->89; 89->90; 90->91; 91->92; 92->93; 93->94; 94->104; 104->108; 108->110; 110->114; 114->122; 121->122; 122->126; 125->126; 126->139; 138->139; 139->143; 142->143; 143->145; }

120,121

124,125

137,138

141,142

104

108

114

122

139

digraph G { subgraph clusterstage_34 { label="Stage 34"; subgraph cluster215 { label="partitionBy"; 97 [labelType="html" label="ShuffledRDD [97] partitionBy at VertexRDD.scala:319"]; } subgraph cluster217 { label="partitionBy"; 99 [labelType="html" label="ShuffledRDD [99] partitionBy at VertexRDD.scala:361"]; } subgraph cluster218 { label="mapPartitions"; 100 [labelType="html" label="MapPartitionsRDD [100] mapPartitions at VertexRDD.scala:361"]; } subgraph cluster219 { label="zipPartitions"; 101 [labelType="html" label="VertexRDD, VertexRDD [101] [Cached] zipPartitions at VertexRDD.scala:322"]; } subgraph cluster221 { label="mapPartitions"; 106 [labelType="html" label="VertexRDD [106] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster224 { label="mapPartitions"; 112 [labelType="html" label="VertexRDD [112] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster226 { label="mapPartitions"; 116 [labelType="html" label="VertexRDD [116] [Cached] mapPartitions at VertexRDDImpl.scala:96"]; } subgraph cluster235 { label="partitionBy"; 129 [labelType="html" label="ShuffledRDD [129] partitionBy at VertexRDDImpl.scala:214"]; } subgraph cluster236 { label="zipPartitions"; 130 [labelType="html" label="VertexRDD [130] [Cached] zipPartitions at VertexRDDImpl.scala:215"]; } subgraph cluster281 { label="zipPartitions"; 133 [labelType="html" label="VertexRDD [133] [Cached] zipPartitions at VertexRDDImpl.scala:156"]; } subgraph cluster290 { label="partitionBy"; 146 [labelType="html" label="ShuffledRDD [146] partitionBy at VertexRDDImpl.scala:214"]; } subgraph cluster291 { label="zipPartitions"; 147 [labelType="html" label="VertexRDD [147] [Cached] zipPartitions at VertexRDDImpl.scala:215"]; } subgraph cluster292 { label="map"; 149 [labelType="html" label="MapPartitionsRDD [149] map at VertexRDDImpl.scala:90"]; } } 99->100; 97->101; 100->101; 101->106; 106->112; 112->116; 116->130; 129->130; 116->133; 130->133; 133->147; 146->147; 147->149; }

86,97

98,99

128,129

145,146

101

106

112

116

130

133

147

Completed Stages (4)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
34	fold at VertexRDDImpl.scala:90 +details org.apache.spark.rdd.RDD.fold(RDD.scala:1151) org.apache.spark.graphx.impl.VertexRDDImpl.count(VertexRDDImpl.scala:90) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:161) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/20 23:05:40	41 ms	1/1	44.6 KiB	3.7 KiB
33	mapPartitions at GraphImpl.scala:208 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.GraphImpl.aggregateMessagesWithActiveSet(GraphImpl.scala:208) org.apache.spark.graphx.GraphXUtils$.mapReduceTriplets(GraphXUtils.scala:69) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:155) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/20 23:05:40	0.8 s	3/3	5.4 MiB	7.6 KiB	3.7 KiB
32	mapPartitions at VertexRDDImpl.scala:247 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.VertexRDDImpl.shipVertexAttributes(VertexRDDImpl.scala:247) org.apache.spark.graphx.impl.ReplicatedVertexView.updateVertices(ReplicatedVertexView.scala:107) org.apache.spark.graphx.impl.GraphImpl.outerJoinVertices(GraphImpl.scala:259) org.apache.spark.graphx.GraphOps.joinVertices(GraphOps.scala:231) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:148) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/20 23:05:39	44 ms	1/1	103.2 KiB		2.2 KiB
31	mapPartitions at VertexRDDImpl.scala:251 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.VertexRDDImpl.shipVertexIds(VertexRDDImpl.scala:251) org.apache.spark.graphx.impl.ReplicatedVertexView.withActiveSet(ReplicatedVertexView.scala:88) org.apache.spark.graphx.impl.GraphImpl.aggregateMessagesWithActiveSet(GraphImpl.scala:201) org.apache.spark.graphx.GraphXUtils$.mapReduceTriplets(GraphXUtils.scala:69) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:155) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/20 23:05:39	56 ms	1/1	16.4 KiB		420.0 B

Page:

Skipped Stages (5)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
30	mapPartitions at GraphImpl.scala:208 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.GraphImpl.aggregateMessagesWithActiveSet(GraphImpl.scala:208) org.apache.spark.graphx.GraphXUtils$.mapReduceTriplets(GraphXUtils.scala:69) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:136) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/3
29	mapPartitions at VertexRDDImpl.scala:247 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.VertexRDDImpl.shipVertexAttributes(VertexRDDImpl.scala:247) org.apache.spark.graphx.impl.ReplicatedVertexView.upgrade(ReplicatedVertexView.scala:66) org.apache.spark.graphx.impl.GraphImpl.aggregateMessagesWithActiveSet(GraphImpl.scala:198) org.apache.spark.graphx.GraphXUtils$.mapReduceTriplets(GraphXUtils.scala:69) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:136) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
28	mapPartitions at VertexRDDImpl.scala:247 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.impl.VertexRDDImpl.shipVertexAttributes(VertexRDDImpl.scala:247) org.apache.spark.graphx.impl.ReplicatedVertexView.updateVertices(ReplicatedVertexView.scala:107) org.apache.spark.graphx.impl.GraphImpl.mapVertices(GraphImpl.scala:133) org.apache.spark.graphx.Pregel$.apply(Pregel.scala:130) org.apache.spark.graphx.lib.ShortestPaths$.run(ShortestPaths.scala:75) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:70) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
27	map at GraphFrame.scala:181 +details org.apache.spark.rdd.RDD.map(RDD.scala:395) org.graphframes.GraphFrame.toGraphX(GraphFrame.scala:181) org.graphframes.GraphFrame.cachedGraphX$lzycompute(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedGraphX(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedTopologyGraphX$lzycompute(GraphFrame.scala:592) org.graphframes.GraphFrame.cachedTopologyGraphX(GraphFrame.scala:591) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:71) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
26	mapPartitions at VertexRDD.scala:356 +details org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:833) org.apache.spark.graphx.VertexRDD$.createRoutingTables(VertexRDD.scala:356) org.apache.spark.graphx.VertexRDD$.apply(VertexRDD.scala:321) org.apache.spark.graphx.VertexRDD$.apply(VertexRDD.scala:299) org.apache.spark.graphx.impl.GraphImpl$.apply(GraphImpl.scala:315) org.apache.spark.graphx.Graph$.apply(Graph.scala:531) org.graphframes.GraphFrame.toGraphX(GraphFrame.scala:184) org.graphframes.GraphFrame.cachedGraphX$lzycompute(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedGraphX(GraphFrame.scala:598) org.graphframes.GraphFrame.cachedTopologyGraphX$lzycompute(GraphFrame.scala:592) org.graphframes.GraphFrame.cachedTopologyGraphX(GraphFrame.scala:591) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:71) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282)	Unknown	Unknown	0/3

Page: