graphframes - Details for Job 11

Details for Job 11

Status: SUCCEEDED
Associated SQL Query: 8
Completed Stages: 1

Enable zooming

digraph G { subgraph clusterstage_29 { label="Stage 29"; 30 [labelType="html" label="SQLExecutionRDD [30] javaToPython at NativeMethodAccessorImpl.java:0"]; 33 [labelType="html" label="PythonRDD [33] RDD at PythonRDD.scala:53"]; subgraph cluster60 { label="Scan csv "; 27 [labelType="html" label="FileScanRDD [27] javaToPython at NativeMethodAccessorImpl.java:0"]; 28 [labelType="html" label="MapPartitionsRDD [28] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster57 { label="WholeStageCodegen (1)"; 29 [labelType="html" label="MapPartitionsRDD [29] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster61 { label="map"; 31 [labelType="html" label="MapPartitionsRDD [31] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster62 { label="mapPartitions"; 32 [labelType="html" label="MapPartitionsRDD [32] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster63 { label="mapPartitions"; 34 [labelType="html" label="MapPartitionsRDD [34] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster64 { label="map"; 35 [labelType="html" label="MapPartitionsRDD [35] map at SerDeUtil.scala:137"]; } subgraph cluster65 { label="mapPartitions"; 36 [labelType="html" label="MapPartitionsRDD [36] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster297 { label="WholeStageCodegen (1)"; 114 [labelType="html" label="MapPartitionsRDD [114] take at GraphXConversions.scala:182"]; } subgraph cluster301 { label="mapPartitionsInternal"; 115 [labelType="html" label="MapPartitionsRDD [115] take at GraphXConversions.scala:182"]; } } 27->28; 28->29; 29->30; 30->31; 31->32; 32->33; 33->34; 34->35; 35->36; 36->114; 114->115; }

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
29	take at GraphXConversions.scala:182 +details org.apache.spark.sql.Dataset.take(Dataset.scala:2904) org.graphframes.lib.GraphXConversions$.integralId(GraphXConversions.scala:182) org.graphframes.lib.ShortestPaths$.$anonfun$run$1(ShortestPaths.scala:69) scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238) scala.collection.Iterator.foreach(Iterator.scala:941) scala.collection.Iterator.foreach$(Iterator.scala:941) scala.collection.AbstractIterator.foreach(Iterator.scala:1429) scala.collection.IterableLike.foreach(IterableLike.scala:74) scala.collection.IterableLike.foreach$(IterableLike.scala:73) scala.collection.AbstractIterable.foreach(Iterable.scala:56) scala.collection.TraversableLike.map(TraversableLike.scala:238) scala.collection.TraversableLike.map$(TraversableLike.scala:231) scala.collection.AbstractTraversable.map(Traversable.scala:108) org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:69) org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)	2024/12/04 20:23:59	57 ms	1/1	9.2 KiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

take at GraphXConversions.scala:182 +details

org.apache.spark.sql.Dataset.take(Dataset.scala:2904)
org.graphframes.lib.GraphXConversions$.integralId(GraphXConversions.scala:182)
org.graphframes.lib.ShortestPaths$.$anonfun$run$1(ShortestPaths.scala:69)
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
scala.collection.Iterator.foreach(Iterator.scala:941)
scala.collection.Iterator.foreach$(Iterator.scala:941)
scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
scala.collection.IterableLike.foreach(IterableLike.scala:74)
scala.collection.IterableLike.foreach$(IterableLike.scala:73)
scala.collection.AbstractIterable.foreach(Iterable.scala:56)
scala.collection.TraversableLike.map(TraversableLike.scala:238)
scala.collection.TraversableLike.map$(TraversableLike.scala:231)
scala.collection.AbstractTraversable.map(Traversable.scala:108)
org.graphframes.lib.ShortestPaths$.org$graphframes$lib$ShortestPaths$$run(ShortestPaths.scala:69)
org.graphframes.lib.ShortestPaths.run(ShortestPaths.scala:61)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)

2024/12/04 20:23:59

57 ms

1/1

9.2 KiB

Page: