graphframes - Details for Job 10

Details for Job 10

Status: SUCCEEDED
Associated SQL Query: 9
Completed Stages: 6

Enable zooming

digraph G { subgraph clusterstage_15 { label="Stage 15"; 32 [labelType="html" label="PythonRDD [32] RDD at PythonRDD.scala:53"]; subgraph cluster60 { label="parallelize"; 31 [labelType="html" label="ParallelCollectionRDD [31] readRDDFromFile at PythonRDD.scala:262"]; } subgraph cluster61 { label="mapPartitions"; 33 [labelType="html" label="MapPartitionsRDD [33] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster62 { label="map"; 34 [labelType="html" label="MapPartitionsRDD [34] map at SerDeUtil.scala:137"]; } subgraph cluster63 { label="mapPartitions"; 35 [labelType="html" label="MapPartitionsRDD [35] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster197 { label="WholeStageCodegen (3)"; 84 [labelType="html" label="MapPartitionsRDD [84] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster196 { label="Exchange"; 85 [labelType="html" label="MapPartitionsRDD [85] count at NativeMethodAccessorImpl.java:0"]; } } 31->32; 32->33; 33->34; 34->35; 35->84; 84->85; }

digraph G { subgraph clusterstage_16 { label="Stage 16"; 51 [labelType="html" label="PythonRDD [51] RDD at PythonRDD.scala:53"]; subgraph cluster87 { label="parallelize"; 50 [labelType="html" label="ParallelCollectionRDD [50] readRDDFromFile at PythonRDD.scala:262"]; } subgraph cluster88 { label="mapPartitions"; 52 [labelType="html" label="MapPartitionsRDD [52] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster89 { label="map"; 53 [labelType="html" label="MapPartitionsRDD [53] map at SerDeUtil.scala:137"]; } subgraph cluster90 { label="mapPartitions"; 54 [labelType="html" label="MapPartitionsRDD [54] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster189 { label="WholeStageCodegen (1)"; 80 [labelType="html" label="MapPartitionsRDD [80] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster188 { label="Exchange"; 81 [labelType="html" label="MapPartitionsRDD [81] count at NativeMethodAccessorImpl.java:0"]; } } 50->51; 51->52; 52->53; 53->54; 54->80; 80->81; }

digraph G { subgraph clusterstage_17 { label="Stage 17"; subgraph cluster188 { label="Exchange"; 82 [labelType="html" label="ShuffledRowRDD [82] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster185 { label="WholeStageCodegen (2)"; 83 [labelType="html" label="MapPartitionsRDD [83] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster196 { label="Exchange"; 86 [labelType="html" label="ShuffledRowRDD [86] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster193 { label="WholeStageCodegen (4)"; 87 [labelType="html" label="MapPartitionsRDD [87] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster182 { label="WholeStageCodegen (5)"; 88 [labelType="html" label="ZippedPartitionsRDD2 [88] count at NativeMethodAccessorImpl.java:0"]; 89 [labelType="html" label="MapPartitionsRDD [89] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster181 { label="Exchange"; 90 [labelType="html" label="MapPartitionsRDD [90] count at NativeMethodAccessorImpl.java:0"]; } } 82->83; 86->87; 83->88; 87->88; 88->89; 89->90; }

81,82

85,86

digraph G { subgraph clusterstage_18 { label="Stage 18"; 32 [labelType="html" label="PythonRDD [32] RDD at PythonRDD.scala:53"]; subgraph cluster60 { label="parallelize"; 31 [labelType="html" label="ParallelCollectionRDD [31] readRDDFromFile at PythonRDD.scala:262"]; } subgraph cluster61 { label="mapPartitions"; 33 [labelType="html" label="MapPartitionsRDD [33] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster62 { label="map"; 34 [labelType="html" label="MapPartitionsRDD [34] map at SerDeUtil.scala:137"]; } subgraph cluster63 { label="mapPartitions"; 35 [labelType="html" label="MapPartitionsRDD [35] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster205 { label="WholeStageCodegen (7)"; 93 [labelType="html" label="MapPartitionsRDD [93] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster204 { label="Exchange"; 94 [labelType="html" label="MapPartitionsRDD [94] count at NativeMethodAccessorImpl.java:0"]; } } 31->32; 32->33; 33->34; 34->35; 35->93; 93->94; }

digraph G { subgraph clusterstage_19 { label="Stage 19"; subgraph cluster181 { label="Exchange"; 91 [labelType="html" label="ShuffledRowRDD [91] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster178 { label="WholeStageCodegen (6)"; 92 [labelType="html" label="MapPartitionsRDD [92] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster204 { label="Exchange"; 95 [labelType="html" label="ShuffledRowRDD [95] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster201 { label="WholeStageCodegen (8)"; 96 [labelType="html" label="MapPartitionsRDD [96] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster173 { label="WholeStageCodegen (9)"; 97 [labelType="html" label="ZippedPartitionsRDD2 [97] count at NativeMethodAccessorImpl.java:0"]; 98 [labelType="html" label="MapPartitionsRDD [98] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster172 { label="Exchange"; 99 [labelType="html" label="MapPartitionsRDD [99] count at NativeMethodAccessorImpl.java:0"]; } } 91->92; 95->96; 92->97; 96->97; 97->98; 98->99; }

90,91

94,95

digraph G { subgraph clusterstage_20 { label="Stage 20"; subgraph cluster172 { label="Exchange"; 100 [labelType="html" label="ShuffledRowRDD [100] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster169 { label="WholeStageCodegen (10)"; 101 [labelType="html" label="MapPartitionsRDD [101] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster208 { label="mapPartitionsInternal"; 102 [labelType="html" label="MapPartitionsRDD [102] count at NativeMethodAccessorImpl.java:0"]; } } 100->101; 101->102; }

99,100

Completed Stages (6)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Shuffle Read	Shuffle Write
20	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:2980) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:28:05	37 ms	1/1	11.4 KiB
19	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:2980) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:28:04	0.5 s	200/200	2.2 MiB	11.4 KiB
18	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:2980) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:28:03	0.1 s	16/16		35.2 KiB
17	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:2980) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:28:04	0.6 s	200/200	2.5 MiB	2.1 MiB
16	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:2980) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:28:03	0.3 s	16/16		2.5 MiB
15	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:2980) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:28:03	0.1 s	16/16		34.9 KiB

Page: