NYC Yellow Taxi Data Analysis - Details for Job 10

Details for Job 10

Status: SUCCEEDED
Associated SQL Query: 5
Completed Stages: 1
Skipped Stages: 4

Enable zooming

digraph G { subgraph clusterstage_29 { label="Stage 29 (skipped)"; 2 [labelType="html" label="PythonRDD [2] RDD at PythonRDD.scala:53"]; 14 [labelType="html" label="SQLExecutionRDD [14] csv at NativeMethodAccessorImpl.java:0"]; subgraph cluster0 { label="textFile"; 0 [labelType="html" label="s3a://data-repository-bkt/ECS765/nyc_taxi/taxi_zone_lookup.csv [0] textFile at NativeMethodAccessorImpl.java:0"]; 1 [labelType="html" label="s3a://data-repository-bkt/ECS765/nyc_taxi/taxi_zone_lookup.csv [1] textFile at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1 { label="map"; 3 [labelType="html" label="MapPartitionsRDD [3] map at NativeMethodAccessorImpl.java:0"]; } subgraph cluster29 { label="Scan"; 11 [labelType="html" label="MapPartitionsRDD [11] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster25 { label="WholeStageCodegen (1)"; 12 [labelType="html" label="MapPartitionsRDD [12] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster24 { label="DeserializeToObject"; 13 [labelType="html" label="MapPartitionsRDD [13] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster30 { label="mapPartitions"; 15 [labelType="html" label="MapPartitionsRDD [15] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster31 { label="mapPartitions"; 16 [labelType="html" label="MapPartitionsRDD [16] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster32 { label="mapPartitions"; 17 [labelType="html" label="MapPartitionsRDD [17] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster205 { label="WholeStageCodegen (2)"; 74 [labelType="html" label="MapPartitionsRDD [74] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster204 { label="Exchange"; 75 [labelType="html" label="MapPartitionsRDD [75] showString at NativeMethodAccessorImpl.java:0"]; } } 0->1; 1->2; 2->3; 3->11; 11->12; 12->13; 13->14; 14->15; 15->16; 16->17; 17->74; 74->75; }

digraph G { subgraph clusterstage_30 { label="Stage 30 (skipped)"; subgraph cluster200 { label="Scan csv "; 69 [labelType="html" label="FileScanRDD [69] showString at NativeMethodAccessorImpl.java:0"]; 70 [labelType="html" label="MapPartitionsRDD [70] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster199 { label="Exchange"; 71 [labelType="html" label="MapPartitionsRDD [71] showString at NativeMethodAccessorImpl.java:0"]; } } 69->70; 70->71; }

digraph G { subgraph clusterstage_31 { label="Stage 31 (skipped)"; subgraph cluster199 { label="Exchange"; 72 [labelType="html" label="ShuffledRowRDD [72] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster196 { label="WholeStageCodegen (1)"; 73 [labelType="html" label="MapPartitionsRDD [73] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster204 { label="Exchange"; 76 [labelType="html" label="ShuffledRowRDD [76] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster201 { label="WholeStageCodegen (3)"; 77 [labelType="html" label="MapPartitionsRDD [77] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster195 { label="SortMergeJoin"; 78 [labelType="html" label="ZippedPartitionsRDD2 [78] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster192 { label="WholeStageCodegen (4)"; 79 [labelType="html" label="MapPartitionsRDD [79] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster191 { label="Exchange"; 80 [labelType="html" label="MapPartitionsRDD [80] showString at NativeMethodAccessorImpl.java:0"]; } } 72->73; 76->77; 73->78; 77->78; 78->79; 79->80; }

71,72

75,76

digraph G { subgraph clusterstage_32 { label="Stage 32 (skipped)"; subgraph cluster204 { label="Exchange"; 76 [labelType="html" label="ShuffledRowRDD [76] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster191 { label="Exchange"; 81 [labelType="html" label="ShuffledRowRDD [81] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster188 { label="WholeStageCodegen (5)"; 82 [labelType="html" label="MapPartitionsRDD [82] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster209 { label="WholeStageCodegen (7)"; 83 [labelType="html" label="MapPartitionsRDD [83] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster187 { label="SortMergeJoin"; 84 [labelType="html" label="ZippedPartitionsRDD2 [84] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster183 { label="WholeStageCodegen (8)"; 85 [labelType="html" label="MapPartitionsRDD [85] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster182 { label="Exchange"; 86 [labelType="html" label="MapPartitionsRDD [86] showString at NativeMethodAccessorImpl.java:0"]; } } 81->82; 76->83; 82->84; 83->84; 84->85; 85->86; }

75,76

80,81

digraph G { subgraph clusterstage_33 { label="Stage 33"; subgraph cluster182 { label="Exchange"; 87 [labelType="html" label="ShuffledRowRDD [87] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster177 { label="WholeStageCodegen (9)"; 88 [labelType="html" label="MapPartitionsRDD [88] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster214 { label="mapPartitionsInternal"; 89 [labelType="html" label="MapPartitionsRDD [89] showString at NativeMethodAccessorImpl.java:0"]; } } 87->88; 88->89; }

86,87

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
33	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/16 22:50:38	94 ms	20/20			65.3 KiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/16 22:50:38

94 ms

20/20

65.3 KiB

Page:

Skipped Stages (4)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
32	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200
31	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200
30	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/20
29	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/2

Page: