task2 - Details for Job 10

Details for Job 10

Status: SUCCEEDED
Associated SQL Query: 7
Completed Stages: 2
Skipped Stages: 3

Enable zooming

digraph G { subgraph clusterstage_35 { label="Stage 35 (skipped)"; subgraph cluster271 { label="Scan csv "; 110 [labelType="html" label="FileScanRDD [110] csv at NativeMethodAccessorImpl.java:0"]; 111 [labelType="html" label="MapPartitionsRDD [111] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster267 { label="WholeStageCodegen (1)"; 112 [labelType="html" label="MapPartitionsRDD [112] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster266 { label="Exchange"; 113 [labelType="html" label="MapPartitionsRDD [113] csv at NativeMethodAccessorImpl.java:0"]; } } 110->111; 111->112; 112->113; }

digraph G { subgraph clusterstage_36 { label="Stage 36 (skipped)"; subgraph cluster281 { label="Scan csv "; 116 [labelType="html" label="FileScanRDD [116] csv at NativeMethodAccessorImpl.java:0"]; 117 [labelType="html" label="MapPartitionsRDD [117] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster277 { label="WholeStageCodegen (3)"; 118 [labelType="html" label="MapPartitionsRDD [118] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster276 { label="Exchange"; 119 [labelType="html" label="MapPartitionsRDD [119] csv at NativeMethodAccessorImpl.java:0"]; } } 116->117; 117->118; 118->119; }

digraph G { subgraph clusterstage_37 { label="Stage 37 (skipped)"; subgraph cluster266 { label="Exchange"; 114 [labelType="html" label="ShuffledRowRDD [114] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster263 { label="WholeStageCodegen (2)"; 115 [labelType="html" label="MapPartitionsRDD [115] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster276 { label="Exchange"; 120 [labelType="html" label="ShuffledRowRDD [120] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster273 { label="WholeStageCodegen (4)"; 121 [labelType="html" label="MapPartitionsRDD [121] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster258 { label="WholeStageCodegen (5)"; 122 [labelType="html" label="ZippedPartitionsRDD2 [122] csv at NativeMethodAccessorImpl.java:0"]; 123 [labelType="html" label="MapPartitionsRDD [123] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster257 { label="Exchange"; 124 [labelType="html" label="MapPartitionsRDD [124] csv at NativeMethodAccessorImpl.java:0"]; } } 114->115; 120->121; 115->122; 121->122; 122->123; 123->124; }

113,114

119,120

digraph G { subgraph clusterstage_38 { label="Stage 38"; subgraph cluster257 { label="Exchange"; 125 [labelType="html" label="ShuffledRowRDD [125] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster254 { label="WholeStageCodegen (6)"; 126 [labelType="html" label="MapPartitionsRDD [126] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster253 { label="Exchange"; 130 [labelType="html" label="MapPartitionsRDD [130] csv at NativeMethodAccessorImpl.java:0"]; } } 125->126; 126->130; }

124,125

digraph G { subgraph clusterstage_39 { label="Stage 39"; subgraph cluster253 { label="Exchange"; 131 [labelType="html" label="ShuffledRowRDD [131] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster302 { label="WholeStageCodegen (7)"; 133 [labelType="html" label="MapPartitionsRDD [133] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster249 { label="Coalesce"; 134 [labelType="html" label="CoalescedRDD [134] csv at NativeMethodAccessorImpl.java:0"]; } } 131->133; 133->134; }

130,131

Completed Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
39	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/10/29 20:23:40	1 s	1/1		1038.0 B	2.5 KiB
38	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/10/29 20:23:40	0.5 s	200/200			467.5 KiB	2.5 KiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

csv at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/10/29 20:23:40

1 s

1/1

1038.0 B

2.5 KiB

csv at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/10/29 20:23:40

0.5 s

200/200

467.5 KiB

2.5 KiB

Page:

Skipped Stages (3)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
37	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200
36	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/2
35	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/5

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

csv at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

Unknown

0/200

csv at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

Unknown

0/2

csv at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

Unknown

0/5

Page: