task2 - Details for Job 9

Details for Job 9

Status: SUCCEEDED
Associated SQL Query: 6
Completed Stages: 2
Skipped Stages: 4

Enable zooming

digraph G { subgraph clusterstage_22 { label="Stage 22 (skipped)"; subgraph cluster220 { label="Scan csv "; 83 [labelType="html" label="FileScanRDD [83] csv at NativeMethodAccessorImpl.java:0"]; 84 [labelType="html" label="MapPartitionsRDD [84] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster216 { label="WholeStageCodegen (3)"; 85 [labelType="html" label="MapPartitionsRDD [85] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster215 { label="Exchange"; 86 [labelType="html" label="MapPartitionsRDD [86] csv at NativeMethodAccessorImpl.java:0"]; } } 83->84; 84->85; 85->86; }

digraph G { subgraph clusterstage_23 { label="Stage 23 (skipped)"; subgraph cluster210 { label="Scan csv "; 77 [labelType="html" label="FileScanRDD [77] csv at NativeMethodAccessorImpl.java:0"]; 78 [labelType="html" label="MapPartitionsRDD [78] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster206 { label="WholeStageCodegen (1)"; 79 [labelType="html" label="MapPartitionsRDD [79] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster205 { label="Exchange"; 80 [labelType="html" label="MapPartitionsRDD [80] csv at NativeMethodAccessorImpl.java:0"]; } } 77->78; 78->79; 79->80; }

digraph G { subgraph clusterstage_24 { label="Stage 24 (skipped)"; subgraph cluster205 { label="Exchange"; 81 [labelType="html" label="ShuffledRowRDD [81] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster202 { label="WholeStageCodegen (2)"; 82 [labelType="html" label="MapPartitionsRDD [82] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster215 { label="Exchange"; 87 [labelType="html" label="ShuffledRowRDD [87] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster212 { label="WholeStageCodegen (4)"; 88 [labelType="html" label="MapPartitionsRDD [88] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster196 { label="WholeStageCodegen (5)"; 89 [labelType="html" label="ZippedPartitionsRDD2 [89] csv at NativeMethodAccessorImpl.java:0"]; 90 [labelType="html" label="MapPartitionsRDD [90] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster195 { label="Exchange"; 91 [labelType="html" label="MapPartitionsRDD [91] csv at NativeMethodAccessorImpl.java:0"]; } } 81->82; 87->88; 82->89; 88->89; 89->90; 90->91; }

80,81

86,87

digraph G { subgraph clusterstage_25 { label="Stage 25 (skipped)"; subgraph cluster195 { label="Exchange"; 92 [labelType="html" label="ShuffledRowRDD [92] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster191 { label="WholeStageCodegen (6)"; 93 [labelType="html" label="MapPartitionsRDD [93] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster190 { label="Exchange"; 94 [labelType="html" label="MapPartitionsRDD [94] csv at NativeMethodAccessorImpl.java:0"]; } } 92->93; 93->94; }

91,92

digraph G { subgraph clusterstage_26 { label="Stage 26"; subgraph cluster190 { label="Exchange"; 95 [labelType="html" label="ShuffledRowRDD [95] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster187 { label="WholeStageCodegen (7)"; 96 [labelType="html" label="MapPartitionsRDD [96] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster186 { label="Exchange"; 100 [labelType="html" label="MapPartitionsRDD [100] csv at NativeMethodAccessorImpl.java:0"]; } } 95->96; 96->100; }

94,95

digraph G { subgraph clusterstage_27 { label="Stage 27"; subgraph cluster186 { label="Exchange"; 101 [labelType="html" label="ShuffledRowRDD [101] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster244 { label="WholeStageCodegen (8)"; 103 [labelType="html" label="MapPartitionsRDD [103] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster182 { label="Coalesce"; 104 [labelType="html" label="CoalescedRDD [104] csv at NativeMethodAccessorImpl.java:0"]; } } 101->103; 103->104; }

100,101

Completed Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
27	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/04 20:23:08	1 s	1/1		661.0 B	2.5 KiB
26	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/04 20:23:07	0.8 s	200/200			430.0 KiB	2.5 KiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

csv at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/04 20:23:08

1 s

1/1

661.0 B

2.5 KiB

csv at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/04 20:23:07

0.8 s

200/200

430.0 KiB

2.5 KiB

Page:

Skipped Stages (4)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
25	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200
24	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200
23	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/2
22	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/5

Page: