task2 - Details for Job 9

Details for Job 9

Status: SUCCEEDED
Associated SQL Query: 6
Completed Stages: 2
Skipped Stages: 4

Enable zooming

digraph G { subgraph clusterstage_21 { label="Stage 21 (skipped)"; subgraph cluster209 { label="Scan csv "; 78 [labelType="html" label="FileScanRDD [78] csv at NativeMethodAccessorImpl.java:0"]; 79 [labelType="html" label="MapPartitionsRDD [79] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster205 { label="WholeStageCodegen (3)"; 80 [labelType="html" label="MapPartitionsRDD [80] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster204 { label="Exchange"; 81 [labelType="html" label="MapPartitionsRDD [81] csv at NativeMethodAccessorImpl.java:0"]; } } 78->79; 79->80; 80->81; }

digraph G { subgraph clusterstage_22 { label="Stage 22 (skipped)"; subgraph cluster199 { label="Scan csv "; 72 [labelType="html" label="FileScanRDD [72] csv at NativeMethodAccessorImpl.java:0"]; 73 [labelType="html" label="MapPartitionsRDD [73] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster195 { label="WholeStageCodegen (1)"; 74 [labelType="html" label="MapPartitionsRDD [74] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster194 { label="Exchange"; 75 [labelType="html" label="MapPartitionsRDD [75] csv at NativeMethodAccessorImpl.java:0"]; } } 72->73; 73->74; 74->75; }

digraph G { subgraph clusterstage_23 { label="Stage 23 (skipped)"; subgraph cluster194 { label="Exchange"; 76 [labelType="html" label="ShuffledRowRDD [76] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster191 { label="WholeStageCodegen (2)"; 77 [labelType="html" label="MapPartitionsRDD [77] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster204 { label="Exchange"; 82 [labelType="html" label="ShuffledRowRDD [82] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster201 { label="WholeStageCodegen (4)"; 83 [labelType="html" label="MapPartitionsRDD [83] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster186 { label="WholeStageCodegen (5)"; 84 [labelType="html" label="ZippedPartitionsRDD2 [84] csv at NativeMethodAccessorImpl.java:0"]; 85 [labelType="html" label="MapPartitionsRDD [85] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster185 { label="Exchange"; 86 [labelType="html" label="MapPartitionsRDD [86] csv at NativeMethodAccessorImpl.java:0"]; } } 76->77; 82->83; 77->84; 83->84; 84->85; 85->86; }

75,76

81,82

digraph G { subgraph clusterstage_24 { label="Stage 24 (skipped)"; subgraph cluster185 { label="Exchange"; 87 [labelType="html" label="ShuffledRowRDD [87] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster181 { label="WholeStageCodegen (6)"; 88 [labelType="html" label="MapPartitionsRDD [88] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster180 { label="Exchange"; 89 [labelType="html" label="MapPartitionsRDD [89] csv at NativeMethodAccessorImpl.java:0"]; } } 87->88; 88->89; }

86,87

digraph G { subgraph clusterstage_25 { label="Stage 25"; subgraph cluster180 { label="Exchange"; 90 [labelType="html" label="ShuffledRowRDD [90] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster177 { label="WholeStageCodegen (7)"; 91 [labelType="html" label="MapPartitionsRDD [91] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster176 { label="Exchange"; 95 [labelType="html" label="MapPartitionsRDD [95] csv at NativeMethodAccessorImpl.java:0"]; } } 90->91; 91->95; }

89,90

digraph G { subgraph clusterstage_26 { label="Stage 26"; subgraph cluster176 { label="Exchange"; 96 [labelType="html" label="ShuffledRowRDD [96] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster173 { label="WholeStageCodegen (8)"; 97 [labelType="html" label="MapPartitionsRDD [97] csv at NativeMethodAccessorImpl.java:0"]; } } 96->97; }

95,96

Completed Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
26	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/04 20:24:01	14 s	30/30		2.2 KiB	2.5 KiB
25	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/04 20:24:00	0.9 s	200/200			425.5 KiB	2.5 KiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

csv at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/04 20:24:01

14 s

30/30

2.2 KiB

2.5 KiB

csv at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/04 20:24:00

0.9 s

200/200

425.5 KiB

2.5 KiB

Page:

Skipped Stages (4)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
24	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200
23	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200
22	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/2
21	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/5

Page: