task2 - Details for Job 6

Details for Job 6

Status: SUCCEEDED
Associated SQL Query: 5
Completed Stages: 2
Skipped Stages: 4

Enable zooming

digraph G { subgraph clusterstage_14 { label="Stage 14 (skipped)"; subgraph cluster129 { label="Scan csv "; 50 [labelType="html" label="FileScanRDD [50] csv at NativeMethodAccessorImpl.java:0"]; 51 [labelType="html" label="MapPartitionsRDD [51] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster125 { label="WholeStageCodegen (1)"; 52 [labelType="html" label="MapPartitionsRDD [52] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster124 { label="Exchange"; 53 [labelType="html" label="MapPartitionsRDD [53] csv at NativeMethodAccessorImpl.java:0"]; } } 50->51; 51->52; 52->53; }

digraph G { subgraph clusterstage_15 { label="Stage 15 (skipped)"; subgraph cluster139 { label="Scan csv "; 56 [labelType="html" label="FileScanRDD [56] csv at NativeMethodAccessorImpl.java:0"]; 57 [labelType="html" label="MapPartitionsRDD [57] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster135 { label="WholeStageCodegen (3)"; 58 [labelType="html" label="MapPartitionsRDD [58] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster134 { label="Exchange"; 59 [labelType="html" label="MapPartitionsRDD [59] csv at NativeMethodAccessorImpl.java:0"]; } } 56->57; 57->58; 58->59; }

digraph G { subgraph clusterstage_16 { label="Stage 16 (skipped)"; subgraph cluster124 { label="Exchange"; 54 [labelType="html" label="ShuffledRowRDD [54] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster121 { label="WholeStageCodegen (2)"; 55 [labelType="html" label="MapPartitionsRDD [55] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster134 { label="Exchange"; 60 [labelType="html" label="ShuffledRowRDD [60] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster131 { label="WholeStageCodegen (4)"; 61 [labelType="html" label="MapPartitionsRDD [61] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (5)"; 62 [labelType="html" label="ZippedPartitionsRDD2 [62] csv at NativeMethodAccessorImpl.java:0"]; 63 [labelType="html" label="MapPartitionsRDD [63] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="Exchange"; 64 [labelType="html" label="MapPartitionsRDD [64] csv at NativeMethodAccessorImpl.java:0"]; } } 54->55; 60->61; 55->62; 61->62; 62->63; 63->64; }

53,54

59,60

digraph G { subgraph clusterstage_17 { label="Stage 17 (skipped)"; subgraph cluster115 { label="Exchange"; 65 [labelType="html" label="ShuffledRowRDD [65] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster111 { label="WholeStageCodegen (6)"; 66 [labelType="html" label="MapPartitionsRDD [66] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster110 { label="Exchange"; 67 [labelType="html" label="MapPartitionsRDD [67] csv at NativeMethodAccessorImpl.java:0"]; } } 65->66; 66->67; }

64,65

digraph G { subgraph clusterstage_18 { label="Stage 18"; subgraph cluster110 { label="Exchange"; 68 [labelType="html" label="ShuffledRowRDD [68] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster107 { label="WholeStageCodegen (7)"; 69 [labelType="html" label="MapPartitionsRDD [69] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster106 { label="Exchange"; 73 [labelType="html" label="MapPartitionsRDD [73] csv at NativeMethodAccessorImpl.java:0"]; } } 68->69; 69->73; }

67,68

digraph G { subgraph clusterstage_19 { label="Stage 19"; subgraph cluster106 { label="Exchange"; 74 [labelType="html" label="ShuffledRowRDD [74] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster163 { label="WholeStageCodegen (8)"; 76 [labelType="html" label="MapPartitionsRDD [76] csv at NativeMethodAccessorImpl.java:0"]; } subgraph cluster102 { label="Coalesce"; 77 [labelType="html" label="CoalescedRDD [77] csv at NativeMethodAccessorImpl.java:0"]; } } 74->76; 76->77; }

73,74

Completed Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
19	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/10/29 20:23:19	1 s	1/1		661.0 B	2.5 KiB
18	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/10/29 20:23:19	0.5 s	200/200			425.4 KiB	2.5 KiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

csv at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/10/29 20:23:19

1 s

1/1

661.0 B

2.5 KiB

csv at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/10/29 20:23:19

0.5 s

200/200

425.4 KiB

2.5 KiB

Page:

Skipped Stages (4)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
17	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200
16	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200
15	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/2
14	csv at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:953) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/5

Page: