NasaLogSparkStreaming - Details for Job 20

Details for Job 20

Status: FAILED
Associated SQL Query: 51
Job Group: 2ad6ec75-02a9-40e4-8f05-f0dd3dbf4b8d
Skipped Stages: 1
Failed Stages: 1

Enable zooming

digraph G { subgraph clusterstage_31 { label="Stage 31"; subgraph cluster480 { label="MicroBatchScan"; 181 [labelType="html" label="DataSourceRDD [181] start at NativeMethodAccessorImpl.java:0"]; 182 [labelType="html" label="MapPartitionsRDD [182] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster477 { label="WholeStageCodegen (1)"; 183 [labelType="html" label="MapPartitionsRDD [183] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster476 { label="Generate"; 184 [labelType="html" label="MapPartitionsRDD [184] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster472 { label="WholeStageCodegen (2)"; 185 [labelType="html" label="MapPartitionsRDD [185] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster471 { label="EventTimeWatermark"; 186 [labelType="html" label="MapPartitionsRDD [186] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster465 { label="WholeStageCodegen (3)"; 187 [labelType="html" label="MapPartitionsRDD [187] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster464 { label="Exchange"; 188 [labelType="html" label="MapPartitionsRDD [188] start at NativeMethodAccessorImpl.java:0"]; } } 181->182; 182->183; 183->184; 184->185; 185->186; 186->187; 187->188; }

digraph G { subgraph clusterstage_32 { label="Stage 32 (skipped)"; subgraph cluster464 { label="Exchange"; 189 [labelType="html" label="ShuffledRowRDD [189] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster461 { label="WholeStageCodegen (4)"; 190 [labelType="html" label="MapPartitionsRDD [190] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster460 { label="StateStoreRestore"; 191 [labelType="html" label="StateStoreRDD [191] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster457 { label="WholeStageCodegen (5)"; 192 [labelType="html" label="MapPartitionsRDD [192] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster456 { label="StateStoreSave"; 193 [labelType="html" label="StateStoreRDD [193] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster453 { label="WholeStageCodegen (6)"; 194 [labelType="html" label="MapPartitionsRDD [194] start at NativeMethodAccessorImpl.java:0"]; } } 189->190; 190->191; 191->192; 192->193; 193->194; }

188,189

Skipped Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
32	start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

Unknown

0/200

Page:

Failed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write	Failure Reason
31	id = 3cd2ce6a-cda3-4e07-9f6b-31eb0ac00a91 runId = 2ad6ec75-02a9-40e4-8f05-f0dd3dbf4b8d batch = 3 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/27 14:33:37	Unknown	0/2					Stage cancelled because SparkContext was shut down

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

Failure Reason

id = 3cd2ce6a-cda3-4e07-9f6b-31eb0ac00a91 runId = 2ad6ec75-02a9-40e4-8f05-f0dd3dbf4b8d batch = 3 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/27 14:33:37

Unknown

0/2

Stage cancelled because SparkContext was shut down

Page: