NasaLogSparkStreaming - Details for Job 5

Details for Job 5

Status: FAILED
Associated SQL Query: 16
Job Group: 10bd72d4-04b3-48e3-97a6-62b1f13a8094
Completed Stages: 1
Failed Stages: 1

Enable zooming

digraph G { subgraph clusterstage_10 { label="Stage 10"; subgraph cluster225 { label="MicroBatchScan"; 70 [labelType="html" label="DataSourceRDD [70] start at NativeMethodAccessorImpl.java:0"]; 71 [labelType="html" label="MapPartitionsRDD [71] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster222 { label="WholeStageCodegen (1)"; 72 [labelType="html" label="MapPartitionsRDD [72] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster221 { label="Generate"; 73 [labelType="html" label="MapPartitionsRDD [73] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster217 { label="WholeStageCodegen (2)"; 74 [labelType="html" label="MapPartitionsRDD [74] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster216 { label="EventTimeWatermark"; 75 [labelType="html" label="MapPartitionsRDD [75] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster212 { label="WholeStageCodegen (3)"; 76 [labelType="html" label="MapPartitionsRDD [76] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster211 { label="Exchange"; 77 [labelType="html" label="MapPartitionsRDD [77] start at NativeMethodAccessorImpl.java:0"]; } } 70->71; 71->72; 72->73; 73->74; 74->75; 75->76; 76->77; }

digraph G { subgraph clusterstage_11 { label="Stage 11"; subgraph cluster211 { label="Exchange"; 78 [labelType="html" label="ShuffledRowRDD [78] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster208 { label="WholeStageCodegen (4)"; 79 [labelType="html" label="MapPartitionsRDD [79] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster207 { label="StateStoreRestore"; 80 [labelType="html" label="StateStoreRDD [80] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster204 { label="WholeStageCodegen (5)"; 81 [labelType="html" label="MapPartitionsRDD [81] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster203 { label="StateStoreSave"; 82 [labelType="html" label="StateStoreRDD [82] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster200 { label="WholeStageCodegen (6)"; 83 [labelType="html" label="MapPartitionsRDD [83] start at NativeMethodAccessorImpl.java:0"]; } } 78->79; 79->80; 80->81; 81->82; 82->83; }

77,78

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
10	id = 98ef06b2-e87a-47c3-b836-bcf5b48519fb runId = 10bd72d4-04b3-48e3-97a6-62b1f13a8094 batch = 5 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/04 20:24:00	39 ms	2/2				160.0 B

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

id = 98ef06b2-e87a-47c3-b836-bcf5b48519fb runId = 10bd72d4-04b3-48e3-97a6-62b1f13a8094 batch = 5 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/04 20:24:00

39 ms

2/2

160.0 B

Page:

Failed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write	Failure Reason
11	id = 98ef06b2-e87a-47c3-b836-bcf5b48519fb runId = 10bd72d4-04b3-48e3-97a6-62b1f13a8094 batch = 5 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/04 20:24:00	2 s	47/200 (2 running)					Job 5 cancelled part of cancelled job group 10bd72d4-04b3-48e3-97a6-62b1f13a8094

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

Failure Reason

id = 98ef06b2-e87a-47c3-b836-bcf5b48519fb runId = 10bd72d4-04b3-48e3-97a6-62b1f13a8094 batch = 5 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/04 20:24:00

2 s

47/200 (2 running)

Job 5 cancelled part of cancelled job group 10bd72d4-04b3-48e3-97a6-62b1f13a8094

Page: