NasaLogSparkStreaming - Details for Job 14

Details for Job 14

Status: SUCCEEDED
Associated SQL Query: 22
Job Group: 8a4155da-36fe-4885-93fd-d46a110c0886
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_35 { label="Stage 35"; subgraph cluster402 { label="MicroBatchScan"; 140 [labelType="html" label="DataSourceRDD [140] start at NativeMethodAccessorImpl.java:0"]; 141 [labelType="html" label="MapPartitionsRDD [141] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster399 { label="WholeStageCodegen (1)"; 142 [labelType="html" label="MapPartitionsRDD [142] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster398 { label="Generate"; 143 [labelType="html" label="MapPartitionsRDD [143] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster395 { label="WholeStageCodegen (2)"; 144 [labelType="html" label="MapPartitionsRDD [144] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster394 { label="EventTimeWatermark"; 145 [labelType="html" label="MapPartitionsRDD [145] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster387 { label="WholeStageCodegen (3)"; 146 [labelType="html" label="MapPartitionsRDD [146] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster386 { label="Exchange"; 147 [labelType="html" label="MapPartitionsRDD [147] start at NativeMethodAccessorImpl.java:0"]; } } 140->141; 141->142; 142->143; 143->144; 144->145; 145->146; 146->147; }

digraph G { subgraph clusterstage_36 { label="Stage 36"; subgraph cluster386 { label="Exchange"; 148 [labelType="html" label="ShuffledRowRDD [148] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster383 { label="WholeStageCodegen (4)"; 149 [labelType="html" label="MapPartitionsRDD [149] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster382 { label="StateStoreRestore"; 150 [labelType="html" label="StateStoreRDD [150] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster379 { label="WholeStageCodegen (5)"; 151 [labelType="html" label="MapPartitionsRDD [151] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster378 { label="StateStoreSave"; 152 [labelType="html" label="StateStoreRDD [152] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster375 { label="WholeStageCodegen (6)"; 153 [labelType="html" label="MapPartitionsRDD [153] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster374 { label="Exchange"; 154 [labelType="html" label="MapPartitionsRDD [154] start at NativeMethodAccessorImpl.java:0"]; 155 [labelType="html" label="MapPartitionsRDD [155] start at NativeMethodAccessorImpl.java:0"]; 156 [labelType="html" label="MapPartitionsRDD [156] start at NativeMethodAccessorImpl.java:0"]; } } 148->149; 149->150; 150->151; 151->152; 152->153; 153->154; 154->155; 155->156; }

147,148

Completed Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
36	id = c9796033-7dc1-49ea-a30d-3d85ced97d46 runId = 8a4155da-36fe-4885-93fd-d46a110c0886 batch = 7 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/12 00:10:16	6 s	200/200			490.0 B
35	id = c9796033-7dc1-49ea-a30d-3d85ced97d46 runId = 8a4155da-36fe-4885-93fd-d46a110c0886 batch = 7 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/12 00:10:16	35 ms	2/2				490.0 B

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

id = c9796033-7dc1-49ea-a30d-3d85ced97d46 runId = 8a4155da-36fe-4885-93fd-d46a110c0886 batch = 7 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/12 00:10:16

6 s

200/200

490.0 B

id = c9796033-7dc1-49ea-a30d-3d85ced97d46 runId = 8a4155da-36fe-4885-93fd-d46a110c0886 batch = 7 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/12 00:10:16

35 ms

2/2

490.0 B

Page: