NasaLogSparkStreaming - Details for Job 8

Details for Job 8

Status: SUCCEEDED
Associated SQL Query: 13
Job Group: 8a4155da-36fe-4885-93fd-d46a110c0886
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_20 { label="Stage 20"; subgraph cluster243 { label="MicroBatchScan"; 80 [labelType="html" label="DataSourceRDD [80] start at NativeMethodAccessorImpl.java:0"]; 81 [labelType="html" label="MapPartitionsRDD [81] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster240 { label="WholeStageCodegen (1)"; 82 [labelType="html" label="MapPartitionsRDD [82] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster239 { label="Generate"; 83 [labelType="html" label="MapPartitionsRDD [83] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster236 { label="WholeStageCodegen (2)"; 84 [labelType="html" label="MapPartitionsRDD [84] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster235 { label="EventTimeWatermark"; 85 [labelType="html" label="MapPartitionsRDD [85] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster228 { label="WholeStageCodegen (3)"; 86 [labelType="html" label="MapPartitionsRDD [86] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster227 { label="Exchange"; 87 [labelType="html" label="MapPartitionsRDD [87] start at NativeMethodAccessorImpl.java:0"]; } } 80->81; 81->82; 82->83; 83->84; 84->85; 85->86; 86->87; }

digraph G { subgraph clusterstage_21 { label="Stage 21"; subgraph cluster227 { label="Exchange"; 88 [labelType="html" label="ShuffledRowRDD [88] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster224 { label="WholeStageCodegen (4)"; 89 [labelType="html" label="MapPartitionsRDD [89] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster223 { label="StateStoreRestore"; 90 [labelType="html" label="StateStoreRDD [90] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster220 { label="WholeStageCodegen (5)"; 91 [labelType="html" label="MapPartitionsRDD [91] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster219 { label="StateStoreSave"; 92 [labelType="html" label="StateStoreRDD [92] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster216 { label="WholeStageCodegen (6)"; 93 [labelType="html" label="MapPartitionsRDD [93] start at NativeMethodAccessorImpl.java:0"]; } subgraph cluster215 { label="Exchange"; 94 [labelType="html" label="MapPartitionsRDD [94] start at NativeMethodAccessorImpl.java:0"]; 95 [labelType="html" label="MapPartitionsRDD [95] start at NativeMethodAccessorImpl.java:0"]; 96 [labelType="html" label="MapPartitionsRDD [96] start at NativeMethodAccessorImpl.java:0"]; } } 88->89; 89->90; 90->91; 91->92; 92->93; 93->94; 94->95; 95->96; }

87,88

Completed Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
21	id = c9796033-7dc1-49ea-a30d-3d85ced97d46 runId = 8a4155da-36fe-4885-93fd-d46a110c0886 batch = 4 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/12 00:09:35	6 s	200/200			641.0 B
20	id = c9796033-7dc1-49ea-a30d-3d85ced97d46 runId = 8a4155da-36fe-4885-93fd-d46a110c0886 batch = 4 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2024/12/12 00:09:35	37 ms	2/2				641.0 B

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

id = c9796033-7dc1-49ea-a30d-3d85ced97d46 runId = 8a4155da-36fe-4885-93fd-d46a110c0886 batch = 4 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/12 00:09:35

6 s

200/200

641.0 B

id = c9796033-7dc1-49ea-a30d-3d85ced97d46 runId = 8a4155da-36fe-4885-93fd-d46a110c0886 batch = 4 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:366)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2024/12/12 00:09:35

37 ms

2/2

641.0 B

Page: