NasaLogSparkStreaming - Details for Query 61

Details for Query 61

Submitted Time: 2024/12/07 23:21:24
Duration: 95 ms
Succeeded Jobs: 20

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" WriteToDataSourceV2 "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n1 ms (0 ms, 1 ms, 1 ms (stage 20.0: task 40))"; 2 [labelType="html" label=" Project "]; } 3 [labelType="html" label=" EventTimeWatermark "]; subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n4 ms (2 ms, 2 ms, 2 ms (stage 20.0: task 41))"; 5 [labelType="html" label=" Project "]; } 6 [labelType="html" label="Generate number of output rows: 1"]; subgraph cluster7 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n12 ms (6 ms, 6 ms, 6 ms (stage 20.0: task 41))"; 8 [labelType="html" label=" Project "]; 9 [labelType="html" label=" Project "]; } 10 [labelType="html" label="MicroBatchScan number of output rows: 1"]; 2->0; 3->2; 5->3; 6->5; 8->6; 9->8; 10->9; }

WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@3f9f929d

Project [logs#19, processing_time#14-T3000ms, split(logs#19, ,, -1)[0] AS idx#22, split(logs#19, ,, -1)[1] AS hostname#26, split(logs#19, ,, -1)[2] AS time#31, split(logs#19, ,, -1)[3] AS method#37, split(logs#19, ,, -1)[4] AS url#44, split(logs#19, ,, -1)[5] AS responsecode#52, split(logs#19, ,, -1)[6] AS bytes#61]

WholeStageCodegen (3)

EventTimeWatermark processing_time#14: timestamp, 3 seconds

Project [logs#19, processing_time#14]

WholeStageCodegen (2)

Generate explode(split(raw_logs#4, , -1)), [processing_time#14], false, [logs#19]

Project [value#0 AS raw_logs#4, cast(split(value#0, , -1)[0] as timestamp) AS processing_time#14]

Project [value#0, timestamp#1]

WholeStageCodegen (1)

MicroBatchScan[value#0, timestamp#1] class org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1

Details

== Parsed Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@3f9f929d
+- Project [logs#19, processing_time#14-T3000ms, idx#22, hostname#26, time#31, method#37, url#44, responsecode#52, split(logs#19, ,, -1)[6] AS bytes#61]
   +- Project [logs#19, processing_time#14-T3000ms, idx#22, hostname#26, time#31, method#37, url#44, split(logs#19, ,, -1)[5] AS responsecode#52]
      +- Project [logs#19, processing_time#14-T3000ms, idx#22, hostname#26, time#31, method#37, split(logs#19, ,, -1)[4] AS url#44]
         +- Project [logs#19, processing_time#14-T3000ms, idx#22, hostname#26, time#31, split(logs#19, ,, -1)[3] AS method#37]
            +- Project [logs#19, processing_time#14-T3000ms, idx#22, hostname#26, split(logs#19, ,, -1)[2] AS time#31]
               +- Project [logs#19, processing_time#14-T3000ms, idx#22, split(logs#19, ,, -1)[1] AS hostname#26]
                  +- Project [logs#19, processing_time#14-T3000ms, split(logs#19, ,, -1)[0] AS idx#22]
                     +- EventTimeWatermark processing_time#14: timestamp, 3 seconds
                        +- Project [logs#19, processing_time#14]
                           +- Generate explode(split(raw_logs#4,  , -1)), false, [logs#19]
                              +- Project [raw_logs#4, timestamp#1, cast(split(raw_logs#4,  , -1)[0] as timestamp) AS processing_time#14]
                                 +- Project [raw_logs#4, timestamp#1, current_batch_timestamp(1733613684817, TimestampType, Some(Dummy TimeZoneId)) AS processing_time#7]
                                    +- Project [value#0 AS raw_logs#4, timestamp#1]
                                       +- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@565361a2, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], 20, 21

== Analyzed Logical Plan ==

WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@3f9f929d
+- Project [logs#19, processing_time#14-T3000ms, idx#22, hostname#26, time#31, method#37, url#44, responsecode#52, split(logs#19, ,, -1)[6] AS bytes#61]
   +- Project [logs#19, processing_time#14-T3000ms, idx#22, hostname#26, time#31, method#37, url#44, split(logs#19, ,, -1)[5] AS responsecode#52]
      +- Project [logs#19, processing_time#14-T3000ms, idx#22, hostname#26, time#31, method#37, split(logs#19, ,, -1)[4] AS url#44]
         +- Project [logs#19, processing_time#14-T3000ms, idx#22, hostname#26, time#31, split(logs#19, ,, -1)[3] AS method#37]
            +- Project [logs#19, processing_time#14-T3000ms, idx#22, hostname#26, split(logs#19, ,, -1)[2] AS time#31]
               +- Project [logs#19, processing_time#14-T3000ms, idx#22, split(logs#19, ,, -1)[1] AS hostname#26]
                  +- Project [logs#19, processing_time#14-T3000ms, split(logs#19, ,, -1)[0] AS idx#22]
                     +- EventTimeWatermark processing_time#14: timestamp, 3 seconds
                        +- Project [logs#19, processing_time#14]
                           +- Generate explode(split(raw_logs#4,  , -1)), false, [logs#19]
                              +- Project [raw_logs#4, timestamp#1, cast(split(raw_logs#4,  , -1)[0] as timestamp) AS processing_time#14]
                                 +- Project [raw_logs#4, timestamp#1, current_batch_timestamp(1733613684817, TimestampType, Some(Dummy TimeZoneId)) AS processing_time#7]
                                    +- Project [value#0 AS raw_logs#4, timestamp#1]
                                       +- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@565361a2, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], 20, 21

== Optimized Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@3f9f929d
+- Project [logs#19, processing_time#14-T3000ms, split(logs#19, ,, -1)[0] AS idx#22, split(logs#19, ,, -1)[1] AS hostname#26, split(logs#19, ,, -1)[2] AS time#31, split(logs#19, ,, -1)[3] AS method#37, split(logs#19, ,, -1)[4] AS url#44, split(logs#19, ,, -1)[5] AS responsecode#52, split(logs#19, ,, -1)[6] AS bytes#61]
   +- EventTimeWatermark processing_time#14: timestamp, 3 seconds
      +- Project [logs#19, processing_time#14]
         +- Generate explode(split(raw_logs#4,  , -1)), [0], false, [logs#19]
            +- Project [value#0 AS raw_logs#4, cast(split(value#0,  , -1)[0] as timestamp) AS processing_time#14]
               +- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@565361a2, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], 20, 21

== Physical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@3f9f929d
+- *(3) Project [logs#19, processing_time#14-T3000ms, split(logs#19, ,, -1)[0] AS idx#22, split(logs#19, ,, -1)[1] AS hostname#26, split(logs#19, ,, -1)[2] AS time#31, split(logs#19, ,, -1)[3] AS method#37, split(logs#19, ,, -1)[4] AS url#44, split(logs#19, ,, -1)[5] AS responsecode#52, split(logs#19, ,, -1)[6] AS bytes#61]
   +- EventTimeWatermark processing_time#14: timestamp, 3 seconds
      +- *(2) Project [logs#19, processing_time#14]
         +- Generate explode(split(raw_logs#4,  , -1)), [processing_time#14], false, [logs#19]
            +- *(1) Project [value#0 AS raw_logs#4, cast(split(value#0,  , -1)[0] as timestamp) AS processing_time#14]
               +- *(1) Project [value#0, timestamp#1]
                  +- MicroBatchScan[value#0, timestamp#1] class org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1