• Submitted Time: 2024/12/04 20:21:04
• Duration: 15 s

== Parsed Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@8118c7
+- Sort [total_bytes#858 DESC NULLS LAST], true
   +- Aggregate [window#859-T3000ms, hostname#140], [window#859-T3000ms AS window#847-T3000ms, hostname#140, sum(cast(bytes#175 as double)) AS total_bytes#858]
      +- Filter ((timestamp#1-T3000ms >= window#859-T3000ms.start) AND (timestamp#1-T3000ms < window#859-T3000ms.end))
         +- Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175)], [window#859-T3000ms, logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175]
            +- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, split(logs#5, ,, -1)[6] AS bytes#175]
               +- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, split(logs#5, ,, -1)[5] AS responsecode#166]
                  +- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, split(logs#5, ,, -1)[4] AS url#158]
                     +- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, split(logs#5, ,, -1)[3] AS method#151]
                        +- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, split(logs#5, ,, -1)[2] AS time#145]
                           +- Project [logs#5, timestamp#1-T3000ms, idx#136, split(logs#5, ,, -1)[1] AS hostname#140]
                              +- Project [logs#5, timestamp#1-T3000ms, split(logs#5, ,, -1)[0] AS idx#136]
                                 +- EventTimeWatermark timestamp#1: timestamp, 3 seconds
                                    +- Project [logs#5, timestamp#1]
                                       +- Generate explode(split(value#0,  , -1)), false, [logs#5]
                                          +- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@6b3ca49a, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], 108, 123

== Analyzed Logical Plan ==

WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@8118c7
+- Sort [total_bytes#858 DESC NULLS LAST], true
   +- Aggregate [window#859-T3000ms, hostname#140], [window#859-T3000ms AS window#847-T3000ms, hostname#140, sum(cast(bytes#175 as double)) AS total_bytes#858]
      +- Filter ((timestamp#1-T3000ms >= window#859-T3000ms.start) AND (timestamp#1-T3000ms < window#859-T3000ms.end))
         +- Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175)], [window#859-T3000ms, logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175]
            +- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, split(logs#5, ,, -1)[6] AS bytes#175]
               +- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, split(logs#5, ,, -1)[5] AS responsecode#166]
                  +- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, split(logs#5, ,, -1)[4] AS url#158]
                     +- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, split(logs#5, ,, -1)[3] AS method#151]
                        +- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, split(logs#5, ,, -1)[2] AS time#145]
                           +- Project [logs#5, timestamp#1-T3000ms, idx#136, split(logs#5, ,, -1)[1] AS hostname#140]
                              +- Project [logs#5, timestamp#1-T3000ms, split(logs#5, ,, -1)[0] AS idx#136]
                                 +- EventTimeWatermark timestamp#1: timestamp, 3 seconds
                                    +- Project [logs#5, timestamp#1]
                                       +- Generate explode(split(value#0,  , -1)), false, [logs#5]
                                          +- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@6b3ca49a, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], 108, 123

== Optimized Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@8118c7
+- Sort [total_bytes#858 DESC NULLS LAST], true
   +- Aggregate [window#859-T3000ms, hostname#140], [window#859-T3000ms AS window#847-T3000ms, hostname#140, sum(cast(bytes#175 as double)) AS total_bytes#858]
      +- Project [window#859-T3000ms, hostname#140, bytes#175]
         +- Filter (((isnotnull(timestamp#1-T3000ms) AND isnotnull(window#859-T3000ms)) AND (timestamp#1-T3000ms >= window#859-T3000ms.start)) AND (timestamp#1-T3000ms < window#859-T3000ms.end))
            +- Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms, hostname#140, bytes#175), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms, hostname#140, bytes#175)], [window#859-T3000ms, timestamp#1-T3000ms, hostname#140, bytes#175]
               +- Project [timestamp#1-T3000ms, split(logs#5, ,, -1)[1] AS hostname#140, split(logs#5, ,, -1)[6] AS bytes#175]
                  +- EventTimeWatermark timestamp#1: timestamp, 3 seconds
                     +- Project [logs#5, timestamp#1]
                        +- Generate explode(split(value#0,  , -1)), [0], false, [logs#5]
                           +- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@6b3ca49a, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], 108, 123

== Physical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@8118c7
+- *(7) Sort [total_bytes#858 DESC NULLS LAST], true, 0
   +- Exchange rangepartitioning(total_bytes#858 DESC NULLS LAST, 200), true, [id=#1878]
      +- *(6) HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[sum(cast(bytes#175 as double))], output=[window#847-T3000ms, hostname#140, total_bytes#858])
         +- StateStoreSave [window#859-T3000ms, hostname#140], state info [ checkpoint = file:/tmp/temporary-fcb72abb-9143-4d38-a65d-25baf17e34c7/state, runId = 49431645-20e3-4737-9dcc-8f334da22f1d, opId = 0, ver = 8, numPartitions = 200], Complete, 1733343645147, 2
            +- *(5) HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[merge_sum(cast(bytes#175 as double))], output=[window#859-T3000ms, hostname#140, sum#869])
               +- StateStoreRestore [window#859-T3000ms, hostname#140], state info [ checkpoint = file:/tmp/temporary-fcb72abb-9143-4d38-a65d-25baf17e34c7/state, runId = 49431645-20e3-4737-9dcc-8f334da22f1d, opId = 0, ver = 8, numPartitions = 200], 2
                  +- *(4) HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[merge_sum(cast(bytes#175 as double))], output=[window#859-T3000ms, hostname#140, sum#869])
                     +- Exchange hashpartitioning(window#859-T3000ms, hostname#140, 200), true, [id=#1866]
                        +- *(3) HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[partial_sum(cast(bytes#175 as double))], output=[window#859-T3000ms, hostname#140, sum#869])
                           +- *(3) Project [window#859-T3000ms, hostname#140, bytes#175]
                              +- *(3) Filter (((isnotnull(timestamp#1-T3000ms) AND isnotnull(window#859-T3000ms)) AND (timestamp#1-T3000ms >= window#859-T3000ms.start)) AND (timestamp#1-T3000ms < window#859-T3000ms.end))
                                 +- *(3) Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms, hostname#140, bytes#175), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms, hostname#140, bytes#175)], [window#859-T3000ms, timestamp#1-T3000ms, hostname#140, bytes#175]
                                    +- *(3) Project [timestamp#1-T3000ms, split(logs#5, ,, -1)[1] AS hostname#140, split(logs#5, ,, -1)[6] AS bytes#175]
                                       +- EventTimeWatermark timestamp#1: timestamp, 3 seconds
                                          +- *(2) Project [logs#5, timestamp#1]
                                             +- Generate explode(split(value#0,  , -1)), [timestamp#1], false, [logs#5]
                                                +- *(1) Project [value#0, timestamp#1]
                                                   +- MicroBatchScan[value#0, timestamp#1] class org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1