digraph G {
0 [labelType="html" label="<br><b>WriteToDataSourceV2</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: 34 ms";
2 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 64.0 KiB<br>spill size: 0.0 B"];
}
3 [labelType="html" label="<br><b>Exchange</b><br><br>"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n785 ms (1 ms, 1 ms, 17 ms (stage 25.0: task 445))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 25.0: task 446))<br>peak memory total (min, med, max (stageId: taskId))<br>100.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 25.0: task 446))"];
}
6 [labelType="html" label="<b>StateStoreSave</b><br><br>memory used by state total (min, med, max (stageId: taskId))<br>81.3 KiB (208.0 B, 208.0 B, 208.0 B (stage 25.0: task 446))<br>estimated size of state only on current version total (min, med, max (stageId: taskId))<br>25.0 KiB (64.0 B, 64.0 B, 64.0 B (stage 25.0: task 446))<br>time to commit changes total (min, med, max (stageId: taskId))<br>19.8 s (34 ms, 37 ms, 88 ms (stage 25.0: task 563))<br>time to remove total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 25.0: task 446))<br>time to update total (min, med, max (stageId: taskId))<br>318 ms (0 ms, 1 ms, 3 ms (stage 25.0: task 502))"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n332 ms (0 ms, 1 ms, 3 ms (stage 25.0: task 445))";
8 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 25.0: task 446))<br>peak memory total (min, med, max (stageId: taskId))<br>100.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 25.0: task 446))"];
}
9 [labelType="html" label="<br><b>StateStoreRestore</b><br><br>"];
subgraph cluster10 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n1.6 s (1 ms, 3 ms, 33 ms (stage 25.0: task 445))";
11 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>1 ms (0 ms, 0 ms, 1 ms (stage 25.0: task 635))<br>peak memory total (min, med, max (stageId: taskId))<br>100.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 25.0: task 446))"];
}
12 [labelType="html" label="<br><b>Exchange</b><br><br>"];
subgraph cluster13 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n120 ms (44 ms, 76 ms, 76 ms (stage 24.0: task 443))";
14 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 24.0: task 444))<br>peak memory total (min, med, max (stageId: taskId))<br>512.0 KiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 24.0: task 444))"];
15 [labelType="html" label="<br><b>Project</b><br><br>"];
16 [labelType="html" label="<br><b>Filter</b><br><br>"];
17 [labelType="html" label="<br><b>Expand</b><br><br>"];
18 [labelType="html" label="<br><b>Project</b><br><br>"];
}
19 [labelType="html" label="<br><b>EventTimeWatermark</b><br><br>"];
subgraph cluster20 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n235 ms (94 ms, 141 ms, 141 ms (stage 24.0: task 443))";
21 [labelType="html" label="<br><b>Project</b><br><br>"];
}
22 [labelType="html" label="<br><b>Generate</b><br><br>"];
subgraph cluster23 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n237 ms (95 ms, 142 ms, 142 ms (stage 24.0: task 443))";
24 [labelType="html" label="<br><b>Project</b><br><br>"];
}
25 [labelType="html" label="<br><b>MicroBatchScan</b><br><br>"];
2->0;
3->2;
5->3;
6->5;
8->6;
9->8;
11->9;
12->11;
14->12;
15->14;
16->15;
17->16;
18->17;
19->18;
21->19;
22->21;
24->22;
25->24;
}
26
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@2bd7894
Sort [total_bytes#858 DESC NULLS LAST], true, 0
WholeStageCodegen (7)
Exchange rangepartitioning(total_bytes#858 DESC NULLS LAST, 200), true, [id=#990]
HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[sum(cast(bytes#175 as double))])
WholeStageCodegen (6)
StateStoreSave [window#859-T3000ms, hostname#140], state info [ checkpoint = file:/tmp/temporary-fcb72abb-9143-4d38-a65d-25baf17e34c7/state, runId = 49431645-20e3-4737-9dcc-8f334da22f1d, opId = 0, ver = 0, numPartitions = 200], Complete, 0, 2
HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[merge_sum(cast(bytes#175 as double))])
WholeStageCodegen (5)
StateStoreRestore [window#859-T3000ms, hostname#140], state info [ checkpoint = file:/tmp/temporary-fcb72abb-9143-4d38-a65d-25baf17e34c7/state, runId = 49431645-20e3-4737-9dcc-8f334da22f1d, opId = 0, ver = 0, numPartitions = 200], 2
HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[merge_sum(cast(bytes#175 as double))])
WholeStageCodegen (4)
Exchange hashpartitioning(window#859-T3000ms, hostname#140, 200), true, [id=#978]
HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[partial_sum(cast(bytes#175 as double))])
Project [window#859-T3000ms, hostname#140, bytes#175]
Filter (((isnotnull(timestamp#1-T3000ms) AND isnotnull(window#859-T3000ms)) AND (timestamp#1-T3000ms >= window#859-T3000ms.start)) AND (timestamp#1-T3000ms < window#859-T3000ms.end))
Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms, hostname#140, bytes#175), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms, hostname#140, bytes#175)], [window#859-T3000ms, timestamp#1-T3000ms, hostname#140, bytes#175]
Project [timestamp#1-T3000ms, split(logs#5, ,, -1)[1] AS hostname#140, split(logs#5, ,, -1)[6] AS bytes#175]
WholeStageCodegen (3)
EventTimeWatermark timestamp#1: timestamp, 3 seconds
Project [logs#5, timestamp#1]
WholeStageCodegen (2)
Generate explode(split(value#0, , -1)), [timestamp#1], false, [logs#5]
Project [value#0, timestamp#1]
WholeStageCodegen (1)
MicroBatchScan[value#0, timestamp#1] class org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1
== Parsed Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@2bd7894
+- Sort [total_bytes#858 DESC NULLS LAST], true
+- Aggregate [window#859-T3000ms, hostname#140], [window#859-T3000ms AS window#847-T3000ms, hostname#140, sum(cast(bytes#175 as double)) AS total_bytes#858]
+- Filter ((timestamp#1-T3000ms >= window#859-T3000ms.start) AND (timestamp#1-T3000ms < window#859-T3000ms.end))
+- Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175)], [window#859-T3000ms, logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, split(logs#5, ,, -1)[6] AS bytes#175]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, split(logs#5, ,, -1)[5] AS responsecode#166]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, split(logs#5, ,, -1)[4] AS url#158]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, split(logs#5, ,, -1)[3] AS method#151]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, split(logs#5, ,, -1)[2] AS time#145]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, split(logs#5, ,, -1)[1] AS hostname#140]
+- Project [logs#5, timestamp#1-T3000ms, split(logs#5, ,, -1)[0] AS idx#136]
+- EventTimeWatermark timestamp#1: timestamp, 3 seconds
+- Project [logs#5, timestamp#1]
+- Generate explode(split(value#0, , -1)), false, [logs#5]
+- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@6b3ca49a, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], -1, -1
== Analyzed Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@2bd7894
+- Sort [total_bytes#858 DESC NULLS LAST], true
+- Aggregate [window#859-T3000ms, hostname#140], [window#859-T3000ms AS window#847-T3000ms, hostname#140, sum(cast(bytes#175 as double)) AS total_bytes#858]
+- Filter ((timestamp#1-T3000ms >= window#859-T3000ms.start) AND (timestamp#1-T3000ms < window#859-T3000ms.end))
+- Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175)], [window#859-T3000ms, logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, bytes#175]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, responsecode#166, split(logs#5, ,, -1)[6] AS bytes#175]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, url#158, split(logs#5, ,, -1)[5] AS responsecode#166]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, method#151, split(logs#5, ,, -1)[4] AS url#158]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, time#145, split(logs#5, ,, -1)[3] AS method#151]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, hostname#140, split(logs#5, ,, -1)[2] AS time#145]
+- Project [logs#5, timestamp#1-T3000ms, idx#136, split(logs#5, ,, -1)[1] AS hostname#140]
+- Project [logs#5, timestamp#1-T3000ms, split(logs#5, ,, -1)[0] AS idx#136]
+- EventTimeWatermark timestamp#1: timestamp, 3 seconds
+- Project [logs#5, timestamp#1]
+- Generate explode(split(value#0, , -1)), false, [logs#5]
+- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@6b3ca49a, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], -1, -1
== Optimized Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@2bd7894
+- Sort [total_bytes#858 DESC NULLS LAST], true
+- Aggregate [window#859-T3000ms, hostname#140], [window#859-T3000ms AS window#847-T3000ms, hostname#140, sum(cast(bytes#175 as double)) AS total_bytes#858]
+- Project [window#859-T3000ms, hostname#140, bytes#175]
+- Filter (((isnotnull(timestamp#1-T3000ms) AND isnotnull(window#859-T3000ms)) AND (timestamp#1-T3000ms >= window#859-T3000ms.start)) AND (timestamp#1-T3000ms < window#859-T3000ms.end))
+- Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms, hostname#140, bytes#175), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms, hostname#140, bytes#175)], [window#859-T3000ms, timestamp#1-T3000ms, hostname#140, bytes#175]
+- Project [timestamp#1-T3000ms, split(logs#5, ,, -1)[1] AS hostname#140, split(logs#5, ,, -1)[6] AS bytes#175]
+- EventTimeWatermark timestamp#1: timestamp, 3 seconds
+- Project [logs#5, timestamp#1]
+- Generate explode(split(value#0, , -1)), [0], false, [logs#5]
+- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@6b3ca49a, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], -1, -1
== Physical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@2bd7894
+- *(7) Sort [total_bytes#858 DESC NULLS LAST], true, 0
+- Exchange rangepartitioning(total_bytes#858 DESC NULLS LAST, 200), true, [id=#990]
+- *(6) HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[sum(cast(bytes#175 as double))], output=[window#847-T3000ms, hostname#140, total_bytes#858])
+- StateStoreSave [window#859-T3000ms, hostname#140], state info [ checkpoint = file:/tmp/temporary-fcb72abb-9143-4d38-a65d-25baf17e34c7/state, runId = 49431645-20e3-4737-9dcc-8f334da22f1d, opId = 0, ver = 0, numPartitions = 200], Complete, 0, 2
+- *(5) HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[merge_sum(cast(bytes#175 as double))], output=[window#859-T3000ms, hostname#140, sum#869])
+- StateStoreRestore [window#859-T3000ms, hostname#140], state info [ checkpoint = file:/tmp/temporary-fcb72abb-9143-4d38-a65d-25baf17e34c7/state, runId = 49431645-20e3-4737-9dcc-8f334da22f1d, opId = 0, ver = 0, numPartitions = 200], 2
+- *(4) HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[merge_sum(cast(bytes#175 as double))], output=[window#859-T3000ms, hostname#140, sum#869])
+- Exchange hashpartitioning(window#859-T3000ms, hostname#140, 200), true, [id=#978]
+- *(3) HashAggregate(keys=[window#859-T3000ms, hostname#140], functions=[partial_sum(cast(bytes#175 as double))], output=[window#859-T3000ms, hostname#140, sum#869])
+- *(3) Project [window#859-T3000ms, hostname#140, bytes#175]
+- *(3) Filter (((isnotnull(timestamp#1-T3000ms) AND isnotnull(window#859-T3000ms)) AND (timestamp#1-T3000ms >= window#859-T3000ms.start)) AND (timestamp#1-T3000ms < window#859-T3000ms.end))
+- *(3) Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms, hostname#140, bytes#175), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms, hostname#140, bytes#175)], [window#859-T3000ms, timestamp#1-T3000ms, hostname#140, bytes#175]
+- *(3) Project [timestamp#1-T3000ms, split(logs#5, ,, -1)[1] AS hostname#140, split(logs#5, ,, -1)[6] AS bytes#175]
+- EventTimeWatermark timestamp#1: timestamp, 3 seconds
+- *(2) Project [logs#5, timestamp#1]
+- Generate explode(split(value#0, , -1)), [timestamp#1], false, [logs#5]
+- *(1) Project [value#0, timestamp#1]
+- MicroBatchScan[value#0, timestamp#1] class org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1
