digraph G {
0 [labelType="html" label="<br><b>WriteToDataSourceV2</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n9.6 s (30 ms, 36 ms, 146 ms (stage 11.0: task 637))";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>9.4 s (30 ms, 35 ms, 144 ms (stage 11.0: task 637))<br>peak memory total (min, med, max (stageId: taskId))<br>178.0 MiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 11.0: task 637))<br>number of output rows: 2<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 11.0: task 637))"];
}
3 [labelType="html" label="<b>StateStoreSave</b><br><br>number of total state rows: 2<br>memory used by state total (min, med, max (stageId: taskId))<br>77.0 KiB (392.0 B, 392.0 B, 592.0 B (stage 11.0: task 637))<br>count of cache hit on states cache in provider: 400<br>number of output rows: 2<br>estimated size of state only on current version total (min, med, max (stageId: taskId))<br>20.7 KiB (104.0 B, 104.0 B, 304.0 B (stage 11.0: task 637))<br>time to commit changes total (min, med, max (stageId: taskId))<br>9.0 s (27 ms, 34 ms, 94 ms (stage 11.0: task 736))<br>time to remove total (min, med, max (stageId: taskId))<br>3 ms (0 ms, 0 ms, 3 ms (stage 11.0: task 624))<br>number of updated state rows: 2<br>time to update total (min, med, max (stageId: taskId))<br>426 ms (0 ms, 1 ms, 68 ms (stage 11.0: task 637))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n500 ms (1 ms, 1 ms, 69 ms (stage 11.0: task 637))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>59 ms (0 ms, 0 ms, 38 ms (stage 11.0: task 637))<br>peak memory total (min, med, max (stageId: taskId))<br>178.0 MiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 11.0: task 637))<br>number of output rows: 2<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 11.0: task 637))"];
}
6 [labelType="html" label="<b>StateStoreRestore</b><br><br>number of output rows: 2"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n939 ms (1 ms, 2 ms, 69 ms (stage 11.0: task 637))";
8 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>36 ms (0 ms, 0 ms, 25 ms (stage 11.0: task 637))<br>peak memory total (min, med, max (stageId: taskId))<br>178.0 MiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 11.0: task 637))<br>number of output rows: 2<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 11.0: task 637))"];
}
9 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 4<br>shuffle write time total (min, med, max (stageId: taskId))<br>17 ms (4 ms, 12 ms, 12 ms (stage 10.0: task 414))<br>records read: 4<br>local bytes read total (min, med, max (stageId: taskId))<br>156.0 B (0.0 B, 0.0 B, 156.0 B (stage 11.0: task 751))<br>fetch wait time total (min, med, max (stageId: taskId))<br>11 ms (0 ms, 0 ms, 11 ms (stage 11.0: task 637))<br>remote bytes read total (min, med, max (stageId: taskId))<br>158.0 B (0.0 B, 0.0 B, 158.0 B (stage 11.0: task 637))<br>local blocks read: 2<br>remote blocks read: 2<br>data size total (min, med, max (stageId: taskId))<br>192.0 B (96.0 B, 96.0 B, 96.0 B (stage 10.0: task 414))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>314.0 B (157.0 B, 157.0 B, 157.0 B (stage 10.0: task 414))"];
subgraph cluster10 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n17 ms (6 ms, 11 ms, 11 ms (stage 10.0: task 414))";
11 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>1 ms (0 ms, 1 ms, 1 ms (stage 10.0: task 414))<br>peak memory total (min, med, max (stageId: taskId))<br>128.5 MiB (64.3 MiB, 64.3 MiB, 64.3 MiB (stage 10.0: task 414))<br>number of output rows: 4<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 10.0: task 414))"];
12 [labelType="html" label="<br><b>Project</b><br><br>"];
13 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 16"];
14 [labelType="html" label="<b>Expand</b><br><br>number of output rows: 16"];
}
15 [labelType="html" label="<br><b>EventTimeWatermark</b><br><br>"];
subgraph cluster16 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n6 ms (3 ms, 3 ms, 3 ms (stage 10.0: task 414))";
17 [labelType="html" label="<br><b>Project</b><br><br>"];
18 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 8"];
}
19 [labelType="html" label="<b>Generate</b><br><br>number of output rows: 13"];
subgraph cluster20 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n8 ms (4 ms, 4 ms, 4 ms (stage 10.0: task 414))";
21 [labelType="html" label="<br><b>Project</b><br><br>"];
}
22 [labelType="html" label="<b>MicroBatchScan</b><br><br>number of output rows: 11"];
2->0;
3->2;
5->3;
6->5;
8->6;
9->8;
11->9;
12->11;
13->12;
14->13;
15->14;
17->15;
18->17;
19->18;
21->19;
22->21;
}
23
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@7dfa1bfd
HashAggregate(keys=[window#81-T3000ms], functions=[count(1)])
WholeStageCodegen (6)
StateStoreSave [window#81-T3000ms], state info [ checkpoint = file:/tmp/temporary-44a37146-c22b-4332-803b-47058dc7d369/state, runId = 2ad6ec75-02a9-40e4-8f05-f0dd3dbf4b8d, opId = 0, ver = 1, numPartitions = 200], Update, 0, 2
HashAggregate(keys=[window#81-T3000ms], functions=[merge_count(1)])
WholeStageCodegen (5)
StateStoreRestore [window#81-T3000ms], state info [ checkpoint = file:/tmp/temporary-44a37146-c22b-4332-803b-47058dc7d369/state, runId = 2ad6ec75-02a9-40e4-8f05-f0dd3dbf4b8d, opId = 0, ver = 1, numPartitions = 200], 2
HashAggregate(keys=[window#81-T3000ms], functions=[merge_count(1)])
WholeStageCodegen (4)
Exchange hashpartitioning(window#81-T3000ms, 200), true, [id=#518]
HashAggregate(keys=[window#81-T3000ms], functions=[partial_count(1)])
Project [window#81-T3000ms]
Filter (((isnotnull(timestamp#1-T3000ms) AND isnotnull(window#81-T3000ms)) AND (timestamp#1-T3000ms >= window#81-T3000ms.start)) AND (timestamp#1-T3000ms < window#81-T3000ms.end))
Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms)], [window#81-T3000ms, timestamp#1-T3000ms]
WholeStageCodegen (3)
EventTimeWatermark timestamp#1: timestamp, 3 seconds
Project [timestamp#1]
Filter Contains(split(logs#5, ,, -1)[4], gif)
WholeStageCodegen (2)
Generate explode(split(value#0, , -1)), [timestamp#1], false, [logs#5]
Project [value#0, timestamp#1]
WholeStageCodegen (1)
MicroBatchScan[value#0, timestamp#1] class org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1
== Parsed Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@7dfa1bfd
+- SubqueryAlias gif_count
+- Aggregate [window#81-T3000ms], [window#81-T3000ms AS window#69-T3000ms, count(1) AS count#80L]
+- Filter ((timestamp#1-T3000ms >= window#81-T3000ms.start) AND (timestamp#1-T3000ms < window#81-T3000ms.end))
+- Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, url#32, responsecode#40, bytes#49), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, url#32, responsecode#40, bytes#49)], [window#81-T3000ms, logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, url#32, responsecode#40, bytes#49]
+- Filter Contains(url#32, gif)
+- Project [logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, url#32, responsecode#40, split(logs#5, ,, -1)[6] AS bytes#49]
+- Project [logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, url#32, split(logs#5, ,, -1)[5] AS responsecode#40]
+- Project [logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, split(logs#5, ,, -1)[4] AS url#32]
+- Project [logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, split(logs#5, ,, -1)[3] AS method#25]
+- Project [logs#5, timestamp#1-T3000ms, idx#10, hostname#14, split(logs#5, ,, -1)[2] AS time#19]
+- Project [logs#5, timestamp#1-T3000ms, idx#10, split(logs#5, ,, -1)[1] AS hostname#14]
+- Project [logs#5, timestamp#1-T3000ms, split(logs#5, ,, -1)[0] AS idx#10]
+- EventTimeWatermark timestamp#1: timestamp, 3 seconds
+- Project [logs#5, timestamp#1]
+- Generate explode(split(value#0, , -1)), false, [logs#5]
+- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@466d1a44, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], -1, 10
== Analyzed Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@7dfa1bfd
+- SubqueryAlias gif_count
+- Aggregate [window#81-T3000ms], [window#81-T3000ms AS window#69-T3000ms, count(1) AS count#80L]
+- Filter ((timestamp#1-T3000ms >= window#81-T3000ms.start) AND (timestamp#1-T3000ms < window#81-T3000ms.end))
+- Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(0 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, url#32, responsecode#40, bytes#49), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion((((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) + cast(1 as bigint)) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / cast(30000000 as double))) END + cast(1 as bigint)) - cast(2 as bigint)) * 30000000) + 0) + 60000000), LongType, TimestampType)), logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, url#32, responsecode#40, bytes#49)], [window#81-T3000ms, logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, url#32, responsecode#40, bytes#49]
+- Filter Contains(url#32, gif)
+- Project [logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, url#32, responsecode#40, split(logs#5, ,, -1)[6] AS bytes#49]
+- Project [logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, url#32, split(logs#5, ,, -1)[5] AS responsecode#40]
+- Project [logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, method#25, split(logs#5, ,, -1)[4] AS url#32]
+- Project [logs#5, timestamp#1-T3000ms, idx#10, hostname#14, time#19, split(logs#5, ,, -1)[3] AS method#25]
+- Project [logs#5, timestamp#1-T3000ms, idx#10, hostname#14, split(logs#5, ,, -1)[2] AS time#19]
+- Project [logs#5, timestamp#1-T3000ms, idx#10, split(logs#5, ,, -1)[1] AS hostname#14]
+- Project [logs#5, timestamp#1-T3000ms, split(logs#5, ,, -1)[0] AS idx#10]
+- EventTimeWatermark timestamp#1: timestamp, 3 seconds
+- Project [logs#5, timestamp#1]
+- Generate explode(split(value#0, , -1)), false, [logs#5]
+- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@466d1a44, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], -1, 10
== Optimized Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@7dfa1bfd
+- Aggregate [window#81-T3000ms], [window#81-T3000ms AS window#69-T3000ms, count(1) AS count#80L]
+- Project [window#81-T3000ms]
+- Filter (((isnotnull(timestamp#1-T3000ms) AND isnotnull(window#81-T3000ms)) AND (timestamp#1-T3000ms >= window#81-T3000ms.start)) AND (timestamp#1-T3000ms < window#81-T3000ms.end))
+- Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms)], [window#81-T3000ms, timestamp#1-T3000ms]
+- EventTimeWatermark timestamp#1: timestamp, 3 seconds
+- Project [timestamp#1]
+- Filter Contains(split(logs#5, ,, -1)[4], gif)
+- Generate explode(split(value#0, , -1)), [0], false, [logs#5]
+- StreamingDataSourceV2Relation [value#0, timestamp#1], org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1@466d1a44, TextSocketV2[host: stream-emulator.data-science-tools.svc.cluster.local, port: 5551], -1, 10
== Physical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite@7dfa1bfd
+- *(6) HashAggregate(keys=[window#81-T3000ms], functions=[count(1)], output=[window#69-T3000ms, count#80L])
+- StateStoreSave [window#81-T3000ms], state info [ checkpoint = file:/tmp/temporary-44a37146-c22b-4332-803b-47058dc7d369/state, runId = 2ad6ec75-02a9-40e4-8f05-f0dd3dbf4b8d, opId = 0, ver = 1, numPartitions = 200], Update, 0, 2
+- *(5) HashAggregate(keys=[window#81-T3000ms], functions=[merge_count(1)], output=[window#81-T3000ms, count#102L])
+- StateStoreRestore [window#81-T3000ms], state info [ checkpoint = file:/tmp/temporary-44a37146-c22b-4332-803b-47058dc7d369/state, runId = 2ad6ec75-02a9-40e4-8f05-f0dd3dbf4b8d, opId = 0, ver = 1, numPartitions = 200], 2
+- *(4) HashAggregate(keys=[window#81-T3000ms], functions=[merge_count(1)], output=[window#81-T3000ms, count#102L])
+- Exchange hashpartitioning(window#81-T3000ms, 200), true, [id=#518]
+- *(3) HashAggregate(keys=[window#81-T3000ms], functions=[partial_count(1)], output=[window#81-T3000ms, count#102L])
+- *(3) Project [window#81-T3000ms]
+- *(3) Filter (((isnotnull(timestamp#1-T3000ms) AND isnotnull(window#81-T3000ms)) AND (timestamp#1-T3000ms >= window#81-T3000ms.start)) AND (timestamp#1-T3000ms < window#81-T3000ms.end))
+- *(3) Expand [ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 0) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms), ArrayBuffer(named_struct(start, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 0), LongType, TimestampType), end, precisetimestampconversion(((((CASE WHEN (cast(CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) as double) = (cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) THEN (CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) + 1) ELSE CEIL((cast((precisetimestampconversion(timestamp#1-T3000ms, TimestampType, LongType) - 0) as double) / 3.0E7)) END + 1) - 2) * 30000000) + 60000000), LongType, TimestampType)), timestamp#1-T3000ms)], [window#81-T3000ms, timestamp#1-T3000ms]
+- EventTimeWatermark timestamp#1: timestamp, 3 seconds
+- *(2) Project [timestamp#1]
+- *(2) Filter Contains(split(logs#5, ,, -1)[4], gif)
+- Generate explode(split(value#0, , -1)), [timestamp#1], false, [logs#5]
+- *(1) Project [value#0, timestamp#1]
+- MicroBatchScan[value#0, timestamp#1] class org.apache.spark.sql.execution.streaming.sources.TextSocketTable$$anon$1
