digraph G {
0 [labelType="html" label="<b>Execute InsertIntoHadoopFsRelationCommand</b><br><br>number of written files: 30<br>written output: 1986.0 B<br>number of output rows: 30<br>number of dynamic part: 0"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (8)\n \nduration: total (min, med, max (stageId: taskId))\n1.2 s (24 ms, 30 ms, 199 ms (stage 32.0: task 2071))";
2 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>1 ms (0 ms, 0 ms, 1 ms (stage 32.0: task 2052))<br>peak memory total (min, med, max (stageId: taskId))<br>1921.9 MiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 32.0: task 2051))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 32.0: task 2051))"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 30<br>shuffle write time total (min, med, max (stageId: taskId))<br>27 ms (0 ms, 0 ms, 2 ms (stage 31.0: task 1860))<br>records read: 30<br>local bytes read total (min, med, max (stageId: taskId))<br>2.0 KiB (68.0 B, 69.0 B, 69.0 B (stage 32.0: task 2051))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 32.0: task 2051))<br>local blocks read: 30<br>data size total (min, med, max (stageId: taskId))<br>960.0 B (0.0 B, 0.0 B, 64.0 B (stage 31.0: task 1854))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.0 KiB (0.0 B, 0.0 B, 138.0 B (stage 31.0: task 1854))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: total (min, med, max (stageId: taskId))\n642 ms (0 ms, 0 ms, 66 ms (stage 31.0: task 1855))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>509 ms (0 ms, 0 ms, 64 ms (stage 31.0: task 1855))<br>peak memory total (min, med, max (stageId: taskId))<br>3.7 GiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 31.0: task 1851))<br>number of output rows: 60<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 31.0: task 1851))"];
}
6 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 5,937<br>shuffle write time total (min, med, max (stageId: taskId))<br>1.1 s (3 ms, 5 ms, 39 ms (stage 25.0: task 1501))<br>records read: 11,874<br>local bytes read total (min, med, max (stageId: taskId))<br>389.7 KiB (0.0 B, 0.0 B, 9.4 KiB (stage 26.0: task 1654))<br>fetch wait time total (min, med, max (stageId: taskId))<br>138 ms (0 ms, 0 ms, 53 ms (stage 31.0: task 1856))<br>remote bytes read total (min, med, max (stageId: taskId))<br>390.3 KiB (0.0 B, 0.0 B, 9.4 KiB (stage 31.0: task 1854))<br>local blocks read: 5,737<br>remote blocks read: 5,745<br>data size total (min, med, max (stageId: taskId))<br>185.5 KiB (864.0 B, 960.0 B, 960.0 B (stage 25.0: task 1451))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>390.0 KiB (1811.0 B, 2018.0 B, 2022.0 B (stage 25.0: task 1472))"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n2.7 s (8 ms, 13 ms, 54 ms (stage 25.0: task 1502))";
8 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>1.1 s (3 ms, 5 ms, 40 ms (stage 25.0: task 1502))<br>peak memory total (min, med, max (stageId: taskId))<br>50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 25.0: task 1451))<br>number of output rows: 5,937"];
9 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>819 ms (2 ms, 4 ms, 38 ms (stage 25.0: task 1502))<br>peak memory total (min, med, max (stageId: taskId))<br>50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 25.0: task 1451))<br>number of output rows: 28,128"];
}
10 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 85,947<br>shuffle write time total (min, med, max (stageId: taskId))<br>2.9 s (9 ms, 12 ms, 200 ms (stage 24.0: task 1426))<br>records read: 85,947<br>local bytes read total (min, med, max (stageId: taskId))<br>3.3 MiB (9.9 KiB, 16.2 KiB, 34.0 KiB (stage 25.0: task 1496))<br>fetch wait time total (min, med, max (stageId: taskId))<br>100 ms (0 ms, 0 ms, 32 ms (stage 25.0: task 1502))<br>remote bytes read total (min, med, max (stageId: taskId))<br>3.2 MiB (9.6 KiB, 16.3 KiB, 28.6 KiB (stage 25.0: task 1472))<br>local blocks read: 17,757<br>remote blocks read: 17,396<br>data size total (min, med, max (stageId: taskId))<br>6.6 MiB (22.8 KiB, 33.4 KiB, 45.2 KiB (stage 24.0: task 1361))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>6.6 MiB (25.4 KiB, 33.5 KiB, 42.1 KiB (stage 24.0: task 1361))"];
subgraph cluster11 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n4.2 s (13 ms, 19 ms, 189 ms (stage 24.0: task 1428))";
12 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>896 ms (2 ms, 3 ms, 164 ms (stage 24.0: task 1428))<br>peak memory total (min, med, max (stageId: taskId))<br>50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 24.0: task 1251))<br>number of output rows: 85,947"];
13 [labelType="html" label="<br><b>Project</b><br><br>"];
14 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 173,805"];
}
subgraph cluster15 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n949 ms (0 ms, 6 ms, 52 ms (stage 24.0: task 1325))";
16 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 24.0: task 1251))<br>peak memory total (min, med, max (stageId: taskId))<br>12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 24.0: task 1251))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 24.0: task 1251))"];
}
17 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 144,178<br>shuffle write time total (min, med, max (stageId: taskId))<br>49 ms (0 ms, 0 ms, 28 ms (stage 22.0: task 1246))<br>records read: 144,178<br>local bytes read total (min, med, max (stageId: taskId))<br>4.6 MiB (19.0 KiB, 23.3 KiB, 29.4 KiB (stage 24.0: task 1286))<br>fetch wait time total (min, med, max (stageId: taskId))<br>260 ms (0 ms, 0 ms, 154 ms (stage 24.0: task 1428))<br>remote bytes read total (min, med, max (stageId: taskId))<br>4.7 MiB (18.5 KiB, 23.9 KiB, 29.1 KiB (stage 24.0: task 1330))<br>local blocks read: 200<br>remote blocks read: 200<br>data size total (min, med, max (stageId: taskId))<br>13.2 MiB (0.0 B, 0.0 B, 7.0 MiB (stage 22.0: task 1245))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>9.3 MiB (0.0 B, 0.0 B, 4.9 MiB (stage 22.0: task 1245))"];
subgraph cluster18 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n8.8 s (174 ms, 2.1 s, 2.5 s (stage 22.0: task 1245))";
19 [labelType="html" label="<br><b>Project</b><br><br>"];
20 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 144,178"];
}
21 [labelType="html" label="<b>Scan csv </b><br><br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 519.0 MiB<br>number of output rows: 500,001"];
subgraph cluster22 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n1.4 s (0 ms, 5 ms, 334 ms (stage 24.0: task 1428))";
23 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>14 ms (0 ms, 0 ms, 2 ms (stage 24.0: task 1447))<br>peak memory total (min, med, max (stageId: taskId))<br>12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 24.0: task 1252))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 24.0: task 1251))"];
}
24 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 504,708<br>shuffle write time total (min, med, max (stageId: taskId))<br>129 ms (59 ms, 70 ms, 70 ms (stage 23.0: task 1250))<br>records read: 504,708<br>local bytes read total (min, med, max (stageId: taskId))<br>26.7 MiB (101.9 KiB, 134.0 KiB, 198.3 KiB (stage 24.0: task 1360))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 24.0: task 1251))<br>remote bytes read total (min, med, max (stageId: taskId))<br>26.1 MiB (106.0 KiB, 131.5 KiB, 171.2 KiB (stage 24.0: task 1337))<br>local blocks read: 200<br>remote blocks read: 200<br>data size total (min, med, max (stageId: taskId))<br>107.8 MiB (50.6 MiB, 57.2 MiB, 57.2 MiB (stage 23.0: task 1249))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>52.8 MiB (24.8 MiB, 27.9 MiB, 27.9 MiB (stage 23.0: task 1249))"];
subgraph cluster25 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n3.9 s (1.9 s, 2.0 s, 2.0 s (stage 23.0: task 1250))";
26 [labelType="html" label="<br><b>Project</b><br><br>"];
27 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 504,708"];
}
28 [labelType="html" label="<b>Scan csv </b><br><br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 173.6 MiB<br>number of output rows: 504,708"];
2->0;
3->2;
5->3;
6->5;
8->6;
9->8;
10->9;
12->10;
13->12;
14->13;
16->14;
17->16;
19->17;
20->19;
21->20;
23->14;
24->23;
26->24;
27->26;
28->27;
}
29
Execute InsertIntoHadoopFsRelationCommand file:/opt/spark/work-dir/visualisations/sept_blocks_senders.csv, false, CSV, Map(header -> true, path -> ./visualisations/sept_blocks_senders.csv), ErrorIfExists, [formatted_date, blocks_produced, unique_senders]
Sort [formatted_date#114 ASC NULLS FIRST], true, 0
WholeStageCodegen (8)
Exchange rangepartitioning(formatted_date#114 ASC NULLS FIRST, 200), true, [id=#399]
HashAggregate(keys=[formatted_date#114], functions=[count(hash#70), count(distinct from_address#75)])
WholeStageCodegen (7)
Exchange hashpartitioning(formatted_date#114, 200), true, [id=#395]
HashAggregate(keys=[formatted_date#114], functions=[merge_count(hash#70), partial_count(distinct from_address#75)])
HashAggregate(keys=[formatted_date#114, from_address#75], functions=[merge_count(hash#70)])
WholeStageCodegen (6)
Exchange hashpartitioning(formatted_date#114, from_address#75, 200), true, [id=#390]
HashAggregate(keys=[formatted_date#114, from_address#75], functions=[partial_count(hash#70)])
Project [formatted_date#114, hash#70, from_address#75]
SortMergeJoin [hash#17], [block_hash#72], Inner
WholeStageCodegen (5)
Sort [hash#17 ASC NULLS FIRST], false, 0
WholeStageCodegen (2)
Exchange hashpartitioning(hash#17, 200), true, [id=#372]
Project [hash#17, cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) AS formatted_date#114]
Filter (((cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) < 16709) AND (cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) > 16678)) AND isnotnull(hash#17))
WholeStageCodegen (1)
FileScan csv [hash#17,timestamp#32] Batched: false, DataFilters: [(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) < 167..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string,timestamp:int>
Sort [block_hash#72 ASC NULLS FIRST], false, 0
WholeStageCodegen (4)
Exchange hashpartitioning(block_hash#72, 200), true, [id=#381]
Project [hash#70, block_hash#72, from_address#75]
Filter isnotnull(block_hash#72)
WholeStageCodegen (3)
FileScan csv [hash#70,block_hash#72,from_address#75] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<hash:string,block_hash:string,from_address:string>