digraph G {
0 [labelType="html" label="<b>Execute InsertIntoHadoopFsRelationCommand</b><br><br>number of written files: 30<br>written output: 1986.0 B<br>number of output rows: 30<br>number of dynamic part: 0"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (8)\n \nduration: total (min, med, max (stageId: taskId))\n1.1 s (23 ms, 33 ms, 121 ms (stage 32.0: task 2052))";
2 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>1 ms (0 ms, 0 ms, 1 ms (stage 32.0: task 2051))<br>peak memory total (min, med, max (stageId: taskId))<br>1921.9 MiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 32.0: task 2051))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 32.0: task 2051))"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 30<br>shuffle write time total (min, med, max (stageId: taskId))<br>24 ms (0 ms, 0 ms, 1 ms (stage 31.0: task 1851))<br>records read: 30<br>local bytes read total (min, med, max (stageId: taskId))<br>2.0 KiB (68.0 B, 69.0 B, 69.0 B (stage 32.0: task 2051))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 32.0: task 2051))<br>local blocks read: 30<br>data size total (min, med, max (stageId: taskId))<br>960.0 B (0.0 B, 0.0 B, 64.0 B (stage 31.0: task 1854))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.0 KiB (0.0 B, 0.0 B, 138.0 B (stage 31.0: task 1854))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: total (min, med, max (stageId: taskId))\n486 ms (0 ms, 0 ms, 16 ms (stage 31.0: task 1852))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>241 ms (0 ms, 0 ms, 13 ms (stage 31.0: task 1852))<br>peak memory total (min, med, max (stageId: taskId))<br>3.7 GiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 31.0: task 1852))<br>number of output rows: 60<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 31.0: task 1852))"];
}
6 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 5,937<br>shuffle write time total (min, med, max (stageId: taskId))<br>1.2 s (3 ms, 5 ms, 17 ms (stage 25.0: task 1496))<br>records read: 11,874<br>local bytes read total (min, med, max (stageId: taskId))<br>392.1 KiB (0.0 B, 0.0 B, 8.7 KiB (stage 26.0: task 1654))<br>fetch wait time total (min, med, max (stageId: taskId))<br>3 ms (0 ms, 0 ms, 1 ms (stage 31.0: task 1852))<br>remote bytes read total (min, med, max (stageId: taskId))<br>387.8 KiB (0.0 B, 0.0 B, 8.7 KiB (stage 31.0: task 1854))<br>local blocks read: 5,773<br>remote blocks read: 5,709<br>data size total (min, med, max (stageId: taskId))<br>185.5 KiB (864.0 B, 960.0 B, 960.0 B (stage 25.0: task 1452))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>390.0 KiB (1811.0 B, 2018.0 B, 2022.0 B (stage 25.0: task 1472))"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n3.0 s (7 ms, 13 ms, 116 ms (stage 25.0: task 1560))";
8 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>1.4 s (2 ms, 6 ms, 101 ms (stage 25.0: task 1560))<br>peak memory total (min, med, max (stageId: taskId))<br>50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 25.0: task 1452))<br>number of output rows: 5,937"];
9 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>1.1 s (1 ms, 4 ms, 99 ms (stage 25.0: task 1560))<br>peak memory total (min, med, max (stageId: taskId))<br>50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 25.0: task 1452))<br>number of output rows: 28,128"];
}
10 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 85,947<br>shuffle write time total (min, med, max (stageId: taskId))<br>2.5 s (8 ms, 12 ms, 21 ms (stage 24.0: task 1315))<br>records read: 85,947<br>local bytes read total (min, med, max (stageId: taskId))<br>3.3 MiB (8.4 KiB, 16.3 KiB, 29.3 KiB (stage 25.0: task 1472))<br>fetch wait time total (min, med, max (stageId: taskId))<br>199 ms (0 ms, 0 ms, 92 ms (stage 25.0: task 1560))<br>remote bytes read total (min, med, max (stageId: taskId))<br>3.2 MiB (9.4 KiB, 15.9 KiB, 34.7 KiB (stage 25.0: task 1496))<br>local blocks read: 17,713<br>remote blocks read: 17,440<br>data size total (min, med, max (stageId: taskId))<br>6.6 MiB (22.8 KiB, 33.4 KiB, 45.2 KiB (stage 24.0: task 1361))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>6.6 MiB (25.4 KiB, 33.5 KiB, 42.1 KiB (stage 24.0: task 1361))"];
subgraph cluster11 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n4.0 s (13 ms, 17 ms, 76 ms (stage 24.0: task 1319))";
12 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>654 ms (2 ms, 3 ms, 43 ms (stage 24.0: task 1390))<br>peak memory total (min, med, max (stageId: taskId))<br>50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 24.0: task 1252))<br>number of output rows: 85,947"];
13 [labelType="html" label="<br><b>Project</b><br><br>"];
14 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 173,805"];
}
subgraph cluster15 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n933 ms (0 ms, 5 ms, 46 ms (stage 24.0: task 1390))";
16 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 24.0: task 1252))<br>peak memory total (min, med, max (stageId: taskId))<br>12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 24.0: task 1252))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 24.0: task 1252))"];
}
17 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 144,178<br>shuffle write time total (min, med, max (stageId: taskId))<br>43 ms (0 ms, 0 ms, 22 ms (stage 23.0: task 1248))<br>records read: 144,178<br>local bytes read total (min, med, max (stageId: taskId))<br>4.6 MiB (18.5 KiB, 23.5 KiB, 29.4 KiB (stage 24.0: task 1286))<br>fetch wait time total (min, med, max (stageId: taskId))<br>62 ms (0 ms, 0 ms, 35 ms (stage 24.0: task 1390))<br>remote bytes read total (min, med, max (stageId: taskId))<br>4.7 MiB (18.5 KiB, 23.9 KiB, 28.8 KiB (stage 24.0: task 1435))<br>local blocks read: 200<br>remote blocks read: 200<br>data size total (min, med, max (stageId: taskId))<br>13.2 MiB (0.0 B, 0.0 B, 7.0 MiB (stage 23.0: task 1247))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>9.3 MiB (0.0 B, 0.0 B, 4.9 MiB (stage 23.0: task 1247))"];
subgraph cluster18 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n8.6 s (152 ms, 2.1 s, 2.2 s (stage 23.0: task 1248))";
19 [labelType="html" label="<br><b>Project</b><br><br>"];
20 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 144,178"];
}
21 [labelType="html" label="<b>Scan csv </b><br><br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 519.0 MiB<br>number of output rows: 500,001"];
subgraph cluster22 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n824 ms (0 ms, 5 ms, 37 ms (stage 24.0: task 1319))";
23 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>7 ms (0 ms, 0 ms, 2 ms (stage 24.0: task 1319))<br>peak memory total (min, med, max (stageId: taskId))<br>12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 24.0: task 1252))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 24.0: task 1252))"];
}
24 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 504,708<br>shuffle write time total (min, med, max (stageId: taskId))<br>111 ms (54 ms, 57 ms, 57 ms (stage 22.0: task 1244))<br>records read: 504,708<br>local bytes read total (min, med, max (stageId: taskId))<br>26.4 MiB (101.9 KiB, 132.7 KiB, 190.0 KiB (stage 24.0: task 1313))<br>fetch wait time total (min, med, max (stageId: taskId))<br>22 ms (0 ms, 0 ms, 22 ms (stage 24.0: task 1319))<br>remote bytes read total (min, med, max (stageId: taskId))<br>26.4 MiB (104.2 KiB, 133.6 KiB, 198.3 KiB (stage 24.0: task 1360))<br>local blocks read: 200<br>remote blocks read: 200<br>data size total (min, med, max (stageId: taskId))<br>107.8 MiB (50.6 MiB, 57.2 MiB, 57.2 MiB (stage 22.0: task 1244))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>52.8 MiB (24.8 MiB, 27.9 MiB, 27.9 MiB (stage 22.0: task 1244))"];
subgraph cluster25 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n3.7 s (1.8 s, 1.9 s, 1.9 s (stage 22.0: task 1245))";
26 [labelType="html" label="<br><b>Project</b><br><br>"];
27 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 504,708"];
}
28 [labelType="html" label="<b>Scan csv </b><br><br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 173.6 MiB<br>number of output rows: 504,708"];
2->0;
3->2;
5->3;
6->5;
8->6;
9->8;
10->9;
12->10;
13->12;
14->13;
16->14;
17->16;
19->17;
20->19;
21->20;
23->14;
24->23;
26->24;
27->26;
28->27;
}
29
Execute InsertIntoHadoopFsRelationCommand file:/opt/spark/work-dir/visualizations/sept_blocks_senders.csv, false, CSV, Map(header -> true, path -> ./visualizations/sept_blocks_senders.csv), ErrorIfExists, [formatted_date, blocks_produced, unique_senders]
Sort [formatted_date#114 ASC NULLS FIRST], true, 0
WholeStageCodegen (8)
Exchange rangepartitioning(formatted_date#114 ASC NULLS FIRST, 200), true, [id=#399]
HashAggregate(keys=[formatted_date#114], functions=[count(hash#70), count(distinct from_address#75)])
WholeStageCodegen (7)
Exchange hashpartitioning(formatted_date#114, 200), true, [id=#395]
HashAggregate(keys=[formatted_date#114], functions=[merge_count(hash#70), partial_count(distinct from_address#75)])
HashAggregate(keys=[formatted_date#114, from_address#75], functions=[merge_count(hash#70)])
WholeStageCodegen (6)
Exchange hashpartitioning(formatted_date#114, from_address#75, 200), true, [id=#390]
HashAggregate(keys=[formatted_date#114, from_address#75], functions=[partial_count(hash#70)])
Project [formatted_date#114, hash#70, from_address#75]
SortMergeJoin [hash#17], [block_hash#72], Inner
WholeStageCodegen (5)
Sort [hash#17 ASC NULLS FIRST], false, 0
WholeStageCodegen (2)
Exchange hashpartitioning(hash#17, 200), true, [id=#372]
Project [hash#17, cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) AS formatted_date#114]
Filter (((cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) < 16709) AND (cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) > 16678)) AND isnotnull(hash#17))
WholeStageCodegen (1)
FileScan csv [hash#17,timestamp#32] Batched: false, DataFilters: [(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) < 167..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string,timestamp:int>
Sort [block_hash#72 ASC NULLS FIRST], false, 0
WholeStageCodegen (4)
Exchange hashpartitioning(block_hash#72, 200), true, [id=#381]
Project [hash#70, block_hash#72, from_address#75]
Filter isnotnull(block_hash#72)
WholeStageCodegen (3)
FileScan csv [hash#70,block_hash#72,from_address#75] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<hash:string,block_hash:string,from_address:string>