EthereumTasks - Details for Query 5

Details for Query 5

Submitted Time: 2024/12/21 16:51:28
Duration: 19 s
Succeeded Jobs: 7

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" TakeOrderedAndProject "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (7)\n \nduration: total (min, med, max (stageId: taskId))\n401 ms (0 ms, 0 ms, 28 ms (stage 15.0: task 635))"; 2 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 136 ms (0 ms, 0 ms, 11 ms (stage 15.0: task 635)) peak memory total (min, med, max (stageId: taskId)) 1906.0 MiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 15.0: task 631)) number of output rows: 30 avg hash probe bucket list iters (min, med, max (stageId: taskId)): (1, 1, 1 (stage 15.0: task 631))"]; } 3 [labelType="html" label="Exchange shuffle records written: 5,937 shuffle write time total (min, med, max (stageId: taskId)) 1.1 s (4 ms, 5 ms, 11 ms (stage 14.0: task 508)) records read: 5,937 local bytes read total (min, med, max (stageId: taskId)) 198.6 KiB (0.0 B, 0.0 B, 9.2 KiB (stage 15.0: task 634)) fetch wait time total (min, med, max (stageId: taskId)) 18 ms (0 ms, 0 ms, 4 ms (stage 15.0: task 654)) remote bytes read total (min, med, max (stageId: taskId)) 191.3 KiB (0.0 B, 0.0 B, 7.5 KiB (stage 15.0: task 634)) local blocks read: 2,922 remote blocks read: 2,819 data size total (min, med, max (stageId: taskId)) 185.5 KiB (864.0 B, 960.0 B, 960.0 B (stage 14.0: task 431)) shuffle bytes written total (min, med, max (stageId: taskId)) 390.0 KiB (1811.0 B, 2018.0 B, 2022.0 B (stage 14.0: task 452))"]; subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n3.5 s (11 ms, 15 ms, 85 ms (stage 14.0: task 536))"; 5 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 1.6 s (4 ms, 6 ms, 73 ms (stage 14.0: task 536)) peak memory total (min, med, max (stageId: taskId)) 50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 14.0: task 431)) number of output rows: 5,937"]; 6 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 1.1 s (2 ms, 4 ms, 71 ms (stage 14.0: task 536)) peak memory total (min, med, max (stageId: taskId)) 50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 14.0: task 431)) number of output rows: 28,128"]; } 7 [labelType="html" label="Exchange shuffle records written: 85,947 shuffle write time total (min, med, max (stageId: taskId)) 2.7 s (9 ms, 12 ms, 32 ms (stage 13.0: task 239)) records read: 85,947 local bytes read total (min, med, max (stageId: taskId)) 3.3 MiB (9.1 KiB, 16.0 KiB, 35.7 KiB (stage 14.0: task 476)) fetch wait time total (min, med, max (stageId: taskId)) 173 ms (0 ms, 0 ms, 62 ms (stage 14.0: task 536)) remote bytes read total (min, med, max (stageId: taskId)) 3.3 MiB (8.6 KiB, 16.5 KiB, 28.5 KiB (stage 14.0: task 500)) local blocks read: 17,732 remote blocks read: 17,421 data size total (min, med, max (stageId: taskId)) 6.6 MiB (22.8 KiB, 33.4 KiB, 45.2 KiB (stage 13.0: task 341)) shuffle bytes written total (min, med, max (stageId: taskId)) 6.6 MiB (25.4 KiB, 33.5 KiB, 42.1 KiB (stage 13.0: task 341))"]; subgraph cluster8 { isCluster="true"; label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n5.8 s (14 ms, 23 ms, 396 ms (stage 13.0: task 231))"; 9 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 1.4 s (2 ms, 3 ms, 236 ms (stage 13.0: task 231)) peak memory total (min, med, max (stageId: taskId)) 50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 13.0: task 232)) number of output rows: 85,947"]; 10 [labelType="html" label=" Project "]; 11 [labelType="html" label="SortMergeJoin number of output rows: 173,805"]; } subgraph cluster12 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n1.7 s (0 ms, 8 ms, 197 ms (stage 13.0: task 302))"; 13 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 7 ms (0 ms, 0 ms, 2 ms (stage 13.0: task 232)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 13.0: task 232)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 13.0: task 232))"]; } 14 [labelType="html" label="Exchange shuffle records written: 144,178 shuffle write time total (min, med, max (stageId: taskId)) 44 ms (0 ms, 0 ms, 22 ms (stage 11.0: task 226)) records read: 144,178 local bytes read total (min, med, max (stageId: taskId)) 4.6 MiB (19.3 KiB, 23.5 KiB, 29.1 KiB (stage 13.0: task 310)) fetch wait time total (min, med, max (stageId: taskId)) 361 ms (0 ms, 0 ms, 181 ms (stage 13.0: task 302)) remote bytes read total (min, med, max (stageId: taskId)) 4.7 MiB (18.5 KiB, 23.8 KiB, 29.4 KiB (stage 13.0: task 266)) local blocks read: 200 remote blocks read: 200 data size total (min, med, max (stageId: taskId)) 13.2 MiB (0.0 B, 0.0 B, 7.0 MiB (stage 11.0: task 225)) shuffle bytes written total (min, med, max (stageId: taskId)) 9.3 MiB (0.0 B, 0.0 B, 4.9 MiB (stage 11.0: task 225))"]; subgraph cluster15 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n10.4 s (184 ms, 2.5 s, 3.1 s (stage 11.0: task 225))"; 16 [labelType="html" label=" Project "]; 17 [labelType="html" label="Filter number of output rows: 144,178"]; } 18 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 519.0 MiB number of output rows: 500,001"]; subgraph cluster19 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n1.8 s (0 ms, 7 ms, 458 ms (stage 13.0: task 231))"; 20 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 43 ms (0 ms, 0 ms, 10 ms (stage 13.0: task 232)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 13.0: task 232)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 13.0: task 232))"]; } 21 [labelType="html" label="Exchange shuffle records written: 504,708 shuffle write time total (min, med, max (stageId: taskId)) 141 ms (57 ms, 83 ms, 83 ms (stage 12.0: task 229)) records read: 504,708 local bytes read total (min, med, max (stageId: taskId)) 26.0 MiB (101.9 KiB, 132.0 KiB, 176.5 KiB (stage 13.0: task 399)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 13.0: task 232)) remote bytes read total (min, med, max (stageId: taskId)) 26.8 MiB (104.2 KiB, 134.3 KiB, 198.3 KiB (stage 13.0: task 340)) local blocks read: 200 remote blocks read: 200 data size total (min, med, max (stageId: taskId)) 107.8 MiB (50.6 MiB, 57.2 MiB, 57.2 MiB (stage 12.0: task 229)) shuffle bytes written total (min, med, max (stageId: taskId)) 52.8 MiB (24.8 MiB, 27.9 MiB, 27.9 MiB (stage 12.0: task 229))"]; subgraph cluster22 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n5.0 s (1.9 s, 3.1 s, 3.1 s (stage 12.0: task 229))"; 23 [labelType="html" label=" Project "]; 24 [labelType="html" label="Filter number of output rows: 504,708"]; } 25 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 173.6 MiB number of output rows: 504,708"]; 2->0; 3->2; 5->3; 6->5; 7->6; 9->7; 10->9; 11->10; 13->11; 14->13; 16->14; 17->16; 18->17; 20->11; 21->20; 23->21; 24->23; 25->24; }

TakeOrderedAndProject(limit=21, orderBy=[formatted_date#114 ASC NULLS FIRST], output=[formatted_date#371,blocks_produced#372,unique_senders#373])

HashAggregate(keys=[formatted_date#114], functions=[count(hash#70), count(distinct from_address#75)])

WholeStageCodegen (7)

Exchange hashpartitioning(formatted_date#114, 200), true, [id=#217]

HashAggregate(keys=[formatted_date#114], functions=[merge_count(hash#70), partial_count(distinct from_address#75)])

HashAggregate(keys=[formatted_date#114, from_address#75], functions=[merge_count(hash#70)])

WholeStageCodegen (6)

Exchange hashpartitioning(formatted_date#114, from_address#75, 200), true, [id=#212]

HashAggregate(keys=[formatted_date#114, from_address#75], functions=[partial_count(hash#70)])

Project [formatted_date#114, hash#70, from_address#75]

SortMergeJoin [hash#17], [block_hash#72], Inner

WholeStageCodegen (5)

Sort [hash#17 ASC NULLS FIRST], false, 0

WholeStageCodegen (2)

Exchange hashpartitioning(hash#17, 200), true, [id=#194]

Project [hash#17, cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) AS formatted_date#114]

Filter (((cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) < 16709) AND (cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) > 16678)) AND isnotnull(hash#17))

WholeStageCodegen (1)

FileScan csv [hash#17,timestamp#32] Batched: false, DataFilters: [(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) < 167..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string,timestamp:int>

Sort [block_hash#72 ASC NULLS FIRST], false, 0

WholeStageCodegen (4)

Exchange hashpartitioning(block_hash#72, 200), true, [id=#203]

Project [hash#70, block_hash#72, from_address#75]

Filter isnotnull(block_hash#72)

WholeStageCodegen (3)

FileScan csv [hash#70,block_hash#72,from_address#75] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<hash:string,block_hash:string,from_address:string>

Details

== Parsed Logical Plan ==
GlobalLimit 21
+- LocalLimit 21
   +- Project [cast(formatted_date#114 as string) AS formatted_date#371, cast(blocks_produced#362L as string) AS blocks_produced#372, cast(unique_senders#363L as string) AS unique_senders#373]
      +- Sort [formatted_date#114 ASC NULLS FIRST], true
         +- Aggregate [formatted_date#114], [formatted_date#114, count(hash#70) AS blocks_produced#362L, count(distinct from_address#75) AS unique_senders#363L]
            +- Filter ((formatted_date#114 < cast(2015-10-01 as date)) AND (formatted_date#114 > cast(2015-08-31 as date)))
               +- Project [number#16, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, formatted_date#114, hash#70, nonce#71, block_hash#72, block_number#73, transaction_index#74, ... 10 more fields]
                  +- Join Inner, (hash#17 = block_hash#72)
                     :- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, to_date(from_unixtime('timestamp, yyyy-MM-dd HH:mm:ss, None), None) AS formatted_date#114]
                     :  +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
                     +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Analyzed Logical Plan ==
formatted_date: string, blocks_produced: string, unique_senders: string
GlobalLimit 21
+- LocalLimit 21
   +- Project [cast(formatted_date#114 as string) AS formatted_date#371, cast(blocks_produced#362L as string) AS blocks_produced#372, cast(unique_senders#363L as string) AS unique_senders#373]
      +- Sort [formatted_date#114 ASC NULLS FIRST], true
         +- Aggregate [formatted_date#114], [formatted_date#114, count(hash#70) AS blocks_produced#362L, count(distinct from_address#75) AS unique_senders#363L]
            +- Filter ((formatted_date#114 < cast(2015-10-01 as date)) AND (formatted_date#114 > cast(2015-08-31 as date)))
               +- Project [number#16, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, formatted_date#114, hash#70, nonce#71, block_hash#72, block_number#73, transaction_index#74, ... 10 more fields]
                  +- Join Inner, (hash#17 = block_hash#72)
                     :- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, to_date(from_unixtime('timestamp, yyyy-MM-dd HH:mm:ss, None), None) AS formatted_date#114]
                     :  +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
                     +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Optimized Logical Plan ==
GlobalLimit 21
+- LocalLimit 21
   +- Project [cast(formatted_date#114 as string) AS formatted_date#371, cast(blocks_produced#362L as string) AS blocks_produced#372, cast(unique_senders#363L as string) AS unique_senders#373]
      +- Sort [formatted_date#114 ASC NULLS FIRST], true
         +- Aggregate [formatted_date#114], [formatted_date#114, count(hash#70) AS blocks_produced#362L, count(distinct from_address#75) AS unique_senders#363L]
            +- Project [formatted_date#114, hash#70, from_address#75]
               +- Join Inner, (hash#17 = block_hash#72)
                  :- Project [hash#17, cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) AS formatted_date#114]
                  :  +- Filter (((cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) < 16709) AND (cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) > 16678)) AND isnotnull(hash#17))
                  :     +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
                  +- Project [hash#70, block_hash#72, from_address#75]
                     +- Filter isnotnull(block_hash#72)
                        +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Physical Plan ==
TakeOrderedAndProject(limit=21, orderBy=[formatted_date#114 ASC NULLS FIRST], output=[formatted_date#371,blocks_produced#372,unique_senders#373])
+- *(7) HashAggregate(keys=[formatted_date#114], functions=[count(hash#70), count(distinct from_address#75)], output=[formatted_date#114, blocks_produced#362L, unique_senders#363L])
   +- Exchange hashpartitioning(formatted_date#114, 200), true, [id=#217]
      +- *(6) HashAggregate(keys=[formatted_date#114], functions=[merge_count(hash#70), partial_count(distinct from_address#75)], output=[formatted_date#114, count#378L, count#381L])
         +- *(6) HashAggregate(keys=[formatted_date#114, from_address#75], functions=[merge_count(hash#70)], output=[formatted_date#114, from_address#75, count#378L])
            +- Exchange hashpartitioning(formatted_date#114, from_address#75, 200), true, [id=#212]
               +- *(5) HashAggregate(keys=[formatted_date#114, from_address#75], functions=[partial_count(hash#70)], output=[formatted_date#114, from_address#75, count#378L])
                  +- *(5) Project [formatted_date#114, hash#70, from_address#75]
                     +- *(5) SortMergeJoin [hash#17], [block_hash#72], Inner
                        :- *(2) Sort [hash#17 ASC NULLS FIRST], false, 0
                        :  +- Exchange hashpartitioning(hash#17, 200), true, [id=#194]
                        :     +- *(1) Project [hash#17, cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) AS formatted_date#114]
                        :        +- *(1) Filter (((cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) < 16709) AND (cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) > 16678)) AND isnotnull(hash#17))
                        :           +- FileScan csv [hash#17,timestamp#32] Batched: false, DataFilters: [(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as date) < 167..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string,timestamp:int>
                        +- *(4) Sort [block_hash#72 ASC NULLS FIRST], false, 0
                           +- Exchange hashpartitioning(block_hash#72, 200), true, [id=#203]
                              +- *(3) Project [hash#70, block_hash#72, from_address#75]
                                 +- *(3) Filter isnotnull(block_hash#72)
                                    +- FileScan csv [hash#70,block_hash#72,from_address#75] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<hash:string,block_hash:string,from_address:string>