task2 - Details for Query 5

Details for Query 5

Submitted Time: 2024/12/04 20:22:31
Duration: 17 s
Succeeded Jobs: 7

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" TakeOrderedAndProject "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (7)\n \nduration: total (min, med, max (stageId: taskId))\n316 ms (0 ms, 0 ms, 18 ms (stage 16.0: task 831))"; 2 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 89 ms (0 ms, 0 ms, 7 ms (stage 16.0: task 835)) peak memory total (min, med, max (stageId: taskId)) 1650.0 MiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 16.0: task 834)) number of output rows: 30 avg hash probe bucket list iters (min, med, max (stageId: taskId)): (1, 1, 1 (stage 16.0: task 834))"]; } 3 [labelType="html" label="Exchange shuffle records written: 6,000 shuffle write time total (min, med, max (stageId: taskId)) 921 ms (3 ms, 3 ms, 16 ms (stage 15.0: task 631)) records read: 6,000 local bytes read total (min, med, max (stageId: taskId)) 223.8 KiB (0.0 B, 0.0 B, 12.6 KiB (stage 16.0: task 832)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 16.0: task 834)) remote bytes read total (min, med, max (stageId: taskId)) 206.2 KiB (0.0 B, 0.0 B, 15.0 KiB (stage 16.0: task 850)) local blocks read: 2,625 remote blocks read: 2,375 data size total (min, med, max (stageId: taskId)) 281.3 KiB (1440.0 B, 1440.0 B, 1440.0 B (stage 15.0: task 632)) shuffle bytes written total (min, med, max (stageId: taskId)) 430.0 KiB (2.1 KiB, 2.1 KiB, 2.2 KiB (stage 15.0: task 719))"]; subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n2.9 s (7 ms, 12 ms, 134 ms (stage 15.0: task 763))"; 5 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 1.1 s (2 ms, 4 ms, 125 ms (stage 15.0: task 764)) peak memory total (min, med, max (stageId: taskId)) 50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 15.0: task 632)) number of output rows: 6,000"]; 6 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 903 ms (2 ms, 3 ms, 125 ms (stage 15.0: task 764)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.3 MiB, 64.3 MiB, 64.3 MiB (stage 15.0: task 632)) number of output rows: 81,749 avg hash probe bucket list iters (min, med, max (stageId: taskId)): (1, 1, 1 (stage 15.0: task 632))"]; } 7 [labelType="html" label="Exchange shuffle records written: 139,568 shuffle write time total (min, med, max (stageId: taskId)) 2.6 s (9 ms, 11 ms, 27 ms (stage 14.0: task 467)) records read: 139,568 local bytes read total (min, med, max (stageId: taskId)) 4.6 MiB (12.6 KiB, 24.3 KiB, 43.3 KiB (stage 15.0: task 795)) fetch wait time total (min, med, max (stageId: taskId)) 203 ms (0 ms, 0 ms, 116 ms (stage 15.0: task 764)) remote bytes read total (min, med, max (stageId: taskId)) 4.1 MiB (12.1 KiB, 18.2 KiB, 37.5 KiB (stage 15.0: task 698)) local blocks read: 20,591 remote blocks read: 18,152 data size total (min, med, max (stageId: taskId)) 11.4 MiB (43.3 KiB, 57.9 KiB, 75.1 KiB (stage 14.0: task 529)) shuffle bytes written total (min, med, max (stageId: taskId)) 8.7 MiB (35.9 KiB, 44.0 KiB, 52.7 KiB (stage 14.0: task 529))"]; subgraph cluster8 { isCluster="true"; label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n5.2 s (13 ms, 22 ms, 151 ms (stage 14.0: task 438))"; 9 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 924 ms (1 ms, 2 ms, 107 ms (stage 14.0: task 439)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.3 MiB, 64.3 MiB, 64.3 MiB (stage 14.0: task 432)) number of output rows: 139,568 avg hash probe bucket list iters (min, med, max (stageId: taskId)): (1, 1, 1.1 (stage 14.0: task 496))"]; 10 [labelType="html" label="Expand number of output rows: 347,610"]; 11 [labelType="html" label=" Project "]; 12 [labelType="html" label="SortMergeJoin number of output rows: 173,805"]; } subgraph cluster13 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n1.2 s (0 ms, 5 ms, 119 ms (stage 14.0: task 431))"; 14 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 15 ms (0 ms, 0 ms, 8 ms (stage 14.0: task 608)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 14.0: task 432)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 432))"]; } 15 [labelType="html" label="Exchange shuffle records written: 504,708 shuffle write time total (min, med, max (stageId: taskId)) 65 ms (26 ms, 39 ms, 39 ms (stage 12.0: task 425)) records read: 504,708 local bytes read total (min, med, max (stageId: taskId)) 10.7 MiB (45.1 KiB, 54.6 KiB, 70.7 KiB (stage 14.0: task 540)) fetch wait time total (min, med, max (stageId: taskId)) 198 ms (0 ms, 0 ms, 103 ms (stage 14.0: task 439)) remote bytes read total (min, med, max (stageId: taskId)) 10.5 MiB (44.8 KiB, 53.0 KiB, 64.9 KiB (stage 14.0: task 579)) local blocks read: 200 remote blocks read: 200 data size total (min, med, max (stageId: taskId)) 69.3 MiB (32.5 MiB, 36.8 MiB, 36.8 MiB (stage 12.0: task 424)) shuffle bytes written total (min, med, max (stageId: taskId)) 21.2 MiB (10.0 MiB, 11.2 MiB, 11.2 MiB (stage 12.0: task 424))"]; subgraph cluster16 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n3.9 s (1.7 s, 2.2 s, 2.2 s (stage 12.0: task 425))"; 17 [labelType="html" label=" Project "]; 18 [labelType="html" label="Filter number of output rows: 504,708"]; } 19 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 173.6 MiB number of output rows: 504,708"]; subgraph cluster20 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n1.3 s (0 ms, 4 ms, 170 ms (stage 14.0: task 490))"; 21 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 14.0: task 432)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 14.0: task 432)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 432))"]; } 22 [labelType="html" label="Exchange shuffle records written: 144,178 shuffle write time total (min, med, max (stageId: taskId)) 45 ms (0 ms, 0 ms, 27 ms (stage 13.0: task 427)) records read: 144,178 local bytes read total (min, med, max (stageId: taskId)) 5.0 MiB (20.1 KiB, 25.1 KiB, 31.6 KiB (stage 14.0: task 510)) fetch wait time total (min, med, max (stageId: taskId)) 76 ms (0 ms, 0 ms, 76 ms (stage 14.0: task 490)) remote bytes read total (min, med, max (stageId: taskId)) 5.1 MiB (22.3 KiB, 26.4 KiB, 31.6 KiB (stage 14.0: task 466)) local blocks read: 200 remote blocks read: 200 data size total (min, med, max (stageId: taskId)) 16.5 MiB (0.0 B, 0.0 B, 8.8 MiB (stage 13.0: task 427)) shuffle bytes written total (min, med, max (stageId: taskId)) 10.1 MiB (0.0 B, 0.0 B, 5.4 MiB (stage 13.0: task 427))"]; subgraph cluster23 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n9.8 s (144 ms, 2.0 s, 3.1 s (stage 13.0: task 427))"; 24 [labelType="html" label=" Project "]; 25 [labelType="html" label="Filter number of output rows: 144,178"]; } 26 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 519.0 MiB number of output rows: 500,001"]; 2->0; 3->2; 5->3; 6->5; 7->6; 9->7; 10->9; 11->10; 12->11; 14->12; 15->14; 17->15; 18->17; 19->18; 21->12; 22->21; 24->22; 25->24; 26->25; }

TakeOrderedAndProject(limit=21, orderBy=[formatted_date#136 ASC NULLS FIRST], output=[formatted_date#136,block_count#362,unique_senders_count_number#363])

HashAggregate(keys=[formatted_date#136], functions=[count(if ((gid#367 = 1)) `number`#368 else null), count(if ((gid#367 = 2)) `from_address`#369 else null)])

WholeStageCodegen (7)

Exchange hashpartitioning(formatted_date#136, 200), true, [id=#244]

HashAggregate(keys=[formatted_date#136], functions=[partial_count(if ((gid#367 = 1)) `number`#368 else null), partial_count(if ((gid#367 = 2)) `from_address`#369 else null)])

HashAggregate(keys=[formatted_date#136, `number`#368, `from_address`#369, gid#367], functions=[])

WholeStageCodegen (6)

Exchange hashpartitioning(formatted_date#136, `number`#368, `from_address`#369, gid#367, 200), true, [id=#239]

HashAggregate(keys=[formatted_date#136, `number`#368, `from_address`#369, gid#367], functions=[])

Expand [ArrayBuffer(formatted_date#136, number#16, null, 1), ArrayBuffer(formatted_date#136, null, from_address#75, 2)], [formatted_date#136, `number`#368, `from_address`#369, gid#367]

Project [from_address#75, number#16, formatted_date#136]

SortMergeJoin [block_hash#72], [hash#17], Inner

WholeStageCodegen (5)

Sort [block_hash#72 ASC NULLS FIRST], false, 0

WholeStageCodegen (2)

Exchange hashpartitioning(block_hash#72, 200), true, [id=#220]

Project [block_hash#72, from_address#75]

Filter isnotnull(block_hash#72)

WholeStageCodegen (1)

FileScan csv [block_hash#72,from_address#75] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<block_hash:string,from_address:string>

Sort [hash#17 ASC NULLS FIRST], false, 0

WholeStageCodegen (4)

Exchange hashpartitioning(hash#17, 200), true, [id=#229]

Project [number#16, hash#17, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]

Filter ((date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM, Some(GMT)) = 2015-09) AND isnotnull(hash#17))

WholeStageCodegen (3)

FileScan csv [number#16,hash#17,timestamp#32] Batched: false, DataFilters: [(date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<number:int,hash:string,timestamp:int>

Details

== Parsed Logical Plan ==
GlobalLimit 21
+- LocalLimit 21
   +- Project [cast(formatted_date#136 as string) AS formatted_date#361, cast(block_count#351L as string) AS block_count#362, cast(unique_senders_count_number#352L as string) AS unique_senders_count_number#363]
      +- Sort [formatted_date#136 ASC NULLS FIRST], true
         +- Aggregate [formatted_date#136], [formatted_date#136, count(distinct number#16) AS block_count#351L, count(distinct from_address#75) AS unique_senders_count_number#352L]
            +- Filter (date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM, Some(GMT)) = 2015-09)
               +- Join Inner, (block_hash#72 = hash#17)
                  :- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv
                  +- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                     +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv

== Analyzed Logical Plan ==
formatted_date: string, block_count: string, unique_senders_count_number: string
GlobalLimit 21
+- LocalLimit 21
   +- Project [cast(formatted_date#136 as string) AS formatted_date#361, cast(block_count#351L as string) AS block_count#362, cast(unique_senders_count_number#352L as string) AS unique_senders_count_number#363]
      +- Sort [formatted_date#136 ASC NULLS FIRST], true
         +- Aggregate [formatted_date#136], [formatted_date#136, count(distinct number#16) AS block_count#351L, count(distinct from_address#75) AS unique_senders_count_number#352L]
            +- Filter (date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM, Some(GMT)) = 2015-09)
               +- Join Inner, (block_hash#72 = hash#17)
                  :- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv
                  +- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                     +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv

== Optimized Logical Plan ==
GlobalLimit 21
+- LocalLimit 21
   +- Project [formatted_date#136, cast(block_count#351L as string) AS block_count#362, cast(unique_senders_count_number#352L as string) AS unique_senders_count_number#363]
      +- Sort [formatted_date#136 ASC NULLS FIRST], true
         +- Aggregate [formatted_date#136], [formatted_date#136, count(if ((gid#367 = 1)) `number`#368 else null) AS block_count#351L, count(if ((gid#367 = 2)) `from_address`#369 else null) AS unique_senders_count_number#352L]
            +- Aggregate [formatted_date#136, `number`#368, `from_address`#369, gid#367], [formatted_date#136, `number`#368, `from_address`#369, gid#367]
               +- Expand [ArrayBuffer(formatted_date#136, number#16, null, 1), ArrayBuffer(formatted_date#136, null, from_address#75, 2)], [formatted_date#136, `number`#368, `from_address`#369, gid#367]
                  +- Project [from_address#75, number#16, formatted_date#136]
                     +- Join Inner, (block_hash#72 = hash#17)
                        :- Project [block_hash#72, from_address#75]
                        :  +- Filter isnotnull(block_hash#72)
                        :     +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv
                        +- Project [number#16, hash#17, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                           +- Filter ((date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM, Some(GMT)) = 2015-09) AND isnotnull(hash#17))
                              +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv

== Physical Plan ==
TakeOrderedAndProject(limit=21, orderBy=[formatted_date#136 ASC NULLS FIRST], output=[formatted_date#136,block_count#362,unique_senders_count_number#363])
+- *(7) HashAggregate(keys=[formatted_date#136], functions=[count(if ((gid#367 = 1)) `number`#368 else null), count(if ((gid#367 = 2)) `from_address`#369 else null)], output=[formatted_date#136, block_count#351L, unique_senders_count_number#352L])
   +- Exchange hashpartitioning(formatted_date#136, 200), true, [id=#244]
      +- *(6) HashAggregate(keys=[formatted_date#136], functions=[partial_count(if ((gid#367 = 1)) `number`#368 else null), partial_count(if ((gid#367 = 2)) `from_address`#369 else null)], output=[formatted_date#136, count#372L, count#373L])
         +- *(6) HashAggregate(keys=[formatted_date#136, `number`#368, `from_address`#369, gid#367], functions=[], output=[formatted_date#136, `number`#368, `from_address`#369, gid#367])
            +- Exchange hashpartitioning(formatted_date#136, `number`#368, `from_address`#369, gid#367, 200), true, [id=#239]
               +- *(5) HashAggregate(keys=[formatted_date#136, `number`#368, `from_address`#369, gid#367], functions=[], output=[formatted_date#136, `number`#368, `from_address`#369, gid#367])
                  +- *(5) Expand [ArrayBuffer(formatted_date#136, number#16, null, 1), ArrayBuffer(formatted_date#136, null, from_address#75, 2)], [formatted_date#136, `number`#368, `from_address`#369, gid#367]
                     +- *(5) Project [from_address#75, number#16, formatted_date#136]
                        +- *(5) SortMergeJoin [block_hash#72], [hash#17], Inner
                           :- *(2) Sort [block_hash#72 ASC NULLS FIRST], false, 0
                           :  +- Exchange hashpartitioning(block_hash#72, 200), true, [id=#220]
                           :     +- *(1) Project [block_hash#72, from_address#75]
                           :        +- *(1) Filter isnotnull(block_hash#72)
                           :           +- FileScan csv [block_hash#72,from_address#75] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<block_hash:string,from_address:string>
                           +- *(4) Sort [hash#17 ASC NULLS FIRST], false, 0
                              +- Exchange hashpartitioning(hash#17, 200), true, [id=#229]
                                 +- *(3) Project [number#16, hash#17, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                                    +- *(3) Filter ((date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM, Some(GMT)) = 2015-09) AND isnotnull(hash#17))
                                       +- FileScan csv [number#16,hash#17,timestamp#32] Batched: false, DataFilters: [(date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<number:int,hash:string,timestamp:int>