task2 - Details for Query 8

Details for Query 8

Submitted Time: 2024/10/29 20:23:42
Duration: 7 s
Succeeded Jobs: 11 12

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" CollectLimit "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (8)\n \nduration: 27 ms"; 2 [labelType="html" label=" Project "]; } 3 [labelType="html" label=" Coalesce "]; subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (7)\n \nduration: 0 ms"; 5 [labelType="html" label="Sort sort time: 0 ms peak memory: 1985.9 MiB spill size: 0.0 B"]; } 6 [labelType="html" label="Exchange shuffle records written: 31 shuffle write time total (min, med, max (stageId: taskId)) 15 ms (0 ms, 0 ms, 0 ms (stage 47.0: task 3048)) records read: 31 local bytes read: 1428.0 B fetch wait time: 0 ms remote bytes read: 1176.0 B local blocks read: 17 remote blocks read: 14 data size total (min, med, max (stageId: taskId)) 1240.0 B (0.0 B, 0.0 B, 120.0 B (stage 47.0: task 3055)) shuffle bytes written total (min, med, max (stageId: taskId)) 2.5 KiB (0.0 B, 0.0 B, 252.0 B (stage 47.0: task 3055))"]; subgraph cluster7 { isCluster="true"; label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n183 ms (0 ms, 0 ms, 5 ms (stage 43.0: task 3022))"; 8 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 124 ms (0 ms, 0 ms, 4 ms (stage 43.0: task 3022)) peak memory total (min, med, max (stageId: taskId)) 3.6 GiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 43.0: task 2847)) number of output rows: 62 avg hash probe bucket list iters (min, med, max (stageId: taskId)): (1, 1, 1 (stage 43.0: task 2847))"]; } 9 [labelType="html" label="Exchange shuffle records written: 6,200 shuffle write time total (min, med, max (stageId: taskId)) 675 ms (3 ms, 3 ms, 9 ms (stage 42.0: task 2761)) records read: 12,400 local bytes read total (min, med, max (stageId: taskId)) 467.2 KiB (0.0 B, 0.0 B, 11.8 KiB (stage 43.0: task 2855)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 43.0: task 2847)) remote bytes read total (min, med, max (stageId: taskId)) 467.7 KiB (0.0 B, 0.0 B, 12.3 KiB (stage 43.0: task 2855)) local blocks read: 5,600 remote blocks read: 5,600 data size total (min, med, max (stageId: taskId)) 242.2 KiB (1240.0 B, 1240.0 B, 1240.0 B (stage 42.0: task 2648)) shuffle bytes written total (min, med, max (stageId: taskId)) 467.5 KiB (2.3 KiB, 2.3 KiB, 2.3 KiB (stage 42.0: task 2722))"]; subgraph cluster10 { isCluster="true"; label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n1.1 s (4 ms, 5 ms, 15 ms (stage 42.0: task 2647))"; 11 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 221 ms (1 ms, 1 ms, 11 ms (stage 42.0: task 2647)) peak memory total (min, med, max (stageId: taskId)) 50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 42.0: task 2648)) number of output rows: 6,200"]; 12 [labelType="html" label=" Project "]; 13 [labelType="html" label="SortMergeJoin number of output rows: 72,118"]; } subgraph cluster14 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n371 ms (0 ms, 3 ms, 14 ms (stage 42.0: task 2647))"; 15 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 42.0: task 2648)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 42.0: task 2648)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 42.0: task 2648))"]; } 16 [labelType="html" label="Exchange shuffle records written: 156,095 shuffle write time total (min, med, max (stageId: taskId)) 28 ms (0 ms, 0 ms, 15 ms (stage 40.0: task 2643)) records read: 156,095 local bytes read total (min, med, max (stageId: taskId)) 5.0 MiB (16.2 KiB, 30.1 KiB, 37.5 KiB (stage 42.0: task 2719)) fetch wait time total (min, med, max (stageId: taskId)) 6 ms (0 ms, 0 ms, 6 ms (stage 42.0: task 2713)) remote bytes read total (min, med, max (stageId: taskId)) 5.0 MiB (15.9 KiB, 20.8 KiB, 36.6 KiB (stage 42.0: task 2779)) local blocks read: 200 remote blocks read: 200 data size total (min, med, max (stageId: taskId)) 16.7 MiB (0.0 B, 0.0 B, 10.7 MiB (stage 40.0: task 2643)) shuffle bytes written total (min, med, max (stageId: taskId)) 10.0 MiB (0.0 B, 0.0 B, 6.4 MiB (stage 40.0: task 2643))"]; subgraph cluster17 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n6.9 s (148 ms, 1.7 s, 2.0 s (stage 40.0: task 2643))"; 18 [labelType="html" label=" Project "]; 19 [labelType="html" label="Filter number of output rows: 156,095"]; } 20 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 519.0 MiB number of output rows: 500,001"]; subgraph cluster21 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n371 ms (0 ms, 2 ms, 9 ms (stage 42.0: task 2713))"; 22 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 42.0: task 2648)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 42.0: task 2648)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 42.0: task 2648))"]; } 23 [labelType="html" label="Exchange shuffle records written: 171,282 shuffle write time total (min, med, max (stageId: taskId)) 29 ms (14 ms, 14 ms, 14 ms (stage 41.0: task 2645)) records read: 171,282 local bytes read total (min, med, max (stageId: taskId)) 5.8 MiB (24.8 KiB, 29.6 KiB, 36.4 KiB (stage 42.0: task 2756)) fetch wait time total (min, med, max (stageId: taskId)) 4 ms (0 ms, 0 ms, 4 ms (stage 42.0: task 2789)) remote bytes read total (min, med, max (stageId: taskId)) 5.9 MiB (25.5 KiB, 30.5 KiB, 36.3 KiB (stage 42.0: task 2817)) local blocks read: 200 remote blocks read: 200 data size total (min, med, max (stageId: taskId)) 20.9 MiB (10.1 MiB, 10.8 MiB, 10.8 MiB (stage 41.0: task 2645)) shuffle bytes written total (min, med, max (stageId: taskId)) 11.8 MiB (5.6 MiB, 6.2 MiB, 6.2 MiB (stage 41.0: task 2645))"]; subgraph cluster24 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n2.5 s (1.2 s, 1.3 s, 1.3 s (stage 41.0: task 2645))"; 25 [labelType="html" label=" Project "]; 26 [labelType="html" label="Filter number of output rows: 171,282"]; } 27 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 173.6 MiB number of output rows: 504,708"]; 2->0; 3->2; 5->3; 6->5; 8->6; 9->8; 11->9; 12->11; 13->12; 15->13; 16->15; 18->16; 19->18; 20->19; 22->13; 23->22; 25->23; 26->25; 27->26; }

CollectLimit 32

Project [formatted_date#136, cast(total_transaction_fee#464 as string) AS total_transaction_fee#480]

WholeStageCodegen (8)

Coalesce 1

Sort [formatted_date#136 ASC NULLS FIRST], true, 0

WholeStageCodegen (7)

Exchange rangepartitioning(formatted_date#136 ASC NULLS FIRST, 200), true, [id=#536]

HashAggregate(keys=[formatted_date#136], functions=[sum(transaction_fee#390)])

WholeStageCodegen (6)

Exchange hashpartitioning(formatted_date#136, 200), true, [id=#532]

HashAggregate(keys=[formatted_date#136], functions=[partial_sum(transaction_fee#390)])

Project [formatted_date#136, (cast(gas#78 as double) * cast(gas_price#79 as double)) AS transaction_fee#390]

SortMergeJoin [hash#17], [block_hash#72], Inner

WholeStageCodegen (5)

Sort [hash#17 ASC NULLS FIRST], false, 0

WholeStageCodegen (2)

Exchange hashpartitioning(hash#17, 200), true, [id=#514]

Project [hash#17, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]

Filter (((date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) >= 2015-10-01) AND (date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) <= 2015-10-31)) AND isnotnull(hash#17))

WholeStageCodegen (1)

FileScan csv [hash#17,timestamp#32] Batched: false, DataFilters: [(date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string,timestamp:string>

Sort [block_hash#72 ASC NULLS FIRST], false, 0

WholeStageCodegen (4)

Exchange hashpartitioning(block_hash#72, 200), true, [id=#523]

Project [block_hash#72, gas#78, gas_price#79]

Filter ((isnotnull(transaction_index#74) AND (cast(transaction_index#74 as int) = 0)) AND isnotnull(block_hash#72))

WholeStageCodegen (3)

FileScan csv [block_hash#72,transaction_index#74,gas#78,gas_price#79] Batched: false, DataFilters: [isnotnull(transaction_index#74), (cast(transaction_index#74 as int) = 0), isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(transaction_index), IsNotNull(block_hash)], ReadSchema: struct<block_hash:string,transaction_index:string,gas:string,gas_price:string>

Details

== Parsed Logical Plan ==
GlobalLimit 32
+- LocalLimit 32
   +- Project [cast(formatted_date#136 as string) AS formatted_date#479, cast(total_transaction_fee#464 as string) AS total_transaction_fee#480]
      +- Repartition 1, false
         +- Sort [formatted_date#136 ASC NULLS FIRST], true
            +- Aggregate [formatted_date#136], [formatted_date#136, sum(transaction_fee#390) AS total_transaction_fee#464]
               +- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, formatted_date#136, hash#70, nonce#71, block_hash#72, block_number#73, ... 12 more fields]
                  +- Filter (((formatted_date#136 >= 2015-10-01) AND (formatted_date#136 <= 2015-10-31)) AND (cast(transaction_index#74 as int) = 0))
                     +- Join Inner, (hash#17 = block_hash#72)
                        :- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                        :  +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
                        +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Analyzed Logical Plan ==
formatted_date: string, total_transaction_fee: string
GlobalLimit 32
+- LocalLimit 32
   +- Project [cast(formatted_date#136 as string) AS formatted_date#479, cast(total_transaction_fee#464 as string) AS total_transaction_fee#480]
      +- Repartition 1, false
         +- Sort [formatted_date#136 ASC NULLS FIRST], true
            +- Aggregate [formatted_date#136], [formatted_date#136, sum(transaction_fee#390) AS total_transaction_fee#464]
               +- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, formatted_date#136, hash#70, nonce#71, block_hash#72, block_number#73, ... 12 more fields]
                  +- Filter (((formatted_date#136 >= 2015-10-01) AND (formatted_date#136 <= 2015-10-31)) AND (cast(transaction_index#74 as int) = 0))
                     +- Join Inner, (hash#17 = block_hash#72)
                        :- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                        :  +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
                        +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Optimized Logical Plan ==
GlobalLimit 32
+- LocalLimit 32
   +- Project [formatted_date#136, cast(total_transaction_fee#464 as string) AS total_transaction_fee#480]
      +- Repartition 1, false
         +- Sort [formatted_date#136 ASC NULLS FIRST], true
            +- Aggregate [formatted_date#136], [formatted_date#136, sum(transaction_fee#390) AS total_transaction_fee#464]
               +- Project [formatted_date#136, (cast(gas#78 as double) * cast(gas_price#79 as double)) AS transaction_fee#390]
                  +- Join Inner, (hash#17 = block_hash#72)
                     :- Project [hash#17, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                     :  +- Filter (((date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) >= 2015-10-01) AND (date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) <= 2015-10-31)) AND isnotnull(hash#17))
                     :     +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
                     +- Project [block_hash#72, gas#78, gas_price#79]
                        +- Filter ((isnotnull(transaction_index#74) AND (cast(transaction_index#74 as int) = 0)) AND isnotnull(block_hash#72))
                           +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Physical Plan ==
CollectLimit 32
+- *(8) Project [formatted_date#136, cast(total_transaction_fee#464 as string) AS total_transaction_fee#480]
   +- Coalesce 1
      +- *(7) Sort [formatted_date#136 ASC NULLS FIRST], true, 0
         +- Exchange rangepartitioning(formatted_date#136 ASC NULLS FIRST, 200), true, [id=#536]
            +- *(6) HashAggregate(keys=[formatted_date#136], functions=[sum(transaction_fee#390)], output=[formatted_date#136, total_transaction_fee#464])
               +- Exchange hashpartitioning(formatted_date#136, 200), true, [id=#532]
                  +- *(5) HashAggregate(keys=[formatted_date#136], functions=[partial_sum(transaction_fee#390)], output=[formatted_date#136, sum#470])
                     +- *(5) Project [formatted_date#136, (cast(gas#78 as double) * cast(gas_price#79 as double)) AS transaction_fee#390]
                        +- *(5) SortMergeJoin [hash#17], [block_hash#72], Inner
                           :- *(2) Sort [hash#17 ASC NULLS FIRST], false, 0
                           :  +- Exchange hashpartitioning(hash#17, 200), true, [id=#514]
                           :     +- *(1) Project [hash#17, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                           :        +- *(1) Filter (((date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) >= 2015-10-01) AND (date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) <= 2015-10-31)) AND isnotnull(hash#17))
                           :           +- FileScan csv [hash#17,timestamp#32] Batched: false, DataFilters: [(date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string,timestamp:string>
                           +- *(4) Sort [block_hash#72 ASC NULLS FIRST], false, 0
                              +- Exchange hashpartitioning(block_hash#72, 200), true, [id=#523]
                                 +- *(3) Project [block_hash#72, gas#78, gas_price#79]
                                    +- *(3) Filter ((isnotnull(transaction_index#74) AND (cast(transaction_index#74 as int) = 0)) AND isnotnull(block_hash#72))
                                       +- FileScan csv [block_hash#72,transaction_index#74,gas#78,gas_price#79] Batched: false, DataFilters: [isnotnull(transaction_index#74), (cast(transaction_index#74 as int) = 0), isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(transaction_index), IsNotNull(block_hash)], ReadSchema: struct<block_hash:string,transaction_index:string,gas:string,gas_price:string>