task2 - Details for Query 4

Details for Query 4

Submitted Time: 2024/11/28 21:55:02
Duration: 6 s
Succeeded Jobs: 6

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { subgraph cluster0 { isCluster="true"; label="WholeStageCodegen (6)\n \nduration: 9 ms"; 1 [labelType="html" label="HashAggregate time in aggregation build: 9 ms number of output rows: 1"]; } 2 [labelType="html" label="Exchange shuffle records written: 200 shuffle write time total (min, med, max (stageId: taskId)) 34 ms (0 ms, 0 ms, 0 ms (stage 9.0: task 259)) records read: 200 local bytes read: 6.2 KiB fetch wait time: 1 ms remote bytes read: 5.4 KiB local blocks read: 107 remote blocks read: 93 data size total (min, med, max (stageId: taskId)) 3.1 KiB (16.0 B, 16.0 B, 16.0 B (stage 9.0: task 222)) shuffle bytes written total (min, med, max (stageId: taskId)) 11.5 KiB (58.0 B, 59.0 B, 59.0 B (stage 9.0: task 222))"]; subgraph cluster3 { isCluster="true"; label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n728 ms (2 ms, 3 ms, 47 ms (stage 9.0: task 222))"; 4 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 674 ms (2 ms, 3 ms, 44 ms (stage 9.0: task 223)) number of output rows: 200"]; 5 [labelType="html" label=" Project "]; 6 [labelType="html" label="SortMergeJoin number of output rows: 504,708"]; } subgraph cluster7 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n321 ms (0 ms, 0 ms, 14 ms (stage 9.0: task 225))"; 8 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 32 ms (0 ms, 0 ms, 9 ms (stage 9.0: task 222)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 9.0: task 222)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 9.0: task 222))"]; } 9 [labelType="html" label="Exchange shuffle records written: 500,001 shuffle write time total (min, med, max (stageId: taskId)) 107 ms (14 ms, 20 ms, 29 ms (stage 8.0: task 218)) records read: 500,001 local bytes read total (min, med, max (stageId: taskId)) 15.5 MiB (73.5 KiB, 79.2 KiB, 85.5 KiB (stage 9.0: task 363)) fetch wait time total (min, med, max (stageId: taskId)) 2 ms (0 ms, 0 ms, 2 ms (stage 9.0: task 264)) remote bytes read total (min, med, max (stageId: taskId)) 15.5 MiB (72.8 KiB, 79.4 KiB, 85.8 KiB (stage 9.0: task 245)) local blocks read: 493 remote blocks read: 507 data size total (min, med, max (stageId: taskId)) 42.0 MiB (573.9 KiB, 10.3 MiB, 10.4 MiB (stage 8.0: task 218)) shuffle bytes written total (min, med, max (stageId: taskId)) 31.0 MiB (471.4 KiB, 7.6 MiB, 7.7 MiB (stage 8.0: task 218))"]; subgraph cluster10 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n6.4 s (148 ms, 1.5 s, 1.6 s (stage 8.0: task 217))"; 11 [labelType="html" label=" Project "]; 12 [labelType="html" label="Filter number of output rows: 500,001"]; } 13 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 519.0 MiB number of output rows: 500,001"]; subgraph cluster14 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n1.6 s (3 ms, 6 ms, 122 ms (stage 9.0: task 222))"; 15 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 18 ms (0 ms, 0 ms, 6 ms (stage 9.0: task 222)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 9.0: task 223)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 9.0: task 222))"]; } 16 [labelType="html" label="Exchange shuffle records written: 504,708 shuffle write time total (min, med, max (stageId: taskId)) 43 ms (19 ms, 24 ms, 24 ms (stage 7.0: task 215)) records read: 504,708 local bytes read total (min, med, max (stageId: taskId)) 5.7 MiB (24.6 KiB, 29.0 KiB, 34.6 KiB (stage 9.0: task 331)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 9.0: task 222)) remote bytes read total (min, med, max (stageId: taskId)) 5.7 MiB (24.6 KiB, 29.0 KiB, 34.1 KiB (stage 9.0: task 392)) local blocks read: 200 remote blocks read: 200 data size total (min, med, max (stageId: taskId)) 42.4 MiB (19.9 MiB, 22.5 MiB, 22.5 MiB (stage 7.0: task 215)) shuffle bytes written total (min, med, max (stageId: taskId)) 11.4 MiB (5.5 MiB, 5.9 MiB, 5.9 MiB (stage 7.0: task 215))"]; subgraph cluster17 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n2.5 s (1.2 s, 1.3 s, 1.3 s (stage 7.0: task 215))"; 18 [labelType="html" label=" Project "]; 19 [labelType="html" label="Filter number of output rows: 504,708"]; } 20 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 173.6 MiB number of output rows: 504,708"]; 2->1; 4->2; 5->4; 6->5; 8->6; 9->8; 11->9; 12->11; 13->12; 15->6; 16->15; 18->16; 19->18; 20->19; }

HashAggregate(keys=[], functions=[count(1)])

WholeStageCodegen (6)

Exchange SinglePartition, true, [id=#137]

HashAggregate(keys=[], functions=[partial_count(1)])

Project

SortMergeJoin [hash#17], [block_hash#72], Inner

WholeStageCodegen (5)

Sort [hash#17 ASC NULLS FIRST], false, 0

WholeStageCodegen (2)

Exchange hashpartitioning(hash#17, 200), true, [id=#119]

Project [hash#17]

Filter isnotnull(hash#17)

WholeStageCodegen (1)

FileScan csv [hash#17] Batched: false, DataFilters: [isnotnull(hash#17)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string>

Sort [block_hash#72 ASC NULLS FIRST], false, 0

WholeStageCodegen (4)

Exchange hashpartitioning(block_hash#72, 200), true, [id=#128]

Project [block_hash#72]

Filter isnotnull(block_hash#72)

WholeStageCodegen (3)

FileScan csv [block_hash#72] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<block_hash:string>

Details

== Parsed Logical Plan ==
Aggregate [count(1) AS count#310L]
+- Join Inner, (hash#17 = block_hash#72)
   :- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
   :  +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
   +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Analyzed Logical Plan ==
count: bigint
Aggregate [count(1) AS count#310L]
+- Join Inner, (hash#17 = block_hash#72)
   :- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
   :  +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
   +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Optimized Logical Plan ==
Aggregate [count(1) AS count#310L]
+- Project
   +- Join Inner, (hash#17 = block_hash#72)
      :- Project [hash#17]
      :  +- Filter isnotnull(hash#17)
      :     +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
      +- Project [block_hash#72]
         +- Filter isnotnull(block_hash#72)
            +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Physical Plan ==
*(6) HashAggregate(keys=[], functions=[count(1)], output=[count#310L])
+- Exchange SinglePartition, true, [id=#137]
   +- *(5) HashAggregate(keys=[], functions=[partial_count(1)], output=[count#313L])
      +- *(5) Project
         +- *(5) SortMergeJoin [hash#17], [block_hash#72], Inner
            :- *(2) Sort [hash#17 ASC NULLS FIRST], false, 0
            :  +- Exchange hashpartitioning(hash#17, 200), true, [id=#119]
            :     +- *(1) Project [hash#17]
            :        +- *(1) Filter isnotnull(hash#17)
            :           +- FileScan csv [hash#17] Batched: false, DataFilters: [isnotnull(hash#17)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string>
            +- *(4) Sort [block_hash#72 ASC NULLS FIRST], false, 0
               +- Exchange hashpartitioning(block_hash#72, 200), true, [id=#128]
                  +- *(3) Project [block_hash#72]
                     +- *(3) Filter isnotnull(block_hash#72)
                        +- FileScan csv [block_hash#72] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<block_hash:string>