task2 - Details for Query 5

Details for Query 5

Submitted Time: 2024/12/17 23:14:18
Duration: 9 s
Succeeded Jobs: 7

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { subgraph cluster0 { isCluster="true"; label="WholeStageCodegen (6)\n \nduration: 10 ms"; 1 [labelType="html" label="HashAggregate time in aggregation build: 9 ms number of output rows: 1"]; } 2 [labelType="html" label="Exchange shuffle records written: 200 shuffle write time total (min, med, max (stageId: taskId)) 70 ms (0 ms, 0 ms, 8 ms (stage 10.0: task 242)) records read: 200 local bytes read: 5.9 KiB fetch wait time: 0 ms remote bytes read: 5.6 KiB local blocks read: 103 remote blocks read: 97 data size total (min, med, max (stageId: taskId)) 3.1 KiB (16.0 B, 16.0 B, 16.0 B (stage 10.0: task 223)) shuffle bytes written total (min, med, max (stageId: taskId)) 11.5 KiB (58.0 B, 59.0 B, 59.0 B (stage 10.0: task 223))"]; subgraph cluster3 { isCluster="true"; label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n1.2 s (3 ms, 4 ms, 83 ms (stage 10.0: task 224))"; 4 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 1.1 s (3 ms, 4 ms, 79 ms (stage 10.0: task 224)) number of output rows: 200"]; 5 [labelType="html" label=" Project "]; 6 [labelType="html" label="SortMergeJoin number of output rows: 504,708"]; } subgraph cluster7 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n2.0 s (6 ms, 8 ms, 169 ms (stage 10.0: task 224))"; 8 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 74 ms (0 ms, 0 ms, 13 ms (stage 10.0: task 223)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 10.0: task 224)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 10.0: task 223))"]; } 9 [labelType="html" label="Exchange shuffle records written: 504,708 shuffle write time total (min, med, max (stageId: taskId)) 52 ms (25 ms, 26 ms, 26 ms (stage 9.0: task 222)) records read: 504,708 local bytes read total (min, med, max (stageId: taskId)) 5.6 MiB (24.6 KiB, 28.9 KiB, 34.0 KiB (stage 10.0: task 259)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 10.0: task 223)) remote bytes read total (min, med, max (stageId: taskId)) 5.7 MiB (25.3 KiB, 29.2 KiB, 34.6 KiB (stage 10.0: task 332)) local blocks read: 200 remote blocks read: 200 data size total (min, med, max (stageId: taskId)) 42.4 MiB (19.9 MiB, 22.5 MiB, 22.5 MiB (stage 9.0: task 221)) shuffle bytes written total (min, med, max (stageId: taskId)) 11.4 MiB (5.5 MiB, 5.9 MiB, 5.9 MiB (stage 9.0: task 221))"]; subgraph cluster10 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n3.4 s (1.5 s, 1.9 s, 1.9 s (stage 9.0: task 221))"; 11 [labelType="html" label=" Project "]; 12 [labelType="html" label="Filter number of output rows: 504,708"]; } 13 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 173.6 MiB number of output rows: 504,708"]; subgraph cluster14 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n377 ms (0 ms, 0 ms, 20 ms (stage 10.0: task 227))"; 15 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 81 ms (0 ms, 0 ms, 6 ms (stage 10.0: task 224)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 10.0: task 223)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 10.0: task 223))"]; } 16 [labelType="html" label="Exchange shuffle records written: 500,001 shuffle write time total (min, med, max (stageId: taskId)) 145 ms (20 ms, 29 ms, 33 ms (stage 8.0: task 217)) records read: 500,001 local bytes read total (min, med, max (stageId: taskId)) 15.5 MiB (71.7 KiB, 79.2 KiB, 86.0 KiB (stage 10.0: task 253)) fetch wait time total (min, med, max (stageId: taskId)) 2 ms (0 ms, 0 ms, 1 ms (stage 10.0: task 315)) remote bytes read total (min, med, max (stageId: taskId)) 15.5 MiB (72.4 KiB, 79.5 KiB, 86.1 KiB (stage 10.0: task 421)) local blocks read: 497 remote blocks read: 503 data size total (min, med, max (stageId: taskId)) 42.0 MiB (573.9 KiB, 10.3 MiB, 10.4 MiB (stage 8.0: task 217)) shuffle bytes written total (min, med, max (stageId: taskId)) 31.0 MiB (471.4 KiB, 7.6 MiB, 7.7 MiB (stage 8.0: task 217))"]; subgraph cluster17 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n10.2 s (183 ms, 2.3 s, 3.0 s (stage 8.0: task 217))"; 18 [labelType="html" label=" Project "]; 19 [labelType="html" label="Filter number of output rows: 500,001"]; } 20 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 519.0 MiB number of output rows: 500,001"]; 2->1; 4->2; 5->4; 6->5; 8->6; 9->8; 11->9; 12->11; 13->12; 15->6; 16->15; 18->16; 19->18; 20->19; }

HashAggregate(keys=[], functions=[count(1)])

WholeStageCodegen (6)

Exchange SinglePartition, true, [id=#149]

HashAggregate(keys=[], functions=[partial_count(1)])

Project

SortMergeJoin [block_hash#72], [hash#17], Inner

WholeStageCodegen (5)

Sort [block_hash#72 ASC NULLS FIRST], false, 0

WholeStageCodegen (2)

Exchange hashpartitioning(block_hash#72, 200), true, [id=#131]

Project [block_hash#72]

Filter isnotnull(block_hash#72)

WholeStageCodegen (1)

FileScan csv [block_hash#72] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<block_hash:string>

Sort [hash#17 ASC NULLS FIRST], false, 0

WholeStageCodegen (4)

Exchange hashpartitioning(hash#17, 200), true, [id=#140]

Project [hash#17]

Filter isnotnull(hash#17)

WholeStageCodegen (3)

FileScan csv [hash#17] Batched: false, DataFilters: [isnotnull(hash#17)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string>

Details

== Parsed Logical Plan ==
Aggregate [count(1) AS count#410L]
+- Join Inner, (block_hash#72 = hash#17)
   :- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv
   +- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
      +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv

== Analyzed Logical Plan ==
count: bigint
Aggregate [count(1) AS count#410L]
+- Join Inner, (block_hash#72 = hash#17)
   :- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv
   +- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
      +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv

== Optimized Logical Plan ==
Aggregate [count(1) AS count#410L]
+- Project
   +- Join Inner, (block_hash#72 = hash#17)
      :- Project [block_hash#72]
      :  +- Filter isnotnull(block_hash#72)
      :     +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv
      +- Project [hash#17]
         +- Filter isnotnull(hash#17)
            +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv

== Physical Plan ==
*(6) HashAggregate(keys=[], functions=[count(1)], output=[count#410L])
+- Exchange SinglePartition, true, [id=#149]
   +- *(5) HashAggregate(keys=[], functions=[partial_count(1)], output=[count#413L])
      +- *(5) Project
         +- *(5) SortMergeJoin [block_hash#72], [hash#17], Inner
            :- *(2) Sort [block_hash#72 ASC NULLS FIRST], false, 0
            :  +- Exchange hashpartitioning(block_hash#72, 200), true, [id=#131]
            :     +- *(1) Project [block_hash#72]
            :        +- *(1) Filter isnotnull(block_hash#72)
            :           +- FileScan csv [block_hash#72] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<block_hash:string>
            +- *(4) Sort [hash#17 ASC NULLS FIRST], false, 0
               +- Exchange hashpartitioning(hash#17, 200), true, [id=#140]
                  +- *(3) Project [hash#17]
                     +- *(3) Filter isnotnull(hash#17)
                        +- FileScan csv [hash#17] Batched: false, DataFilters: [isnotnull(hash#17)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string>