task2 - Details for Query 6

Details for Query 6

Submitted Time: 2024/11/28 21:55:16
Duration: 15 s
Succeeded Jobs: 8 9 10 11

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" CollectLimit "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (7)\n \nduration: total (min, med, max (stageId: taskId))\n110 ms (0 ms, 1 ms, 13 ms (stage 18.0: task 838))"; 2 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 28 ms (0 ms, 0 ms, 4 ms (stage 28.0: task 843)) peak memory total (min, med, max (stageId: taskId)) 729.0 MiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 28.0: task 843)) number of output rows: 13 avg hash probe bucket list iters (min, med, max (stageId: taskId)): (1, 1, 1 (stage 28.0: task 843))"]; } 3 [labelType="html" label="Exchange shuffle records written: 5,933 shuffle write time total (min, med, max (stageId: taskId)) 541 ms (2 ms, 2 ms, 6 ms (stage 17.0: task 732)) records read: 2,575 local bytes read total (min, med, max (stageId: taskId)) 186.9 KiB (0.0 B, 0.0 B, 20.4 KiB (stage 28.0: task 843)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 28.0: task 843)) local blocks read: 2,182 data size total (min, med, max (stageId: taskId)) 278.1 KiB (1344.0 B, 1440.0 B, 1440.0 B (stage 17.0: task 640)) shuffle bytes written total (min, med, max (stageId: taskId)) 425.4 KiB (2011.0 B, 2.1 KiB, 2.2 KiB (stage 17.0: task 734))"]; subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n1.5 s (6 ms, 7 ms, 58 ms (stage 17.0: task 734))"; 5 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 629 ms (2 ms, 3 ms, 50 ms (stage 17.0: task 734)) peak memory total (min, med, max (stageId: taskId)) 50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 17.0: task 638)) number of output rows: 5,933"]; 6 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 420 ms (1 ms, 2 ms, 48 ms (stage 17.0: task 734)) peak memory total (min, med, max (stageId: taskId)) 50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 17.0: task 638)) number of output rows: 28,128"]; } 7 [labelType="html" label="Exchange shuffle records written: 85,947 shuffle write time total (min, med, max (stageId: taskId)) 1.6 s (6 ms, 7 ms, 16 ms (stage 16.0: task 529)) records read: 85,947 local bytes read total (min, med, max (stageId: taskId)) 7.2 MiB (24.8 KiB, 35.8 KiB, 61.1 KiB (stage 17.0: task 755)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 17.0: task 638)) local blocks read: 35,312 data size total (min, med, max (stageId: taskId)) 7.9 MiB (27.4 KiB, 40.0 KiB, 54.3 KiB (stage 16.0: task 548)) shuffle bytes written total (min, med, max (stageId: taskId)) 7.2 MiB (28.0 KiB, 36.9 KiB, 46.0 KiB (stage 16.0: task 548))"]; subgraph cluster8 { isCluster="true"; label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n2.8 s (10 ms, 12 ms, 96 ms (stage 16.0: task 442))"; 9 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 398 ms (1 ms, 2 ms, 39 ms (stage 16.0: task 438)) peak memory total (min, med, max (stageId: taskId)) 50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 16.0: task 438)) number of output rows: 85,947"]; 10 [labelType="html" label=" Project "]; 11 [labelType="html" label="SortMergeJoin number of output rows: 173,805"]; } subgraph cluster12 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n692 ms (0 ms, 4 ms, 12 ms (stage 16.0: task 531))"; 13 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 16.0: task 438)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 16.0: task 438)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 16.0: task 438))"]; } 14 [labelType="html" label="Exchange shuffle records written: 144,178 shuffle write time total (min, med, max (stageId: taskId)) 34 ms (0 ms, 0 ms, 19 ms (stage 14.0: task 433)) records read: 144,178 local bytes read total (min, med, max (stageId: taskId)) 10.1 MiB (47.3 KiB, 51.6 KiB, 56.4 KiB (stage 16.0: task 461)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 16.0: task 438)) local blocks read: 400 data size total (min, med, max (stageId: taskId)) 16.5 MiB (0.0 B, 0.0 B, 8.8 MiB (stage 14.0: task 432)) shuffle bytes written total (min, med, max (stageId: taskId)) 10.1 MiB (0.0 B, 0.0 B, 5.4 MiB (stage 14.0: task 432))"]; subgraph cluster15 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n6.1 s (145 ms, 1.4 s, 1.6 s (stage 14.0: task 434))"; 16 [labelType="html" label=" Project "]; 17 [labelType="html" label="Filter number of output rows: 144,178"]; } 18 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 519.0 MiB number of output rows: 500,001"]; subgraph cluster19 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n752 ms (0 ms, 4 ms, 80 ms (stage 16.0: task 438))"; 20 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 6 ms (0 ms, 0 ms, 1 ms (stage 16.0: task 508)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 16.0: task 439)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 16.0: task 438))"]; } 21 [labelType="html" label="Exchange shuffle records written: 504,708 shuffle write time total (min, med, max (stageId: taskId)) 52 ms (25 ms, 26 ms, 26 ms (stage 15.0: task 437)) records read: 504,708 local bytes read total (min, med, max (stageId: taskId)) 11.2 MiB (46.2 KiB, 56.8 KiB, 70.7 KiB (stage 16.0: task 547)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 16.0: task 438)) remote bytes read total (min, med, max (stageId: taskId)) 10.0 MiB (44.8 KiB, 51.1 KiB, 59.3 KiB (stage 16.0: task 469)) local blocks read: 200 remote blocks read: 200 data size total (min, med, max (stageId: taskId)) 69.3 MiB (32.5 MiB, 36.8 MiB, 36.8 MiB (stage 15.0: task 436)) shuffle bytes written total (min, med, max (stageId: taskId)) 21.2 MiB (10.0 MiB, 11.2 MiB, 11.2 MiB (stage 15.0: task 436))"]; subgraph cluster22 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n2.7 s (1.2 s, 1.5 s, 1.5 s (stage 15.0: task 436))"; 23 [labelType="html" label=" Project "]; 24 [labelType="html" label="Filter number of output rows: 504,708"]; } 25 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 173.6 MiB number of output rows: 504,708"]; 2->0; 3->2; 5->3; 6->5; 7->6; 9->7; 10->9; 11->10; 13->11; 14->13; 16->14; 17->16; 18->17; 20->11; 21->20; 23->21; 24->23; 25->24; }

CollectLimit 6

HashAggregate(keys=[formatted_date#136], functions=[count(number#16), count(distinct from_address#75)])

WholeStageCodegen (7)

Exchange hashpartitioning(formatted_date#136, 200), true, [id=#279]

HashAggregate(keys=[formatted_date#136], functions=[merge_count(number#16), partial_count(distinct from_address#75)])

HashAggregate(keys=[formatted_date#136, from_address#75], functions=[merge_count(number#16)])

WholeStageCodegen (6)

Exchange hashpartitioning(formatted_date#136, from_address#75, 200), true, [id=#274]

HashAggregate(keys=[formatted_date#136, from_address#75], functions=[partial_count(number#16)])

Project [number#16, formatted_date#136, from_address#75]

SortMergeJoin [hash#17], [block_hash#72], Inner

WholeStageCodegen (5)

Sort [hash#17 ASC NULLS FIRST], false, 0

WholeStageCodegen (2)

Exchange hashpartitioning(hash#17, 200), true, [id=#256]

Project [number#16, hash#17, from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]

Filter ((from_unixtime(cast(timestamp#32 as bigint), yyyy-MM, Some(GMT)) = 2015-09) AND isnotnull(hash#17))

WholeStageCodegen (1)

FileScan csv [number#16,hash#17,timestamp#32] Batched: false, DataFilters: [(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM, Some(GMT)) = 2015-09), isnotnull(hash#17)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<number:int,hash:string,timestamp:int>

Sort [block_hash#72 ASC NULLS FIRST], false, 0

WholeStageCodegen (4)

Exchange hashpartitioning(block_hash#72, 200), true, [id=#265]

Project [block_hash#72, from_address#75]

Filter isnotnull(block_hash#72)

WholeStageCodegen (3)

FileScan csv [block_hash#72,from_address#75] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<block_hash:string,from_address:string>

Details

== Parsed Logical Plan ==
GlobalLimit 6
+- LocalLimit 6
   +- Project [cast(formatted_date#136 as string) AS formatted_date#536, cast(count(number)#527L as string) AS count(number)#537, cast(count(from_address)#528L as string) AS count(from_address)#538]
      +- Aggregate [formatted_date#136], [formatted_date#136, count(number#16) AS count(number)#527L, count(distinct from_address#75) AS count(from_address)#528L]
         +- Filter (from_unixtime(cast(timestamp#32 as bigint), yyyy-MM, Some(GMT)) = 2015-09)
            +- Join Inner, (hash#17 = block_hash#72)
               :- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
               :  +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
               +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Analyzed Logical Plan ==
formatted_date: string, count(number): string, count(from_address): string
GlobalLimit 6
+- LocalLimit 6
   +- Project [cast(formatted_date#136 as string) AS formatted_date#536, cast(count(number)#527L as string) AS count(number)#537, cast(count(from_address)#528L as string) AS count(from_address)#538]
      +- Aggregate [formatted_date#136], [formatted_date#136, count(number#16) AS count(number)#527L, count(distinct from_address#75) AS count(from_address)#528L]
         +- Filter (from_unixtime(cast(timestamp#32 as bigint), yyyy-MM, Some(GMT)) = 2015-09)
            +- Join Inner, (hash#17 = block_hash#72)
               :- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
               :  +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
               +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Optimized Logical Plan ==
GlobalLimit 6
+- LocalLimit 6
   +- Aggregate [formatted_date#136], [formatted_date#136, cast(count(number#16) as string) AS count(number)#537, cast(count(distinct from_address#75) as string) AS count(from_address)#538]
      +- Project [number#16, formatted_date#136, from_address#75]
         +- Join Inner, (hash#17 = block_hash#72)
            :- Project [number#16, hash#17, from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
            :  +- Filter ((from_unixtime(cast(timestamp#32 as bigint), yyyy-MM, Some(GMT)) = 2015-09) AND isnotnull(hash#17))
            :     +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv
            +- Project [block_hash#72, from_address#75]
               +- Filter isnotnull(block_hash#72)
                  +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv

== Physical Plan ==
CollectLimit 6
+- *(7) HashAggregate(keys=[formatted_date#136], functions=[count(number#16), count(distinct from_address#75)], output=[formatted_date#136, count(number)#537, count(from_address)#538])
   +- Exchange hashpartitioning(formatted_date#136, 200), true, [id=#279]
      +- *(6) HashAggregate(keys=[formatted_date#136], functions=[merge_count(number#16), partial_count(distinct from_address#75)], output=[formatted_date#136, count#543L, count#546L])
         +- *(6) HashAggregate(keys=[formatted_date#136, from_address#75], functions=[merge_count(number#16)], output=[formatted_date#136, from_address#75, count#543L])
            +- Exchange hashpartitioning(formatted_date#136, from_address#75, 200), true, [id=#274]
               +- *(5) HashAggregate(keys=[formatted_date#136, from_address#75], functions=[partial_count(number#16)], output=[formatted_date#136, from_address#75, count#543L])
                  +- *(5) Project [number#16, formatted_date#136, from_address#75]
                     +- *(5) SortMergeJoin [hash#17], [block_hash#72], Inner
                        :- *(2) Sort [hash#17 ASC NULLS FIRST], false, 0
                        :  +- Exchange hashpartitioning(hash#17, 200), true, [id=#256]
                        :     +- *(1) Project [number#16, hash#17, from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                        :        +- *(1) Filter ((from_unixtime(cast(timestamp#32 as bigint), yyyy-MM, Some(GMT)) = 2015-09) AND isnotnull(hash#17))
                        :           +- FileScan csv [number#16,hash#17,timestamp#32] Batched: false, DataFilters: [(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM, Some(GMT)) = 2015-09), isnotnull(hash#17)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<number:int,hash:string,timestamp:int>
                        +- *(4) Sort [block_hash#72 ASC NULLS FIRST], false, 0
                           +- Exchange hashpartitioning(block_hash#72, 200), true, [id=#265]
                              +- *(3) Project [block_hash#72, from_address#75]
                                 +- *(3) Filter isnotnull(block_hash#72)
                                    +- FileScan csv [block_hash#72,from_address#75] Batched: false, DataFilters: [isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(block_hash)], ReadSchema: struct<block_hash:string,from_address:string>