task2 - Details for Query 8

Details for Query 8

Submitted Time: 2024/12/04 20:23:21
Duration: 12 s
Succeeded Jobs: 11 12

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label="Execute InsertIntoHadoopFsRelationCommand number of written files: 1 written output: 1046.0 B number of output rows: 31 number of dynamic part: 0"]; 1 [labelType="html" label=" Coalesce "]; subgraph cluster2 { isCluster="true"; label="WholeStageCodegen (7)\n \nduration: 1 ms"; 3 [labelType="html" label="Sort sort time: 0 ms peak memory: 1985.9 MiB spill size: 0.0 B"]; } 4 [labelType="html" label="Exchange shuffle records written: 31 shuffle write time total (min, med, max (stageId: taskId)) 15 ms (0 ms, 0 ms, 0 ms (stage 39.0: task 2712)) records read: 31 local bytes read: 2.5 KiB fetch wait time: 0 ms local blocks read: 31 data size total (min, med, max (stageId: taskId)) 1240.0 B (0.0 B, 0.0 B, 120.0 B (stage 39.0: task 2671)) shuffle bytes written total (min, med, max (stageId: taskId)) 2.5 KiB (0.0 B, 0.0 B, 249.0 B (stage 39.0: task 2671))"]; subgraph cluster5 { isCluster="true"; label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n395 ms (0 ms, 1 ms, 7 ms (stage 39.0: task 2709))"; 6 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 128 ms (0 ms, 0 ms, 5 ms (stage 39.0: task 2709)) peak memory total (min, med, max (stageId: taskId)) 3.6 GiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 39.0: task 2653)) number of output rows: 62 avg hash probe bucket list iters (min, med, max (stageId: taskId)): (1, 1, 1 (stage 39.0: task 2653))"]; } 7 [labelType="html" label="Exchange shuffle records written: 6,200 shuffle write time total (min, med, max (stageId: taskId)) 705 ms (3 ms, 3 ms, 15 ms (stage 34.0: task 2317)) records read: 12,400 local bytes read total (min, med, max (stageId: taskId)) 915.0 KiB (0.0 B, 0.0 B, 23.3 KiB (stage 39.0: task 2671)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 39.0: task 2653)) local blocks read: 11,200 data size total (min, med, max (stageId: taskId)) 242.2 KiB (1240.0 B, 1240.0 B, 1240.0 B (stage 34.0: task 2253)) shuffle bytes written total (min, med, max (stageId: taskId)) 457.5 KiB (2.3 KiB, 2.3 KiB, 2.3 KiB (stage 34.0: task 2413))"]; subgraph cluster8 { isCluster="true"; label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n1.1 s (5 ms, 5 ms, 14 ms (stage 34.0: task 2261))"; 9 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 198 ms (0 ms, 1 ms, 1 ms (stage 34.0: task 2253)) peak memory total (min, med, max (stageId: taskId)) 50.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 34.0: task 2253)) number of output rows: 6,200"]; 10 [labelType="html" label=" Project "]; 11 [labelType="html" label="SortMergeJoin number of output rows: 72,118"]; } subgraph cluster12 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n385 ms (0 ms, 3 ms, 4 ms (stage 34.0: task 2277))"; 13 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 34.0: task 2253)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 34.0: task 2253)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 34.0: task 2253))"]; } 14 [labelType="html" label="Exchange shuffle records written: 171,282 shuffle write time total (min, med, max (stageId: taskId)) 34 ms (15 ms, 18 ms, 18 ms (stage 33.0: task 2251)) records read: 171,282 local bytes read total (min, med, max (stageId: taskId)) 11.4 MiB (53.0 KiB, 58.5 KiB, 64.6 KiB (stage 34.0: task 2342)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 34.0: task 2253)) local blocks read: 400 data size total (min, med, max (stageId: taskId)) 17.0 MiB (8.2 MiB, 8.8 MiB, 8.8 MiB (stage 33.0: task 2251)) shuffle bytes written total (min, med, max (stageId: taskId)) 11.4 MiB (5.5 MiB, 6.0 MiB, 6.0 MiB (stage 33.0: task 2251))"]; subgraph cluster15 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.4 s (1.2 s, 1.2 s, 1.2 s (stage 33.0: task 2251))"; 16 [labelType="html" label=" Project "]; 17 [labelType="html" label="Filter number of output rows: 171,282"]; } 18 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 173.6 MiB number of output rows: 171,282"]; subgraph cluster19 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n349 ms (0 ms, 2 ms, 7 ms (stage 34.0: task 2276))"; 20 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 34.0: task 2253)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 34.0: task 2253)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 34.0: task 2253))"]; } 21 [labelType="html" label="Exchange shuffle records written: 156,095 shuffle write time total (min, med, max (stageId: taskId)) 45 ms (0 ms, 0 ms, 31 ms (stage 32.0: task 2249)) records read: 156,095 local bytes read total (min, med, max (stageId: taskId)) 3.6 MiB (15.9 KiB, 18.3 KiB, 21.2 KiB (stage 34.0: task 2324)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 34.0: task 2253)) remote bytes read total (min, med, max (stageId: taskId)) 6.4 MiB (28.5 KiB, 32.9 KiB, 37.5 KiB (stage 34.0: task 2325)) local blocks read: 200 remote blocks read: 200 data size total (min, med, max (stageId: taskId)) 16.7 MiB (0.0 B, 0.0 B, 10.7 MiB (stage 32.0: task 2249)) shuffle bytes written total (min, med, max (stageId: taskId)) 10.0 MiB (0.0 B, 0.0 B, 6.4 MiB (stage 32.0: task 2249))"]; subgraph cluster22 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n7.9 s (123 ms, 1.6 s, 2.6 s (stage 32.0: task 2249))"; 23 [labelType="html" label=" Project "]; 24 [labelType="html" label="Filter number of output rows: 156,095"]; } 25 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 519.0 MiB number of output rows: 500,001"]; 1->0; 3->1; 4->3; 6->4; 7->6; 9->7; 10->9; 11->10; 13->11; 14->13; 16->14; 17->16; 18->17; 20->11; 21->20; 23->21; 24->23; 25->24; }

Execute InsertIntoHadoopFsRelationCommand s3a://object-bucket-ec22981-9bc23785-4078-4ea5-b872-e6c98a05d411/04-12-2024_20:23:20, false, CSV, Map(header -> true, path -> s3a://object-bucket-ec22981-9bc23785-4078-4ea5-b872-e6c98a05d411/04-12-2024_20:23:20/), ErrorIfExists, [formatted_date, total_transaction_fee]

Coalesce 1

Sort [formatted_date#136 ASC NULLS FIRST], true, 0

WholeStageCodegen (7)

Exchange rangepartitioning(formatted_date#136 ASC NULLS FIRST, 200), true, [id=#517]

HashAggregate(keys=[formatted_date#136], functions=[sum(cast((cast(gas#78 as float) * cast(gas_price#79L as float)) as double))])

WholeStageCodegen (6)

Exchange hashpartitioning(formatted_date#136, 200), true, [id=#513]

HashAggregate(keys=[formatted_date#136], functions=[partial_sum(cast((cast(gas#78 as float) * cast(gas_price#79L as float)) as double))])

Project [gas#78, gas_price#79L, formatted_date#136]

SortMergeJoin [block_hash#72], [hash#17], Inner

WholeStageCodegen (5)

Sort [block_hash#72 ASC NULLS FIRST], false, 0

WholeStageCodegen (2)

Exchange hashpartitioning(block_hash#72, 200), true, [id=#495]

Project [block_hash#72, gas#78, gas_price#79L]

Filter ((isnotnull(transaction_index#74) AND (transaction_index#74 = 0)) AND isnotnull(block_hash#72))

WholeStageCodegen (1)

FileScan csv [block_hash#72,transaction_index#74,gas#78,gas_price#79L] Batched: false, DataFilters: [isnotnull(transaction_index#74), (transaction_index#74 = 0), isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(transaction_index), EqualTo(transaction_index,0), IsNotNull(block_hash)], ReadSchema: struct<block_hash:string,transaction_index:int,gas:int,gas_price:bigint>

Sort [hash#17 ASC NULLS FIRST], false, 0

WholeStageCodegen (4)

Exchange hashpartitioning(hash#17, 200), true, [id=#504]

Project [hash#17, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]

Filter ((date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM, Some(GMT)) = 2015-10) AND isnotnull(hash#17))

WholeStageCodegen (3)

FileScan csv [hash#17,timestamp#32] Batched: false, DataFilters: [(date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string,timestamp:int>

Details

== Parsed Logical Plan ==
InsertIntoHadoopFsRelationCommand s3a://object-bucket-ec22981-9bc23785-4078-4ea5-b872-e6c98a05d411/04-12-2024_20:23:20, false, CSV, Map(header -> true, path -> s3a://object-bucket-ec22981-9bc23785-4078-4ea5-b872-e6c98a05d411/04-12-2024_20:23:20/), ErrorIfExists, [formatted_date, total_transaction_fee]
+- Repartition 1, false
   +- Sort [formatted_date#136 ASC NULLS FIRST], true
      +- Aggregate [formatted_date#136], [formatted_date#136, sum(cast((cast(gas#78 as float) * cast(gas_price#79L as float)) as double)) AS total_transaction_fee#434]
         +- Filter (transaction_index#74 = 0)
            +- Filter (date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM, Some(GMT)) = 2015-10)
               +- Join Inner, (block_hash#72 = hash#17)
                  :- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv
                  +- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                     +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv

== Analyzed Logical Plan ==

InsertIntoHadoopFsRelationCommand s3a://object-bucket-ec22981-9bc23785-4078-4ea5-b872-e6c98a05d411/04-12-2024_20:23:20, false, CSV, Map(header -> true, path -> s3a://object-bucket-ec22981-9bc23785-4078-4ea5-b872-e6c98a05d411/04-12-2024_20:23:20/), ErrorIfExists, [formatted_date, total_transaction_fee]
+- Repartition 1, false
   +- Sort [formatted_date#136 ASC NULLS FIRST], true
      +- Aggregate [formatted_date#136], [formatted_date#136, sum(cast((cast(gas#78 as float) * cast(gas_price#79L as float)) as double)) AS total_transaction_fee#434]
         +- Filter (transaction_index#74 = 0)
            +- Filter (date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM, Some(GMT)) = 2015-10)
               +- Join Inner, (block_hash#72 = hash#17)
                  :- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv
                  +- Project [number#16, hash#17, parent_hash#18, nonce#19, sha3_uncles#20, logs_bloom#21, transactions_root#22, state_root#23, receipts_root#24, miner#25, difficulty#26L, total_difficulty#27, size#28, extra_data#29, gas_limit#30, gas_used#31, timestamp#32, transaction_count#33, base_fee_per_gas#34, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                     +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv

== Optimized Logical Plan ==
InsertIntoHadoopFsRelationCommand s3a://object-bucket-ec22981-9bc23785-4078-4ea5-b872-e6c98a05d411/04-12-2024_20:23:20, false, CSV, Map(header -> true, path -> s3a://object-bucket-ec22981-9bc23785-4078-4ea5-b872-e6c98a05d411/04-12-2024_20:23:20/), ErrorIfExists, [formatted_date, total_transaction_fee]
+- Repartition 1, false
   +- Sort [formatted_date#136 ASC NULLS FIRST], true
      +- Aggregate [formatted_date#136], [formatted_date#136, sum(cast((cast(gas#78 as float) * cast(gas_price#79L as float)) as double)) AS total_transaction_fee#434]
         +- Project [gas#78, gas_price#79L, formatted_date#136]
            +- Join Inner, (block_hash#72 = hash#17)
               :- Project [block_hash#72, gas#78, gas_price#79L]
               :  +- Filter ((isnotnull(transaction_index#74) AND (transaction_index#74 = 0)) AND isnotnull(block_hash#72))
               :     +- Relation[hash#70,nonce#71,block_hash#72,block_number#73,transaction_index#74,from_address#75,to_address#76,value#77,gas#78,gas_price#79L,input#80,block_timestamp#81,max_fee_per_gas#82,max_priority_fee_per_gas#83,transaction_type#84] csv
               +- Project [hash#17, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                  +- Filter ((date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM, Some(GMT)) = 2015-10) AND isnotnull(hash#17))
                     +- Relation[number#16,hash#17,parent_hash#18,nonce#19,sha3_uncles#20,logs_bloom#21,transactions_root#22,state_root#23,receipts_root#24,miner#25,difficulty#26L,total_difficulty#27,size#28,extra_data#29,gas_limit#30,gas_used#31,timestamp#32,transaction_count#33,base_fee_per_gas#34] csv

== Physical Plan ==
Execute InsertIntoHadoopFsRelationCommand s3a://object-bucket-ec22981-9bc23785-4078-4ea5-b872-e6c98a05d411/04-12-2024_20:23:20, false, CSV, Map(header -> true, path -> s3a://object-bucket-ec22981-9bc23785-4078-4ea5-b872-e6c98a05d411/04-12-2024_20:23:20/), ErrorIfExists, [formatted_date, total_transaction_fee]
+- Coalesce 1
   +- *(7) Sort [formatted_date#136 ASC NULLS FIRST], true, 0
      +- Exchange rangepartitioning(formatted_date#136 ASC NULLS FIRST, 200), true, [id=#517]
         +- *(6) HashAggregate(keys=[formatted_date#136], functions=[sum(cast((cast(gas#78 as float) * cast(gas_price#79L as float)) as double))], output=[formatted_date#136, total_transaction_fee#434])
            +- Exchange hashpartitioning(formatted_date#136, 200), true, [id=#513]
               +- *(5) HashAggregate(keys=[formatted_date#136], functions=[partial_sum(cast((cast(gas#78 as float) * cast(gas_price#79L as float)) as double))], output=[formatted_date#136, sum#444])
                  +- *(5) Project [gas#78, gas_price#79L, formatted_date#136]
                     +- *(5) SortMergeJoin [block_hash#72], [hash#17], Inner
                        :- *(2) Sort [block_hash#72 ASC NULLS FIRST], false, 0
                        :  +- Exchange hashpartitioning(block_hash#72, 200), true, [id=#495]
                        :     +- *(1) Project [block_hash#72, gas#78, gas_price#79L]
                        :        +- *(1) Filter ((isnotnull(transaction_index#74) AND (transaction_index#74 = 0)) AND isnotnull(block_hash#72))
                        :           +- FileScan csv [block_hash#72,transaction_index#74,gas#78,gas_price#79L] Batched: false, DataFilters: [isnotnull(transaction_index#74), (transaction_index#74 = 0), isnotnull(block_hash#72)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/transactions.csv], PartitionFilters: [], PushedFilters: [IsNotNull(transaction_index), EqualTo(transaction_index,0), IsNotNull(block_hash)], ReadSchema: struct<block_hash:string,transaction_index:int,gas:int,gas_price:bigint>
                        +- *(4) Sort [hash#17 ASC NULLS FIRST], false, 0
                           +- Exchange hashpartitioning(hash#17, 200), true, [id=#504]
                              +- *(3) Project [hash#17, date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM-dd, Some(GMT)) AS formatted_date#136]
                                 +- *(3) Filter ((date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as timestamp), yyyy-MM, Some(GMT)) = 2015-10) AND isnotnull(hash#17))
                                    +- FileScan csv [hash#17,timestamp#32] Batched: false, DataFilters: [(date_format(cast(from_unixtime(cast(timestamp#32 as bigint), yyyy-MM-dd HH:mm:ss, Some(GMT)) as..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/ethereum/blocks.csv], PartitionFilters: [], PushedFilters: [IsNotNull(hash)], ReadSchema: struct<hash:string,timestamp:int>