task1 - Details for Query 4

Details for Query 4

Submitted Time: 2024/12/08 01:26:08
Duration: 0.6 s
Succeeded Jobs: 4 5

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" CollectLimit "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (3)"; 2 [labelType="html" label=" Project "]; 3 [labelType="html" label="BroadcastHashJoin number of output rows: 11"]; 4 [labelType="html" label=" Project "]; 5 [labelType="html" label="BroadcastHashJoin number of output rows: 11"]; 6 [labelType="html" label=" Project "]; 7 [labelType="html" label="Filter number of output rows: 11"]; } 8 [labelType="html" label="Scan csv number of files read: 7 metadata time: 0 ms size of files read: 2.2 GiB number of output rows: 11"]; 9 [labelType="html" label="BroadcastExchange data size: 1040.0 KiB time to collect: 196 ms time to build: 34 ms time to broadcast: 6 ms"]; subgraph cluster10 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: 76 ms"; 11 [labelType="html" label=" Project "]; 12 [labelType="html" label="Filter number of output rows: 265"]; } 13 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 10.5 KiB number of output rows: 265"]; 2->0; 3->2; 4->3; 5->4; 6->5; 7->6; 8->7; 9->5; 11->9; 12->11; 13->12; 9->3; }

CollectLimit 11

Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208, Borough#17 AS Dropoff_Borough#440, Zone#18 AS Dropoff_Zone#441, service_zone#19 AS Dropoff_service_zone#442, concat_ws( to , Pickup_Borough#206, Borough#17) AS route#443, cast(month(cast(tpep_pickup_datetime#40 as date)) as string) AS Month#444]

BroadcastHashJoin [DOLocationID#45], [LocationID#16], Inner, BuildRight

Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, DOLocationID#45, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Borough#17 AS Pickup_Borough#206, Zone#18 AS Pickup_Zone#207, service_zone#19 AS Pickup_service_zone#208]

BroadcastHashJoin [PULocationID#44], [LocationID#16], Inner, BuildRight

Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, PULocationID#44, DOLocationID#45, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55]

Filter (isnotnull(PULocationID#44) AND isnotnull(DOLocationID#45))

WholeStageCodegen (3)

FileScan csv [tpep_pickup_datetime#40,tpep_dropoff_datetime#41,passenger_count#42,trip_distance#43,PULocationID#44,DOLocationID#45,payment_type#46,fare_amount#47,extra#48,mta_tax#49,tip_amount#50,tolls_amount#51,total_amount#52,congestion_surcharge#53,airport_fee#54,taxi_type#55] Batched: false, DataFilters: [isnotnull(PULocationID#44), isnotnull(DOLocationID#45)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi/yellow_tripdata/2023], PartitionFilters: [], PushedFilters: [IsNotNull(PULocationID), IsNotNull(DOLocationID)], ReadSchema: struct<tpep_pickup_datetime:string,tpep_dropoff_datetime:string,passenger_count:string,trip_dista...

BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true])), [id=#144]

Project [LocationID#16, Borough#17, Zone#18, service_zone#19]

Filter isnotnull(LocationID#16)

WholeStageCodegen (1)

FileScan csv [LocationID#16,Borough#17,Zone#18,service_zone#19] Batched: false, DataFilters: [isnotnull(LocationID#16)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi], PartitionFilters: [], PushedFilters: [IsNotNull(LocationID)], ReadSchema: struct<LocationID:string,Borough:string,Zone:string,service_zone:string>

Details

== Parsed Logical Plan ==
GlobalLimit 11
+- LocalLimit 11
   +- Project [cast(tpep_pickup_datetime#40 as string) AS tpep_pickup_datetime#423, cast(tpep_dropoff_datetime#41 as string) AS tpep_dropoff_datetime#424, cast(passenger_count#42 as string) AS passenger_count#425, cast(trip_distance#43 as string) AS trip_distance#426, cast(payment_type#46 as string) AS payment_type#427, cast(fare_amount#47 as string) AS fare_amount#428, cast(extra#48 as string) AS extra#429, cast(mta_tax#49 as string) AS mta_tax#430, cast(tip_amount#50 as string) AS tip_amount#431, cast(tolls_amount#51 as string) AS tolls_amount#432, cast(total_amount#52 as string) AS total_amount#433, cast(congestion_surcharge#53 as string) AS congestion_surcharge#434, cast(airport_fee#54 as string) AS airport_fee#435, cast(taxi_type#55 as string) AS taxi_type#436, cast(Pickup_Borough#206 as string) AS Pickup_Borough#437, cast(Pickup_Zone#207 as string) AS Pickup_Zone#438, cast(Pickup_service_zone#208 as string) AS Pickup_service_zone#439, cast(Dropoff_Borough#312 as string) AS Dropoff_Borough#440, cast(Dropoff_Zone#313 as string) AS Dropoff_Zone#441, cast(Dropoff_service_zone#314 as string) AS Dropoff_service_zone#442, cast(route#356 as string) AS route#443, cast(Month#378 as string) AS Month#444]
      +- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208, Dropoff_Borough#312, Dropoff_Zone#313, Dropoff_service_zone#314, route#356, month(cast(tpep_pickup_datetime#40 as date)) AS Month#378]
         +- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208, Dropoff_Borough#312, Dropoff_Zone#313, Dropoff_service_zone#314, concat_ws( to , Pickup_Borough#206, Dropoff_Borough#312) AS route#356]
            +- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208, Dropoff_Borough#312, Dropoff_Zone#313, Dropoff_service_zone#314]
               +- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, DOLocationID#45, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208, Borough#17 AS Dropoff_Borough#312, Zone#18 AS Dropoff_Zone#313, service_zone#19 AS Dropoff_service_zone#314]
                  +- Join Inner, (DOLocationID#45 = LocationID#16)
                     :- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, DOLocationID#45, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208]
                     :  +- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, PULocationID#44, DOLocationID#45, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Borough#17 AS Pickup_Borough#206, Zone#18 AS Pickup_Zone#207, service_zone#19 AS Pickup_service_zone#208]
                     :     +- Join Inner, (PULocationID#44 = LocationID#16)
                     :        :- Relation[tpep_pickup_datetime#40,tpep_dropoff_datetime#41,passenger_count#42,trip_distance#43,PULocationID#44,DOLocationID#45,payment_type#46,fare_amount#47,extra#48,mta_tax#49,tip_amount#50,tolls_amount#51,total_amount#52,congestion_surcharge#53,airport_fee#54,taxi_type#55] csv
                     :        +- Relation[LocationID#16,Borough#17,Zone#18,service_zone#19] csv
                     +- Relation[LocationID#16,Borough#17,Zone#18,service_zone#19] csv

== Analyzed Logical Plan ==
tpep_pickup_datetime: string, tpep_dropoff_datetime: string, passenger_count: string, trip_distance: string, payment_type: string, fare_amount: string, extra: string, mta_tax: string, tip_amount: string, tolls_amount: string, total_amount: string, congestion_surcharge: string, airport_fee: string, taxi_type: string, Pickup_Borough: string, Pickup_Zone: string, Pickup_service_zone: string, Dropoff_Borough: string, Dropoff_Zone: string, Dropoff_service_zone: string, route: string, Month: string
GlobalLimit 11
+- LocalLimit 11
   +- Project [cast(tpep_pickup_datetime#40 as string) AS tpep_pickup_datetime#423, cast(tpep_dropoff_datetime#41 as string) AS tpep_dropoff_datetime#424, cast(passenger_count#42 as string) AS passenger_count#425, cast(trip_distance#43 as string) AS trip_distance#426, cast(payment_type#46 as string) AS payment_type#427, cast(fare_amount#47 as string) AS fare_amount#428, cast(extra#48 as string) AS extra#429, cast(mta_tax#49 as string) AS mta_tax#430, cast(tip_amount#50 as string) AS tip_amount#431, cast(tolls_amount#51 as string) AS tolls_amount#432, cast(total_amount#52 as string) AS total_amount#433, cast(congestion_surcharge#53 as string) AS congestion_surcharge#434, cast(airport_fee#54 as string) AS airport_fee#435, cast(taxi_type#55 as string) AS taxi_type#436, cast(Pickup_Borough#206 as string) AS Pickup_Borough#437, cast(Pickup_Zone#207 as string) AS Pickup_Zone#438, cast(Pickup_service_zone#208 as string) AS Pickup_service_zone#439, cast(Dropoff_Borough#312 as string) AS Dropoff_Borough#440, cast(Dropoff_Zone#313 as string) AS Dropoff_Zone#441, cast(Dropoff_service_zone#314 as string) AS Dropoff_service_zone#442, cast(route#356 as string) AS route#443, cast(Month#378 as string) AS Month#444]
      +- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208, Dropoff_Borough#312, Dropoff_Zone#313, Dropoff_service_zone#314, route#356, month(cast(tpep_pickup_datetime#40 as date)) AS Month#378]
         +- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208, Dropoff_Borough#312, Dropoff_Zone#313, Dropoff_service_zone#314, concat_ws( to , Pickup_Borough#206, Dropoff_Borough#312) AS route#356]
            +- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208, Dropoff_Borough#312, Dropoff_Zone#313, Dropoff_service_zone#314]
               +- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, DOLocationID#45, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208, Borough#17 AS Dropoff_Borough#312, Zone#18 AS Dropoff_Zone#313, service_zone#19 AS Dropoff_service_zone#314]
                  +- Join Inner, (DOLocationID#45 = LocationID#16)
                     :- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, DOLocationID#45, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208]
                     :  +- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, PULocationID#44, DOLocationID#45, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Borough#17 AS Pickup_Borough#206, Zone#18 AS Pickup_Zone#207, service_zone#19 AS Pickup_service_zone#208]
                     :     +- Join Inner, (PULocationID#44 = LocationID#16)
                     :        :- Relation[tpep_pickup_datetime#40,tpep_dropoff_datetime#41,passenger_count#42,trip_distance#43,PULocationID#44,DOLocationID#45,payment_type#46,fare_amount#47,extra#48,mta_tax#49,tip_amount#50,tolls_amount#51,total_amount#52,congestion_surcharge#53,airport_fee#54,taxi_type#55] csv
                     :        +- Relation[LocationID#16,Borough#17,Zone#18,service_zone#19] csv
                     +- Relation[LocationID#16,Borough#17,Zone#18,service_zone#19] csv

== Optimized Logical Plan ==
GlobalLimit 11
+- LocalLimit 11
   +- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208, Borough#17 AS Dropoff_Borough#440, Zone#18 AS Dropoff_Zone#441, service_zone#19 AS Dropoff_service_zone#442, concat_ws( to , Pickup_Borough#206, Borough#17) AS route#443, cast(month(cast(tpep_pickup_datetime#40 as date)) as string) AS Month#444]
      +- Join Inner, (DOLocationID#45 = LocationID#16)
         :- Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, DOLocationID#45, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Borough#17 AS Pickup_Borough#206, Zone#18 AS Pickup_Zone#207, service_zone#19 AS Pickup_service_zone#208]
         :  +- Join Inner, (PULocationID#44 = LocationID#16)
         :     :- Filter (isnotnull(PULocationID#44) AND isnotnull(DOLocationID#45))
         :     :  +- Relation[tpep_pickup_datetime#40,tpep_dropoff_datetime#41,passenger_count#42,trip_distance#43,PULocationID#44,DOLocationID#45,payment_type#46,fare_amount#47,extra#48,mta_tax#49,tip_amount#50,tolls_amount#51,total_amount#52,congestion_surcharge#53,airport_fee#54,taxi_type#55] csv
         :     +- Filter isnotnull(LocationID#16)
         :        +- Relation[LocationID#16,Borough#17,Zone#18,service_zone#19] csv
         +- Filter isnotnull(LocationID#16)
            +- Relation[LocationID#16,Borough#17,Zone#18,service_zone#19] csv

== Physical Plan ==
CollectLimit 11
+- *(3) Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Pickup_Borough#206, Pickup_Zone#207, Pickup_service_zone#208, Borough#17 AS Dropoff_Borough#440, Zone#18 AS Dropoff_Zone#441, service_zone#19 AS Dropoff_service_zone#442, concat_ws( to , Pickup_Borough#206, Borough#17) AS route#443, cast(month(cast(tpep_pickup_datetime#40 as date)) as string) AS Month#444]
   +- *(3) BroadcastHashJoin [DOLocationID#45], [LocationID#16], Inner, BuildRight
      :- *(3) Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, DOLocationID#45, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55, Borough#17 AS Pickup_Borough#206, Zone#18 AS Pickup_Zone#207, service_zone#19 AS Pickup_service_zone#208]
      :  +- *(3) BroadcastHashJoin [PULocationID#44], [LocationID#16], Inner, BuildRight
      :     :- *(3) Project [tpep_pickup_datetime#40, tpep_dropoff_datetime#41, passenger_count#42, trip_distance#43, PULocationID#44, DOLocationID#45, payment_type#46, fare_amount#47, extra#48, mta_tax#49, tip_amount#50, tolls_amount#51, total_amount#52, congestion_surcharge#53, airport_fee#54, taxi_type#55]
      :     :  +- *(3) Filter (isnotnull(PULocationID#44) AND isnotnull(DOLocationID#45))
      :     :     +- FileScan csv [tpep_pickup_datetime#40,tpep_dropoff_datetime#41,passenger_count#42,trip_distance#43,PULocationID#44,DOLocationID#45,payment_type#46,fare_amount#47,extra#48,mta_tax#49,tip_amount#50,tolls_amount#51,total_amount#52,congestion_surcharge#53,airport_fee#54,taxi_type#55] Batched: false, DataFilters: [isnotnull(PULocationID#44), isnotnull(DOLocationID#45)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi/yellow_tripdata/2023], PartitionFilters: [], PushedFilters: [IsNotNull(PULocationID), IsNotNull(DOLocationID)], ReadSchema: struct<tpep_pickup_datetime:string,tpep_dropoff_datetime:string,passenger_count:string,trip_dista...
      :     +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true])), [id=#144]
      :        +- *(1) Project [LocationID#16, Borough#17, Zone#18, service_zone#19]
      :           +- *(1) Filter isnotnull(LocationID#16)
      :              +- FileScan csv [LocationID#16,Borough#17,Zone#18,service_zone#19] Batched: false, DataFilters: [isnotnull(LocationID#16)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi], PartitionFilters: [], PushedFilters: [IsNotNull(LocationID)], ReadSchema: struct<LocationID:string,Borough:string,Zone:string,service_zone:string>
      +- ReusedExchange [LocationID#16, Borough#17, Zone#18, service_zone#19], BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true])), [id=#144]