== Parsed Logical Plan == GlobalLimit 11 +- LocalLimit 11 +- Project [cast(tpep_pickup_datetime#16 as string) AS tpep_pickup_datetime#490, cast(tpep_dropoff_datetime#17 as string) AS tpep_dropoff_datetime#491, cast(passenger_count#18 as string) AS passenger_count#492, cast(trip_distance#19 as string) AS trip_distance#493, cast(payment_type#22 as string) AS payment_type#494, cast(fare_amount#23 as string) AS fare_amount#495, cast(extra#24 as string) AS extra#496, cast(mta_tax#25 as string) AS mta_tax#497, cast(tip_amount#26 as string) AS tip_amount#498, cast(tolls_amount#27 as string) AS tolls_amount#499, cast(total_amount#28 as string) AS total_amount#500, cast(congestion_surcharge#29 as string) AS congestion_surcharge#501, cast(airport_fee#30 as string) AS airport_fee#502, cast(taxi_type#31 as string) AS taxi_type#503, cast(Pickup_Borough#152 as string) AS Pickup_Borough#504, cast(Pickup_Zone#173 as string) AS Pickup_Zone#505, cast(Pickup_service_zone#194 as string) AS Pickup_service_zone#506, cast(Dropoff_Borough#318 as string) AS Dropoff_Borough#507, cast(Dropoff_Zone#341 as string) AS Dropoff_Zone#508, cast(Dropoff_service_zone#364 as string) AS Dropoff_service_zone#509] +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, Dropoff_Borough#318, Dropoff_Zone#341, Dropoff_service_zone#364] +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, Dropoff_Borough#318, Dropoff_Zone#341, Dropoff_service_zone#364] +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, LocationID#64, Dropoff_Borough#318, Dropoff_Zone#341, service_zone#67 AS Dropoff_service_zone#364] +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, LocationID#64, Dropoff_Borough#318, Zone#66 AS Dropoff_Zone#341, service_zone#67] +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, LocationID#64, Borough#65 AS Dropoff_Borough#318, Zone#66, service_zone#67] +- Join Inner, (DOLocationID#21 = LocationID#64) :- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194] : +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, PULocationID#20, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194] : +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, PULocationID#20, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, LocationID#64, Pickup_Borough#152, Pickup_Zone#173, service_zone#67 AS Pickup_service_zone#194] : +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, PULocationID#20, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, LocationID#64, Pickup_Borough#152, Zone#66 AS Pickup_Zone#173, service_zone#67] : +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, PULocationID#20, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, LocationID#64, Borough#65 AS Pickup_Borough#152, Zone#66, service_zone#67] : +- Join Inner, (PULocationID#20 = LocationID#64) : :- Relation[tpep_pickup_datetime#16,tpep_dropoff_datetime#17,passenger_count#18,trip_distance#19,PULocationID#20,DOLocationID#21,payment_type#22,fare_amount#23,extra#24,mta_tax#25,tip_amount#26,tolls_amount#27,total_amount#28,congestion_surcharge#29,airport_fee#30,taxi_type#31] csv : +- Relation[LocationID#64,Borough#65,Zone#66,service_zone#67] csv +- Relation[LocationID#64,Borough#65,Zone#66,service_zone#67] csv == Analyzed Logical Plan == tpep_pickup_datetime: string, tpep_dropoff_datetime: string, passenger_count: string, trip_distance: string, payment_type: string, fare_amount: string, extra: string, mta_tax: string, tip_amount: string, tolls_amount: string, total_amount: string, congestion_surcharge: string, airport_fee: string, taxi_type: string, Pickup_Borough: string, Pickup_Zone: string, Pickup_service_zone: string, Dropoff_Borough: string, Dropoff_Zone: string, Dropoff_service_zone: string GlobalLimit 11 +- LocalLimit 11 +- Project [cast(tpep_pickup_datetime#16 as string) AS tpep_pickup_datetime#490, cast(tpep_dropoff_datetime#17 as string) AS tpep_dropoff_datetime#491, cast(passenger_count#18 as string) AS passenger_count#492, cast(trip_distance#19 as string) AS trip_distance#493, cast(payment_type#22 as string) AS payment_type#494, cast(fare_amount#23 as string) AS fare_amount#495, cast(extra#24 as string) AS extra#496, cast(mta_tax#25 as string) AS mta_tax#497, cast(tip_amount#26 as string) AS tip_amount#498, cast(tolls_amount#27 as string) AS tolls_amount#499, cast(total_amount#28 as string) AS total_amount#500, cast(congestion_surcharge#29 as string) AS congestion_surcharge#501, cast(airport_fee#30 as string) AS airport_fee#502, cast(taxi_type#31 as string) AS taxi_type#503, cast(Pickup_Borough#152 as string) AS Pickup_Borough#504, cast(Pickup_Zone#173 as string) AS Pickup_Zone#505, cast(Pickup_service_zone#194 as string) AS Pickup_service_zone#506, cast(Dropoff_Borough#318 as string) AS Dropoff_Borough#507, cast(Dropoff_Zone#341 as string) AS Dropoff_Zone#508, cast(Dropoff_service_zone#364 as string) AS Dropoff_service_zone#509] +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, Dropoff_Borough#318, Dropoff_Zone#341, Dropoff_service_zone#364] +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, Dropoff_Borough#318, Dropoff_Zone#341, Dropoff_service_zone#364] +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, LocationID#64, Dropoff_Borough#318, Dropoff_Zone#341, service_zone#67 AS Dropoff_service_zone#364] +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, LocationID#64, Dropoff_Borough#318, Zone#66 AS Dropoff_Zone#341, service_zone#67] +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, LocationID#64, Borough#65 AS Dropoff_Borough#318, Zone#66, service_zone#67] +- Join Inner, (DOLocationID#21 = LocationID#64) :- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194] : +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, PULocationID#20, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194] : +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, PULocationID#20, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, LocationID#64, Pickup_Borough#152, Pickup_Zone#173, service_zone#67 AS Pickup_service_zone#194] : +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, PULocationID#20, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, LocationID#64, Pickup_Borough#152, Zone#66 AS Pickup_Zone#173, service_zone#67] : +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, PULocationID#20, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, LocationID#64, Borough#65 AS Pickup_Borough#152, Zone#66, service_zone#67] : +- Join Inner, (PULocationID#20 = LocationID#64) : :- Relation[tpep_pickup_datetime#16,tpep_dropoff_datetime#17,passenger_count#18,trip_distance#19,PULocationID#20,DOLocationID#21,payment_type#22,fare_amount#23,extra#24,mta_tax#25,tip_amount#26,tolls_amount#27,total_amount#28,congestion_surcharge#29,airport_fee#30,taxi_type#31] csv : +- Relation[LocationID#64,Borough#65,Zone#66,service_zone#67] csv +- Relation[LocationID#64,Borough#65,Zone#66,service_zone#67] csv == Optimized Logical Plan == GlobalLimit 11 +- LocalLimit 11 +- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, Borough#65 AS Dropoff_Borough#507, Zone#66 AS Dropoff_Zone#508, service_zone#67 AS Dropoff_service_zone#509] +- Join Inner, (DOLocationID#21 = LocationID#64) :- Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Borough#65 AS Pickup_Borough#152, Zone#66 AS Pickup_Zone#173, service_zone#67 AS Pickup_service_zone#194] : +- Join Inner, (PULocationID#20 = LocationID#64) : :- Filter (isnotnull(PULocationID#20) AND isnotnull(DOLocationID#21)) : : +- Relation[tpep_pickup_datetime#16,tpep_dropoff_datetime#17,passenger_count#18,trip_distance#19,PULocationID#20,DOLocationID#21,payment_type#22,fare_amount#23,extra#24,mta_tax#25,tip_amount#26,tolls_amount#27,total_amount#28,congestion_surcharge#29,airport_fee#30,taxi_type#31] csv : +- Filter isnotnull(LocationID#64) : +- Relation[LocationID#64,Borough#65,Zone#66,service_zone#67] csv +- Filter isnotnull(LocationID#64) +- Relation[LocationID#64,Borough#65,Zone#66,service_zone#67] csv == Physical Plan == CollectLimit 11 +- *(3) Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Pickup_Borough#152, Pickup_Zone#173, Pickup_service_zone#194, Borough#65 AS Dropoff_Borough#507, Zone#66 AS Dropoff_Zone#508, service_zone#67 AS Dropoff_service_zone#509] +- *(3) BroadcastHashJoin [DOLocationID#21], [LocationID#64], Inner, BuildRight :- *(3) Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31, Borough#65 AS Pickup_Borough#152, Zone#66 AS Pickup_Zone#173, service_zone#67 AS Pickup_service_zone#194] : +- *(3) BroadcastHashJoin [PULocationID#20], [LocationID#64], Inner, BuildRight : :- *(3) Project [tpep_pickup_datetime#16, tpep_dropoff_datetime#17, passenger_count#18, trip_distance#19, PULocationID#20, DOLocationID#21, payment_type#22, fare_amount#23, extra#24, mta_tax#25, tip_amount#26, tolls_amount#27, total_amount#28, congestion_surcharge#29, airport_fee#30, taxi_type#31] : : +- *(3) Filter (isnotnull(PULocationID#20) AND isnotnull(DOLocationID#21)) : : +- FileScan csv [tpep_pickup_datetime#16,tpep_dropoff_datetime#17,passenger_count#18,trip_distance#19,PULocationID#20,DOLocationID#21,payment_type#22,fare_amount#23,extra#24,mta_tax#25,tip_amount#26,tolls_amount#27,total_amount#28,congestion_surcharge#29,airport_fee#30,taxi_type#31] Batched: false, DataFilters: [isnotnull(PULocationID#20), isnotnull(DOLocationID#21)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi/yellow_tripdata/2023], PartitionFilters: [], PushedFilters: [IsNotNull(PULocationID), IsNotNull(DOLocationID)], ReadSchema: struct<tpep_pickup_datetime:string,tpep_dropoff_datetime:string,passenger_count:string,trip_dista... : +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true])), [id=#98] : +- *(1) Project [LocationID#64, Borough#65, Zone#66, service_zone#67] : +- *(1) Filter isnotnull(LocationID#64) : +- FileScan csv [LocationID#64,Borough#65,Zone#66,service_zone#67] Batched: false, DataFilters: [isnotnull(LocationID#64)], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi/taxi_zone_lookup.csv], PartitionFilters: [], PushedFilters: [IsNotNull(LocationID)], ReadSchema: struct<LocationID:string,Borough:string,Zone:string,service_zone:string> +- ReusedExchange [LocationID#64, Borough#65, Zone#66, service_zone#67], BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true])), [id=#98]