digraph G {
0 [labelType="html" label="<br><b>TakeOrderedAndProject</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n440 ms (1 ms, 2 ms, 22 ms (stage 8.0: task 65))";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>2 ms (0 ms, 0 ms, 1 ms (stage 8.0: task 65))<br>peak memory total (min, med, max (stageId: taskId))<br>498.0 MiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 8.0: task 65))<br>number of output rows: 7<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 8.0: task 65))"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 14<br>shuffle write time total (min, med, max (stageId: taskId))<br>19 ms (0 ms, 0 ms, 13 ms (stage 7.0: task 47))<br>records read: 14<br>local bytes read total (min, med, max (stageId: taskId))<br>1127.0 B (0.0 B, 0.0 B, 161.0 B (stage 8.0: task 65))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 8.0: task 65))<br>local blocks read: 14<br>data size total (min, med, max (stageId: taskId))<br>560.0 B (0.0 B, 0.0 B, 280.0 B (stage 7.0: task 47))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>1127.0 B (0.0 B, 0.0 B, 567.0 B (stage 7.0: task 47))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n1.2 m (1.1 s, 3.8 s, 4.3 s (stage 7.0: task 46))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>1.1 m (1.1 s, 3.8 s, 4.2 s (stage 7.0: task 46))<br>peak memory total (min, med, max (stageId: taskId))<br>5.0 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 7.0: task 45))<br>number of output rows: 14"];
6 [labelType="html" label="<br><b>Project</b><br><br>"];
7 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 2,202"];
}
8 [labelType="html" label="<b>Scan csv </b><br><br>number of files read: 7<br>metadata time: 0 ms<br>size of files read: 2.2 GiB<br>number of output rows: 22,400,728"];
2->0;
3->2;
5->3;
6->5;
7->6;
8->7;
}
9
TakeOrderedAndProject(limit=21, orderBy=[trip_date#114 ASC NULLS FIRST], output=[trip_date#114,trip_count#156])
HashAggregate(keys=[trip_date#114], functions=[count(1)])
WholeStageCodegen (2)
Exchange hashpartitioning(trip_date#114, 200), true, [id=#94]
HashAggregate(keys=[trip_date#114], functions=[partial_count(1)])
Project [date_format(cast(tpep_pickup_datetime#61 as timestamp), yyyy-MM-dd, Some(GMT)) AS trip_date#114]
Filter (((((isnotnull(trip_distance#64) AND isnotnull(fare_amount#68)) AND (cast(fare_amount#68 as int) > 50)) AND (cast(trip_distance#64 as int) < 1)) AND (date_format(cast(tpep_pickup_datetime#61 as timestamp), yyyy-MM-dd, Some(GMT)) >= 2023-02-01)) AND (date_format(cast(tpep_pickup_datetime#61 as timestamp), yyyy-MM-dd, Some(GMT)) <= 2023-02-07))
WholeStageCodegen (1)
FileScan csv [tpep_pickup_datetime#61,trip_distance#64,fare_amount#68] Batched: false, DataFilters: [isnotnull(trip_distance#64), isnotnull(fare_amount#68), (cast(fare_amount#68 as int) > 50), (cas..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi/yellow_tripdata/2023/yellow_tripdata_..., PartitionFilters: [], PushedFilters: [IsNotNull(trip_distance), IsNotNull(fare_amount)], ReadSchema: struct<tpep_pickup_datetime:string,trip_distance:string,fare_amount:string>
== Parsed Logical Plan ==
GlobalLimit 21
+- LocalLimit 21
+- Project [cast(trip_date#114 as string) AS trip_date#155, cast(trip_count#150L as string) AS trip_count#156]
+- Sort [trip_date#114 ASC NULLS FIRST], true
+- Aggregate [trip_date#114], [trip_date#114, count(1) AS trip_count#150L]
+- Filter (((cast(fare_amount#68 as int) > 50) AND (cast(trip_distance#64 as int) < 1)) AND ((trip_date#114 >= 2023-02-01) AND (trip_date#114 <= 2023-02-07)))
+- Project [tpep_pickup_datetime#61, tpep_dropoff_datetime#62, passenger_count#63, trip_distance#64, PULocationID#65, DOLocationID#66, payment_type#67, fare_amount#68, extra#69, mta_tax#70, tip_amount#71, tolls_amount#72, total_amount#73, congestion_surcharge#74, airport_fee#75, taxi_type#76, date_format(cast(tpep_pickup_datetime#61 as timestamp), yyyy-MM-dd, Some(GMT)) AS trip_date#114]
+- Relation[tpep_pickup_datetime#61,tpep_dropoff_datetime#62,passenger_count#63,trip_distance#64,PULocationID#65,DOLocationID#66,payment_type#67,fare_amount#68,extra#69,mta_tax#70,tip_amount#71,tolls_amount#72,total_amount#73,congestion_surcharge#74,airport_fee#75,taxi_type#76] csv
== Analyzed Logical Plan ==
trip_date: string, trip_count: string
GlobalLimit 21
+- LocalLimit 21
+- Project [cast(trip_date#114 as string) AS trip_date#155, cast(trip_count#150L as string) AS trip_count#156]
+- Sort [trip_date#114 ASC NULLS FIRST], true
+- Aggregate [trip_date#114], [trip_date#114, count(1) AS trip_count#150L]
+- Filter (((cast(fare_amount#68 as int) > 50) AND (cast(trip_distance#64 as int) < 1)) AND ((trip_date#114 >= 2023-02-01) AND (trip_date#114 <= 2023-02-07)))
+- Project [tpep_pickup_datetime#61, tpep_dropoff_datetime#62, passenger_count#63, trip_distance#64, PULocationID#65, DOLocationID#66, payment_type#67, fare_amount#68, extra#69, mta_tax#70, tip_amount#71, tolls_amount#72, total_amount#73, congestion_surcharge#74, airport_fee#75, taxi_type#76, date_format(cast(tpep_pickup_datetime#61 as timestamp), yyyy-MM-dd, Some(GMT)) AS trip_date#114]
+- Relation[tpep_pickup_datetime#61,tpep_dropoff_datetime#62,passenger_count#63,trip_distance#64,PULocationID#65,DOLocationID#66,payment_type#67,fare_amount#68,extra#69,mta_tax#70,tip_amount#71,tolls_amount#72,total_amount#73,congestion_surcharge#74,airport_fee#75,taxi_type#76] csv
== Optimized Logical Plan ==
GlobalLimit 21
+- LocalLimit 21
+- Project [trip_date#114, cast(trip_count#150L as string) AS trip_count#156]
+- Sort [trip_date#114 ASC NULLS FIRST], true
+- Aggregate [trip_date#114], [trip_date#114, count(1) AS trip_count#150L]
+- Project [date_format(cast(tpep_pickup_datetime#61 as timestamp), yyyy-MM-dd, Some(GMT)) AS trip_date#114]
+- Filter (((((isnotnull(trip_distance#64) AND isnotnull(fare_amount#68)) AND (cast(fare_amount#68 as int) > 50)) AND (cast(trip_distance#64 as int) < 1)) AND (date_format(cast(tpep_pickup_datetime#61 as timestamp), yyyy-MM-dd, Some(GMT)) >= 2023-02-01)) AND (date_format(cast(tpep_pickup_datetime#61 as timestamp), yyyy-MM-dd, Some(GMT)) <= 2023-02-07))
+- Relation[tpep_pickup_datetime#61,tpep_dropoff_datetime#62,passenger_count#63,trip_distance#64,PULocationID#65,DOLocationID#66,payment_type#67,fare_amount#68,extra#69,mta_tax#70,tip_amount#71,tolls_amount#72,total_amount#73,congestion_surcharge#74,airport_fee#75,taxi_type#76] csv
== Physical Plan ==
TakeOrderedAndProject(limit=21, orderBy=[trip_date#114 ASC NULLS FIRST], output=[trip_date#114,trip_count#156])
+- *(2) HashAggregate(keys=[trip_date#114], functions=[count(1)], output=[trip_date#114, trip_count#150L])
+- Exchange hashpartitioning(trip_date#114, 200), true, [id=#94]
+- *(1) HashAggregate(keys=[trip_date#114], functions=[partial_count(1)], output=[trip_date#114, count#160L])
+- *(1) Project [date_format(cast(tpep_pickup_datetime#61 as timestamp), yyyy-MM-dd, Some(GMT)) AS trip_date#114]
+- *(1) Filter (((((isnotnull(trip_distance#64) AND isnotnull(fare_amount#68)) AND (cast(fare_amount#68 as int) > 50)) AND (cast(trip_distance#64 as int) < 1)) AND (date_format(cast(tpep_pickup_datetime#61 as timestamp), yyyy-MM-dd, Some(GMT)) >= 2023-02-01)) AND (date_format(cast(tpep_pickup_datetime#61 as timestamp), yyyy-MM-dd, Some(GMT)) <= 2023-02-07))
+- FileScan csv [tpep_pickup_datetime#61,trip_distance#64,fare_amount#68] Batched: false, DataFilters: [isnotnull(trip_distance#64), isnotnull(fare_amount#68), (cast(fare_amount#68 as int) > 50), (cas..., Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi/yellow_tripdata/2023/yellow_tripdata_..., PartitionFilters: [], PushedFilters: [IsNotNull(trip_distance), IsNotNull(fare_amount)], ReadSchema: struct<tpep_pickup_datetime:string,trip_distance:string,fare_amount:string>
