graphframes - Details for Query 4

Details for Query 4

Submitted Time: 2024/12/20 23:05:29
Duration: 0.7 s
Succeeded Jobs: 5 6 7

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" CollectLimit "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (3)"; 2 [labelType="html" label=" Project "]; 3 [labelType="html" label="BroadcastHashJoin number of output rows: 11"]; 4 [labelType="html" label="BroadcastHashJoin number of output rows: 11"]; 5 [labelType="html" label=" Project "]; } 6 [labelType="html" label="Scan csv number of files read: 7 metadata time: 0 ms size of files read: 47.0 MiB number of output rows: 11"]; 7 [labelType="html" label="BroadcastExchange data size: 1026.1 KiB time to collect: 252 ms time to build: 46 ms time to broadcast: 6 ms"]; subgraph cluster8 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: 106 ms"; 9 [labelType="html" label=" Project "]; } 10 [labelType="html" label="Scan csv number of files read: 1 metadata time: 0 ms size of files read: 10.5 KiB number of output rows: 265"]; 11 [labelType="html" label="BroadcastExchange data size: 1026.1 KiB time to collect: 248 ms time to build: 47 ms time to broadcast: 6 ms"]; subgraph cluster12 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 99 ms"; 13 [labelType="html" label=" Project "]; } 14 [labelType="html" label="Scan csv number of files read: 1 metadata time: 1 ms size of files read: 10.5 KiB number of output rows: 265"]; 2->0; 3->2; 4->3; 5->4; 6->5; 7->4; 9->7; 10->9; 11->3; 13->11; 14->13; }

CollectLimit 11

Project [cast(src#123 as string) AS src#152, cast(edge#121 as string) AS edge#153, cast(dst#125 as string) AS dst#154]

BroadcastHashJoin [edge#121.dst], [dst#125.id], Inner, BuildRight

BroadcastHashJoin [edge#121.src], [src#123.id], Inner, BuildRight

Project [struct(src, PULocationID#18, dst, DOLocationID#19) AS edge#121]

WholeStageCodegen (3)

FileScan csv [PULocationID#18,DOLocationID#19] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi/green_tripdata/2023/green_tripdata_20..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<PULocationID:int,DOLocationID:int>

BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, struct<id:int,Borough:string,Zone:string,service_zone:string>, false].id as bigint))), [id=#106]

Project [struct(id, LocationID#72, Borough, Borough#73, Zone, Zone#74, service_zone, service_zone#75) AS src#123]

WholeStageCodegen (1)

FileScan csv [LocationID#72,Borough#73,Zone#74,service_zone#75] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi/taxi_zone_lookup.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<LocationID:int,Borough:string,Zone:string,service_zone:string>

BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, struct<id:int,Borough:string,Zone:string,service_zone:string>, false].id as bigint))), [id=#112]

Project [struct(id, LocationID#72, Borough, Borough#73, Zone, Zone#74, service_zone, service_zone#75) AS dst#125]

WholeStageCodegen (2)

Details

== Parsed Logical Plan ==
GlobalLimit 11
+- LocalLimit 11
   +- Project [cast(src#123 as string) AS src#152, cast(edge#121 as string) AS edge#153, cast(dst#125 as string) AS dst#154]
      +- Project [src#123, edge#121, dst#125]
         +- Join Inner, (edge#121.dst = dst#125.id)
            :- Join Inner, (edge#121.src = src#123.id)
            :  :- Project [struct(src, src#85, dst, dst#86) AS edge#121]
            :  :  +- Project [PULocationID#18 AS src#85, DOLocationID#19 AS dst#86]
            :  :     +- Relation[lpep_pickup_datetime#16,lpep_dropoff_datetime#17,PULocationID#18,DOLocationID#19,passenger_count#20,trip_distance#21,fare_amount#22,extra#23,mta_tax#24,tip_amount#25,tolls_amount#26,ehail_fee#27,total_amount#28,payment_type#29,trip_type#30,congestion_surcharge#31,taxi_type#32] csv
            :  +- Project [struct(id, id#80, Borough, Borough#73, Zone, Zone#74, service_zone, service_zone#75) AS src#123]
            :     +- Project [LocationID#72 AS id#80, Borough#73, Zone#74, service_zone#75]
            :        +- Relation[LocationID#72,Borough#73,Zone#74,service_zone#75] csv
            +- Project [struct(id, id#80, Borough, Borough#73, Zone, Zone#74, service_zone, service_zone#75) AS dst#125]
               +- Project [LocationID#72 AS id#80, Borough#73, Zone#74, service_zone#75]
                  +- Relation[LocationID#72,Borough#73,Zone#74,service_zone#75] csv

== Analyzed Logical Plan ==
src: string, edge: string, dst: string
GlobalLimit 11
+- LocalLimit 11
   +- Project [cast(src#123 as string) AS src#152, cast(edge#121 as string) AS edge#153, cast(dst#125 as string) AS dst#154]
      +- Project [src#123, edge#121, dst#125]
         +- Join Inner, (edge#121.dst = dst#125.id)
            :- Join Inner, (edge#121.src = src#123.id)
            :  :- Project [struct(src, src#85, dst, dst#86) AS edge#121]
            :  :  +- Project [PULocationID#18 AS src#85, DOLocationID#19 AS dst#86]
            :  :     +- Relation[lpep_pickup_datetime#16,lpep_dropoff_datetime#17,PULocationID#18,DOLocationID#19,passenger_count#20,trip_distance#21,fare_amount#22,extra#23,mta_tax#24,tip_amount#25,tolls_amount#26,ehail_fee#27,total_amount#28,payment_type#29,trip_type#30,congestion_surcharge#31,taxi_type#32] csv
            :  +- Project [struct(id, id#80, Borough, Borough#73, Zone, Zone#74, service_zone, service_zone#75) AS src#123]
            :     +- Project [LocationID#72 AS id#80, Borough#73, Zone#74, service_zone#75]
            :        +- Relation[LocationID#72,Borough#73,Zone#74,service_zone#75] csv
            +- Project [struct(id, id#80, Borough, Borough#73, Zone, Zone#74, service_zone, service_zone#75) AS dst#125]
               +- Project [LocationID#72 AS id#80, Borough#73, Zone#74, service_zone#75]
                  +- Relation[LocationID#72,Borough#73,Zone#74,service_zone#75] csv

== Optimized Logical Plan ==
GlobalLimit 11
+- LocalLimit 11
   +- Project [cast(src#123 as string) AS src#152, cast(edge#121 as string) AS edge#153, cast(dst#125 as string) AS dst#154]
      +- Join Inner, (edge#121.dst = dst#125.id)
         :- Join Inner, (edge#121.src = src#123.id)
         :  :- Project [struct(src, PULocationID#18, dst, DOLocationID#19) AS edge#121]
         :  :  +- Relation[lpep_pickup_datetime#16,lpep_dropoff_datetime#17,PULocationID#18,DOLocationID#19,passenger_count#20,trip_distance#21,fare_amount#22,extra#23,mta_tax#24,tip_amount#25,tolls_amount#26,ehail_fee#27,total_amount#28,payment_type#29,trip_type#30,congestion_surcharge#31,taxi_type#32] csv
         :  +- Project [struct(id, LocationID#72, Borough, Borough#73, Zone, Zone#74, service_zone, service_zone#75) AS src#123]
         :     +- Relation[LocationID#72,Borough#73,Zone#74,service_zone#75] csv
         +- Project [struct(id, LocationID#72, Borough, Borough#73, Zone, Zone#74, service_zone, service_zone#75) AS dst#125]
            +- Relation[LocationID#72,Borough#73,Zone#74,service_zone#75] csv

== Physical Plan ==
CollectLimit 11
+- *(3) Project [cast(src#123 as string) AS src#152, cast(edge#121 as string) AS edge#153, cast(dst#125 as string) AS dst#154]
   +- *(3) BroadcastHashJoin [edge#121.dst], [dst#125.id], Inner, BuildRight
      :- *(3) BroadcastHashJoin [edge#121.src], [src#123.id], Inner, BuildRight
      :  :- *(3) Project [struct(src, PULocationID#18, dst, DOLocationID#19) AS edge#121]
      :  :  +- FileScan csv [PULocationID#18,DOLocationID#19] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi/green_tripdata/2023/green_tripdata_20..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<PULocationID:int,DOLocationID:int>
      :  +- BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, struct<id:int,Borough:string,Zone:string,service_zone:string>, false].id as bigint))), [id=#106]
      :     +- *(1) Project [struct(id, LocationID#72, Borough, Borough#73, Zone, Zone#74, service_zone, service_zone#75) AS src#123]
      :        +- FileScan csv [LocationID#72,Borough#73,Zone#74,service_zone#75] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi/taxi_zone_lookup.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<LocationID:int,Borough:string,Zone:string,service_zone:string>
      +- BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, struct<id:int,Borough:string,Zone:string,service_zone:string>, false].id as bigint))), [id=#112]
         +- *(2) Project [struct(id, LocationID#72, Borough, Borough#73, Zone, Zone#74, service_zone, service_zone#75) AS dst#125]
            +- FileScan csv [LocationID#72,Borough#73,Zone#74,service_zone#75] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3a://data-repository-bkt/ECS765/nyc_taxi/taxi_zone_lookup.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<LocationID:int,Borough:string,Zone:string,service_zone:string>