graphframes - Details for Query 12

Details for Query 12

Submitted Time: 2024/12/27 14:28:18
Duration: 0.5 s
Succeeded Jobs: 53

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" TakeOrderedAndProject "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n1.2 s (0 ms, 0 ms, 132 ms (stage 2149.0: task 3492))"; 2 [labelType="html" label=" Project "]; 3 [labelType="html" label="SortMergeJoin number of output rows: 265"]; } subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n2.3 s (0 ms, 2 ms, 152 ms (stage 2149.0: task 3485))"; 5 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 2149.0: task 3469)) peak memory total (min, med, max (stageId: taskId)) 2.3 GiB (64.0 KiB, 16.1 MiB, 16.1 MiB (stage 2149.0: task 3469)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 2149.0: task 3469))"]; } 6 [labelType="html" label="Exchange shuffle records written: 265 shuffle write time total (min, med, max (stageId: taskId)) 176 ms (7 ms, 11 ms, 13 ms (stage 2148.0: task 3457)) records read: 265 local bytes read total (min, med, max (stageId: taskId)) 26.8 KiB (0.0 B, 137.0 B, 418.0 B (stage 2149.0: task 3519)) fetch wait time total (min, med, max (stageId: taskId)) 280 ms (0 ms, 0 ms, 65 ms (stage 2149.0: task 3499)) remote bytes read total (min, med, max (stageId: taskId)) 8.3 KiB (0.0 B, 0.0 B, 351.0 B (stage 2149.0: task 3519)) local blocks read: 196 remote blocks read: 60 data size total (min, med, max (stageId: taskId)) 28.4 KiB (1680.0 B, 1760.0 B, 2.7 KiB (stage 2148.0: task 3467)) shuffle bytes written total (min, med, max (stageId: taskId)) 35.1 KiB (2.0 KiB, 2.1 KiB, 3.2 KiB (stage 2148.0: task 3467))"]; subgraph cluster7 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n831 ms (49 ms, 51 ms, 57 ms (stage 2148.0: task 3458))"; 8 [labelType="html" label=" Project "]; 9 [labelType="html" label="Filter number of output rows: 265"]; 10 [labelType="html" label="Scan ExistingRDD number of output rows: 265"]; } subgraph cluster11 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n885 ms (0 ms, 0 ms, 67 ms (stage 2149.0: task 3494))"; 12 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 2149.0: task 3469)) peak memory total (min, med, max (stageId: taskId)) 2.3 GiB (0.0 B, 16.1 MiB, 16.1 MiB (stage 2149.0: task 3469)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 2149.0: task 3469))"]; } 13 [labelType="html" label="Exchange shuffle records written: 265 shuffle write time total (min, med, max (stageId: taskId)) 156 ms (5 ms, 10 ms, 10 ms (stage 2147.0: task 3443)) records read: 265 local bytes read total (min, med, max (stageId: taskId)) 8.8 KiB (0.0 B, 0.0 B, 241.0 B (stage 2149.0: task 3519)) fetch wait time total (min, med, max (stageId: taskId)) 269 ms (0 ms, 0 ms, 66 ms (stage 2149.0: task 3494)) remote bytes read total (min, med, max (stageId: taskId)) 9.7 KiB (0.0 B, 74.0 B, 222.0 B (stage 2149.0: task 3473)) local blocks read: 119 remote blocks read: 133 data size total (min, med, max (stageId: taskId)) 10.4 KiB (640.0 B, 680.0 B, 680.0 B (stage 2147.0: task 3445)) shuffle bytes written total (min, med, max (stageId: taskId)) 18.4 KiB (1070.0 B, 1196.0 B, 1259.0 B (stage 2147.0: task 3441))"]; subgraph cluster14 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n216 ms (10 ms, 13 ms, 17 ms (stage 2147.0: task 3439))"; 15 [labelType="html" label=" Project "]; 16 [labelType="html" label=" SerializeFromObject "]; } 17 [labelType="html" label="Scan number of output rows: 265"]; 2->0; 3->2; 5->3; 6->5; 8->6; 9->8; 10->9; 12->3; 13->12; 15->13; 16->15; 17->16; }

TakeOrderedAndProject(limit=6, orderBy=[pagerank#461 DESC NULLS LAST], output=[id#513,pagerank#514])

Project [attr#323.id AS id#457, graphx_attr#424.pagerank AS pagerank#461]

SortMergeJoin [new_id#325L], [new_id#419L], Inner

WholeStageCodegen (5)

Sort [new_id#325L ASC NULLS FIRST], false, 0

WholeStageCodegen (2)

Exchange hashpartitioning(new_id#325L, 200), true, [id=#497]

Project [cast(id#105 as bigint) AS new_id#325L, struct(id, id#105, Borough, Borough#106, Zone, Zone#107, service_zone, service_zone#108) AS attr#323]

Filter isnotnull(cast(id#105 as bigint))

Scan ExistingRDD[id#105,Borough#106,Zone#107,service_zone#108]

WholeStageCodegen (1)

Sort [new_id#419L ASC NULLS FIRST], false, 0

WholeStageCodegen (4)

Exchange hashpartitioning(new_id#419L, 200), true, [id=#506]

Project [struct(pagerank, _2#416._1) AS graphx_attr#424, _1#415L AS new_id#419L]

SerializeFromObject [knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1 AS _1#415L, if (isnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)) null else named_struct(_1, knownnotnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)._1) AS _2#416]

WholeStageCodegen (3)

Scan[obj#414]

Details

== Parsed Logical Plan ==
GlobalLimit 6
+- LocalLimit 6
   +- Project [cast(id#457 as string) AS id#513, cast(pagerank#461 as string) AS pagerank#514]
      +- Sort [pagerank#461 DESC NULLS LAST], true
         +- Project [id#457, pagerank#461]
            +- Project [attr#323.id AS id#457, attr#323.Borough AS Borough#458, attr#323.Zone AS Zone#459, attr#323.service_zone AS service_zone#460, graphx_attr#424.pagerank AS pagerank#461]
               +- Project [attr#323, graphx_attr#424]
                  +- Project [new_id#325L, attr#323, graphx_attr#424]
                     +- Join Inner, (new_id#325L = new_id#419L)
                        :- Project [new_id#325L, attr#323]
                        :  +- Project [cast(attr#323.id as bigint) AS new_id#325L, attr#323.id AS id#326, attr#323]
                        :     +- Project [struct(id, id#105, Borough, Borough#106, Zone, Zone#107, service_zone, service_zone#108) AS attr#323]
                        :        +- LogicalRDD [id#105, Borough#106, Zone#107, service_zone#108], false
                        +- Project [struct(pagerank, graphx_attr#420._1) AS graphx_attr#424, new_id#419L]
                           +- Project [_1#415L AS new_id#419L, _2#416 AS graphx_attr#420]
                              +- SerializeFromObject [knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1 AS _1#415L, if (isnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)) null else named_struct(_1, knownnotnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)._1) AS _2#416]
                                 +- ExternalRDD [obj#414]

== Analyzed Logical Plan ==
id: string, pagerank: string
GlobalLimit 6
+- LocalLimit 6
   +- Project [cast(id#457 as string) AS id#513, cast(pagerank#461 as string) AS pagerank#514]
      +- Sort [pagerank#461 DESC NULLS LAST], true
         +- Project [id#457, pagerank#461]
            +- Project [attr#323.id AS id#457, attr#323.Borough AS Borough#458, attr#323.Zone AS Zone#459, attr#323.service_zone AS service_zone#460, graphx_attr#424.pagerank AS pagerank#461]
               +- Project [attr#323, graphx_attr#424]
                  +- Project [new_id#325L, attr#323, graphx_attr#424]
                     +- Join Inner, (new_id#325L = new_id#419L)
                        :- Project [new_id#325L, attr#323]
                        :  +- Project [cast(attr#323.id as bigint) AS new_id#325L, attr#323.id AS id#326, attr#323]
                        :     +- Project [struct(id, id#105, Borough, Borough#106, Zone, Zone#107, service_zone, service_zone#108) AS attr#323]
                        :        +- LogicalRDD [id#105, Borough#106, Zone#107, service_zone#108], false
                        +- Project [struct(pagerank, graphx_attr#420._1) AS graphx_attr#424, new_id#419L]
                           +- Project [_1#415L AS new_id#419L, _2#416 AS graphx_attr#420]
                              +- SerializeFromObject [knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1 AS _1#415L, if (isnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)) null else named_struct(_1, knownnotnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)._1) AS _2#416]
                                 +- ExternalRDD [obj#414]

== Optimized Logical Plan ==
GlobalLimit 6
+- LocalLimit 6
   +- Project [cast(id#457 as string) AS id#513, cast(pagerank#461 as string) AS pagerank#514]
      +- Sort [pagerank#461 DESC NULLS LAST], true
         +- Project [attr#323.id AS id#457, graphx_attr#424.pagerank AS pagerank#461]
            +- Join Inner, (new_id#325L = new_id#419L)
               :- Project [cast(id#105 as bigint) AS new_id#325L, struct(id, id#105, Borough, Borough#106, Zone, Zone#107, service_zone, service_zone#108) AS attr#323]
               :  +- Filter isnotnull(cast(id#105 as bigint))
               :     +- LogicalRDD [id#105, Borough#106, Zone#107, service_zone#108], false
               +- Project [struct(pagerank, _2#416._1) AS graphx_attr#424, _1#415L AS new_id#419L]
                  +- SerializeFromObject [knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1 AS _1#415L, if (isnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)) null else named_struct(_1, knownnotnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)._1) AS _2#416]
                     +- ExternalRDD [obj#414]

== Physical Plan ==
TakeOrderedAndProject(limit=6, orderBy=[pagerank#461 DESC NULLS LAST], output=[id#513,pagerank#514])
+- *(5) Project [attr#323.id AS id#457, graphx_attr#424.pagerank AS pagerank#461]
   +- *(5) SortMergeJoin [new_id#325L], [new_id#419L], Inner
      :- *(2) Sort [new_id#325L ASC NULLS FIRST], false, 0
      :  +- Exchange hashpartitioning(new_id#325L, 200), true, [id=#497]
      :     +- *(1) Project [cast(id#105 as bigint) AS new_id#325L, struct(id, id#105, Borough, Borough#106, Zone, Zone#107, service_zone, service_zone#108) AS attr#323]
      :        +- *(1) Filter isnotnull(cast(id#105 as bigint))
      :           +- *(1) Scan ExistingRDD[id#105,Borough#106,Zone#107,service_zone#108]
      +- *(4) Sort [new_id#419L ASC NULLS FIRST], false, 0
         +- Exchange hashpartitioning(new_id#419L, 200), true, [id=#506]
            +- *(3) Project [struct(pagerank, _2#416._1) AS graphx_attr#424, _1#415L AS new_id#419L]
               +- *(3) SerializeFromObject [knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1 AS _1#415L, if (isnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)) null else named_struct(_1, knownnotnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)._1) AS _2#416]
                  +- Scan[obj#414]