Task3 - Details for Query 4

Details for Query 4

Submitted Time: 2025/04/10 23:50:48
Duration: 9 s
Succeeded Jobs: 30

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" TakeOrderedAndProject "]; subgraph cluster1 { isCluster="true"; label="WholeStageCodegen (7)\n \nduration: total (min, med, max (stageId: taskId))\n62 ms (0 ms, 0 ms, 30 ms (stage 1054.0: task 14817))"; 2 [labelType="html" label=" Project "]; 3 [labelType="html" label="SortMergeJoin number of output rows: 77"]; } subgraph cluster4 { isCluster="true"; label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n579 ms (0 ms, 1 ms, 138 ms (stage 1054.0: task 14818))"; 5 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 1054.0: task 14820)) peak memory total (min, med, max (stageId: taskId)) 3.7 GiB (64.0 KiB, 64.0 KiB, 64.1 MiB (stage 1054.0: task 14820)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 1054.0: task 14820))"]; } 6 [labelType="html" label="Exchange shuffle records written: 77 shuffle write time total (min, med, max (stageId: taskId)) 207 ms (0 ms, 0 ms, 5 ms (stage 1053.0: task 14635)) records read: 77 local bytes read total (min, med, max (stageId: taskId)) 7.6 KiB (0.0 B, 0.0 B, 236.0 B (stage 1054.0: task 14824)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 1054.0: task 14820)) remote bytes read total (min, med, max (stageId: taskId)) 1156.0 B (0.0 B, 0.0 B, 236.0 B (stage 1054.0: task 14865)) local blocks read: 67 remote blocks read: 10 data size total (min, med, max (stageId: taskId)) 6.5 KiB (0.0 B, 0.0 B, 256.0 B (stage 1053.0: task 14638)) shuffle bytes written total (min, med, max (stageId: taskId)) 8.7 KiB (0.0 B, 0.0 B, 346.0 B (stage 1053.0: task 14638))"]; subgraph cluster7 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n201 ms (0 ms, 0 ms, 22 ms (stage 1053.0: task 14618))"; 8 [labelType="html" label=" Project "]; } 9 [labelType="html" label="SortAggregate number of output rows: 77"]; subgraph cluster10 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n366 ms (0 ms, 0 ms, 28 ms (stage 1053.0: task 14618))"; 11 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 1053.0: task 14617)) peak memory total (min, med, max (stageId: taskId)) 3.9 GiB (64.0 KiB, 64.0 KiB, 64.1 MiB (stage 1053.0: task 14617)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 1053.0: task 14617))"]; } 12 [labelType="html" label="Exchange shuffle records written: 77 shuffle write time total (min, med, max (stageId: taskId)) 381 ms (0 ms, 0 ms, 7 ms (stage 1052.0: task 14465)) records read: 77 local bytes read total (min, med, max (stageId: taskId)) 7.5 KiB (0.0 B, 0.0 B, 207.0 B (stage 1053.0: task 14638)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 1053.0: task 14617)) local blocks read: 63 data size total (min, med, max (stageId: taskId)) 6.5 KiB (0.0 B, 0.0 B, 256.0 B (stage 1052.0: task 14447)) shuffle bytes written total (min, med, max (stageId: taskId)) 7.5 KiB (0.0 B, 0.0 B, 207.0 B (stage 1052.0: task 14447))"]; 13 [labelType="html" label="SortAggregate number of output rows: 77"]; subgraph cluster14 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n9.4 s (41 ms, 43 ms, 63 ms (stage 1052.0: task 14418))"; 15 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 1052.0: task 14408)) peak memory total (min, med, max (stageId: taskId)) 3.9 GiB (64.0 KiB, 64.0 KiB, 64.1 MiB (stage 1052.0: task 14408)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 1052.0: task 14408))"]; 16 [labelType="html" label="Scan ExistingRDD number of output rows: 77"]; } subgraph cluster17 { isCluster="true"; label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n110 ms (0 ms, 0 ms, 42 ms (stage 1054.0: task 14817))"; 18 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 1054.0: task 14820)) peak memory total (min, med, max (stageId: taskId)) 3.7 GiB (0.0 B, 0.0 B, 64.1 MiB (stage 1054.0: task 14820)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 1054.0: task 14820))"]; } 19 [labelType="html" label="Exchange shuffle records written: 77 shuffle write time total (min, med, max (stageId: taskId)) 263 ms (0 ms, 0 ms, 10 ms (stage 1051.0: task 14272)) records read: 77 local bytes read total (min, med, max (stageId: taskId)) 3.3 KiB (0.0 B, 0.0 B, 148.0 B (stage 1054.0: task 14818)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 1054.0: task 14820)) remote bytes read total (min, med, max (stageId: taskId)) 2.2 KiB (0.0 B, 0.0 B, 148.0 B (stage 1054.0: task 14843)) local blocks read: 46 remote blocks read: 31 data size total (min, med, max (stageId: taskId)) 3.0 KiB (0.0 B, 0.0 B, 40.0 B (stage 1051.0: task 14219)) shuffle bytes written total (min, med, max (stageId: taskId)) 5.6 KiB (0.0 B, 0.0 B, 74.0 B (stage 1051.0: task 14219))"]; subgraph cluster20 { isCluster="true"; label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n276 ms (0 ms, 0 ms, 17 ms (stage 1051.0: task 14218))"; 21 [labelType="html" label=" Project "]; 22 [labelType="html" label=" SerializeFromObject "]; } 23 [labelType="html" label="Scan number of output rows: 77"]; 2->0; 3->2; 5->3; 6->5; 8->6; 9->8; 11->9; 12->11; 13->12; 15->13; 16->15; 18->3; 19->18; 21->19; 22->21; 23->22; }

TakeOrderedAndProject(limit=6, orderBy=[pagerank#297 DESC NULLS LAST], output=[id#369,pagerank#370])

Project [attr#277.id AS id#293, graphx_attr#251.pagerank AS pagerank#297]

SortMergeJoin [new_id#279L], [new_id#246L], Inner

WholeStageCodegen (7)

Sort [new_id#279L ASC NULLS FIRST], false, 0

WholeStageCodegen (4)

Exchange hashpartitioning(new_id#279L, 200), true, [id=#230]

Project [cast(id#151 as bigint) AS new_id#279L, struct(id, id#151, Latitude, Latitude#152, Longitude, Longitude#153, Census Tract, Census Tract#154) AS attr#277]

WholeStageCodegen (3)

SortAggregate(key=[id#151], functions=[first(Latitude#152, false), first(Longitude#153, false), first(Census Tract#154, false)])

Sort [id#151 ASC NULLS FIRST], false, 0

WholeStageCodegen (2)

Exchange hashpartitioning(id#151, 200), true, [id=#222]

SortAggregate(key=[id#151], functions=[partial_first(Latitude#152, false), partial_first(Longitude#153, false), partial_first(Census Tract#154, false)])

Sort [id#151 ASC NULLS FIRST], false, 0

Scan ExistingRDD[id#151,Latitude#152,Longitude#153,Census Tract#154]

WholeStageCodegen (1)

Sort [new_id#246L ASC NULLS FIRST], false, 0

WholeStageCodegen (6)

Exchange hashpartitioning(new_id#246L, 200), true, [id=#239]

Project [struct(pagerank, _2#243._1) AS graphx_attr#251, _1#242L AS new_id#246L]

SerializeFromObject [knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1 AS _1#242L, if (isnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)) null else named_struct(_1, knownnotnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)._1) AS _2#243]

WholeStageCodegen (5)

Scan[obj#241]

Details

== Parsed Logical Plan ==
GlobalLimit 6
+- LocalLimit 6
   +- Project [cast(id#293 as string) AS id#369, cast(pagerank#297 as string) AS pagerank#370]
      +- Sort [pagerank#297 DESC NULLS LAST], true
         +- Project [id#293, pagerank#297]
            +- Project [attr#277.id AS id#293, attr#277.Latitude AS Latitude#294, attr#277.Longitude AS Longitude#295, attr#277.Census Tract AS Census Tract#296, graphx_attr#251.pagerank AS pagerank#297]
               +- Project [attr#277, graphx_attr#251]
                  +- Project [new_id#279L, attr#277, graphx_attr#251]
                     +- Join Inner, (new_id#279L = new_id#246L)
                        :- Project [new_id#279L, attr#277]
                        :  +- Project [cast(attr#277.id as bigint) AS new_id#279L, attr#277.id AS id#280, attr#277]
                        :     +- Project [struct(id, id#151, Latitude, Latitude#152, Longitude, Longitude#153, Census Tract, Census Tract#154) AS attr#277]
                        :        +- Deduplicate [id#151]
                        :           +- LogicalRDD [id#151, Latitude#152, Longitude#153, Census Tract#154], false
                        +- Project [struct(pagerank, graphx_attr#247._1) AS graphx_attr#251, new_id#246L]
                           +- Project [_1#242L AS new_id#246L, _2#243 AS graphx_attr#247]
                              +- SerializeFromObject [knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1 AS _1#242L, if (isnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)) null else named_struct(_1, knownnotnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)._1) AS _2#243]
                                 +- ExternalRDD [obj#241]

== Analyzed Logical Plan ==
id: string, pagerank: string
GlobalLimit 6
+- LocalLimit 6
   +- Project [cast(id#293 as string) AS id#369, cast(pagerank#297 as string) AS pagerank#370]
      +- Sort [pagerank#297 DESC NULLS LAST], true
         +- Project [id#293, pagerank#297]
            +- Project [attr#277.id AS id#293, attr#277.Latitude AS Latitude#294, attr#277.Longitude AS Longitude#295, attr#277.Census Tract AS Census Tract#296, graphx_attr#251.pagerank AS pagerank#297]
               +- Project [attr#277, graphx_attr#251]
                  +- Project [new_id#279L, attr#277, graphx_attr#251]
                     +- Join Inner, (new_id#279L = new_id#246L)
                        :- Project [new_id#279L, attr#277]
                        :  +- Project [cast(attr#277.id as bigint) AS new_id#279L, attr#277.id AS id#280, attr#277]
                        :     +- Project [struct(id, id#151, Latitude, Latitude#152, Longitude, Longitude#153, Census Tract, Census Tract#154) AS attr#277]
                        :        +- Deduplicate [id#151]
                        :           +- LogicalRDD [id#151, Latitude#152, Longitude#153, Census Tract#154], false
                        +- Project [struct(pagerank, graphx_attr#247._1) AS graphx_attr#251, new_id#246L]
                           +- Project [_1#242L AS new_id#246L, _2#243 AS graphx_attr#247]
                              +- SerializeFromObject [knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1 AS _1#242L, if (isnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)) null else named_struct(_1, knownnotnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)._1) AS _2#243]
                                 +- ExternalRDD [obj#241]

== Optimized Logical Plan ==
GlobalLimit 6
+- LocalLimit 6
   +- Project [cast(id#293 as string) AS id#369, cast(pagerank#297 as string) AS pagerank#370]
      +- Sort [pagerank#297 DESC NULLS LAST], true
         +- Project [attr#277.id AS id#293, graphx_attr#251.pagerank AS pagerank#297]
            +- Join Inner, (new_id#279L = new_id#246L)
               :- Project [cast(id#151 as bigint) AS new_id#279L, struct(id, id#151, Latitude, Latitude#152, Longitude, Longitude#153, Census Tract, Census Tract#154) AS attr#277]
               :  +- Aggregate [id#151], [id#151, first(Latitude#152, false) AS Latitude#152, first(Longitude#153, false) AS Longitude#153, first(Census Tract#154, false) AS Census Tract#154]
               :     +- LogicalRDD [id#151, Latitude#152, Longitude#153, Census Tract#154], false
               +- Project [struct(pagerank, _2#243._1) AS graphx_attr#251, _1#242L AS new_id#246L]
                  +- SerializeFromObject [knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1 AS _1#242L, if (isnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)) null else named_struct(_1, knownnotnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)._1) AS _2#243]
                     +- ExternalRDD [obj#241]

== Physical Plan ==
TakeOrderedAndProject(limit=6, orderBy=[pagerank#297 DESC NULLS LAST], output=[id#369,pagerank#370])
+- *(7) Project [attr#277.id AS id#293, graphx_attr#251.pagerank AS pagerank#297]
   +- *(7) SortMergeJoin [new_id#279L], [new_id#246L], Inner
      :- *(4) Sort [new_id#279L ASC NULLS FIRST], false, 0
      :  +- Exchange hashpartitioning(new_id#279L, 200), true, [id=#230]
      :     +- *(3) Project [cast(id#151 as bigint) AS new_id#279L, struct(id, id#151, Latitude, Latitude#152, Longitude, Longitude#153, Census Tract, Census Tract#154) AS attr#277]
      :        +- SortAggregate(key=[id#151], functions=[first(Latitude#152, false), first(Longitude#153, false), first(Census Tract#154, false)], output=[id#151, Latitude#152, Longitude#153, Census Tract#154])
      :           +- *(2) Sort [id#151 ASC NULLS FIRST], false, 0
      :              +- Exchange hashpartitioning(id#151, 200), true, [id=#222]
      :                 +- SortAggregate(key=[id#151], functions=[partial_first(Latitude#152, false), partial_first(Longitude#153, false), partial_first(Census Tract#154, false)], output=[id#151, first#382, valueSet#383, first#384, valueSet#385, first#386, valueSet#387])
      :                    +- *(1) Sort [id#151 ASC NULLS FIRST], false, 0
      :                       +- *(1) Scan ExistingRDD[id#151,Latitude#152,Longitude#153,Census Tract#154]
      +- *(6) Sort [new_id#246L ASC NULLS FIRST], false, 0
         +- Exchange hashpartitioning(new_id#246L, 200), true, [id=#239]
            +- *(5) Project [struct(pagerank, _2#243._1) AS graphx_attr#251, _1#242L AS new_id#246L]
               +- *(5) SerializeFromObject [knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1 AS _1#242L, if (isnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)) null else named_struct(_1, knownnotnull(knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2)._1) AS _2#243]
                  +- Scan[obj#241]