digraph G {
subgraph cluster0 {
isCluster="true";
label="WholeStageCodegen (10)\n \nduration: 3 ms";
1 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 3 ms<br>number of output rows: 1"];
}
2 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 200<br>shuffle write time total (min, med, max (stageId: taskId))<br>39 ms (0 ms, 0 ms, 0 ms (stage 19.0: task 549))<br>records read: 200<br>local bytes read: 7.5 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 3.9 KiB<br>local blocks read: 132<br>remote blocks read: 68<br>data size total (min, med, max (stageId: taskId))<br>3.1 KiB (16.0 B, 16.0 B, 16.0 B (stage 19.0: task 560))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>11.4 KiB (56.0 B, 59.0 B, 59.0 B (stage 19.0: task 560))"];
subgraph cluster3 {
isCluster="true";
label="WholeStageCodegen (9)\n \nduration: total (min, med, max (stageId: taskId))\n2.6 s (0 ms, 6 ms, 84 ms (stage 19.0: task 557))";
4 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>2.6 s (0 ms, 5 ms, 84 ms (stage 19.0: task 557))<br>number of output rows: 200"];
5 [labelType="html" label="<br><b>Project</b><br><br>"];
6 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 285,732"];
}
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n1.0 s (0 ms, 0 ms, 160 ms (stage 19.0: task 551))";
8 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>751 ms (0 ms, 0 ms, 52 ms (stage 19.0: task 547))<br>peak memory total (min, med, max (stageId: taskId))<br>3.1 GiB (16.1 MiB, 16.1 MiB, 17.0 MiB (stage 19.0: task 604))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 19.0: task 560))"];
}
9 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 466,523<br>shuffle write time total (min, med, max (stageId: taskId))<br>1.8 s (0 ms, 9 ms, 43 ms (stage 17.0: task 419))<br>records read: 466,523<br>local bytes read total (min, med, max (stageId: taskId))<br>1183.4 KiB (288.0 B, 5.1 KiB, 26.5 KiB (stage 19.0: task 604))<br>fetch wait time total (min, med, max (stageId: taskId))<br>637 ms (0 ms, 0 ms, 46 ms (stage 19.0: task 640))<br>remote bytes read total (min, med, max (stageId: taskId))<br>987.7 KiB (604.0 B, 4.0 KiB, 31.3 KiB (stage 19.0: task 638))<br>local blocks read: 5,628<br>remote blocks read: 4,729<br>data size total (min, med, max (stageId: taskId))<br>60.6 MiB (0.0 B, 16.3 KiB, 12.7 MiB (stage 17.0: task 486))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.1 MiB (0.0 B, 7.1 KiB, 135.7 KiB (stage 17.0: task 486))"];
subgraph cluster10 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n4.9 s (0 ms, 12 ms, 304 ms (stage 17.0: task 431))";
11 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 466,523"];
}
subgraph cluster12 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n5.6 s (2 ms, 16 ms, 307 ms (stage 17.0: task 431))";
13 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>28 ms (0 ms, 0 ms, 10 ms (stage 17.0: task 431))<br>peak memory total (min, med, max (stageId: taskId))<br>2.2 GiB (64.0 KiB, 16.1 MiB, 20.0 MiB (stage 17.0: task 486))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 17.0: task 346))"];
}
14 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 466,523<br>shuffle write time total (min, med, max (stageId: taskId))<br>432 ms (12 ms, 27 ms, 33 ms (stage 16.0: task 325))<br>records read: 466,523<br>local bytes read total (min, med, max (stageId: taskId))<br>1274.1 KiB (0.0 B, 1046.0 B, 201.5 KiB (stage 17.0: task 486))<br>fetch wait time total (min, med, max (stageId: taskId))<br>521 ms (0 ms, 0 ms, 66 ms (stage 17.0: task 488))<br>remote bytes read total (min, med, max (stageId: taskId))<br>1271.4 KiB (0.0 B, 1208.0 B, 190.6 KiB (stage 17.0: task 486))<br>local blocks read: 1,048<br>remote blocks read: 1,037<br>data size total (min, med, max (stageId: taskId))<br>17.8 MiB (1120.0 KiB, 1143.6 KiB, 1160.0 KiB (stage 16.0: task 315))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.5 MiB (151.4 KiB, 159.2 KiB, 170.0 KiB (stage 16.0: task 329))"];
subgraph cluster15 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.2 s (53 ms, 181 ms, 225 ms (stage 16.0: task 324))";
16 [labelType="html" label="<br><b>Project</b><br><br>"];
17 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 466,523"];
}
subgraph cluster18 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n3.7 s (0 ms, 7 ms, 293 ms (stage 17.0: task 431))";
19 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 17.0: task 346))<br>peak memory total (min, med, max (stageId: taskId))<br>2.2 GiB (0.0 B, 16.1 MiB, 16.1 MiB (stage 17.0: task 346))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 17.0: task 346))"];
}
20 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 265<br>shuffle write time total (min, med, max (stageId: taskId))<br>220 ms (6 ms, 16 ms, 18 ms (stage 15.0: task 311))<br>records read: 261<br>local bytes read total (min, med, max (stageId: taskId))<br>17.0 KiB (0.0 B, 0.0 B, 417.0 B (stage 17.0: task 435))<br>fetch wait time total (min, med, max (stageId: taskId))<br>84 ms (0 ms, 0 ms, 60 ms (stage 17.0: task 480))<br>remote bytes read total (min, med, max (stageId: taskId))<br>17.9 KiB (0.0 B, 122.0 B, 424.0 B (stage 17.0: task 431))<br>local blocks read: 124<br>remote blocks read: 131<br>data size total (min, med, max (stageId: taskId))<br>26.3 KiB (1552.0 B, 1632.0 B, 2.5 KiB (stage 15.0: task 313))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>34.9 KiB (2042.0 B, 2.1 KiB, 3.4 KiB (stage 15.0: task 313))"];
subgraph cluster21 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n739 ms (41 ms, 46 ms, 52 ms (stage 15.0: task 306))";
22 [labelType="html" label="<br><b>Project</b><br><br>"];
23 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 265"];
}
subgraph cluster24 {
isCluster="true";
label="WholeStageCodegen (8)\n \nduration: total (min, med, max (stageId: taskId))\n5.4 s (0 ms, 10 ms, 230 ms (stage 19.0: task 557))";
25 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 19.0: task 560))<br>peak memory total (min, med, max (stageId: taskId))<br>2.5 GiB (64.0 KiB, 16.1 MiB, 16.1 MiB (stage 19.0: task 560))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 19.0: task 560))"];
}
26 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 265<br>shuffle write time total (min, med, max (stageId: taskId))<br>143 ms (5 ms, 9 ms, 10 ms (stage 18.0: task 333))<br>records read: 265<br>local bytes read total (min, med, max (stageId: taskId))<br>22.8 KiB (0.0 B, 126.0 B, 550.0 B (stage 19.0: task 584))<br>fetch wait time total (min, med, max (stageId: taskId))<br>127 ms (0 ms, 0 ms, 9 ms (stage 19.0: task 632))<br>remote bytes read total (min, med, max (stageId: taskId))<br>12.4 KiB (0.0 B, 0.0 B, 433.0 B (stage 19.0: task 597))<br>local blocks read: 168<br>remote blocks read: 91<br>data size total (min, med, max (stageId: taskId))<br>26.3 KiB (1552.0 B, 1632.0 B, 2.5 KiB (stage 18.0: task 345))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>35.2 KiB (2.1 KiB, 2.1 KiB, 3.3 KiB (stage 18.0: task 345))"];
subgraph cluster27 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: total (min, med, max (stageId: taskId))\n784 ms (46 ms, 50 ms, 52 ms (stage 18.0: task 345))";
28 [labelType="html" label="<br><b>Project</b><br><br>"];
29 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 265"];
}
2->1;
4->2;
5->4;
6->5;
8->6;
9->8;
11->9;
13->11;
14->13;
16->14;
17->16;
19->11;
20->19;
22->20;
23->22;
25->6;
26->25;
28->26;
29->28;
}
30
HashAggregate(keys=[], functions=[count(1)])
WholeStageCodegen (10)
Exchange SinglePartition, true, [id=#281]
HashAggregate(keys=[], functions=[partial_count(1)])
Project
SortMergeJoin [src#205.Borough, src#205.service_zone, edge#203.dst], [dst#207.Borough, dst#207.service_zone, dst#207.id], Inner
WholeStageCodegen (9)
Sort [src#205.Borough ASC NULLS FIRST, src#205.service_zone ASC NULLS FIRST, edge#203.dst ASC NULLS FIRST], false, 0
WholeStageCodegen (6)
Exchange hashpartitioning(src#205.Borough, src#205.service_zone, edge#203.dst, 200), true, [id=#266]
SortMergeJoin [edge#203.src], [src#205.id], Inner
WholeStageCodegen (5)
Sort [edge#203.src ASC NULLS FIRST], false, 0
WholeStageCodegen (2)
Exchange hashpartitioning(edge#203.src, 200), true, [id=#253]
Project [struct(src, src#169, dst, dst#170) AS edge#203]
Scan ExistingRDD[src#169,dst#170]
WholeStageCodegen (1)
Sort [src#205.id ASC NULLS FIRST], false, 0
WholeStageCodegen (4)
Exchange hashpartitioning(src#205.id, 200), true, [id=#259]
Project [struct(id, id#105, Borough, Borough#106, Zone, Zone#107, service_zone, service_zone#108) AS src#205]
Scan ExistingRDD[id#105,Borough#106,Zone#107,service_zone#108]
WholeStageCodegen (3)
Sort [dst#207.Borough ASC NULLS FIRST, dst#207.service_zone ASC NULLS FIRST, dst#207.id ASC NULLS FIRST], false, 0
WholeStageCodegen (8)
Exchange hashpartitioning(dst#207.Borough, dst#207.service_zone, dst#207.id, 200), true, [id=#272]
Project [struct(id, id#105, Borough, Borough#106, Zone, Zone#107, service_zone, service_zone#108) AS dst#207]
Scan ExistingRDD[id#105,Borough#106,Zone#107,service_zone#108]
WholeStageCodegen (7)
