digraph G {
0 [labelType="html" label="<br><b>CollectLimit</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (9)";
2 [labelType="html" label="<br><b>Project</b><br><br>"];
3 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 11"];
}
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (6)";
5 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 16.1 MiB<br>spill size: 0.0 B"];
}
6 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 466,523<br>shuffle write time total (min, med, max (stageId: taskId))<br>1.9 s (0 ms, 8 ms, 73 ms (stage 13.0: task 191))<br>records read: 793<br>local bytes read: 9.8 KiB<br>fetch wait time: 0 ms<br>local blocks read: 58<br>data size total (min, med, max (stageId: taskId))<br>60.6 MiB (0.0 B, 16.3 KiB, 12.7 MiB (stage 13.0: task 292))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.1 MiB (0.0 B, 7.0 KiB, 169.2 KiB (stage 13.0: task 292))"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n3.7 s (1 ms, 12 ms, 116 ms (stage 13.0: task 278))";
8 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 466,523"];
}
subgraph cluster9 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n6.7 s (4 ms, 17 ms, 210 ms (stage 13.0: task 104))";
10 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>8 ms (0 ms, 0 ms, 4 ms (stage 13.0: task 207))<br>peak memory total (min, med, max (stageId: taskId))<br>2.2 GiB (64.0 KiB, 16.1 MiB, 20.0 MiB (stage 13.0: task 292))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 13.0: task 107))"];
}
11 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 466,523<br>shuffle write time total (min, med, max (stageId: taskId))<br>403 ms (13 ms, 24 ms, 36 ms (stage 12.0: task 86))<br>records read: 466,523<br>local bytes read total (min, med, max (stageId: taskId))<br>2.5 MiB (0.0 B, 2.2 KiB, 392.1 KiB (stage 13.0: task 292))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 13.0: task 99))<br>local blocks read: 2,085<br>data size total (min, med, max (stageId: taskId))<br>17.8 MiB (1120.0 KiB, 1143.6 KiB, 1160.0 KiB (stage 12.0: task 84))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.5 MiB (151.4 KiB, 159.2 KiB, 170.0 KiB (stage 12.0: task 96))"];
subgraph cluster12 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.5 s (60 ms, 202 ms, 243 ms (stage 12.0: task 88))";
13 [labelType="html" label="<br><b>Project</b><br><br>"];
14 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 466,523"];
}
subgraph cluster15 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n3.0 s (0 ms, 5 ms, 128 ms (stage 13.0: task 292))";
16 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 13.0: task 99))<br>peak memory total (min, med, max (stageId: taskId))<br>2.2 GiB (0.0 B, 16.1 MiB, 16.1 MiB (stage 13.0: task 99))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 13.0: task 99))"];
}
17 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 265<br>shuffle write time total (min, med, max (stageId: taskId))<br>206 ms (8 ms, 12 ms, 17 ms (stage 11.0: task 68))<br>records read: 261<br>local bytes read total (min, med, max (stageId: taskId))<br>34.9 KiB (0.0 B, 141.0 B, 681.0 B (stage 13.0: task 176))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 13.0: task 99))<br>local blocks read: 255<br>data size total (min, med, max (stageId: taskId))<br>26.3 KiB (1552.0 B, 1632.0 B, 2.5 KiB (stage 11.0: task 80))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>34.9 KiB (2042.0 B, 2.1 KiB, 3.4 KiB (stage 11.0: task 80))"];
subgraph cluster18 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n730 ms (26 ms, 51 ms, 55 ms (stage 11.0: task 78))";
19 [labelType="html" label="<br><b>Project</b><br><br>"];
20 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 265"];
}
subgraph cluster21 {
isCluster="true";
label="WholeStageCodegen (8)\n \nduration: 17 ms";
22 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 16.1 MiB<br>spill size: 0.0 B"];
}
23 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 265<br>shuffle write time total (min, med, max (stageId: taskId))<br>428 ms (23 ms, 27 ms, 29 ms (stage 10.0: task 53))<br>records read: 1<br>local bytes read: 141.0 B<br>fetch wait time: 0 ms<br>local blocks read: 1<br>data size total (min, med, max (stageId: taskId))<br>26.3 KiB (1552.0 B, 1632.0 B, 2.5 KiB (stage 10.0: task 64))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>34.9 KiB (2042.0 B, 2.1 KiB, 3.4 KiB (stage 10.0: task 64))"];
subgraph cluster24 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: total (min, med, max (stageId: taskId))\n2.3 s (89 ms, 178 ms, 207 ms (stage 10.0: task 62))";
25 [labelType="html" label="<br><b>Project</b><br><br>"];
26 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 265"];
}
2->0;
3->2;
5->3;
6->5;
8->6;
10->8;
11->10;
13->11;
14->13;
16->8;
17->16;
19->17;
20->19;
22->3;
23->22;
25->23;
26->25;
}
27
CollectLimit 11
Project [cast(src#205 as string) AS src#237, cast(edge#203 as string) AS edge#238, cast(dst#207 as string) AS dst#239]
SortMergeJoin [edge#203.dst], [dst#207.id], Inner
WholeStageCodegen (9)
Sort [edge#203.dst ASC NULLS FIRST], false, 0
WholeStageCodegen (6)
Exchange hashpartitioning(edge#203.dst, 200), true, [id=#177]
SortMergeJoin [edge#203.src], [src#205.id], Inner
WholeStageCodegen (5)
Sort [edge#203.src ASC NULLS FIRST], false, 0
WholeStageCodegen (2)
Exchange hashpartitioning(edge#203.src, 200), true, [id=#164]
Project [struct(src, src#169, dst, dst#170) AS edge#203]
Scan ExistingRDD[src#169,dst#170]
WholeStageCodegen (1)
Sort [src#205.id ASC NULLS FIRST], false, 0
WholeStageCodegen (4)
Exchange hashpartitioning(src#205.id, 200), true, [id=#170]
Project [struct(id, id#105, Borough, Borough#106, Zone, Zone#107, service_zone, service_zone#108) AS src#205]
Scan ExistingRDD[id#105,Borough#106,Zone#107,service_zone#108]
WholeStageCodegen (3)
Sort [dst#207.id ASC NULLS FIRST], false, 0
WholeStageCodegen (8)
Exchange hashpartitioning(dst#207.id, 200), true, [id=#183]
Project [struct(id, id#105, Borough, Borough#106, Zone, Zone#107, service_zone, service_zone#108) AS dst#207]
Scan ExistingRDD[id#105,Borough#106,Zone#107,service_zone#108]
WholeStageCodegen (7)
