digraph G {
0 [labelType="html" label="<br><b>CollectLimit</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (9)";
2 [labelType="html" label="<br><b>Project</b><br><br>"];
3 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 11"];
}
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (6)";
5 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 16.1 MiB<br>spill size: 0.0 B"];
}
6 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 466,523<br>shuffle write time total (min, med, max (stageId: taskId))<br>1.7 s (0 ms, 7 ms, 40 ms (stage 24.0: task 849))<br>records read: 1,280<br>local bytes read: 4.3 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 5.4 KiB<br>local blocks read: 25<br>remote blocks read: 26<br>data size total (min, med, max (stageId: taskId))<br>60.6 MiB (0.0 B, 16.3 KiB, 12.7 MiB (stage 24.0: task 935))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.1 MiB (0.0 B, 7.1 KiB, 135.9 KiB (stage 24.0: task 935))"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n2.1 s (0 ms, 8 ms, 70 ms (stage 24.0: task 850))";
8 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 466,523"];
}
subgraph cluster9 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n2.6 s (2 ms, 10 ms, 72 ms (stage 24.0: task 850))";
10 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 24.0: task 796))<br>peak memory total (min, med, max (stageId: taskId))<br>2.2 GiB (64.0 KiB, 16.1 MiB, 20.0 MiB (stage 24.0: task 935))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 24.0: task 796))"];
}
11 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 466,523<br>shuffle write time total (min, med, max (stageId: taskId))<br>354 ms (13 ms, 21 ms, 32 ms (stage 22.0: task 778))<br>records read: 466,523<br>local bytes read total (min, med, max (stageId: taskId))<br>1274.7 KiB (0.0 B, 1131.0 B, 188.2 KiB (stage 24.0: task 935))<br>fetch wait time total (min, med, max (stageId: taskId))<br>3 ms (0 ms, 0 ms, 2 ms (stage 24.0: task 808))<br>remote bytes read total (min, med, max (stageId: taskId))<br>1270.8 KiB (0.0 B, 1079.0 B, 203.9 KiB (stage 24.0: task 935))<br>local blocks read: 1,054<br>remote blocks read: 1,031<br>data size total (min, med, max (stageId: taskId))<br>17.8 MiB (1120.0 KiB, 1143.6 KiB, 1160.0 KiB (stage 22.0: task 764))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.5 MiB (151.4 KiB, 159.2 KiB, 170.0 KiB (stage 22.0: task 778))"];
subgraph cluster12 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n973 ms (39 ms, 61 ms, 75 ms (stage 22.0: task 778))";
13 [labelType="html" label="<br><b>Project</b><br><br>"];
14 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 466,523"];
}
subgraph cluster15 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n1.3 s (0 ms, 3 ms, 70 ms (stage 24.0: task 850))";
16 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 24.0: task 796))<br>peak memory total (min, med, max (stageId: taskId))<br>2.2 GiB (0.0 B, 16.1 MiB, 16.1 MiB (stage 24.0: task 796))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 24.0: task 796))"];
}
17 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 265<br>shuffle write time total (min, med, max (stageId: taskId))<br>140 ms (4 ms, 8 ms, 11 ms (stage 23.0: task 794))<br>records read: 261<br>local bytes read total (min, med, max (stageId: taskId))<br>20.1 KiB (0.0 B, 130.0 B, 531.0 B (stage 24.0: task 819))<br>fetch wait time total (min, med, max (stageId: taskId))<br>15 ms (0 ms, 0 ms, 4 ms (stage 24.0: task 801))<br>remote bytes read total (min, med, max (stageId: taskId))<br>14.8 KiB (0.0 B, 0.0 B, 345.0 B (stage 24.0: task 896))<br>local blocks read: 146<br>remote blocks read: 109<br>data size total (min, med, max (stageId: taskId))<br>26.3 KiB (1552.0 B, 1632.0 B, 2.5 KiB (stage 23.0: task 794))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>34.9 KiB (2042.0 B, 2.1 KiB, 3.4 KiB (stage 23.0: task 794))"];
subgraph cluster18 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n779 ms (45 ms, 49 ms, 50 ms (stage 23.0: task 783))";
19 [labelType="html" label="<br><b>Project</b><br><br>"];
20 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 265"];
}
subgraph cluster21 {
isCluster="true";
label="WholeStageCodegen (8)";
22 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 16.1 MiB<br>spill size: 0.0 B"];
}
23 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 265<br>shuffle write time total (min, med, max (stageId: taskId))<br>166 ms (7 ms, 10 ms, 12 ms (stage 21.0: task 752))<br>records read: 2<br>local bytes read: 290.0 B<br>fetch wait time: 0 ms<br>local blocks read: 2<br>data size total (min, med, max (stageId: taskId))<br>26.3 KiB (1552.0 B, 1632.0 B, 2.5 KiB (stage 21.0: task 762))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>35.2 KiB (2.1 KiB, 2.1 KiB, 3.3 KiB (stage 21.0: task 762))"];
subgraph cluster24 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: total (min, med, max (stageId: taskId))\n797 ms (44 ms, 50 ms, 55 ms (stage 21.0: task 747))";
25 [labelType="html" label="<br><b>Project</b><br><br>"];
26 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 265"];
}
2->0;
3->2;
5->3;
6->5;
8->6;
10->8;
11->10;
13->11;
14->13;
16->8;
17->16;
19->17;
20->19;
22->3;
23->22;
25->23;
26->25;
}
27
CollectLimit 11
Project [cast(src#205.id as string) AS src_id#271, cast(dst#207.id as string) AS dst_id#272, src#205.Borough AS Borough#273, src#205.service_zone AS service_zone#274]
SortMergeJoin [src#205.Borough, src#205.service_zone, edge#203.dst], [dst#207.Borough, dst#207.service_zone, dst#207.id], Inner
WholeStageCodegen (9)
Sort [src#205.Borough ASC NULLS FIRST, src#205.service_zone ASC NULLS FIRST, edge#203.dst ASC NULLS FIRST], false, 0
WholeStageCodegen (6)
Exchange hashpartitioning(src#205.Borough, src#205.service_zone, edge#203.dst, 200), true, [id=#352]
SortMergeJoin [edge#203.src], [src#205.id], Inner
WholeStageCodegen (5)
Sort [edge#203.src ASC NULLS FIRST], false, 0
WholeStageCodegen (2)
Exchange hashpartitioning(edge#203.src, 200), true, [id=#339]
Project [struct(src, src#169, dst, dst#170) AS edge#203]
Scan ExistingRDD[src#169,dst#170]
WholeStageCodegen (1)
Sort [src#205.id ASC NULLS FIRST], false, 0
WholeStageCodegen (4)
Exchange hashpartitioning(src#205.id, 200), true, [id=#345]
Project [struct(id, id#105, Borough, Borough#106, Zone, Zone#107, service_zone, service_zone#108) AS src#205]
Scan ExistingRDD[id#105,Borough#106,Zone#107,service_zone#108]
WholeStageCodegen (3)
Sort [dst#207.Borough ASC NULLS FIRST, dst#207.service_zone ASC NULLS FIRST, dst#207.id ASC NULLS FIRST], false, 0
WholeStageCodegen (8)
Exchange hashpartitioning(dst#207.Borough, dst#207.service_zone, dst#207.id, 200), true, [id=#358]
Project [struct(id, id#105, Borough, Borough#106, Zone, Zone#107, service_zone, service_zone#108) AS dst#207]
Scan ExistingRDD[id#105,Borough#106,Zone#107,service_zone#108]
WholeStageCodegen (7)
