digraph G {
0 [labelType="html" label="<br><b>CollectLimit</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (9)\n \nduration: 2 ms";
2 [labelType="html" label="<br><b>Project</b><br><br>"];
3 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 44"];
}
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (6)";
5 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>10 ms (0 ms, 0 ms, 9 ms (stage 28.0: task 831))<br>peak memory total (min, med, max (stageId: taskId))<br>320.6 MiB (64.1 MiB, 64.1 MiB, 64.3 MiB (stage 28.0: task 831))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 28.0: task 830))"];
}
6 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 466,523<br>shuffle write time total (min, med, max (stageId: taskId))<br>794 ms (0 ms, 4 ms, 11 ms (stage 21.0: task 714))<br>records read: 12,908<br>local bytes read total (min, med, max (stageId: taskId))<br>20.5 KiB (2.1 KiB, 4.4 KiB, 5.2 KiB (stage 28.0: task 832))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 28.0: task 830))<br>remote bytes read total (min, med, max (stageId: taskId))<br>23.9 KiB (1629.0 B, 3.8 KiB, 11.7 KiB (stage 28.0: task 831))<br>local blocks read: 151<br>remote blocks read: 137<br>data size total (min, med, max (stageId: taskId))<br>30.5 MiB (0.0 B, 8.5 KiB, 6.4 MiB (stage 21.0: task 769))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>1401.5 KiB (0.0 B, 4.7 KiB, 71.1 KiB (stage 21.0: task 769))"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n1.4 s (0 ms, 5 ms, 50 ms (stage 21.0: task 769))";
8 [labelType="html" label="<br><b>Project</b><br><br>"];
9 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 466,523"];
}
subgraph cluster10 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n1.7 s (1 ms, 7 ms, 51 ms (stage 21.0: task 769))";
11 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 21.0: task 629))<br>peak memory total (min, med, max (stageId: taskId))<br>8.9 GiB (64.0 KiB, 64.1 MiB, 68.0 MiB (stage 21.0: task 769))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 21.0: task 629))"];
}
12 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 466,523<br>shuffle write time total (min, med, max (stageId: taskId))<br>30 ms (8 ms, 10 ms, 11 ms (stage 20.0: task 625))<br>records read: 466,523<br>local bytes read total (min, med, max (stageId: taskId))<br>1137.4 KiB (0.0 B, 628.0 B, 225.1 KiB (stage 21.0: task 769))<br>fetch wait time total (min, med, max (stageId: taskId))<br>41 ms (0 ms, 0 ms, 41 ms (stage 21.0: task 645))<br>remote bytes read total (min, med, max (stageId: taskId))<br>1063.1 KiB (0.0 B, 488.0 B, 159.1 KiB (stage 21.0: task 769))<br>local blocks read: 215<br>remote blocks read: 199<br>data size total (min, med, max (stageId: taskId))<br>10.7 MiB (1437.7 KiB, 4.5 MiB, 4.8 MiB (stage 20.0: task 625))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.1 MiB (304.2 KiB, 920.3 KiB, 976.0 KiB (stage 20.0: task 625))"];
subgraph cluster13 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.4 s (333 ms, 1.0 s, 1.1 s (stage 20.0: task 625))";
14 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 466,523"];
15 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 466,523"];
}
subgraph cluster16 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n800 ms (0 ms, 1 ms, 50 ms (stage 21.0: task 769))";
17 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 21.0: task 629))<br>peak memory total (min, med, max (stageId: taskId))<br>8.9 GiB (0.0 B, 64.1 MiB, 64.1 MiB (stage 21.0: task 629))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 21.0: task 629))"];
}
18 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 265<br>shuffle write time: 8 ms<br>records read: 261<br>local bytes read total (min, med, max (stageId: taskId))<br>8.0 KiB (0.0 B, 0.0 B, 176.0 B (stage 21.0: task 678))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 21.0: task 629))<br>remote bytes read total (min, med, max (stageId: taskId))<br>9.5 KiB (0.0 B, 0.0 B, 199.0 B (stage 21.0: task 653))<br>local blocks read: 67<br>remote blocks read: 78<br>data size: 15.1 KiB<br>shuffle bytes written: 17.4 KiB"];
subgraph cluster19 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 61 ms";
20 [labelType="html" label="<br><b>Project</b><br><br>"];
21 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 265"];
22 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 265"];
}
subgraph cluster23 {
isCluster="true";
label="WholeStageCodegen (8)\n \nduration: total (min, med, max (stageId: taskId))\n17 ms (0 ms, 0 ms, 15 ms (stage 23.0: task 829))";
24 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 28.0: task 830))<br>peak memory total (min, med, max (stageId: taskId))<br>256.3 MiB (64.0 KiB, 64.1 MiB, 64.1 MiB (stage 28.0: task 830))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 28.0: task 830))"];
}
25 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 265<br>shuffle write time: 8 ms<br>records read: 9<br>local bytes read total (min, med, max (stageId: taskId))<br>148.0 B (0.0 B, 0.0 B, 148.0 B (stage 28.0: task 831))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 28.0: task 830))<br>remote bytes read total (min, med, max (stageId: taskId))<br>350.0 B (0.0 B, 121.0 B, 128.0 B (stage 28.0: task 833))<br>local blocks read: 1<br>remote blocks read: 3<br>data size: 15.1 KiB<br>shuffle bytes written: 17.1 KiB"];
subgraph cluster26 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: 57 ms";
27 [labelType="html" label="<br><b>Project</b><br><br>"];
28 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 265"];
29 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 265"];
}
2->0;
3->2;
5->3;
6->5;
8->6;
9->8;
11->9;
12->11;
14->12;
15->14;
17->9;
18->17;
20->18;
21->20;
22->21;
24->3;
25->24;
27->25;
28->27;
29->28;
}
30
CollectLimit 11
Project [cast(src#119 as string) AS id#293, cast(dst#120 as string) AS id#294, src_Borough#195 AS Borough#295, src_service_zone#200 AS service_zone#296]
SortMergeJoin [src_service_zone#200, dst#120, src_Borough#195], [dst_service_zone#215, dst_id#205, dst_Borough#210], Inner
WholeStageCodegen (9)
Sort [src_service_zone#200 ASC NULLS FIRST, dst#120 ASC NULLS FIRST, src_Borough#195 ASC NULLS FIRST], false, 0
WholeStageCodegen (6)
Exchange hashpartitioning(src_service_zone#200, dst#120, src_Borough#195, 200), true, [id=#367]
Project [src#119, dst#120, src_Borough#195, src_service_zone#200]
SortMergeJoin [src#119], [src_id#190], Inner
WholeStageCodegen (5)
Sort [src#119 ASC NULLS FIRST], false, 0
WholeStageCodegen (2)
Exchange hashpartitioning(src#119, 200), true, [id=#353]
Filter (isnotnull(src#119) AND isnotnull(dst#120))
Scan ExistingRDD[src#119,dst#120]
WholeStageCodegen (1)
Sort [src_id#190 ASC NULLS FIRST], false, 0
WholeStageCodegen (4)
Exchange hashpartitioning(src_id#190, 200), true, [id=#359]
Project [id#105 AS src_id#190, Borough#106 AS src_Borough#195, service_zone#108 AS src_service_zone#200]
Filter ((isnotnull(id#105) AND isnotnull(service_zone#108)) AND isnotnull(Borough#106))
Scan ExistingRDD[id#105,Borough#106,Zone#107,service_zone#108]
WholeStageCodegen (3)
Sort [dst_service_zone#215 ASC NULLS FIRST, dst_id#205 ASC NULLS FIRST, dst_Borough#210 ASC NULLS FIRST], false, 0
WholeStageCodegen (8)
Exchange hashpartitioning(dst_service_zone#215, dst_id#205, dst_Borough#210, 200), true, [id=#373]
Project [id#238 AS dst_id#205, Borough#239 AS dst_Borough#210, service_zone#241 AS dst_service_zone#215]
Filter ((isnotnull(service_zone#241) AND isnotnull(id#238)) AND isnotnull(Borough#239))
Scan ExistingRDD[id#238,Borough#239,Zone#240,service_zone#241]
WholeStageCodegen (7)
