digraph G {
subgraph cluster0 {
isCluster="true";
label="WholeStageCodegen (10)\n \nduration: 2 ms";
1 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 2 ms<br>number of output rows: 1"];
}
2 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 200<br>shuffle write time total (min, med, max (stageId: taskId))<br>65 ms (0 ms, 0 ms, 15 ms (stage 17.0: task 517))<br>records read: 200<br>local bytes read: 5.6 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 5.8 KiB<br>local blocks read: 98<br>remote blocks read: 102<br>data size total (min, med, max (stageId: taskId))<br>3.1 KiB (16.0 B, 16.0 B, 16.0 B (stage 17.0: task 423))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>11.3 KiB (56.0 B, 59.0 B, 59.0 B (stage 17.0: task 424))"];
subgraph cluster3 {
isCluster="true";
label="WholeStageCodegen (9)\n \nduration: total (min, med, max (stageId: taskId))\n472 ms (0 ms, 1 ms, 19 ms (stage 17.0: task 425))";
4 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>431 ms (0 ms, 1 ms, 19 ms (stage 17.0: task 425))<br>number of output rows: 200"];
5 [labelType="html" label="<br><b>Project</b><br><br>"];
6 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 285,732"];
}
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n280 ms (0 ms, 0 ms, 21 ms (stage 17.0: task 425))";
8 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>43 ms (0 ms, 0 ms, 10 ms (stage 17.0: task 425))<br>peak memory total (min, med, max (stageId: taskId))<br>12.5 GiB (64.1 MiB, 64.1 MiB, 65.0 MiB (stage 17.0: task 476))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 17.0: task 423))"];
}
9 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 466,523<br>shuffle write time total (min, med, max (stageId: taskId))<br>870 ms (0 ms, 4 ms, 50 ms (stage 15.0: task 292))<br>records read: 466,523<br>local bytes read total (min, med, max (stageId: taskId))<br>660.3 KiB (196.0 B, 3.1 KiB, 13.9 KiB (stage 17.0: task 497))<br>fetch wait time total (min, med, max (stageId: taskId))<br>6 ms (0 ms, 0 ms, 6 ms (stage 17.0: task 551))<br>remote bytes read total (min, med, max (stageId: taskId))<br>667.2 KiB (360.0 B, 3.1 KiB, 16.7 KiB (stage 17.0: task 497))<br>local blocks read: 5,158<br>remote blocks read: 5,170<br>data size total (min, med, max (stageId: taskId))<br>26.9 MiB (0.0 B, 7.4 KiB, 5.6 MiB (stage 15.0: task 363))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>1327.5 KiB (0.0 B, 4.5 KiB, 64.4 KiB (stage 15.0: task 363))"];
subgraph cluster10 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n2.1 s (0 ms, 6 ms, 239 ms (stage 15.0: task 308))";
11 [labelType="html" label="<br><b>Project</b><br><br>"];
12 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 466,523"];
}
subgraph cluster13 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n2.6 s (1 ms, 8 ms, 241 ms (stage 15.0: task 308))";
14 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 15.0: task 223))<br>peak memory total (min, med, max (stageId: taskId))<br>8.9 GiB (64.0 KiB, 64.1 MiB, 68.0 MiB (stage 15.0: task 363))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 15.0: task 223))"];
}
15 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 466,523<br>shuffle write time total (min, med, max (stageId: taskId))<br>30 ms (9 ms, 10 ms, 10 ms (stage 14.0: task 219))<br>records read: 466,523<br>local bytes read total (min, med, max (stageId: taskId))<br>1084.2 KiB (0.0 B, 491.0 B, 225.1 KiB (stage 15.0: task 363))<br>fetch wait time total (min, med, max (stageId: taskId))<br>197 ms (0 ms, 0 ms, 170 ms (stage 15.0: task 310))<br>remote bytes read total (min, med, max (stageId: taskId))<br>1116.3 KiB (0.0 B, 572.0 B, 159.1 KiB (stage 15.0: task 363))<br>local blocks read: 207<br>remote blocks read: 207<br>data size total (min, med, max (stageId: taskId))<br>10.7 MiB (1437.7 KiB, 4.5 MiB, 4.8 MiB (stage 14.0: task 219))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.1 MiB (304.2 KiB, 920.3 KiB, 976.0 KiB (stage 14.0: task 219))"];
subgraph cluster16 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.5 s (347 ms, 1.0 s, 1.1 s (stage 14.0: task 219))";
17 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 466,523"];
18 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 466,523"];
}
subgraph cluster19 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n1.4 s (0 ms, 2 ms, 228 ms (stage 15.0: task 308))";
20 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 15.0: task 223))<br>peak memory total (min, med, max (stageId: taskId))<br>8.9 GiB (0.0 B, 64.1 MiB, 64.1 MiB (stage 15.0: task 223))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 15.0: task 223))"];
}
21 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 265<br>shuffle write time: 9 ms<br>records read: 261<br>local bytes read total (min, med, max (stageId: taskId))<br>8.9 KiB (0.0 B, 0.0 B, 174.0 B (stage 15.0: task 232))<br>fetch wait time total (min, med, max (stageId: taskId))<br>5 ms (0 ms, 0 ms, 5 ms (stage 15.0: task 320))<br>remote bytes read total (min, med, max (stageId: taskId))<br>8.6 KiB (0.0 B, 0.0 B, 199.0 B (stage 15.0: task 247))<br>local blocks read: 75<br>remote blocks read: 70<br>data size: 15.1 KiB<br>shuffle bytes written: 17.4 KiB"];
subgraph cluster22 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 50 ms";
23 [labelType="html" label="<br><b>Project</b><br><br>"];
24 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 265"];
25 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 265"];
}
subgraph cluster26 {
isCluster="true";
label="WholeStageCodegen (8)\n \nduration: total (min, med, max (stageId: taskId))\n919 ms (0 ms, 3 ms, 58 ms (stage 17.0: task 424))";
27 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 17.0: task 423))<br>peak memory total (min, med, max (stageId: taskId))<br>8.9 GiB (64.0 KiB, 64.1 MiB, 64.1 MiB (stage 17.0: task 424))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 17.0: task 423))"];
}
28 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 265<br>shuffle write time: 10 ms<br>records read: 265<br>local bytes read total (min, med, max (stageId: taskId))<br>8.9 KiB (0.0 B, 0.0 B, 225.0 B (stage 17.0: task 513))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 17.0: task 424))<br>remote bytes read total (min, med, max (stageId: taskId))<br>8.1 KiB (0.0 B, 0.0 B, 217.0 B (stage 17.0: task 497))<br>local blocks read: 74<br>remote blocks read: 69<br>data size: 15.1 KiB<br>shuffle bytes written: 17.1 KiB"];
subgraph cluster29 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: 48 ms";
30 [labelType="html" label="<br><b>Project</b><br><br>"];
31 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 265"];
32 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 265"];
}
2->1;
4->2;
5->4;
6->5;
8->6;
9->8;
11->9;
12->11;
14->12;
15->14;
17->15;
18->17;
20->12;
21->20;
23->21;
24->23;
25->24;
27->6;
28->27;
30->28;
31->30;
32->31;
}
33
HashAggregate(keys=[], functions=[count(1)])
WholeStageCodegen (10)
Exchange SinglePartition, true, [id=#282]
HashAggregate(keys=[], functions=[partial_count(1)])
Project
SortMergeJoin [src_service_zone#200, dst#120, src_Borough#195], [dst_service_zone#215, dst_id#205, dst_Borough#210], Inner
WholeStageCodegen (9)
Sort [src_service_zone#200 ASC NULLS FIRST, dst#120 ASC NULLS FIRST, src_Borough#195 ASC NULLS FIRST], false, 0
WholeStageCodegen (6)
Exchange hashpartitioning(src_service_zone#200, dst#120, src_Borough#195, 200), true, [id=#267]
Project [dst#120, src_Borough#195, src_service_zone#200]
SortMergeJoin [src#119], [src_id#190], Inner
WholeStageCodegen (5)
Sort [src#119 ASC NULLS FIRST], false, 0
WholeStageCodegen (2)
Exchange hashpartitioning(src#119, 200), true, [id=#253]
Filter (isnotnull(src#119) AND isnotnull(dst#120))
Scan ExistingRDD[src#119,dst#120]
WholeStageCodegen (1)
Sort [src_id#190 ASC NULLS FIRST], false, 0
WholeStageCodegen (4)
Exchange hashpartitioning(src_id#190, 200), true, [id=#259]
Project [id#105 AS src_id#190, Borough#106 AS src_Borough#195, service_zone#108 AS src_service_zone#200]
Filter ((isnotnull(id#105) AND isnotnull(service_zone#108)) AND isnotnull(Borough#106))
Scan ExistingRDD[id#105,Borough#106,Zone#107,service_zone#108]
WholeStageCodegen (3)
Sort [dst_service_zone#215 ASC NULLS FIRST, dst_id#205 ASC NULLS FIRST, dst_Borough#210 ASC NULLS FIRST], false, 0
WholeStageCodegen (8)
Exchange hashpartitioning(dst_service_zone#215, dst_id#205, dst_Borough#210, 200), true, [id=#273]
Project [id#238 AS dst_id#205, Borough#239 AS dst_Borough#210, service_zone#241 AS dst_service_zone#215]
Filter ((isnotnull(service_zone#241) AND isnotnull(id#238)) AND isnotnull(Borough#239))
Scan ExistingRDD[id#238,Borough#239,Zone#240,service_zone#241]
WholeStageCodegen (7)
