digraph G {
0 [labelType="html" label="<br><b>CollectLimit</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: 0 ms";
2 [labelType="html" label="<br><b>Project</b><br><br>"];
}
3 [labelType="html" label="<b>TakeOrderedAndProject</b><br><br>shuffle records written: 2,000<br>shuffle write time total (min, med, max (stageId: taskId))<br>50 ms (0 ms, 0 ms, 0 ms (stage 80.0: task 2345))<br>records read: 2,000<br>local bytes read: 62.9 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 63.1 KiB<br>local blocks read: 100<br>remote blocks read: 100<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>126.0 KiB (549.0 B, 643.0 B, 784.0 B (stage 80.0: task 2383))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n1.5 s (5 ms, 7 ms, 55 ms (stage 80.0: task 2468))";
5 [labelType="html" label="<br><b>Project</b><br><br>"];
6 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 83,239"];
}
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n2.0 s (6 ms, 8 ms, 57 ms (stage 80.0: task 2468))";
8 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 80.0: task 2325))<br>peak memory total (min, med, max (stageId: taskId))<br>12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 80.0: task 2325))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 80.0: task 2325))"];
9 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>1.2 s (4 ms, 5 ms, 52 ms (stage 80.0: task 2468))<br>peak memory total (min, med, max (stageId: taskId))<br>12.5 GiB (64.3 MiB, 64.3 MiB, 64.3 MiB (stage 80.0: task 2325))<br>number of output rows: 83,239<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 80.0: task 2325))"];
}
10 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1,824,534<br>shuffle write time total (min, med, max (stageId: taskId))<br>2.8 s (10 ms, 13 ms, 25 ms (stage 79.0: task 2305))<br>records read: 1,824,534<br>local bytes read total (min, med, max (stageId: taskId))<br>12.6 MiB (48.2 KiB, 64.1 KiB, 82.4 KiB (stage 80.0: task 2500))<br>fetch wait time total (min, med, max (stageId: taskId))<br>51 ms (0 ms, 0 ms, 43 ms (stage 80.0: task 2469))<br>remote bytes read total (min, med, max (stageId: taskId))<br>12.6 MiB (48.2 KiB, 64.0 KiB, 83.4 KiB (stage 80.0: task 2500))<br>local blocks read: 19,986<br>remote blocks read: 19,978<br>data size total (min, med, max (stageId: taskId))<br>55.7 MiB (20.8 KiB, 277.6 KiB, 578.7 KiB (stage 79.0: task 2316))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>25.2 MiB (21.4 KiB, 134.2 KiB, 222.0 KiB (stage 79.0: task 2316))"];
subgraph cluster11 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n22.1 s (74 ms, 108 ms, 218 ms (stage 79.0: task 2126))";
12 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>16.9 s (51 ms, 85 ms, 156 ms (stage 79.0: task 2126))<br>peak memory total (min, med, max (stageId: taskId))<br>4.9 GiB (256.0 KiB, 256.0 KiB, 64.3 MiB (stage 79.0: task 2248))<br>number of output rows: 1,824,534<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 79.0: task 2248))"];
13 [labelType="html" label="<br><b>Project</b><br><br>"];
14 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>6.9 s (13 ms, 35 ms, 53 ms (stage 79.0: task 2126))<br>peak memory total (min, med, max (stageId: taskId))<br>13.5 GiB (68.0 MiB, 68.0 MiB, 72.0 MiB (stage 79.0: task 2126))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 79.0: task 2125))"];
}
15 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 33,832,162<br>shuffle write time total (min, med, max (stageId: taskId))<br>1.0 s (28 ms, 38 ms, 48 ms (stage 77.0: task 2098))<br>records read: 33,832,162<br>local bytes read total (min, med, max (stageId: taskId))<br>149.6 MiB (502.0 KiB, 762.3 KiB, 1083.5 KiB (stage 79.0: task 2289))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 79.0: task 2125))<br>remote bytes read total (min, med, max (stageId: taskId))<br>151.6 MiB (458.2 KiB, 754.7 KiB, 1113.8 KiB (stage 79.0: task 2246))<br>local blocks read: 2,796<br>remote blocks read: 2,804<br>data size total (min, med, max (stageId: taskId))<br>1548.7 MiB (45.8 MiB, 55.0 MiB, 59.0 MiB (stage 77.0: task 2097))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>301.2 MiB (8.9 MiB, 10.7 MiB, 11.5 MiB (stage 77.0: task 2097))"];
subgraph cluster16 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n4.8 m (3.5 s, 5.0 s, 6.3 s (stage 77.0: task 2100))";
17 [labelType="html" label="<br><b>Project</b><br><br>"];
18 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 67,664,324"];
19 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 67,664,324"];
}
subgraph cluster20 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n1.7 s (0 ms, 7 ms, 56 ms (stage 80.0: task 2468))";
21 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 80.0: task 2325))<br>peak memory total (min, med, max (stageId: taskId))<br>12.5 GiB (64.1 MiB, 64.1 MiB, 64.1 MiB (stage 80.0: task 2325))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 80.0: task 2325))"];
}
22 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 86,537<br>shuffle write time total (min, med, max (stageId: taskId))<br>35 ms (16 ms, 18 ms, 18 ms (stage 78.0: task 2096))<br>records read: 86,537<br>local bytes read total (min, med, max (stageId: taskId))<br>1936.9 KiB (7.9 KiB, 9.7 KiB, 12.4 KiB (stage 80.0: task 2423))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 80.0: task 2325))<br>remote bytes read total (min, med, max (stageId: taskId))<br>1931.9 KiB (8.2 KiB, 9.7 KiB, 11.1 KiB (stage 80.0: task 2460))<br>local blocks read: 200<br>remote blocks read: 200<br>data size total (min, med, max (stageId: taskId))<br>6.4 MiB (3.2 MiB, 3.2 MiB, 3.2 MiB (stage 78.0: task 2096))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>3.8 MiB (1924.7 KiB, 1944.1 KiB, 1944.1 KiB (stage 78.0: task 2095))"];
subgraph cluster23 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n725 ms (309 ms, 416 ms, 416 ms (stage 78.0: task 2096))";
24 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 86,537"];
25 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 86,537"];
}
2->0;
3->2;
5->3;
6->5;
8->6;
9->8;
10->9;
12->10;
13->12;
14->13;
15->14;
17->15;
18->17;
19->18;
21->6;
22->21;
24->22;
25->24;
}
26
CollectLimit 11
Project [cast(movieId#1L as string) AS movieId#346, cast(avg_rating#335 as string) AS avg_rating#347, title#201, genres#202]
WholeStageCodegen (7)
TakeOrderedAndProject(limit=10, orderBy=[avg_rating#335 DESC NULLS LAST], output=[movieId#1L,avg_rating#335,title#201,genres#202])
Project [movieId#1L, avg_rating#335, title#201, genres#202]
SortMergeJoin [movieId#1L], [movieId#200L], Inner
WholeStageCodegen (6)
Sort [movieId#1L ASC NULLS FIRST], false, 0
HashAggregate(keys=[movieId#1L], functions=[avg(rating#2)])
WholeStageCodegen (3)
Exchange hashpartitioning(movieId#1L, 200), true, [id=#463]
HashAggregate(keys=[movieId#1L], functions=[partial_avg(rating#2)])
Project [movieId#1L, rating#2]
Sort [date#39 ASC NULLS FIRST], true, 0
WholeStageCodegen (2)
Exchange rangepartitioning(date#39 ASC NULLS FIRST, 200), true, [id=#457]
Project [movieId#1L, rating#2, from_unixtime(timestamp_str#3L, yyyy-MM-dd, Some(GMT)) AS date#39]
Filter isnotnull(movieId#1L)
Scan ExistingRDD[userId#0L,movieId#1L,rating#2,timestamp_str#3L]
WholeStageCodegen (1)
Sort [movieId#200L ASC NULLS FIRST], false, 0
WholeStageCodegen (5)
Exchange hashpartitioning(movieId#200L, 200), true, [id=#470]
Filter isnotnull(movieId#200L)
Scan ExistingRDD[movieId#200L,title#201,genres#202]
WholeStageCodegen (4)
