digraph G {
subgraph cluster0 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 12 ms";
1 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 12 ms<br>number of output rows: 1"];
}
2 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 200<br>shuffle write time total (min, med, max (stageId: taskId))<br>64 ms (0 ms, 0 ms, 16 ms (stage 10.0: task 383))<br>records read: 200<br>local bytes read: 5.2 KiB<br>fetch wait time: 4 ms<br>remote bytes read: 6.3 KiB<br>local blocks read: 90<br>remote blocks read: 110<br>data size total (min, med, max (stageId: taskId))<br>3.1 KiB (16.0 B, 16.0 B, 16.0 B (stage 10.0: task 372))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>11.5 KiB (59.0 B, 59.0 B, 59.0 B (stage 10.0: task 372))"];
subgraph cluster3 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n723 ms (2 ms, 3 ms, 19 ms (stage 10.0: task 384))";
4 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>681 ms (2 ms, 3 ms, 19 ms (stage 10.0: task 384))<br>number of output rows: 200"];
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>435 ms (1 ms, 2 ms, 17 ms (stage 10.0: task 384))<br>peak memory total (min, med, max (stageId: taskId))<br>12.5 GiB (64.3 MiB, 64.3 MiB, 64.3 MiB (stage 10.0: task 372))<br>number of output rows: 83,239<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 10.0: task 372))"];
}
6 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 816,055<br>shuffle write time total (min, med, max (stageId: taskId))<br>521 ms (15 ms, 18 ms, 24 ms (stage 9.0: task 353))<br>records read: 816,055<br>local bytes read total (min, med, max (stageId: taskId))<br>2.7 MiB (10.2 KiB, 13.9 KiB, 17.3 KiB (stage 10.0: task 521))<br>fetch wait time total (min, med, max (stageId: taskId))<br>42 ms (0 ms, 0 ms, 16 ms (stage 10.0: task 384))<br>remote bytes read total (min, med, max (stageId: taskId))<br>2.7 MiB (10.3 KiB, 13.6 KiB, 18.4 KiB (stage 10.0: task 546))<br>local blocks read: 2,840<br>remote blocks read: 2,760<br>data size total (min, med, max (stageId: taskId))<br>12.5 MiB (402.8 KiB, 444.7 KiB, 702.0 KiB (stage 9.0: task 358))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>5.4 MiB (159.7 KiB, 195.9 KiB, 280.0 KiB (stage 9.0: task 358))"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.4 m (3.5 s, 5.8 s, 6.5 s (stage 9.0: task 364))";
8 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>2.4 m (3.5 s, 5.8 s, 6.5 s (stage 9.0: task 364))<br>peak memory total (min, med, max (stageId: taskId))<br>1821.0 MiB (65.0 MiB, 65.0 MiB, 66.0 MiB (stage 9.0: task 358))<br>number of output rows: 816,055<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1.1 (stage 9.0: task 361))"];
9 [labelType="html" label="<br><b>Project</b><br><br>"];
10 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 33,832,162"];
}
2->1;
4->2;
5->4;
6->5;
8->6;
9->8;
10->9;
}
11
HashAggregate(keys=[], functions=[count(1)])
WholeStageCodegen (3)
Exchange SinglePartition, true, [id=#82]
HashAggregate(keys=[], functions=[partial_count(1)])
HashAggregate(keys=[movieId#1L], functions=[])
WholeStageCodegen (2)
Exchange hashpartitioning(movieId#1L, 200), true, [id=#77]
HashAggregate(keys=[movieId#1L], functions=[])
Project [movieId#1L]
Scan ExistingRDD[userId#0L,movieId#1L,rating#2,timestamp_str#3L]
WholeStageCodegen (1)
