digraph G {
subgraph cluster0 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 3 ms";
1 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 3 ms<br>number of output rows: 1"];
}
2 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 200<br>shuffle write time total (min, med, max (stageId: taskId))<br>46 ms (0 ms, 0 ms, 2 ms (stage 5.0: task 380))<br>records read: 200<br>local bytes read: 5.3 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 6.2 KiB<br>local blocks read: 92<br>remote blocks read: 108<br>data size total (min, med, max (stageId: taskId))<br>3.1 KiB (16.0 B, 16.0 B, 16.0 B (stage 5.0: task 259))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>11.5 KiB (59.0 B, 59.0 B, 59.0 B (stage 5.0: task 259))"];
subgraph cluster3 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n679 ms (2 ms, 3 ms, 10 ms (stage 5.0: task 259))";
4 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>631 ms (1 ms, 3 ms, 10 ms (stage 5.0: task 259))<br>number of output rows: 200"];
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>396 ms (1 ms, 2 ms, 8 ms (stage 5.0: task 259))<br>peak memory total (min, med, max (stageId: taskId))<br>12.5 GiB (64.3 MiB, 64.3 MiB, 64.3 MiB (stage 5.0: task 259))<br>number of output rows: 83,239<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 5.0: task 259))"];
}
6 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 816,055<br>shuffle write time total (min, med, max (stageId: taskId))<br>483 ms (12 ms, 17 ms, 24 ms (stage 4.0: task 252))<br>records read: 816,055<br>local bytes read total (min, med, max (stageId: taskId))<br>2.7 MiB (10.3 KiB, 13.9 KiB, 18.1 KiB (stage 5.0: task 433))<br>fetch wait time total (min, med, max (stageId: taskId))<br>10 ms (0 ms, 0 ms, 4 ms (stage 5.0: task 259))<br>remote bytes read total (min, med, max (stageId: taskId))<br>2.7 MiB (10.8 KiB, 13.6 KiB, 17.2 KiB (stage 5.0: task 290))<br>local blocks read: 2,816<br>remote blocks read: 2,784<br>data size total (min, med, max (stageId: taskId))<br>12.5 MiB (402.8 KiB, 444.7 KiB, 702.0 KiB (stage 4.0: task 245))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>5.4 MiB (159.7 KiB, 195.9 KiB, 280.0 KiB (stage 4.0: task 245))"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.5 m (4.6 s, 5.0 s, 6.2 s (stage 4.0: task 231))";
8 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>2.5 m (4.6 s, 5.0 s, 6.2 s (stage 4.0: task 231))<br>peak memory total (min, med, max (stageId: taskId))<br>1821.0 MiB (65.0 MiB, 65.0 MiB, 66.0 MiB (stage 4.0: task 245))<br>number of output rows: 816,055<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1.1 (stage 4.0: task 248))"];
9 [labelType="html" label="<br><b>Project</b><br><br>"];
10 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 33,832,162"];
}
2->1;
4->2;
5->4;
6->5;
8->6;
9->8;
10->9;
}
11
HashAggregate(keys=[], functions=[count(1)])
WholeStageCodegen (3)
Exchange SinglePartition, true, [id=#70]
HashAggregate(keys=[], functions=[partial_count(1)])
HashAggregate(keys=[movieId#1L], functions=[])
WholeStageCodegen (2)
Exchange hashpartitioning(movieId#1L, 200), true, [id=#65]
HashAggregate(keys=[movieId#1L], functions=[])
Project [movieId#1L]
Scan ExistingRDD[userId#0L,movieId#1L,rating#2,timestamp_str#3L]
WholeStageCodegen (1)
