movieratings - Details for Query 1

digraph G { subgraph cluster0 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: 3 ms"; 1 [labelType="html" label="HashAggregate time in aggregation build: 3 ms number of output rows: 1"]; } 2 [labelType="html" label="Exchange shuffle records written: 200 shuffle write time total (min, med, max (stageId: taskId)) 60 ms (0 ms, 0 ms, 1 ms (stage 7.0: task 267)) records read: 200 local bytes read: 5.7 KiB fetch wait time: 0 ms remote bytes read: 5.8 KiB local blocks read: 99 remote blocks read: 101 data size total (min, med, max (stageId: taskId)) 3.1 KiB (16.0 B, 16.0 B, 16.0 B (stage 7.0: task 142)) shuffle bytes written total (min, med, max (stageId: taskId)) 11.5 KiB (59.0 B, 59.0 B, 59.0 B (stage 7.0: task 142))"]; subgraph cluster3 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n1.1 s (3 ms, 5 ms, 21 ms (stage 7.0: task 143))"; 4 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 1.0 s (3 ms, 5 ms, 19 ms (stage 7.0: task 143)) number of output rows: 200"]; 5 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 583 ms (1 ms, 2 ms, 15 ms (stage 7.0: task 143)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.3 MiB, 64.3 MiB, 64.3 MiB (stage 7.0: task 142)) number of output rows: 330,975 avg hash probe bucket list iters (min, med, max (stageId: taskId)): (1, 1, 1 (stage 7.0: task 142))"]; } 6 [labelType="html" label="Exchange shuffle records written: 331,002 shuffle write time total (min, med, max (stageId: taskId)) 623 ms (14 ms, 20 ms, 48 ms (stage 6.0: task 114)) records read: 331,002 local bytes read total (min, med, max (stageId: taskId)) 1070.6 KiB (4.2 KiB, 5.7 KiB, 6.5 KiB (stage 7.0: task 151)) fetch wait time total (min, med, max (stageId: taskId)) 43 ms (0 ms, 0 ms, 3 ms (stage 7.0: task 144)) remote bytes read total (min, med, max (stageId: taskId)) 1069.8 KiB (4.3 KiB, 5.0 KiB, 6.7 KiB (stage 7.0: task 331)) local blocks read: 2,804 remote blocks read: 2,796 data size total (min, med, max (stageId: taskId)) 5.1 MiB (155.4 KiB, 186.1 KiB, 196.1 KiB (stage 6.0: task 121)) shuffle bytes written total (min, med, max (stageId: taskId)) 2.1 MiB (60.1 KiB, 69.6 KiB, 111.6 KiB (stage 6.0: task 118))"]; subgraph cluster7 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.4 m (3.5 s, 5.2 s, 6.5 s (stage 6.0: task 114))"; 8 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 2.4 m (3.5 s, 5.2 s, 6.2 s (stage 6.0: task 114)) peak memory total (min, med, max (stageId: taskId)) 1806.0 MiB (64.5 MiB, 64.5 MiB, 64.5 MiB (stage 6.0: task 115)) number of output rows: 331,002 avg hash probe bucket list iters (min, med, max (stageId: taskId)): (1.4, 1.4, 1.4 (stage 6.0: task 115))"]; 9 [labelType="html" label=" Project "]; 10 [labelType="html" label="Scan ExistingRDD number of output rows: 33,832,162"]; } 2->1; 4->2; 5->4; 6->5; 8->6; 9->8; 10->9; }

HashAggregate(keys=[], functions=[count(1)])

WholeStageCodegen (3)

Exchange SinglePartition, true, [id=#45]

HashAggregate(keys=[], functions=[partial_count(1)])

HashAggregate(keys=[userId#0L], functions=[])

WholeStageCodegen (2)

Exchange hashpartitioning(userId#0L, 200), true, [id=#40]

HashAggregate(keys=[userId#0L], functions=[])

Project [userId#0L]

Scan ExistingRDD[userId#0L,movieId#1L,rating#2,timestamp_str#3L]

WholeStageCodegen (1)

== Parsed Logical Plan ==
Aggregate [count(1) AS count#28L]
+- Deduplicate [userId#0L]
   +- Project [userId#0L]
      +- LogicalRDD [userId#0L, movieId#1L, rating#2, timestamp_str#3L], false

== Analyzed Logical Plan ==
count: bigint
Aggregate [count(1) AS count#28L]
+- Deduplicate [userId#0L]
   +- Project [userId#0L]
      +- LogicalRDD [userId#0L, movieId#1L, rating#2, timestamp_str#3L], false

== Optimized Logical Plan ==
Aggregate [count(1) AS count#28L]
+- Aggregate [userId#0L]
   +- Project [userId#0L]
      +- LogicalRDD [userId#0L, movieId#1L, rating#2, timestamp_str#3L], false

== Physical Plan ==
*(3) HashAggregate(keys=[], functions=[count(1)], output=[count#28L])
+- Exchange SinglePartition, true, [id=#45]
   +- *(2) HashAggregate(keys=[], functions=[partial_count(1)], output=[count#31L])
      +- *(2) HashAggregate(keys=[userId#0L], functions=[], output=[])
         +- Exchange hashpartitioning(userId#0L, 200), true, [id=#40]
            +- *(1) HashAggregate(keys=[userId#0L], functions=[], output=[userId#0L])
               +- *(1) Project [userId#0L]
                  +- *(1) Scan ExistingRDD[userId#0L,movieId#1L,rating#2,timestamp_str#3L]