movieratings - Details for Query 0

digraph G { subgraph cluster0 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: 2 ms"; 1 [labelType="html" label="HashAggregate time in aggregation build: 2 ms number of output rows: 1"]; } 2 [labelType="html" label="Exchange shuffle records written: 200 shuffle write time total (min, med, max (stageId: taskId)) 44 ms (0 ms, 0 ms, 0 ms (stage 2.0: task 87)) records read: 200 local bytes read: 7.0 KiB fetch wait time: 0 ms remote bytes read: 4.6 KiB local blocks read: 121 remote blocks read: 79 data size total (min, med, max (stageId: taskId)) 3.1 KiB (16.0 B, 16.0 B, 16.0 B (stage 2.0: task 29)) shuffle bytes written total (min, med, max (stageId: taskId)) 11.5 KiB (59.0 B, 59.0 B, 59.0 B (stage 2.0: task 29))"]; subgraph cluster3 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n1.1 s (2 ms, 4 ms, 32 ms (stage 2.0: task 30))"; 4 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 1.0 s (2 ms, 4 ms, 30 ms (stage 2.0: task 30)) number of output rows: 200"]; 5 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 564 ms (0 ms, 2 ms, 16 ms (stage 2.0: task 30)) peak memory total (min, med, max (stageId: taskId)) 12.5 GiB (64.3 MiB, 64.3 MiB, 64.3 MiB (stage 2.0: task 29)) number of output rows: 330,975 avg hash probe bucket list iters (min, med, max (stageId: taskId)): (1, 1, 1 (stage 2.0: task 29))"]; } 6 [labelType="html" label="Exchange shuffle records written: 331,002 shuffle write time total (min, med, max (stageId: taskId)) 662 ms (14 ms, 19 ms, 70 ms (stage 1.0: task 1)) records read: 331,002 local bytes read total (min, med, max (stageId: taskId)) 1102.7 KiB (4.2 KiB, 6.0 KiB, 6.6 KiB (stage 2.0: task 106)) fetch wait time total (min, med, max (stageId: taskId)) 60 ms (0 ms, 0 ms, 11 ms (stage 2.0: task 209)) remote bytes read total (min, med, max (stageId: taskId)) 1037.6 KiB (4.2 KiB, 4.7 KiB, 6.7 KiB (stage 2.0: task 225)) local blocks read: 2,884 remote blocks read: 2,716 data size total (min, med, max (stageId: taskId)) 5.1 MiB (155.4 KiB, 186.1 KiB, 196.1 KiB (stage 1.0: task 8)) shuffle bytes written total (min, med, max (stageId: taskId)) 2.1 MiB (60.1 KiB, 69.6 KiB, 111.6 KiB (stage 1.0: task 5))"]; subgraph cluster7 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.5 m (4.0 s, 5.0 s, 6.4 s (stage 1.0: task 7))"; 8 [labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 2.5 m (4.0 s, 5.0 s, 6.3 s (stage 1.0: task 7)) peak memory total (min, med, max (stageId: taskId)) 1806.0 MiB (64.5 MiB, 64.5 MiB, 64.5 MiB (stage 1.0: task 2)) number of output rows: 331,002 avg hash probe bucket list iters (min, med, max (stageId: taskId)): (1.4, 1.4, 1.4 (stage 1.0: task 2))"]; 9 [labelType="html" label=" Project "]; 10 [labelType="html" label="Scan ExistingRDD number of output rows: 33,832,162"]; } 2->1; 4->2; 5->4; 6->5; 8->6; 9->8; 10->9; }

HashAggregate(keys=[], functions=[count(1)])

WholeStageCodegen (3)

Exchange SinglePartition, true, [id=#33]

HashAggregate(keys=[], functions=[partial_count(1)])

HashAggregate(keys=[userId#0L], functions=[])

WholeStageCodegen (2)

Exchange hashpartitioning(userId#0L, 200), true, [id=#28]

HashAggregate(keys=[userId#0L], functions=[])

Project [userId#0L]

Scan ExistingRDD[userId#0L,movieId#1L,rating#2,timestamp_str#3L]

WholeStageCodegen (1)

== Parsed Logical Plan ==
Aggregate [count(1) AS count#11L]
+- Deduplicate [userId#0L]
   +- Project [userId#0L]
      +- LogicalRDD [userId#0L, movieId#1L, rating#2, timestamp_str#3L], false

== Analyzed Logical Plan ==
count: bigint
Aggregate [count(1) AS count#11L]
+- Deduplicate [userId#0L]
   +- Project [userId#0L]
      +- LogicalRDD [userId#0L, movieId#1L, rating#2, timestamp_str#3L], false

== Optimized Logical Plan ==
Aggregate [count(1) AS count#11L]
+- Aggregate [userId#0L]
   +- Project [userId#0L]
      +- LogicalRDD [userId#0L, movieId#1L, rating#2, timestamp_str#3L], false

== Physical Plan ==
*(3) HashAggregate(keys=[], functions=[count(1)], output=[count#11L])
+- Exchange SinglePartition, true, [id=#33]
   +- *(2) HashAggregate(keys=[], functions=[partial_count(1)], output=[count#14L])
      +- *(2) HashAggregate(keys=[userId#0L], functions=[], output=[])
         +- Exchange hashpartitioning(userId#0L, 200), true, [id=#28]
            +- *(1) HashAggregate(keys=[userId#0L], functions=[], output=[userId#0L])
               +- *(1) Project [userId#0L]
                  +- *(1) Scan ExistingRDD[userId#0L,movieId#1L,rating#2,timestamp_str#3L]