movieratings - Details for Job 9

Details for Job 9

Status: SUCCEEDED
Associated SQL Query: 2
Completed Stages: 1
Skipped Stages: 3

Enable zooming

digraph G { subgraph clusterstage_21 { label="Stage 21 (skipped)"; 3 [labelType="html" label="PythonRDD [3] RDD at PythonRDD.scala:53"]; subgraph cluster0 { label="textFile"; 0 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/ratings.csv [0] textFile at NativeMethodAccessorImpl.java:0"]; 1 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/ratings.csv [1] textFile at NativeMethodAccessorImpl.java:0"]; } subgraph cluster4 { label="mapPartitions"; 4 [labelType="html" label="MapPartitionsRDD [4] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster5 { label="map"; 5 [labelType="html" label="MapPartitionsRDD [5] map at SerDeUtil.scala:137"]; } subgraph cluster6 { label="mapPartitions"; 6 [labelType="html" label="MapPartitionsRDD [6] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster163 { label="WholeStageCodegen (4)"; 66 [labelType="html" label="MapPartitionsRDD [66] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster162 { label="Exchange"; 67 [labelType="html" label="MapPartitionsRDD [67] showString at NativeMethodAccessorImpl.java:0"]; } } 0->1; 1->3; 3->4; 4->5; 5->6; 6->66; 66->67; }

digraph G { subgraph clusterstage_22 { label="Stage 22 (skipped)"; 56 [labelType="html" label="PythonRDD [56] RDD at PythonRDD.scala:53"]; subgraph cluster130 { label="textFile"; 53 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/movies.csv [53] textFile at NativeMethodAccessorImpl.java:0"]; 54 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/movies.csv [54] textFile at NativeMethodAccessorImpl.java:0"]; } subgraph cluster134 { label="mapPartitions"; 57 [labelType="html" label="MapPartitionsRDD [57] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster135 { label="map"; 58 [labelType="html" label="MapPartitionsRDD [58] map at SerDeUtil.scala:137"]; } subgraph cluster136 { label="mapPartitions"; 59 [labelType="html" label="MapPartitionsRDD [59] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster154 { label="WholeStageCodegen (1)"; 60 [labelType="html" label="MapPartitionsRDD [60] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster153 { label="Generate"; 61 [labelType="html" label="MapPartitionsRDD [61] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster150 { label="WholeStageCodegen (2)"; 62 [labelType="html" label="MapPartitionsRDD [62] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster149 { label="Exchange"; 63 [labelType="html" label="MapPartitionsRDD [63] showString at NativeMethodAccessorImpl.java:0"]; } } 53->54; 54->56; 56->57; 57->58; 58->59; 59->60; 60->61; 61->62; 62->63; }

digraph G { subgraph clusterstage_23 { label="Stage 23 (skipped)"; subgraph cluster149 { label="Exchange"; 64 [labelType="html" label="ShuffledRowRDD [64] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster146 { label="WholeStageCodegen (3)"; 65 [labelType="html" label="MapPartitionsRDD [65] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster162 { label="Exchange"; 68 [labelType="html" label="ShuffledRowRDD [68] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster159 { label="WholeStageCodegen (5)"; 69 [labelType="html" label="MapPartitionsRDD [69] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster141 { label="WholeStageCodegen (6)"; 70 [labelType="html" label="ZippedPartitionsRDD2 [70] showString at NativeMethodAccessorImpl.java:0"]; 71 [labelType="html" label="MapPartitionsRDD [71] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster140 { label="Exchange"; 72 [labelType="html" label="MapPartitionsRDD [72] showString at NativeMethodAccessorImpl.java:0"]; } } 64->65; 68->69; 65->70; 69->70; 70->71; 71->72; }

63,64

67,68

digraph G { subgraph clusterstage_24 { label="Stage 24"; subgraph cluster140 { label="Exchange"; 73 [labelType="html" label="ShuffledRowRDD [73] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster137 { label="WholeStageCodegen (7)"; 74 [labelType="html" label="MapPartitionsRDD [74] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster167 { label="mapPartitionsInternal"; 75 [labelType="html" label="MapPartitionsRDD [75] showString at NativeMethodAccessorImpl.java:0"]; } } 73->74; 74->75; }

72,73

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
24	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 13:50:22	66 ms	4/4

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2025/03/08 13:50:22

66 ms

4/4

Page:

Skipped Stages (3)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
23	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200
22	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/2
21	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/28

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

Unknown

0/200

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

Unknown

0/2

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

Unknown

0/28

Page: