movieratings - Details for Job 30

Details for Job 30

Status: SUCCEEDED
Associated SQL Query: 9
Completed Stages: 1
Skipped Stages: 3

Enable zooming

digraph G { subgraph clusterstage_52 { label="Stage 52 (skipped)"; 7 [labelType="html" label="PythonRDD [7] RDD at PythonRDD.scala:53"]; subgraph cluster0 { label="textFile"; 0 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/ratings.csv [0] textFile at NativeMethodAccessorImpl.java:0"]; 1 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/ratings.csv [1] textFile at NativeMethodAccessorImpl.java:0"]; } subgraph cluster16 { label="mapPartitions"; 8 [labelType="html" label="MapPartitionsRDD [8] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster17 { label="map"; 9 [labelType="html" label="MapPartitionsRDD [9] map at SerDeUtil.scala:137"]; } subgraph cluster18 { label="mapPartitions"; 10 [labelType="html" label="MapPartitionsRDD [10] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster257 { label="WholeStageCodegen (4)"; 90 [labelType="html" label="MapPartitionsRDD [90] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster256 { label="Exchange"; 91 [labelType="html" label="MapPartitionsRDD [91] showString at NativeMethodAccessorImpl.java:0"]; } } 0->1; 1->7; 7->8; 8->9; 9->10; 10->90; 90->91; }

digraph G { subgraph clusterstage_53 { label="Stage 53 (skipped)"; 69 [labelType="html" label="PythonRDD [69] RDD at PythonRDD.scala:53"]; subgraph cluster174 { label="textFile"; 62 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/movies.csv [62] textFile at NativeMethodAccessorImpl.java:0"]; 63 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/movies.csv [63] textFile at NativeMethodAccessorImpl.java:0"]; } subgraph cluster190 { label="mapPartitions"; 70 [labelType="html" label="MapPartitionsRDD [70] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster191 { label="map"; 71 [labelType="html" label="MapPartitionsRDD [71] map at SerDeUtil.scala:137"]; } subgraph cluster192 { label="mapPartitions"; 72 [labelType="html" label="MapPartitionsRDD [72] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster248 { label="WholeStageCodegen (1)"; 84 [labelType="html" label="MapPartitionsRDD [84] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster247 { label="Generate"; 85 [labelType="html" label="MapPartitionsRDD [85] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster244 { label="WholeStageCodegen (2)"; 86 [labelType="html" label="MapPartitionsRDD [86] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster243 { label="Exchange"; 87 [labelType="html" label="MapPartitionsRDD [87] showString at NativeMethodAccessorImpl.java:0"]; } } 62->63; 63->69; 69->70; 70->71; 71->72; 72->84; 84->85; 85->86; 86->87; }

digraph G { subgraph clusterstage_54 { label="Stage 54 (skipped)"; subgraph cluster243 { label="Exchange"; 88 [labelType="html" label="ShuffledRowRDD [88] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster240 { label="WholeStageCodegen (3)"; 89 [labelType="html" label="MapPartitionsRDD [89] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster256 { label="Exchange"; 92 [labelType="html" label="ShuffledRowRDD [92] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster253 { label="WholeStageCodegen (5)"; 93 [labelType="html" label="MapPartitionsRDD [93] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster235 { label="WholeStageCodegen (6)"; 94 [labelType="html" label="ZippedPartitionsRDD2 [94] showString at NativeMethodAccessorImpl.java:0"]; 95 [labelType="html" label="MapPartitionsRDD [95] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster234 { label="Exchange"; 96 [labelType="html" label="MapPartitionsRDD [96] showString at NativeMethodAccessorImpl.java:0"]; } } 88->89; 92->93; 89->94; 93->94; 94->95; 95->96; }

87,88

91,92

digraph G { subgraph clusterstage_55 { label="Stage 55"; subgraph cluster234 { label="Exchange"; 97 [labelType="html" label="ShuffledRowRDD [97] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster231 { label="WholeStageCodegen (7)"; 98 [labelType="html" label="MapPartitionsRDD [98] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster261 { label="mapPartitionsInternal"; 99 [labelType="html" label="MapPartitionsRDD [99] showString at NativeMethodAccessorImpl.java:0"]; } } 97->98; 98->99; }

96,97

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
55	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 12:38:46	63 ms	4/4

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2025/03/08 12:38:46

63 ms

4/4

Page:

Skipped Stages (3)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
54	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/200
53	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/2
52	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/28

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

Unknown

0/200

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

Unknown

0/2

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

Unknown

0/28

Page: