movieratings - Details for Job 41

DAG Visualization

digraph G { subgraph clusterstage_82 { label="Stage 82"; 7 [labelType="html" label="PythonRDD [7] RDD at PythonRDD.scala:53"]; subgraph cluster0 { label="textFile"; 0 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/ratings.csv [0] textFile at NativeMethodAccessorImpl.java:0"]; 1 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/ratings.csv [1] textFile at NativeMethodAccessorImpl.java:0"]; } subgraph cluster16 { label="mapPartitions"; 8 [labelType="html" label="MapPartitionsRDD [8] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster17 { label="map"; 9 [labelType="html" label="MapPartitionsRDD [9] map at SerDeUtil.scala:137"]; } subgraph cluster18 { label="mapPartitions"; 10 [labelType="html" label="MapPartitionsRDD [10] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster365 { label="WholeStageCodegen (1)"; 128 [labelType="html" label="MapPartitionsRDD [128] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster364 { label="Exchange"; 129 [labelType="html" label="MapPartitionsRDD [129] showString at NativeMethodAccessorImpl.java:0"]; } } 0->1; 1->7; 7->8; 8->9; 9->10; 10->128; 128->129; }

digraph G { subgraph clusterstage_83 { label="Stage 83"; subgraph cluster364 { label="Exchange"; 130 [labelType="html" label="ShuffledRowRDD [130] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster361 { label="WholeStageCodegen (2)"; 131 [labelType="html" label="MapPartitionsRDD [131] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster369 { label="map"; 132 [labelType="html" label="MapPartitionsRDD [132] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster370 { label="takeOrdered"; 133 [labelType="html" label="MapPartitionsRDD [133] showString at NativeMethodAccessorImpl.java:0"]; } } 130->131; 131->132; 132->133; }

129,130

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
83	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 12:43:59	0.9 s	200/200			4.4 MiB
82	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 12:42:45	1.2 min	28/28	890.8 MiB			4.4 MiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2025/03/08 12:43:59

0.9 s

200/200

4.4 MiB

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2025/03/08 12:42:45

1.2 min

28/28

890.8 MiB

4.4 MiB

Page:

Details for Job 41

Completed Stages (2)