movieratings - Details for Job 16

Details for Job 16

Status: SUCCEEDED
Completed Stages: 5

Enable zooming

digraph G { subgraph clusterstage_44 { label="Stage 44"; 3 [labelType="html" label="PythonRDD [3] RDD at PythonRDD.scala:53"]; subgraph cluster0 { label="textFile"; 0 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/ratings.csv [0] textFile at NativeMethodAccessorImpl.java:0"]; 1 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/ratings.csv [1] textFile at NativeMethodAccessorImpl.java:0"]; } subgraph cluster4 { label="mapPartitions"; 4 [labelType="html" label="MapPartitionsRDD [4] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster5 { label="map"; 5 [labelType="html" label="MapPartitionsRDD [5] map at SerDeUtil.scala:137"]; } subgraph cluster6 { label="mapPartitions"; 6 [labelType="html" label="MapPartitionsRDD [6] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster284 { label="WholeStageCodegen (1)"; 103 [labelType="html" label="MapPartitionsRDD [103] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster283 { label="Exchange"; 107 [labelType="html" label="MapPartitionsRDD [107] javaToPython at NativeMethodAccessorImpl.java:0"]; } } 0->1; 1->3; 3->4; 4->5; 5->6; 6->103; 103->107; }

digraph G { subgraph clusterstage_45 { label="Stage 45"; 56 [labelType="html" label="PythonRDD [56] RDD at PythonRDD.scala:53"]; subgraph cluster130 { label="textFile"; 53 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/movies.csv [53] textFile at NativeMethodAccessorImpl.java:0"]; 54 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/movies.csv [54] textFile at NativeMethodAccessorImpl.java:0"]; } subgraph cluster134 { label="mapPartitions"; 57 [labelType="html" label="MapPartitionsRDD [57] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster135 { label="map"; 58 [labelType="html" label="MapPartitionsRDD [58] map at SerDeUtil.scala:137"]; } subgraph cluster136 { label="mapPartitions"; 59 [labelType="html" label="MapPartitionsRDD [59] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster298 { label="WholeStageCodegen (4)"; 113 [labelType="html" label="MapPartitionsRDD [113] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster297 { label="Exchange"; 114 [labelType="html" label="MapPartitionsRDD [114] javaToPython at NativeMethodAccessorImpl.java:0"]; } } 53->54; 54->56; 56->57; 57->58; 58->59; 59->113; 113->114; }

digraph G { subgraph clusterstage_46 { label="Stage 46"; subgraph cluster283 { label="Exchange"; 108 [labelType="html" label="ShuffledRowRDD [108] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster278 { label="WholeStageCodegen (2)"; 109 [labelType="html" label="MapPartitionsRDD [109] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster277 { label="Exchange"; 110 [labelType="html" label="MapPartitionsRDD [110] javaToPython at NativeMethodAccessorImpl.java:0"]; } } 108->109; 109->110; }

107,108

digraph G { subgraph clusterstage_47 { label="Stage 47"; subgraph cluster277 { label="Exchange"; 111 [labelType="html" label="ShuffledRowRDD [111] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster273 { label="WholeStageCodegen (3)"; 112 [labelType="html" label="MapPartitionsRDD [112] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster297 { label="Exchange"; 115 [labelType="html" label="ShuffledRowRDD [115] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster294 { label="WholeStageCodegen (5)"; 116 [labelType="html" label="MapPartitionsRDD [116] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster269 { label="WholeStageCodegen (6)"; 117 [labelType="html" label="ZippedPartitionsRDD2 [117] javaToPython at NativeMethodAccessorImpl.java:0"]; 118 [labelType="html" label="MapPartitionsRDD [118] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster268 { label="TakeOrderedAndProject"; 119 [labelType="html" label="MapPartitionsRDD [119] javaToPython at NativeMethodAccessorImpl.java:0"]; 120 [labelType="html" label="MapPartitionsRDD [120] javaToPython at NativeMethodAccessorImpl.java:0"]; 121 [labelType="html" label="MapPartitionsRDD [121] javaToPython at NativeMethodAccessorImpl.java:0"]; } } 111->112; 115->116; 112->117; 116->117; 117->118; 118->119; 119->120; 120->121; }

110,111

114,115

digraph G { subgraph clusterstage_48 { label="Stage 48"; 124 [labelType="html" label="SQLExecutionRDD [124] javaToPython at NativeMethodAccessorImpl.java:0"]; 127 [labelType="html" label="PythonRDD [127] collect at /tmp/spark-fbc0b603-1a1f-4a06-9cf5-9a35ac12a849/taskb.py:274"]; subgraph cluster268 { label="TakeOrderedAndProject"; 122 [labelType="html" label="ShuffledRowRDD [122] javaToPython at NativeMethodAccessorImpl.java:0"]; 123 [labelType="html" label="MapPartitionsRDD [123] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster301 { label="map"; 125 [labelType="html" label="MapPartitionsRDD [125] javaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster302 { label="mapPartitions"; 126 [labelType="html" label="MapPartitionsRDD [126] javaToPython at NativeMethodAccessorImpl.java:0"]; } } 122->123; 123->124; 124->125; 125->126; 126->127; }

121,122

Completed Stages (5)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
48	collect at /tmp/spark-fbc0b603-1a1f-4a06-9cf5-9a35ac12a849/taskb.py:274 +details org.apache.spark.rdd.RDD.collect(RDD.scala:1003) org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:168) org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 13:56:15	72 ms	1/1		126.0 KiB
47	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3441) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 13:56:13	2 s	200/200		28.9 MiB	126.0 KiB
46	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3441) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 13:56:02	10 s	200/200		301.1 MiB	25.1 MiB
45	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3441) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 13:54:45	0.5 s	2/2	4.0 MiB		3.8 MiB
44	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3441) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 13:54:45	1.3 min	28/28	890.8 MiB		301.1 MiB

Page: