movieratings - Details for Job 8

Details for Job 8

Status: SUCCEEDED
Associated SQL Query: 2
Completed Stages: 4

Enable zooming

digraph G { subgraph clusterstage_17 { label="Stage 17"; 3 [labelType="html" label="PythonRDD [3] RDD at PythonRDD.scala:53"]; subgraph cluster0 { label="textFile"; 0 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/ratings.csv [0] textFile at NativeMethodAccessorImpl.java:0"]; 1 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/ratings.csv [1] textFile at NativeMethodAccessorImpl.java:0"]; } subgraph cluster4 { label="mapPartitions"; 4 [labelType="html" label="MapPartitionsRDD [4] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster5 { label="map"; 5 [labelType="html" label="MapPartitionsRDD [5] map at SerDeUtil.scala:137"]; } subgraph cluster6 { label="mapPartitions"; 6 [labelType="html" label="MapPartitionsRDD [6] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster163 { label="WholeStageCodegen (4)"; 66 [labelType="html" label="MapPartitionsRDD [66] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster162 { label="Exchange"; 67 [labelType="html" label="MapPartitionsRDD [67] showString at NativeMethodAccessorImpl.java:0"]; } } 0->1; 1->3; 3->4; 4->5; 5->6; 6->66; 66->67; }

digraph G { subgraph clusterstage_18 { label="Stage 18"; 56 [labelType="html" label="PythonRDD [56] RDD at PythonRDD.scala:53"]; subgraph cluster130 { label="textFile"; 53 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/movies.csv [53] textFile at NativeMethodAccessorImpl.java:0"]; 54 [labelType="html" label="s3a://data-repository-bkt/ECS765/MovieLens/movies.csv [54] textFile at NativeMethodAccessorImpl.java:0"]; } subgraph cluster134 { label="mapPartitions"; 57 [labelType="html" label="MapPartitionsRDD [57] mapPartitions at SerDeUtil.scala:184"]; } subgraph cluster135 { label="map"; 58 [labelType="html" label="MapPartitionsRDD [58] map at SerDeUtil.scala:137"]; } subgraph cluster136 { label="mapPartitions"; 59 [labelType="html" label="MapPartitionsRDD [59] applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0"]; } subgraph cluster154 { label="WholeStageCodegen (1)"; 60 [labelType="html" label="MapPartitionsRDD [60] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster153 { label="Generate"; 61 [labelType="html" label="MapPartitionsRDD [61] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster150 { label="WholeStageCodegen (2)"; 62 [labelType="html" label="MapPartitionsRDD [62] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster149 { label="Exchange"; 63 [labelType="html" label="MapPartitionsRDD [63] showString at NativeMethodAccessorImpl.java:0"]; } } 53->54; 54->56; 56->57; 57->58; 58->59; 59->60; 60->61; 61->62; 62->63; }

digraph G { subgraph clusterstage_19 { label="Stage 19"; subgraph cluster149 { label="Exchange"; 64 [labelType="html" label="ShuffledRowRDD [64] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster146 { label="WholeStageCodegen (3)"; 65 [labelType="html" label="MapPartitionsRDD [65] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster162 { label="Exchange"; 68 [labelType="html" label="ShuffledRowRDD [68] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster159 { label="WholeStageCodegen (5)"; 69 [labelType="html" label="MapPartitionsRDD [69] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster141 { label="WholeStageCodegen (6)"; 70 [labelType="html" label="ZippedPartitionsRDD2 [70] showString at NativeMethodAccessorImpl.java:0"]; 71 [labelType="html" label="MapPartitionsRDD [71] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster140 { label="Exchange"; 72 [labelType="html" label="MapPartitionsRDD [72] showString at NativeMethodAccessorImpl.java:0"]; } } 64->65; 68->69; 65->70; 69->70; 70->71; 71->72; }

63,64

67,68

digraph G { subgraph clusterstage_20 { label="Stage 20"; subgraph cluster140 { label="Exchange"; 73 [labelType="html" label="ShuffledRowRDD [73] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster137 { label="WholeStageCodegen (7)"; 74 [labelType="html" label="MapPartitionsRDD [74] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster167 { label="mapPartitionsInternal"; 75 [labelType="html" label="MapPartitionsRDD [75] showString at NativeMethodAccessorImpl.java:0"]; } } 73->74; 74->75; }

72,73

Completed Stages (4)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
20	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 13:50:22	60 ms	1/1
19	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 13:50:01	20 s	200/200		118.2 MiB	257.2 KiB
18	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 13:48:43	1 s	2/2	4.0 MiB		1393.5 KiB
17	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2025/03/08 13:48:43	1.3 min	28/28	890.8 MiB		116.8 MiB

Page: