Environment

Runtime Information

NameValue
Java Home/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.332.b09-2.el8_6.x86_64/jre
Java Version1.8.0_332 (Red Hat, Inc.)
Scala Versionversion 2.12.10

Spark Properties

NameValue
spark.app.idspark-d9f6760a13d04a009667445dd0635c0e
spark.app.nameNasaLogSparkStreaming
spark.app.submitTime1735309940546
spark.driver.bindAddress10.133.106.15
spark.driver.blockManager.port7079
spark.driver.cores1
spark.driver.extraJavaOptions-Divy.cache.dir=/tmp -Divy.home=/tmp
spark.driver.hosttask4-starter-kit-76b7df94088857b4-driver-svc.data-science-ec21637.svc
spark.driver.memory4g
spark.driver.port7078
spark.dynamicAllocation.enabledfalse
spark.dynamicAllocation.executorAllocationRatio0.5
spark.dynamicAllocation.initialExecutors2
spark.dynamicAllocation.maxExecutors24
spark.dynamicAllocation.minExecutors2
spark.dynamicAllocation.shuffleTracking.enabledtrue
spark.eventLog.dirs3a://spark-hs-bkt-89306845-2a18-47bf-bc82-302765918961/logs-dir/
spark.eventLog.enabledtrue
spark.executor.cores1
spark.executor.iddriver
spark.executor.instances2
spark.executor.memory4g
spark.hadoop.fs.s3a.access.key92GGC717PSCSTE1T6D4Q
spark.hadoop.fs.s3a.aws.credentials.providerorg.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.bucket.spark-hs-bkt-89306845-2a18-47bf-bc82-302765918961.access.keyHKZIJEUXSN0FMYEEXGQP
spark.hadoop.fs.s3a.bucket.spark-hs-bkt-89306845-2a18-47bf-bc82-302765918961.connection.ssl.enabledfalse
spark.hadoop.fs.s3a.bucket.spark-hs-bkt-89306845-2a18-47bf-bc82-302765918961.endpointrook-ceph-rgw-ceph-storage-object.rook-ceph.svc:8080
spark.hadoop.fs.s3a.bucket.spark-hs-bkt-89306845-2a18-47bf-bc82-302765918961.path.style.accesstrue
spark.hadoop.fs.s3a.bucket.spark-hs-bkt-89306845-2a18-47bf-bc82-302765918961.secret.key*********(redacted)
spark.hadoop.fs.s3a.connection.ssl.enabledfalse
spark.hadoop.fs.s3a.endpointhttps://ceph-a-object-rgw.comp-teach.qmul.ac.uk
spark.hadoop.fs.s3a.fast.uploadtrue
spark.hadoop.fs.s3a.implorg.apache.hadoop.fs.s3a.S3AFileSystem
spark.hadoop.fs.s3a.path.style.accesstrue
spark.hadoop.fs.s3a.secret.key*********(redacted)
spark.jarslocal:///opt/spark/jars/graphframes-0.8.2-spark3.0-s_2.12.jar
spark.jars.packagesorg.apache.hadoop:hadoop-aws:3.2.2
spark.kubernetes.authenticate.driver.serviceAccountNamespark
spark.kubernetes.container.imageimage-registry.openshift-image-registry.svc:5000/spark-images/pyspark-odh-boto3@sha256:69b91b09212ff7ab3fbc31032e6842d4984e3add4021e4ff68fea99f82d52bda
spark.kubernetes.container.image.pullPolicyAlways
spark.kubernetes.driver.pod.featureStepsorg.apache.spark.deploy.k8s.features.VolcanoFeatureStep
spark.kubernetes.driver.pod.nametask4-starter-kit-76b7df94088857b4-driver
spark.kubernetes.driver.secretKeyRef.AWS_ACCESS_KEY_ID*********(redacted)
spark.kubernetes.driver.secretKeyRef.AWS_SECRET_ACCESS_KEY*********(redacted)
spark.kubernetes.driverEnv.BUCKET_NAMEobject-bucket-ec21637-6b4850ab-f469-4932-8127-1573e74313be
spark.kubernetes.driverEnv.BUCKET_PORT443
spark.kubernetes.driverEnv.DATA_REPOSITORY_BUCKETdata-repository-bkt
spark.kubernetes.driverEnv.S3_ENDPOINT_URLhttps://ceph-a-object-rgw.comp-teach.qmul.ac.uk
spark.kubernetes.driverEnv.SPARK_USERec21637
spark.kubernetes.driverEnv.STREAMING_SERVERstream-emulator.data-science-tools.svc.cluster.local
spark.kubernetes.driverEnv.STREAMING_SERVER_PORT5551
spark.kubernetes.executor.pod.featureStepsorg.apache.spark.deploy.k8s.features.VolcanoFeatureStep
spark.kubernetes.executor.podNamePrefixnasalogsparkstreaming-e9905a9408888c17
spark.kubernetes.executor.secretKeyRef.AWS_ACCESS_KEY_ID*********(redacted)
spark.kubernetes.executor.secretKeyRef.AWS_SECRET_ACCESS_KEY*********(redacted)
spark.kubernetes.executorEnv.BUCKET_NAMEobject-bucket-ec21637-6b4850ab-f469-4932-8127-1573e74313be
spark.kubernetes.executorEnv.BUCKET_PORT443
spark.kubernetes.executorEnv.DATA_REPOSITORY_BUCKETdata-repository-bkt
spark.kubernetes.executorEnv.S3_ENDPOINT_URLhttps://ceph-a-object-rgw.comp-teach.qmul.ac.uk
spark.kubernetes.executorEnv.SPARK_USERec21637
spark.kubernetes.executorEnv.STREAMING_SERVERstream-emulator.data-science-tools.svc.cluster.local
spark.kubernetes.executorEnv.STREAMING_SERVER_PORT5551
spark.kubernetes.file.upload.paths3a://object-bucket-ec21637-6b4850ab-f469-4932-8127-1573e74313be/spark-hs
spark.kubernetes.memoryOverheadFactor0.4
spark.kubernetes.namespacedata-science-ec21637
spark.kubernetes.resource.typepython
spark.kubernetes.scheduler.namevolcano
spark.kubernetes.scheduler.volcano.podGroupTemplateFile/opt/spark/podgroup-template-volcano.yaml
spark.kubernetes.submitInDrivertrue
spark.masterk8s://https://api.a.comp-teach.qmul.ac.uk:6443
spark.metrics.appStatusSource.enabledtrue
spark.metrics.conf.*.source.jvm.classorg.apache.spark.metrics.source.JvmSource
spark.rdd.compressTrue
spark.repl.local.jarslocal:///opt/spark/jars/graphframes-0.8.2-spark3.0-s_2.12.jar
spark.scheduler.modeFIFO
spark.serializer.objectStreamReset100
spark.submit.deployModeclient
spark.submit.pyFiles

Hadoop Properties

System Properties

Classpath Entries