refactor/pyspark4.1: Upgrade Spark Docker Image and subproject to 4.1.1 (#175)

* Updates Spark image to `apache/spark:4.1.1`
* Updates compose and config filenames to match version (4.1)
* Bumps pyspark dependency on `pyspark-pipeline` to 4.1.1 to match Spark Cluster

Author: iobruno

…essing/compose.spark-4.0-standalone.yaml …essing/compose.spark-4.1-standalone.yamlmodule5-batch-processing/compose.spark-4.0-standalone.yaml renamed to module5-batch-processing/compose.spark-4.1-standalone.yaml module5-batch-processing/compose.spark-4.0-standalone.yaml renamed to module5-batch-processing/compose.spark-4.1-standalone.yaml

@@ -1,4 +1,4 @@
-x-spark-image: &spark-image spark:${SPARK_VERSION:-4.0.1-scala2.13-java21-python3-ubuntu}
+x-spark-image: &spark-image apache/spark:${SPARK_VERSION:-4.1.1-scala2.13-java21-python3-ubuntu}
 x-hive-image:  &hive-image  apache/hive:${HIVE_VERSION:-4.2.0}
 x-postgres-image: &postgres-image postgres:${POSTGRES_VERSION:-18.1-alpine}
 
@@ -13,7 +13,7 @@ x-spark-common:
   volumes:
     &spark-common-vol
     - ./logs/:/opt/spark/logs/
-    - ./spark-4.0-standalone.conf:/opt/spark/conf/spark-standalone.conf
+    - ./spark-4.1-standalone.conf:/opt/spark/conf/spark-standalone.conf
     - ~/.gcp/spark_credentials.json:/secrets/gcp_credentials.json
     - vol-spark-extra-jars:/opt/spark/extra-jars/
   depends_on:

module5-batch-processing/pyspark-4.x/README.md

@@ -1,6 +1,6 @@
 # Batch processing with PySpark 4.x
 
-![Python](https://img.shields.io/badge/Python-3.13_|_3.12-4B8BBE.svg?style=flat&logo=python&logoColor=FFD43B&labelColor=306998)
+![Python](https://img.shields.io/badge/Python-3.14_|_3.13_|_3.12-4B8BBE.svg?style=flat&logo=python&logoColor=FFD43B&labelColor=306998)
 [![PySpark](https://img.shields.io/badge/PySpark-4.x-262A38?style=flat-square&logo=apachespark&logoColor=E36B22&labelColor=262A38)](https://spark.apache.org/docs/4.0.2/api/python/user_guide)
 [![Hadoop](https://img.shields.io/badge/Hadoop-3.4.x-262A38?style=flat-square&logo=apachehadoop&logoColor=FDEE21&labelColor=262A38)](https://spark.apache.org/docs/4.0.2/api/python/user_guide)
 [![Scala](https://img.shields.io/badge/Scala-2.13-262A38?style=flat-square&logo=scala&logoColor=E03E3C&labelColor=262A38)](https://sdkman.io/usage/)

module5-batch-processing/pyspark-4.x/pyproject.toml

@@ -3,10 +3,10 @@ name = "pyspark-4.0-pipeline"
 version = "2.0"
 description = "Batch processing with PySpark 4"
 readme = "README.md"
-requires-python = ">=3.12,<3.14"
+requires-python = ">=3.12,<3.15"
 
 dependencies = [
-    "pyspark[connect]==4.0.1",
+    "pyspark[connect]==4.1.1",
     "pyarrow>=23.0.0,<24.0",
 ]