Start using spark4-preview versions

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 2024-10-22
+
+Affected: `pyspark-notebook` and `all-spark-notebook` images users
+
+- **Breaking:** Start using Spark 4.0.0 preview versions ([#2159](https://github.com/jupyter/docker-stacks/pull/2159)).
+
+  Reason: Spark v3 is not compatible with Python 3.12, and [the voting group has decided](https://github.com/jupyter/docker-stacks/pull/2072#issuecomment-2414123851) to switch to Spark v4 preview version.
+
 ## 2024-10-09
 
 Affected: users building a custom set of images

images/pyspark-notebook/Dockerfile

@@ -63,7 +63,7 @@ USER ${NB_UID}
 RUN mamba install --yes \
     'grpcio-status' \
     'grpcio' \
-    'pandas=2.0.3' \
+    'pandas=2.2.2' \
     'pyarrow' && \
     mamba clean --all -f -y && \
     fix-permissions "${CONDA_DIR}" && \

images/pyspark-notebook/setup_spark.py

@@ -36,7 +36,7 @@ def get_latest_spark_version() -> str:
     stable_versions = [
         ref.removeprefix("spark-").removesuffix("/")
         for ref in all_refs
-        if ref.startswith("spark-") and "incubating" not in ref and "preview" not in ref
+        if ref.startswith("spark-") and "incubating" not in ref
     ]
 
     # Compare versions semantically

tests/pyspark-notebook/test_spark.py

@@ -3,12 +3,20 @@
 import logging
 
 from tests.conftest import TrackedContainer
-from tests.run_command import run_command
 
 LOGGER = logging.getLogger(__name__)
 
 
 def test_spark_shell(container: TrackedContainer) -> None:
     """Checking if Spark (spark-shell) is running properly"""
-    logs = run_command(container, 'spark-shell <<< "1+1"', timeout=60)
+    logs = container.run_and_wait(
+        timeout=60,
+        no_warnings=False,
+        tty=True,
+        command=["bash", "-c", 'spark-shell <<< "1+1"'],
+    )
+    warnings = TrackedContainer.get_warnings(logs)
+    assert len(warnings) == 1
+    assert "Using incubator modules: jdk.incubator.vector" in warnings[0]
+
     assert "res0: Int = 2" in logs, "spark-shell does not work"

tests/pyspark-notebook/units/unit_pandas_version.py

@@ -2,4 +2,4 @@
 # Distributed under the terms of the Modified BSD License.
 import pandas
 
-assert pandas.__version__ == "2.0.3"
+assert pandas.__version__ == "2.2.2"