compat: Dask 2025.1 (#1390)

datashader/core.py

@@ -1287,7 +1287,11 @@ def _source_from_geopandas(self, source):
 
         with contextlib.suppress(ImportError):
             import dask_geopandas
-            if Version(dask_geopandas.__version__) >= Version("0.4.0"):
+
+            DGP_VERSION = Version(dask_geopandas.__version__).release
+            # Version 0.4.0 added support for dask_expr and 0.4.3 removed
+            # support for classic DataFrame
+            if (0, 4, 0) <= DGP_VERSION < (0, 4, 3):
                 from dask_geopandas.core import GeoDataFrame as gdf1
                 dfs.append(gdf1)
 

datashader/data_libraries/dask.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-from contextlib import suppress
 
 import numpy as np
 import pandas as pd
@@ -51,13 +50,7 @@ def dask_pipeline(df, schema, canvas, glyph, summary, *, antialias=False, cuda=F
     return scheduler(dsk, name)
 
 
-# Classic Dask.DataFrame
-bypixel.pipeline.register(dd.core.DataFrame)(dask_pipeline)
-
-with suppress(ImportError):
-    import dask_expr
-
-    bypixel.pipeline.register(dask_expr.DataFrame)(dask_pipeline)
+bypixel.pipeline.register(dd.DataFrame)(dask_pipeline)
 
 
 def shape_bounds_st_and_axis(df, canvas, glyph):

datashader/tests/utils.py

@@ -2,16 +2,50 @@
 from contextlib import contextmanager
 from importlib import reload
 from importlib.util import find_spec
+from contextlib import suppress
+from functools import lru_cache
 
 import pytest
+from packaging.version import Version
 
-__all__ = ("dask_switcher", "DASK_UNAVAILABLE", "EXPR_UNAVAILABLE", "dask_skip")
+__all__ = ("dask_switcher", "DASK_UNAVAILABLE", "dask_skip")
 
 DASK_UNAVAILABLE = find_spec("dask") is None
-EXPR_UNAVAILABLE = find_spec("dask_expr") is None
 
 dask_skip = pytest.mark.skipif(DASK_UNAVAILABLE, reason="dask is not available")
 
+
+@lru_cache
+def _dask_setup():
+    """
+    Set-up both dask dataframes, using lru_cahce to only do it once
+
+    """
+    import dask
+    from datashader.data_libraries.dask import bypixel, dask_pipeline
+
+    classic, expr = False, False
+
+    # Removed in Dask 2025.1, and will raise AttributeError
+    if Version(dask.__version__).release < (2025, 1, 0):
+        import dask.dataframe as dd
+
+        bypixel.pipeline.register(dd.core.DataFrame)(dask_pipeline)
+        classic = True
+    else:
+        # dask_expr import below will now fail with:
+        # cannot import name '_Frame' from 'dask.dataframe.core'
+        expr = True
+
+    with suppress(ImportError):
+        import dask_expr
+
+        bypixel.pipeline.register(dask_expr.DataFrame)(dask_pipeline)
+        expr = True
+
+    return classic, expr
+
+
 @contextmanager
 def dask_switcher(*, query=False, extras=None):
     """
@@ -22,7 +56,12 @@ def dask_switcher(*, query=False, extras=None):
     """
     if DASK_UNAVAILABLE:
         pytest.skip("dask is not available")
-    if query and EXPR_UNAVAILABLE:
+
+    classic, expr = _dask_setup()
+
+    if not query and not classic:
+        pytest.skip("Classic DataFrame no longer supported by dask")
+    if query and not expr:
         pytest.skip("dask-expr is not available")
 
     import dask

examples/conftest.py

@@ -1,4 +1,5 @@
 from importlib.util import find_spec
+from packaging.version import Version
 
 collect_ignore_glob = [
     "tiling.ipynb",
@@ -14,3 +15,22 @@
         "user_guide/7_Networks.ipynb",
         "user_guide/8_Polygons.ipynb",
     ]
+
+if find_spec("dask") is not None:
+    import dask
+
+    # Spatialpandas does not support dask-expr, which is
+    # only available from this version.
+    if Version(dask.__version__).release >= (2025, 1, 0):
+        collect_ignore_glob += [
+            "user_guide/8_Polygons.ipynb",
+        ]
+
+
+def pytest_sessionfinish(session, exitstatus):
+    # Can be removed when spatialpandas work with dask-expr
+
+    from pytest import ExitCode
+
+    if exitstatus == ExitCode.NO_TESTS_COLLECTED:
+        session.exitstatus = ExitCode.OK

pixi.toml

@@ -22,7 +22,8 @@ build = ["py311", "build"]
 lint = ["py311", "lint"]
 
 [dependencies]
-numba = "*"  # At top as this dictates Python version
+python = "<3.13"
+numba = "*" # At top as this dictates Python version
 colorcet = "*"
 multipledispatch = "*"
 numpy = "*"
@@ -67,10 +68,10 @@ holoviews = "*"
 matplotlib-base = ">=3.3"
 networkx = "*"
 panel = ">1.1"
-pyogrio =  "*"
+pyogrio = "*"
 python-graphviz = "*"
 python-snappy = "*"
-rasterio = "!=1.4.0"  # 2024-11: Errors and will not solve to latest on Linux + Python 3.9
+rasterio = "!=1.4.0" # 2024-11: Errors and will not solve to latest on Linux + Python 3.9
 scikit-image = "*"
 shapely = ">=2.0.0"
 spatialpandas = "*"
@@ -100,8 +101,8 @@ geodatasets = "*"
 geopandas-base = "*"
 netcdf4 = "*"
 pyarrow = "*"
-pyogrio =  "*"
-rasterio = "!=1.4.0"  # 2024-11: Errors and will not solve to latest on Linux + Python 3.9
+pyogrio = "*"
+rasterio = "!=1.4.0" # 2024-11: Errors and will not solve to latest on Linux + Python 3.9
 rioxarray = "*"
 scikit-image = "*"
 shapely = ">=2.0.0"
@@ -111,7 +112,9 @@ spatialpandas = "*"
 nbval = "*"
 
 [feature.test-example.tasks]
-test-example = { cmd = 'pytest -n logical --dist loadscope --nbval-lax examples --benchmark-skip', env = { DASK_DATAFRAME__QUERY_PLANNING = "False" } }
+_test-example-1 = 'pytest -n logical --dist loadscope --nbval-lax examples --benchmark-skip -k "not Polygon"'
+_test-example-2 = { cmd = 'pytest -n logical --dist loadscope --nbval-lax examples --benchmark-skip -k "Polygon"', env = { DASK_DATAFRAME__QUERY_PLANNING = "False" } }
+test-example = { depends-on = ["_test-example-1", "_test-example-2"] }
 
 [feature.test-gpu]
 channels = ["rapidsai"]
@@ -141,6 +144,7 @@ nbsite = ">=0.8.4,<0.9.0"
 numpydoc = "*"
 sphinxcontrib-mermaid = "*"
 sphinx-reredirects = "*"
+dask-core = "<2025.1" # Spatialpandas does not currently support dask-expr
 
 [feature.doc.activation.env]
 DASK_DATAFRAME__QUERY_PLANNING = "False"
@@ -165,7 +169,6 @@ build-pip = 'python -m build .'
 
 [feature.build.activation.env]
 MPLBACKEND = "Agg"
-DASK_DATAFRAME__QUERY_PLANNING = "False"
 
 # =============================================
 # =================== LINT ====================