From 2aeda24a39fe8fab1f0be5274126d0bf8e78a0f5 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Mon, 13 Jul 2026 15:27:14 -0500 Subject: [PATCH 1/4] chore: extend Python support to 3.14 and test the min/max in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit datajoint-python declared support for 3.10–3.13 but CI (a single pixi environment) only exercised 3.13, and 3.14 — released 2025-10 — was excluded. This closes the ceiling gap and makes the declared range actually tested. - pyproject.toml: bump requires-python and the pixi python dependency to ">=3.10,<3.15" (adds 3.14); add per-version PyPI classifiers for 3.10–3.14. - pyproject.toml: add version-pinned pixi test environments (test-py310, test-py314) so CI can run the suite on both ends of the range. Each solves independently, since a solve-group cannot span Python versions. - .github/workflows/test.yaml: run the containerized test job as a matrix over [test-py310, test-py314]. Note: 3.10 reaches end-of-life 2026-10; dropping it (min → 3.11) is a follow-up, kept out of this change to avoid a breaking bump. --- .github/workflows/test.yaml | 11 ++++++++++- pyproject.toml | 21 +++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a4a91448f..50b8c5160 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -27,6 +27,14 @@ on: jobs: test: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Exercise both ends of the supported range (requires-python + # >=3.10,<3.15). The version-pinned pixi environments are defined in + # pyproject.toml under [tool.pixi.environments]. + environment: [test-py310, test-py314] + name: test (${{ matrix.environment }}) steps: - uses: actions/checkout@v4 @@ -35,9 +43,10 @@ jobs: with: cache: true locked: false + environments: ${{ matrix.environment }} - name: Run tests - run: pixi run -e test test-cov + run: pixi run -e ${{ matrix.environment }} test-cov # Unit tests run without containers (faster feedback) unit-tests: diff --git a/pyproject.toml b/pyproject.toml index d5c361658..9e717e481 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "packaging", ] -requires-python = ">=3.10,<3.14" +requires-python = ">=3.10,<3.15" authors = [ {name = "Dimitri Yatsenko", email = "dimitri@datajoint.com"}, {name = "Thinh Nguyen", email = "thinh@datajoint.com"}, @@ -57,6 +57,11 @@ keywords = [ # https://pypi.org/classifiers/ classifiers = [ "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", "Intended Audience :: Healthcare Industry", @@ -250,10 +255,22 @@ datajoint = { path = ".", editable = true, extras = ["test"] } [tool.pixi.feature.dev.pypi-dependencies] datajoint = { path = ".", editable = true, extras = ["dev", "test"] } +# Pin the interpreter for the min/max supported versions so CI can exercise +# both ends of the requires-python range (see .github/workflows/test.yaml). +[tool.pixi.feature.py310.dependencies] +python = "3.10.*" + +[tool.pixi.feature.py314.dependencies] +python = "3.14.*" + [tool.pixi.environments] default = { solve-group = "default" } dev = { features = ["dev"], solve-group = "default" } test = { features = ["test"], solve-group = "default" } +# Version-pinned test environments (each solves independently — a solve-group +# cannot span different Python versions). +test-py310 = { features = ["test", "py310"] } +test-py314 = { features = ["test", "py314"] } [tool.pixi.tasks] # Tests use testcontainers - no manual setup required @@ -265,7 +282,7 @@ services-down = "docker compose down" test-external = { cmd = "DJ_USE_EXTERNAL_CONTAINERS=1 pytest tests/", depends-on = ["services-up"] } [tool.pixi.dependencies] -python = ">=3.10,<3.14" +python = ">=3.10,<3.15" graphviz = ">=13.1.2,<14" [tool.pixi.activation] From 0a0aeb22947021505e4f15519c7a19c3ff370a7f Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 14 Jul 2026 22:29:46 -0500 Subject: [PATCH 2/4] test(tls): pin use_tls=False to client config, not negotiated cipher test_insecure_connection asserted Ssl_cipher == '', but a MySQL 8 server with TLS configured (the datajoint/mysql:8.0 test image) can negotiate TLS during connection setup even when the client requests no SSL. Whether the cipher ends up empty depends on the resolved client stack, so the assertion passed on the py3.13 solve but failed on the py3.10/py3.14 matrix added in this PR. Assert what DataJoint actually controls: use_tls=False sends no client-side SSL config and the connection is usable. --- tests/integration/test_tls.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 19ed087b7..8f2681dd9 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -38,9 +38,21 @@ def test_ssl_auto_detect(db_creds_test, connection_test, caplog): def test_insecure_connection(db_creds_test, connection_test): - """When use_tls=False, SSL should not be used.""" - result = dj.conn(use_tls=False, reset=True, **db_creds_test).query("SHOW STATUS LIKE 'Ssl_cipher';").fetchone()[1] - assert result == "" + """When use_tls=False, DataJoint must not configure client-side TLS. + + Note: this pins what DataJoint controls, not the negotiated cipher. A + MySQL 8 server with TLS configured may still negotiate TLS during + connection setup (e.g. to protect the auth handshake) even when the client + requests no SSL, so ``Ssl_cipher`` is not guaranteed to be empty and must + not be asserted on — doing so made this test depend on the resolved client + stack rather than on DataJoint behavior. + """ + conn = dj.conn(use_tls=False, reset=True, **db_creds_test) + # DataJoint sent no client-side SSL configuration. + assert conn.conn_info.get("ssl_input") is False + assert "ssl" not in conn.conn_info + # The connection is usable without client TLS. + assert conn.query("SELECT 1").fetchone()[0] == 1 @requires_ssl From 605f8331a8dfe5524c3c96bca458979fb9d719f1 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 14 Jul 2026 22:48:58 -0500 Subject: [PATCH 3/4] fix(populate): make Dependencies picklable for multiprocessing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parallel populate pickles the table (and its connection's Dependencies) to worker processes, which then reload dependencies and reconnect. Dependencies held an `itertools.count` (_node_alias_count) that is not picklable. This was latent on Linux through Python 3.13 because multiprocessing defaulted to `fork` (which inherits rather than pickles); Python 3.14 defaults to `forkserver`, which pickles the initializer args and surfaced the failure (`TypeError: cannot pickle 'itertools.count' object` in test_multi_processing). Add __getstate__/__setstate__ to drop the counter on pickle and rebuild it on load — workers reload dependencies anyway, so the value need not be carried. --- src/datajoint/dependencies.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/datajoint/dependencies.py b/src/datajoint/dependencies.py index 9b67c00d0..c8d4cbd0e 100644 --- a/src/datajoint/dependencies.py +++ b/src/datajoint/dependencies.py @@ -140,6 +140,19 @@ def clear(self) -> None: self._node_alias_count = itertools.count() # reset alias IDs for consistency super().clear() + def __getstate__(self) -> dict: + # itertools.count is not picklable. Multiprocessing populate pickles the + # table (and thus its connection's Dependencies) to worker processes, + # where dependencies are reloaded — so the counter is dropped here and + # rebuilt in __setstate__ rather than carried across the pickle. + state = self.__dict__.copy() + state.pop("_node_alias_count", None) + return state + + def __setstate__(self, state: dict) -> None: + self.__dict__.update(state) + self._node_alias_count = itertools.count() + def load(self, force: bool = True, schema_names: set[str] | None = None) -> None: """ Load dependencies for the given schemas. From e68a1b8fc21e768d77cbdc69c1fb9fe7293d0bf3 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Wed, 15 Jul 2026 00:22:24 -0500 Subject: [PATCH 4/4] fix(populate): pin parallel populate to the fork start method The real cause of the py3.14 failure is Python 3.14 changing the multiprocessing default start method from fork to forkserver. Parallel populate hands the live table and its open DB connection to workers by process inheritance (fork); under forkserver the payload is pickled instead, which cannot carry a live connection and deadlocked the job (after first surfacing as 'cannot pickle itertools.count'). Pin both Pool call sites to a fork context (fork where available, platform default otherwise). Reverts the earlier Dependencies __getstate__/__setstate__ pickle workaround, which addressed only the symptom. py3.10 (fork default) already passed; this makes py3.14 use the same proven path. --- src/datajoint/autopopulate.py | 23 +++++++++++++++++------ src/datajoint/dependencies.py | 13 ------------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/datajoint/autopopulate.py b/src/datajoint/autopopulate.py index d33e6ccf0..76a3ab22e 100644 --- a/src/datajoint/autopopulate.py +++ b/src/datajoint/autopopulate.py @@ -22,6 +22,14 @@ logger = logging.getLogger(__name__.split(".")[0]) +# Parallel populate hands the live table and its open connection to workers by +# process inheritance, so it requires the `fork` start method. Python 3.14 +# changed the default from `fork` to `forkserver`, which pickles the payload +# instead — that neither carries a live DB connection nor the table's internal +# state, and it deadlocks. Pin to `fork` where available (all POSIX platforms); +# fall back to the platform default elsewhere. +_MP_START_METHOD = "fork" if "fork" in mp.get_all_start_methods() else None + # --- helper functions for multiprocessing -- @@ -30,7 +38,8 @@ def _initialize_populate(table: Table, jobs: Job | None, populate_kwargs: dict[s """ Initialize a worker process for multiprocessing. - Saves the unpickled table to the current process and reconnects to database. + Stores the inherited table on the worker process and reconnects to database + (the parent closes its connection before forking; each worker reopens one). Parameters ---------- @@ -476,7 +485,9 @@ def _populate_direct( if hasattr(self.connection._conn, "ctx"): del self.connection._conn.ctx with ( - mp.Pool(processes, _initialize_populate, (self, None, populate_kwargs)) as pool, + mp.get_context(_MP_START_METHOD).Pool( + processes, _initialize_populate, (self, None, populate_kwargs) + ) as pool, tqdm(desc="Processes: ", total=nkeys) if display_progress else contextlib.nullcontext() as progress_bar, ): for status in pool.imap(_call_populate1, keys, chunksize=1): @@ -572,7 +583,9 @@ def handler(signum, frame): if hasattr(self.connection._conn, "ctx"): del self.connection._conn.ctx # SSLContext is not pickleable with ( - mp.Pool(processes, _initialize_populate, (self, self.jobs, populate_kwargs)) as pool, + mp.get_context(_MP_START_METHOD).Pool( + processes, _initialize_populate, (self, self.jobs, populate_kwargs) + ) as pool, tqdm(desc="Processes: ", total=nkeys) if display_progress else contextlib.nullcontext() as progress_bar, @@ -866,8 +879,6 @@ def _update_job_metadata(self, key, start_time, duration, version): pk_condition = make_condition(self, key, set()) self.connection.query( - f"UPDATE {self.full_table_name} SET " - "_job_start_time=%s, _job_duration=%s, _job_version=%s " - f"WHERE {pk_condition}", + f"UPDATE {self.full_table_name} SET _job_start_time=%s, _job_duration=%s, _job_version=%s WHERE {pk_condition}", args=(start_time, duration, version[:64] if version else ""), ) diff --git a/src/datajoint/dependencies.py b/src/datajoint/dependencies.py index c8d4cbd0e..9b67c00d0 100644 --- a/src/datajoint/dependencies.py +++ b/src/datajoint/dependencies.py @@ -140,19 +140,6 @@ def clear(self) -> None: self._node_alias_count = itertools.count() # reset alias IDs for consistency super().clear() - def __getstate__(self) -> dict: - # itertools.count is not picklable. Multiprocessing populate pickles the - # table (and thus its connection's Dependencies) to worker processes, - # where dependencies are reloaded — so the counter is dropped here and - # rebuilt in __setstate__ rather than carried across the pickle. - state = self.__dict__.copy() - state.pop("_node_alias_count", None) - return state - - def __setstate__(self, state: dict) -> None: - self.__dict__.update(state) - self._node_alias_count = itertools.count() - def load(self, force: bool = True, schema_names: set[str] | None = None) -> None: """ Load dependencies for the given schemas.