From bfb2c4dd838b3802784c52a1a22d0c646cb7ae17 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Thu, 11 Jun 2026 15:27:32 +0530 Subject: [PATCH 01/44] =?UTF-8?q?UN-3534=20[FEAT]=20PG=20Queue=20Phase=208?= =?UTF-8?q?a=20=E2=80=94=20queue-transport=20routing=20gate=20+=20scaffold?= =?UTF-8?q?=20(#2033)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3534 [FEAT] PG Queue Phase 8a — queue-transport routing gate + scaffold Add the Strangler-Fig routing seam that lets PG Queue (PGMQ) coexist with Celery so task types can be migrated one at a time. Scaffold only: the PG branch is a Celery-routing stub (no PG consumer exists yet), so this is zero-behaviour-change by construction. - queue_backend/routing.py: QueueBackend{CELERY,PG} + select_backend(task_name) reading the WORKER_PG_QUEUE_ENABLED_TASKS allow-list (default empty -> all Celery). Tolerant CSV parsing; never raises. - dispatch(): consults select_backend(); PG-selected tasks are logged but still dispatched via Celery. The send_task call sits outside the PG branch so the wire is byte-identical regardless of the routing decision. - queue_backend/pg_queue/: scaffold subpackage. PGMQ is a core transport substrate, so it lives in the seam beside dispatch/routing/barrier, not under the git-ignored plugins/ cloud overlay. - sample.env: documents the flag (default-safe, OSS-friendly, no Flipt server). - tests: 12 routing tests incl. the byte-identical-dispatch characterisation pinning the inert-scaffold invariant. Barrier axis untouched (WORKER_BARRIER_BACKEND stays chord). Per-org routing intentionally deferred to the rollout phase. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3534 [FEAT] Address Phase 8a review feedback - test seam: drop stale WORKER_PG_QUEUE_ENABLED_ORGS reference — the org axis was removed, that flag never existed [must-fix] - observability: routing log DEBUG -> INFO so a cutover survives a default log config; log-once per task name bounds volume. Log the configured allow-list once per process so a typo'd task name is eyeballable at boot even when it never matches a real dispatch [important] - tests: pin the routing branch with caplog assertions (PG -> log fires, Celery -> no log, bounded to once) so the inert gate can't be silently deleted; assert allow-list logging too [important — closes test gap] - QueueBackend: document the is-not-== discipline (StrEnum makes a typo'd "== cellery" a silent False) [suggestion] - routing: drop dead _parse_allow_list(env_var) param, read the constant directly; one-pass strip; test imports the constant (single source of truth) [nits] - pg_queue docstring: clarify plugins/ subdirs are git-ignored while the dir itself is tracked; soften volatile labs branch/section/filename references [nits] Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3534 [DOCS] Note migration-coherence constraint in routing gate Document that the per-task allow-list may only split independent/leaf tasks across substrates. The coupled execution pipeline (async_execute_bin -> file processing -> callback, with the barrier fan-in) must run a single execution entirely on one transport — its migration unit is the execution, not the task. The next phase resolves transport once at kickoff and carries it in ExecutionContext; select_backend then honours that carried marker over the per-task env. Until then, only leaf tasks should be enabled here. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3534 [DOCS] Fix sample.env example to a leaf task (not pipeline) async_execute_bin is the pipeline kickoff — exactly the task the coherence note says must NOT be split per-task. Switch the example to a leaf task (send_webhook_notification) and warn against listing coupled pipeline tasks until ExecutionContext carries the transport choice. Addresses Greptile review feedback. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- workers/queue_backend/__init__.py | 12 ++ workers/queue_backend/dispatch.py | 31 ++++ workers/queue_backend/pg_queue/__init__.py | 24 +++ workers/queue_backend/routing.py | 134 ++++++++++++++ workers/sample.env | 28 +++ workers/tests/test_queue_backend_seam.py | 9 +- workers/tests/test_routing.py | 198 +++++++++++++++++++++ 7 files changed, 434 insertions(+), 2 deletions(-) create mode 100644 workers/queue_backend/pg_queue/__init__.py create mode 100644 workers/queue_backend/routing.py create mode 100644 workers/tests/test_routing.py diff --git a/workers/queue_backend/__init__.py b/workers/queue_backend/__init__.py index 3fcb1bab62..c4310f5eff 100644 --- a/workers/queue_backend/__init__.py +++ b/workers/queue_backend/__init__.py @@ -23,6 +23,15 @@ Unknown values raise — operators get a loud error at module import time rather than silently falling back, which would mask a typo'd flag in a production env. + +**Queue-transport routing (separate axis).** ``dispatch()`` consults +:func:`~queue_backend.routing.select_backend`, which reads the +PG-queue routing table (``WORKER_PG_QUEUE_ENABLED_TASKS``) to decide +Celery-vs-PG per task. This is orthogonal to the barrier choice above: +the barrier is *how a batch's fan-in fires the callback*; the transport +is *how messages travel*. Both default to Celery. The routing table is +a scaffold today — PG-selected tasks still ride Celery (no PG consumer +yet) — so it is observable but inert. """ import os @@ -37,6 +46,7 @@ barrier_abort, barrier_decr_and_check, ) +from .routing import QueueBackend, select_backend __all__ = [ "Barrier", @@ -44,11 +54,13 @@ "BarrierHandle", "CeleryChordBarrier", "FairnessKey", + "QueueBackend", "RedisDecrBarrier", "barrier_abort", "barrier_decr_and_check", "dispatch", "get_barrier", + "select_backend", "worker_task", ] diff --git a/workers/queue_backend/dispatch.py b/workers/queue_backend/dispatch.py index cf8d2c274e..e986ecb6f4 100644 --- a/workers/queue_backend/dispatch.py +++ b/workers/queue_backend/dispatch.py @@ -7,6 +7,7 @@ from __future__ import annotations +import logging from collections.abc import Mapping, Sequence from typing import Any @@ -14,6 +15,14 @@ from .fairness import FairnessKey from .handle import DispatchHandle +from .routing import QueueBackend, select_backend + +logger = logging.getLogger(__name__) + +# Task names already logged as PG-routed in this process. Bounds the +# routing log to once per task name (per prefork child) so an opted-in +# high-throughput task doesn't log on every dispatch. +_pg_routing_logged: set[str] = set() def dispatch( @@ -28,7 +37,29 @@ def dispatch( ``fairness`` is attached as the ``x-fairness-key`` header (not in kwargs). Pass ``None`` for non-workflow worker tasks. + + The transport is chosen by :func:`select_backend` from the PG-queue + routing table (``WORKER_PG_QUEUE_ENABLED_TASKS``). In this phase the + table is a scaffold: PG-selected tasks are *logged* but still + dispatched via Celery, because no PG consumer exists yet. The + ``QueueBackend.PG`` branch below is the seam where the real PG + enqueue lands in a later phase — keeping the ``send_task`` call + outside it guarantees today's wire is byte-identical regardless of + the routing decision. """ + if ( + select_backend(task_name) is QueueBackend.PG + and task_name not in _pg_routing_logged + ): + # INFO (not DEBUG) so the cutover is visible under a default log + # config; log-once per task name keeps the volume bounded. + _pg_routing_logged.add(task_name) + logger.info( + "PG-queue routing selected for task=%r; dispatching via " + "Celery (scaffold — no PG consumer yet)", + task_name, + ) + headers = fairness.as_header() if fairness is not None else None return current_app.send_task( task_name, diff --git a/workers/queue_backend/pg_queue/__init__.py b/workers/queue_backend/pg_queue/__init__.py new file mode 100644 index 0000000000..a85622cfe8 --- /dev/null +++ b/workers/queue_backend/pg_queue/__init__.py @@ -0,0 +1,24 @@ +"""PG Queue (PGMQ) transport substrate — scaffold. + +Reserved home for the PostgreSQL-backed queue substrate (PGMQ + +``SKIP LOCKED``) that will run alongside Celery during the Strangler-Fig +migration. PGMQ is a *core* worker transport — the intended primary +backend — so it lives inside the queue-backend seam next to its sibling +substrates (``dispatch``, ``routing``, ``barrier``, ``redis_barrier``), +**not** under ``workers/plugins/``, whose plugin *implementation +subdirectories* are the git-ignored overlay copied in at build time (the +directory itself — ``__init__.py``, ``plugin_manager.py`` — is tracked). + +A subpackage (rather than a single module like the barriers) because the +real implementation will likely span several modules (config, consumer +poll loop, orchestrator) — exact layout TBD. + +Empty by design in this phase. Routing decisions are made by +:func:`queue_backend.routing.select_backend`; until a consumer exists +here, PG-selected dispatches still ride Celery (see +``queue_backend.dispatch``). + +Design reference: the PG Queue implementation guide in the labs repo +(``workflow-execution-architecture``). Branch and section pointers move, +so they're tracked on UN-3534 / the PR rather than baked in here. +""" diff --git a/workers/queue_backend/routing.py b/workers/queue_backend/routing.py new file mode 100644 index 0000000000..abc43e8029 --- /dev/null +++ b/workers/queue_backend/routing.py @@ -0,0 +1,134 @@ +"""Queue-transport routing gate (Strangler-Fig). + +Decides whether a given dispatch should ride the **Celery** transport +(today's only real path) or the future **PG Queue** (PGMQ) transport, +based on a per-task-type opt-in allow-list read from env: + +- ``WORKER_PG_QUEUE_ENABLED_TASKS`` — comma-separated task names routed + to PG. Empty / unset → everything routes to Celery. + +**Why an allow-list, not a global boolean.** A single +``WORKER_QUEUE_BACKEND=pg`` switch would move *all* traffic at once — +the big-bang migration the PG Queue rollout explicitly forbids. The +allow-list lets an operator migrate one task type at a time +(drain-and-cutover) and roll back instantly by removing an entry. + +(Per-org granularity is intentionally out of scope here. When tenant-level +canarying is needed it slots in behind :func:`select_backend` as a small +additive change — no call site touches the routing decision directly.) + +**Migration coherence — what the allow-list may and may not split.** +Per-task routing is the right granularity for *independent / leaf* tasks +(e.g. ``process_log_history``, ``send_webhook_notification``): no chain, +no barrier, so one can ride PG while the rest stay on Celery — these are +the safe first migration candidates. It is **not** valid to split the +*coupled execution pipeline* (``async_execute_bin`` → file processing → +callback, with the barrier fan-in) across substrates: a chain hands off +stage→stage and the barrier coordinates a batch, so a single execution +must run **entirely on one transport**. The migration unit for the +pipeline is therefore the *execution*, not the task — the next phase will +resolve the transport once at kickoff and carry it in ``ExecutionContext`` +(like the fairness key), and downstream dispatches will honour that +carried choice over the per-task env. ``select_backend`` then becomes: +"if the dispatch carries an execution-transport marker, use it; otherwise +consult this allow-list." Until then, only enable *leaf* tasks here. + +**Scaffold posture.** This module only makes the routing *decision*. +In the current phase there is no PG consumer, so ``dispatch()`` still +sends PG-selected tasks via Celery (the decision is observable in logs +but inert). ``select_backend()`` is the seam where the real PG dispatch +lands in a later phase. + +**Observability.** Because the gate is silent-by-construction (a +misrouted task still runs on Celery), the only signals are logs, emitted +at INFO so they survive a default log config: the configured allow-list +is logged once per process (:func:`_log_allow_list_once`, so a typo is +visible even if it never matches a real task), and the first time each +task is routed to PG (``dispatch()``, log-once per task name). + +**Fail-safe parsing.** Unlike ``get_barrier()`` (which raises on an +unrecognised value, because a typo'd substrate must not silently fall +back), the routing table is a membership set with no "invalid value" +concept: blanks and stray whitespace are simply dropped. Malformed or +empty config can only ever resolve to the safe ``CELERY`` default — +``select_backend()`` never raises. +""" + +from __future__ import annotations + +import logging +import os +from enum import StrEnum + +logger = logging.getLogger(__name__) + +# Env var name kept as a module literal (mirrors how the sibling +# ``WORKER_BARRIER_BACKEND`` flag is read in ``queue_backend.__init__``) +# so the queue-backend seam stays self-contained. +_ENABLED_TASKS_ENV_VAR = "WORKER_PG_QUEUE_ENABLED_TASKS" + +# One-shot guard so the configured allow-list is logged once per process +# (see ``_log_allow_list_once``). Module-level → per prefork child. +_allow_list_logged = False + + +class QueueBackend(StrEnum): + """Transport a dispatch is routed to. + + ``StrEnum`` (3.11+) is inherited for symmetry with ``BarrierBackend``, + but unlike that enum this one is never read from / written to env — + only the task-name allow-list is. So callers MUST compare by identity + (``backend is QueueBackend.PG``), never ``== "pg"``: ``StrEnum`` makes a + typo'd ``== "cellery"`` a silent ``False`` rather than an error. + """ + + CELERY = "celery" + PG = "pg" + + +def _parse_allow_list() -> frozenset[str]: + """Parse ``WORKER_PG_QUEUE_ENABLED_TASKS`` into a set of task names. + + Trims surrounding whitespace per entry and drops blanks, so + ``"a, b ,, c"`` → ``{"a", "b", "c"}`` and ``""`` / unset → ``frozenset()``. + Read at call time (not import) so test harnesses that + ``monkeypatch.setenv`` flip the table per-test without a reload. + """ + stripped = ( + entry.strip() for entry in os.getenv(_ENABLED_TASKS_ENV_VAR, "").split(",") + ) + return frozenset(entry for entry in stripped if entry) + + +def _log_allow_list_once(allow_list: frozenset[str]) -> None: + """Log the configured allow-list once per process, at INFO. + + Makes a misconfiguration eyeballable at boot: a typo'd task name + (``async_execute_bni``) silently routes everything to Celery, but it + still shows up here so an operator can spot it. Only fires for a + *non-empty* allow-list — the default (feature off) stays silent so + the scaffold is truly inert when unused. + """ + global _allow_list_logged + if _allow_list_logged or not allow_list: + return + _allow_list_logged = True + logger.info( + "PG-queue routing enabled for tasks: %s (all others → Celery)", + sorted(allow_list), + ) + + +def select_backend(task_name: str) -> QueueBackend: + """Return the transport a dispatch should ride. + + ``PG`` if ``task_name`` is in ``WORKER_PG_QUEUE_ENABLED_TASKS``; + otherwise ``CELERY``. Empty / unset allow-list → always ``CELERY``. + + Never raises — the worst case is the safe ``CELERY`` default. + """ + allow_list = _parse_allow_list() + _log_allow_list_once(allow_list) + if task_name in allow_list: + return QueueBackend.PG + return QueueBackend.CELERY diff --git a/workers/sample.env b/workers/sample.env index 4764fb0c5a..6e9aaf2d2f 100644 --- a/workers/sample.env +++ b/workers/sample.env @@ -131,6 +131,34 @@ WORKER_BARRIER_BACKEND=chord # workflows can plausibly span >6h. WORKER_BARRIER_KEY_TTL_SECONDS=21600 +# ============================================================================= +# PG Queue Routing (Strangler-Fig) +# ============================================================================= +# Selects the queue *transport* per task — Celery (default) or PG Queue +# (PGMQ). Separate axis from the barrier above: the barrier is how a +# batch's fan-in fires the callback; this is how messages travel. +# +# A comma-separated allow-list drives a gradual, per-task-type migration +# (NOT a global on/off switch). A task routes to PG if its name is +# listed. Unset / empty (the default) → everything routes to Celery, +# exactly as today. +# +# Scaffold note: in this release there is no PG consumer yet, so +# PG-selected tasks are logged but still dispatched via Celery. Setting +# this flag is therefore safe to test routing decisions locally with +# zero behaviour change. No Flipt server required. +# +# Roll back instantly by removing an entry; flips take effect on worker +# restart (same posture as every other WORKER_* env). +# +# Only list *leaf* tasks dispatched via queue_backend.dispatch() — i.e. no +# downstream chain or barrier fan-in (e.g. send_webhook_notification). Do +# NOT list coupled pipeline tasks like async_execute_bin here: one execution +# must run wholly on one transport, so the pipeline migrates per-execution +# (via ExecutionContext) in a later phase, not per-task. See the routing.py +# module docstring for the full constraint. +# WORKER_PG_QUEUE_ENABLED_TASKS=send_webhook_notification + # Database URL (for fallback usage) DATABASE_URL=postgresql://unstract_dev:unstract_pass@unstract-db:5432/unstract_db diff --git a/workers/tests/test_queue_backend_seam.py b/workers/tests/test_queue_backend_seam.py index 8cd827cc10..8112e95d49 100644 --- a/workers/tests/test_queue_backend_seam.py +++ b/workers/tests/test_queue_backend_seam.py @@ -193,9 +193,9 @@ def some_function(): def test_worker_task_matches_shared_task_registration(self): """A function decorated with @worker_task is the same kind of object as one decorated with @shared_task — same registration semantics, same - invocation interface.""" + invocation interface. + """ from celery import shared_task - from queue_backend import worker_task @worker_task(name="queue_backend_test.via_seam") @@ -280,17 +280,22 @@ def test_all_exports(self): # (registered as a Celery task on import) + the BarrierBackend # enum + the get_barrier factory that the WORKER_BARRIER_BACKEND # env flag drives. + # Phase 8a adds QueueBackend + select_backend — the queue-transport + # routing gate that the WORKER_PG_QUEUE_ENABLED_TASKS allow-list + # drives. assert set(queue_backend.__all__) == { "Barrier", "BarrierBackend", "BarrierHandle", "CeleryChordBarrier", "FairnessKey", + "QueueBackend", "RedisDecrBarrier", "barrier_abort", "barrier_decr_and_check", "dispatch", "get_barrier", + "select_backend", "worker_task", } diff --git a/workers/tests/test_routing.py b/workers/tests/test_routing.py new file mode 100644 index 0000000000..f8b55473b4 --- /dev/null +++ b/workers/tests/test_routing.py @@ -0,0 +1,198 @@ +"""Tests for the PG-queue routing gate (``queue_backend.routing``). + +Three layers: + +1. **select_backend()** resolves the per-task allow-list correctly, + defaults to ``CELERY`` when unset, and never raises on malformed + input. + +2. **Observability** — the configured allow-list is logged once, and a + PG cutover emits a log. These are the gate's only observable effect, + so they're asserted explicitly (a future "the gate is inert, delete + it" refactor must fail a test, not pass silently). + +3. **dispatch() is inert under routing** — the ``current_app.send_task`` + call is byte-identical whether the routing table selects ``CELERY`` + or ``PG``. +""" + +from __future__ import annotations + +import importlib +import logging +from unittest.mock import patch + +import pytest +from queue_backend import QueueBackend, dispatch, select_backend +from queue_backend import routing as routing_mod +from queue_backend.fairness import FairnessKey, WorkloadType +from queue_backend.routing import _ENABLED_TASKS_ENV_VAR as ENABLED_TASKS_ENV + +# ``queue_backend.__init__`` binds ``dispatch`` to the *function*, shadowing +# the submodule attribute — import the module explicitly to reach its globals. +dispatch_mod = importlib.import_module("queue_backend.dispatch") + + +@pytest.fixture(autouse=True) +def _reset_routing_state(monkeypatch): + """Empty allow-list + cleared log-once guards before each test. + + The allow-list-logged flag and the per-task routing-logged set are + process-global one-shot guards; reset them so caplog assertions are + deterministic regardless of test order. + """ + monkeypatch.delenv(ENABLED_TASKS_ENV, raising=False) + routing_mod._allow_list_logged = False + dispatch_mod._pg_routing_logged.clear() + + +# --- select_backend() resolution --- + + +class TestSelectBackend: + def test_empty_table_routes_to_celery(self): + assert select_backend("any_task") is QueueBackend.CELERY + + def test_task_opted_in_routes_to_pg(self, monkeypatch): + monkeypatch.setenv(ENABLED_TASKS_ENV, "process_log_history") + assert select_backend("process_log_history") is QueueBackend.PG + + def test_task_not_opted_in_routes_to_celery(self, monkeypatch): + monkeypatch.setenv(ENABLED_TASKS_ENV, "process_log_history") + assert select_backend("some_other_task") is QueueBackend.CELERY + + def test_multiple_entries_membership(self, monkeypatch): + monkeypatch.setenv(ENABLED_TASKS_ENV, "a,b,c") + assert select_backend("b") is QueueBackend.PG + assert select_backend("d") is QueueBackend.CELERY + + +class TestSelectBackendTolerantParsing: + """Malformed / whitespace input resolves to the safe default, never raises.""" + + def test_whitespace_around_entries_trimmed(self, monkeypatch): + monkeypatch.setenv(ENABLED_TASKS_ENV, " a , b , c ") + assert select_backend("b") is QueueBackend.PG + + def test_empty_and_blank_segments_ignored(self, monkeypatch): + monkeypatch.setenv(ENABLED_TASKS_ENV, "a,, ,b") + assert select_backend("a") is QueueBackend.PG + # A blank segment must not become a matchable empty-string entry. + assert select_backend("") is QueueBackend.CELERY + + def test_empty_string_env_routes_to_celery(self, monkeypatch): + monkeypatch.setenv(ENABLED_TASKS_ENV, " ") + assert select_backend("any_task") is QueueBackend.CELERY + + def test_does_not_raise_on_any_input(self, monkeypatch): + for value in ("", " ", ",,,", "a,b", "\t\n", ",a,"): + monkeypatch.setenv(ENABLED_TASKS_ENV, value) + # Must resolve to an enum member, never raise. + assert select_backend("a") in (QueueBackend.CELERY, QueueBackend.PG) + + +class TestQueueBackendEnum: + def test_string_values(self): + assert QueueBackend.CELERY == "celery" + assert QueueBackend.PG == "pg" + + def test_members(self): + assert {b.value for b in QueueBackend} == {"celery", "pg"} + + +# --- Observability (the gate's only observable effect) --- + + +class TestObservability: + def test_allow_list_logged_once_when_configured(self, monkeypatch, caplog): + monkeypatch.setenv(ENABLED_TASKS_ENV, "async_execute_bin") + with caplog.at_level(logging.INFO, logger="queue_backend.routing"): + select_backend("x") + select_backend("y") # second call must NOT re-log + hits = [r for r in caplog.records if "PG-queue routing enabled" in r.getMessage()] + assert len(hits) == 1 + assert "async_execute_bin" in hits[0].getMessage() + + def test_empty_allow_list_logs_nothing(self, caplog): + """Default (feature off) stays silent — truly inert.""" + with caplog.at_level(logging.INFO, logger="queue_backend.routing"): + select_backend("x") + assert "PG-queue routing enabled" not in caplog.text + + def test_pg_selection_emits_routing_log(self, monkeypatch, caplog): + """Pins the dispatch() routing branch: deleting it must fail here.""" + monkeypatch.setenv(ENABLED_TASKS_ENV, "t1") + with ( + patch("queue_backend.dispatch.current_app"), + caplog.at_level(logging.INFO, logger="queue_backend.dispatch"), + ): + dispatch("t1") + assert "PG-queue routing selected" in caplog.text + + def test_celery_selection_emits_no_routing_log(self, caplog): + """Negative case — guards against the gate being made unconditional.""" + with ( + patch("queue_backend.dispatch.current_app"), + caplog.at_level(logging.INFO, logger="queue_backend.dispatch"), + ): + dispatch("t1") # empty allow-list → CELERY + assert "PG-queue routing selected" not in caplog.text + + def test_routing_log_bounded_to_once_per_task(self, monkeypatch, caplog): + monkeypatch.setenv(ENABLED_TASKS_ENV, "t1") + with ( + patch("queue_backend.dispatch.current_app"), + caplog.at_level(logging.INFO, logger="queue_backend.dispatch"), + ): + dispatch("t1") + dispatch("t1") + dispatch("t1") + hits = [r for r in caplog.records if "PG-queue routing selected" in r.getMessage()] + assert len(hits) == 1 + + +# --- dispatch() is inert under routing (the scaffold invariant) --- + + +class TestDispatchByteIdenticalRegardlessOfRouting: + """``dispatch()`` produces the same send_task call whether routed PG or Celery.""" + + def _capture_send_task(self, task_name, fairness): + with patch("queue_backend.dispatch.current_app") as mock_app: + dispatch( + task_name, + args=["a", 1], + kwargs={"k": "v"}, + queue="general", + fairness=fairness, + ) + return mock_app.send_task.call_args + + def test_pg_selection_does_not_change_the_wire(self, monkeypatch): + fairness = FairnessKey(org_id="org-1", workload_type=WorkloadType.API) + + # Celery path (empty table). + celery_call = self._capture_send_task("t1", fairness) + + # PG path (task opted in) — same inputs. + monkeypatch.setenv(ENABLED_TASKS_ENV, "t1") + pg_call = self._capture_send_task("t1", fairness) + + assert celery_call == pg_call + assert pg_call.args[0] == "t1" + assert pg_call.kwargs["args"] == ["a", 1] + assert pg_call.kwargs["kwargs"] == {"k": "v"} + assert pg_call.kwargs["queue"] == "general" + + def test_dispatch_without_fairness_still_routes_by_task(self, monkeypatch): + """Routing keys on task name only; fairness is irrelevant to the decision.""" + monkeypatch.setenv(ENABLED_TASKS_ENV, "bare_task") + with patch("queue_backend.dispatch.current_app") as mock_app: + dispatch("bare_task") + # Still dispatched (via Celery), header is None (no fairness). + assert mock_app.send_task.call_args.args[0] == "bare_task" + assert mock_app.send_task.call_args.kwargs["headers"] is None + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 85c36ffaa807e63b1a9bd34fd09fb9ac440015c5 Mon Sep 17 00:00:00 2001 From: ali Date: Thu, 11 Jun 2026 16:07:39 +0530 Subject: [PATCH 02/44] UN-3534 [DOCS] Standardize on "PG Queue" naming; drop PGMQ branding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't use the pgmq project (github.com/pgmq/pgmq) — no extension, no Python package, no copied SQL. The queue is a bespoke SKIP LOCKED schema (see the extension-free decision on UN-3533). Rename the 5 prose spots that called our substrate "PGMQ" to "PG Queue" so the code no longer reads as if it depends on that project. PGMQ stays only as prior-art reference in the decision record, not as the name of our substrate. Docs-only; no code or behaviour change. Co-Authored-By: Claude Opus 4.8 (1M context) --- workers/queue_backend/pg_queue/__init__.py | 11 ++++++----- workers/queue_backend/routing.py | 2 +- workers/sample.env | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/workers/queue_backend/pg_queue/__init__.py b/workers/queue_backend/pg_queue/__init__.py index a85622cfe8..b1d520f4dc 100644 --- a/workers/queue_backend/pg_queue/__init__.py +++ b/workers/queue_backend/pg_queue/__init__.py @@ -1,9 +1,10 @@ -"""PG Queue (PGMQ) transport substrate — scaffold. +"""PG Queue transport substrate — scaffold. -Reserved home for the PostgreSQL-backed queue substrate (PGMQ + -``SKIP LOCKED``) that will run alongside Celery during the Strangler-Fig -migration. PGMQ is a *core* worker transport — the intended primary -backend — so it lives inside the queue-backend seam next to its sibling +Reserved home for the PostgreSQL-backed queue substrate (a bespoke +``SKIP LOCKED`` queue, no extension) that will run alongside Celery during +the Strangler-Fig migration. PG Queue is a *core* worker transport — the +intended primary backend — so it lives inside the queue-backend seam next +to its sibling substrates (``dispatch``, ``routing``, ``barrier``, ``redis_barrier``), **not** under ``workers/plugins/``, whose plugin *implementation subdirectories* are the git-ignored overlay copied in at build time (the diff --git a/workers/queue_backend/routing.py b/workers/queue_backend/routing.py index abc43e8029..82cce0d67e 100644 --- a/workers/queue_backend/routing.py +++ b/workers/queue_backend/routing.py @@ -1,7 +1,7 @@ """Queue-transport routing gate (Strangler-Fig). Decides whether a given dispatch should ride the **Celery** transport -(today's only real path) or the future **PG Queue** (PGMQ) transport, +(today's only real path) or the future **PG Queue** transport, based on a per-task-type opt-in allow-list read from env: - ``WORKER_PG_QUEUE_ENABLED_TASKS`` — comma-separated task names routed diff --git a/workers/sample.env b/workers/sample.env index 6e9aaf2d2f..9d6f6a61fe 100644 --- a/workers/sample.env +++ b/workers/sample.env @@ -135,7 +135,7 @@ WORKER_BARRIER_KEY_TTL_SECONDS=21600 # PG Queue Routing (Strangler-Fig) # ============================================================================= # Selects the queue *transport* per task — Celery (default) or PG Queue -# (PGMQ). Separate axis from the barrier above: the barrier is how a +# (PG Queue). Separate axis from the barrier above: the barrier is how a # batch's fan-in fires the callback; this is how messages travel. # # A comma-separated allow-list drives a gradual, per-task-type migration From 2b6714f7b11863db1d9719d0d953476ae30777c5 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Thu, 11 Jun 2026 21:35:25 +0530 Subject: [PATCH 03/44] =?UTF-8?q?UN-3537=20[FEAT]=20PG=20Queue=20Phase=209?= =?UTF-8?q?a=20=E2=80=94=20extension-free=20queue=20schema=20+=20SKIP=20LO?= =?UTF-8?q?CKED=20dequeue=20(#2036)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3537 [FEAT] PG Queue Phase 9a — extension-free queue schema + SKIP LOCKED dequeue The storage + dequeue primitive the routing seam will route to. Inert: nothing in dispatch() calls it yet (the PG branch still routes to Celery), so zero behaviour change even on the integration branch. Backend (schema only — SHARED_APPS, cross-org infra, shared schema): - new pg_queue app; 0001 is 100% makemigrations-generated (pg_queue_message table + dequeue index). No CREATE EXTENSION, no DB-side function — plain Django. managed=True model doubles as a typed read handle. Workers (the client; first direct-DB worker capability): - queue_backend/pg_queue: send / read / delete + QueueMessage. read() runs one atomic UPDATE ... FOR UPDATE SKIP LOCKED ... RETURNING (visibility- timeout pattern): claim+commit, process outside the txn, delete on success; a crash lets vt expire and the row redelivers (at-least-once, no double-delivery, VACUUM-safe, PgBouncer txn-pooling compatible). - connection.py reuses the backend DB_* env -> PgBouncer in cloud, direct in OSS (UN-3533 decision). - psycopg2-binary==2.9.9 (matches backend), promoted to a direct dep. Tests: 4 unit (mocked SQL shape) + 4 integration (real Postgres) proving roundtrip, vt-hiding, vt-expiry redelivery, and no-double-delivery. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3537 [FIX] org_id empty-string default, not null=True (Django S6553) A string-based field shouldn't have two "no data" values (NULL and ""). Use default="" for "no org" (leaf tasks) instead of null=True; regenerate the generated 0001 accordingly. The client coerces None -> "" since the column is now non-null. Addresses SonarCloud S6553. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3537 [FIX] Address Phase 9a review feedback Robustness + docstring-accuracy fixes on the PG-queue primitive: - client: roll back on error and recover a dead connection (drop the cached conn on OperationalError/InterfaceError -> next call reconnects); add close() + context manager. One connection blip no longer wedges the 9c consumer. - client.read(): raise on non-positive vt_seconds/qty (vt<=0 is a silent double-delivery window). - client.delete(): WARN when no row was removed (the at-least-once re-delivery signal was previously swallowed). - QueueMessage: slots=True + doc that the payload is decoded JSONB (the dict stays mutable; frozen freezes the binding only). - connection: parameterise create_pg_connection(env_prefix); wrap connect with a self-identifying error log (non-secret host/port/dbname/schema); drop the stale pg_queue_read reference; soften the DB_HOST default claim. - models: fix stale docstrings describing a removed 0002 / DB-level defaults / pg_queue_read function. - tests: narrow integration skip to OperationalError (don't mask ImportError/bugs/permission errors as a green skip); rollback before the teardown DELETE; integration conn delegates to create_pg_connection(env_prefix="TEST_DB_"); add unit coverage for create_pg_connection, read() validation, and error-rollback. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3537 [FIX] Address Phase 9a review round 2 + SonarCloud S2068 - perf: ORDER BY (vt, msg_id) so the (queue_name, vt, msg_id) index drives an indexed top-N instead of sorting the whole visible backlog on each read() — a real regression on a deep queue. - recovery: a failed rollback now proves the connection is dead, so a poisoned connection is recycled regardless of which psycopg2 error subclass was raised (not only Operational/Interface); also checks conn.closed. Closes the "one blip wedges the consumer" gap more fully. - docs: clarify the contract — at-least-once means a message CAN be processed more than once after vt-expiry; SKIP LOCKED only prevents concurrent double-claim. "(no double-delivery)" -> "(no concurrent double-claim)". - tests: cover the recovery/ownership branches (owned conn recycled on OperationalError and on failed rollback; injected conn never closed; close() owned-vs-injected). - security (SonarCloud S2068): the create_pg_connection test mapped a literal "PASSWORD": "p" -> use a runtime uuid token so it isn't flagged as a hard-coded credential. Resolves the Security Rating C gate failure. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/backend/settings/base.py | 2 + backend/pg_queue/__init__.py | 0 backend/pg_queue/apps.py | 7 + backend/pg_queue/migrations/0001_initial.py | 34 ++ backend/pg_queue/migrations/__init__.py | 0 backend/pg_queue/models.py | 37 +++ workers/pyproject.toml | 3 + workers/queue_backend/pg_queue/__init__.py | 21 +- workers/queue_backend/pg_queue/client.py | 215 +++++++++++++ workers/queue_backend/pg_queue/connection.py | 67 ++++ workers/tests/test_pg_queue_client.py | 321 +++++++++++++++++++ workers/uv.lock | 2 + 12 files changed, 704 insertions(+), 5 deletions(-) create mode 100644 backend/pg_queue/__init__.py create mode 100644 backend/pg_queue/apps.py create mode 100644 backend/pg_queue/migrations/0001_initial.py create mode 100644 backend/pg_queue/migrations/__init__.py create mode 100644 backend/pg_queue/models.py create mode 100644 workers/queue_backend/pg_queue/client.py create mode 100644 workers/queue_backend/pg_queue/connection.py create mode 100644 workers/tests/test_pg_queue_client.py diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index 479cb5476b..787a7603e6 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -306,6 +306,8 @@ def filter(self, record): # For the organization model "account_v2", "account_usage", + # PG Queue — extension-free bespoke queue (cross-org infra, shared schema) + "pg_queue", # Django apps should go below this line "django.contrib.admin", "django.contrib.auth", diff --git a/backend/pg_queue/__init__.py b/backend/pg_queue/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/pg_queue/apps.py b/backend/pg_queue/apps.py new file mode 100644 index 0000000000..93333d2fe7 --- /dev/null +++ b/backend/pg_queue/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class PgQueueConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "pg_queue" + verbose_name = "PG Queue" diff --git a/backend/pg_queue/migrations/0001_initial.py b/backend/pg_queue/migrations/0001_initial.py new file mode 100644 index 0000000000..770f06aed0 --- /dev/null +++ b/backend/pg_queue/migrations/0001_initial.py @@ -0,0 +1,34 @@ +# Generated by Django 4.2.1 on 2026-06-11 14:42 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="PgQueueMessage", + fields=[ + ("msg_id", models.BigAutoField(primary_key=True, serialize=False)), + ("queue_name", models.TextField()), + ("message", models.JSONField()), + ("org_id", models.TextField(blank=True, default="")), + ("enqueued_at", models.DateTimeField(default=django.utils.timezone.now)), + ("vt", models.DateTimeField(default=django.utils.timezone.now)), + ("read_ct", models.IntegerField(default=0)), + ], + options={ + "db_table": "pg_queue_message", + "indexes": [ + models.Index( + fields=["queue_name", "vt", "msg_id"], + name="pg_queue_message_dequeue_idx", + ) + ], + }, + ), + ] diff --git a/backend/pg_queue/migrations/__init__.py b/backend/pg_queue/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py new file mode 100644 index 0000000000..49c63f4b9a --- /dev/null +++ b/backend/pg_queue/models.py @@ -0,0 +1,37 @@ +from django.db import models +from django.utils import timezone + + +class PgQueueMessage(models.Model): + """Bespoke extension-free queue table (``SKIP LOCKED`` + visibility timeout). + + The table and its dequeue index are fully Django-managed (generated by + ``makemigrations``, migration 0001) — no hand-written migration, no DB + function, no ``CREATE EXTENSION`` (UN-3533 extension-free decision). The + ``SKIP LOCKED`` dequeue itself is a single statement in the workers' + ``pg_queue`` client, not a DB-side function. This model doubles as a + typed read handle for the backend producer side. + """ + + msg_id = models.BigAutoField(primary_key=True) + queue_name = models.TextField() + message = models.JSONField() + # "" = no org (leaf tasks); the fair-admission query (a later phase) + # uses it for the coupled pipeline. Empty string rather than NULL — + # a string field shouldn't have two "no data" values (Django S6553). + org_id = models.TextField(blank=True, default="") + # Python-level defaults let ORM ``.create()`` work without these; the + # raw-SQL client supplies them explicitly (``now()`` / ``0``), so no + # DB-level default is required. + enqueued_at = models.DateTimeField(default=timezone.now) + vt = models.DateTimeField(default=timezone.now) # visibility timeout + read_ct = models.IntegerField(default=0) + + class Meta: + db_table = "pg_queue_message" + indexes = [ + models.Index( + fields=["queue_name", "vt", "msg_id"], + name="pg_queue_message_dequeue_idx", + ) + ] diff --git a/workers/pyproject.toml b/workers/pyproject.toml index 6344f8798d..e3368a4eea 100644 --- a/workers/pyproject.toml +++ b/workers/pyproject.toml @@ -34,6 +34,9 @@ dependencies = [ "unstract-filesystem", # Caching "redis>=4.5.0,<6.0.0", # Redis client for worker cache access + # PG Queue — direct Postgres access for the bespoke SKIP LOCKED queue + # (reuses the backend's DB_* connection: PgBouncer in cloud, direct in OSS) + "psycopg2-binary==2.9.9", # matches backend; first direct-DB worker capability # Streaming utilities for bulk downloads (pluggable worker) "zipstream-ng>=1.7.0", # Streaming zip creation for memory-efficient bulk downloads # Note: Using dataclasses instead of pydantic for lightweight typing diff --git a/workers/queue_backend/pg_queue/__init__.py b/workers/queue_backend/pg_queue/__init__.py index b1d520f4dc..4b0aee5782 100644 --- a/workers/queue_backend/pg_queue/__init__.py +++ b/workers/queue_backend/pg_queue/__init__.py @@ -14,12 +14,23 @@ real implementation will likely span several modules (config, consumer poll loop, orchestrator) — exact layout TBD. -Empty by design in this phase. Routing decisions are made by -:func:`queue_backend.routing.select_backend`; until a consumer exists -here, PG-selected dispatches still ride Celery (see -``queue_backend.dispatch``). +Phase 9a adds the storage + dequeue primitive (:class:`PgQueueClient` +over the ``pg_queue_message`` table; the ``SKIP LOCKED`` dequeue lives in +the client). It is still **inert**: routing decisions are made by +:func:`queue_backend.routing.select_backend`, and until the enqueue +wiring (9b) and consumer (9c) land, PG-selected dispatches still ride +Celery (see ``queue_backend.dispatch``). Design reference: the PG Queue implementation guide in the labs repo (``workflow-execution-architecture``). Branch and section pointers move, -so they're tracked on UN-3534 / the PR rather than baked in here. +so they're tracked on the ticket / PR rather than baked in here. """ + +from .client import PgQueueClient, QueueMessage +from .connection import create_pg_connection + +__all__ = [ + "PgQueueClient", + "QueueMessage", + "create_pg_connection", +] diff --git a/workers/queue_backend/pg_queue/client.py b/workers/queue_backend/pg_queue/client.py new file mode 100644 index 0000000000..fb61147b8c --- /dev/null +++ b/workers/queue_backend/pg_queue/client.py @@ -0,0 +1,215 @@ +"""Thin client over the bespoke PG queue (extension-free, ``SKIP LOCKED``). + +**Inert in this phase** — nothing in ``dispatch()`` calls this yet (the +routing gate's PG branch still routes to Celery). This is the storage + +dequeue primitive that the enqueue wiring (9b) and the consumer poll +loop (9c) build on. + +Dequeue uses the visibility-timeout pattern: :meth:`PgQueueClient.read` +runs a single atomic ``UPDATE … WHERE msg_id IN (SELECT … FOR UPDATE +SKIP LOCKED …) RETURNING …`` (committed immediately), the caller +processes the message *outside* the transaction, then +:meth:`PgQueueClient.delete` acks on success. A crash before delete +leaves the row to reappear once its ``vt`` expires — **at-least-once** +delivery: SKIP LOCKED stops two *concurrent* readers from claiming the +same visible row, but a message can still be delivered more than once if +a reader crashes before ``delete()`` and the row's ``vt`` expires, so the +consumer must be idempotent. The whole queue contract lives here, in one place; +the schema (``pg_queue_message`` table + dequeue index) is a plain +Django migration with no DB-side function. + +The cached connection is kept usable across calls: every operation rolls +back on error, and a connection that goes bad (dropped socket / PgBouncer +recycle) is discarded so the next call reconnects — so one blip can't +permanently wedge the 9c consumer. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +from collections.abc import Iterator +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Self + +import psycopg2 + +from .connection import create_pg_connection + +if TYPE_CHECKING: + from psycopg2.extensions import connection as PgConnection + +logger = logging.getLogger(__name__) + +# Atomic claim. Takes up to %(qty)s ready messages no other transaction +# holds, makes them invisible for %(vt)s seconds, returns them. SKIP LOCKED +# => concurrent readers never claim the same visible row (no concurrent +# double-claim). The caller commits immediately, processes OUTSIDE the txn, +# DELETEs on success; a crash leaves the row to reappear when vt expires +# (at-least-once — a message CAN be processed more than once). No lock held +# during processing -> VACUUM-safe and PgBouncer txn-pooling compatible. +# +# ORDER BY (vt, msg_id) — not just msg_id — so the (queue_name, vt, msg_id) +# index drives an indexed top-N: for a fixed queue the index is ordered by +# (vt, msg_id), so the vt<=now() range scan emits rows already sorted and PG +# applies LIMIT without sorting the whole visible backlog. Still effectively +# FIFO (vt is set to now() at enqueue; msg_id is the deterministic tiebreak). +_DEQUEUE_SQL = """ +UPDATE pg_queue_message + SET vt = now() + make_interval(secs => %s), + read_ct = read_ct + 1 + WHERE msg_id IN ( + SELECT msg_id + FROM pg_queue_message + WHERE queue_name = %s + AND vt <= now() + ORDER BY vt, msg_id + FOR UPDATE SKIP LOCKED + LIMIT %s + ) +RETURNING msg_id, message +""" + + +@dataclass(frozen=True, slots=True) +class QueueMessage: + """A claimed queue message. + + ``message`` is the already-decoded JSONB payload (psycopg2 parses + ``jsonb`` to a Python ``dict``). ``frozen`` freezes the binding only — + the ``dict`` itself is mutable, so treat the payload as read-only by + convention. + """ + + msg_id: int + message: dict[str, Any] + + +class PgQueueClient: + """``send`` / ``read`` / ``delete`` over ``pg_queue_message``. + + A connection may be injected (tests); otherwise one is created lazily + from the backend ``DB_*`` env on first use and owned by this client + (closed by :meth:`close`, recovered automatically after a connection + error). Usable as a context manager. + """ + + def __init__(self, conn: PgConnection | None = None) -> None: + self._conn = conn + # Injected connections belong to the caller — never close/recycle them. + self._owns_conn = conn is None + + @property + def conn(self) -> PgConnection: + if self._conn is None: + self._conn = create_pg_connection() + return self._conn + + @contextlib.contextmanager + def _cursor(self) -> Iterator[Any]: + """Yield a cursor; commit on success, roll back + recover on error. + + Keeps the cached connection usable: a failed statement leaves the + connection in an aborted transaction, so we always roll back; a + dead connection can't be reused, so (when we own it) we drop the + cached handle and the next call reconnects. + """ + conn = self.conn + try: + with conn.cursor() as cur: + yield cur + conn.commit() + except Exception as exc: + # A failed rollback proves the connection is unusable, regardless + # of which psycopg2 error subclass was raised (a server-side + # termination mid-statement can surface as a bare DatabaseError). + conn_dead = isinstance( + exc, (psycopg2.OperationalError, psycopg2.InterfaceError) + ) + try: + conn.rollback() + except Exception: + conn_dead = True + # Discard an unusable connection so the next call reconnects — + # only when we own it (an injected connection is the caller's). + if self._owns_conn and (conn_dead or conn.closed): + with contextlib.suppress(Exception): + conn.close() + self._conn = None + raise + + def send( + self, queue_name: str, message: dict[str, Any], *, org_id: str | None = None + ) -> int: + """Enqueue a message; returns its ``msg_id``. + + Immediately visible — ``vt`` is set to ``now()`` (DB clock). The + timestamp/counter columns are supplied here rather than via DB + defaults so the schema stays a plain Django migration. + """ + with self._cursor() as cur: + cur.execute( + "INSERT INTO pg_queue_message " + "(queue_name, message, org_id, enqueued_at, vt, read_ct) " + "VALUES (%s, %s::jsonb, %s, now(), now(), 0) RETURNING msg_id", + # "" rather than NULL for "no org" — the column is non-null + # (string fields shouldn't have two empty values; Django S6553). + (queue_name, json.dumps(message), org_id if org_id is not None else ""), + ) + msg_id = cur.fetchone()[0] + return int(msg_id) + + def read( + self, queue_name: str, *, vt_seconds: int = 30, qty: int = 1 + ) -> list[QueueMessage]: + """Atomically claim up to ``qty`` ready messages, hiding them for ``vt_seconds``. + + Commits immediately so the row lock is released and the ``vt`` + bump persists — claimed messages are then invisible to other + readers until ``vt`` expires or :meth:`delete` removes them. + + Raises ``ValueError`` for non-positive ``vt_seconds`` (which would + make a claimed message immediately re-visible — a double-delivery + window) or ``qty`` (a pointless / erroring ``LIMIT``). + """ + if vt_seconds <= 0: + raise ValueError(f"vt_seconds must be positive, got {vt_seconds}") + if qty <= 0: + raise ValueError(f"qty must be positive, got {qty}") + with self._cursor() as cur: + cur.execute(_DEQUEUE_SQL, (vt_seconds, queue_name, qty)) + rows = cur.fetchall() + return [QueueMessage(msg_id=int(r[0]), message=r[1]) for r in rows] + + def delete(self, msg_id: int) -> bool: + """Ack a processed message. Returns ``True`` if a row was removed. + + ``False`` means the row was already gone — typically its visibility + timeout expired during processing and another worker (re)claimed it, + i.e. the work may be processed twice. Logged at WARNING so this + at-least-once condition is visible rather than silently swallowed. + """ + with self._cursor() as cur: + cur.execute("DELETE FROM pg_queue_message WHERE msg_id = %s", (msg_id,)) + deleted = cur.rowcount + if deleted == 0: + logger.warning( + "PG-queue: delete(msg_id=%s) removed no row — message was already " + "gone (vt likely expired during processing; possible re-delivery).", + msg_id, + ) + return deleted == 1 + + def close(self) -> None: + """Close the connection if this client owns it (no-op if injected).""" + if self._owns_conn and self._conn is not None: + with contextlib.suppress(Exception): + self._conn.close() + self._conn = None + + def __enter__(self) -> Self: + return self + + def __exit__(self, *_exc: object) -> None: + self.close() diff --git a/workers/queue_backend/pg_queue/connection.py b/workers/queue_backend/pg_queue/connection.py new file mode 100644 index 0000000000..53c1f414f9 --- /dev/null +++ b/workers/queue_backend/pg_queue/connection.py @@ -0,0 +1,67 @@ +"""Direct Postgres connection for the PG-queue client. + +The PG queue is the first worker component to open a *direct* Postgres +connection — workers otherwise reach state via the Internal API, with +Redis the only other direct store. It reuses the **backend's** ``DB_*`` +env so the target follows the deployment: in cloud those vars point at +PgBouncer (connection pooling); in OSS / on-prem they point straight at +Postgres (UN-3533 decision). + +``search_path`` is set to ``DB_SCHEMA`` so the bespoke +``pg_queue_message`` table resolves in the same schema the backend +manages. +""" + +from __future__ import annotations + +import logging +import os +from typing import TYPE_CHECKING + +import psycopg2 + +if TYPE_CHECKING: + from psycopg2.extensions import connection as PgConnection + +logger = logging.getLogger(__name__) + + +def create_pg_connection(env_prefix: str = "DB_") -> PgConnection: + """Open a direct Postgres connection from ``{env_prefix}*`` env. + + The default ``DB_`` prefix reads the same variable *names* the backend + uses (``DB_HOST`` / ``DB_PORT`` / ``DB_NAME`` / ``DB_USER`` / + ``DB_PASSWORD`` / ``DB_SCHEMA``), so the queue connects to the + backend's database and schema. Tests pass ``env_prefix="TEST_DB_"`` to + target a dev DB. The fallbacks below are dev-compose values (the host + is the compose service name); real deployments always set the env + explicitly, so the fallback host intentionally differs from the + backend's own ``base.py`` default. + """ + host = os.getenv(f"{env_prefix}HOST", "unstract-db") + port = os.getenv(f"{env_prefix}PORT", "5432") + dbname = os.getenv(f"{env_prefix}NAME", "unstract_db") + user = os.getenv(f"{env_prefix}USER", "unstract_dev") + schema = os.getenv(f"{env_prefix}SCHEMA", "unstract") + try: + return psycopg2.connect( + host=host, + port=port, + dbname=dbname, + user=user, + password=os.getenv(f"{env_prefix}PASSWORD", "unstract_pass"), + options=f"-c search_path={schema}", + ) + except psycopg2.OperationalError: + # First direct-DB worker component — make the failure self-identifying + # so a misconfigured DB_* var is obvious. Secrets (password) omitted. + logger.error( + "PG-queue: failed to connect to Postgres " + "(host=%s port=%s dbname=%s schema=%s, env_prefix=%r)", + host, + port, + dbname, + schema, + env_prefix, + ) + raise diff --git a/workers/tests/test_pg_queue_client.py b/workers/tests/test_pg_queue_client.py new file mode 100644 index 0000000000..fbc4bcd579 --- /dev/null +++ b/workers/tests/test_pg_queue_client.py @@ -0,0 +1,321 @@ +"""Tests for the PG-queue client (``queue_backend.pg_queue``). + +Two layers: + +1. **Unit** (mocked connection) — ``send`` / ``read`` / ``delete`` issue + the right SQL + params and commit. Always runs, no DB needed. + +2. **Integration** (real Postgres) — exercises the actual ``SKIP LOCKED`` + + visibility-timeout semantics against a live DB. Skips gracefully if + Postgres isn't reachable or the ``pg_queue`` migration hasn't been + applied. +""" + +from __future__ import annotations + +import os +import time +import uuid +from unittest.mock import MagicMock + +import psycopg2 +import pytest +from queue_backend.pg_queue import PgQueueClient, QueueMessage +from queue_backend.pg_queue.connection import create_pg_connection + +# --- Unit: SQL shape against a mocked connection --- + + +class _CursorCtx: + """Mimic a psycopg2 cursor used as a context manager.""" + + def __init__(self, cursor): + self._cursor = cursor + + def __enter__(self): + return self._cursor + + def __exit__(self, *_): + return False + + +def _mock_conn(*, fetchone=None, fetchall=None, rowcount=0): + cur = MagicMock() + cur.fetchone.return_value = fetchone + cur.fetchall.return_value = fetchall or [] + cur.rowcount = rowcount + conn = MagicMock() + conn.cursor.return_value = _CursorCtx(cur) + return conn, cur + + +class TestPgQueueClientUnit: + def test_send_inserts_and_returns_msg_id(self): + conn, cur = _mock_conn(fetchone=(42,)) + msg_id = PgQueueClient(conn=conn).send("q1", {"a": 1}, org_id="org-9") + assert msg_id == 42 + sql, params = cur.execute.call_args.args + assert "INSERT INTO pg_queue_message" in sql + assert params[0] == "q1" + assert params[2] == "org-9" + assert '"a": 1' in params[1] # message JSON-serialised + conn.commit.assert_called_once() + + def test_send_coerces_missing_org_to_empty_string(self): + # org_id column is non-null (Django S6553) — None must become "". + conn, cur = _mock_conn(fetchone=(1,)) + PgQueueClient(conn=conn).send("q1", {"a": 1}) + _, params = cur.execute.call_args.args + assert params[2] == "" + + def test_read_runs_skip_locked_dequeue(self): + conn, cur = _mock_conn(fetchall=[(7, {"k": "v"})]) + msgs = PgQueueClient(conn=conn).read("q1", vt_seconds=15, qty=3) + sql, params = cur.execute.call_args.args + assert "FOR UPDATE SKIP LOCKED" in sql + assert "UPDATE pg_queue_message" in sql + # Param order follows the %s positions: vt_seconds, queue_name, qty. + assert params == (15, "q1", 3) + assert msgs == [QueueMessage(msg_id=7, message={"k": "v"})] + conn.commit.assert_called_once() + + def test_delete_returns_true_when_row_removed(self): + conn, cur = _mock_conn(rowcount=1) + assert PgQueueClient(conn=conn).delete(7) is True + sql, params = cur.execute.call_args.args + assert "DELETE FROM pg_queue_message" in sql + assert params == (7,) + conn.commit.assert_called_once() + + def test_delete_returns_false_when_no_row(self): + conn, _ = _mock_conn(rowcount=0) + assert PgQueueClient(conn=conn).delete(999) is False + + def test_read_rejects_non_positive_vt(self): + conn, _ = _mock_conn() + with pytest.raises(ValueError, match="vt_seconds"): + PgQueueClient(conn=conn).read("q1", vt_seconds=0) + + def test_read_rejects_non_positive_qty(self): + conn, _ = _mock_conn() + with pytest.raises(ValueError, match="qty"): + PgQueueClient(conn=conn).read("q1", qty=0) + + def test_error_rolls_back_and_reraises(self): + conn, cur = _mock_conn() + cur.execute.side_effect = RuntimeError("boom") + with pytest.raises(RuntimeError): + PgQueueClient(conn=conn).send("q1", {"a": 1}) + conn.rollback.assert_called_once() + conn.commit.assert_not_called() + + +class TestConnectionLifecycle: + """Recovery + ownership branches of ``_cursor()`` / ``close()``.""" + + @staticmethod + def _owned_client(monkeypatch, *, execute_raises=None): + cur = MagicMock() + if execute_raises is not None: + cur.execute.side_effect = execute_raises + conn = MagicMock() + conn.closed = 0 + conn.cursor.return_value = _CursorCtx(cur) + factory = MagicMock(return_value=conn) + monkeypatch.setattr( + "queue_backend.pg_queue.client.create_pg_connection", factory + ) + return PgQueueClient(), conn, factory + + def test_owned_conn_recovered_on_operational_error(self, monkeypatch): + client, conn, factory = self._owned_client( + monkeypatch, execute_raises=psycopg2.OperationalError("dead") + ) + with pytest.raises(psycopg2.OperationalError): + client.send("q", {"a": 1}) + conn.rollback.assert_called_once() + conn.close.assert_called_once() + assert client._conn is None # cached handle dropped + _ = client.conn # next op reconnects + assert factory.call_count == 2 + + def test_owned_conn_recovered_when_rollback_fails(self, monkeypatch): + # A non-Operational error whose rollback fails → still treated as dead. + client, conn, _ = self._owned_client( + monkeypatch, execute_raises=RuntimeError("boom") + ) + conn.rollback.side_effect = psycopg2.DatabaseError("socket gone") + with pytest.raises(RuntimeError): + client.send("q", {"a": 1}) + conn.close.assert_called_once() + assert client._conn is None + + def test_injected_conn_never_closed_on_error(self): + cur = MagicMock() + cur.execute.side_effect = psycopg2.OperationalError("dead") + conn = MagicMock() + conn.closed = 0 + conn.cursor.return_value = _CursorCtx(cur) + client = PgQueueClient(conn=conn) + with pytest.raises(psycopg2.OperationalError): + client.send("q", {"a": 1}) + conn.rollback.assert_called_once() + conn.close.assert_not_called() + assert client._conn is conn # caller's connection untouched + + def test_close_closes_owned_not_injected(self, monkeypatch): + client, conn, _ = self._owned_client(monkeypatch) + _ = client.conn # lazily create + client.close() + conn.close.assert_called_once() + assert client._conn is None + + injected = MagicMock() + PgQueueClient(conn=injected).close() + injected.close.assert_not_called() + + +class TestCreatePgConnection: + """Unit coverage for the connection factory (no real DB).""" + + def test_reads_env_prefix_and_sets_search_path(self, monkeypatch): + captured = {} + monkeypatch.setattr( + "queue_backend.pg_queue.connection.psycopg2.connect", + lambda **kw: captured.update(kw) or object(), + ) + # Password is a runtime token (not a hard-coded literal) so it isn't + # mistaken for a credential, while still proving the env is read. + secret = uuid.uuid4().hex + for k, v in { + "HOST": "h", + "PORT": "6432", + "NAME": "n", + "USER": "u", + "PASSWORD": secret, + "SCHEMA": "s", + }.items(): + monkeypatch.setenv(f"DB_{k}", v) + create_pg_connection() + assert captured["host"] == "h" + assert captured["port"] == "6432" + assert captured["dbname"] == "n" + assert captured["user"] == "u" + assert captured["password"] == secret + assert captured["options"] == "-c search_path=s" + + def test_env_prefix_override(self, monkeypatch): + captured = {} + monkeypatch.setattr( + "queue_backend.pg_queue.connection.psycopg2.connect", + lambda **kw: captured.update(kw) or object(), + ) + monkeypatch.setenv("TEST_DB_HOST", "test-host") + create_pg_connection(env_prefix="TEST_DB_") + assert captured["host"] == "test-host" + + def test_connect_failure_is_logged_and_reraised(self, monkeypatch, caplog): + import logging + + def boom(**_): + raise psycopg2.OperationalError("nope") + + monkeypatch.setattr( + "queue_backend.pg_queue.connection.psycopg2.connect", boom + ) + with ( + caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.connection"), + pytest.raises(psycopg2.OperationalError), + ): + create_pg_connection() + assert "failed to connect" in caplog.text + + +# --- Integration: real Postgres --- + + +def _integration_conn(): + # Reuse create_pg_connection (single source of connection logic) via the + # dedicated TEST_DB_* prefix — NOT the generic DB_*, which the suite's + # conftest sets to DB_USER=test for unit isolation (wrong real-DB creds). + # Default the host to the dev-compose published port on localhost. + os.environ.setdefault("TEST_DB_HOST", "127.0.0.1") + return create_pg_connection(env_prefix="TEST_DB_") + + +@pytest.fixture +def pg_conn(): + try: + conn = _integration_conn() + except psycopg2.OperationalError as exc: + # Only an unreachable/unauthenticated DB skips — ImportError, bugs, + # and schema/permission errors surface as real failures. + pytest.skip(f"Postgres not reachable: {exc}") + with conn.cursor() as cur: + cur.execute("SELECT to_regclass('pg_queue_message')") + (table,) = cur.fetchone() + if table is None: + conn.close() + pytest.skip("pg_queue migration not applied (run backend migrate)") + yield conn + conn.rollback() + conn.close() + + +@pytest.fixture +def queue_name(pg_conn): + # Unique per test for isolation; clean up rows afterwards. + name = f"test_q_{os.getpid()}_{int(time.time() * 1000)}" + yield name + # A failed test body can leave the connection in an aborted transaction; + # roll back first so this cleanup doesn't raise InFailedSqlTransaction. + pg_conn.rollback() + with pg_conn.cursor() as cur: + cur.execute("DELETE FROM pg_queue_message WHERE queue_name = %s", (name,)) + pg_conn.commit() + + +class TestPgQueueClientIntegration: + def test_send_read_delete_roundtrip(self, pg_conn, queue_name): + client = PgQueueClient(conn=pg_conn) + msg_id = client.send(queue_name, {"hello": "world"}) + msgs = client.read(queue_name, vt_seconds=30, qty=10) + assert [m.msg_id for m in msgs] == [msg_id] + assert msgs[0].message == {"hello": "world"} + assert client.delete(msg_id) is True + assert client.read(queue_name, vt_seconds=30, qty=10) == [] # gone + + def test_read_hides_message_for_vt(self, pg_conn, queue_name): + client = PgQueueClient(conn=pg_conn) + client.send(queue_name, {"n": 1}) + assert len(client.read(queue_name, vt_seconds=30, qty=10)) == 1 + # Second read within vt sees nothing. + assert client.read(queue_name, vt_seconds=30, qty=10) == [] + + def test_vt_expiry_redelivers(self, pg_conn, queue_name): + client = PgQueueClient(conn=pg_conn) + client.send(queue_name, {"n": 1}) + claimed = client.read(queue_name, vt_seconds=1, qty=10) + assert len(claimed) == 1 + time.sleep(1.3) # let vt expire + again = client.read(queue_name, vt_seconds=30, qty=10) + assert [m.msg_id for m in again] == [c.msg_id for c in claimed] + + def test_no_double_delivery_across_readers(self, pg_conn, queue_name): + """Two readers never claim the same message (SKIP LOCKED + vt).""" + client_a = PgQueueClient(conn=pg_conn) + for i in range(5): + client_a.send(queue_name, {"i": i}) + conn_b = _integration_conn() + try: + client_b = PgQueueClient(conn=conn_b) + ids_a = {m.msg_id for m in client_a.read(queue_name, vt_seconds=30, qty=3)} + ids_b = {m.msg_id for m in client_b.read(queue_name, vt_seconds=30, qty=3)} + assert ids_a.isdisjoint(ids_b) # no double-delivery + assert len(ids_a) + len(ids_b) == 5 # all delivered exactly once + finally: + conn_b.close() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/workers/uv.lock b/workers/uv.lock index cfd1e7c96d..6f625ec13b 100644 --- a/workers/uv.lock +++ b/workers/uv.lock @@ -4903,6 +4903,7 @@ dependencies = [ { name = "httpx" }, { name = "prometheus-client" }, { name = "psutil" }, + { name = "psycopg2-binary" }, { name = "python-dotenv" }, { name = "python-socketio" }, { name = "redis" }, @@ -4951,6 +4952,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.27.0" }, { name = "prometheus-client", specifier = ">=0.17.0,<1.0.0" }, { name = "psutil", specifier = ">=5.9.0,<6.0.0" }, + { name = "psycopg2-binary", specifier = "==2.9.9" }, { name = "python-dotenv", specifier = ">=1.2.2,<2.0.0" }, { name = "python-socketio", specifier = ">=5.9.0" }, { name = "redis", specifier = ">=4.5.0,<6.0.0" }, From 808767717387f1a07af22e2e9287fbeffcddebd2 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Thu, 11 Jun 2026 22:35:42 +0530 Subject: [PATCH 04/44] =?UTF-8?q?UN-3538=20[FEAT]=20PG=20Queue=20Phase=209?= =?UTF-8?q?b=20=E2=80=94=20enqueue=20PG-routed=20tasks=20to=20Postgres=20(?= =?UTF-8?q?#2043)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3538 [FEAT] PG Queue Phase 9b — enqueue PG-routed tasks to Postgres When a task is opted into WORKER_PG_QUEUE_ENABLED_TASKS, dispatch() now serialises it as a TaskPayload and enqueues to pg_queue_message instead of Celery, returning a PgDispatchHandle (.id = msg_id). A process-singleton PgQueueClient is reused across dispatches. The Celery path is unchanged and the default-empty flag routes everything to Celery — zero behaviour change. - queue_backend/pg_queue/task_payload.py: TaskPayload TypedDict + to_payload() — the producer<->consumer wire contract. This is the *contents* of the pg_queue_message.message JSONB column, distinct from the backend's PgQueueMessage *row* model (envelope vs payload — they nest, not duplicate). - dispatch(): PG branch enqueues + returns; cutover log (INFO, once per task); .warning docstring that an opt-in requires the 9c consumer running, else the task enqueues but never executes. - sample.env: same warning on the flag. - backend pg_queue model: note that `message` holds a TaskPayload. - tests: TestDispatchRouting (PG->enqueue, Celery->send_task), TestCutoverLog, to_payload shape + real-PG integration (dispatch lands a decodable row). Leaf-only (migration-coherence) — pipeline tasks stay on Celery until execution-level routing (9e). INERT by default. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3538 [FIX] Address Phase 9b review feedback - dispatch: per-thread PG client via threading.local — a libpq connection isn't safe for concurrent use across threads; correct under prefork AND a -P threads pool (gevent would need a pool — noted, out of scope). - dispatch: log the routing *decision* BEFORE send() so a first-dispatch failure (DB down / unmigrated) doesn't suppress the one announcement; wrap the enqueue with a logger.exception breadcrumb (a raw psycopg2.Error or a json.dumps TypeError on a non-serialisable arg otherwise propagate with no "PG-routed dispatch" context). No Celery fallback retained. - dispatch: hoist `pg_queue = queue or _DEFAULT_PG_QUEUE` (one source). - fairness: share a FairnessPayload TypedDict; to_dict() -> FairnessPayload; TaskPayload.fairness uses it instead of a loose dict[str, Any]. - pg_queue/__init__ docstring: describe 9b state (no longer "inert, rides Celery"); sample.env: drop a duplicated routes-to-Celery sentence. - tests: no-fallback-on-enqueue-failure (Critical gap) + log-ordering pin; lazy-init + reuse of the per-thread client; cutover-log assertions updated for the new message. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/pg_queue/models.py | 3 + workers/queue_backend/dispatch.py | 134 +++++++++++---- workers/queue_backend/fairness.py | 26 ++- workers/queue_backend/pg_queue/__init__.py | 18 +- .../queue_backend/pg_queue/task_payload.py | 53 ++++++ workers/sample.env | 14 +- workers/tests/test_dispatch_pg.py | 137 ++++++++++++++++ workers/tests/test_routing.py | 155 +++++++++++------- 8 files changed, 427 insertions(+), 113 deletions(-) create mode 100644 workers/queue_backend/pg_queue/task_payload.py create mode 100644 workers/tests/test_dispatch_pg.py diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py index 49c63f4b9a..771fd909e6 100644 --- a/backend/pg_queue/models.py +++ b/backend/pg_queue/models.py @@ -15,6 +15,9 @@ class PgQueueMessage(models.Model): msg_id = models.BigAutoField(primary_key=True) queue_name = models.TextField() + # Opaque JSONB payload. For task dispatches the workers write a task + # payload here (``queue_backend.pg_queue.TaskPayload``: task_name / args + # / kwargs / queue / fairness); the queue itself stays payload-agnostic. message = models.JSONField() # "" = no org (leaf tasks); the fair-admission query (a later phase) # uses it for the coupled pipeline. Empty string rather than NULL — diff --git a/workers/queue_backend/dispatch.py b/workers/queue_backend/dispatch.py index e986ecb6f4..68261293df 100644 --- a/workers/queue_backend/dispatch.py +++ b/workers/queue_backend/dispatch.py @@ -1,29 +1,74 @@ """Transport-agnostic task dispatch. -Thin pass-through to ``celery.current_app.send_task``; the indirection -is the seam — a future per-task router can land here without touching -call sites. +Routes each task to its transport via :func:`select_backend`: + +- **Celery** (default) — a thin pass-through to ``current_app.send_task``. +- **PG Queue** — when a task is opted into ``WORKER_PG_QUEUE_ENABLED_TASKS``, + the task is serialised and enqueued to ``pg_queue_message`` (9b); the PG + consumer (9c) drains and runs it. + +The default (empty allow-list) routes everything to Celery, so dispatch is +unchanged unless an operator explicitly opts a task in. + +.. warning:: + A task opted into the PG queue **requires the PG consumer to be running** + — otherwise the message is durably enqueued but never executed. Only opt + in tasks once the consumer is deployed (and, per the migration-coherence + decision, only *leaf* tasks until execution-level routing exists). """ from __future__ import annotations import logging +import threading from collections.abc import Mapping, Sequence +from dataclasses import dataclass from typing import Any from celery import current_app from .fairness import FairnessKey from .handle import DispatchHandle +from .pg_queue import PgQueueClient, to_payload from .routing import QueueBackend, select_backend logger = logging.getLogger(__name__) -# Task names already logged as PG-routed in this process. Bounds the -# routing log to once per task name (per prefork child) so an opted-in -# high-throughput task doesn't log on every dispatch. +# pg_queue_message.queue_name used when a dispatch carries no Celery queue. +_DEFAULT_PG_QUEUE = "default" + +# Task names already logged as PG-routed in this process — bounds the cutover +# log to once per task name (per prefork child). _pg_routing_logged: set[str] = set() +# Per-thread client, reused across dispatches (it self-recovers a dead +# connection). The worker pool is prefork (each child a single thread), so +# this is effectively one client per child; ``threading.local`` keeps it +# correct under a ``-P threads`` pool too, since a libpq connection is not +# safe for concurrent use across threads. (A gevent pool shares one thread +# across greenlets and would need a connection pool — out of scope while +# prefork is the deployment.) +_pg_local = threading.local() + + +def _get_pg_client() -> PgQueueClient: + client = getattr(_pg_local, "client", None) + if client is None: + client = PgQueueClient() + _pg_local.client = client + return client + + +@dataclass(frozen=True, slots=True) +class PgDispatchHandle: + """:class:`~queue_backend.handle.TaskHandle` for a PG-enqueued task. + + ``id`` is the ``pg_queue_message.msg_id`` (as a string), so call sites + that read ``handle.id`` keep working across the Celery/PG boundary. + """ + + id: str + def dispatch( task_name: str, @@ -33,32 +78,14 @@ def dispatch( queue: str | None = None, fairness: FairnessKey | None = None, ) -> DispatchHandle: - """Enqueue a task by name. - - ``fairness`` is attached as the ``x-fairness-key`` header (not in - kwargs). Pass ``None`` for non-workflow worker tasks. - - The transport is chosen by :func:`select_backend` from the PG-queue - routing table (``WORKER_PG_QUEUE_ENABLED_TASKS``). In this phase the - table is a scaffold: PG-selected tasks are *logged* but still - dispatched via Celery, because no PG consumer exists yet. The - ``QueueBackend.PG`` branch below is the seam where the real PG - enqueue lands in a later phase — keeping the ``send_task`` call - outside it guarantees today's wire is byte-identical regardless of - the routing decision. + """Enqueue a task by name onto its selected transport. + + ``fairness`` is attached as the ``x-fairness-key`` header on the Celery + path / serialised into the message on the PG path. Pass ``None`` for + non-workflow worker tasks. """ - if ( - select_backend(task_name) is QueueBackend.PG - and task_name not in _pg_routing_logged - ): - # INFO (not DEBUG) so the cutover is visible under a default log - # config; log-once per task name keeps the volume bounded. - _pg_routing_logged.add(task_name) - logger.info( - "PG-queue routing selected for task=%r; dispatching via " - "Celery (scaffold — no PG consumer yet)", - task_name, - ) + if select_backend(task_name) is QueueBackend.PG: + return _enqueue_pg(task_name, args, kwargs, queue, fairness) headers = fairness.as_header() if fairness is not None else None return current_app.send_task( @@ -68,3 +95,48 @@ def dispatch( queue=queue, headers=headers, ) + + +def _enqueue_pg( + task_name: str, + args: Sequence[Any] | None, + kwargs: Mapping[str, Any] | None, + queue: str | None, + fairness: FairnessKey | None, +) -> PgDispatchHandle: + """Serialise + enqueue a PG-routed task to ``pg_queue_message``. + + A PG enqueue failure raises (no silent Celery fallback — that would + hide the failure or risk double-dispatch). + """ + pg_queue = queue or _DEFAULT_PG_QUEUE + if task_name not in _pg_routing_logged: + # Log the routing *decision* once per task, BEFORE the send — it's + # true regardless of send outcome, so a first-dispatch failure + # (DB down / unmigrated) must not suppress the one announcement. + # INFO so it survives a default log config; once-per-task bounds it. + _pg_routing_logged.add(task_name) + logger.info( + "PG-queue: routing task=%r to Postgres (queue=%r). Requires the " + "PG consumer to be running, or it will not execute.", + task_name, + pg_queue, + ) + payload = to_payload( + task_name, args=args, kwargs=kwargs, queue=queue, fairness=fairness + ) + try: + msg_id = _get_pg_client().send( + pg_queue, + payload, + org_id=fairness.org_id if fairness is not None else None, + ) + except Exception: + # Re-raise with a breadcrumb (raw psycopg2.Error / a json.dumps + # TypeError on a non-serialisable arg would otherwise propagate with + # no "this was a PG-routed dispatch" context). No Celery fallback. + logger.exception( + "PG-queue: failed to enqueue task=%r to queue=%r", task_name, pg_queue + ) + raise + return PgDispatchHandle(id=str(msg_id)) diff --git a/workers/queue_backend/fairness.py b/workers/queue_backend/fairness.py index 83af4fea1d..160b393e7b 100644 --- a/workers/queue_backend/fairness.py +++ b/workers/queue_backend/fairness.py @@ -9,7 +9,19 @@ from dataclasses import dataclass from enum import StrEnum -from typing import Final +from typing import Final, TypedDict + + +class FairnessPayload(TypedDict): + """Serialised :class:`FairnessKey` (``to_dict()`` / wire shape). + + Shared so producers and the PG ``TaskPayload`` wire contract agree on + the exact sub-shape instead of a loose ``dict[str, Any]``. + """ + + org_id: str | None + workload_type: str + pipeline_priority: int class WorkloadType(StrEnum): @@ -47,12 +59,12 @@ def __post_init__(self) -> None: f"[{MIN_PRIORITY}, {MAX_PRIORITY}]: {self.pipeline_priority}" ) - def to_dict(self) -> dict[str, str | int | None]: - return { - "org_id": self.org_id, - "workload_type": self.workload_type.value, - "pipeline_priority": self.pipeline_priority, - } + def to_dict(self) -> FairnessPayload: + return FairnessPayload( + org_id=self.org_id, + workload_type=self.workload_type.value, + pipeline_priority=self.pipeline_priority, + ) def as_header(self) -> dict[str, dict[str, str | int | None]]: """Celery ``send_task(headers=...)`` payload. diff --git a/workers/queue_backend/pg_queue/__init__.py b/workers/queue_backend/pg_queue/__init__.py index 4b0aee5782..31f51e253d 100644 --- a/workers/queue_backend/pg_queue/__init__.py +++ b/workers/queue_backend/pg_queue/__init__.py @@ -14,12 +14,15 @@ real implementation will likely span several modules (config, consumer poll loop, orchestrator) — exact layout TBD. -Phase 9a adds the storage + dequeue primitive (:class:`PgQueueClient` -over the ``pg_queue_message`` table; the ``SKIP LOCKED`` dequeue lives in -the client). It is still **inert**: routing decisions are made by -:func:`queue_backend.routing.select_backend`, and until the enqueue -wiring (9b) and consumer (9c) land, PG-selected dispatches still ride -Celery (see ``queue_backend.dispatch``). +9a added the storage + dequeue primitive (:class:`PgQueueClient` over the +``pg_queue_message`` table; the ``SKIP LOCKED`` dequeue lives in the +client). 9b wires :func:`queue_backend.dispatch.dispatch` to enqueue +PG-selected tasks here as a :class:`TaskPayload` instead of sending to +Celery. The consumer poll loop that drains + runs them is 9c — until it +lands, an opted-in task is durably enqueued but not executed. + +Default-empty ``WORKER_PG_QUEUE_ENABLED_TASKS`` → everything still routes +to Celery, so this is inert unless a task is explicitly opted in. Design reference: the PG Queue implementation guide in the labs repo (``workflow-execution-architecture``). Branch and section pointers move, @@ -28,9 +31,12 @@ from .client import PgQueueClient, QueueMessage from .connection import create_pg_connection +from .task_payload import TaskPayload, to_payload __all__ = [ "PgQueueClient", "QueueMessage", + "TaskPayload", "create_pg_connection", + "to_payload", ] diff --git a/workers/queue_backend/pg_queue/task_payload.py b/workers/queue_backend/pg_queue/task_payload.py new file mode 100644 index 0000000000..3a7cc5e7fd --- /dev/null +++ b/workers/queue_backend/pg_queue/task_payload.py @@ -0,0 +1,53 @@ +"""Payload carried *inside* a PG queue row, for a dispatched task. + +Not to be confused with the queue **row** itself (the backend's +``PgQueueMessage`` model / ``pg_queue_message`` table). This is the shape +of the row's ``message`` JSONB column when the message is a task: a +``dispatch()`` that routes to PG (9b) serialises a :class:`TaskPayload` +into that column, and the consumer poll loop (9c) decodes it and runs the +task. Producer↔consumer contract — keep both sides reading the same keys. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Any, TypedDict + +from ..fairness import FairnessPayload + +if TYPE_CHECKING: + from ..fairness import FairnessKey + + +class TaskPayload(TypedDict): + """Everything the 9c consumer needs to run a PG-routed task. + + ``fairness`` is the serialised :class:`FairnessKey` + (:class:`~queue_backend.fairness.FairnessPayload`) or ``None`` — the + consumer rebuilds the ``x-fairness-key`` header from it, mirroring the + Celery dispatch path. + """ + + task_name: str + args: list[Any] + kwargs: dict[str, Any] + queue: str | None + fairness: FairnessPayload | None + + +def to_payload( + task_name: str, + *, + args: Sequence[Any] | None = None, + kwargs: Mapping[str, Any] | None = None, + queue: str | None = None, + fairness: FairnessKey | None = None, +) -> TaskPayload: + """Build the JSON-serialisable task payload for the PG queue.""" + return TaskPayload( + task_name=task_name, + args=list(args) if args is not None else [], + kwargs=dict(kwargs) if kwargs is not None else {}, + queue=queue, + fairness=fairness.to_dict() if fairness is not None else None, + ) diff --git a/workers/sample.env b/workers/sample.env index 9d6f6a61fe..8ad7df3612 100644 --- a/workers/sample.env +++ b/workers/sample.env @@ -139,14 +139,14 @@ WORKER_BARRIER_KEY_TTL_SECONDS=21600 # batch's fan-in fires the callback; this is how messages travel. # # A comma-separated allow-list drives a gradual, per-task-type migration -# (NOT a global on/off switch). A task routes to PG if its name is -# listed. Unset / empty (the default) → everything routes to Celery, -# exactly as today. +# (NOT a global on/off switch). A task routes to PG if its name is listed. +# Unset / empty (the default) → everything routes to Celery, zero +# behaviour change. No Flipt server required. # -# Scaffold note: in this release there is no PG consumer yet, so -# PG-selected tasks are logged but still dispatched via Celery. Setting -# this flag is therefore safe to test routing decisions locally with -# zero behaviour change. No Flipt server required. +# ⚠️ A listed task is enqueued to Postgres instead of Celery. It will only +# execute if the PG consumer is running — otherwise the message is durably +# enqueued but never run. Do NOT opt a task in unless the consumer is +# deployed. # # Roll back instantly by removing an entry; flips take effect on worker # restart (same posture as every other WORKER_* env). diff --git a/workers/tests/test_dispatch_pg.py b/workers/tests/test_dispatch_pg.py new file mode 100644 index 0000000000..6c2f974f74 --- /dev/null +++ b/workers/tests/test_dispatch_pg.py @@ -0,0 +1,137 @@ +"""dispatch() PG enqueue (9b) + the ``TaskPayload`` wire shape. + +The mocked routing/handle behaviour lives in ``test_routing.py``; this +file covers the message serialisation contract and an end-to-end +integration check (a PG-selected ``dispatch()`` lands a decodable row in +``pg_queue_message``). The integration test skips if Postgres is +unreachable or unmigrated. +""" + +from __future__ import annotations + +import importlib +import json +import os +from unittest.mock import patch + +import psycopg2 +import pytest +from queue_backend import dispatch +from queue_backend.fairness import FairnessKey, WorkloadType +from queue_backend.pg_queue import PgQueueClient, to_payload +from queue_backend.pg_queue.connection import create_pg_connection +from queue_backend.routing import _ENABLED_TASKS_ENV_VAR as ENABLED_TASKS_ENV + +# ``queue_backend.dispatch`` the attribute is the function (shadows the +# submodule) — import the module explicitly to reach its globals. +dispatch_mod = importlib.import_module("queue_backend.dispatch") + + +@pytest.fixture(autouse=True) +def _reset(monkeypatch): + monkeypatch.delenv(ENABLED_TASKS_ENV, raising=False) + dispatch_mod._pg_routing_logged.clear() + dispatch_mod._pg_local.client = None + + +# --- to_payload wire shape --- + + +class TestToPayload: + def test_minimal(self): + assert to_payload("t") == { + "task_name": "t", + "args": [], + "kwargs": {}, + "queue": None, + "fairness": None, + } + + def test_full(self): + fairness = FairnessKey( + org_id="o", workload_type=WorkloadType.API, pipeline_priority=7 + ) + payload = to_payload( + "t", args=("a", 1), kwargs={"k": "v"}, queue="q", fairness=fairness + ) + assert payload["args"] == ["a", 1] # tuple → list + assert payload["queue"] == "q" + assert payload["fairness"] == { + "org_id": "o", + "workload_type": "api", + "pipeline_priority": 7, + } + + def test_json_serialisable(self): + fairness = FairnessKey(org_id="o", workload_type=WorkloadType.NON_API) + # Must round-trip through JSON (it's stored in a JSONB column). + json.dumps(to_payload("t", args=[1], kwargs={"a": "b"}, fairness=fairness)) + + +# --- Integration: dispatch() lands a decodable row in pg_queue_message --- + + +def _test_conn(): + os.environ.setdefault("TEST_DB_HOST", "127.0.0.1") + return create_pg_connection(env_prefix="TEST_DB_") + + +@pytest.fixture +def pg_client(): + try: + conn = _test_conn() + except psycopg2.OperationalError as exc: + pytest.skip(f"Postgres not reachable: {exc}") + with conn.cursor() as cur: + cur.execute("SELECT to_regclass('pg_queue_message')") + if cur.fetchone()[0] is None: + conn.close() + pytest.skip("pg_queue migration not applied (run backend migrate)") + yield PgQueueClient(conn=conn) + conn.rollback() + conn.close() + + +class TestDispatchEnqueueIntegration: + def test_dispatch_lands_decodable_row(self, monkeypatch, pg_client): + queue_name = f"test_dispatch_{os.getpid()}" + monkeypatch.setenv(ENABLED_TASKS_ENV, "leaf_task") + monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: pg_client) + fairness = FairnessKey(org_id="org-x", workload_type=WorkloadType.API) + try: + handle = dispatch( + "leaf_task", + args=[1, 2], + kwargs={"k": "v"}, + queue=queue_name, + fairness=fairness, + ) + msgs = pg_client.read(queue_name, vt_seconds=30, qty=10) + assert len(msgs) == 1 + assert str(msgs[0].msg_id) == handle.id + message = msgs[0].message + assert message["task_name"] == "leaf_task" + assert message["args"] == [1, 2] + assert message["kwargs"] == {"k": "v"} + assert message["queue"] == queue_name + assert message["fairness"]["org_id"] == "org-x" + finally: + with pg_client.conn.cursor() as cur: + cur.execute( + "DELETE FROM pg_queue_message WHERE queue_name = %s", (queue_name,) + ) + pg_client.conn.commit() + + def test_celery_dispatch_unaffected(self): + # No opt-in → Celery path; never touches the PG client. + with ( + patch("queue_backend.dispatch.current_app") as mock_app, + patch("queue_backend.dispatch._get_pg_client") as mock_get, + ): + dispatch("some_task", args=["x"], queue="general") + mock_app.send_task.assert_called_once() + mock_get.assert_not_called() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/workers/tests/test_routing.py b/workers/tests/test_routing.py index f8b55473b4..7c3e6447e8 100644 --- a/workers/tests/test_routing.py +++ b/workers/tests/test_routing.py @@ -20,7 +20,7 @@ import importlib import logging -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from queue_backend import QueueBackend, dispatch, select_backend @@ -44,6 +44,7 @@ def _reset_routing_state(monkeypatch): monkeypatch.delenv(ENABLED_TASKS_ENV, raising=False) routing_mod._allow_list_logged = False dispatch_mod._pg_routing_logged.clear() + dispatch_mod._pg_local.client = None # drop the per-thread PG client # --- select_backend() resolution --- @@ -119,79 +120,109 @@ def test_empty_allow_list_logs_nothing(self, caplog): select_backend("x") assert "PG-queue routing enabled" not in caplog.text - def test_pg_selection_emits_routing_log(self, monkeypatch, caplog): - """Pins the dispatch() routing branch: deleting it must fail here.""" - monkeypatch.setenv(ENABLED_TASKS_ENV, "t1") - with ( - patch("queue_backend.dispatch.current_app"), - caplog.at_level(logging.INFO, logger="queue_backend.dispatch"), - ): - dispatch("t1") - assert "PG-queue routing selected" in caplog.text - def test_celery_selection_emits_no_routing_log(self, caplog): - """Negative case — guards against the gate being made unconditional.""" +# --- dispatch() routing behaviour (9b: PG-selected enqueues to Postgres) --- + + +def _mock_pg_client(monkeypatch, *, msg_id=99): + client = MagicMock() + client.send.return_value = msg_id + # Patch on the module object (string target would navigate the shadowing + # ``dispatch`` *function*, not the submodule). + monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: client) + return client + + +class TestDispatchRouting: + """Celery path unchanged; a PG-selected task enqueues to Postgres, not Celery.""" + + def test_celery_path_sends_to_celery_and_never_touches_pg(self): with ( - patch("queue_backend.dispatch.current_app"), - caplog.at_level(logging.INFO, logger="queue_backend.dispatch"), + patch("queue_backend.dispatch.current_app") as mock_app, + patch("queue_backend.dispatch._get_pg_client") as mock_get, ): - dispatch("t1") # empty allow-list → CELERY - assert "PG-queue routing selected" not in caplog.text + dispatch("t1", args=["a"], kwargs={"k": "v"}, queue="general") + mock_app.send_task.assert_called_once() + mock_get.assert_not_called() - def test_routing_log_bounded_to_once_per_task(self, monkeypatch, caplog): + def test_pg_selected_enqueues_to_pg_not_celery(self, monkeypatch): + monkeypatch.setenv(ENABLED_TASKS_ENV, "t1") + client = _mock_pg_client(monkeypatch, msg_id=99) + fairness = FairnessKey(org_id="org-1", workload_type=WorkloadType.API) + with patch("queue_backend.dispatch.current_app") as mock_app: + handle = dispatch( + "t1", args=["a", 1], kwargs={"k": "v"}, queue="general", fairness=fairness + ) + mock_app.send_task.assert_not_called() + client.send.assert_called_once() + queue_name, message = client.send.call_args.args + assert queue_name == "general" + assert message["task_name"] == "t1" + assert message["args"] == ["a", 1] + assert message["kwargs"] == {"k": "v"} + assert message["queue"] == "general" + assert message["fairness"]["org_id"] == "org-1" + assert client.send.call_args.kwargs["org_id"] == "org-1" + # Handle satisfies TaskHandle (.id) and carries the msg_id. + assert handle.id == "99" + + def test_pg_default_queue_name_when_unset(self, monkeypatch): + monkeypatch.setenv(ENABLED_TASKS_ENV, "t1") + client = _mock_pg_client(monkeypatch) + dispatch("t1") + assert client.send.call_args.args[0] == "default" + # No fairness → org_id None (client coerces to ""). + assert client.send.call_args.kwargs["org_id"] is None + + def test_pg_enqueue_failure_propagates_and_does_not_fall_back(self, monkeypatch): + """A PG enqueue failure raises — no silent Celery fallback.""" monkeypatch.setenv(ENABLED_TASKS_ENV, "t1") + client = MagicMock() + client.send.side_effect = RuntimeError("db down") + monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: client) with ( - patch("queue_backend.dispatch.current_app"), - caplog.at_level(logging.INFO, logger="queue_backend.dispatch"), + patch("queue_backend.dispatch.current_app") as mock_app, + pytest.raises(RuntimeError), ): dispatch("t1") + mock_app.send_task.assert_not_called() # never falls back to Celery + # The routing decision is logged BEFORE the send, so even a failing + # first dispatch announced it (pins the log-ordering property). + assert "t1" in dispatch_mod._pg_routing_logged + + def test_get_pg_client_lazily_inits_and_reuses(self, monkeypatch): + """The per-thread client is constructed once and reused (connection reuse).""" + dispatch_mod._pg_local.client = None + sentinel = object() + ctor = MagicMock(return_value=sentinel) + monkeypatch.setattr(dispatch_mod, "PgQueueClient", ctor) + assert dispatch_mod._get_pg_client() is sentinel + assert dispatch_mod._get_pg_client() is sentinel + ctor.assert_called_once() + + +class TestCutoverLog: + """The PG cutover log: visible (INFO), once per task name.""" + + def test_pg_enqueue_logs_once_per_task(self, monkeypatch, caplog): + monkeypatch.setenv(ENABLED_TASKS_ENV, "t1") + _mock_pg_client(monkeypatch) + with caplog.at_level(logging.INFO, logger="queue_backend.dispatch"): + dispatch("t1") dispatch("t1") dispatch("t1") - hits = [r for r in caplog.records if "PG-queue routing selected" in r.getMessage()] + hits = [ + r for r in caplog.records if "routing task=" in r.getMessage() + ] assert len(hits) == 1 - -# --- dispatch() is inert under routing (the scaffold invariant) --- - - -class TestDispatchByteIdenticalRegardlessOfRouting: - """``dispatch()`` produces the same send_task call whether routed PG or Celery.""" - - def _capture_send_task(self, task_name, fairness): - with patch("queue_backend.dispatch.current_app") as mock_app: - dispatch( - task_name, - args=["a", 1], - kwargs={"k": "v"}, - queue="general", - fairness=fairness, - ) - return mock_app.send_task.call_args - - def test_pg_selection_does_not_change_the_wire(self, monkeypatch): - fairness = FairnessKey(org_id="org-1", workload_type=WorkloadType.API) - - # Celery path (empty table). - celery_call = self._capture_send_task("t1", fairness) - - # PG path (task opted in) — same inputs. - monkeypatch.setenv(ENABLED_TASKS_ENV, "t1") - pg_call = self._capture_send_task("t1", fairness) - - assert celery_call == pg_call - assert pg_call.args[0] == "t1" - assert pg_call.kwargs["args"] == ["a", 1] - assert pg_call.kwargs["kwargs"] == {"k": "v"} - assert pg_call.kwargs["queue"] == "general" - - def test_dispatch_without_fairness_still_routes_by_task(self, monkeypatch): - """Routing keys on task name only; fairness is irrelevant to the decision.""" - monkeypatch.setenv(ENABLED_TASKS_ENV, "bare_task") - with patch("queue_backend.dispatch.current_app") as mock_app: - dispatch("bare_task") - # Still dispatched (via Celery), header is None (no fairness). - assert mock_app.send_task.call_args.args[0] == "bare_task" - assert mock_app.send_task.call_args.kwargs["headers"] is None + def test_celery_path_emits_no_pg_log(self, caplog): + with ( + patch("queue_backend.dispatch.current_app"), + caplog.at_level(logging.INFO, logger="queue_backend.dispatch"), + ): + dispatch("t1") # empty allow-list → Celery + assert "routing task=" not in caplog.text if __name__ == "__main__": From 12631c1025e2f9a58a3ebcee4e53d4604dcc45af Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Fri, 12 Jun 2026 11:19:45 +0530 Subject: [PATCH 05/44] =?UTF-8?q?UN-3539=20[FEAT]=20PG=20Queue=20Phase=209?= =?UTF-8?q?c=20=E2=80=94=20consumer=20poll=20loop=20(claim=20=E2=86=92=20r?= =?UTF-8?q?un=20=E2=86=92=20ack)=20(#2045)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3539 [FEAT] PG Queue Phase 9c — consumer poll loop (claim → run → ack) PgQueueConsumer drains pg_queue_message and runs each claimed task in-process: poll_once() claims a batch (SKIP LOCKED + vt via PgQueueClient.read), runs it via current_app.tasks[name].apply(throw=True), and acks by deleting on success. Task failure -> leave the row (vt expiry redelivers, at-least-once); unknown task -> drop + error (no poison loop). run() adds an empty-queue backoff loop + SIGTERM/SIGINT graceful stop; main() is a `python -m` entrypoint (env-configured). Completes the leaf-first end-to-end path: 8a route -> 9b enqueue -> 9a store -> 9c consume+run. Validated live on the dev stack: a real send_webhook_notification routed to PG was claimed by the consumer, POSTed to Slack (HTTP 200), and acked (row removed). Deployment note: the consumer PROCESS must bootstrap the worker app (import the task modules) so current_app.tasks resolves the task — an entrypoint/rollout concern, not consumer logic. Documented in the module docstring; the rollout phase wires a consumer container that boots the app. Tests: 5 unit (run+ack, fail->no-ack, unknown->drop, empty, graceful stop) + 1 real-PG integration (enqueue -> poll -> execute -> ack). Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3539 [FIX] Dedup PG integration test fixtures into conftest (SonarCloud) The connect-to-dev-DB + skip-if-unreachable/unmigrated block was copy-pasted across test_pg_queue_client / test_dispatch_pg / test_pg_queue_consumer (6.3% duplication on new code, over the 3% gate). Extracted into shared pg_conn / pg_client fixtures + an integration_pg_conn() helper in tests/conftest.py; the three files now use them. Behaviour unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3539 [FIX] Address Phase 9c review feedback - fairness header rebuilt from the payload on the PG run path (was dropped) — a PG-routed run now mirrors the Celery dispatch contract. - poison-message guard: surface read_ct (QueueMessage + dequeue RETURNING), drop a task that keeps failing past max_attempts (default 5, env WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS) with a loud ERROR carrying the payload, instead of redelivering forever. - malformed payload (missing task_name) -> distinct "missing task_name" drop log, not a misleading "unknown task None". - run() wraps poll_once() so a transient read/DB blip backs off and continues instead of tearing down the loop (the client self-recovers). - ack: WARN when delete() finds no row (task exceeded vt -> possible double-run). - __init__ validates positive tuning params; main() wires backoff_max + max_attempts via env (prefix helper); non-main-thread signal-install failure -> WARNING. - tests: poison drop, missing task_name, fairness-header propagation, multi-message batch, ack-no-row warn, construction validation, backoff growth/reset, poll-error resilience; fixed the read mock for read_ct. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3539 [FIX] Address Greptile review on the 9c consumer - [P1] batch shared-vt window: default batch_size 10 -> 1 so each message gets its own visibility window. The batch's vt is set atomically at claim but messages run sequentially, so with batch_size>1 the tail could exceed vt and be re-claimed mid-run (double-run). Batching stays opt-in; doc the vt > batch x worst-case-duration constraint. - [P2] QueueMessage.read_ct: drop the misleading =0 default (a "never claimed" state the dequeue can't produce — read_ct is always >=1). 0 would silently bypass the poison guard; now required, all callers supply it. - [P2] __init__: reject backoff_max < poll_interval (else min(poll*2, max) shrinks the backoff below poll_interval instead of growing). - [P2] dedup the ack-miss warning: client.delete() now logs at DEBUG; the consumer keeps the contextual WARNING (it names the task). Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- workers/queue_backend/pg_queue/client.py | 23 +- workers/queue_backend/pg_queue/consumer.py | 243 +++++++++++++++++++++ workers/tests/conftest.py | 44 ++++ workers/tests/test_dispatch_pg.py | 26 +-- workers/tests/test_pg_queue_client.py | 37 +--- workers/tests/test_pg_queue_consumer.py | 238 ++++++++++++++++++++ 6 files changed, 547 insertions(+), 64 deletions(-) create mode 100644 workers/queue_backend/pg_queue/consumer.py create mode 100644 workers/tests/test_pg_queue_consumer.py diff --git a/workers/queue_backend/pg_queue/client.py b/workers/queue_backend/pg_queue/client.py index fb61147b8c..030f8d8921 100644 --- a/workers/queue_backend/pg_queue/client.py +++ b/workers/queue_backend/pg_queue/client.py @@ -68,7 +68,7 @@ FOR UPDATE SKIP LOCKED LIMIT %s ) -RETURNING msg_id, message +RETURNING msg_id, message, read_ct """ @@ -79,11 +79,15 @@ class QueueMessage: ``message`` is the already-decoded JSONB payload (psycopg2 parses ``jsonb`` to a Python ``dict``). ``frozen`` freezes the binding only — the ``dict`` itself is mutable, so treat the payload as read-only by - convention. + convention. ``read_ct`` is the post-claim delivery count — always ``>= 1`` + from the dequeue (``read_ct = read_ct + 1 RETURNING``), so consumers can + cap redelivery of a poison message. No default: a ``0`` ("never claimed") + can't come from the dequeue and would silently bypass the poison guard. """ msg_id: int message: dict[str, Any] + read_ct: int class PgQueueClient: @@ -180,23 +184,26 @@ def read( with self._cursor() as cur: cur.execute(_DEQUEUE_SQL, (vt_seconds, queue_name, qty)) rows = cur.fetchall() - return [QueueMessage(msg_id=int(r[0]), message=r[1]) for r in rows] + return [ + QueueMessage(msg_id=int(r[0]), message=r[1], read_ct=int(r[2])) for r in rows + ] def delete(self, msg_id: int) -> bool: """Ack a processed message. Returns ``True`` if a row was removed. ``False`` means the row was already gone — typically its visibility timeout expired during processing and another worker (re)claimed it, - i.e. the work may be processed twice. Logged at WARNING so this - at-least-once condition is visible rather than silently swallowed. + i.e. the work may be processed twice. Logged at DEBUG here; the + consumer emits the contextual WARNING (it names the task), so this + avoids a duplicate warning per double-run. """ with self._cursor() as cur: cur.execute("DELETE FROM pg_queue_message WHERE msg_id = %s", (msg_id,)) deleted = cur.rowcount if deleted == 0: - logger.warning( - "PG-queue: delete(msg_id=%s) removed no row — message was already " - "gone (vt likely expired during processing; possible re-delivery).", + logger.debug( + "PG-queue: delete(msg_id=%s) removed no row — already gone " + "(vt likely expired; consumer logs the contextual warning).", msg_id, ) return deleted == 1 diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py new file mode 100644 index 0000000000..a5c2024201 --- /dev/null +++ b/workers/queue_backend/pg_queue/consumer.py @@ -0,0 +1,243 @@ +"""PG queue consumer — claims tasks from ``pg_queue_message`` and runs them. + +The producer side (9b) enqueues a :class:`~queue_backend.pg_queue.TaskPayload` +when a task is routed to PG. This is the other half: it polls the queue with +``SKIP LOCKED`` + a visibility timeout (via :class:`PgQueueClient`), runs each +claimed task **in-process** (no Celery broker), and acks by deleting the row. + +A task that fails — or a crash before ack — is redelivered once its ``vt`` +expires (at-least-once; tasks must be idempotent), bounded by ``max_attempts`` +(``read_ct``): a task that keeps failing past the cap is dropped as a poison +message (logged with its payload) rather than redelivered forever. A message +with no ``task_name`` (malformed/foreign) or a name not in the registry can +never run, so it is likewise dropped with a loud log. The fairness header is +rebuilt from the payload so a PG-routed run mirrors the Celery dispatch path. + +Run as ``python -m queue_backend.pg_queue.consumer`` (config via env). The +worker bootstrap must have imported/registered the Celery tasks so they +resolve in ``current_app.tasks``. +""" + +from __future__ import annotations + +import logging +import os +import signal +import time +from typing import TYPE_CHECKING + +from celery import current_app + +from ..fairness import FAIRNESS_HEADER_NAME +from .client import PgQueueClient + +if TYPE_CHECKING: + from celery import Celery + + from .client import QueueMessage + +logger = logging.getLogger(__name__) + +_DEFAULT_QUEUE = "default" +# Default 1: the whole batch shares one vt window (set atomically at claim), +# but messages run sequentially — so with batch_size > 1 the tail can exceed +# its vt and be re-claimed mid-run (double-run). Batching is opt-in; if you +# raise it, keep vt_seconds > batch_size x worst-case task duration. +_DEFAULT_BATCH = 1 +_DEFAULT_VT_SECONDS = 30 +_DEFAULT_POLL_INTERVAL = 0.1 +_DEFAULT_BACKOFF_MAX = 2.0 +# A task claimed more than this many times keeps failing — drop it (poison) +# rather than redeliver forever. +_DEFAULT_MAX_ATTEMPTS = 5 + + +class PgQueueConsumer: + """Polls one PG queue, runs each claimed task in-process, acks on success.""" + + def __init__( + self, + queue_name: str, + *, + client: PgQueueClient | None = None, + app: Celery | None = None, + batch_size: int = _DEFAULT_BATCH, + vt_seconds: int = _DEFAULT_VT_SECONDS, + poll_interval: float = _DEFAULT_POLL_INTERVAL, + backoff_max: float = _DEFAULT_BACKOFF_MAX, + max_attempts: int = _DEFAULT_MAX_ATTEMPTS, + ) -> None: + # Validate at construction so a misconfigured consumer fails here + # rather than batch-after-batch once the loop starts. + for name, value in ( + ("batch_size", batch_size), + ("vt_seconds", vt_seconds), + ("poll_interval", poll_interval), + ("backoff_max", backoff_max), + ("max_attempts", max_attempts), + ): + if value <= 0: + raise ValueError(f"{name} must be positive, got {value!r}") + if backoff_max < poll_interval: + # Otherwise min(poll_interval*2, backoff_max) shrinks the backoff + # below poll_interval — it would decrease instead of grow. + raise ValueError( + f"backoff_max ({backoff_max}) must be >= poll_interval " + f"({poll_interval})" + ) + self.queue_name = queue_name + self._client = client if client is not None else PgQueueClient() + self._app = app if app is not None else current_app + self.batch_size = batch_size + self.vt_seconds = vt_seconds + self.poll_interval = poll_interval + self.backoff_max = backoff_max + self.max_attempts = max_attempts + self._running = False + + def poll_once(self) -> int: + """Claim + process one batch; returns the number of messages claimed.""" + messages = self._client.read( + self.queue_name, vt_seconds=self.vt_seconds, qty=self.batch_size + ) + for message in messages: + self._handle(message) + return len(messages) + + def _handle(self, message: QueueMessage) -> None: + payload = message.message + task_name = payload.get("task_name") + + # Malformed / foreign payload: no task name → can't run; drop with a + # log that points at the payload, not at task registration. + if not task_name: + logger.error( + "PG-queue consumer: payload missing task_name (msg_id=%s) — " + "dropping malformed message: %r", + message.msg_id, + payload, + ) + self._client.delete(message.msg_id) + return + + # Poison message: a task re-claimed past the cap keeps failing. Drop + # it (with the payload, so it's recoverable from logs) instead of + # redelivering on every vt expiry forever. + if message.read_ct > self.max_attempts: + logger.error( + "PG-queue consumer: task %r (msg_id=%s) exceeded max_attempts=%s " + "(read_ct=%s) — dropping poison message: %r", + task_name, + message.msg_id, + self.max_attempts, + message.read_ct, + payload, + ) + self._client.delete(message.msg_id) + return + + task = self._app.tasks.get(task_name) + if task is None: + # A named-but-unregistered task can never run → drop and shout. + logger.error( + "PG-queue consumer: unknown task %r (msg_id=%s) — dropping", + task_name, + message.msg_id, + ) + self._client.delete(message.msg_id) + return + + try: + # Run the task body in-process (eager), carrying the fairness + # header so a PG-routed run mirrors the Celery dispatch path. + fairness = payload.get("fairness") + headers = {FAIRNESS_HEADER_NAME: fairness} if fairness else None + task.apply( + args=payload.get("args") or [], + kwargs=payload.get("kwargs") or {}, + headers=headers, + throw=True, + ) + except Exception: + # Leave the row: its vt expires and it is redelivered (bounded by + # max_attempts above). + logger.exception( + "PG-queue consumer: task %r (msg_id=%s, read_ct=%s) failed — " + "leaving for vt-expiry redelivery", + task_name, + message.msg_id, + message.read_ct, + ) + return + + if not self._client.delete(message.msg_id): # ack + logger.warning( + "PG-queue consumer: ack found no row for task %r (msg_id=%s) — " + "it likely exceeded vt and was re-claimed (possible double-run)", + task_name, + message.msg_id, + ) + + def run(self, *, install_signals: bool = True) -> None: + """Poll loop with empty-queue backoff and graceful shutdown.""" + self._running = True + if install_signals: + self._install_signal_handlers() + logger.info( + "PG-queue consumer started (queue=%r, batch=%s, vt=%ss)", + self.queue_name, + self.batch_size, + self.vt_seconds, + ) + backoff = self.poll_interval + while self._running: + try: + claimed = self.poll_once() + except Exception: + # A transient read/DB blip must not tear down the loop — the + # client self-recovers its connection, so log and back off. + logger.exception( + "PG-queue consumer: poll cycle failed; backing off and continuing" + ) + claimed = 0 + if claimed: + backoff = self.poll_interval + else: + time.sleep(backoff) + backoff = min(backoff * 2, self.backoff_max) + logger.info("PG-queue consumer stopped (queue=%r)", self.queue_name) + + def stop(self, *_: object) -> None: + """Request a graceful stop after the current batch.""" + self._running = False + + def _install_signal_handlers(self) -> None: + # signal.signal only works in the main thread. + try: + signal.signal(signal.SIGTERM, self.stop) + signal.signal(signal.SIGINT, self.stop) + except ValueError: + logger.warning( + "PG-queue consumer: signal handlers not installed (non-main " + "thread) — SIGTERM/SIGINT will not trigger graceful shutdown" + ) + + +def main() -> None: + logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) + + def _env(suffix: str, default: object, cast: type) -> object: + return cast(os.getenv(f"WORKER_PG_QUEUE_CONSUMER_{suffix}", default)) + + PgQueueConsumer( + queue_name=_env("QUEUE", _DEFAULT_QUEUE, str), + batch_size=_env("BATCH", _DEFAULT_BATCH, int), + vt_seconds=_env("VT_SECONDS", _DEFAULT_VT_SECONDS, int), + poll_interval=_env("POLL_INTERVAL", _DEFAULT_POLL_INTERVAL, float), + backoff_max=_env("BACKOFF_MAX", _DEFAULT_BACKOFF_MAX, float), + max_attempts=_env("MAX_ATTEMPTS", _DEFAULT_MAX_ATTEMPTS, int), + ).run() + + +if __name__ == "__main__": + main() diff --git a/workers/tests/conftest.py b/workers/tests/conftest.py index 084a8ef88c..2939c1aa89 100644 --- a/workers/tests/conftest.py +++ b/workers/tests/conftest.py @@ -6,9 +6,53 @@ time if INTERNAL_API_BASE_URL is not set. """ +import os from pathlib import Path +import psycopg2 +import pytest from dotenv import load_dotenv _env_test = Path(__file__).resolve().parent.parent / ".env.test" load_dotenv(_env_test) + + +# --- Shared PG-queue integration fixtures (real Postgres) --- +# +# Connect to the dev DB via TEST_DB_* — NOT the generic DB_*, which the +# suite's unit isolation sets to DB_USER=test. Skip gracefully when Postgres +# is unreachable or the pg_queue migration hasn't been applied, so CI without +# a migrated DB doesn't fail (only the real-DB seam is gated). + + +def integration_pg_conn(): + """Raw psycopg2 connection to the dev DB (host defaults to localhost).""" + from queue_backend.pg_queue.connection import create_pg_connection + + os.environ.setdefault("TEST_DB_HOST", "127.0.0.1") + return create_pg_connection(env_prefix="TEST_DB_") + + +@pytest.fixture +def pg_conn(): + """A real connection; skips if Postgres is unreachable or unmigrated.""" + try: + conn = integration_pg_conn() + except psycopg2.OperationalError as exc: + pytest.skip(f"Postgres not reachable: {exc}") + with conn.cursor() as cur: + cur.execute("SELECT to_regclass('pg_queue_message')") + if cur.fetchone()[0] is None: + conn.close() + pytest.skip("pg_queue migration not applied (run backend migrate)") + yield conn + conn.rollback() + conn.close() + + +@pytest.fixture +def pg_client(pg_conn): + """A :class:`PgQueueClient` over the integration connection.""" + from queue_backend.pg_queue import PgQueueClient + + return PgQueueClient(conn=pg_conn) diff --git a/workers/tests/test_dispatch_pg.py b/workers/tests/test_dispatch_pg.py index 6c2f974f74..75d840962f 100644 --- a/workers/tests/test_dispatch_pg.py +++ b/workers/tests/test_dispatch_pg.py @@ -14,12 +14,10 @@ import os from unittest.mock import patch -import psycopg2 import pytest from queue_backend import dispatch from queue_backend.fairness import FairnessKey, WorkloadType -from queue_backend.pg_queue import PgQueueClient, to_payload -from queue_backend.pg_queue.connection import create_pg_connection +from queue_backend.pg_queue import to_payload from queue_backend.routing import _ENABLED_TASKS_ENV_VAR as ENABLED_TASKS_ENV # ``queue_backend.dispatch`` the attribute is the function (shadows the @@ -69,27 +67,7 @@ def test_json_serialisable(self): # --- Integration: dispatch() lands a decodable row in pg_queue_message --- - - -def _test_conn(): - os.environ.setdefault("TEST_DB_HOST", "127.0.0.1") - return create_pg_connection(env_prefix="TEST_DB_") - - -@pytest.fixture -def pg_client(): - try: - conn = _test_conn() - except psycopg2.OperationalError as exc: - pytest.skip(f"Postgres not reachable: {exc}") - with conn.cursor() as cur: - cur.execute("SELECT to_regclass('pg_queue_message')") - if cur.fetchone()[0] is None: - conn.close() - pytest.skip("pg_queue migration not applied (run backend migrate)") - yield PgQueueClient(conn=conn) - conn.rollback() - conn.close() +# Uses the shared ``pg_client`` fixture from conftest.py. class TestDispatchEnqueueIntegration: diff --git a/workers/tests/test_pg_queue_client.py b/workers/tests/test_pg_queue_client.py index fbc4bcd579..20bb72a4ba 100644 --- a/workers/tests/test_pg_queue_client.py +++ b/workers/tests/test_pg_queue_client.py @@ -69,14 +69,14 @@ def test_send_coerces_missing_org_to_empty_string(self): assert params[2] == "" def test_read_runs_skip_locked_dequeue(self): - conn, cur = _mock_conn(fetchall=[(7, {"k": "v"})]) + conn, cur = _mock_conn(fetchall=[(7, {"k": "v"}, 1)]) msgs = PgQueueClient(conn=conn).read("q1", vt_seconds=15, qty=3) sql, params = cur.execute.call_args.args assert "FOR UPDATE SKIP LOCKED" in sql assert "UPDATE pg_queue_message" in sql # Param order follows the %s positions: vt_seconds, queue_name, qty. assert params == (15, "q1", 3) - assert msgs == [QueueMessage(msg_id=7, message={"k": "v"})] + assert msgs == [QueueMessage(msg_id=7, message={"k": "v"}, read_ct=1)] conn.commit.assert_called_once() def test_delete_returns_true_when_row_removed(self): @@ -231,35 +231,7 @@ def boom(**_): assert "failed to connect" in caplog.text -# --- Integration: real Postgres --- - - -def _integration_conn(): - # Reuse create_pg_connection (single source of connection logic) via the - # dedicated TEST_DB_* prefix — NOT the generic DB_*, which the suite's - # conftest sets to DB_USER=test for unit isolation (wrong real-DB creds). - # Default the host to the dev-compose published port on localhost. - os.environ.setdefault("TEST_DB_HOST", "127.0.0.1") - return create_pg_connection(env_prefix="TEST_DB_") - - -@pytest.fixture -def pg_conn(): - try: - conn = _integration_conn() - except psycopg2.OperationalError as exc: - # Only an unreachable/unauthenticated DB skips — ImportError, bugs, - # and schema/permission errors surface as real failures. - pytest.skip(f"Postgres not reachable: {exc}") - with conn.cursor() as cur: - cur.execute("SELECT to_regclass('pg_queue_message')") - (table,) = cur.fetchone() - if table is None: - conn.close() - pytest.skip("pg_queue migration not applied (run backend migrate)") - yield conn - conn.rollback() - conn.close() +# --- Integration: real Postgres (pg_conn fixture from conftest.py) --- @pytest.fixture @@ -306,7 +278,8 @@ def test_no_double_delivery_across_readers(self, pg_conn, queue_name): client_a = PgQueueClient(conn=pg_conn) for i in range(5): client_a.send(queue_name, {"i": i}) - conn_b = _integration_conn() + # Second connection (TEST_DB_HOST already set by the pg_conn fixture). + conn_b = create_pg_connection(env_prefix="TEST_DB_") try: client_b = PgQueueClient(conn=conn_b) ids_a = {m.msg_id for m in client_a.read(queue_name, vt_seconds=30, qty=3)} diff --git a/workers/tests/test_pg_queue_consumer.py b/workers/tests/test_pg_queue_consumer.py new file mode 100644 index 0000000000..02e7d9c9cd --- /dev/null +++ b/workers/tests/test_pg_queue_consumer.py @@ -0,0 +1,238 @@ +"""Tests for the PG queue consumer (claim → run → ack). + +Unit tests drive ``poll_once`` with a mocked client and *real* registered +Celery tasks (so ``apply()`` actually runs the body). The integration test +exercises the full enqueue → poll → execute → ack loop against real Postgres +(skips if unreachable / unmigrated). +""" + +from __future__ import annotations + +import logging +import os +from unittest.mock import MagicMock + +import pytest +from celery import shared_task +from queue_backend.fairness import FAIRNESS_HEADER_NAME +from queue_backend.pg_queue import to_payload +from queue_backend.pg_queue.client import QueueMessage +from queue_backend.pg_queue.consumer import PgQueueConsumer + +# Registered test tasks (namespaced). apply() runs their bodies in-process. +_calls: list = [] + + +@shared_task(name="test_pg_consumer.ok") +def _ok_task(x, y=0): + _calls.append((x, y)) + return x + y + + +@shared_task(name="test_pg_consumer.boom") +def _boom_task(): + raise RuntimeError("boom") + + +@pytest.fixture(autouse=True) +def _clear_calls(): + _calls.clear() + + +def _msg(msg_id, payload, *, read_ct=1): + return QueueMessage(msg_id=msg_id, message=payload, read_ct=read_ct) + + +def _ok_payload(x, y=0): + return {"task_name": "test_pg_consumer.ok", "args": [x], "kwargs": {"y": y}} + + +# --- poll_once (mocked client, real tasks) --- + + +class TestPollOnce: + def test_runs_task_and_acks(self): + client = MagicMock() + client.read.return_value = [_msg(1, _ok_payload(3, 4))] + assert PgQueueConsumer("q", client=client).poll_once() == 1 + assert _calls == [(3, 4)] # task body ran + client.delete.assert_called_once_with(1) # acked + + def test_failed_task_is_not_acked_and_logs(self, caplog): + client = MagicMock() + client.read.return_value = [ + _msg(2, {"task_name": "test_pg_consumer.boom", "args": [], "kwargs": {}}) + ] + with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"): + PgQueueConsumer("q", client=client).poll_once() + client.delete.assert_not_called() # left for vt-expiry redelivery + assert "failed" in caplog.text # the cycling signal is logged + + def test_unknown_task_is_dropped(self, caplog): + client = MagicMock() + client.read.return_value = [ + _msg(3, {"task_name": "nope.nope", "args": [], "kwargs": {}}) + ] + with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"): + PgQueueConsumer("q", client=client).poll_once() + client.delete.assert_called_once_with(3) # dropped, not redelivered + assert "unknown task" in caplog.text + + def test_missing_task_name_dropped_as_malformed(self, caplog): + client = MagicMock() + client.read.return_value = [_msg(4, {"args": [], "kwargs": {}})] # no task_name + with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"): + PgQueueConsumer("q", client=client).poll_once() + client.delete.assert_called_once_with(4) + assert "missing task_name" in caplog.text # distinct from "unknown task" + + def test_poison_message_dropped_past_max_attempts(self, caplog): + client = MagicMock() + # boom task, claimed more than max_attempts times → drop instead of redeliver. + client.read.return_value = [ + _msg(5, {"task_name": "test_pg_consumer.boom"}, read_ct=6) + ] + with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"): + PgQueueConsumer("q", client=client, max_attempts=5).poll_once() + client.delete.assert_called_once_with(5) # dropped as poison + assert "poison" in caplog.text + + def test_fairness_header_rebuilt_for_run(self): + # Mock app so we can inspect the apply() headers. + fairness = {"org_id": "o", "workload_type": "api", "pipeline_priority": 5} + task = MagicMock() + app = MagicMock() + app.tasks.get.return_value = task + client = MagicMock() + client.read.return_value = [ + _msg(6, {"task_name": "t", "args": [1], "kwargs": {"k": "v"}, "fairness": fairness}) + ] + PgQueueConsumer("q", client=client, app=app).poll_once() + kwargs = task.apply.call_args.kwargs + assert kwargs["args"] == [1] + assert kwargs["kwargs"] == {"k": "v"} + assert kwargs["headers"] == {FAIRNESS_HEADER_NAME: fairness} + + def test_ack_finding_no_row_warns(self, caplog): + client = MagicMock() + client.delete.return_value = False # row already gone (vt expired mid-run) + client.read.return_value = [_msg(7, _ok_payload(1))] + with caplog.at_level(logging.WARNING, logger="queue_backend.pg_queue.consumer"): + PgQueueConsumer("q", client=client).poll_once() + assert "possible double-run" in caplog.text + + def test_multi_message_batch(self): + client = MagicMock() + client.read.return_value = [ + _msg(10, _ok_payload(1)), + _msg(11, {"task_name": "test_pg_consumer.boom"}), + _msg(12, {"task_name": "nope.nope"}), + _msg(13, _ok_payload(2)), + ] + assert PgQueueConsumer("q", client=client).poll_once() == 4 + assert _calls == [(1, 0), (2, 0)] # ok tasks ran in order + deleted = {c.args[0] for c in client.delete.call_args_list} + assert deleted == {10, 12, 13} # ok acked + unknown dropped; boom NOT acked + + def test_empty_batch_acks_nothing(self): + client = MagicMock() + client.read.return_value = [] + assert PgQueueConsumer("q", client=client).poll_once() == 0 + client.delete.assert_not_called() + + +class TestConstruction: + def test_rejects_non_positive_params(self): + for kw in ( + {"batch_size": 0}, + {"vt_seconds": -1}, + {"poll_interval": 0}, + {"backoff_max": 0}, + {"max_attempts": 0}, + ): + with pytest.raises(ValueError): + PgQueueConsumer("q", client=MagicMock(), **kw) + + def test_rejects_backoff_max_below_poll_interval(self): + # Otherwise backoff would shrink below poll_interval instead of growing. + with pytest.raises(ValueError, match="backoff_max"): + PgQueueConsumer( + "q", client=MagicMock(), poll_interval=0.5, backoff_max=0.1 + ) + + +class TestRunLoop: + def test_run_stops_gracefully(self, monkeypatch): + client = MagicMock() + client.read.return_value = [] # always empty → backoff path + consumer = PgQueueConsumer("q", client=client) + # First empty poll → sleep (patched to request stop) → loop exits. + monkeypatch.setattr( + "queue_backend.pg_queue.consumer.time.sleep", lambda _s: consumer.stop() + ) + consumer.run(install_signals=False) + assert consumer._running is False + + def test_backoff_grows_then_resets(self, monkeypatch): + client = MagicMock() + # empty, empty, one message, empty → backoff doubles, resets, doubles. + client.read.side_effect = [[], [], [_msg(1, _ok_payload(1))], []] + consumer = PgQueueConsumer( + "q", client=client, poll_interval=0.1, backoff_max=0.25 + ) + sleeps: list[float] = [] + + def _sleep(secs): + sleeps.append(secs) + if len(sleeps) == 3: # stop after the third sleep + consumer.stop() + + monkeypatch.setattr("queue_backend.pg_queue.consumer.time.sleep", _sleep) + consumer.run(install_signals=False) + # empty→0.1, empty→0.2 (doubled), [msg] resets, empty→0.1 again. + assert sleeps == [0.1, 0.2, 0.1] + + def test_poll_error_does_not_kill_loop(self, monkeypatch): + client = MagicMock() + # first poll raises (transient), then empty → loop must survive the raise. + client.read.side_effect = [RuntimeError("blip"), []] + consumer = PgQueueConsumer("q", client=client) + sleeps: list[float] = [] + + def _sleep(secs): + sleeps.append(secs) + if len(sleeps) == 2: # stop after the post-error poll + consumer.stop() + + monkeypatch.setattr("queue_backend.pg_queue.consumer.time.sleep", _sleep) + consumer.run(install_signals=False) # must not raise + assert client.read.call_count == 2 # kept polling after the error + + +# --- Integration: full enqueue → poll → execute → ack against real PG --- +# Uses the shared ``pg_client`` fixture from conftest.py. + + +class TestConsumerIntegration: + def test_enqueue_poll_execute_ack(self, pg_client): + queue_name = f"test_consumer_{os.getpid()}" + try: + pg_client.send( + queue_name, + to_payload("test_pg_consumer.ok", args=[5], kwargs={"y": 6}), + ) + claimed = PgQueueConsumer(queue_name, client=pg_client).poll_once() + assert claimed == 1 + assert (5, 6) in _calls # task actually executed off Postgres + # Row was acked (deleted) — nothing left to claim. + assert pg_client.read(queue_name, vt_seconds=30, qty=10) == [] + finally: + with pg_client.conn.cursor() as cur: + cur.execute( + "DELETE FROM pg_queue_message WHERE queue_name = %s", (queue_name,) + ) + pg_client.conn.commit() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From b58ee1e3247acab4fff78dd6080d5c59711b63f6 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Fri, 12 Jun 2026 15:45:53 +0530 Subject: [PATCH 06/44] UN-3541 [FEAT] Wire PG-queue consumer into run-worker.sh + bootstrap guard (#2047) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3541 [FEAT] Wire PG-queue consumer into run-worker.sh + bootstrap guard Make the 9c PG-queue consumer (UN-3539) runnable via the normal worker flow, safely. Split from #2045 to keep that PR focused on consumer logic. - Launcher (pg_queue_consumer/__main__.py): set WORKER_TYPE to the source worker (default notification) BEFORE `import worker`, so the right tasks register. worker.py loads exactly one worker type's tasks; a bare import would load the general worker's and drop every notification as unknown. - run-worker.sh: `pg-queue-consumer` type runs `python -m pg_queue_consumer` (not a celery command) from the workers root; queue via env. - Startup guard: PgQueueConsumer.run() refuses to start on an empty task registry — fail loud instead of silently dropping every message. - Drop the hard-coded 8086 health port (consumer runs no health server). Integration fixes found during live dev-test (real send_webhook_notification end-to-end → Slack HTTP 200): - Opt-in status: consumer is not part of `all`; shown only when running. - Log path: detach writes to an absolute $worker_dir/$type.log so -L/-C find it (also fixes the same latent bug for pluggable workers). - PID discovery: get_worker_pids matches the `python -m` invocation, so --status / -k / -r work for the consumer. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3541 [FIX] Address PR #2047 review feedback - run-worker.sh: verify liveness after a detached launch (kill -0 + tail); `set -e` doesn't apply to `&`, so a fork that died on startup was reported as "started" — acute for the health-port-less consumer (High). - consumer.py: log the registered application task names at startup so a *wrong* (non-empty but mismatched) registry is diagnosable — the guard only catches an *empty* one (Medium observability). - consumer.py: type _env() with a TypeVar (was `cast: type -> object`, erasing types at the typed __init__) and name the offending var on a bad cast instead of a context-free ValueError (Medium). - tests: add the guard's positive / require_tasks=False bypass / built-in filter arms (only the failure arm was covered). - get_worker_pids: warn on pgrep rc>1 (operational/regex error) instead of collapsing it to "not running" (Low). - run-worker.sh: extract the repeated "pg_queue_consumer" literal into a readonly constant (SonarCloud S1192). - Comment accuracy: build_celery_app configures but does not import tasks; `notification` is the worker that owns the leaf task, not the task itself; generalise the "every notification dropped" example. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3541 [FIX] Address Greptile review feedback - __main__.py: move the WORKER_TYPE mutation + `import worker` bootstrap into a guarded _bootstrap_and_run() called only under `__name__ == "__main__"`, so an accidental import (test/IDE/type-checker walking __main__) no longer overwrites WORKER_TYPE or triggers a full worker-app bootstrap. - run-worker.sh: list `pg-queue-consumer` in the usage/--help worker types, plus a note for its WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE / _QUEUE overrides. - run-worker.sh: clarify the post-launch liveness check is a best-effort fast-fail for *immediate* (sub-second) crash-on-import/bad-config faults, not a connectivity check; kept general (an immediate crash can hit any worker) and noted the `all` subshells overlap the 1s with inter-launch sleep. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- workers/pg_queue_consumer/__init__.py | 12 +++ workers/pg_queue_consumer/__main__.py | 46 ++++++++++ workers/queue_backend/pg_queue/consumer.py | 55 ++++++++++-- workers/run-worker.sh | 100 +++++++++++++++++++-- workers/tests/test_pg_queue_consumer.py | 57 ++++++++++++ 5 files changed, 257 insertions(+), 13 deletions(-) create mode 100644 workers/pg_queue_consumer/__init__.py create mode 100644 workers/pg_queue_consumer/__main__.py diff --git a/workers/pg_queue_consumer/__init__.py b/workers/pg_queue_consumer/__init__.py new file mode 100644 index 0000000000..24ee4777ae --- /dev/null +++ b/workers/pg_queue_consumer/__init__.py @@ -0,0 +1,12 @@ +"""PG-queue consumer worker — a standalone process that drains ``pg_queue_message``. + +It bootstraps a source worker's Celery app (registering that worker type's +tasks, like ``celery -A worker worker`` for a single type) so the consumer can +resolve and run them, then runs the +:class:`~queue_backend.pg_queue.consumer.PgQueueConsumer` poll loop. The source +worker type is selected by ``WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE`` (default +``notification``); see ``__main__`` for why this must precede ``import worker``. + +Launch via ``python -m pg_queue_consumer`` or ``./run-worker.sh +pg-queue-consumer``. Config via ``WORKER_PG_QUEUE_CONSUMER_*`` env. +""" diff --git a/workers/pg_queue_consumer/__main__.py b/workers/pg_queue_consumer/__main__.py new file mode 100644 index 0000000000..1be588708b --- /dev/null +++ b/workers/pg_queue_consumer/__main__.py @@ -0,0 +1,46 @@ +"""Entry point: bootstrap a worker app's tasks, then run the PG-queue consumer. + +The root ``worker`` module loads exactly ONE worker type's tasks, chosen by the +``WORKER_TYPE`` env: ``worker.py`` builds the Celery app via +``WorkerBuilder.build_celery_app`` (which configures, but does NOT import +tasks), then — in separate module-level logic in ``worker.py`` — ``importlib``- +loads that type's ``tasks.py``. The PG consumer drains a specific source +worker's queue, so it must register THAT worker's tasks: we set ``WORKER_TYPE`` +from ``WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE`` (default ``notification`` — the +worker that owns the first migrated leaf task, ``send_webhook_notification``) +BEFORE importing ``worker``. + +This override is required: ``run-worker.sh`` exports its own ``WORKER_TYPE`` +(the consumer pseudo-type ``pg_queue_consumer``, which owns no tasks), so a +plain import would fall back to the ``general`` worker's tasks and every +message for the intended worker would be dropped as an unknown task. The +consumer's startup guard only catches an *empty* registry, not a *wrong* one — +so the right worker type must be selected here. + +Launch via ``python -m pg_queue_consumer`` or ``./run-worker.sh pg-queue-consumer``. +""" + +import os + + +def _bootstrap_and_run() -> None: + # Select the source worker whose tasks back this consumer's queue. Must run + # BEFORE `import worker`, which reads WORKER_TYPE at import time. We overwrite + # (not setdefault) because the launcher's own WORKER_TYPE owns no tasks. + # + # Kept inside this guarded function (not at module scope) so the env mutation + # and the heavy `import worker` bootstrap only happen on the `python -m` + # entry — never as a side-effect if this module is imported by a test, IDE, + # or type-checker walking `__main__` files. + os.environ["WORKER_TYPE"] = os.environ.get( + "WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE", "notification" + ) + + import worker # noqa: F401 — side-effect: registers the source worker's tasks + from queue_backend.pg_queue.consumer import main + + main() + + +if __name__ == "__main__": + _bootstrap_and_run() diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py index a5c2024201..d7f527f705 100644 --- a/workers/queue_backend/pg_queue/consumer.py +++ b/workers/queue_backend/pg_queue/consumer.py @@ -24,7 +24,8 @@ import os import signal import time -from typing import TYPE_CHECKING +from collections.abc import Callable +from typing import TYPE_CHECKING, TypeVar from celery import current_app @@ -38,6 +39,8 @@ logger = logging.getLogger(__name__) +_T = TypeVar("_T") + _DEFAULT_QUEUE = "default" # Default 1: the whole batch shares one vt window (set atomically at claim), # but messages run sequentially — so with batch_size > 1 the tail can exceed @@ -178,16 +181,46 @@ def _handle(self, message: QueueMessage) -> None: message.msg_id, ) - def run(self, *, install_signals: bool = True) -> None: - """Poll loop with empty-queue backoff and graceful shutdown.""" + def _registered_task_count(self) -> int: + """Count application tasks (excluding Celery's built-ins).""" + return sum(1 for name in self._app.tasks if not name.startswith("celery.")) + + def run(self, *, install_signals: bool = True, require_tasks: bool = True) -> None: + """Poll loop with empty-queue backoff and graceful shutdown. + + Refuses to start if no application tasks are registered — a strong + signal the worker app wasn't bootstrapped, in which case *every* + message would be dropped as "unknown task". This makes a + misconfigured launch fail loudly instead of silently destroying data. + """ + if require_tasks and self._registered_task_count() == 0: + raise RuntimeError( + "PG-queue consumer: no application tasks are registered — the " + "worker app was not bootstrapped. Launch via " + "`python -m pg_queue_consumer` (or ./run-worker.sh " + "pg-queue-consumer), not bare " + "`python -m queue_backend.pg_queue.consumer`. Refusing to start " + "to avoid dropping every message as an unknown task." + ) self._running = True if install_signals: self._install_signal_handlers() + # Log the registered application tasks at startup. The guard above only + # catches an *empty* registry; a *wrong* one (e.g. the launcher selected + # the wrong source worker type) is non-empty but missing the target + # task, so each message would be dropped as "unknown task". Surfacing + # the registry here makes a wrong-type boot diagnosable from one line. + app_tasks = sorted( + name for name in self._app.tasks if not name.startswith("celery.") + ) logger.info( - "PG-queue consumer started (queue=%r, batch=%s, vt=%ss)", + "PG-queue consumer started (queue=%r, batch=%s, vt=%ss) — " + "%d application task(s) registered: %s", self.queue_name, self.batch_size, self.vt_seconds, + len(app_tasks), + ", ".join(app_tasks) or "(none)", ) backoff = self.poll_interval while self._running: @@ -226,8 +259,18 @@ def _install_signal_handlers(self) -> None: def main() -> None: logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) - def _env(suffix: str, default: object, cast: type) -> object: - return cast(os.getenv(f"WORKER_PG_QUEUE_CONSUMER_{suffix}", default)) + def _env(suffix: str, default: _T, cast: Callable[[str], _T]) -> _T: + # Preserve the default's type through to PgQueueConsumer's typed + # __init__ (a bare `type` would erase it). On a bad value, fail with the + # offending var name instead of a context-free `int('abc')` ValueError. + var = f"WORKER_PG_QUEUE_CONSUMER_{suffix}" + raw = os.getenv(var) + if raw is None: + return default + try: + return cast(raw) + except (ValueError, TypeError) as exc: + raise ValueError(f"Invalid {var}={raw!r}: {exc}") from exc PgQueueConsumer( queue_name=_env("QUEUE", _DEFAULT_QUEUE, str), diff --git a/workers/run-worker.sh b/workers/run-worker.sh index 6597907a1a..99751f007e 100755 --- a/workers/run-worker.sh +++ b/workers/run-worker.sh @@ -24,6 +24,9 @@ ENV_FILE="$WORKERS_DIR/.env" # Worker type constant for the executor worker readonly EXECUTOR_WORKER_TYPE="executor" readonly IDE_CALLBACK_WORKER_TYPE="ide_callback" +# Canonical name of the PG-queue consumer worker (referenced in several maps +# and special-cases below; a constant keeps them in sync). +readonly PG_QUEUE_CONSUMER_TYPE="pg_queue_consumer" # Available workers declare -A WORKERS=( @@ -44,6 +47,10 @@ declare -A WORKERS=( ["${EXECUTOR_WORKER_TYPE}"]="${EXECUTOR_WORKER_TYPE}" ["ide-callback"]="${IDE_CALLBACK_WORKER_TYPE}" ["${IDE_CALLBACK_WORKER_TYPE}"]="${IDE_CALLBACK_WORKER_TYPE}" + # PG Queue consumer — polls Postgres (SKIP LOCKED), not RabbitMQ via Celery + ["pg-queue-consumer"]="$PG_QUEUE_CONSUMER_TYPE" + ["$PG_QUEUE_CONSUMER_TYPE"]="$PG_QUEUE_CONSUMER_TYPE" + ["pg-consumer"]="$PG_QUEUE_CONSUMER_TYPE" ["all"]="all" ) @@ -61,6 +68,9 @@ declare -A WORKER_QUEUES=( ["scheduler"]="scheduler" ["${EXECUTOR_WORKER_TYPE}"]="celery_executor_legacy" ["${IDE_CALLBACK_WORKER_TYPE}"]="${IDE_CALLBACK_WORKER_TYPE}" + # The PG queue (in pg_queue_message) this consumer polls — exported as + # WORKER_PG_QUEUE_CONSUMER_QUEUE, not a Celery --queues value. + ["$PG_QUEUE_CONSUMER_TYPE"]="notifications" ) # Worker health ports @@ -74,6 +84,17 @@ declare -A WORKER_HEALTH_PORTS=( ["scheduler"]="8087" ["${EXECUTOR_WORKER_TYPE}"]="8088" ["${IDE_CALLBACK_WORKER_TYPE}"]="8089" + # pg_queue_consumer: no entry — it runs no health server, so it binds no + # port (avoids a hard-coded port that could collide). A liveness endpoint, + # if added later, should read WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT. +) + +# Opt-in workers: experimental and NOT part of the default "all" fleet, so +# they're started only on explicit request. Status shows them only when they +# are actually running, so a deliberate non-start isn't reported as a STOPPED +# failure (they'd otherwise show STOPPED after every `all`). +declare -A OPTIN_WORKERS=( + ["$PG_QUEUE_CONSUMER_TYPE"]=1 ) # Function to display usage @@ -93,9 +114,12 @@ WORKER_TYPE: scheduler, schedule Run scheduler worker (scheduled pipeline tasks) executor Run executor worker (extraction execution tasks) ide-callback Run IDE callback worker (Prompt Studio post-execution callbacks) + pg-queue-consumer Run PG-queue poll-loop consumer (opt-in; not part of 'all') all Run all workers (in separate processes, includes auto-discovered pluggable workers) Note: Pluggable workers in pluggable_worker/ directory are automatically discovered and can be run by name. +Note: pg-queue-consumer overrides: WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE (source worker whose + tasks to load, default notification) and WORKER_PG_QUEUE_CONSUMER_QUEUE (queue to poll). OPTIONS: -e, --env-file FILE Use specific environment file (default: .env) @@ -300,8 +324,26 @@ validate_env() { # --hostname=callback-worker@%h (default, no WORKER_INSTANCE_ID) # --hostname=callback-worker-${id}@%h (when WORKER_INSTANCE_ID is set) get_worker_pids() { - local worker_type=$1 - pgrep -f -- "[^[:alnum:]_]${worker_type}-worker(@|-)" || true + local worker_type=$1 pattern out rc + # The PG-queue consumer runs as `python -m pg_queue_consumer`, not a Celery + # `-worker@host` process, so it has no `-worker` token to anchor on. + # Match its module invocation instead (covers both the `uv run python` + # parent and the `python -m` child). Keeps --status / -k / -r working for it. + if [[ "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" ]]; then + pattern="-m[[:space:]]+${worker_type}([[:space:]]|\$)" + else + pattern="[^[:alnum:]_]${worker_type}-worker(@|-)" + fi + # pgrep exits 1 for "no match" (normal — absorbed) but >=2 for an + # operational/regex error. Distinguish them: collapsing rc>=2 to empty would + # make a live worker look absent (a -k no-op, or a duplicate spawn on -r). + out=$(pgrep -f -- "$pattern") + rc=$? + if (( rc > 1 )); then + print_status "$YELLOW" "warning: pgrep failed (rc=$rc) while matching $worker_type" >&2 + fi + [[ -n "$out" ]] && printf '%s\n' "$out" + return 0 } # Returns get_worker_pids output as a single space-separated string with @@ -475,6 +517,12 @@ show_status() { local pids pids=$(get_worker_pids_oneline "$worker") + # Opt-in workers aren't part of `all`; only surface them when running + # so an intentional non-start doesn't read as a STOPPED failure. + if [[ -z "$pids" && -n "${OPTIN_WORKERS[$worker]:-}" ]]; then + continue + fi + printf ' %-22s ' "$worker:" if [[ -n "$pids" ]]; then @@ -647,28 +695,66 @@ run_worker() { esac fi + # PG queue consumer is a plain Python poll-loop (polls Postgres via + # SKIP LOCKED, not a Celery/RabbitMQ worker) — override the celery command + # with the bootstrapping launcher and route the queue via env. + if [[ "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" ]]; then + export WORKER_PG_QUEUE_CONSUMER_QUEUE="$queues" + # The consumer registers ONE source worker's tasks (the launcher sets + # WORKER_TYPE from this before `import worker`). Default: notification — + # the worker that owns the first migrated leaf task, + # send_webhook_notification; override to drain another worker's queue. + export WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE="${WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE:-notification}" + cmd_args=("uv" "run" "python" "-m" "$PG_QUEUE_CONSUMER_TYPE") + fi + print_status $GREEN "Starting $worker_type worker..." print_status $BLUE "Directory: $worker_dir" print_status $BLUE "Worker Name: $worker_instance_name" print_status $BLUE "Queues: $queues" - print_status $BLUE "Health Port: ${WORKER_HEALTH_PORTS[$worker_type]}" + print_status $BLUE "Health Port: ${WORKER_HEALTH_PORTS[$worker_type]:-n/a}" print_status $BLUE "Command: ${cmd_args[*]}" # Change to appropriate directory # For pluggable workers, stay at workers root to allow module imports # For core workers, change to worker directory - if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" ]]; then + if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" || "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" ]]; then + # Run from the workers root so `python -m pg_queue_consumer` (and the + # `worker` app it bootstraps) resolve. cd "$WORKERS_DIR" else cd "$worker_dir" fi if [[ "$detach" == "true" ]]; then - # Run in background - nohup "${cmd_args[@]}" > "$worker_type.log" 2>&1 & + # Run in background. Write to an ABSOLUTE log path ($worker_dir is + # absolute) so the file lands where resolve_log_file() / -L / -C look, + # regardless of cwd. Workers that run from the workers root (pluggable + # workers, pg_queue_consumer) would otherwise drop a relative + # "$worker_type.log" at the root, where -L/-C can't find it. + local log_file="$worker_dir/$worker_type.log" + nohup "${cmd_args[@]}" > "$log_file" 2>&1 & local pid=$! + # set -e does not apply to backgrounded jobs, so a fork that dies on + # startup (e.g. the consumer's require_tasks RuntimeError, an + # `import worker` failure, a bad env cast) would still be reported as + # "started" — and for pg_queue_consumer, which has no health port, a + # dead process then just reads as absent in --status. Catch an + # *immediate* exit. This is a best-effort fast-fail for crash-on-import + # / bad-config faults (sub-second), NOT a connectivity check: a worker + # that dies slowly (e.g. a broker connect timing out after >1s) still + # passes here and surfaces later via its health port / --status. Run all + # detached workers share it — an immediate crash can hit any worker type; + # in `all` the subshells are backgrounded, so this 1s overlaps the + # inter-launch sleep rather than serializing. + sleep 1 + if ! kill -0 "$pid" 2>/dev/null; then + print_status $RED "$worker_type worker failed to start (PID $pid exited) — last log lines:" + tail -n 20 "$log_file" 2>/dev/null + return 1 + fi print_status $GREEN "$worker_type worker started in background (PID: $pid)" - print_status $BLUE "Logs: $worker_dir/$worker_type.log" + print_status $BLUE "Logs: $log_file" else # Run in foreground exec "${cmd_args[@]}" diff --git a/workers/tests/test_pg_queue_consumer.py b/workers/tests/test_pg_queue_consumer.py index 02e7d9c9cd..fa56e67833 100644 --- a/workers/tests/test_pg_queue_consumer.py +++ b/workers/tests/test_pg_queue_consumer.py @@ -208,6 +208,63 @@ def _sleep(secs): consumer.run(install_signals=False) # must not raise assert client.read.call_count == 2 # kept polling after the error + def test_run_refuses_to_start_with_empty_registry(self): + # A non-bootstrapped consumer would drop every message as "unknown" — + # fail loudly instead. + from celery import Celery + + empty_app = Celery("empty-no-tasks") # only celery.* built-ins + consumer = PgQueueConsumer("q", client=MagicMock(), app=empty_app) + with pytest.raises(RuntimeError, match="no application tasks"): + consumer.run(install_signals=False) + + def test_run_starts_with_registered_tasks(self, monkeypatch): + # Positive arm: a non-empty registry starts and polls. current_app has + # the module's @shared_task tasks registered, so the guard passes. + client = MagicMock() + client.read.return_value = [] # empty → loop sleeps → we stop it + consumer = PgQueueConsumer("q", client=client) + monkeypatch.setattr( + "queue_backend.pg_queue.consumer.time.sleep", lambda _s: consumer.stop() + ) + consumer.run(install_signals=False) # must not raise + assert client.read.called + + def test_run_bypasses_guard_when_require_tasks_false(self, monkeypatch): + # require_tasks=False skips the guard even on an empty registry (lets a + # caller opt out, e.g. a deliberately task-less smoke run). + from celery import Celery + + empty_app = Celery("empty-no-tasks") + client = MagicMock() + client.read.return_value = [] + consumer = PgQueueConsumer("q", client=client, app=empty_app) + monkeypatch.setattr( + "queue_backend.pg_queue.consumer.time.sleep", lambda _s: consumer.stop() + ) + consumer.run(install_signals=False, require_tasks=False) # must not raise + assert client.read.called + + def test_registered_task_count_excludes_celery_builtins(self): + # The guard counts application tasks only — celery.* built-ins (present + # in every app) must not mask an un-bootstrapped registry. + from celery import Celery + + empty_app = Celery("empty-no-tasks") + assert ( + PgQueueConsumer("q", client=MagicMock(), app=empty_app)._registered_task_count() + == 0 + ) + + @empty_app.task(name="test_pg_consumer.demo") + def _demo(): + return 1 + + assert ( + PgQueueConsumer("q", client=MagicMock(), app=empty_app)._registered_task_count() + == 1 + ) + # --- Integration: full enqueue → poll → execute → ack against real PG --- # Uses the shared ``pg_client`` fixture from conftest.py. From 9d7124feafcd62ee27cee2dde4dbdfecfe711cf3 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Fri, 12 Jun 2026 17:03:11 +0530 Subject: [PATCH 07/44] UN-3544 [FEAT] PG-queue consumer liveness endpoint (poll-loop heartbeat) (#2051) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3544 [FEAT] PG-queue consumer liveness endpoint (poll-loop heartbeat) Give the consumer a /health HTTP endpoint for K8s liveness probing, like every other worker — but keyed only on a poll-loop heartbeat. - consumer.py: track _last_poll_monotonic (refreshed at the top of poll_once, so a loop wedged on a long task goes stale and is detectable — which pgrep-based --status and the launch-liveness check cannot see). Expose seconds_since_last_poll() / is_poll_stale(). main() starts a LivenessServer when WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT is set (opt-in), stops it on exit. - LivenessServer: tiny stdlib HTTP server (/health, /healthz, /livez) → 200 while fresh, 503 once stale. Deliberately lean: a liveness probe must report only "is this process making progress?", NOT broker/API reachability or resource pressure (those would crash-loop a healthy consumer on a blip). So it does NOT reuse the shared HealthChecker (which also bundles an api_connectivity check that is both wrong for liveness and currently broken — its `from .api_client_singleton` import points at a non-existent module; tracked separately). - run-worker.sh: default port 8090 (outside the 8080-8089 core range, no collision), exported opt-in; documented in --help. - tests: heartbeat fresh/stale + a real bind-and-GET 200->503 endpoint test. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3544 [FIX] Address PR #2051 review feedback Important: - Liveness bind failure (OSError/port-in-use) now degrades gracefully: log and continue probe-less instead of aborting the consumer before it polls. Verified live (2nd consumer on a taken :8090 keeps draining). - run-worker.sh: 8090 collided with the first auto-discovered pluggable worker (8090 + count); pluggable discovery now starts at 8091, 8090 reserved. - LivenessServer.stop() is defensive (can't raise into main()'s finally and mask the real run() exception) and warns if the thread outlives the join. - Empty WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT now hits the clean opt-out (_env treats "" as unset) instead of int("") crashing at launch. Suggestions: - LivenessServer: guard double start(); reset state in stop(); wrap serve_forever so a thread crash is logged; route handler errors to the logger (log_message=pass was hiding log_error too); guard wfile.write against client disconnects; single clock read per request. - Type _httpd/_thread as HTTPServer|None / Thread|None via TYPE_CHECKING (drops the Any import); restores static checking. - run-worker.sh: status line shows the effective -p override, not the map default. - Docstrings: phrase the helper trigger in terms of `port`; note 0.0.0.0 bind; document that a fast-failing loop stays healthy by design (liveness must not couple to backend reachability). - tests: heartbeat-stamped-before-read (pins top-of-poll), /healthz + /livez aliases + unknown-path 404, double-start rejection. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3544 [FIX] Address SonarCloud issues - consumer.py: use logger.exception() in the liveness-bind except block (preserves the traceback; S6679). - test: lift the walrus assignment out of the PgQueueConsumer() argument list — plain `client = MagicMock()` first (clearer; S6328). Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3544 [FIX] Address Greptile review feedback - LivenessServer handler strips the query string before matching paths (self.path includes it), so /health?probe=k8s returns 200 not 404. Added a query-string case to the alias test. - Document the stale-threshold trade-off prominently: the heartbeat is frozen during task execution, so WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS is also an upper bound on single-task wall-clock (a longer task trips the probe → restart → redelivery). 60s suits the sub-second leaf; raise it above max(batch_size x worst_case_task_seconds, backoff_max) for longer tasks. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- workers/queue_backend/pg_queue/consumer.py | 215 ++++++++++++++++++++- workers/run-worker.sh | 24 ++- workers/tests/test_pg_queue_consumer.py | 113 +++++++++++ 3 files changed, 342 insertions(+), 10 deletions(-) diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py index d7f527f705..856a038a26 100644 --- a/workers/queue_backend/pg_queue/consumer.py +++ b/workers/queue_backend/pg_queue/consumer.py @@ -33,6 +33,9 @@ from .client import PgQueueClient if TYPE_CHECKING: + from http.server import HTTPServer + from threading import Thread + from celery import Celery from .client import QueueMessage @@ -53,6 +56,15 @@ # A task claimed more than this many times keeps failing — drop it (poison) # rather than redeliver forever. _DEFAULT_MAX_ATTEMPTS = 5 +# Liveness: a poll loop that hasn't cycled in this many seconds is reported +# unhealthy. The heartbeat is stamped at the top of each poll_once and frozen +# during task execution, so this threshold doubles as an UPPER BOUND on a single +# task's wall-clock: a task running longer than it trips the probe → pod restart +# → the in-flight task is killed and (at-least-once) redelivered. 60s suits the +# current sub-second leaf (send_webhook_notification); for longer-running tasks, +# raise WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS above +# max(batch_size x worst_case_task_seconds, backoff_max). +_DEFAULT_HEALTH_STALE_SECONDS = 60.0 class PgQueueConsumer: @@ -97,9 +109,16 @@ def __init__( self.backoff_max = backoff_max self.max_attempts = max_attempts self._running = False + # Heartbeat for the liveness probe: monotonic timestamp of the most + # recent poll attempt. Seeded at construction so a just-started consumer + # reads healthy. Updated at the TOP of poll_once, so a loop wedged on a + # long-running task (poll_once not returning) goes stale and is caught — + # something pgrep-based --status and the launch-time check cannot see. + self._last_poll_monotonic = time.monotonic() def poll_once(self) -> int: """Claim + process one batch; returns the number of messages claimed.""" + self._last_poll_monotonic = time.monotonic() messages = self._client.read( self.queue_name, vt_seconds=self.vt_seconds, qty=self.batch_size ) @@ -185,6 +204,30 @@ def _registered_task_count(self) -> int: """Count application tasks (excluding Celery's built-ins).""" return sum(1 for name in self._app.tasks if not name.startswith("celery.")) + def seconds_since_last_poll(self) -> float: + """Seconds since the last poll attempt (for the liveness heartbeat).""" + return time.monotonic() - self._last_poll_monotonic + + def is_poll_stale(self, stale_after_seconds: float) -> bool: + """True if the poll loop hasn't cycled within ``stale_after_seconds``. + + Drives the health endpoint: a stale loop means the consumer is wedged + (deadlock, or a single task running longer than the threshold), so the + liveness probe should report unhealthy and let the orchestrator restart + it. Pick a threshold comfortably above ``backoff_max`` and the longest + expected task so normal idle/backoff never trips it. + + Note the heartbeat is stamped at the *top* of ``poll_once`` (before the + DB read), so a loop that fails fast every cycle — e.g. ``read()`` raising + on an unreachable DB, caught and backed off by ``run()`` — keeps stamping + and stays *healthy*. That is deliberate: a liveness probe must not couple + to backend reachability (a restart can't fix a DB outage, and coupling + would crash-loop every consumer during one). Surfacing a permanent + config fault (bad creds, missing schema) is a readiness/alerting concern, + not liveness. + """ + return self.seconds_since_last_poll() > stale_after_seconds + def run(self, *, install_signals: bool = True, require_tasks: bool = True) -> None: """Poll loop with empty-queue backoff and graceful shutdown. @@ -265,21 +308,187 @@ def _env(suffix: str, default: _T, cast: Callable[[str], _T]) -> _T: # offending var name instead of a context-free `int('abc')` ValueError. var = f"WORKER_PG_QUEUE_CONSUMER_{suffix}" raw = os.getenv(var) - if raw is None: + # Treat empty-string as unset: an empty HEALTH_PORT (e.g. a run-worker.sh + # fallback resolving empty) must hit the clean opt-out, not int("") crash. + if raw is None or raw == "": return default try: return cast(raw) except (ValueError, TypeError) as exc: raise ValueError(f"Invalid {var}={raw!r}: {exc}") from exc - PgQueueConsumer( + consumer = PgQueueConsumer( queue_name=_env("QUEUE", _DEFAULT_QUEUE, str), batch_size=_env("BATCH", _DEFAULT_BATCH, int), vt_seconds=_env("VT_SECONDS", _DEFAULT_VT_SECONDS, int), poll_interval=_env("POLL_INTERVAL", _DEFAULT_POLL_INTERVAL, float), backoff_max=_env("BACKOFF_MAX", _DEFAULT_BACKOFF_MAX, float), max_attempts=_env("MAX_ATTEMPTS", _DEFAULT_MAX_ATTEMPTS, int), - ).run() + ) + health_server = _maybe_start_health_server( + consumer, + port=_env("HEALTH_PORT", None, int), + stale_after=_env("HEALTH_STALE_SECONDS", _DEFAULT_HEALTH_STALE_SECONDS, float), + ) + try: + consumer.run() + finally: + if health_server is not None: + health_server.stop() + + +def _maybe_start_health_server( + consumer: PgQueueConsumer, *, port: int | None, stale_after: float +) -> LivenessServer | None: + """Start the liveness server when ``port`` is not None; else ``None``. + + ``main()`` wires ``port`` from ``WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT`` (unset + → ``None`` → no server, no stray port). A bind failure degrades gracefully: + the probe is auxiliary, so it must never stop the consumer from draining the + queue — we log and continue probe-less rather than abort startup. + """ + if port is None: + logger.info( + "PG-queue consumer: WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT unset — " + "liveness server disabled" + ) + return None + server = LivenessServer(consumer, port=port, stale_after=stale_after) + try: + server.start() + except OSError: + logger.exception( + "PG-queue consumer: liveness server could not bind :%s — " + "continuing WITHOUT a probe", + port, + ) + return None + logger.info( + "PG-queue consumer: liveness server on :%s/health (stale after %ss)", + server.bound_port, + stale_after, + ) + return server + + +class LivenessServer: + """Tiny HTTP liveness probe: 200 while the poll loop is fresh, else 503. + + Deliberately lean. A *liveness* probe must answer one question — "is this + process still making progress?" — and nothing else. It must NOT depend on + broker/API reachability or resource pressure: a transient backend blip or a + busy moment would otherwise make the orchestrator crash-loop an + otherwise-healthy consumer. So this intentionally does *not* reuse the + shared ``HealthChecker`` (which bundles api-connectivity / system-resource + checks meant for richer health reporting, not liveness) — it reports solely + on the poll-loop heartbeat (:meth:`PgQueueConsumer.is_poll_stale`). + + Serves ``/health`` (also ``/healthz``, ``/livez``) on ``0.0.0.0`` (all + interfaces — a container/k8s probe reaches it from outside the process) in a + daemon thread. Bind ``port=0`` to let the OS pick a free port (read back via + :attr:`bound_port`) — used in tests. Start once; :meth:`stop` returns it to + the inert state. + """ + + _PATHS = frozenset({"/health", "/healthz", "/livez"}) + + def __init__( + self, consumer: PgQueueConsumer, *, port: int, stale_after: float + ) -> None: + self._consumer = consumer + self._port = port + self._stale_after = stale_after + self._httpd: HTTPServer | None = None + self._thread: Thread | None = None + + def start(self) -> None: + import json + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + from urllib.parse import urlsplit + + if self._httpd is not None: + raise RuntimeError("LivenessServer already started") + + consumer = self._consumer + stale_after = self._stale_after + paths = self._PATHS + + class _Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + # Strip any query string — self.path includes it, so a probe + # like /health?foo=bar must still match. + if urlsplit(self.path).path not in paths: + self.send_response(404) + self.end_headers() + return + # One clock read so age and the healthy/stale verdict are + # derived from the same instant. + age = consumer.seconds_since_last_poll() + stale = age > stale_after + body = json.dumps( + { + "status": "unhealthy" if stale else "healthy", + "check": "pg_queue_poll", + "seconds_since_last_poll": round(age, 3), + "stale_after_seconds": stale_after, + } + ).encode() + self.send_response(503 if stale else 200) + self.send_header("Content-Type", "application/json") + self.end_headers() + try: + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + pass # client (probe) hung up mid-response — not our problem + + def log_message(self, *_: object) -> None: + pass # silence per-request access logging + + def log_error(self, fmt: str, *args: object) -> None: + # BaseHTTPRequestHandler routes errors through log_message too; + # don't let the pass above swallow them — surface to our logger. + logger.warning("pg-queue liveness handler: " + fmt, *args) + + def _serve(httpd: HTTPServer) -> None: + try: + httpd.serve_forever() + except Exception: + # A daemon thread dying silently would make /health stop + # answering (connection refused) with no breadcrumb. + logger.exception("pg-queue liveness server thread crashed") + + httpd = HTTPServer(("0.0.0.0", self._port), _Handler) + self._httpd = httpd + self._thread = threading.Thread( + target=_serve, args=(httpd,), daemon=True, name="pg-consumer-liveness" + ) + self._thread.start() + + @property + def bound_port(self) -> int: + """Actual listening port (resolves ``port=0``); the requested port if not started.""" + if self._httpd is not None: + return self._httpd.server_address[1] + return self._port + + def stop(self) -> None: + """Shut the server down. Defensive: never raises (called from a finally).""" + try: + if self._httpd is not None: + self._httpd.shutdown() + self._httpd.server_close() + if self._thread is not None: + self._thread.join(timeout=5) + if self._thread.is_alive(): + logger.warning( + "PG-queue consumer: liveness thread did not stop within 5s" + ) + except Exception: + logger.exception("PG-queue consumer: error stopping liveness server") + finally: + self._httpd = None + self._thread = None if __name__ == "__main__": diff --git a/workers/run-worker.sh b/workers/run-worker.sh index 99751f007e..01875712e3 100755 --- a/workers/run-worker.sh +++ b/workers/run-worker.sh @@ -84,9 +84,12 @@ declare -A WORKER_HEALTH_PORTS=( ["scheduler"]="8087" ["${EXECUTOR_WORKER_TYPE}"]="8088" ["${IDE_CALLBACK_WORKER_TYPE}"]="8089" - # pg_queue_consumer: no entry — it runs no health server, so it binds no - # port (avoids a hard-coded port that could collide). A liveness endpoint, - # if added later, should read WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT. + # pg_queue_consumer: 8090 — reserved here, just past the 8080-8089 core + # range and just below where pluggable-worker discovery starts allocating + # (8091+, see below), so it collides with neither. The consumer binds it + # only when WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT is exported (below); a bare + # `python -m pg_queue_consumer` binds nothing. + ["$PG_QUEUE_CONSUMER_TYPE"]="8090" ) # Opt-in workers: experimental and NOT part of the default "all" fleet, so @@ -119,7 +122,8 @@ WORKER_TYPE: Note: Pluggable workers in pluggable_worker/ directory are automatically discovered and can be run by name. Note: pg-queue-consumer overrides: WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE (source worker whose - tasks to load, default notification) and WORKER_PG_QUEUE_CONSUMER_QUEUE (queue to poll). + tasks to load, default notification), WORKER_PG_QUEUE_CONSUMER_QUEUE (queue to poll), + and WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT (liveness server port, default 8090). OPTIONS: -e, --env-file FILE Use specific environment file (default: .env) @@ -272,9 +276,11 @@ discover_pluggable_workers() { WORKER_QUEUES["$worker_name"]="$worker_name" fi - # Assign health port dynamically (starting from 8090) + # Assign health port dynamically (starting from 8091; 8090 is + # reserved for pg_queue_consumer, so the first pluggable worker + # doesn't collide with it). if [[ -z "${WORKER_HEALTH_PORTS[$worker_name]:-}" ]]; then - WORKER_HEALTH_PORTS["$worker_name"]=$((8090 + discovered_count)) + WORKER_HEALTH_PORTS["$worker_name"]=$((8091 + discovered_count)) fi print_status $GREEN "Discovered pluggable worker: $worker_name" @@ -705,6 +711,9 @@ run_worker() { # the worker that owns the first migrated leaf task, # send_webhook_notification; override to drain another worker's queue. export WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE="${WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE:-notification}" + # Liveness HTTP server port (-p override wins, else the map default). + # Exported so the launcher's main() opts into the health server. + export WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT="${health_port:-${WORKER_HEALTH_PORTS[$worker_type]}}" cmd_args=("uv" "run" "python" "-m" "$PG_QUEUE_CONSUMER_TYPE") fi @@ -712,7 +721,8 @@ run_worker() { print_status $BLUE "Directory: $worker_dir" print_status $BLUE "Worker Name: $worker_instance_name" print_status $BLUE "Queues: $queues" - print_status $BLUE "Health Port: ${WORKER_HEALTH_PORTS[$worker_type]:-n/a}" + # Show the effective port: a -p/--health-port override wins over the map. + print_status $BLUE "Health Port: ${health_port:-${WORKER_HEALTH_PORTS[$worker_type]:-n/a}}" print_status $BLUE "Command: ${cmd_args[*]}" # Change to appropriate directory diff --git a/workers/tests/test_pg_queue_consumer.py b/workers/tests/test_pg_queue_consumer.py index fa56e67833..372252770a 100644 --- a/workers/tests/test_pg_queue_consumer.py +++ b/workers/tests/test_pg_queue_consumer.py @@ -266,6 +266,119 @@ def _demo(): ) +# --- Liveness heartbeat (drives the health endpoint) --- + + +class TestPollHeartbeat: + def test_poll_once_refreshes_heartbeat(self): + client = MagicMock() + client.read.return_value = [] + consumer = PgQueueConsumer("q", client=client) + # Simulate a long-idle consumer, then poll → heartbeat resets to ~now. + consumer._last_poll_monotonic -= 120 + assert consumer.seconds_since_last_poll() > 100 + consumer.poll_once() + assert consumer.seconds_since_last_poll() < 1.0 + + def test_heartbeat_stamped_before_read(self): + # Pins the headline design: the stamp lands at the TOP of poll_once + # (before read), so a task running longer than the threshold still trips + # the probe. A bottom-of-poll stamp would pass test_poll_once_refreshes + # but fail here. + client = MagicMock() + consumer = PgQueueConsumer("q", client=client) + before = consumer._last_poll_monotonic + seen: dict[str, float] = {} + client.read.side_effect = lambda *a, **k: ( + seen.setdefault("during", consumer._last_poll_monotonic), + [], + )[1] + consumer.poll_once() + assert seen["during"] > before # refreshed BEFORE read ran, not after + + def test_is_poll_stale_threshold(self): + consumer = PgQueueConsumer("q", client=MagicMock()) + consumer._last_poll_monotonic -= 120 # last poll 120s ago + assert consumer.is_poll_stale(60) is True # past threshold → stale + assert consumer.is_poll_stale(200) is False # within threshold → fresh + + def test_fresh_consumer_is_not_stale(self): + # Seeded at construction, so a just-started consumer reads healthy. + consumer = PgQueueConsumer("q", client=MagicMock()) + assert consumer.is_poll_stale(60) is False + + def test_health_server_disabled_without_port(self): + # No port configured → no server bound (opt-in). + from queue_backend.pg_queue.consumer import _maybe_start_health_server + + consumer = PgQueueConsumer("q", client=MagicMock()) + assert _maybe_start_health_server(consumer, port=None, stale_after=60) is None + + def test_liveness_server_reports_200_then_503(self): + # Real endpoint: 200 while the poll loop is fresh, 503 once it goes + # stale. Bind port 0 so the OS picks a free port (no fixed-port clash). + import json + import urllib.error + import urllib.request + + from queue_backend.pg_queue.consumer import LivenessServer + + consumer = PgQueueConsumer("q", client=MagicMock()) + server = LivenessServer(consumer, port=0, stale_after=60) + server.start() + try: + url = f"http://127.0.0.1:{server.bound_port}/health" + with urllib.request.urlopen(url, timeout=5) as resp: + assert resp.status == 200 + assert json.loads(resp.read())["status"] == "healthy" + + consumer._last_poll_monotonic -= 120 # force the loop stale + with pytest.raises(urllib.error.HTTPError) as ei: + urllib.request.urlopen(url, timeout=5) + assert ei.value.code == 503 + assert json.loads(ei.value.read())["status"] == "unhealthy" + finally: + server.stop() + + def test_liveness_aliases_and_unknown_path(self): + # All three probe aliases answer 200 (different orchestrators probe + # different paths); an unknown path is 404 (guards against a regression + # that makes every path pass). + import urllib.error + import urllib.request + + from queue_backend.pg_queue.consumer import LivenessServer + + consumer = PgQueueConsumer("q", client=MagicMock()) + server = LivenessServer(consumer, port=0, stale_after=60) + server.start() + try: + base = f"http://127.0.0.1:{server.bound_port}" + # Aliases, plus a query string (self.path includes it) must match. + for path in ("/health", "/healthz", "/livez", "/health?probe=k8s"): + with urllib.request.urlopen(f"{base}{path}", timeout=5) as resp: + assert resp.status == 200, path + with pytest.raises(urllib.error.HTTPError) as ei: + urllib.request.urlopen(f"{base}/nope", timeout=5) + assert ei.value.code == 404 + finally: + server.stop() + + def test_double_start_is_rejected(self): + from queue_backend.pg_queue.consumer import LivenessServer + + server = LivenessServer(PgQueueConsumer("q", client=MagicMock()), port=0, stale_after=60) + server.start() + try: + with pytest.raises(RuntimeError, match="already started"): + server.start() + finally: + server.stop() + # stop() returns it to the inert state → can start again. + server.start() + server.stop() + + # --- Integration: full enqueue → poll → execute → ack against real PG --- # Uses the shared ``pg_client`` fixture from conftest.py. From 0955df981f87b7b2978ccaa03148a7e99787d162 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Fri, 12 Jun 2026 20:08:39 +0530 Subject: [PATCH 08/44] UN-3546 [FEAT] Priority-ordered PG-queue dequeue + concurrency-safe claim (#2052) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3546 [FEAT] Priority-ordered PG-queue dequeue + concurrency-safe claim Start enforcing the load-independent part of fairness — pipeline_priority (L3) — directly in the single-table dequeue: higher priority is claimed first, FIFO (msg_id) within a priority. The org-tier (L1) / workload (L2) axes + burst_max admission stay deferred to the fair-admission orchestrator. - Schema: add `priority` (smallint, default 5 = FairnessKey.DEFAULT_PRIORITY) to pg_queue_message; swap the dequeue index to (queue_name, priority DESC, msg_id) so the priority-ordered claim stays an indexed top-N. - Enqueue: dispatch() writes priority from fairness.pipeline_priority; a bare dispatch (fairness=None) writes the neutral default. - Dequeue: ORDER BY priority DESC, msg_id. Also fixes a latent concurrency bug in the original dequeue (9a): `UPDATE ... WHERE msg_id IN (SELECT ... FOR UPDATE SKIP LOCKED LIMIT n)` can OVER-CLAIM under concurrent writers — EvalPlanQual re-evaluates the LIMIT subquery when a row it tried to lock was concurrently touched, so one claim can return more than n rows. Switched to the canonical PGMQ-safe shape: lock candidates in a CTE, then `UPDATE ... FROM locked WHERE q.msg_id = locked.msg_id`, which locks exactly n rows once. The trailing SELECT re-orders RETURNING (which is otherwise unspecified) so batched claims come back in priority order too. Tests: priority selection (one-at-a-time) + batch ordering against real Postgres; send writes priority; dispatch wiring (fairness + neutral default); read param order. Verified live end-to-end via dispatch()->claim-order (9>7>3a>3b>1). Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3546 [FIX] Address PR #2052 review feedback - Validate priority at the write boundary: client.send() raises ValueError on out-of-range (mirrors its vt_seconds/qty guards) — an out-of-range value would silently jump/sink the row in the priority DESC claim order. - Add a DB CheckConstraint (priority 1..10) as the backstop no ORM/raw writer can bypass (migration 0003). check= (not condition=) — repo is on Django 4.2. - Soften the "indexed top-N" comments (client.py + models.py): the dequeue is an index walk with vt<=now() as a per-row filter, NOT a guaranteed top-N — vt is not in the index, so in-flight (future-vt) high-priority rows are scanned past on each claim; the orchestrator's admission is the high-backlog answer. Update the module docstring to the CTE FROM-join shape; fix the fairness.DEFAULT_PRIORITY symbol reference; drop the duplicated param-order comment. - Tests: send() range-guard (parametrized) + DB CheckConstraint backstop; concurrent-writer over-claim guard (two readers, no batch exceeds qty — the regression test for the EvalPlanQual fix); vt × priority (visible low beats invisible high); FIFO-within-band for multi-member batch bands. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3546 [FIX] Address Greptile review feedback - Concurrency test: assert the drain worker terminated after join (a hung worker now fails the test instead of passing silently while conn_b.close() races its in-flight queries). - Priority-bounds drift guard: backend models.py and workers fairness.py are separate codebases that can't import each other, so the DB constraint bounds (1/10) duplicate fairness.MIN/MAX_PRIORITY. Replaced the hardcoded "42" reject test with test_db_check_constraint_matches_fairness_bounds — raw-inserts at MIN/MAX (accepted) and MIN-1/MAX+1 (CheckViolation), pinning the DB constraint to the fairness range so a future widening that misses one side fails loudly. Documented the canonical source in the model comment. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3546 [DOCS] Note check=/condition= handling on the constraint migration Breadcrumb for a future Django upgrade: `check=` is correct on the pinned Django 4.2 (deprecated 5.1, removed 6.0); fresh installs always replay under the shipped Django, so leave it. When the pin reaches >= 6.0, squash (or do the behaviour-preserving check= -> condition= edit) so a from-scratch migrate runs. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- ...e_pg_queue_message_dequeue_idx_and_more.py | 30 ++++ ...message_pg_queue_message_priority_range.py | 25 ++++ backend/pg_queue/models.py | 30 +++- workers/queue_backend/dispatch.py | 8 +- workers/queue_backend/pg_queue/client.py | 108 ++++++++++---- workers/tests/test_dispatch_pg.py | 35 +++++ workers/tests/test_pg_queue_client.py | 137 +++++++++++++++++- 7 files changed, 339 insertions(+), 34 deletions(-) create mode 100644 backend/pg_queue/migrations/0002_remove_pgqueuemessage_pg_queue_message_dequeue_idx_and_more.py create mode 100644 backend/pg_queue/migrations/0003_pgqueuemessage_pg_queue_message_priority_range.py diff --git a/backend/pg_queue/migrations/0002_remove_pgqueuemessage_pg_queue_message_dequeue_idx_and_more.py b/backend/pg_queue/migrations/0002_remove_pgqueuemessage_pg_queue_message_dequeue_idx_and_more.py new file mode 100644 index 0000000000..08cd58106e --- /dev/null +++ b/backend/pg_queue/migrations/0002_remove_pgqueuemessage_pg_queue_message_dequeue_idx_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.1 on 2026-06-12 11:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("pg_queue", "0001_initial"), + ] + + operations = [ + migrations.RemoveIndex( + model_name="pgqueuemessage", + name="pg_queue_message_dequeue_idx", + ), + migrations.AddField( + model_name="pgqueuemessage", + name="priority", + field=models.SmallIntegerField(default=5), + ), + migrations.AddIndex( + model_name="pgqueuemessage", + index=models.Index( + models.F("queue_name"), + models.OrderBy(models.F("priority"), descending=True), + models.F("msg_id"), + name="pg_queue_message_dequeue_idx", + ), + ), + ] diff --git a/backend/pg_queue/migrations/0003_pgqueuemessage_pg_queue_message_priority_range.py b/backend/pg_queue/migrations/0003_pgqueuemessage_pg_queue_message_priority_range.py new file mode 100644 index 0000000000..68e141c8fc --- /dev/null +++ b/backend/pg_queue/migrations/0003_pgqueuemessage_pg_queue_message_priority_range.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.1 on 2026-06-12 14:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("pg_queue", "0002_remove_pgqueuemessage_pg_queue_message_dequeue_idx_and_more"), + ] + + operations = [ + # `check=` is correct for the pinned Django (4.2). It's deprecated in + # 5.1 in favour of `condition=` and removed in 6.0 — but fresh installs + # always replay this under the Django we ship, so leave it as-is. + # When the pin is bumped to >= 6.0, squash these migrations (or do the + # behaviour-preserving `check=` -> `condition=` edit) as part of that + # upgrade so a from-scratch migrate still runs. + migrations.AddConstraint( + model_name="pgqueuemessage", + constraint=models.CheckConstraint( + check=models.Q(("priority__gte", 1), ("priority__lte", 10)), + name="pg_queue_message_priority_range", + ), + ), + ] diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py index 771fd909e6..8b3386cb27 100644 --- a/backend/pg_queue/models.py +++ b/backend/pg_queue/models.py @@ -1,4 +1,5 @@ from django.db import models +from django.db.models import F from django.utils import timezone @@ -29,12 +30,39 @@ class PgQueueMessage(models.Model): enqueued_at = models.DateTimeField(default=timezone.now) vt = models.DateTimeField(default=timezone.now) # visibility timeout read_ct = models.IntegerField(default=0) + # fairness L3: higher value is claimed sooner. The canonical range lives in + # workers/queue_backend/fairness.py (MIN_PRIORITY=1, MAX_PRIORITY=10, + # DEFAULT_PRIORITY=5); these literals mirror it because backend and workers + # are separate codebases that can't import each other. Keep them in sync — a + # workers integration test (test_db_check_constraint_matches_fairness_bounds) + # asserts this DB constraint matches the fairness bounds, so a divergence + # fails loudly. The dispatch writes priority from fairness.pipeline_priority; + # leaf tasks with no fairness get the neutral default. L1 (org tier) / L2 + # (workload) ordering + burst_max admission are deferred to the fair-admission + # orchestrator. The CheckConstraint is the one backstop no writer can bypass. + priority = models.SmallIntegerField(default=5) class Meta: db_table = "pg_queue_message" + constraints = [ + models.CheckConstraint( + check=models.Q(priority__gte=1) & models.Q(priority__lte=10), + name="pg_queue_message_priority_range", + ), + ] indexes = [ + # The dequeue walks one queue in (priority DESC, msg_id ASC) order + # and applies vt <= now() as a per-row filter during the walk — + # claim high-priority first, FIFO (msg_id, monotonic and stable + # across re-claims) within a band. Note this is NOT a guaranteed + # top-N: vt is intentionally not in the index, so claimed-but-unacked + # rows (future vt) sit at the front of their band and are scanned + # past on each claim. Fine at low in-flight depth; the orchestrator's + # staging→task admission is the answer if that backlog grows large. models.Index( - fields=["queue_name", "vt", "msg_id"], + F("queue_name"), + F("priority").desc(), + F("msg_id"), name="pg_queue_message_dequeue_idx", ) ] diff --git a/workers/queue_backend/dispatch.py b/workers/queue_backend/dispatch.py index 68261293df..7012b0a7e4 100644 --- a/workers/queue_backend/dispatch.py +++ b/workers/queue_backend/dispatch.py @@ -27,7 +27,7 @@ from celery import current_app -from .fairness import FairnessKey +from .fairness import DEFAULT_PRIORITY, FairnessKey from .handle import DispatchHandle from .pg_queue import PgQueueClient, to_payload from .routing import QueueBackend, select_backend @@ -126,10 +126,16 @@ def _enqueue_pg( task_name, args=args, kwargs=kwargs, queue=queue, fairness=fairness ) try: + # Carry org_id + L3 priority onto the row so the dequeue can order by + # priority. A bare dispatch (fairness=None) writes the neutral defaults + # (org_id None → "" / DEFAULT_PRIORITY). msg_id = _get_pg_client().send( pg_queue, payload, org_id=fairness.org_id if fairness is not None else None, + priority=( + fairness.pipeline_priority if fairness is not None else DEFAULT_PRIORITY + ), ) except Exception: # Re-raise with a breadcrumb (raw psycopg2.Error / a json.dumps diff --git a/workers/queue_backend/pg_queue/client.py b/workers/queue_backend/pg_queue/client.py index 030f8d8921..8024f11ebc 100644 --- a/workers/queue_backend/pg_queue/client.py +++ b/workers/queue_backend/pg_queue/client.py @@ -1,13 +1,13 @@ """Thin client over the bespoke PG queue (extension-free, ``SKIP LOCKED``). -**Inert in this phase** — nothing in ``dispatch()`` calls this yet (the -routing gate's PG branch still routes to Celery). This is the storage + -dequeue primitive that the enqueue wiring (9b) and the consumer poll -loop (9c) build on. +This is the storage + dequeue primitive the enqueue wiring (9b) and the +consumer poll loop (9c) build on; ``dispatch()`` routes PG-opted tasks here. Dequeue uses the visibility-timeout pattern: :meth:`PgQueueClient.read` -runs a single atomic ``UPDATE … WHERE msg_id IN (SELECT … FOR UPDATE -SKIP LOCKED …) RETURNING …`` (committed immediately), the caller +runs a single atomic statement — candidate rows are locked in a CTE +(``SELECT … FOR UPDATE SKIP LOCKED LIMIT n``) and that CTE is joined into +the ``UPDATE … FROM locked`` (the EvalPlanQual-safe shape; see +``_DEQUEUE_SQL``) — committed immediately, the caller processes the message *outside* the transaction, then :meth:`PgQueueClient.delete` acks on success. A crash before delete leaves the row to reappear once its ``vt`` expires — **at-least-once** @@ -35,6 +35,7 @@ import psycopg2 +from ..fairness import DEFAULT_PRIORITY, MAX_PRIORITY, MIN_PRIORITY from .connection import create_pg_connection if TYPE_CHECKING: @@ -50,25 +51,53 @@ # (at-least-once — a message CAN be processed more than once). No lock held # during processing -> VACUUM-safe and PgBouncer txn-pooling compatible. # -# ORDER BY (vt, msg_id) — not just msg_id — so the (queue_name, vt, msg_id) +# ORDER BY (priority DESC, msg_id) — the (queue_name, priority DESC, msg_id) # index drives an indexed top-N: for a fixed queue the index is ordered by -# (vt, msg_id), so the vt<=now() range scan emits rows already sorted and PG -# applies LIMIT without sorting the whole visible backlog. Still effectively -# FIFO (vt is set to now() at enqueue; msg_id is the deterministic tiebreak). +# (priority DESC, msg_id), so PG walks rows highest-priority-first, applies the +# vt<=now() visibility filter as it goes, and stops at LIMIT — no sort of the +# whole visible backlog. Higher priority is claimed sooner; msg_id ASC is the +# FIFO tiebreak within a priority (monotonic, and unlike vt it never moves when +# a row is re-claimed). Fairness L1 (org tier) / L2 (workload) + burst_max +# admission are deferred to the fair-admission orchestrator (a later phase). +# The inner ORDER BY selects WHICH rows are claimed (the top-N by priority when +# LIMIT < available). The outer SELECT re-applies it because UPDATE ... RETURNING +# yields rows in an unspecified order — so for batch_size > 1 the caller still +# gets the batch in priority order, not physical/update order. (At the default +# batch_size = 1 only the single highest-priority row is claimed, so the outer +# sort is a no-op there.) +# Canonical PGMQ-safe shape: lock the candidate rows in a CTE, then UPDATE by +# JOINING that CTE (``UPDATE ... FROM locked WHERE msg_id = locked.msg_id``). +# The alternative — ``UPDATE ... WHERE msg_id IN (SELECT ... FOR UPDATE SKIP +# LOCKED LIMIT n)`` — can over-claim under concurrent writers: EvalPlanQual +# re-evaluates the LIMIT subquery when a row it tried to lock was concurrently +# touched, so a single claim can return more than ``n`` rows. The FROM-join form +# locks exactly ``n`` rows once and updates precisely those. The trailing SELECT +# re-applies the order because UPDATE ... RETURNING is otherwise unordered. +# +# Ordering is an index walk over (queue_name, priority DESC, msg_id) with +# ``vt <= now()`` applied as a per-row filter — not a guaranteed top-N: vt is +# not in the index, so claimed-but-unacked rows (future vt) at the front of a +# priority band are scanned past on each claim. Cheap at low in-flight depth. _DEQUEUE_SQL = """ -UPDATE pg_queue_message - SET vt = now() + make_interval(secs => %s), - read_ct = read_ct + 1 - WHERE msg_id IN ( - SELECT msg_id - FROM pg_queue_message - WHERE queue_name = %s - AND vt <= now() - ORDER BY vt, msg_id - FOR UPDATE SKIP LOCKED - LIMIT %s - ) -RETURNING msg_id, message, read_ct +WITH locked AS ( + SELECT msg_id + FROM pg_queue_message + WHERE queue_name = %s + AND vt <= now() + ORDER BY priority DESC, msg_id + FOR UPDATE SKIP LOCKED + LIMIT %s +), claimed AS ( + UPDATE pg_queue_message q + SET vt = now() + make_interval(secs => %s), + read_ct = read_ct + 1 + FROM locked + WHERE q.msg_id = locked.msg_id + RETURNING q.msg_id, q.message, q.read_ct, q.priority +) +SELECT msg_id, message, read_ct + FROM claimed + ORDER BY priority DESC, msg_id """ @@ -144,22 +173,43 @@ def _cursor(self) -> Iterator[Any]: raise def send( - self, queue_name: str, message: dict[str, Any], *, org_id: str | None = None + self, + queue_name: str, + message: dict[str, Any], + *, + org_id: str | None = None, + priority: int = DEFAULT_PRIORITY, ) -> int: """Enqueue a message; returns its ``msg_id``. Immediately visible — ``vt`` is set to ``now()`` (DB clock). The timestamp/counter columns are supplied here rather than via DB defaults so the schema stays a plain Django migration. + + ``priority`` (fairness L3) controls dequeue order — higher is claimed + sooner. Defaults to the neutral ``DEFAULT_PRIORITY`` for tasks dispatched + without a fairness key (leaf tasks). Must be in ``[MIN_PRIORITY, + MAX_PRIORITY]`` — out of range raises (it would silently jump/sink the + row in the ``priority DESC`` claim order), mirroring ``read()``'s guards. + The DB ``CheckConstraint`` is the backstop for any ORM/raw writer. """ + if not MIN_PRIORITY <= priority <= MAX_PRIORITY: + raise ValueError( + f"priority out of range [{MIN_PRIORITY}, {MAX_PRIORITY}]: {priority!r}" + ) with self._cursor() as cur: cur.execute( "INSERT INTO pg_queue_message " - "(queue_name, message, org_id, enqueued_at, vt, read_ct) " - "VALUES (%s, %s::jsonb, %s, now(), now(), 0) RETURNING msg_id", + "(queue_name, message, org_id, priority, enqueued_at, vt, read_ct) " + "VALUES (%s, %s::jsonb, %s, %s, now(), now(), 0) RETURNING msg_id", # "" rather than NULL for "no org" — the column is non-null # (string fields shouldn't have two empty values; Django S6553). - (queue_name, json.dumps(message), org_id if org_id is not None else ""), + ( + queue_name, + json.dumps(message), + org_id if org_id is not None else "", + priority, + ), ) msg_id = cur.fetchone()[0] return int(msg_id) @@ -182,7 +232,9 @@ def read( if qty <= 0: raise ValueError(f"qty must be positive, got {qty}") with self._cursor() as cur: - cur.execute(_DEQUEUE_SQL, (vt_seconds, queue_name, qty)) + # Param order matches the %s positions in _DEQUEUE_SQL: + # queue_name (locked CTE), qty (LIMIT), vt_seconds (UPDATE SET). + cur.execute(_DEQUEUE_SQL, (queue_name, qty, vt_seconds)) rows = cur.fetchall() return [ QueueMessage(msg_id=int(r[0]), message=r[1], read_ct=int(r[2])) for r in rows diff --git a/workers/tests/test_dispatch_pg.py b/workers/tests/test_dispatch_pg.py index 75d840962f..e86d99193a 100644 --- a/workers/tests/test_dispatch_pg.py +++ b/workers/tests/test_dispatch_pg.py @@ -111,5 +111,40 @@ def test_celery_dispatch_unaffected(self): mock_get.assert_not_called() +class TestDispatchPriorityWiring: + """dispatch() carries fairness.pipeline_priority onto the PG row (mocked).""" + + @staticmethod + def _capture_send(monkeypatch): + captured: dict = {} + + class _Client: + def send(self, queue_name, payload, **kwargs): + captured.update(queue=queue_name, **kwargs) + return 7 + + monkeypatch.setenv(ENABLED_TASKS_ENV, "leaf_task") + monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client()) + return captured + + def test_priority_from_fairness(self, monkeypatch): + captured = self._capture_send(monkeypatch) + fairness = FairnessKey( + org_id="o", workload_type=WorkloadType.API, pipeline_priority=8 + ) + dispatch("leaf_task", fairness=fairness) + assert captured["priority"] == 8 + assert captured["org_id"] == "o" + + def test_no_fairness_uses_neutral_defaults(self, monkeypatch): + # Bare dispatch → org_id None (client coerces to "") + DEFAULT_PRIORITY. + from queue_backend.fairness import DEFAULT_PRIORITY + + captured = self._capture_send(monkeypatch) + dispatch("leaf_task") + assert captured["priority"] == DEFAULT_PRIORITY + assert captured["org_id"] is None + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/workers/tests/test_pg_queue_client.py b/workers/tests/test_pg_queue_client.py index 20bb72a4ba..4a473af01d 100644 --- a/workers/tests/test_pg_queue_client.py +++ b/workers/tests/test_pg_queue_client.py @@ -20,6 +20,7 @@ import psycopg2 import pytest +from queue_backend.fairness import DEFAULT_PRIORITY, MAX_PRIORITY, MIN_PRIORITY from queue_backend.pg_queue import PgQueueClient, QueueMessage from queue_backend.pg_queue.connection import create_pg_connection @@ -68,14 +69,36 @@ def test_send_coerces_missing_org_to_empty_string(self): _, params = cur.execute.call_args.args assert params[2] == "" + def test_send_writes_priority(self): + # Default is the neutral DEFAULT_PRIORITY; an explicit value passes through. + conn, cur = _mock_conn(fetchone=(1,)) + PgQueueClient(conn=conn).send("q1", {"a": 1}) + sql, params = cur.execute.call_args.args + assert "priority" in sql + assert params[3] == DEFAULT_PRIORITY + + conn, cur = _mock_conn(fetchone=(2,)) + PgQueueClient(conn=conn).send("q1", {"a": 1}, priority=9) + _, params = cur.execute.call_args.args + assert params[3] == 9 + + @pytest.mark.parametrize("bad", [0, -1, 11, 99]) + def test_send_rejects_out_of_range_priority(self, bad): + # An out-of-range priority would silently jump/sink the row in the + # priority DESC claim order — reject at the write boundary. + conn, _ = _mock_conn(fetchone=(1,)) + with pytest.raises(ValueError, match="priority out of range"): + PgQueueClient(conn=conn).send("q1", {"a": 1}, priority=bad) + def test_read_runs_skip_locked_dequeue(self): conn, cur = _mock_conn(fetchall=[(7, {"k": "v"}, 1)]) msgs = PgQueueClient(conn=conn).read("q1", vt_seconds=15, qty=3) sql, params = cur.execute.call_args.args assert "FOR UPDATE SKIP LOCKED" in sql assert "UPDATE pg_queue_message" in sql - # Param order follows the %s positions: vt_seconds, queue_name, qty. - assert params == (15, "q1", 3) + assert "ORDER BY priority DESC" in sql # fairness L3 claim order + # Param order follows the %s positions: queue_name, qty, vt_seconds. + assert params == ("q1", 3, 15) assert msgs == [QueueMessage(msg_id=7, message={"k": "v"}, read_ct=1)] conn.commit.assert_called_once() @@ -236,8 +259,9 @@ def boom(**_): @pytest.fixture def queue_name(pg_conn): - # Unique per test for isolation; clean up rows afterwards. - name = f"test_q_{os.getpid()}_{int(time.time() * 1000)}" + # Unique per test for isolation (uuid — a ms timestamp collides when + # fast tests run within the same millisecond); clean up rows afterwards. + name = f"test_q_{os.getpid()}_{uuid.uuid4().hex}" yield name # A failed test body can leave the connection in an aborted transaction; # roll back first so this cleanup doesn't raise InFailedSqlTransaction. @@ -264,6 +288,111 @@ def test_read_hides_message_for_vt(self, pg_conn, queue_name): # Second read within vt sees nothing. assert client.read(queue_name, vt_seconds=30, qty=10) == [] + def test_priority_orders_dequeue(self, pg_conn, queue_name): + # Higher priority is claimed first; FIFO (msg_id) within a priority — + # regardless of enqueue order. Read one at a time (the default + # batch_size=1 path), so each claim selects the current top-priority row. + client = PgQueueClient(conn=pg_conn) + client.send(queue_name, {"n": "low1"}, priority=1) + client.send(queue_name, {"n": "high"}, priority=9) + client.send(queue_name, {"n": "low2"}, priority=1) + client.send(queue_name, {"n": "mid"}, priority=5) + claimed = [] + for _ in range(4): + msgs = client.read(queue_name, vt_seconds=30, qty=1) + assert len(msgs) == 1 + claimed.append(msgs[0].message["n"]) + client.delete(msgs[0].msg_id) + assert claimed == ["high", "mid", "low1", "low2"] + + def test_priority_orders_batch_claim(self, pg_conn, queue_name): + # A batched claim (qty > 1) returns the batch in priority order too — + # the CTE re-sorts RETURNING, which is otherwise unspecified. + client = PgQueueClient(conn=pg_conn) + client.send(queue_name, {"n": "a"}, priority=1) + client.send(queue_name, {"n": "b"}, priority=9) + client.send(queue_name, {"n": "c"}, priority=5) + msgs = client.read(queue_name, vt_seconds=30, qty=10) + assert [m.message["n"] for m in msgs] == ["b", "c", "a"] + + def test_batch_fifo_within_priority_band(self, pg_conn, queue_name): + # Two rows per band, interleaved enqueue → strict (band DESC, msg_id ASC). + client = PgQueueClient(conn=pg_conn) + for label, prio in [("9a", 9), ("1a", 1), ("9b", 9), ("1b", 1)]: + client.send(queue_name, {"n": label}, priority=prio) + msgs = client.read(queue_name, vt_seconds=30, qty=10) + assert [m.message["n"] for m in msgs] == ["9a", "9b", "1a", "1b"] + + def test_visible_low_priority_beats_invisible_high(self, pg_conn, queue_name): + # A claimed-but-unacked high-priority row (future vt) must not block a + # visible lower-priority row — exercises vt × priority interaction. + client = PgQueueClient(conn=pg_conn) + high_id = client.send(queue_name, {"n": "high"}, priority=9) + # Claim the high row → its vt jumps 30s ahead (now invisible). + assert [m.msg_id for m in client.read(queue_name, vt_seconds=30, qty=1)] == [ + high_id + ] + client.send(queue_name, {"n": "low"}, priority=1) + msgs = client.read(queue_name, vt_seconds=30, qty=1) + assert [m.message["n"] for m in msgs] == ["low"] + + def test_concurrent_claims_never_exceed_qty(self, pg_conn, queue_name): + # The CTE FROM-join claims exactly qty even under concurrent writers; + # the old IN(SELECT ... LIMIT) form could over-claim via EvalPlanQual. + # Two readers drain a backlog in parallel; no batch may exceed qty. + import threading + + client_a = PgQueueClient(conn=pg_conn) + for i in range(50): + client_a.send(queue_name, {"i": i}) + conn_b = create_pg_connection(env_prefix="TEST_DB_") + violations: list[int] = [] + + def drain(client): + while True: + msgs = client.read(queue_name, vt_seconds=30, qty=1) + if not msgs: + return + if len(msgs) > 1: # over-claim → the bug this rewrite fixes + violations.append(len(msgs)) + for m in msgs: + client.delete(m.msg_id) + + try: + worker = threading.Thread(target=drain, args=(PgQueueClient(conn=conn_b),)) + worker.start() + drain(client_a) + worker.join(timeout=15) + # Assert termination: a hung drain must fail the test, not pass + # silently while conn_b.close() races its in-flight queries. + assert not worker.is_alive(), "drain worker did not finish within 15s" + finally: + conn_b.close() + assert violations == [] + + def test_db_check_constraint_matches_fairness_bounds(self, pg_conn, queue_name): + # The DB CheckConstraint is the backstop for any raw/ORM writer that + # bypasses send()'s guard. It also pins the constraint to the app's + # fairness range: models.py and fairness.py live in separate codebases + # that can't import each other, so this boundary check is the guard + # against silent drift — in-range accepted, just outside rejected. + def _raw_insert(prio): + with pg_conn.cursor() as cur: + cur.execute( + "INSERT INTO pg_queue_message " + "(queue_name, message, org_id, priority, enqueued_at, vt, read_ct) " + "VALUES (%s, '{}'::jsonb, '', %s, now(), now(), 0)", + (queue_name, prio), + ) + + for prio in (MIN_PRIORITY, MAX_PRIORITY): # in-range boundaries accepted + _raw_insert(prio) + pg_conn.commit() + for prio in (MIN_PRIORITY - 1, MAX_PRIORITY + 1): # out-of-range rejected + with pytest.raises(psycopg2.errors.CheckViolation): + _raw_insert(prio) + pg_conn.rollback() + def test_vt_expiry_redelivers(self, pg_conn, queue_name): client = PgQueueClient(conn=pg_conn) client.send(queue_name, {"n": 1}) From 7c7617b3558cc7be7cc8b45df9c69dbca4128b58 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:37:36 +0530 Subject: [PATCH 09/44] =?UTF-8?q?UN-3548=20[FEAT]=20PgBarrier=20=E2=80=94?= =?UTF-8?q?=20Postgres=20fan-in=20barrier=20(3rd=20WORKER=5FBARRIER=5FBACK?= =?UTF-8?q?END)=20(#2053)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3548 [FEAT] PgBarrier — Postgres fan-in barrier (3rd WORKER_BARRIER_BACKEND) Add a Postgres Barrier substrate selected by WORKER_BARRIER_BACKEND=pg (default stays chord). Moves the fan-in aggregation ("wait for N header tasks, then fire the callback with their results") onto a pg_barrier_state row — the same DB that holds the PG queue, so an execution can coordinate without Redis/RabbitMQ. The 9e pipeline on-ramp primitive. Mirrors RedisDecrBarrier 1:1 — same Barrier protocol, fairness plumbing, Celery-dispatched header tasks with .link/.link_error, empty->None, missing-execution_id->raise, mid-loop dispatch cleanup. Defaults-off, zero behaviour change until the flag flips. - Schema (backend/pg_queue): pg_barrier_state (execution_id PK, remaining, results jsonb, aborted, expires_at) + migration 0004. - pg_barrier.py: PgBarrier + barrier_pg_decr_and_check / barrier_pg_abort. Atomic decrement is ONE statement (UPDATE ... SET remaining = remaining-1, results = results || jsonb_build_array(%s) ... RETURNING remaining, results, aborted) — row lock serialises concurrent decrements so exactly one sees 0; no Lua. Guards: reads aborted in the same statement (never fires partial), row-missing / negative-remaining clean up without firing, callback dispatched BEFORE row delete. Orphan bound via expires_at + opportunistic sweep in enqueue (periodic sweep is the backstop). - __init__.py: BarrierBackend.PG -> PgBarrier() in get_barrier(). Tests: protocol shape, TTL env validation, enqueue (upsert/links/fairness/ stale-reset/expiry-sweep/mid-loop-cleanup), decr paths (pending/complete-fires/ aborted/negative/missing/unserialisable), abort (claim+delete/dedup), and a real two-connection decrement-atomicity check (exactly one sees 0). Selector PG case. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3548 [FIX] Address PR #2053 review feedback High: - Abort is now ONE atomic statement: `WITH claimed AS (DELETE ... RETURNING) ...` — claim+teardown in a single transaction (no claimed-but-not-deleted window; a crash rolls back so a sibling retries). This makes the `aborted` column redundant — dropped it; the decrement's "row missing -> abandoned" branch now covers the failed-task case. The callback can only fire when remaining hits 0 (all tasks succeeded), so a failed task (which deletes the row) can never let a partial-results fire. - Dropped the per-enqueue global orphan sweep (unbounded DELETE on the hot path, deadlock-prone, shared the UPSERT txn). Reclaim is a future periodic sweep. - A NUL byte survives json.dumps but jsonb rejects it -> catch the DataError and tear the barrier down (fail fast) instead of hanging to expiry. Medium: - Post-dispatch row delete is best-effort (logged, not raised) so a delete error can't mask the already-fired callback; documented the no-double-fire invariant (last decrement + max_retries=0). - Added a DB CheckConstraint (expires_at > created_at) — the one writer-proof invariant; Meta comment warns off a `remaining >= 0` check (teardown needs negative). Softened the "periodic sweep" comments to future/not-yet-shipped. Low: - Extracted shared `barrier_ttl_seconds()` + `CallbackDescriptor` into barrier.py; both backends import them (redis keeps back-compat aliases). signature_kwargs dict instead of inline ** spread. Atomicity comment notes the per-transaction premise. Tests: callback-dispatch-failure-preserves-row; decrement-after-abort-no-fire; atomicity through barrier_pg_decr_and_check (two threads, exactly one fires); list-result-as-single-element; NUL-byte teardown; DB-constraint; max_retries=0. 92 barrier tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) * UN-3548 [DOCS] Drop stale aborted-column reference in PgBarrier docstring The wire-model docstring's enqueue step still listed `aborted = false` as an UPSERT column after the column was removed (abort now dedups via DELETE … RETURNING / row existence). Remove it so a reader doesn't hunt for — or re-add — a column that no longer exists. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../0004_pgbarrierstate_and_more.py | 36 ++ backend/pg_queue/models.py | 49 ++ workers/queue_backend/__init__.py | 11 + workers/queue_backend/barrier.py | 48 +- workers/queue_backend/pg_barrier.py | 419 ++++++++++++++++++ workers/queue_backend/redis_barrier.py | 68 +-- .../tests/test_barrier_backend_selection.py | 7 + workers/tests/test_pg_barrier.py | 407 +++++++++++++++++ 8 files changed, 987 insertions(+), 58 deletions(-) create mode 100644 backend/pg_queue/migrations/0004_pgbarrierstate_and_more.py create mode 100644 workers/queue_backend/pg_barrier.py create mode 100644 workers/tests/test_pg_barrier.py diff --git a/backend/pg_queue/migrations/0004_pgbarrierstate_and_more.py b/backend/pg_queue/migrations/0004_pgbarrierstate_and_more.py new file mode 100644 index 0000000000..c79b29a75a --- /dev/null +++ b/backend/pg_queue/migrations/0004_pgbarrierstate_and_more.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.1 on 2026-06-15 06:55 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("pg_queue", "0003_pgqueuemessage_pg_queue_message_priority_range"), + ] + + operations = [ + migrations.CreateModel( + name="PgBarrierState", + fields=[ + ("execution_id", models.TextField(primary_key=True, serialize=False)), + ("remaining", models.IntegerField()), + ("results", models.JSONField(default=list)), + ("created_at", models.DateTimeField(default=django.utils.timezone.now)), + ("expires_at", models.DateTimeField()), + ], + options={ + "db_table": "pg_barrier_state", + "indexes": [ + models.Index(fields=["expires_at"], name="pg_barrier_expires_idx") + ], + }, + ), + migrations.AddConstraint( + model_name="pgbarrierstate", + constraint=models.CheckConstraint( + check=models.Q(("expires_at__gt", models.F("created_at"))), + name="pg_barrier_expires_after_created", + ), + ), + ] diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py index 8b3386cb27..0eeb75e635 100644 --- a/backend/pg_queue/models.py +++ b/backend/pg_queue/models.py @@ -66,3 +66,52 @@ class Meta: name="pg_queue_message_dequeue_idx", ) ] + + +class PgBarrierState(models.Model): + """Per-execution fan-in barrier state for ``PgBarrier`` (the Postgres + ``WORKER_BARRIER_BACKEND``). + + One row per in-flight barrier (keyed by ``execution_id``). The worker-side + ``barrier_pg_decr_and_check`` link task atomically decrements ``remaining`` + and appends to ``results`` in a single ``UPDATE … RETURNING``; the task that + drives ``remaining`` to 0 dispatches the aggregating callback and deletes the + row. A header-task failure aborts the barrier by deleting the row outright + (``DELETE … RETURNING`` — atomic claim+teardown), so the callback can never + fire with partial results. ``expires_at`` bounds an orphaned barrier (header + tasks that never complete); a periodic sweep job (not yet implemented) is the + intended reclaim backstop. + + Managed=True / generated migration — no DB-side function, extension-free + (UN-3533), same posture as ``PgQueueMessage``. + """ + + execution_id = models.TextField(primary_key=True) + # Header tasks still pending. The last task to decrement it to 0 fires the + # callback. A value < 0 (decrement after expiry/cleanup) means the barrier + # was already torn down — the task cleans up without firing. + remaining = models.IntegerField() + # Aggregated header-task results, appended in completion order (JSONB array). + results = models.JSONField(default=list) + created_at = models.DateTimeField(default=timezone.now) + # Orphan bound (Redis-TTL equivalent): a barrier whose header tasks never + # complete is reclaimable past this. Must exceed the longest execution + # wall-clock, same budgeting as WORKER_BARRIER_KEY_TTL_SECONDS. + expires_at = models.DateTimeField() + + class Meta: + db_table = "pg_barrier_state" + constraints = [ + # The one writer-proof invariant (the worker SQL can't import this + # model). Deliberately NO `remaining >= 0` check — the teardown path + # relies on `remaining` going negative as a "barrier already gone" + # signal, so a non-negative constraint would break it. + models.CheckConstraint( + check=models.Q(expires_at__gt=models.F("created_at")), + name="pg_barrier_expires_after_created", + ), + ] + indexes = [ + # Drives the (future) periodic expiry-sweep job. + models.Index(fields=["expires_at"], name="pg_barrier_expires_idx"), + ] diff --git a/workers/queue_backend/__init__.py b/workers/queue_backend/__init__.py index c4310f5eff..d1c9154aa6 100644 --- a/workers/queue_backend/__init__.py +++ b/workers/queue_backend/__init__.py @@ -41,6 +41,11 @@ from .decorator import worker_task from .dispatch import dispatch from .fairness import FairnessKey +from .pg_barrier import ( + PgBarrier, + barrier_pg_abort, + barrier_pg_decr_and_check, +) from .redis_barrier import ( RedisDecrBarrier, barrier_abort, @@ -54,10 +59,13 @@ "BarrierHandle", "CeleryChordBarrier", "FairnessKey", + "PgBarrier", "QueueBackend", "RedisDecrBarrier", "barrier_abort", "barrier_decr_and_check", + "barrier_pg_abort", + "barrier_pg_decr_and_check", "dispatch", "get_barrier", "select_backend", @@ -77,6 +85,7 @@ class BarrierBackend(StrEnum): CHORD = "chord" REDIS = "redis" + PG = "pg" def get_barrier() -> Barrier: @@ -110,6 +119,8 @@ def get_barrier() -> Barrier: return CeleryChordBarrier() if backend is BarrierBackend.REDIS: return RedisDecrBarrier() + if backend is BarrierBackend.PG: + return PgBarrier() # Unreachable — StrEnum constructor would have raised above for # anything not in the enum. Defensive raise so the type checker # sees an exhaustive match. diff --git a/workers/queue_backend/barrier.py b/workers/queue_backend/barrier.py index 38b4045c16..d56348ce42 100644 --- a/workers/queue_backend/barrier.py +++ b/workers/queue_backend/barrier.py @@ -35,7 +35,8 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Protocol +import os +from typing import TYPE_CHECKING, Any, Protocol, TypedDict from celery import chord @@ -47,6 +48,51 @@ logger = logging.getLogger(__name__) +# Shared barrier-key TTL — both the Redis and PG backends bound an orphaned +# barrier (header tasks that never complete) by the same env var, since only one +# backend is active per deployment. One definition here prevents drift. +_DEFAULT_BARRIER_TTL_SECONDS = 6 * 60 * 60 # 6h + + +def barrier_ttl_seconds() -> int: + """Barrier TTL from ``WORKER_BARRIER_KEY_TTL_SECONDS`` (default 6h). + + Read at call time so tests can flip it. Invalid / non-positive values raise, + matching ``get_barrier()``'s loud-on-misconfig posture — a TTL shorter than + execution wall-clock would tear barriers down early (spurious behaviour). + """ + raw = os.getenv("WORKER_BARRIER_KEY_TTL_SECONDS") + if raw is None: + return _DEFAULT_BARRIER_TTL_SECONDS + try: + value = int(raw) + except ValueError as exc: + raise ValueError( + f"WORKER_BARRIER_KEY_TTL_SECONDS={raw!r} is not an integer. Unset it " + f"to default to {_DEFAULT_BARRIER_TTL_SECONDS}s (6h)." + ) from exc + if value <= 0: + raise ValueError( + f"WORKER_BARRIER_KEY_TTL_SECONDS={value} must be a positive integer. " + f"Unset it to default to {_DEFAULT_BARRIER_TTL_SECONDS}s (6h)." + ) + return value + + +class CallbackDescriptor(TypedDict): + """Serialisable aggregating-callback spec baked into a barrier link signature. + + Crosses a Celery serialisation boundary (producer → broker → worker), so the + four-key contract is typed to catch a typo/rename before it surfaces as a + remote ``KeyError`` mid-aggregation. Shared by both the Redis and PG + backends. ``fairness_headers`` is ``None`` when the producer passed no key. + """ + + task_name: str + kwargs: dict[str, Any] + queue: str + fairness_headers: dict[str, Any] | None + class Barrier(Protocol): """Fan-out-then-callback primitive. diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.py new file mode 100644 index 0000000000..4ebc46be87 --- /dev/null +++ b/workers/queue_backend/pg_barrier.py @@ -0,0 +1,419 @@ +"""PG fan-in barrier — Postgres substrate for the ``Barrier`` Protocol. + +Third ``WORKER_BARRIER_BACKEND`` option (``pg``) alongside ``chord`` and +``redis``. Mirrors :class:`~queue_backend.redis_barrier.RedisDecrBarrier` +exactly — same ``enqueue`` signature, same ``BarrierHandle | None`` contract, +same fairness plumbing, same Celery-dispatched header tasks with +``.link``/``.link_error`` — but moves the **aggregation** ("wait for N tasks, +then fire the callback with their results") from a Redis ``DECR`` counter to a +Postgres row. Selected at runtime by ``queue_backend.get_barrier``; default +stays ``chord``. + +**Why a Postgres substrate.** It lets an execution coordinate in the *same* +Postgres that holds the PG queue — no Redis (or RabbitMQ chord backend) needed +for the fan-in. The transport for the header tasks themselves is unchanged +(still Celery); only the coordination moves. + +**Wire model.** + +1. ``enqueue``: UPSERT one ``pg_barrier_state`` row (``remaining = N``, + ``results = []``, ``expires_at = now() + ttl``) — the UPSERT clears any stale + state from a prior run reusing the same ``execution_id``. Each header task is + dispatched with + ``.link(barrier_pg_decr_and_check)`` (success) and + ``.link_error(barrier_pg_abort)`` (failure). +2. Per-task success: ``barrier_pg_decr_and_check`` runs ONE atomic statement — + ``UPDATE … SET remaining = remaining - 1, results = results || + jsonb_build_array(result) … RETURNING remaining, results``. The row lock + serialises concurrent decrements, so exactly one task observes ``remaining = + 0``; that task dispatches the callback with the aggregated results, then + deletes the row. (No Lua — a single ``UPDATE … RETURNING`` is atomic in + Postgres. The guarantee relies on each decrement committing in its own + transaction — do NOT batch decrements into a shared transaction, or the row + lock would hold and the serialisation that makes exactly one see 0 breaks.) +3. Per-task failure: ``barrier_pg_abort`` runs as a ``link_error``. It tears the + barrier down with ONE atomic statement (``DELETE … RETURNING`` in a single + transaction): the row's existence is the dedup token, so N concurrent + failures collapse to a single cleanup, and a crash mid-abort rolls back + (leaving the row for a sibling to retry) — there is no claimed-but-not-deleted + window. Mirrors chord's default error semantic (callback not invoked on + header failure). +4. Orphan bound: like the Redis backend's key TTL, ``expires_at`` bounds a + barrier whose header tasks never complete. **No expiry reclaim ships in this + phase** — a periodic sweep job (keyed on ``pg_barrier_expires_idx``) is the + intended backstop and is future work. + +**Failure-masking guards.** The callback can only fire when ``remaining`` hits +exactly 0, which requires ALL N header tasks to have decremented — i.e. all +succeeded. A failed task runs the abort (which deletes the row) instead of a +decrement, so the count never reaches 0 and a late in-flight decrement finds no +row → abandons. A decrement that drives ``remaining`` negative (expiry/replay) +also cleans up without firing. The callback is dispatched BEFORE the row is +deleted, so a callback ``apply_async`` failure leaves the row (and its expiry) +in place rather than stranding the execution; the post-dispatch delete is +best-effort (logged, not raised) since the callback has already fired. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import threading +from collections.abc import Iterator +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import psycopg2 + +from .barrier import CallbackDescriptor, barrier_ttl_seconds +from .decorator import worker_task +from .fairness import FairnessKey +from .handle import BarrierHandle +from .pg_queue.connection import create_pg_connection + +if TYPE_CHECKING: + from celery.canvas import Signature + from psycopg2.extensions import connection as PgConnection + +logger = logging.getLogger(__name__) + + +# Thread-local owned connection (prefork → one per child; thread-local keeps it +# correct under -P threads too, since a libpq connection is not concurrency-safe +# across threads). Self-recovers a dropped socket / PgBouncer recycle — same +# posture as queue_backend.dispatch / PgQueueClient. +_local = threading.local() + + +def _get_conn() -> PgConnection: + conn = getattr(_local, "conn", None) + if conn is None or conn.closed: + conn = create_pg_connection(env_prefix="DB_") + _local.conn = conn + return conn + + +@contextlib.contextmanager +def _cursor() -> Iterator[Any]: + """Yield a cursor; commit on success, roll back + recover on error.""" + conn = _get_conn() + try: + with conn.cursor() as cur: + yield cur + conn.commit() + except Exception as exc: + conn_dead = isinstance(exc, (psycopg2.OperationalError, psycopg2.InterfaceError)) + try: + conn.rollback() + except Exception: + conn_dead = True + if conn_dead or conn.closed: + with contextlib.suppress(Exception): + conn.close() + _local.conn = None + raise + + +def _delete_barrier(execution_id: str) -> None: + with _cursor() as cur: + cur.execute( + "DELETE FROM pg_barrier_state WHERE execution_id = %s", (execution_id,) + ) + + +@dataclass(frozen=True, slots=True) +class _PgBarrierHandle: + """Minimal ``BarrierHandle`` — ``id`` is the execution id (what call sites + log for chord-id tracing), same as ``_RedisBarrierHandle``. + """ + + id: str + + +class PgBarrier: + """``Barrier`` implementation via a Postgres ``pg_barrier_state`` row. + + Drop-in for ``CeleryChordBarrier`` / ``RedisDecrBarrier`` from the call + sites' perspective. See the module docstring for the wire model and the + failure-masking guards. + """ + + def enqueue( + self, + header_tasks: list[Signature], + *, + callback_task_name: str, + callback_kwargs: dict[str, Any], + callback_queue: str, + app_instance: Any, + fairness: FairnessKey | None = None, + ) -> BarrierHandle | None: + """See :class:`queue_backend.barrier.Barrier.enqueue`. + + Empty ``header_tasks`` → ``None`` (caller owns the zero-files contract); + any substrate failure raises. ``app_instance`` is accepted for Protocol + parity but unused — the callback is built inside the link task via + ``current_app`` so it runs against the worker's app. + """ + del app_instance # Protocol parity; callback built in the link task. + if not header_tasks: + logger.info( + f"[exec:{callback_kwargs.get('execution_id')}] " + f"[pipeline:{callback_kwargs.get('pipeline_id')}] " + "Zero header tasks detected — skipping barrier enqueue " + "(parent should handle pipeline status updates directly)" + ) + return None + + execution_id = callback_kwargs.get("execution_id") + if not execution_id: + raise ValueError( + "PgBarrier requires execution_id in callback_kwargs — it's the " + "primary key of the per-execution pg_barrier_state row" + ) + execution_id = str(execution_id) + + try: + fairness_headers = fairness.as_header() if fairness else None + callback_descriptor: CallbackDescriptor = { + "task_name": callback_task_name, + "kwargs": callback_kwargs, + "queue": callback_queue, + "fairness_headers": fairness_headers, + } + ttl_seconds = barrier_ttl_seconds() + + with _cursor() as cur: + # UPSERT clears any leftover state from a prior run with this id. + # No inline expiry sweep here — an unbounded global DELETE on the + # enqueue hot path risks lock contention / deadlocks between + # concurrent enqueues; orphan reclaim is a separate (future) + # periodic sweep keyed on pg_barrier_expires_idx. + cur.execute( + "INSERT INTO pg_barrier_state " + "(execution_id, remaining, results, created_at, expires_at) " + "VALUES (%s, %s, '[]'::jsonb, now(), " + " now() + make_interval(secs => %s)) " + "ON CONFLICT (execution_id) DO UPDATE SET " + " remaining = EXCLUDED.remaining, results = '[]'::jsonb, " + " created_at = now(), expires_at = EXCLUDED.expires_at", + (execution_id, len(header_tasks), ttl_seconds), + ) + + link_signature = barrier_pg_decr_and_check.s( + execution_id=execution_id, + callback_descriptor=callback_descriptor, + ) + link_error_signature = barrier_pg_abort.s(execution_id=execution_id) + for i, task in enumerate(header_tasks): + try: + cloned = task.clone() + if fairness_headers: + cloned.set(headers=fairness_headers) + cloned.link(link_signature) + cloned.link_error(link_error_signature) + cloned.apply_async() + except Exception: + # Mid-loop dispatch failure: i of N never reached the broker, + # so the counter can't reach 0. Delete the row so in-flight + # links' decrement finds no row and cleans up; re-raise so the + # caller marks the workflow ERROR. + with contextlib.suppress(Exception): + _delete_barrier(execution_id) + logger.exception( + f"[exec:{execution_id}] apply_async failed at task " + f"{i}/{len(header_tasks)}; barrier row deleted to prevent " + f"spurious callback fires from the orphan tasks' links." + ) + raise + + logger.info( + f"Barrier enqueued via PgBarrier — exec_id={execution_id}, " + f"header_tasks={len(header_tasks)}, callback={callback_task_name}, " + f"queue={callback_queue}" + ) + return _PgBarrierHandle(id=execution_id) + + except Exception: + logger.exception( + f"[exec:{execution_id}] " + f"[pipeline:{callback_kwargs.get('pipeline_id')}] " + f"Failed to enqueue barrier via Postgres " + f"(callback={callback_task_name}, queue={callback_queue}, " + f"header_tasks={len(header_tasks)})" + ) + raise + + +@worker_task(name="barrier_pg_decr_and_check", max_retries=0) +def barrier_pg_decr_and_check( + result: Any, + *, + execution_id: str, + callback_descriptor: CallbackDescriptor, +) -> dict[str, Any]: + """Per-task ``link`` callback for :class:`PgBarrier`. + + Atomically appends this task's result and decrements ``remaining``. The + single task that drives ``remaining`` to 0 dispatches the aggregating + callback (then deletes the row). ``max_retries=0`` — a Celery retry would + replay the decrement and corrupt the count; an orphaned barrier is bounded + by ``expires_at`` instead. + """ + from celery import current_app + + try: + # No default=str — a non-JSON-safe leaf must fail loudly here (it would + # signal a BatchExecutionResult.to_dict() typed-boundary regression). + result_json = json.dumps(result) + except (TypeError, ValueError): + logger.exception( + f"[exec:{execution_id}] Header task result is not JSON-serialisable " + f"— barrier aggregation cannot proceed (typed-boundary regression)." + ) + raise + + # jsonb_build_array(...) appends exactly one element regardless of the + # result's shape (``||`` would concatenate if the result were itself a list). + try: + with _cursor() as cur: + cur.execute( + "UPDATE pg_barrier_state " + " SET remaining = remaining - 1, " + " results = results || jsonb_build_array(%s::jsonb) " + " WHERE execution_id = %s " + "RETURNING remaining, results", + (result_json, execution_id), + ) + row = cur.fetchone() + except psycopg2.DataError: + # json.dumps accepts a few bytes jsonb rejects — notably a NUL (0x00) + # in a string. The cast above then raises, the decrement never lands, and + # the barrier would hang to expires_at (~6h). Tear it down so the + # execution fails fast and visibly instead. + logger.exception( + f"[exec:{execution_id}] Header result rejected by jsonb (e.g. a NUL " + f"byte) — tearing down the barrier so the execution fails fast " + f"rather than hanging until expiry." + ) + with contextlib.suppress(Exception): + _delete_barrier(execution_id) + raise + + if row is None: + # Barrier already torn down (a header failed → abort deleted the row, or + # an expiry sweep removed it). No callback. + logger.error( + f"[exec:{execution_id}] PgBarrier decrement found no row — barrier " + f"already torn down. No callback dispatched." + ) + return {"status": "abandoned", "remaining": None} + + remaining, all_results = int(row[0]), row[1] + logger.info(f"[exec:{execution_id}] PgBarrier decrement → remaining={remaining}") + + if remaining > 0: + return {"status": "pending", "remaining": remaining} + + if remaining < 0: + # Decremented past 0 — expiry/replay. Clean up, no fire. + _delete_barrier(execution_id) + logger.error( + f"[exec:{execution_id}] PgBarrier abandoned — remaining={remaining} " + f"(expired or torn down). No callback dispatched; execution likely " + f"in an inconsistent terminal state and needs investigation." + ) + return {"status": "abandoned", "remaining": remaining} + + # remaining == 0: we are the last task. psycopg2 decodes the jsonb array to + # a Python list already — no per-element json.loads needed. Build the kwargs + # explicitly (headers only when truthy) — clearer than an inline ** spread, + # matching CeleryChordBarrier's idiom. + signature_kwargs: dict[str, Any] = { + "args": [all_results], + "kwargs": callback_descriptor["kwargs"], + "queue": callback_descriptor["queue"], + } + if callback_descriptor.get("fairness_headers"): + signature_kwargs["headers"] = callback_descriptor["fairness_headers"] + callback_signature = current_app.signature( + callback_descriptor["task_name"], **signature_kwargs + ) + # Dispatch FIRST; delete the row only after dispatch succeeds, so a callback + # apply_async failure leaves the row (and its expiry) in place rather than + # stranding the execution with no state and no recovery path. + callback_result = callback_signature.apply_async() + # The post-dispatch delete must not mask the successful dispatch: the callback + # already fired, so a delete error here is logged (not raised) — the row + # lingers until expiry rather than re-running the callback. No double-fire is + # possible: this is the last decrement (remaining hit 0) and max_retries=0, so + # the link task is never replayed. + try: + _delete_barrier(execution_id) + except Exception: + logger.exception( + f"[exec:{execution_id}] Barrier callback dispatched " + f"(callback_task_id={callback_result.id}) but the post-dispatch row " + f"delete failed — row will be reclaimed at expiry. Callback NOT " + f"re-run (max_retries=0)." + ) + + logger.info( + f"[exec:{execution_id}] Barrier complete — fired callback " + f"{callback_descriptor['task_name']} on {callback_descriptor['queue']} " + f"with {len(all_results)} aggregated results " + f"(callback_task_id={callback_result.id})" + ) + return { + "status": "complete", + "callback_task_id": callback_result.id, + "aggregated_count": len(all_results), + } + + +@worker_task(name="barrier_pg_abort", max_retries=0) +def barrier_pg_abort( + request: Any = None, + exc: Any = None, + traceback: Any = None, + *, + execution_id: str, +) -> dict[str, Any]: + """``link_error`` callback: a header task failed → tear down barrier state. + + Mirrors chord's default error semantic (callback not invoked on header + failure). The claim and the teardown are a SINGLE atomic statement — a + ``DELETE … RETURNING`` in one transaction. The row's existence is the dedup + token: the first abort deletes it (and so "wins"); every concurrent sibling + finds nothing to delete and short-circuits. Because it's one transaction, a + crash/failure mid-abort rolls the whole thing back (the row survives) so a + sibling can retry — there is no "claimed-but-not-deleted" window. A late + in-flight successful decrement then finds no row and abandons (no partial + fire). + + The ``request``/``exc``/``traceback`` defaults keep the old-style + ``errback(task_id)`` invocation (mixed-version deploy / ``bind=True``) from + raising a ``TypeError``. This task does NOT drive workflow terminal status — + the outer orchestrators own that. + """ + del request, exc, traceback # logged by the outer task; unused here + with _cursor() as cur: + cur.execute( + "WITH claimed AS (" + " DELETE FROM pg_barrier_state WHERE execution_id = %s " + " RETURNING execution_id" + ") SELECT execution_id FROM claimed", + (execution_id,), + ) + claimed = cur.fetchone() is not None + + if not claimed: + # Another failure already aborted this execution (or it's already gone). + return {"status": "already_aborted", "execution_id": execution_id} + + logger.error( + f"[exec:{execution_id}] PgBarrier aborted — a header task failed; " + f"barrier state cleaned up (aggregating callback will not fire)." + ) + return {"status": "aborted", "execution_id": execution_id} diff --git a/workers/queue_backend/redis_barrier.py b/workers/queue_backend/redis_barrier.py index 5f79db9cd8..346e44f8b7 100644 --- a/workers/queue_backend/redis_barrier.py +++ b/workers/queue_backend/redis_barrier.py @@ -64,12 +64,17 @@ import logging import os from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, TypedDict +from typing import TYPE_CHECKING, Any from celery.canvas import Signature from unstract.core.cache.redis_client import create_redis_client +from .barrier import ( + _DEFAULT_BARRIER_TTL_SECONDS, + CallbackDescriptor, + barrier_ttl_seconds, +) from .decorator import worker_task from .fairness import FairnessKey from .handle import BarrierHandle @@ -111,41 +116,11 @@ # longer workflows (e.g. multi-step pipelines with chained barriers) # or shorter known max-execution-time should tune via # ``WORKER_BARRIER_KEY_TTL_SECONDS``. -_KEY_TTL_DEFAULT_SECONDS = 6 * 60 * 60 # 6h - - -def _key_ttl_seconds() -> int: - """Read the TTL from env, applying the default only on absence. - - Read at call time (not module import) so a test - ``monkeypatch.setenv`` flips the value without a module reload. - - Invalid values (non-int, negative, zero) **raise**, matching the - posture in ``get_barrier()`` where ``WORKER_BARRIER_BACKEND=rediz`` - raises rather than silently falling back to chord. A misconfigured - TTL shorter than execution wall-clock is a correctness issue per - this file's docstring (would cause spurious callback fires) — - operators get the same loud-on-misconfig signal as for the - backend flag. - """ - raw = os.getenv("WORKER_BARRIER_KEY_TTL_SECONDS") - if raw is None: - return _KEY_TTL_DEFAULT_SECONDS - try: - value = int(raw) - except ValueError as e: - raise ValueError( - f"WORKER_BARRIER_KEY_TTL_SECONDS={raw!r} is not an integer. " - f"Unset the env var to default to {_KEY_TTL_DEFAULT_SECONDS}s " - f"(6h)." - ) from e - if value <= 0: - raise ValueError( - f"WORKER_BARRIER_KEY_TTL_SECONDS={value} must be a positive " - f"integer. Unset the env var to default to " - f"{_KEY_TTL_DEFAULT_SECONDS}s (6h)." - ) - return value +# TTL lives in barrier.py (shared with PgBarrier — both bound an orphaned +# barrier by the same env var). Aliased so this module's internal call sites and +# tests keep their names. +_key_ttl_seconds = barrier_ttl_seconds +_KEY_TTL_DEFAULT_SECONDS = _DEFAULT_BARRIER_TTL_SECONDS # Atomic ``RPUSH + EXPIRE + DECR``: returns ``(remaining, results)``. @@ -511,27 +486,6 @@ def enqueue( raise -class CallbackDescriptor(TypedDict): - """Shape of the dict baked into the link signature and re-read on - the worker that runs ``barrier_decr_and_check``. - - This descriptor crosses a serialisation boundary (Celery - ``signature(...)`` → broker → consumer worker), so the four-key - contract is otherwise enforced only by string literals duplicated - across producer and consumer. Typing it as a ``TypedDict`` gives - the type checker a chance to catch typos / renames before they - surface as remote ``KeyError`` mid-aggregation. - - ``fairness_headers`` is always present in the dict; ``None`` when - the producer passed no ``FairnessKey``. - """ - - task_name: str - kwargs: dict[str, Any] - queue: str - fairness_headers: dict[str, Any] | None - - @dataclass(frozen=True, slots=True) class _RedisBarrierHandle: """Minimal ``BarrierHandle`` implementation. diff --git a/workers/tests/test_barrier_backend_selection.py b/workers/tests/test_barrier_backend_selection.py index 7a452652e5..0948814e67 100644 --- a/workers/tests/test_barrier_backend_selection.py +++ b/workers/tests/test_barrier_backend_selection.py @@ -26,6 +26,7 @@ from queue_backend import ( BarrierBackend, CeleryChordBarrier, + PgBarrier, RedisDecrBarrier, get_barrier, ) @@ -39,6 +40,7 @@ def test_chord_value_is_canonical_string(self): documented env value).""" assert BarrierBackend.CHORD.value == "chord" assert BarrierBackend.REDIS.value == "redis" + assert BarrierBackend.PG.value == "pg" def test_str_enum_string_equality(self): """``StrEnum`` members compare equal to their string values — @@ -46,6 +48,7 @@ def test_str_enum_string_equality(self): interchangeably without conversion.""" assert BarrierBackend.CHORD == "chord" assert BarrierBackend.REDIS == "redis" + assert BarrierBackend.PG == "pg" class TestGetBarrierFactory: @@ -66,6 +69,10 @@ def test_redis_selects_redis_decr_barrier(self, monkeypatch): monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.REDIS) assert isinstance(get_barrier(), RedisDecrBarrier) + def test_pg_selects_pg_barrier(self, monkeypatch): + monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.PG) + assert isinstance(get_barrier(), PgBarrier) + def test_unknown_value_raises_loudly(self, monkeypatch): """Silent fallback to default would mask a typo'd production env (``rediz``, ``REDDIS``, ``redis-decr``) — the canary diff --git a/workers/tests/test_pg_barrier.py b/workers/tests/test_pg_barrier.py new file mode 100644 index 0000000000..0c1a635c1d --- /dev/null +++ b/workers/tests/test_pg_barrier.py @@ -0,0 +1,407 @@ +"""Tests for :class:`queue_backend.pg_barrier.PgBarrier`. + +Three layers: + +1. **Protocol / TTL** — no DB, no Celery. +2. **Enqueue + link/abort tasks** — a real autocommit Postgres connection is + injected into the module thread-local (the barrier's SQL runs for real); the + Celery header-task dispatch + callback are mocked. Skips if Postgres is + unreachable or the ``pg_barrier_state`` migration is unapplied. +3. **Atomicity** — two real connections race the decrement SQL directly. +""" + +from __future__ import annotations + +import os +from unittest.mock import MagicMock, patch + +import psycopg2 +import pytest +from queue_backend import barrier as barrier_mod +from queue_backend import pg_barrier +from queue_backend.fairness import FAIRNESS_HEADER_NAME, FairnessKey, WorkloadType +from queue_backend.handle import BarrierHandle +from queue_backend.pg_barrier import ( + PgBarrier, + barrier_pg_abort, + barrier_pg_decr_and_check, +) +from queue_backend.pg_queue.connection import create_pg_connection + +_CALLBACK = { + "task_name": "process_batch_callback_api", + "kwargs": {"execution_id": "exec-1", "pipeline_id": "pipe-1"}, + "queue": "general", + "fairness_headers": None, +} + + +def _mock_header_task(): + """A header-task Signature whose clone() records link/link_error/apply_async.""" + cloned = MagicMock(name="cloned_signature") + task = MagicMock(name="header_signature") + task.clone.return_value = cloned + return task, cloned + + +# --- Layer 1: protocol shape + TTL (no DB) --- + + +class TestPgBarrierProtocolShape: + def test_satisfies_barrier_protocol(self): + barrier: barrier_mod.Barrier = PgBarrier() + assert callable(getattr(barrier, "enqueue", None)) + + def test_handle_satisfies_barrier_handle(self): + handle: BarrierHandle = pg_barrier._PgBarrierHandle(id="exec-1") + assert handle.id == "exec-1" + assert isinstance(handle.id, str) + + +class TestTtlEnv: + # PgBarrier shares barrier.barrier_ttl_seconds() with the Redis backend. + def test_default_is_six_hours(self, monkeypatch): + monkeypatch.delenv("WORKER_BARRIER_KEY_TTL_SECONDS", raising=False) + assert barrier_mod.barrier_ttl_seconds() == 6 * 60 * 60 + + def test_overridable(self, monkeypatch): + monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "120") + assert barrier_mod.barrier_ttl_seconds() == 120 + + @pytest.mark.parametrize("bad", ["abc", "0", "-5"]) + def test_invalid_raises(self, monkeypatch, bad): + monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", bad) + with pytest.raises(ValueError, match="WORKER_BARRIER_KEY_TTL_SECONDS"): + barrier_mod.barrier_ttl_seconds() + + +class TestEnqueueShortCircuits: + def test_empty_header_returns_none(self): + # Returns before any DB / dispatch. + assert ( + PgBarrier().enqueue( + [], + callback_task_name="cb", + callback_kwargs={"execution_id": "e"}, + callback_queue="general", + app_instance=None, + ) + is None + ) + + def test_missing_execution_id_raises(self): + with pytest.raises(ValueError, match="execution_id"): + PgBarrier().enqueue( + [_mock_header_task()[0]], + callback_task_name="cb", + callback_kwargs={}, # no execution_id + callback_queue="general", + app_instance=None, + ) + + +# --- Layer 2: enqueue + link/abort with a real injected connection --- + + +@pytest.fixture +def barrier_db(): + """Inject a real autocommit connection into pg_barrier's thread-local. + + Autocommit so each statement commits independently (like the worker tasks in + production) and the barrier's ``_cursor`` commit() is a no-op. The link/abort + tasks run in this same (test) thread, so they see the injected connection. + """ + os.environ.setdefault("TEST_DB_HOST", "127.0.0.1") + try: + conn = create_pg_connection(env_prefix="TEST_DB_") + except psycopg2.OperationalError as exc: + pytest.skip(f"Postgres not reachable: {exc}") + conn.autocommit = True + with conn.cursor() as cur: + cur.execute("SELECT to_regclass('pg_barrier_state')") + if cur.fetchone()[0] is None: + conn.close() + pytest.skip("pg_barrier_state migration not applied (run backend migrate)") + cur.execute("DELETE FROM pg_barrier_state") + pg_barrier._local.conn = conn + yield conn + with conn.cursor() as cur: + cur.execute("DELETE FROM pg_barrier_state") + conn.close() + pg_barrier._local.conn = None + + +def _row(conn, execution_id): + with conn.cursor() as cur: + cur.execute( + "SELECT remaining, results FROM pg_barrier_state WHERE execution_id = %s", + (execution_id,), + ) + return cur.fetchone() + + +class TestPgBarrierEnqueue: + def test_upsert_creates_row_and_attaches_links(self, barrier_db): + tasks = [_mock_header_task() for _ in range(3)] + handle = PgBarrier().enqueue( + [t for t, _ in tasks], + callback_task_name="cb", + callback_kwargs={"execution_id": "exec-A"}, + callback_queue="general", + app_instance=None, + ) + assert handle.id == "exec-A" + assert _row(barrier_db, "exec-A") == (3, []) + for _, cloned in tasks: + cloned.link.assert_called_once() + cloned.link_error.assert_called_once() + cloned.apply_async.assert_called_once() + + def test_fairness_header_stamped(self, barrier_db): + task, cloned = _mock_header_task() + PgBarrier().enqueue( + [task], + callback_task_name="cb", + callback_kwargs={"execution_id": "exec-F"}, + callback_queue="general", + app_instance=None, + fairness=FairnessKey(org_id="o", workload_type=WorkloadType.API), + ) + headers = cloned.set.call_args.kwargs["headers"] + assert FAIRNESS_HEADER_NAME in headers + + def test_upsert_overwrites_stale_state(self, barrier_db): + # A prior run left a row at remaining=1 with results; the new enqueue + # resets it. + with barrier_db.cursor() as cur: + cur.execute( + "INSERT INTO pg_barrier_state " + "(execution_id, remaining, results, created_at, expires_at) " + "VALUES ('exec-R', 1, '[1,2]'::jsonb, now(), now() + interval '1h')" + ) + task, _ = _mock_header_task() + PgBarrier().enqueue( + [task, _mock_header_task()[0]], + callback_task_name="cb", + callback_kwargs={"execution_id": "exec-R"}, + callback_queue="general", + app_instance=None, + ) + assert _row(barrier_db, "exec-R") == (2, []) + + def test_mid_loop_dispatch_failure_deletes_row(self, barrier_db): + good, _ = _mock_header_task() + bad, bad_cloned = _mock_header_task() + bad_cloned.apply_async.side_effect = RuntimeError("broker down") + with pytest.raises(RuntimeError): + PgBarrier().enqueue( + [good, bad], + callback_task_name="cb", + callback_kwargs={"execution_id": "exec-D"}, + callback_queue="general", + app_instance=None, + ) + assert _row(barrier_db, "exec-D") is None # cleaned up + + +def _seed(conn, execution_id, remaining, *, results="[]"): + with conn.cursor() as cur: + cur.execute( + "INSERT INTO pg_barrier_state " + "(execution_id, remaining, results, created_at, expires_at) " + "VALUES (%s, %s, %s::jsonb, now(), now() + interval '1h')", + (execution_id, remaining, results), + ) + + +class TestDecrAndCheck: + def test_pending_decrements_only(self, barrier_db): + _seed(barrier_db, "exec-P", 3) + out = barrier_pg_decr_and_check( + {"f": 1}, execution_id="exec-P", callback_descriptor=_CALLBACK + ) + assert out["status"] == "pending" + remaining, results = _row(barrier_db, "exec-P") + assert remaining == 2 + assert results == [{"f": 1}] + + def test_complete_fires_callback_with_aggregated_results(self, barrier_db): + _seed(barrier_db, "exec-C", 1, results='[{"f": "a"}]') + with patch("celery.current_app.signature") as sig: + sig.return_value.apply_async.return_value = MagicMock(id="cb-task-1") + out = barrier_pg_decr_and_check( + {"f": "b"}, execution_id="exec-C", callback_descriptor=_CALLBACK + ) + assert out["status"] == "complete" + # Callback got the full aggregated list as its first positional arg. + assert sig.call_args.kwargs["args"] == [[{"f": "a"}, {"f": "b"}]] + assert _row(barrier_db, "exec-C") is None # row deleted after dispatch + + def test_complete_path_passes_fairness_header(self, barrier_db): + _seed(barrier_db, "exec-FH", 1) + descriptor = {**_CALLBACK, "fairness_headers": {FAIRNESS_HEADER_NAME: {"o": 1}}} + with patch("celery.current_app.signature") as sig: + sig.return_value.apply_async.return_value = MagicMock(id="cb") + barrier_pg_decr_and_check( + {"f": 1}, execution_id="exec-FH", callback_descriptor=descriptor + ) + assert sig.call_args.kwargs["headers"] == {FAIRNESS_HEADER_NAME: {"o": 1}} + + def test_callback_dispatch_failure_preserves_row(self, barrier_db): + # The central failure-masking invariant: dispatch happens BEFORE the row + # is deleted, so an apply_async failure leaves the row in place (reclaimed + # by expiry) — guards against a delete-before-dispatch reorder. + _seed(barrier_db, "exec-CF", 1) + with patch("celery.current_app.signature") as sig: + sig.return_value.apply_async.side_effect = RuntimeError("broker down") + with pytest.raises(RuntimeError): + barrier_pg_decr_and_check( + {"f": 1}, execution_id="exec-CF", callback_descriptor=_CALLBACK + ) + assert _row(barrier_db, "exec-CF") is not None # row survives for TTL reclaim + + def test_list_result_appended_as_single_element(self, barrier_db): + # jsonb_build_array() must append a list-shaped result as ONE element + # (plain `||` would concatenate it). Guards that choice. + _seed(barrier_db, "exec-L", 2) + barrier_pg_decr_and_check( + [1, 2], execution_id="exec-L", callback_descriptor=_CALLBACK + ) + remaining, results = _row(barrier_db, "exec-L") + assert remaining == 1 + assert results == [[1, 2]] # one element that is the list, not [1, 2] + + def test_nul_byte_result_tears_down_barrier(self, barrier_db): + # A NUL byte survives json.dumps but jsonb rejects it. The barrier must + # be torn down (fail fast) rather than hang to expiry. + _seed(barrier_db, "exec-NB", 1) + with pytest.raises(psycopg2.DataError): + barrier_pg_decr_and_check( + {"f": "bad\x00value"}, + execution_id="exec-NB", + callback_descriptor=_CALLBACK, + ) + assert _row(barrier_db, "exec-NB") is None # torn down, not left hanging + + def test_decrement_after_abort_does_not_fire(self, barrier_db): + # The new failure-masking model: an aborted barrier is GONE (abort + # deletes the row), so a late in-flight decrement finds no row and never + # fires — even when it would otherwise have hit remaining == 0. + _seed(barrier_db, "exec-DA", 1) + barrier_pg_abort(execution_id="exec-DA") # header failed → row deleted + with patch("celery.current_app.signature") as sig: + out = barrier_pg_decr_and_check( + {"f": 1}, execution_id="exec-DA", callback_descriptor=_CALLBACK + ) + assert out["status"] == "abandoned" + sig.assert_not_called() + + def test_negative_remaining_does_not_fire(self, barrier_db): + _seed(barrier_db, "exec-N", 0) # decrement → -1 + with patch("celery.current_app.signature") as sig: + out = barrier_pg_decr_and_check( + {"f": 1}, execution_id="exec-N", callback_descriptor=_CALLBACK + ) + assert out["status"] == "abandoned" + sig.assert_not_called() + assert _row(barrier_db, "exec-N") is None + + def test_missing_row_does_not_fire(self, barrier_db): + with patch("celery.current_app.signature") as sig: + out = barrier_pg_decr_and_check( + {"f": 1}, execution_id="nope", callback_descriptor=_CALLBACK + ) + assert out["status"] == "abandoned" + sig.assert_not_called() + + def test_unserialisable_result_raises(self, barrier_db): + _seed(barrier_db, "exec-U", 1) + with pytest.raises(TypeError): + barrier_pg_decr_and_check( + {object()}, execution_id="exec-U", callback_descriptor=_CALLBACK + ) + + def test_registered_under_canonical_name(self): + assert barrier_pg_decr_and_check.name == "barrier_pg_decr_and_check" + + +class TestAbort: + def test_claims_and_deletes(self, barrier_db): + _seed(barrier_db, "exec-X", 2) + out = barrier_pg_abort(execution_id="exec-X") + assert out["status"] == "aborted" + assert _row(barrier_db, "exec-X") is None + + def test_concurrent_aborts_deduplicate(self, barrier_db): + _seed(barrier_db, "exec-Y", 2) + first = barrier_pg_abort(execution_id="exec-Y") + second = barrier_pg_abort(execution_id="exec-Y") + assert first["status"] == "aborted" + assert second["status"] == "already_aborted" # row gone / claim lost + + def test_registered_under_canonical_name(self): + assert barrier_pg_abort.name == "barrier_pg_abort" + + + def test_max_retries_zero(self): + # A Celery retry would replay the decrement and corrupt the count. + assert barrier_pg_decr_and_check.max_retries == 0 + assert barrier_pg_abort.max_retries == 0 + + +# --- Layer 3: atomicity through the real task + DB constraint --- + + +class TestDecrementAtomicityThroughTask: + def test_exactly_one_task_fires_the_callback(self, barrier_db): + # Two header-task links decrement a remaining=2 barrier concurrently, + # each through barrier_pg_decr_and_check itself (not raw SQL). The row + # lock serialises them so exactly one returns "complete" and the callback + # is dispatched exactly once — the real single-fire guarantee. + import threading + + _seed(barrier_db, "exec-Z", 2) + statuses: dict[str, str] = {} + conns = [] + + def run(label): + conn = create_pg_connection(env_prefix="TEST_DB_") + conn.autocommit = True + conns.append(conn) + pg_barrier._local.conn = conn # this thread's own connection + try: + out = barrier_pg_decr_and_check( + {"t": label}, execution_id="exec-Z", callback_descriptor=_CALLBACK + ) + statuses[label] = out["status"] + finally: + pg_barrier._local.conn = None + + with patch("celery.current_app.signature") as sig: + sig.return_value.apply_async.return_value = MagicMock(id="cb") + threads = [threading.Thread(target=run, args=(x,)) for x in ("a", "b")] + for t in threads: + t.start() + for t in threads: + t.join(timeout=10) + assert all(not t.is_alive() for t in threads) + assert sorted(statuses.values()) == ["complete", "pending"] + assert sig.return_value.apply_async.call_count == 1 # single fire + for c in conns: + c.close() + + +class TestDbConstraint: + def test_expires_at_must_exceed_created_at(self, barrier_db): + # The one writer-proof invariant (workers SQL can't import the model). + with pytest.raises(psycopg2.errors.CheckViolation): + with barrier_db.cursor() as cur: + cur.execute( + "INSERT INTO pg_barrier_state " + "(execution_id, remaining, results, created_at, expires_at) " + "VALUES ('bad', 1, '[]'::jsonb, now(), now())" # expires == created + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 52a28fda1384ca329facfbed069d6e8db46b2694 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Mon, 15 Jun 2026 21:14:08 +0530 Subject: [PATCH 10/44] =?UTF-8?q?UN-3553=20[FEAT]=20PG=20Queue=209d=20slic?= =?UTF-8?q?e=201=20=E2=80=94=20leader-election=20lease=20(orchestrator=5Fl?= =?UTF-8?q?ock)=20(#2056)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3553 [FEAT] PG Queue 9d slice 1 — leader-election lease (orchestrator_lock) First slice of 9d (orchestrator/reaper): the singleton-guarantee primitive the reaper loop and a future fair-admission gate hang off. Ships dark — nothing acquires leadership yet, so merging changes no runtime behaviour. Why now: 9d was skipped in the merged spine (9c -> liveness -> priority -> PgBarrier) and is the safety net 9e needs — without a reaper, every at-least-once hang / orphaned barrier bottoms out at the 6h TTL with no recovery. The reaper must run as exactly one instance, so leader election is the foundation. Lease, not advisory lock: leadership is a TTL'd row UPDATE (take it if the leader is free or its lease is stale), not pg_advisory_lock. Session-scoped advisory locks don't survive the transaction-pooled PgBouncer the queue connects through (UN-3533) — a plain UPDATE is one transaction, pooling-safe. All time comparisons are server-side (now()), so candidate clock skew can't split leadership. - backend/pg_queue: PgOrchestratorLock single-row model (id PK, leader, acquired_at) + CheckConstraint(id=1); generated migration 0005 + a reversible RunPython seeding the one free row. Free = empty leader (follows the PgQueueMessage.org_id no-nullable-text convention). - workers/queue_backend/pg_queue/leader_election.py: LeaderLease (try_acquire/renew/release), lease_seconds_from_env() (default 10s, loud-on-misconfig), default_worker_id(). Instance-owned self-recovering connection. - tests: 20 real-PG tests. Load-bearing properties — concurrent try_acquire yields exactly one winner; renew returns False after a stale-lease takeover (the signal that stops a stalled leader). Plus lease-expiry takeover, release-frees-immediately, non-holder no-ops, env validation, single-row constraint. Co-Authored-By: Claude Opus 4.8 * UN-3553 [FEAT] Address review: connection ownership, logging, recovery tests Toolkit + SonarCloud review on #2056: - [High] _owns_conn ownership guard: LeaderLease now mirrors PgQueueClient — an injected connection is never closed/swapped on a transient error (it would otherwise silently re-point an injected TEST_DB_/caller connection at a fresh DB_-env one). _get_conn only recreates an OWNED missing/closed conn. - [Medium] Log the owned-connection discard in _cursor (worker_id + exc type) — a silent rebuild on the reaper singleton correlates with missed renews. - [Medium] Test the recovery machinery: owned-conn recovered on OperationalError, owned-conn recovered when rollback fails, injected-conn never swapped. Plus two documented-invariant gaps — same-holder re-acquire on a fresh lease returns False, and release after a takeover is a no-op. - [Low] release() branches on rowcount — only logs "released" when it really freed the lease; a no-op release logs debug (truthful post-mortems). - [Low] Scope the lease_seconds<=0 guard to the explicit-arg branch (dead on the env path, which already rejects <=0). - [Low] Document the exception-propagation contract (raise == "leadership unknown, stop acting"), relabel the durable Usage example. - [Low] Migration: note the seed row is load-bearing (future reaper-bootstrap should self-heal with INSERT ... ON CONFLICT DO NOTHING). - SonarCloud S117: rename the migration's get_model local to lock_model. 25 leader-election tests pass; makemigrations --check clean. Co-Authored-By: Claude Opus 4.8 * UN-3553 [FEAT] Address Greptile: idempotent worker id + clearer renew log - default_worker_id() is now cached (functools.cache) → idempotent per process, so a caller passing it inline in a retry/restart loop can't drift the worker id out from under renew()/release(). Lazy (first-call), so it's fixed after a fork rather than shared across children. Test asserts idempotency. - renew()'s failure warning now reads "not the current leader (taken over by another candidate, or the lease was never held)" — accurate for the non-holder-renew case too, and fixes the "took over"->"taken over" grammar. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .../migrations/0005_pgorchestratorlock.py | 50 +++ backend/pg_queue/models.py | 46 +++ workers/queue_backend/pg_queue/__init__.py | 4 + .../queue_backend/pg_queue/leader_election.py | 263 ++++++++++++++ workers/tests/test_leader_election.py | 321 ++++++++++++++++++ 5 files changed, 684 insertions(+) create mode 100644 backend/pg_queue/migrations/0005_pgorchestratorlock.py create mode 100644 workers/queue_backend/pg_queue/leader_election.py create mode 100644 workers/tests/test_leader_election.py diff --git a/backend/pg_queue/migrations/0005_pgorchestratorlock.py b/backend/pg_queue/migrations/0005_pgorchestratorlock.py new file mode 100644 index 0000000000..f3adf28cbb --- /dev/null +++ b/backend/pg_queue/migrations/0005_pgorchestratorlock.py @@ -0,0 +1,50 @@ +# Generated by Django 4.2.1 on 2026-06-15 10:35 + +import django.utils.timezone +from django.db import migrations, models + + +def seed_single_row(apps, schema_editor): + """Seed the one lock row (free: leader=''). + + Leader-election is a pure ``UPDATE ... WHERE id = 1`` — the row must exist + for any candidate to ever win; if it's ever absent, every ``try_acquire`` + silently returns ``False`` forever (permanent leaderlessness, no error). So + the future reaper-bootstrap slice should also self-heal with an idempotent + ``INSERT ... ON CONFLICT (id) DO NOTHING`` on startup. ``get_or_create`` + keeps re-runs / squashes idempotent here. + """ + lock_model = apps.get_model("pg_queue", "PgOrchestratorLock") + lock_model.objects.get_or_create(id=1, defaults={"leader": ""}) + + +def unseed_single_row(apps, schema_editor): + lock_model = apps.get_model("pg_queue", "PgOrchestratorLock") + lock_model.objects.filter(id=1).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("pg_queue", "0004_pgbarrierstate_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="PgOrchestratorLock", + fields=[ + ("id", models.IntegerField(default=1, primary_key=True, serialize=False)), + ("leader", models.TextField(blank=True, default="")), + ("acquired_at", models.DateTimeField(default=django.utils.timezone.now)), + ], + options={ + "db_table": "pg_orchestrator_lock", + }, + ), + migrations.AddConstraint( + model_name="pgorchestratorlock", + constraint=models.CheckConstraint( + check=models.Q(("id", 1)), name="pg_orchestrator_lock_single_row" + ), + ), + migrations.RunPython(seed_single_row, unseed_single_row), + ] diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py index 0eeb75e635..bb086c7783 100644 --- a/backend/pg_queue/models.py +++ b/backend/pg_queue/models.py @@ -68,6 +68,52 @@ class Meta: ] +class PgOrchestratorLock(models.Model): + """Single-row leader-election lease for the orchestrator/reaper singleton. + + The orchestrator (admit) and reaper (stuck-task recovery, barrier-orphan + sweep) must run as **exactly one** active instance — multiple would contend + on ``SKIP LOCKED`` and double-act on recovery. HA is leader-elected: one or + more candidate processes race to hold this single row; the holder renews its + lease each cycle, and if it dies a standby takes over once the lease goes + stale. + + **Lease, not advisory lock.** Leadership is a TTL'd ``UPDATE`` (acquire if + ``leader`` is free or ``acquired_at`` is older than the lease window), *not* + ``pg_advisory_lock``. Session-scoped advisory locks do not survive + **transaction-pooled PgBouncer** (the queue's connection path — UN-3533), + since the pooler hands a candidate a different backend per transaction. A + plain ``UPDATE`` is one transaction → pooling-safe. All time comparisons use + the **DB clock** (``now()``), so candidate clock skew can't split leadership. + + **Free = empty ``leader``** (not NULL), following the ``PgQueueMessage.org_id`` + convention — a string field shouldn't carry two "no holder" states (S6553). + ``acquired_at`` is only meaningful while ``leader`` is non-empty. + + Single row enforced by ``CheckConstraint(id = 1)``; the row is seeded by the + migration. Managed=True / generated migration, extension-free, same posture + as the sibling models. + """ + + id = models.IntegerField(primary_key=True, default=1) + # Current leader's worker id; "" means the lease is free (no holder). + leader = models.TextField(blank=True, default="") + # When the current leader last acquired/renewed. Only read while leader != ""; + # a free row's value is immaterial. now() default for ORM .create(); the raw + # leader-election SQL always sets it explicitly. + acquired_at = models.DateTimeField(default=timezone.now) + + class Meta: + db_table = "pg_orchestrator_lock" + constraints = [ + # The whole point: at most one lock row. Any second row is a bug. + models.CheckConstraint( + check=models.Q(id=1), + name="pg_orchestrator_lock_single_row", + ), + ] + + class PgBarrierState(models.Model): """Per-execution fan-in barrier state for ``PgBarrier`` (the Postgres ``WORKER_BARRIER_BACKEND``). diff --git a/workers/queue_backend/pg_queue/__init__.py b/workers/queue_backend/pg_queue/__init__.py index 31f51e253d..7e882df8ce 100644 --- a/workers/queue_backend/pg_queue/__init__.py +++ b/workers/queue_backend/pg_queue/__init__.py @@ -31,12 +31,16 @@ from .client import PgQueueClient, QueueMessage from .connection import create_pg_connection +from .leader_election import LeaderLease, default_worker_id, lease_seconds_from_env from .task_payload import TaskPayload, to_payload __all__ = [ + "LeaderLease", "PgQueueClient", "QueueMessage", "TaskPayload", "create_pg_connection", + "default_worker_id", + "lease_seconds_from_env", "to_payload", ] diff --git a/workers/queue_backend/pg_queue/leader_election.py b/workers/queue_backend/pg_queue/leader_election.py new file mode 100644 index 0000000000..0603d60ff5 --- /dev/null +++ b/workers/queue_backend/pg_queue/leader_election.py @@ -0,0 +1,263 @@ +"""Leader-election lease for the orchestrator/reaper singleton. + +The orchestrator (admit) and reaper (stuck-task recovery, barrier-orphan sweep) +must run as **exactly one** active instance — several would contend on +``SKIP LOCKED`` and double-act on recovery. This module is the HA primitive that +makes that safe: candidate processes race to hold the single +``pg_orchestrator_lock`` row; the holder renews each cycle, and a standby takes +over once the lease goes stale. + +**Lease, not advisory lock.** Leadership is a TTL'd ``UPDATE`` (take it if the +``leader`` is free or its ``acquired_at`` is older than the lease window), *not* +``pg_advisory_lock``. Session-scoped advisory locks do not survive the +transaction-pooled PgBouncer the queue connects through (UN-3533) — the pooler +hands out a different backend per transaction, so a session-held lock would be +silently dropped. A plain ``UPDATE`` is one transaction → pooling-safe. Every +time comparison is server-side (``now()``), so candidate clock skew can't split +leadership. + +**Primitive only.** This module is the lease mechanism — acquiring and driving +the lease is the caller's responsibility (the reaper loop). ``try_acquire`` / +``renew`` / ``release`` return ``bool`` for their *expected* outcomes; an +unexpected DB error propagates (fail-loud, by design for an HA primitive). A +caller must treat a raised exception the same as a ``False`` from ``renew`` — +"leadership state unknown, stop acting" — not assume bool-only. + +Usage: + + lease = LeaderLease(default_worker_id()) + if lease.try_acquire(): + try: + while running: + do_one_cycle() + try: + still_leader = lease.renew() + except Exception: + break # leadership unknown — stop acting + if not still_leader: # lost it (we stalled past the TTL) + break # stop acting immediately + sleep(cycle) + finally: + lease.release() +""" + +from __future__ import annotations + +import contextlib +import functools +import logging +import os +import socket +import uuid +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any + +import psycopg2 + +from .connection import create_pg_connection + +if TYPE_CHECKING: + from psycopg2.extensions import connection as PgConnection + +logger = logging.getLogger(__name__) + +# Lease window: a leader that hasn't renewed within this many seconds is +# considered dead and a standby may take over. The holder must renew well +# inside this window (the reaper loop renews every cycle). Mirrors the labs +# orchestrator's 10s lease. +_DEFAULT_LEASE_SECONDS = 10 + + +def lease_seconds_from_env() -> int: + """Lease window from ``WORKER_PG_ORCHESTRATOR_LEASE_SECONDS`` (default 10s). + + Read at call time so tests can flip it. Invalid / non-positive values raise, + matching the barrier-TTL posture — a non-positive lease would let every + candidate believe leadership is always stale and act simultaneously, which is + exactly the split-brain this primitive exists to prevent. + """ + raw = os.getenv("WORKER_PG_ORCHESTRATOR_LEASE_SECONDS") + if raw is None: + return _DEFAULT_LEASE_SECONDS + try: + value = int(raw) + except ValueError as exc: + raise ValueError( + f"WORKER_PG_ORCHESTRATOR_LEASE_SECONDS={raw!r} is not an integer. " + f"Unset it to default to {_DEFAULT_LEASE_SECONDS}s." + ) from exc + if value <= 0: + raise ValueError( + f"WORKER_PG_ORCHESTRATOR_LEASE_SECONDS={value} must be a positive " + f"integer. Unset it to default to {_DEFAULT_LEASE_SECONDS}s." + ) + return value + + +@functools.cache +def default_worker_id() -> str: + """The process's candidate id (``host:pid:rand``), fixed on first call. + + ``functools.cache`` makes this idempotent: every call within a process returns the + same id, so ``renew``/``release`` match the row this process wrote even if a + caller passes ``default_worker_id()`` inline in a retry/restart loop rather + than capturing it once. The random suffix disambiguates two candidates that + share a host+pid across container restarts. Lazy (computed on first call, not + at import) so it's fixed after any fork rather than shared across children. + """ + return f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}" + + +class LeaderLease: + """A single candidate's handle on the ``pg_orchestrator_lock`` lease. + + One instance owns one Postgres connection (lazily opened, self-recovering on + a dropped socket / PgBouncer recycle — same posture as ``PgBarrier`` / + ``PgQueueClient``). Construct one per candidate process; in tests, inject a + connection to race two instances. + """ + + def __init__( + self, + worker_id: str, + *, + lease_seconds: int | None = None, + conn: PgConnection | None = None, + ) -> None: + if not worker_id or not worker_id.strip(): + # "" is the free sentinel — a candidate must never identify as "". + raise ValueError("LeaderLease worker_id must be a non-empty string") + self._worker_id = worker_id + if lease_seconds is None: + # lease_seconds_from_env already rejects <= 0. + self._lease_seconds = lease_seconds_from_env() + else: + if lease_seconds <= 0: + raise ValueError("lease_seconds must be a positive integer") + self._lease_seconds = lease_seconds + self._conn = conn + # An injected connection belongs to the caller — never close/recycle it + # (mirrors PgQueueClient). Critical: without this, a transient error + # would silently swap an injected TEST_DB_ / caller connection for a + # fresh DB_-env one, re-pointing the lease at a different database. + self._owns_conn = conn is None + + @property + def worker_id(self) -> str: + return self._worker_id + + def _get_conn(self) -> PgConnection: + # Recreate only an OWNED connection that's missing/closed. An injected + # (caller-owned) connection is never swapped — if it's dead, the next + # statement raises rather than silently re-pointing at the DB_ env. + if self._conn is None or (self._owns_conn and self._conn.closed): + self._conn = create_pg_connection(env_prefix="DB_") + return self._conn + + @contextlib.contextmanager + def _cursor(self) -> Iterator[Any]: + """Yield a cursor; commit on success, roll back + recover on error.""" + conn = self._get_conn() + try: + with conn.cursor() as cur: + yield cur + conn.commit() + except Exception as exc: + # A failed rollback proves the connection is unusable regardless of + # the psycopg2 error subclass (a server-side termination can surface + # as a bare DatabaseError). + conn_dead = isinstance( + exc, (psycopg2.OperationalError, psycopg2.InterfaceError) + ) + try: + conn.rollback() + except Exception: + conn_dead = True + # Discard an unusable connection so the next call reconnects — only + # when we own it (an injected connection is the caller's). A silent + # rebuild on the orchestrator/reaper singleton correlates with missed + # renew cycles, so log it rather than swallowing the transition. + if self._owns_conn and (conn_dead or conn.closed): + logger.warning( + "LeaderLease[%s]: connection error (%s) — discarding the " + "owned connection; the next call will reconnect.", + self._worker_id, + type(exc).__name__, + ) + with contextlib.suppress(Exception): + conn.close() + self._conn = None + raise + + def try_acquire(self) -> bool: + """Take leadership if the lease is free or stale. Returns whether we won. + + Wins iff ``leader`` is empty (free) OR ``acquired_at`` is older than the + lease window (previous holder died). The current holder re-affirming + leadership uses :meth:`renew`, not this — a fresh, held lease returns + ``False`` here by design. + """ + with self._cursor() as cur: + cur.execute( + "UPDATE pg_orchestrator_lock " + " SET leader = %s, acquired_at = now() " + " WHERE id = 1 " + " AND (leader = '' " + " OR acquired_at < now() - make_interval(secs => %s)) " + "RETURNING id", + (self._worker_id, self._lease_seconds), + ) + won = cur.fetchone() is not None + if won: + logger.info("LeaderLease: %s acquired leadership", self._worker_id) + return won + + def renew(self) -> bool: + """Extend our lease. Returns ``False`` if we are no longer the leader. + + A ``False`` is the critical safety signal: it means a standby took over + while we stalled past the lease window. The caller **must stop acting** + immediately — continuing would double-drive recovery against the new + leader. A raised DB error means leadership state is *unknown* and must be + treated the same as ``False`` (stop acting) — see the module docstring. + """ + with self._cursor() as cur: + cur.execute( + "UPDATE pg_orchestrator_lock SET acquired_at = now() " + "WHERE id = 1 AND leader = %s RETURNING id", + (self._worker_id,), + ) + still_leader = cur.fetchone() is not None + if not still_leader: + # Fires both when a held lease was taken over (stale → standby won) + # and when a non-holder calls renew — phrase it for both. + logger.warning( + "LeaderLease: %s renew failed — not the current leader " + "(taken over by another candidate, or the lease was never held)", + self._worker_id, + ) + return still_leader + + def release(self) -> None: + """Free the lease on graceful shutdown so a standby takes over at once. + + Only frees it if we still hold it (``leader = us``); if we already lost + leadership this is a no-op, so a late release can't wipe the new holder. + """ + with self._cursor() as cur: + cur.execute( + "UPDATE pg_orchestrator_lock SET leader = '', acquired_at = now() " + "WHERE id = 1 AND leader = %s", + (self._worker_id,), + ) + freed = cur.rowcount == 1 + if freed: + logger.info("LeaderLease: %s released leadership", self._worker_id) + else: + # We already lost the lease to a standby — don't misreport that this + # process freed it (misleading in the split-brain post-mortem this + # primitive exists to support). + logger.debug( + "LeaderLease: %s release was a no-op (not the current holder)", + self._worker_id, + ) diff --git a/workers/tests/test_leader_election.py b/workers/tests/test_leader_election.py new file mode 100644 index 0000000000..24c0217905 --- /dev/null +++ b/workers/tests/test_leader_election.py @@ -0,0 +1,321 @@ +"""Tests for :class:`queue_backend.pg_queue.leader_election.LeaderLease`. + +Two layers: + +1. **Env / construction** — no DB (``lease_seconds_from_env``, worker-id guard). +2. **Lease semantics** — real Postgres. Each ``LeaderLease`` is given its own + autocommit connection so two instances genuinely race the single + ``pg_orchestrator_lock`` row. Skips if Postgres is unreachable or the + ``pg_orchestrator_lock`` migration is unapplied. + +The two load-bearing correctness properties: +- concurrent ``try_acquire`` on a free lock → **exactly one** winner; +- ``renew`` returns ``False`` after a standby took over a stale lease (the + signal that tells a stalled leader to stop acting). +""" + +from __future__ import annotations + +import contextlib +import os +import threading +from types import SimpleNamespace +from unittest.mock import MagicMock + +import psycopg2 +import pytest +from queue_backend.pg_queue.connection import create_pg_connection +from queue_backend.pg_queue.leader_election import ( + LeaderLease, + default_worker_id, + lease_seconds_from_env, +) + +# --- Layer 1: env + construction (no DB) --- + + +class TestLeaseSecondsEnv: + def test_default_is_ten_seconds(self, monkeypatch): + monkeypatch.delenv("WORKER_PG_ORCHESTRATOR_LEASE_SECONDS", raising=False) + assert lease_seconds_from_env() == 10 + + def test_overridable(self, monkeypatch): + monkeypatch.setenv("WORKER_PG_ORCHESTRATOR_LEASE_SECONDS", "30") + assert lease_seconds_from_env() == 30 + + @pytest.mark.parametrize("bad", ["0", "-5", "abc", "1.5"]) + def test_invalid_raises(self, monkeypatch, bad): + monkeypatch.setenv("WORKER_PG_ORCHESTRATOR_LEASE_SECONDS", bad) + with pytest.raises(ValueError): + lease_seconds_from_env() + + +class TestConstruction: + @pytest.mark.parametrize("bad", ["", " "]) + def test_empty_worker_id_rejected(self, bad): + with pytest.raises(ValueError, match="non-empty"): + LeaderLease(bad, conn=object()) # conn unused — guard fires first + + def test_non_positive_lease_seconds_rejected(self): + with pytest.raises(ValueError): + LeaderLease("w1", lease_seconds=0, conn=object()) + + def test_default_worker_id_shape_and_idempotent(self): + default_worker_id.cache_clear() + wid = default_worker_id() + # host:pid:rand shape, three colon-separated parts. + assert len(wid.split(":")) == 3 + # Idempotent within a process (lru_cache) — repeated calls match, so a + # caller can pass it inline without drifting the worker id. + assert default_worker_id() == wid + default_worker_id.cache_clear() + + +# --- Layer 1b: connection recovery / ownership (mocked, no DB) --- + + +def _mock_conn(*, execute_exc=None, rollback_exc=None, fetch=(1,)): + cur = MagicMock() + if execute_exc is not None: + cur.execute.side_effect = execute_exc + cur.fetchone.return_value = fetch + cur.rowcount = 1 + conn = MagicMock() + conn.closed = 0 + conn.cursor.return_value.__enter__.return_value = cur + conn.cursor.return_value.__exit__.return_value = False + if rollback_exc is not None: + conn.rollback.side_effect = rollback_exc + return conn + + +_FACTORY_PATH = "queue_backend.pg_queue.leader_election.create_pg_connection" + + +class TestConnectionRecovery: + """Owned vs injected recovery branches of ``_cursor`` — the load-bearing + self-recovery the docstrings advertise (mirrors PgQueueClient's suite).""" + + def test_owned_conn_recovered_on_operational_error(self, monkeypatch): + bad = _mock_conn(execute_exc=psycopg2.OperationalError("dead")) + good = _mock_conn(fetch=(1,)) + factory = MagicMock(side_effect=[bad, good]) + monkeypatch.setattr(_FACTORY_PATH, factory) + lease = LeaderLease("w") # owns its connection (conn=None) + with pytest.raises(psycopg2.OperationalError): + lease.try_acquire() + bad.rollback.assert_called_once() + bad.close.assert_called_once() + assert lease._conn is None # dead owned handle dropped + assert lease.try_acquire() is True # next call reconnects to good + assert factory.call_count == 2 + + def test_owned_conn_recovered_when_rollback_fails(self, monkeypatch): + # Non-Operational error whose rollback also fails → still treated dead. + bad = _mock_conn( + execute_exc=RuntimeError("boom"), + rollback_exc=psycopg2.DatabaseError("socket gone"), + ) + monkeypatch.setattr(_FACTORY_PATH, MagicMock(side_effect=[bad])) + lease = LeaderLease("w") + with pytest.raises(RuntimeError): + lease.try_acquire() + bad.close.assert_called_once() + assert lease._conn is None + + def test_injected_conn_never_swapped_on_error(self): + # The High finding: an injected (caller-owned) connection must NOT be + # closed + silently replaced with a DB_-env one on a transient error. + bad = _mock_conn(execute_exc=psycopg2.OperationalError("dead")) + lease = LeaderLease("w", conn=bad) + with pytest.raises(psycopg2.OperationalError): + lease.try_acquire() + bad.rollback.assert_called_once() + bad.close.assert_not_called() + assert lease._conn is bad # caller's connection untouched + + +# --- Layer 2: lease semantics (real Postgres) --- + + +def _new_conn(): + os.environ.setdefault("TEST_DB_HOST", "127.0.0.1") + conn = create_pg_connection(env_prefix="TEST_DB_") + conn.autocommit = True + return conn + + +@pytest.fixture +def lock_db(): + """Reset the single lock row to free before/after each test. + + Yields a helper connection (for assertions / ageing the lease); each + ``LeaderLease`` under test gets its OWN connection so concurrency is real. + Skips when Postgres is unreachable or the migration is unapplied. + """ + try: + conn = _new_conn() + except psycopg2.OperationalError as exc: + pytest.skip(f"Postgres not reachable: {exc}") + with conn.cursor() as cur: + cur.execute("SELECT to_regclass('pg_orchestrator_lock')") + if cur.fetchone()[0] is None: + conn.close() + pytest.skip( + "pg_orchestrator_lock migration not applied (run backend migrate)" + ) + # Ensure the singleton row exists and is free (the migration seeds it, + # but reset defensively so tests are order-independent). + cur.execute( + "INSERT INTO pg_orchestrator_lock (id, leader, acquired_at) " + "VALUES (1, '', now()) " + "ON CONFLICT (id) DO UPDATE SET leader = '', acquired_at = now()" + ) + extra_conns: list = [] + + def make_lease(worker_id, **kw): + c = _new_conn() + extra_conns.append(c) + return LeaderLease(worker_id, conn=c, **kw) + + yield SimpleNamespace(conn=conn, make_lease=make_lease) + with conn.cursor() as cur: + cur.execute( + "UPDATE pg_orchestrator_lock SET leader = '', acquired_at = now() " + "WHERE id = 1" + ) + for c in extra_conns: + with contextlib.suppress(Exception): + c.close() + conn.close() + + +def _leader(conn) -> str: + with conn.cursor() as cur: + cur.execute("SELECT leader FROM pg_orchestrator_lock WHERE id = 1") + return cur.fetchone()[0] + + +def _age_lease(conn, seconds: int) -> None: + """Push acquired_at into the past so the lease looks stale.""" + with conn.cursor() as cur: + cur.execute( + "UPDATE pg_orchestrator_lock " + "SET acquired_at = now() - make_interval(secs => %s) WHERE id = 1", + (seconds,), + ) + + +class TestLeaderLease: + def test_acquire_on_free_lock_succeeds(self, lock_db): + lease = lock_db.make_lease("w1") + assert lease.try_acquire() is True + assert _leader(lock_db.conn) == "w1" + + def test_second_acquirer_fails_while_fresh(self, lock_db): + a = lock_db.make_lease("a") + b = lock_db.make_lease("b") + assert a.try_acquire() is True + assert b.try_acquire() is False + assert _leader(lock_db.conn) == "a" + + def test_stale_lease_allows_takeover(self, lock_db): + a = lock_db.make_lease("a", lease_seconds=10) + b = lock_db.make_lease("b", lease_seconds=10) + assert a.try_acquire() is True + _age_lease(lock_db.conn, 30) # older than the 10s window + assert b.try_acquire() is True + assert _leader(lock_db.conn) == "b" + + def test_same_holder_reacquire_while_fresh_returns_false(self, lock_db): + # The current holder can't re-acquire a fresh lease (the WHERE rejects + # even the holder until it goes stale) — renew is the holder's path. + a = lock_db.make_lease("a") + assert a.try_acquire() is True + assert a.try_acquire() is False + assert a.renew() is True # ...but renew keeps it + assert _leader(lock_db.conn) == "a" + + def test_renew_keeps_leadership(self, lock_db): + a = lock_db.make_lease("a") + b = lock_db.make_lease("b") + assert a.try_acquire() is True + assert a.renew() is True + assert b.try_acquire() is False # still fresh after renew + + def test_renew_by_non_holder_returns_false(self, lock_db): + a = lock_db.make_lease("a") + b = lock_db.make_lease("b") + assert a.try_acquire() is True + assert b.renew() is False + assert _leader(lock_db.conn) == "a" # unchanged + + def test_renew_after_takeover_returns_false(self, lock_db): + # The critical safety signal: a stalled leader learns it lost the lease. + a = lock_db.make_lease("a", lease_seconds=10) + b = lock_db.make_lease("b", lease_seconds=10) + assert a.try_acquire() is True + _age_lease(lock_db.conn, 30) + assert b.try_acquire() is True # standby takes over + assert a.renew() is False # original must stop acting + + def test_release_frees_immediately(self, lock_db): + a = lock_db.make_lease("a") + b = lock_db.make_lease("b") + assert a.try_acquire() is True + a.release() + assert _leader(lock_db.conn) == "" + assert b.try_acquire() is True # no need to wait out the TTL + + def test_release_by_non_holder_is_noop(self, lock_db): + a = lock_db.make_lease("a") + b = lock_db.make_lease("b") + assert a.try_acquire() is True + b.release() # b doesn't hold it + assert _leader(lock_db.conn) == "a" # a still leader + assert a.renew() is True + + def test_release_after_takeover_is_noop(self, lock_db): + # A stalled leader's late release must not wipe the standby that took + # over ("a late release can't wipe the new holder"). + a = lock_db.make_lease("a", lease_seconds=10) + b = lock_db.make_lease("b", lease_seconds=10) + assert a.try_acquire() is True + _age_lease(lock_db.conn, 30) + assert b.try_acquire() is True # standby took over + a.release() # late release from the deposed leader + assert _leader(lock_db.conn) == "b" # new holder intact + + def test_concurrent_acquire_exactly_one_wins(self, lock_db): + # N candidates, each its own connection, race try_acquire on a free lock. + # The row lock serialises the conditional UPDATE → exactly one RETURNING. + results: dict[str, bool] = {} + + def run(label): + lease = lock_db.make_lease(label) + results[label] = lease.try_acquire() + + labels = [f"c{i}" for i in range(5)] + threads = [threading.Thread(target=run, args=(x,)) for x in labels] + for t in threads: + t.start() + for t in threads: + t.join(timeout=10) + assert all(not t.is_alive() for t in threads) + assert sum(results.values()) == 1 # exactly one winner + winner = next(label for label, won in results.items() if won) + assert _leader(lock_db.conn) == winner + + +class TestSingleRowConstraint: + def test_second_row_rejected(self, lock_db): + with pytest.raises(psycopg2.errors.CheckViolation): + with lock_db.conn.cursor() as cur: + cur.execute( + "INSERT INTO pg_orchestrator_lock (id, leader, acquired_at) " + "VALUES (2, '', now())" + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 649e06bd431dbfb7a542ce552b440bd9fe678cd6 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Tue, 16 Jun 2026 11:10:08 +0530 Subject: [PATCH 11/44] =?UTF-8?q?UN-3554=20[FEAT]=20PG=20Queue=209d=20slic?= =?UTF-8?q?e=202=20=E2=80=94=20reaper=20process=20+=20barrier-orphan=20swe?= =?UTF-8?q?ep=20(#2058)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3554 [FEAT] PG Queue 9d slice 2 — reaper process + barrier-orphan sweep Builds on the leader-election lease (UN-3553): stands up the reaper process — the leader-elected recovery loop — with its first recovery job, the barrier-orphan sweep. Ships dark (launched explicitly, never in the default worker set). - reaper.py: - sweep_expired_barriers(conn): DELETE pg_barrier_state WHERE expires_at < now() RETURNING + loud per-orphan WARNING. The documented PgBarrier backstop — reclaims barriers whose header tasks never all completed; a late in-flight decrement then finds no row and abandons (existing semantics). Execution terminal-status recovery is 9e's job. - PgReaper: leader-elected loop. Each cycle renews (steps down to standby if renew() returns False), else tries to acquire; sweeps ONLY while leader. run() loops with graceful SIGTERM/SIGINT shutdown + lease release on exit. Guard: cycle interval must be shorter than the lease window, or the leader thrashes leadership between renews. - reaper_interval_from_env() (WORKER_PG_REAPER_INTERVAL_SECONDS, default 5s), main(), python -m queue_backend.pg_queue.reaper entrypoint. - leader_election.py: expose lease_seconds property (reaper validates its cycle against it). - test_pg_reaper.py: 15 tests — env/construction guards (interval < lease), leadership gating (sweeps only when leader, steps down on renew-fail, releases on stop), real-PG sweep (reclaims only expired, leaves fresh). Out of scope: run-worker.sh wiring + liveness (followup, like 9c-followup); pipeline recovery (counter reconstruction, per-stage re-enqueue) deferred to 9e where there's a real PG pipeline to test against. Co-Authored-By: Claude Opus 4.8 * UN-3554 [FEAT] Address review: sweep rollback, conn recovery, tick contract Toolkit + SonarCloud review on #2058: - [CRITICAL] sweep_expired_barriers now rolls back on error before re-raising (conn is manual-commit; an un-rolled-back failure left it in an aborted-txn state, poisoning every later cycle → silent self-perpetuating stall). This also clears SonarCloud's C-reliability gate. - [HIGH] On a failed sweep PgReaper discards its OWNED connection so the next tick reconnects — covers a poisoned/dead handle that `.closed` alone misses. - [MEDIUM] renew() raising now sets _is_leader=False before propagating (honours the lease's "raise == stop acting" contract). - [MEDIUM] release() failure on shutdown is logged (with the lease-window note) instead of silently suppressed. - [MEDIUM] signal-handler ValueError is re-raised unless we're off the main thread (don't mislabel an unrelated ValueError). - [MEDIUM/type-design] tick() returns a TickOutcome(was_leader, reclaimed) NamedTuple instead of an overloaded `-1` int sentinel; added an is_leader property; lease param typed against a new LeaderLeaseLike Protocol. - [LOW] Reworded the sweep race comment, the step-down log (same-cycle re-acquire), and the run() self-recovery comment for accuracy. - Tests: +8 — run() swallows a tick error; owned-conn recreated-when-closed; injected-conn never swapped; failed-sweep discards owned conn; sweep SQL contract (no DB); sweep rolls back on error; step-down-then-reacquire; renew-raising steps down. 23 total; drive paths via is_leader, no private-flag poking. Co-Authored-By: Claude Opus 4.8 * UN-3554 [FIX] Use pytest.approx for float-equality asserts (SonarCloud S1244) The two reaper-interval asserts compared float returns with ==; the values are exactly representable so it was harmless, but pytest.approx is the correct idiom and clears the S1244 reliability bugs. Co-Authored-By: Claude Opus 4.8 * UN-3554 [FEAT] Address Greptile: manual-commit Layer-4 fixture, close owned conn - The real-Postgres fixture (barrier_conn) was autocommit, which made sweep_expired_barriers' own commit() a no-op and its rollback unreachable — so Layer 4 tested a different mode than the production reaper (create_pg_connection is manual-commit). Switched the fixture to manual-commit and added explicit commits to the seed/read/cleanup helpers, so the real-DB tests now exercise the sweep's commit (and rollback) in production mode. - run() now closes its OWNED sweep connection on shutdown (an injected one is the caller's). Harmless for the main() process but keeps PgReaper clean if ever embedded / test-driven. 23 tests pass; ruff clean. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- workers/queue_backend/pg_queue/__init__.py | 12 + .../queue_backend/pg_queue/leader_election.py | 4 + workers/queue_backend/pg_queue/reaper.py | 307 ++++++++++++++++ workers/tests/test_pg_reaper.py | 329 ++++++++++++++++++ 4 files changed, 652 insertions(+) create mode 100644 workers/queue_backend/pg_queue/reaper.py create mode 100644 workers/tests/test_pg_reaper.py diff --git a/workers/queue_backend/pg_queue/__init__.py b/workers/queue_backend/pg_queue/__init__.py index 7e882df8ce..88d163ac14 100644 --- a/workers/queue_backend/pg_queue/__init__.py +++ b/workers/queue_backend/pg_queue/__init__.py @@ -32,15 +32,27 @@ from .client import PgQueueClient, QueueMessage from .connection import create_pg_connection from .leader_election import LeaderLease, default_worker_id, lease_seconds_from_env +from .reaper import ( + LeaderLeaseLike, + PgReaper, + TickOutcome, + reaper_interval_from_env, + sweep_expired_barriers, +) from .task_payload import TaskPayload, to_payload __all__ = [ "LeaderLease", + "LeaderLeaseLike", "PgQueueClient", + "PgReaper", "QueueMessage", "TaskPayload", + "TickOutcome", "create_pg_connection", "default_worker_id", "lease_seconds_from_env", + "reaper_interval_from_env", + "sweep_expired_barriers", "to_payload", ] diff --git a/workers/queue_backend/pg_queue/leader_election.py b/workers/queue_backend/pg_queue/leader_election.py index 0603d60ff5..0a1d8415d8 100644 --- a/workers/queue_backend/pg_queue/leader_election.py +++ b/workers/queue_backend/pg_queue/leader_election.py @@ -146,6 +146,10 @@ def __init__( def worker_id(self) -> str: return self._worker_id + @property + def lease_seconds(self) -> int: + return self._lease_seconds + def _get_conn(self) -> PgConnection: # Recreate only an OWNED connection that's missing/closed. An injected # (caller-owned) connection is never swapped — if it's dead, the next diff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.py new file mode 100644 index 0000000000..64ac120965 --- /dev/null +++ b/workers/queue_backend/pg_queue/reaper.py @@ -0,0 +1,307 @@ +"""Reaper — the leader-elected recovery process for the PG queue. + +A singleton, guarded by :class:`LeaderLease` over ``pg_orchestrator_lock``: only +the elected leader runs recovery work each cycle (several reapers would contend +and double-act). This slice ships the process *harness* (lease-maintenance loop ++ graceful shutdown) plus ONE recovery job — the **barrier-orphan sweep**. + +**Barrier-orphan sweep.** Reclaims ``pg_barrier_state`` rows past their +``expires_at`` — a barrier whose header tasks never all completed (the documented +:class:`PgBarrier` backstop). It ``DELETE``s the orphaned row; by PgBarrier's +existing semantics a late in-flight decrement then finds no row and abandons (no +spurious callback). The owning execution is logged loudly. Marking that +execution *terminal* (ERROR) is recovery that needs the backend and the +pipeline's PG shape — that's 9e, not here; this slice is the storage/orphan +backstop only. + +**Deferred to 9e.** Pipeline recovery (counter reconstruction from +``WorkflowFileExecution``, per-stage re-enqueue of stuck file executions) is +defined against the coupled pipeline running on PG, which doesn't exist yet — so +it lands with 9e, against a real PG pipeline it can be tested on. + +**Lease maintenance.** Each cycle the leader renews; if ``renew()`` returns +``False`` (or raises) it lost / can't confirm the lease and steps down to +standby. A standby tries to acquire each cycle. The cycle interval MUST be +shorter than the lease window, or the leader would lose the lease between +renews — enforced in :meth:`PgReaper.__init__`. + +**Ships dark.** Launched explicitly (``python -m queue_backend.pg_queue.reaper`` +or, later, ``run-worker.sh``); never part of the default worker set. With +``WORKER_BARRIER_BACKEND`` left at ``chord`` (default) there are no +``pg_barrier_state`` rows, so the sweep is a no-op until the PG barrier is used. +""" + +from __future__ import annotations + +import contextlib +import logging +import os +import signal +import threading +import time +from typing import TYPE_CHECKING, NamedTuple, Protocol + +from .connection import create_pg_connection +from .leader_election import LeaderLease, default_worker_id + +if TYPE_CHECKING: + from psycopg2.extensions import connection as PgConnection + +logger = logging.getLogger(__name__) + +# Cadence: how often the leader renews + runs recovery. Enforced shorter than +# the lease window in PgReaper.__init__. +_DEFAULT_REAPER_INTERVAL_SECONDS = 5.0 + + +class LeaderLeaseLike(Protocol): + """Structural contract :class:`PgReaper` needs from a lease. + + The dependency is structural (the tests substitute a duck-typed fake), so the + param is typed against this Protocol — :class:`LeaderLease` satisfies it, and + a fake conforms without inheritance. Same convention as the ``Barrier`` + Protocol elsewhere in the package. + """ + + @property + def lease_seconds(self) -> int: ... + + @property + def worker_id(self) -> str: ... + + def try_acquire(self) -> bool: ... + + def renew(self) -> bool: ... + + def release(self) -> None: ... + + +class TickOutcome(NamedTuple): + """Result of one :meth:`PgReaper.tick` — keeps "was I leader" and "how much + work" on separate channels (an ``int`` sentinel like ``-1`` is truthy and + conflates the two). + """ + + was_leader: bool + reclaimed: int # 0 when standby + + +def reaper_interval_from_env() -> float: + """Cycle interval from ``WORKER_PG_REAPER_INTERVAL_SECONDS`` (default 5s). + + Read at call time. Invalid / non-positive values raise (loud-on-misconfig, + matching the lease/barrier-TTL posture). + """ + raw = os.getenv("WORKER_PG_REAPER_INTERVAL_SECONDS") + if raw is None: + return _DEFAULT_REAPER_INTERVAL_SECONDS + try: + value = float(raw) + except ValueError as exc: + raise ValueError( + f"WORKER_PG_REAPER_INTERVAL_SECONDS={raw!r} is not a number. " + f"Unset it to default to {_DEFAULT_REAPER_INTERVAL_SECONDS}s." + ) from exc + if value <= 0: + raise ValueError( + f"WORKER_PG_REAPER_INTERVAL_SECONDS={value} must be positive. " + f"Unset it to default to {_DEFAULT_REAPER_INTERVAL_SECONDS}s." + ) + return value + + +def sweep_expired_barriers(conn: PgConnection) -> list[str]: + """Reclaim ``pg_barrier_state`` rows past ``expires_at``. Returns their ids. + + A single atomic ``DELETE … RETURNING``: concurrent sweepers would each + reclaim a disjoint subset (``RETURNING`` reports only the rows *this* + statement deleted), so it stays correct even if leadership gating ever fails — + in practice only the leader calls it. Each reclaimed barrier is logged at + WARNING: an orphaned barrier means an execution's header tasks never all + completed, worth surfacing even though deleting the row is the right backstop. + + ``conn`` runs in manual-commit mode, so on any error we roll back before + re-raising — otherwise the connection is left in an aborted-transaction state + and every later statement on it fails with ``InFailedSqlTransaction``. + """ + try: + with conn.cursor() as cur: + cur.execute( + "DELETE FROM pg_barrier_state WHERE expires_at < now() " + "RETURNING execution_id" + ) + reclaimed = [row[0] for row in cur.fetchall()] + conn.commit() + except Exception: + with contextlib.suppress(Exception): + conn.rollback() + raise + for execution_id in reclaimed: + logger.warning( + "Reaper: reclaimed orphaned barrier for execution %s — header tasks " + "never all completed before expiry; barrier deleted (no callback " + "fired). Execution terminal-status recovery is 9e's job.", + execution_id, + ) + return reclaimed + + +class PgReaper: + """Leader-elected recovery loop. Only the lease holder runs recovery work.""" + + def __init__( + self, + lease: LeaderLeaseLike, + *, + interval_seconds: float | None = None, + sweep_conn: PgConnection | None = None, + ) -> None: + self._lease = lease + self._interval = ( + interval_seconds + if interval_seconds is not None + else reaper_interval_from_env() + ) + # Load-bearing even though reaper_interval_from_env validates: an + # explicitly-injected interval_seconds<=0 reaches here unvalidated. + if self._interval <= 0: + raise ValueError("interval_seconds must be positive") + if self._interval >= lease.lease_seconds: + # A leader that ticks slower than its lease window loses the lease + # between renews → it would thrash leadership every cycle. + raise ValueError( + f"reaper interval {self._interval}s must be shorter than the " + f"lease window {lease.lease_seconds}s, or the leader loses the " + f"lease between renews" + ) + self._sweep_conn = sweep_conn + self._owns_sweep_conn = sweep_conn is None + self._running = False + self._is_leader = False + + @property + def is_leader(self) -> bool: + """Whether this process currently holds leadership (last tick's view).""" + return self._is_leader + + def _get_sweep_conn(self) -> PgConnection: + # Recreate only an OWNED missing/closed connection; an injected one is the + # caller's and is never swapped (mirrors LeaderLease / PgQueueClient). + if self._sweep_conn is None or ( + self._owns_sweep_conn and self._sweep_conn.closed + ): + self._sweep_conn = create_pg_connection(env_prefix="DB_") + return self._sweep_conn + + def _discard_owned_sweep_conn(self) -> None: + # After a sweep error, drop an owned connection so the next tick + # reconnects — covers a poisoned (aborted-txn) or dead-socket handle that + # `.closed` alone wouldn't catch. + if self._owns_sweep_conn and self._sweep_conn is not None: + with contextlib.suppress(Exception): + self._sweep_conn.close() + self._sweep_conn = None + logger.warning( + "Reaper: discarded sweep connection after a failed sweep; " + "reconnecting next cycle" + ) + + def tick(self) -> TickOutcome: + """One cycle: maintain leadership, then sweep iff leader.""" + if self._is_leader: + try: + still_leader = self._lease.renew() + except Exception: + # A raised renew == "leadership unknown": stop acting (honour the + # lease's documented contract) before letting it propagate. + self._is_leader = False + raise + if not still_leader: + logger.warning( + "Reaper: lost leadership (lease taken over) — stepping down " + "to standby" + ) + self._is_leader = False + if not self._is_leader and self._lease.try_acquire(): + self._is_leader = True + logger.info("Reaper: acquired leadership") + if not self._is_leader: + return TickOutcome(was_leader=False, reclaimed=0) + try: + reclaimed = len(sweep_expired_barriers(self._get_sweep_conn())) + except Exception: + self._discard_owned_sweep_conn() + raise + return TickOutcome(was_leader=True, reclaimed=reclaimed) + + def run(self, *, install_signals: bool = True) -> None: + """Lease-maintenance + recovery loop until stopped; releases on exit.""" + self._running = True + if install_signals: + self._install_signal_handlers() + logger.info( + "Reaper started (interval=%ss, lease=%ss, worker_id=%s)", + self._interval, + self._lease.lease_seconds, + self._lease.worker_id, + ) + try: + while self._running: + try: + self.tick() + except Exception: + # A transient DB blip must not tear the loop down — the lease + # connection rolls back + discards a dead handle, and a failed + # sweep discards its owned connection, so log and keep cycling. + logger.exception("Reaper: cycle failed; continuing") + time.sleep(self._interval) + finally: + # Hand the lease over promptly so a standby takes leadership without + # waiting out the full lease window. + if self._is_leader: + try: + self._lease.release() + except Exception: + logger.warning( + "Reaper: failed to release lease on shutdown; a standby " + "will take over after the lease window (~%ss) instead of " + "immediately", + self._lease.lease_seconds, + exc_info=True, + ) + # Close our owned sweep connection (an injected one is the caller's). + # Harmless for the main() process — the OS reclaims it — but keeps the + # class clean if it's ever embedded / driven from a test. + if self._owns_sweep_conn and self._sweep_conn is not None: + with contextlib.suppress(Exception): + self._sweep_conn.close() + self._sweep_conn = None + logger.info("Reaper stopped") + + def stop(self, *_: object) -> None: + """Request a graceful stop after the current cycle.""" + self._running = False + + def _install_signal_handlers(self) -> None: + try: + signal.signal(signal.SIGTERM, self.stop) + signal.signal(signal.SIGINT, self.stop) + except ValueError: + # signal.signal raises ValueError off the main thread — assert that + # cause rather than mislabelling an unrelated ValueError. + if threading.current_thread() is threading.main_thread(): + raise + logger.warning( + "Reaper: signal handlers not installed (non-main thread) — " + "SIGTERM/SIGINT will not trigger graceful shutdown" + ) + + +def main() -> None: + logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) + lease = LeaderLease(default_worker_id()) + PgReaper(lease).run() + + +if __name__ == "__main__": + main() diff --git a/workers/tests/test_pg_reaper.py b/workers/tests/test_pg_reaper.py new file mode 100644 index 0000000000..b9fe9f5984 --- /dev/null +++ b/workers/tests/test_pg_reaper.py @@ -0,0 +1,329 @@ +"""Tests for the PG-queue reaper (:mod:`queue_backend.pg_queue.reaper`). + +Layers: + +1. **Env / construction** — no DB (``reaper_interval_from_env``, the + interval-shorter-than-lease guard). +2. **Leadership gating** — a fake lease + a patched sweep, no DB: recovery runs + only while leader; a lost lease steps the reaper down; it re-acquires a later + cycle; the lease is released on shutdown; ``run`` swallows a tick error. +3. **Connection handling** — mocked: a failed sweep rolls back + discards the + owned connection; an injected connection is never swapped; the SQL contract. +4. **Barrier-orphan sweep** — real Postgres: only rows past ``expires_at`` are + reclaimed; fresh barriers are left intact. +""" + +from __future__ import annotations + +import os +import threading +import time +from unittest.mock import MagicMock, patch + +import psycopg2 +import pytest +from queue_backend.pg_queue import reaper as reaper_mod +from queue_backend.pg_queue.connection import create_pg_connection +from queue_backend.pg_queue.reaper import ( + PgReaper, + reaper_interval_from_env, + sweep_expired_barriers, +) + +# --- Layer 1: env + construction (no DB) --- + + +class TestIntervalEnv: + def test_default_is_five_seconds(self, monkeypatch): + monkeypatch.delenv("WORKER_PG_REAPER_INTERVAL_SECONDS", raising=False) + assert reaper_interval_from_env() == pytest.approx(5.0) + + def test_overridable(self, monkeypatch): + monkeypatch.setenv("WORKER_PG_REAPER_INTERVAL_SECONDS", "2.5") + assert reaper_interval_from_env() == pytest.approx(2.5) + + @pytest.mark.parametrize("bad", ["0", "-1", "abc"]) + def test_invalid_raises(self, monkeypatch, bad): + monkeypatch.setenv("WORKER_PG_REAPER_INTERVAL_SECONDS", bad) + with pytest.raises(ValueError): + reaper_interval_from_env() + + +class _FakeLease: + """Duck-typed LeaderLease. ``acquires``/``renews`` accept a bool (constant) + or a list (one outcome popped per call, then ``False``). + """ + + def __init__(self, *, acquires=True, renews=True, lease_seconds=10): + self._acquires = acquires + self._renews = renews + self.lease_seconds = lease_seconds + self.worker_id = "fake" + self.released = False + self.acquire_calls = 0 + self.renew_calls = 0 + + @staticmethod + def _next(val): + if isinstance(val, list): + return val.pop(0) if val else False + return val + + def try_acquire(self): + self.acquire_calls += 1 + return self._next(self._acquires) + + def renew(self): + self.renew_calls += 1 + return self._next(self._renews) + + def release(self): + self.released = True + + +class TestConstruction: + def test_interval_must_be_shorter_than_lease(self): + with pytest.raises(ValueError, match="shorter than the lease"): + PgReaper(_FakeLease(lease_seconds=5), interval_seconds=5, sweep_conn=object()) + + def test_non_positive_interval_rejected(self): + with pytest.raises(ValueError): + PgReaper(_FakeLease(), interval_seconds=0, sweep_conn=object()) + + def test_valid_interval_accepted(self): + PgReaper(_FakeLease(lease_seconds=10), interval_seconds=3, sweep_conn=object()) + + +# --- Layer 2: leadership gating (fake lease + patched sweep, no DB) --- + + +class TestLeadershipGating: + def _reaper(self, lease): + return PgReaper(lease, interval_seconds=0.01, sweep_conn=object()) + + def test_sweeps_when_leader(self): + reaper = self._reaper(_FakeLease(acquires=True, renews=True)) + with patch.object( + reaper_mod, "sweep_expired_barriers", return_value=["x"] + ) as sweep: + outcome = reaper.tick() # acquires leadership → sweeps + assert outcome.was_leader is True + assert outcome.reclaimed == 1 + assert reaper.is_leader is True + sweep.assert_called_once() + + def test_standby_does_not_sweep(self): + reaper = self._reaper(_FakeLease(acquires=False)) # can't get the lease + with patch.object(reaper_mod, "sweep_expired_barriers") as sweep: + outcome = reaper.tick() + assert outcome == (False, 0) + assert reaper.is_leader is False + sweep.assert_not_called() + + def test_steps_down_when_renew_fails(self): + # tick 1 acquires; tick 2 renew fails → step down, acquire also fails → + # standby. Driven through ticks, no private-flag poking. + reaper = self._reaper(_FakeLease(acquires=[True, False], renews=[False])) + with patch.object(reaper_mod, "sweep_expired_barriers", return_value=[]): + assert reaper.tick().was_leader is True + assert reaper.tick().was_leader is False + assert reaper.is_leader is False + + def test_steps_down_then_reacquires(self): + # leader → lose the lease one cycle → re-acquire the next and resume. + reaper = self._reaper(_FakeLease(acquires=[True, False, True], renews=[False])) + with patch.object(reaper_mod, "sweep_expired_barriers", return_value=[]): + assert reaper.tick().was_leader is True # acquired + assert reaper.tick().was_leader is False # renew failed → standby + assert reaper.tick().was_leader is True # re-acquired + assert reaper.is_leader is True + + def test_renew_raising_steps_down(self): + lease = _FakeLease(acquires=True, renews=True) + reaper = self._reaper(lease) + with patch.object(reaper_mod, "sweep_expired_barriers", return_value=[]): + reaper.tick() # becomes leader + lease.renew = MagicMock(side_effect=psycopg2.OperationalError("boom")) + with pytest.raises(psycopg2.OperationalError): + reaper.tick() + assert reaper.is_leader is False # raised renew == stop acting + + def test_release_on_stop_when_leader(self): + lease = _FakeLease(acquires=True, renews=True) + reaper = self._reaper(lease) + with patch.object(reaper_mod, "sweep_expired_barriers", return_value=[]): + t = threading.Thread(target=reaper.run, kwargs={"install_signals": False}) + t.start() + time.sleep(0.05) + reaper.stop() + t.join(timeout=5) + assert not t.is_alive() + assert lease.released is True + + def test_run_swallows_tick_exception(self): + reaper = PgReaper(_FakeLease(), interval_seconds=0.01, sweep_conn=object()) + calls = {"n": 0} + + def boom(): + calls["n"] += 1 + reaper.stop() # one cycle only + raise RuntimeError("transient blip") + + with patch.object(reaper, "tick", side_effect=boom): + with patch.object(reaper_mod.logger, "exception") as logexc: + reaper.run(install_signals=False) # must not propagate + assert calls["n"] == 1 + logexc.assert_called_once() + + +# --- Layer 3: connection handling (mocked, no DB) --- + + +class TestSweepConnection: + def test_sql_contract(self): + cur = MagicMock() + cur.fetchall.return_value = [("e1",), ("e2",)] + conn = MagicMock() + conn.cursor.return_value.__enter__.return_value = cur + assert sweep_expired_barriers(conn) == ["e1", "e2"] + sql = cur.execute.call_args[0][0] + assert "DELETE FROM pg_barrier_state" in sql + assert "expires_at < now()" in sql + assert "RETURNING execution_id" in sql + conn.commit.assert_called_once() + + def test_rolls_back_on_error(self): + cur = MagicMock() + cur.execute.side_effect = psycopg2.OperationalError("dead") + conn = MagicMock() + conn.cursor.return_value.__enter__.return_value = cur + with pytest.raises(psycopg2.OperationalError): + sweep_expired_barriers(conn) + conn.rollback.assert_called_once() + conn.commit.assert_not_called() + + def test_owned_conn_recreated_when_closed(self, monkeypatch): + dead = MagicMock(closed=True) + fresh = MagicMock(closed=False) + factory = MagicMock(side_effect=[dead, fresh]) + monkeypatch.setattr(reaper_mod, "create_pg_connection", factory) + reaper = PgReaper(_FakeLease(), interval_seconds=1) # owns its conn + assert reaper._get_sweep_conn() is dead + assert reaper._get_sweep_conn() is fresh # dead.closed → recreate + assert factory.call_count == 2 + + def test_injected_conn_never_swapped(self): + injected = MagicMock(closed=True) # even closed, it's the caller's + reaper = PgReaper(_FakeLease(), interval_seconds=1, sweep_conn=injected) + assert reaper._get_sweep_conn() is injected + + def test_failed_sweep_discards_owned_conn(self, monkeypatch): + conn = MagicMock(closed=False) + monkeypatch.setattr( + reaper_mod, "create_pg_connection", MagicMock(return_value=conn) + ) + reaper = PgReaper(_FakeLease(acquires=True, renews=True), interval_seconds=1) + with patch.object( + reaper_mod, + "sweep_expired_barriers", + side_effect=psycopg2.OperationalError("x"), + ): + with pytest.raises(psycopg2.OperationalError): + reaper.tick() + conn.close.assert_called_once() + assert reaper._sweep_conn is None # next tick reconnects + + +# --- Layer 4: barrier-orphan sweep (real Postgres) --- + + +def _new_conn(): + os.environ.setdefault("TEST_DB_HOST", "127.0.0.1") + # Manual-commit — exactly as the production reaper opens it + # (create_pg_connection default). NOT autocommit: that would make + # sweep_expired_barriers' own commit() a no-op and its rollback unreachable, + # so Layer 4 would test a different mode than the real reaper runs in. + return create_pg_connection(env_prefix="TEST_DB_") + + +@pytest.fixture +def barrier_conn(): + try: + conn = _new_conn() + except psycopg2.OperationalError as exc: + pytest.skip(f"Postgres not reachable: {exc}") + with conn.cursor() as cur: + cur.execute("SELECT to_regclass('pg_barrier_state')") + if cur.fetchone()[0] is None: + conn.close() + pytest.skip("pg_barrier_state migration not applied (run backend migrate)") + cur.execute("DELETE FROM pg_barrier_state") + conn.commit() + yield conn + with conn.cursor() as cur: + cur.execute("DELETE FROM pg_barrier_state") + conn.commit() + conn.close() + + +def _seed(conn, execution_id, *, expired): + # created_at must precede expires_at (CheckConstraint + # pg_barrier_expires_after_created). Commit so the seed is durable like a + # real barrier row (written by PgBarrier in another transaction) — and so the + # manual-commit sweep's own commit() is what persists the DELETE. + with conn.cursor() as cur: + if expired: + cur.execute( + "INSERT INTO pg_barrier_state " + "(execution_id, remaining, results, created_at, expires_at) " + "VALUES (%s, 1, '[]'::jsonb, now() - interval '2 hours', " + " now() - interval '1 hour')", + (execution_id,), + ) + else: + cur.execute( + "INSERT INTO pg_barrier_state " + "(execution_id, remaining, results, created_at, expires_at) " + "VALUES (%s, 1, '[]'::jsonb, now(), now() + interval '6 hours')", + (execution_id,), + ) + conn.commit() + + +def _ids(conn): + with conn.cursor() as cur: + cur.execute("SELECT execution_id FROM pg_barrier_state ORDER BY execution_id") + rows = [r[0] for r in cur.fetchall()] + conn.commit() # end the read transaction (manual-commit conn) + return rows + + +class TestSweepExpiredBarriers: + def test_reclaims_only_expired(self, barrier_conn): + _seed(barrier_conn, "exp-1", expired=True) + _seed(barrier_conn, "exp-2", expired=True) + _seed(barrier_conn, "fresh-1", expired=False) + reclaimed = sweep_expired_barriers(barrier_conn) + assert sorted(reclaimed) == ["exp-1", "exp-2"] + assert _ids(barrier_conn) == ["fresh-1"] # fresh barrier untouched + + def test_noop_when_nothing_expired(self, barrier_conn): + _seed(barrier_conn, "fresh-1", expired=False) + assert sweep_expired_barriers(barrier_conn) == [] + assert _ids(barrier_conn) == ["fresh-1"] + + def test_tick_sweeps_via_real_conn(self, barrier_conn): + _seed(barrier_conn, "exp-1", expired=True) + reaper = PgReaper( + _FakeLease(acquires=True, renews=True), + interval_seconds=1, + sweep_conn=barrier_conn, + ) + outcome = reaper.tick() # became leader and reclaimed the orphan + assert outcome.was_leader is True + assert outcome.reclaimed == 1 + assert _ids(barrier_conn) == [] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 35c735f315e3e44b72ef2e1eaad1e9ed2fec976e Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Tue, 16 Jun 2026 12:51:06 +0530 Subject: [PATCH 12/44] =?UTF-8?q?UN-3555=20[FEAT]=20PG=20Queue=209d=20slic?= =?UTF-8?q?e-2=20followup=20=E2=80=94=20run-worker.sh=20reaper=20type=20+?= =?UTF-8?q?=20pg-queue=20set=20(#2059)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3555 [FEAT] PG Queue 9d slice-2 followup — run-worker.sh reaper type + pg-queue set Wires the reaper (UN-3554) into run-worker.sh and adds a pg-queue set so the whole PG-queue group launches in one shot. Mirrors the 9c -> 9c-followup split (launcher wiring as its own slice). Liveness probe is a separate follow-on. - workers/pg_queue_reaper/: thin entrypoint package (python -m pg_queue_reaper -> queue_backend.pg_queue.reaper.main). No worker-app bootstrap (the reaper runs no Celery tasks), unlike pg_queue_consumer; exists so the process has a stable name run-worker.sh can launch + pgrep-match. - run-worker.sh: - reaper / pg-queue-reaper type — opt-in (NOT in `all`), launches `python -m pg_queue_reaper`, runs from workers root, --status/-k/-r match via the `-m` pgrep branch (now covers consumer + reaper). Lease/interval env documented in --help. - pg / pg-queue SET — run_pg_queue_set launches consumer + reaper together (always detached, like `all`). Restart (-r pg-queue) kills both members then relaunches. list_core_worker_dirs skips the set alias (no phantom status entry). Help documents the Celery `all` set and the PG `pg-queue` set as independent, runnable in parallel for a dual-transport (strangler-fig) setup. Dev-tested live: `run-worker.sh reaper -d` acquires leadership + ticks and shows RUNNING in --status; `run-worker.sh pg-queue` brings up consumer + reaper, both RUNNING, reaper leader, no phantom set entry; bash -n clean; --help renders. Co-Authored-By: Claude Opus 4.8 * UN-3555 [FEAT] Address review: set start-failure propagation, restart-kill guard Toolkit review on #2059: - [High] run_pg_queue_set swallowed member start-failures (backgrounded subshells' status was lost, banner+return 0 unconditional). Now each member runs in a FOREGROUND subshell so run_worker's own `return 1` on a crash-on-start is captured; the set returns non-zero if any member fails. (The reviewer's kill -0 on `$!` would false-fail — that PID is the launcher subshell, which exits the instant it backgrounds the nohup'd worker; the foreground-subshell return value is the correct signal and isolates `cd`.) Documented that the set always runs detached (ignores -d). - [Medium] Dispatch now `|| exit 1` so a member start-failure reaches the script exit code — the only programmatic startup signal (reaper has no health port yet). - [Medium] Set-restart aggregates kill_one_worker failures and aborts the relaunch if a member survives SIGKILL (avoids a duplicate consumer double-polling Postgres). Mirrors kill_workers' discipline. - [Medium/minor] Startup banner prints `Queues: n/a` when empty (reaper). - [Low] Reworded pg_queue_reaper/__main__ docstring: the launcher DOES export WORKER_TYPE for every worker; the accurate claim is the reaper neither reads nor mutates it (vs the consumer overwriting it before `import worker`). - [Low] Added a smoke test that pg_queue_reaper.__main__ re-exports the real reaper main (guards the `python -m pg_queue_reaper` launch path against an ImportError regression). Dev-tested: `run-worker.sh pg-queue` returns exit 0 with both members up; 24 reaper tests pass; bash -n + ruff clean. Co-Authored-By: Claude Opus 4.8 * UN-3555 [FEAT] Address Greptile: set partial-start teardown + --logs set alias - run_pg_queue_set: on a partial start-failure (one member up, the other crashed) tear the whole set down before returning 1 — kill both members so a restart-on-failure relaunch can't spawn a second instance over the survivor (the consumer would double-poll Postgres). All-or-nothing, mirroring the restart path's discipline. - tail_logs: handle the pg/pg-queue set alias — `--logs pg-queue` now tails both member logs (pg_queue_consumer + pg_queue_reaper) instead of looking for a non-existent workers/pg-queue/pg-queue.log and printing a misleading "no log file" error. Mirrors how list_core_worker_dirs skips the set value. Dev-tested: `--logs pg-queue` tails 2 files (consumer + reaper); bash -n clean; 24 reaper tests pass. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- workers/pg_queue_reaper/__init__.py | 7 ++ workers/pg_queue_reaper/__main__.py | 18 ++++ workers/run-worker.sh | 131 ++++++++++++++++++++++++---- workers/tests/test_pg_reaper.py | 16 ++++ 4 files changed, 155 insertions(+), 17 deletions(-) create mode 100644 workers/pg_queue_reaper/__init__.py create mode 100644 workers/pg_queue_reaper/__main__.py diff --git a/workers/pg_queue_reaper/__init__.py b/workers/pg_queue_reaper/__init__.py new file mode 100644 index 0000000000..ed931d2681 --- /dev/null +++ b/workers/pg_queue_reaper/__init__.py @@ -0,0 +1,7 @@ +"""Launcher package for the PG-queue reaper process. + +A thin top-level package so the reaper has a stable ``python -m pg_queue_reaper`` +invocation that ``run-worker.sh`` can launch and pgrep-match — mirroring +``pg_queue_consumer``. The actual loop lives in +``queue_backend.pg_queue.reaper``. +""" diff --git a/workers/pg_queue_reaper/__main__.py b/workers/pg_queue_reaper/__main__.py new file mode 100644 index 0000000000..6a7dc58c55 --- /dev/null +++ b/workers/pg_queue_reaper/__main__.py @@ -0,0 +1,18 @@ +"""Entry point: run the PG-queue reaper (leader-elected recovery loop). + +Unlike the consumer, the reaper runs no Celery tasks — it only does SQL recovery +(the barrier-orphan sweep) — so it needs **no** worker-app bootstrap: it never +``import``s ``worker``. (``run-worker.sh`` still exports ``WORKER_TYPE`` for every +worker, reaper included, but — unlike the consumer, which overwrites it before +``import worker`` to select which tasks to register — the reaper neither reads +nor mutates it.) This package exists purely so the process has a stable +``python -m pg_queue_reaper`` name that ``run-worker.sh`` can launch and match, +parallel to ``pg_queue_consumer``. + +Launch via ``python -m pg_queue_reaper`` or ``./run-worker.sh reaper``. +""" + +from queue_backend.pg_queue.reaper import main + +if __name__ == "__main__": + main() diff --git a/workers/run-worker.sh b/workers/run-worker.sh index 01875712e3..46c4b941fb 100755 --- a/workers/run-worker.sh +++ b/workers/run-worker.sh @@ -27,6 +27,10 @@ readonly IDE_CALLBACK_WORKER_TYPE="ide_callback" # Canonical name of the PG-queue consumer worker (referenced in several maps # and special-cases below; a constant keeps them in sync). readonly PG_QUEUE_CONSUMER_TYPE="pg_queue_consumer" +# Canonical name of the PG-queue reaper (leader-elected recovery process). +readonly PG_QUEUE_REAPER_TYPE="pg_queue_reaper" +# Set alias that launches the whole PG-queue group (consumer + reaper) together. +readonly PG_QUEUE_SET="pg-queue" # Available workers declare -A WORKERS=( @@ -51,6 +55,13 @@ declare -A WORKERS=( ["pg-queue-consumer"]="$PG_QUEUE_CONSUMER_TYPE" ["$PG_QUEUE_CONSUMER_TYPE"]="$PG_QUEUE_CONSUMER_TYPE" ["pg-consumer"]="$PG_QUEUE_CONSUMER_TYPE" + # PG Queue reaper — leader-elected recovery loop (barrier-orphan sweep) + ["reaper"]="$PG_QUEUE_REAPER_TYPE" + ["pg-queue-reaper"]="$PG_QUEUE_REAPER_TYPE" + ["$PG_QUEUE_REAPER_TYPE"]="$PG_QUEUE_REAPER_TYPE" + # Set: launch the whole PG-queue group (consumer + reaper) in one shot + ["$PG_QUEUE_SET"]="$PG_QUEUE_SET" + ["pg"]="$PG_QUEUE_SET" ["all"]="all" ) @@ -98,6 +109,7 @@ declare -A WORKER_HEALTH_PORTS=( # failure (they'd otherwise show STOPPED after every `all`). declare -A OPTIN_WORKERS=( ["$PG_QUEUE_CONSUMER_TYPE"]=1 + ["$PG_QUEUE_REAPER_TYPE"]=1 ) # Function to display usage @@ -118,12 +130,18 @@ WORKER_TYPE: executor Run executor worker (extraction execution tasks) ide-callback Run IDE callback worker (Prompt Studio post-execution callbacks) pg-queue-consumer Run PG-queue poll-loop consumer (opt-in; not part of 'all') + reaper, pg-queue-reaper Run PG-queue reaper (leader-elected recovery; opt-in) + pg, pg-queue Run the PG-queue set (consumer + reaper) together all Run all workers (in separate processes, includes auto-discovered pluggable workers) Note: Pluggable workers in pluggable_worker/ directory are automatically discovered and can be run by name. +Note: 'all' is the Celery worker set; 'pg-queue' is the PG-queue set. They are independent — + run both in parallel for a dual-transport (strangler-fig) setup. Note: pg-queue-consumer overrides: WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE (source worker whose tasks to load, default notification), WORKER_PG_QUEUE_CONSUMER_QUEUE (queue to poll), and WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT (liveness server port, default 8090). +Note: reaper overrides: WORKER_PG_ORCHESTRATOR_LEASE_SECONDS (lease window, default 10), + WORKER_PG_REAPER_INTERVAL_SECONDS (cycle interval, default 5). OPTIONS: -e, --env-file FILE Use specific environment file (default: .env) @@ -331,11 +349,11 @@ validate_env() { # --hostname=callback-worker-${id}@%h (when WORKER_INSTANCE_ID is set) get_worker_pids() { local worker_type=$1 pattern out rc - # The PG-queue consumer runs as `python -m pg_queue_consumer`, not a Celery - # `-worker@host` process, so it has no `-worker` token to anchor on. - # Match its module invocation instead (covers both the `uv run python` - # parent and the `python -m` child). Keeps --status / -k / -r working for it. - if [[ "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" ]]; then + # The PG-queue consumer and reaper run as `python -m `, not a Celery + # `-worker@host` process, so they have no `-worker` token to anchor on. + # Match the module invocation instead (covers both the `uv run python` parent + # and the `python -m` child). Keeps --status / -k / -r working for them. + if [[ "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" || "$worker_type" == "$PG_QUEUE_REAPER_TYPE" ]]; then pattern="-m[[:space:]]+${worker_type}([[:space:]]|\$)" else pattern="[^[:alnum:]_]${worker_type}-worker(@|-)" @@ -366,7 +384,9 @@ list_core_worker_dirs() { local seen="" for key in "${!WORKERS[@]}"; do local value="${WORKERS[$key]}" - if [[ "$value" == "all" ]]; then + # Skip set aliases ("all", "pg-queue") — they're groups, not real worker + # dirs/processes, so status must not list them as phantom STOPPED workers. + if [[ "$value" == "all" || "$value" == "$PG_QUEUE_SET" ]]; then continue fi if [[ "$seen" == *" $value "* ]]; then @@ -419,13 +439,22 @@ tail_logs() { print_status $BLUE "Tip: omit the worker type to tail all logs" exit 1 fi - local f - f=$(resolve_log_file "$canonical") - if [[ -z "$f" ]]; then - print_status $YELLOW "No log file found for $canonical. Did you start it with -d?" - exit 0 + if [[ "$canonical" == "$PG_QUEUE_SET" ]]; then + # The set alias maps to no single dir — tail both member logs. + for d in "$PG_QUEUE_CONSUMER_TYPE" "$PG_QUEUE_REAPER_TYPE"; do + local member_log + member_log=$(resolve_log_file "$d") + [[ -n "$member_log" ]] && log_files+=("$member_log") + done + else + local f + f=$(resolve_log_file "$canonical") + if [[ -z "$f" ]]; then + print_status $YELLOW "No log file found for $canonical. Did you start it with -d?" + exit 0 + fi + log_files+=("$f") fi - log_files+=("$f") fi if [[ ${#log_files[@]} -eq 0 ]]; then @@ -717,10 +746,18 @@ run_worker() { cmd_args=("uv" "run" "python" "-m" "$PG_QUEUE_CONSUMER_TYPE") fi + # PG queue reaper — a leader-elected SQL recovery loop (no Celery, no task + # bootstrap). Override the celery command with the plain `python -m` entry. + # Tunables (lease window, cycle interval) come from env; no liveness port is + # wired yet (that's a follow-on slice), so it binds nothing. + if [[ "$worker_type" == "$PG_QUEUE_REAPER_TYPE" ]]; then + cmd_args=("uv" "run" "python" "-m" "$PG_QUEUE_REAPER_TYPE") + fi + print_status $GREEN "Starting $worker_type worker..." print_status $BLUE "Directory: $worker_dir" print_status $BLUE "Worker Name: $worker_instance_name" - print_status $BLUE "Queues: $queues" + print_status $BLUE "Queues: ${queues:-n/a}" # Show the effective port: a -p/--health-port override wins over the map. print_status $BLUE "Health Port: ${health_port:-${WORKER_HEALTH_PORTS[$worker_type]:-n/a}}" print_status $BLUE "Command: ${cmd_args[*]}" @@ -728,9 +765,9 @@ run_worker() { # Change to appropriate directory # For pluggable workers, stay at workers root to allow module imports # For core workers, change to worker directory - if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" || "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" ]]; then - # Run from the workers root so `python -m pg_queue_consumer` (and the - # `worker` app it bootstraps) resolve. + if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" || "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" || "$worker_type" == "$PG_QUEUE_REAPER_TYPE" ]]; then + # Run from the workers root so `python -m pg_queue_consumer` / + # `python -m pg_queue_reaper` (and what they import) resolve. cd "$WORKERS_DIR" else cd "$worker_dir" @@ -826,6 +863,47 @@ run_all_workers() { fi } +# Function to run the PG-queue worker set (consumer + reaper) together. +# A set is multiple processes, so — like 'all' — it ALWAYS runs detached +# (it ignores -d/--detach; there's no foreground form). The PG-queue counterpart +# to 'all' (the Celery set): run both for a dual-transport (strangler-fig) setup. +# The reaper is a leader-elected singleton, so running it on several hosts is +# safe (only one wins the lease). Returns non-zero if any member dies on start — +# the reaper has no health port yet, so this launch check is the only +# programmatic startup signal a caller (systemd/CI) gets. +run_pg_queue_set() { + local log_level=$1 + local concurrency=$2 + local pool_type=$3 + + print_status $GREEN "Starting PG-queue set (consumer + reaper)..." + local failed=0 + for worker in "$PG_QUEUE_CONSUMER_TYPE" "$PG_QUEUE_REAPER_TYPE"; do + print_status $BLUE "Starting $worker in background..." + # A FOREGROUND subshell: run_worker (detach=true) nohup-backgrounds the + # actual worker and returns 1 on an immediate crash-on-start — so the + # subshell's exit status IS that signal. The subshell isolates run_worker's + # `cd` from this loop; the nohup'd worker survives the subshell exiting. + # (A background `( … ) &` would lose the status — its `$!` is the launcher + # subshell, which exits the instant it backgrounds the worker.) + ( run_worker "$worker" "true" "$log_level" "$concurrency" "" "" "$pool_type" "" ) \ + || failed=1 + done + if [[ $failed -ne 0 ]]; then + print_status $RED "PG-queue set: a member failed to start — tearing down the set (see logs above)" + # Don't leave a survivor: a restart-on-failure relaunch would spawn a + # second instance on top of it (the consumer would double-poll Postgres). + # Kill both members (the crashed one is already gone → no-op). Same + # all-or-nothing discipline as the restart path. + kill_one_worker "$PG_QUEUE_CONSUMER_TYPE" + kill_one_worker "$PG_QUEUE_REAPER_TYPE" + show_status + return 1 + fi + print_status $GREEN "PG-queue set started in background" + show_status +} + # Parse command line arguments DETACH=false LOG_LEVEL="" @@ -940,7 +1018,21 @@ fi # WORKER_TYPE as "all" once we set it below. if [[ "$RESTART_MODE" == "true" ]]; then discover_pluggable_workers - if [[ -n "$WORKER_TYPE" && "$WORKER_TYPE" != "all" ]]; then + if [[ "${WORKERS[$WORKER_TYPE]:-}" == "$PG_QUEUE_SET" ]]; then + # Restart the PG-queue set: kill both members, then fall through to the + # launch path (which runs run_pg_queue_set since WORKER_TYPE is the set). + # Aggregate kill failures (kill_one_worker returns 1 if a process survives + # SIGKILL) and abort rather than relaunch over a survivor — a second + # consumer would double-poll Postgres. Mirrors kill_workers' discipline. + print_status $BLUE "Restarting PG-queue set..." + restart_failed=0 + kill_one_worker "$PG_QUEUE_CONSUMER_TYPE" || restart_failed=1 + kill_one_worker "$PG_QUEUE_REAPER_TYPE" || restart_failed=1 + if [[ $restart_failed -ne 0 ]]; then + print_status $RED "Cannot restart PG-queue set: a member survived SIGKILL; aborting to avoid duplicate processes" + exit 1 + fi + elif [[ -n "$WORKER_TYPE" && "$WORKER_TYPE" != "all" ]]; then restart_target_dir="${WORKERS[$WORKER_TYPE]:-${PLUGGABLE_WORKERS[$WORKER_TYPE]:-}}" if [[ -z "$restart_target_dir" ]]; then print_status $RED "Error: Unknown worker type for restart: $WORKER_TYPE" @@ -994,6 +1086,11 @@ export PYTHONPATH="$WORKERS_DIR:${PYTHONPATH:-}" # Run the requested worker(s) if [[ "$WORKER_TYPE" == "all" ]]; then run_all_workers "$DETACH" "$LOG_LEVEL" "$CONCURRENCY" "$POOL_TYPE" +elif [[ "${WORKERS[$WORKER_TYPE]:-}" == "$PG_QUEUE_SET" ]]; then + # The PG-queue set (consumer + reaper). Always backgrounded (multiple procs). + # Propagate a member start-failure to the script exit code — the reaper has + # no health port, so this is the only programmatic startup signal. + run_pg_queue_set "$LOG_LEVEL" "$CONCURRENCY" "$POOL_TYPE" || exit 1 else # Resolve worker directory name from either WORKERS or PLUGGABLE_WORKERS WORKER_DIR_NAME="${WORKERS[$WORKER_TYPE]}" diff --git a/workers/tests/test_pg_reaper.py b/workers/tests/test_pg_reaper.py index b9fe9f5984..91173dd722 100644 --- a/workers/tests/test_pg_reaper.py +++ b/workers/tests/test_pg_reaper.py @@ -325,5 +325,21 @@ def test_tick_sweeps_via_real_conn(self, barrier_conn): assert _ids(barrier_conn) == [] +# --- Entry point (the `python -m pg_queue_reaper` launch path) --- + + +class TestEntryPoint: + def test_main_module_reexports_real_main(self): + # The single point where `run-worker.sh reaper` / `python -m + # pg_queue_reaper` either works or dies on ImportError. Pin that the + # launcher module re-exports the real reaper main. + import importlib + + from queue_backend.pg_queue.reaper import main as real_main + + module = importlib.import_module("pg_queue_reaper.__main__") + assert module.main is real_main + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From f31238828b05bfe4b4629b78521f4a12f4a42fa3 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Tue, 16 Jun 2026 17:50:15 +0530 Subject: [PATCH 13/44] =?UTF-8?q?UN-3556=20[FEAT]=20PG=20Queue=209d=20?= =?UTF-8?q?=E2=80=94=20reaper=20liveness=20probe=20(heartbeat=20+=20is=5Fl?= =?UTF-8?q?eader)=20(#2061)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3556 [FEAT] PG Queue 9d — reaper liveness probe (heartbeat + is_leader) Closes the gap flagged in #2059's review: a reaper that crashes after startup was invisible (opt-in skip in --status + no health port). Mirrors the consumer's liveness (UN-3544). - PgReaper heartbeat: _last_tick_monotonic stamped at the START of every tick (a standby tick counts as progress — liveness tracks the loop, not leadership) + seconds_since_last_tick() / is_tick_stale(). - ReaperLivenessServer: lean HTTP probe (mirrors the consumer's LivenessServer) — /health (also /healthz, /livez) returns 200 while the tick loop is fresh, 503 when stale. Payload also surfaces is_leader (which pod holds the lease — useful for 9e debugging). The 200/503 verdict is PURELY the heartbeat, never leadership (a standby is healthy) or DB reachability (a blip must not crash-loop a fine process). - main() wires it from WORKER_PG_REAPER_HEALTH_PORT (unset → no server, no stray port); staleness window from WORKER_PG_REAPER_HEALTH_STALE_SECONDS (default 30s, comfortably above the 5s tick interval). Bind failure degrades gracefully (logs, runs probe-less). - run-worker.sh: reserve port 8086 for the reaper, export the health-port env in the reaper special-case, document the two new env vars in --help. - Tests (+13, 37 total): heartbeat fresh/stale + tick refresh; liveness server 200-fresh / 503-stale / is_leader reflected / 404 / double-start; health staleness env default+override+invalid; server-disabled-when-no-port. Dev-tested live: `run-worker.sh reaper -d` → GET :8086/health → {"status":"healthy","check":"pg_reaper_tick","is_leader":true,...}. Co-Authored-By: Claude Opus 4.8 * UN-3556 [REFACTOR] Extract shared LivenessServer (SonarCloud duplication) SonarCloud flagged the new ReaperLivenessServer as duplicating the consumer's LivenessServer (~19 lines of HTTP-probe boilerplate, over the 3% new-code gate). Extracted one generic LivenessServer into queue_backend/pg_queue/liveness.py — parameterised by a freshness callable + the payload's check/age labels + an optional extra-status callable (the reaper's is_leader). Both sides are now thin subclasses that preserve their exact constructor signatures and wire payloads: - consumer LivenessServer(consumer, port=, stale_after=) → check="pg_queue_poll", seconds_since_last_poll (unchanged on the wire; its tests pass untouched). - ReaperLivenessServer(reaper, port=, stale_after=) → check="pg_reaper_tick", seconds_since_last_tick, is_leader. The boilerplate now lives once → duplication cleared, consumer behaviour preserved. reaper 37 tests + consumer liveness/health tests green; ruff clean. Co-Authored-By: Claude Opus 4.8 * UN-3556 [FEAT] Address review: validate health port, type/guard liveness, tests Toolkit review (10 findings; several already resolved by the dedup refactor 0d2b3f721 — the threading-alias, the query-strip comment, and the extract-shared-server follow-up itself): - [Medium] Port parse: extracted _reaper_health_port_from_env() — names the var on a bad value (no more context-free int('abc') crash) and range-checks 0-65535 at parse time, so an out-of-range value can't escape the bind catch as OverflowError inside start(). main() uses it. - [Medium] liveness.py: typed _httpd/_thread as HTTPServer|None / Thread|None via TYPE_CHECKING (was Any in the shared server) — restores the lifecycle invariant + type-checking on .shutdown()/.join()/etc. - [Low] LivenessServer.__init__ re-validates stale_after > 0 (a direct caller could otherwise build an always-503 probe). - [Low] bound_port docstring: clarified the port=0 / not-started case. - [Low] _DEFAULT_HEALTH_STALE_SECONDS comment references the interval constant, not a hard-coded "5s". - [Medium/Low test gaps] +9 tests: port-env helper (unset/empty/valid/non-int named/out-of-range); main() wiring (parsed port reaches the wiring + health stopped in finally; port=None when unset); _maybe_start_health_server OSError graceful-degrade → None + logger.exception; stale_after<=0 constructor guard. reaper 46 + consumer liveness 10 green; ruff clean. The shared-server extraction (flagged as a follow-up) was already done in 0d2b3f721. Co-Authored-By: Claude Opus 4.8 * UN-3556 [FEAT] Address Greptile: protect core payload + per-process log label - extra_status_fn could silently clobber core payload fields (status / check / age_key / stale_after_seconds) that a monitor reads. Now the handler builds extra fields first and overlays the core fields, so core ALWAYS wins — a future caller's extra dict can't corrupt the status a monitor parses. Test: an extra_status_fn returning {"status":"HACKED",...} leaves status "healthy" and check intact, while a non-reserved extra key is preserved. - The dedup refactor moved the consumer's liveness warnings to the shared liveness logger with generic text, so log-based filtering keyed on the old "PG-queue consumer: ..." would miss them. Added a log_label param (default "pg-queue"); consumer passes "pg-queue consumer", reaper "pg-queue reaper", so the messages stay attributable to the source process. 57 tests green (reaper 47 + consumer liveness 10); ruff clean. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- workers/queue_backend/pg_queue/__init__.py | 4 + workers/queue_backend/pg_queue/consumer.py | 132 ++---------- workers/queue_backend/pg_queue/liveness.py | 178 ++++++++++++++++ workers/queue_backend/pg_queue/reaper.py | 125 +++++++++++- workers/run-worker.sh | 15 +- workers/tests/test_pg_reaper.py | 227 +++++++++++++++++++++ 6 files changed, 560 insertions(+), 121 deletions(-) create mode 100644 workers/queue_backend/pg_queue/liveness.py diff --git a/workers/queue_backend/pg_queue/__init__.py b/workers/queue_backend/pg_queue/__init__.py index 88d163ac14..ec228f4617 100644 --- a/workers/queue_backend/pg_queue/__init__.py +++ b/workers/queue_backend/pg_queue/__init__.py @@ -32,9 +32,11 @@ from .client import PgQueueClient, QueueMessage from .connection import create_pg_connection from .leader_election import LeaderLease, default_worker_id, lease_seconds_from_env +from .liveness import LivenessServer from .reaper import ( LeaderLeaseLike, PgReaper, + ReaperLivenessServer, TickOutcome, reaper_interval_from_env, sweep_expired_barriers, @@ -44,9 +46,11 @@ __all__ = [ "LeaderLease", "LeaderLeaseLike", + "LivenessServer", "PgQueueClient", "PgReaper", "QueueMessage", + "ReaperLivenessServer", "TaskPayload", "TickOutcome", "create_pg_connection", diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py index 856a038a26..62771921c4 100644 --- a/workers/queue_backend/pg_queue/consumer.py +++ b/workers/queue_backend/pg_queue/consumer.py @@ -31,11 +31,9 @@ from ..fairness import FAIRNESS_HEADER_NAME from .client import PgQueueClient +from .liveness import LivenessServer as _BaseLivenessServer if TYPE_CHECKING: - from http.server import HTTPServer - from threading import Thread - from celery import Celery from .client import QueueMessage @@ -97,8 +95,7 @@ def __init__( # Otherwise min(poll_interval*2, backoff_max) shrinks the backoff # below poll_interval — it would decrease instead of grow. raise ValueError( - f"backoff_max ({backoff_max}) must be >= poll_interval " - f"({poll_interval})" + f"backoff_max ({backoff_max}) must be >= poll_interval ({poll_interval})" ) self.queue_name = queue_name self._client = client if client is not None else PgQueueClient() @@ -371,124 +368,25 @@ def _maybe_start_health_server( return server -class LivenessServer: - """Tiny HTTP liveness probe: 200 while the poll loop is fresh, else 503. - - Deliberately lean. A *liveness* probe must answer one question — "is this - process still making progress?" — and nothing else. It must NOT depend on - broker/API reachability or resource pressure: a transient backend blip or a - busy moment would otherwise make the orchestrator crash-loop an - otherwise-healthy consumer. So this intentionally does *not* reuse the - shared ``HealthChecker`` (which bundles api-connectivity / system-resource - checks meant for richer health reporting, not liveness) — it reports solely - on the poll-loop heartbeat (:meth:`PgQueueConsumer.is_poll_stale`). - - Serves ``/health`` (also ``/healthz``, ``/livez``) on ``0.0.0.0`` (all - interfaces — a container/k8s probe reaches it from outside the process) in a - daemon thread. Bind ``port=0`` to let the OS pick a free port (read back via - :attr:`bound_port`) — used in tests. Start once; :meth:`stop` returns it to - the inert state. +class LivenessServer(_BaseLivenessServer): + """Consumer poll-loop liveness — a thin wrapper over the shared + :class:`queue_backend.pg_queue.liveness.LivenessServer`, bound to the + consumer's heartbeat (``seconds_since_last_poll``). Same wire shape as before + (``/health`` → 200 fresh / 503 stale, ``check="pg_queue_poll"``). """ - _PATHS = frozenset({"/health", "/healthz", "/livez"}) - def __init__( self, consumer: PgQueueConsumer, *, port: int, stale_after: float ) -> None: - self._consumer = consumer - self._port = port - self._stale_after = stale_after - self._httpd: HTTPServer | None = None - self._thread: Thread | None = None - - def start(self) -> None: - import json - import threading - from http.server import BaseHTTPRequestHandler, HTTPServer - from urllib.parse import urlsplit - - if self._httpd is not None: - raise RuntimeError("LivenessServer already started") - - consumer = self._consumer - stale_after = self._stale_after - paths = self._PATHS - - class _Handler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - # Strip any query string — self.path includes it, so a probe - # like /health?foo=bar must still match. - if urlsplit(self.path).path not in paths: - self.send_response(404) - self.end_headers() - return - # One clock read so age and the healthy/stale verdict are - # derived from the same instant. - age = consumer.seconds_since_last_poll() - stale = age > stale_after - body = json.dumps( - { - "status": "unhealthy" if stale else "healthy", - "check": "pg_queue_poll", - "seconds_since_last_poll": round(age, 3), - "stale_after_seconds": stale_after, - } - ).encode() - self.send_response(503 if stale else 200) - self.send_header("Content-Type", "application/json") - self.end_headers() - try: - self.wfile.write(body) - except (BrokenPipeError, ConnectionResetError): - pass # client (probe) hung up mid-response — not our problem - - def log_message(self, *_: object) -> None: - pass # silence per-request access logging - - def log_error(self, fmt: str, *args: object) -> None: - # BaseHTTPRequestHandler routes errors through log_message too; - # don't let the pass above swallow them — surface to our logger. - logger.warning("pg-queue liveness handler: " + fmt, *args) - - def _serve(httpd: HTTPServer) -> None: - try: - httpd.serve_forever() - except Exception: - # A daemon thread dying silently would make /health stop - # answering (connection refused) with no breadcrumb. - logger.exception("pg-queue liveness server thread crashed") - - httpd = HTTPServer(("0.0.0.0", self._port), _Handler) - self._httpd = httpd - self._thread = threading.Thread( - target=_serve, args=(httpd,), daemon=True, name="pg-consumer-liveness" + super().__init__( + freshness_fn=consumer.seconds_since_last_poll, + stale_after=stale_after, + port=port, + check_name="pg_queue_poll", + age_key="seconds_since_last_poll", + thread_name="pg-consumer-liveness", + log_label="pg-queue consumer", ) - self._thread.start() - - @property - def bound_port(self) -> int: - """Actual listening port (resolves ``port=0``); the requested port if not started.""" - if self._httpd is not None: - return self._httpd.server_address[1] - return self._port - - def stop(self) -> None: - """Shut the server down. Defensive: never raises (called from a finally).""" - try: - if self._httpd is not None: - self._httpd.shutdown() - self._httpd.server_close() - if self._thread is not None: - self._thread.join(timeout=5) - if self._thread.is_alive(): - logger.warning( - "PG-queue consumer: liveness thread did not stop within 5s" - ) - except Exception: - logger.exception("PG-queue consumer: error stopping liveness server") - finally: - self._httpd = None - self._thread = None if __name__ == "__main__": diff --git a/workers/queue_backend/pg_queue/liveness.py b/workers/queue_backend/pg_queue/liveness.py new file mode 100644 index 0000000000..7256a0e2a4 --- /dev/null +++ b/workers/queue_backend/pg_queue/liveness.py @@ -0,0 +1,178 @@ +"""Shared tiny HTTP liveness probe for PG-queue processes. + +Both the consumer (poll loop) and the reaper (tick loop) need the same probe: +"is this loop still making progress?". This is the single implementation, +parameterised by a freshness callable + the payload's ``check``/age labels (and +an optional extra-status callable — e.g. the reaper's ``is_leader``). Each side +wraps it in a thin subclass with its own constructor shape. + +Deliberately lean. A *liveness* probe must answer one question — progress — and +nothing else. It must NOT depend on broker/DB reachability or resource pressure: +a transient backend blip or a busy moment would otherwise make the orchestrator +crash-loop an otherwise-healthy process. So this does not reuse the shared +``HealthChecker`` (which bundles api-connectivity / system-resource checks meant +for richer health reporting, not liveness). + +Serves ``/health`` (also ``/healthz``, ``/livez``) on ``0.0.0.0`` (a +container/k8s probe reaches it from outside the process) in a daemon thread. +Bind ``port=0`` to let the OS pick a free port (read back via :attr:`bound_port`) +— used in tests. Start once; :meth:`stop` returns it to the inert state. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from http.server import HTTPServer + from threading import Thread + +logger = logging.getLogger(__name__) + + +class LivenessServer: + """Generic liveness probe: 200 while ``freshness_fn()`` is within + ``stale_after`` seconds, else 503. + + ``check_name`` / ``age_key`` label the JSON payload; ``extra_status_fn`` + (optional) merges extra fields in (informational — it never affects the + 200/503 verdict, which is purely the freshness heartbeat). + """ + + _PATHS = frozenset({"/health", "/healthz", "/livez"}) + + def __init__( + self, + *, + freshness_fn: Callable[[], float], + stale_after: float, + port: int, + check_name: str, + age_key: str, + extra_status_fn: Callable[[], dict[str, Any]] | None = None, + thread_name: str = "pg-queue-liveness", + log_label: str = "pg-queue", + ) -> None: + # Re-validate here (not only at the env boundary): a direct caller could + # otherwise build an always-503 probe that crash-loops the pod. Mirrors + # the codebase's load-bearing re-validation convention (PgReaper.__init__). + if stale_after <= 0: + raise ValueError(f"stale_after must be positive, got {stale_after!r}") + self._freshness_fn = freshness_fn + self._stale_after = stale_after + self._port = port + self._check_name = check_name + self._age_key = age_key + self._extra_status_fn = extra_status_fn + self._thread_name = thread_name + # Prefixes the (now-shared) log messages so they stay attributable to the + # source process after the consumer/reaper extraction (e.g. "pg-queue + # consumer" / "pg-queue reaper") — they all log via this module's logger. + self._log_label = log_label + self._httpd: HTTPServer | None = None + self._thread: Thread | None = None + + def start(self) -> None: + import json + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + from urllib.parse import urlsplit + + if self._httpd is not None: + raise RuntimeError("LivenessServer already started") + + freshness_fn = self._freshness_fn + stale_after = self._stale_after + paths = self._PATHS + check_name = self._check_name + age_key = self._age_key + extra_status_fn = self._extra_status_fn + log_label = self._log_label + + class _Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + # Strip any query string — a probe like /health?foo=bar must match. + if urlsplit(self.path).path not in paths: + self.send_response(404) + self.end_headers() + return + # One clock read so age and the healthy/stale verdict share an + # instant. The verdict is purely freshness — extra_status_fn + # fields are informational and never flip it. + age = freshness_fn() + stale = age > stale_after + # Extra fields first, then overlay the core fields — so a caller's + # extra_status_fn can NEVER clobber status/check/age_key/ + # stale_after_seconds (which a monitor reads): core always wins. + payload: dict[str, Any] = {} + if extra_status_fn is not None: + payload.update(extra_status_fn()) + payload.update( + { + "status": "unhealthy" if stale else "healthy", + "check": check_name, + age_key: round(age, 3), + "stale_after_seconds": stale_after, + } + ) + body = json.dumps(payload).encode() + self.send_response(503 if stale else 200) + self.send_header("Content-Type", "application/json") + self.end_headers() + try: + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + pass # client (probe) hung up mid-response — not our problem + + def log_message(self, *_: object) -> None: + pass # silence per-request access logging + + def log_error(self, fmt: str, *args: object) -> None: + # BaseHTTPRequestHandler routes errors through log_message too; + # don't let the pass above swallow them — surface to our logger. + logger.warning(f"{log_label} liveness handler: " + fmt, *args) + + def _serve(httpd: HTTPServer) -> None: + try: + httpd.serve_forever() + except Exception: + # A daemon thread dying silently would make /health stop + # answering (connection refused) with no breadcrumb. + logger.exception("%s liveness server thread crashed", log_label) + + httpd = HTTPServer(("0.0.0.0", self._port), _Handler) + self._httpd = httpd + self._thread = threading.Thread( + target=_serve, args=(httpd,), daemon=True, name=self._thread_name + ) + self._thread.start() + + @property + def bound_port(self) -> int: + """Actual listening port once started (resolves ``port=0`` to the + OS-chosen port). Before :meth:`start` / after :meth:`stop`, returns the + configured port value (which is ``0`` when the OS is asked to choose). + """ + if self._httpd is not None: + return self._httpd.server_address[1] + return self._port + + def stop(self) -> None: + """Shut the server down. Defensive: never raises (called from a finally).""" + try: + if self._httpd is not None: + self._httpd.shutdown() + self._httpd.server_close() + if self._thread is not None: + self._thread.join(timeout=5) + if self._thread.is_alive(): + logger.warning( + "%s liveness thread did not stop within 5s", self._log_label + ) + except Exception: + logger.exception("%s: error stopping liveness server", self._log_label) + finally: + self._httpd = None + self._thread = None diff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.py index 64ac120965..ae05d8812b 100644 --- a/workers/queue_backend/pg_queue/reaper.py +++ b/workers/queue_backend/pg_queue/reaper.py @@ -43,6 +43,7 @@ from .connection import create_pg_connection from .leader_election import LeaderLease, default_worker_id +from .liveness import LivenessServer as _BaseLivenessServer if TYPE_CHECKING: from psycopg2.extensions import connection as PgConnection @@ -178,12 +179,24 @@ def __init__( self._owns_sweep_conn = sweep_conn is None self._running = False self._is_leader = False + # Liveness heartbeat: monotonic timestamp of the last tick start. A + # standby tick counts as progress too (the loop is alive), so this tracks + # loop liveness, not leadership. + self._last_tick_monotonic = time.monotonic() @property def is_leader(self) -> bool: """Whether this process currently holds leadership (last tick's view).""" return self._is_leader + def seconds_since_last_tick(self) -> float: + """Seconds since the last tick started — the liveness heartbeat age.""" + return time.monotonic() - self._last_tick_monotonic + + def is_tick_stale(self, stale_after_seconds: float) -> bool: + """Whether the loop has gone quiet past ``stale_after_seconds``.""" + return self.seconds_since_last_tick() > stale_after_seconds + def _get_sweep_conn(self) -> PgConnection: # Recreate only an OWNED missing/closed connection; an injected one is the # caller's and is never swapped (mirrors LeaderLease / PgQueueClient). @@ -208,6 +221,9 @@ def _discard_owned_sweep_conn(self) -> None: def tick(self) -> TickOutcome: """One cycle: maintain leadership, then sweep iff leader.""" + # Heartbeat at the START of the cycle: a tick that begins but then errors + # still proves the loop is running (the error path is caught by run()). + self._last_tick_monotonic = time.monotonic() if self._is_leader: try: still_leader = self._lease.renew() @@ -297,10 +313,117 @@ def _install_signal_handlers(self) -> None: ) +# Default staleness window for the liveness probe. Comfortably above the default +# tick interval (_DEFAULT_REAPER_INTERVAL_SECONDS) so a single slow cycle (a long +# sweep / DB blip) doesn't flap the probe; the durable rationale is the headroom +# ratio — an operator tightening the interval should tighten this too. +_DEFAULT_HEALTH_STALE_SECONDS = 30.0 + + +class ReaperLivenessServer(_BaseLivenessServer): + """Reaper tick-loop liveness — a thin wrapper over the shared + :class:`queue_backend.pg_queue.liveness.LivenessServer`, bound to the reaper's + heartbeat (``seconds_since_last_tick``) and surfacing ``is_leader`` (which pod + holds the lease — informational; the 200/503 verdict is purely the heartbeat, + so a standby is healthy). + """ + + def __init__(self, reaper: PgReaper, *, port: int, stale_after: float) -> None: + super().__init__( + freshness_fn=reaper.seconds_since_last_tick, + stale_after=stale_after, + port=port, + check_name="pg_reaper_tick", + age_key="seconds_since_last_tick", + extra_status_fn=lambda: {"is_leader": reaper.is_leader}, + thread_name="pg-reaper-liveness", + log_label="pg-queue reaper", + ) + + +def _reaper_health_stale_from_env() -> float: + raw = os.getenv("WORKER_PG_REAPER_HEALTH_STALE_SECONDS") + if raw is None or raw == "": + return _DEFAULT_HEALTH_STALE_SECONDS + try: + value = float(raw) + except ValueError as exc: + raise ValueError( + f"WORKER_PG_REAPER_HEALTH_STALE_SECONDS={raw!r} is not a number." + ) from exc + if value <= 0: + raise ValueError( + f"WORKER_PG_REAPER_HEALTH_STALE_SECONDS={value} must be positive." + ) + return value + + +def _reaper_health_port_from_env() -> int | None: + """Liveness port from ``WORKER_PG_REAPER_HEALTH_PORT`` (unset/empty → None, + i.e. no server). Validates + names the var here so a garbled value (a common + run-worker.sh shell-fallback mistake) fails with a clear message rather than a + context-free ``int('abc')`` crash — and so an out-of-range value is rejected + at parse time rather than escaping the bind catch as ``OverflowError`` inside + ``start()``. + """ + raw = os.getenv("WORKER_PG_REAPER_HEALTH_PORT") + if raw is None or raw == "": + return None + try: + port = int(raw) + except ValueError as exc: + raise ValueError(f"Invalid WORKER_PG_REAPER_HEALTH_PORT={raw!r}: {exc}") from exc + if not (0 <= port <= 65535): + raise ValueError(f"WORKER_PG_REAPER_HEALTH_PORT={port} out of range 0-65535") + return port + + +def _maybe_start_health_server( + reaper: PgReaper, *, port: int | None, stale_after: float +) -> ReaperLivenessServer | None: + """Start the liveness server when ``port`` is not None; else ``None``. + + A bind failure degrades gracefully — the probe is auxiliary and must never + stop the reaper from running; we log and continue probe-less. + """ + if port is None: + logger.info( + "PG-queue reaper: WORKER_PG_REAPER_HEALTH_PORT unset — liveness " + "server disabled" + ) + return None + server = ReaperLivenessServer(reaper, port=port, stale_after=stale_after) + try: + server.start() + except OSError: + logger.exception( + "PG-queue reaper: liveness server could not bind :%s — continuing " + "WITHOUT a probe", + port, + ) + return None + logger.info( + "PG-queue reaper: liveness server on :%s/health (stale after %ss)", + server.bound_port, + stale_after, + ) + return server + + def main() -> None: logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) lease = LeaderLease(default_worker_id()) - PgReaper(lease).run() + reaper = PgReaper(lease) + health = _maybe_start_health_server( + reaper, + port=_reaper_health_port_from_env(), + stale_after=_reaper_health_stale_from_env(), + ) + try: + reaper.run() + finally: + if health is not None: + health.stop() if __name__ == "__main__": diff --git a/workers/run-worker.sh b/workers/run-worker.sh index 46c4b941fb..94ba46732f 100755 --- a/workers/run-worker.sh +++ b/workers/run-worker.sh @@ -101,6 +101,11 @@ declare -A WORKER_HEALTH_PORTS=( # only when WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT is exported (below); a bare # `python -m pg_queue_consumer` binds nothing. ["$PG_QUEUE_CONSUMER_TYPE"]="8090" + # pg_queue_reaper: 8086 — the one free slot in the 8080-8090 band (8086 sits + # between callback's 8085 and scheduler's 8087). Bound only when + # WORKER_PG_REAPER_HEALTH_PORT is exported (below); a bare + # `python -m pg_queue_reaper` binds nothing. + ["$PG_QUEUE_REAPER_TYPE"]="8086" ) # Opt-in workers: experimental and NOT part of the default "all" fleet, so @@ -141,7 +146,9 @@ Note: pg-queue-consumer overrides: WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE (source tasks to load, default notification), WORKER_PG_QUEUE_CONSUMER_QUEUE (queue to poll), and WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT (liveness server port, default 8090). Note: reaper overrides: WORKER_PG_ORCHESTRATOR_LEASE_SECONDS (lease window, default 10), - WORKER_PG_REAPER_INTERVAL_SECONDS (cycle interval, default 5). + WORKER_PG_REAPER_INTERVAL_SECONDS (cycle interval, default 5), + WORKER_PG_REAPER_HEALTH_PORT (liveness server port, default 8086), + WORKER_PG_REAPER_HEALTH_STALE_SECONDS (liveness staleness window, default 30). OPTIONS: -e, --env-file FILE Use specific environment file (default: .env) @@ -748,9 +755,11 @@ run_worker() { # PG queue reaper — a leader-elected SQL recovery loop (no Celery, no task # bootstrap). Override the celery command with the plain `python -m` entry. - # Tunables (lease window, cycle interval) come from env; no liveness port is - # wired yet (that's a follow-on slice), so it binds nothing. + # Tunables (lease window, cycle interval) come from env. The liveness server + # binds when WORKER_PG_REAPER_HEALTH_PORT is set (-p override wins, else the + # map default) — exported so the reaper's main() opts in. if [[ "$worker_type" == "$PG_QUEUE_REAPER_TYPE" ]]; then + export WORKER_PG_REAPER_HEALTH_PORT="${health_port:-${WORKER_HEALTH_PORTS[$worker_type]}}" cmd_args=("uv" "run" "python" "-m" "$PG_QUEUE_REAPER_TYPE") fi diff --git a/workers/tests/test_pg_reaper.py b/workers/tests/test_pg_reaper.py index 91173dd722..a37db4494a 100644 --- a/workers/tests/test_pg_reaper.py +++ b/workers/tests/test_pg_reaper.py @@ -325,6 +325,233 @@ def test_tick_sweeps_via_real_conn(self, barrier_conn): assert _ids(barrier_conn) == [] +# --- Heartbeat + liveness probe --- + + +class TestHeartbeat: + def test_fresh_after_init(self): + reaper = PgReaper(_FakeLease(), interval_seconds=1, sweep_conn=object()) + assert reaper.seconds_since_last_tick() < 5 + assert reaper.is_tick_stale(1) is False + + def test_tick_refreshes_heartbeat(self): + # Even a standby tick (no sweep) counts as loop progress. + reaper = PgReaper( + _FakeLease(acquires=False), interval_seconds=0.01, sweep_conn=object() + ) + reaper._last_tick_monotonic = time.monotonic() - 100 # force stale + assert reaper.is_tick_stale(1) is True + reaper.tick() + assert reaper.is_tick_stale(1) is False + + +def _http_get(server, path="/health"): + import http.client + import json as _json + + conn = http.client.HTTPConnection("127.0.0.1", server.bound_port, timeout=3) + try: + conn.request("GET", path) + resp = conn.getresponse() + raw = resp.read() + return resp.status, (_json.loads(raw) if raw else None) + finally: + conn.close() + + +class TestLivenessServer: + def _server(self, reaper, *, stale_after=30.0): + server = reaper_mod.ReaperLivenessServer(reaper, port=0, stale_after=stale_after) + server.start() + return server + + def test_fresh_returns_200(self): + reaper = PgReaper( + _FakeLease(acquires=False), interval_seconds=0.01, sweep_conn=object() + ) + server = self._server(reaper) + try: + status, body = _http_get(server) + finally: + server.stop() + assert status == 200 + assert body["status"] == "healthy" + assert body["check"] == "pg_reaper_tick" + assert body["is_leader"] is False + + def test_stale_returns_503(self): + reaper = PgReaper(_FakeLease(), interval_seconds=0.01, sweep_conn=object()) + reaper._last_tick_monotonic = time.monotonic() - 100 + server = self._server(reaper, stale_after=1.0) + try: + status, body = _http_get(server) + finally: + server.stop() + assert status == 503 + assert body["status"] == "unhealthy" + + def test_is_leader_reflected(self): + reaper = PgReaper( + _FakeLease(acquires=True, renews=True), + interval_seconds=0.01, + sweep_conn=object(), + ) + with patch.object(reaper_mod, "sweep_expired_barriers", return_value=[]): + reaper.tick() # becomes leader + server = self._server(reaper) + try: + _, body = _http_get(server) + finally: + server.stop() + assert body["is_leader"] is True + + def test_extra_status_cannot_clobber_core_fields(self): + # A future extra_status_fn returning a reserved key must not corrupt the + # core payload a monitor reads — core fields always win. + from queue_backend.pg_queue.liveness import LivenessServer + + server = LivenessServer( + freshness_fn=lambda: 0.0, + stale_after=30, + port=0, + check_name="x", + age_key="age", + extra_status_fn=lambda: {"status": "HACKED", "check": "HACKED", "extra": 1}, + ) + server.start() + try: + status, body = _http_get(server) + finally: + server.stop() + assert status == 200 + assert body["status"] == "healthy" # core not clobbered + assert body["check"] == "x" # core not clobbered + assert body["extra"] == 1 # non-reserved extra preserved + + def test_unknown_path_404(self): + reaper = PgReaper(_FakeLease(), interval_seconds=1, sweep_conn=object()) + server = self._server(reaper) + try: + status, _ = _http_get(server, "/nope") + finally: + server.stop() + assert status == 404 + + def test_double_start_raises(self): + reaper = PgReaper(_FakeLease(), interval_seconds=1, sweep_conn=object()) + server = self._server(reaper) + try: + with pytest.raises(RuntimeError): + server.start() + finally: + server.stop() + + +class TestHealthEnv: + def test_stale_default_is_thirty(self, monkeypatch): + monkeypatch.delenv("WORKER_PG_REAPER_HEALTH_STALE_SECONDS", raising=False) + assert reaper_mod._reaper_health_stale_from_env() == pytest.approx(30.0) + + def test_stale_overridable(self, monkeypatch): + monkeypatch.setenv("WORKER_PG_REAPER_HEALTH_STALE_SECONDS", "10") + assert reaper_mod._reaper_health_stale_from_env() == pytest.approx(10.0) + + @pytest.mark.parametrize("bad", ["0", "-1", "x"]) + def test_stale_invalid_raises(self, monkeypatch, bad): + monkeypatch.setenv("WORKER_PG_REAPER_HEALTH_STALE_SECONDS", bad) + with pytest.raises(ValueError): + reaper_mod._reaper_health_stale_from_env() + + def test_health_server_disabled_when_no_port(self): + reaper = PgReaper(_FakeLease(), interval_seconds=1, sweep_conn=object()) + assert ( + reaper_mod._maybe_start_health_server(reaper, port=None, stale_after=30) + is None + ) + + def test_health_server_bind_failure_degrades_to_none(self, monkeypatch): + # The documented graceful-degrade path: a bind failure must NOT stop the + # reaper — log and run probe-less. + reaper = PgReaper(_FakeLease(), interval_seconds=1, sweep_conn=object()) + monkeypatch.setattr( + reaper_mod.ReaperLivenessServer, + "start", + MagicMock(side_effect=OSError("address already in use")), + ) + with patch.object(reaper_mod.logger, "exception") as logexc: + result = reaper_mod._maybe_start_health_server( + reaper, port=12345, stale_after=30 + ) + assert result is None + logexc.assert_called_once() + + def test_stale_after_non_positive_rejected(self): + # Constructor re-validates (not only the env reader) — an always-503 probe + # would crash-loop the pod. + reaper = PgReaper(_FakeLease(), interval_seconds=1, sweep_conn=object()) + with pytest.raises(ValueError, match="stale_after"): + reaper_mod.ReaperLivenessServer(reaper, port=0, stale_after=0) + + +class TestHealthPortEnv: + def test_unset_is_none(self, monkeypatch): + monkeypatch.delenv("WORKER_PG_REAPER_HEALTH_PORT", raising=False) + assert reaper_mod._reaper_health_port_from_env() is None + + def test_empty_is_none(self, monkeypatch): + monkeypatch.setenv("WORKER_PG_REAPER_HEALTH_PORT", "") + assert reaper_mod._reaper_health_port_from_env() is None + + def test_valid_port(self, monkeypatch): + monkeypatch.setenv("WORKER_PG_REAPER_HEALTH_PORT", "8086") + assert reaper_mod._reaper_health_port_from_env() == 8086 + + def test_non_int_raises_named(self, monkeypatch): + monkeypatch.setenv("WORKER_PG_REAPER_HEALTH_PORT", "abc") + with pytest.raises(ValueError, match="WORKER_PG_REAPER_HEALTH_PORT"): + reaper_mod._reaper_health_port_from_env() + + def test_out_of_range_raises(self, monkeypatch): + monkeypatch.setenv("WORKER_PG_REAPER_HEALTH_PORT", "99999") + with pytest.raises(ValueError, match="out of range"): + reaper_mod._reaper_health_port_from_env() + + +class TestMainWiring: + def _patch_main(self, monkeypatch): + # Don't open a DB connection or run the loop. + monkeypatch.setattr(reaper_mod, "LeaderLease", lambda _wid: _FakeLease()) + monkeypatch.setattr(reaper_mod.PgReaper, "run", lambda self, **_kw: None) + + def test_wires_health_port_and_stops_on_exit(self, monkeypatch): + self._patch_main(monkeypatch) + monkeypatch.setenv("WORKER_PG_REAPER_HEALTH_PORT", "0") + fake_health = MagicMock() + captured: dict = {} + + def fake_start(_reaper, *, port, stale_after): + captured["port"] = port + return fake_health + + monkeypatch.setattr(reaper_mod, "_maybe_start_health_server", fake_start) + reaper_mod.main() + assert captured["port"] == 0 # parsed int reached the wiring + fake_health.stop.assert_called_once() # stopped in the finally + + def test_no_health_when_port_unset(self, monkeypatch): + self._patch_main(monkeypatch) + monkeypatch.delenv("WORKER_PG_REAPER_HEALTH_PORT", raising=False) + captured: dict = {} + + def fake_start(_reaper, *, port, stale_after): + captured["port"] = port + return None + + monkeypatch.setattr(reaper_mod, "_maybe_start_health_server", fake_start) + reaper_mod.main() # must not raise even though health is None + assert captured["port"] is None + + # --- Entry point (the `python -m pg_queue_reaper` launch path) --- From 473603a97101fa291a93ce8dea6768a8f7a57994 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Wed, 17 Jun 2026 11:12:53 +0530 Subject: [PATCH 14/44] =?UTF-8?q?UN-3560=20[FEAT]=20PG=20Queue=209=20?= =?UTF-8?q?=E2=80=94=20run-worker.sh=20`-L=20celery`=20log=20alias=20(#206?= =?UTF-8?q?6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3560 [FEAT] PG Queue 9 — run-worker.sh `-L celery` log alias `./run-worker.sh -L pg-queue` already tails the PG-queue set's logs, but there was no symmetric way to tail only the Celery set: `-L` (no arg) tails EVERYTHING (Celery + PG-queue consumer/reaper), since list_core_worker_dirs includes the PG worker dirs. Add a `-L celery` log alias (mirror of `-L pg-queue`): tails every worker log EXCEPT the PG-queue members (pg_queue_consumer, pg_queue_reaper). Logs-only — the Celery set is still run via 'all'. run-worker.sh only: CELERY_SET constant + a branch in tail_logs() + usage/examples. Dev-tested with stub PG logs: -L = 11 files, -L celery = 9 (PG excluded), -L pg-queue = 2 (PG only). bash -n clean. Co-Authored-By: Claude Opus 4.8 * UN-3560 [FIX] -L celery review — complement wording + single-source PG members + dedup Address PR #2066 toolkit review: - Comment accuracy (P1, x2): reword 'mirrors PG_QUEUE_SET' / 'mirror of the pg-queue alias' — the celery set is the COMPLEMENT (all minus the two PG members), in a different branch, not a mirror. Drop the directional 'below'. - Modeling (P2): add a single 'PG_QUEUE_MEMBERS' source of truth (readonly assoc array) so 'celery = all - pg-queue' stays correct by construction; the celery branch tests membership via it instead of hand-rolling consumer/reaper. - Simplification (P1): collapse the near-duplicate 'all' and 'celery' tail_logs branches into one loop with a membership-guarded skip. Deferred (P2, inherited): set-aware empty-result message for the shared zero-log guard — spans all three set aliases, best done as one follow-up. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- workers/run-worker.sh | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/workers/run-worker.sh b/workers/run-worker.sh index 94ba46732f..68bb23aa11 100755 --- a/workers/run-worker.sh +++ b/workers/run-worker.sh @@ -31,6 +31,21 @@ readonly PG_QUEUE_CONSUMER_TYPE="pg_queue_consumer" readonly PG_QUEUE_REAPER_TYPE="pg_queue_reaper" # Set alias that launches the whole PG-queue group (consumer + reaper) together. readonly PG_QUEUE_SET="pg-queue" +# The PG-queue member worker dirs, as a set — the single source of truth for +# "which workers belong to the PG-queue transport". Keeps the `celery = all − +# pg-queue` complement correct by construction if a third member is ever added +# (callers test membership via ${PG_QUEUE_MEMBERS[$dir]:-} instead of +# hand-rolling the consumer/reaper pair). +declare -rA PG_QUEUE_MEMBERS=( + ["$PG_QUEUE_CONSUMER_TYPE"]=1 + ["$PG_QUEUE_REAPER_TYPE"]=1 +) +# Log-tail alias for the Celery transport: every worker EXCEPT the PG-queue +# members — the *complement* of the 'pg-queue' tail alias, so the two +# transports' logs can be tailed separately (-L celery vs -L pg-queue). +# Tail-only — there is no 'celery' run alias; this set is started via 'all', +# which by design omits the opt-in PG-queue workers. +readonly CELERY_SET="celery" # Available workers declare -A WORKERS=( @@ -163,7 +178,9 @@ OPTIONS: -r, --restart Kill matching worker(s) then relaunch (with WORKER_TYPE, restarts only that worker; without it, restarts all) -s, --status Show status of running workers - -L, --logs [WORKER] Live-tail worker log files (all if WORKER omitted) + -L, --logs [WORKER] Live-tail worker log files (all if WORKER omitted). + WORKER may also be a set: 'celery' (all but the + PG-queue members) or 'pg-queue' (consumer + reaper). -C, --clear-logs Delete worker .log files created by -d / 'all' runs -h, --help Show this help message @@ -200,6 +217,12 @@ EXAMPLES: # Live-tail all worker logs $0 -L + # Tail just the Celery set's logs (excludes the PG-queue workers) + $0 -L celery + + # Tail just the PG-queue set's logs (consumer + reaper) + $0 -L pg-queue + # Tail just one worker's log $0 -L general @@ -432,8 +455,15 @@ tail_logs() { local requested=$1 # may be empty → tail all local log_files=() - if [[ -z "$requested" || "$requested" == "all" ]]; then + if [[ -z "$requested" || "$requested" == "all" || "$requested" == "$CELERY_SET" ]]; then + # 'all' (and empty) tails every worker; 'celery' tails every worker + # EXCEPT the PG-queue members — the complement of the 'pg-queue' alias + # (handled in the else-branch) — so the two transports' logs can be + # tailed separately. Same resolve-and-append body for both, single-sourced. for d in $(list_core_worker_dirs) $(list_pluggable_worker_dirs); do + if [[ "$requested" == "$CELERY_SET" && -n "${PG_QUEUE_MEMBERS[$d]:-}" ]]; then + continue + fi local f f=$(resolve_log_file "$d") [[ -n "$f" ]] && log_files+=("$f") From d6cead2114012883918c4577283044f091417733 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Wed, 17 Jun 2026 11:20:33 +0530 Subject: [PATCH 15/44] =?UTF-8?q?UN-3559=20[FEAT]=20PG=20Queue=209e=20PR?= =?UTF-8?q?=201=20=E2=80=94=20execution=20transport=20seam=20(inert)=20(#2?= =?UTF-8?q?062)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3559 [FEAT] PG Queue 9e PR 1 — execution transport seam (inert) Establish the per-execution transport seam for the 9e coupled-pipeline migration: the transport a workflow execution rides ("celery" | "pg_queue") is resolved once at the creation chokepoint and carried in the task payload. Inert — transport always resolves to "celery", so behaviour is unchanged. Design (chosen = payload-carry, not a WorkflowExecution column): workers/queue_backend/pg_queue/9e-design.md. - core: WorkflowTransport enum + DEFAULT_WORKFLOW_TRANSPORT (shared vocabulary). - backend: resolve_transport() hardwired to celery (signature shaped for PR 3's Flipt wiring); create-execution internal API returns "transport"; execute_workflow_async adds it to the async_execute_bin kwargs. - workers: scheduler threads transport from the create response into the dispatch kwargs; async_execute_bin_general / _execute_general_workflow carry it onto the live WorkflowContextData. - tests: backend resolver + enum (5), worker WorkflowContextData carry/default (2), dispatch characterisation updated (+ backend-resolved-transport thread). Out of scope: live PG routing + per-batch idempotency key (PR 2); Flipt canary wiring (PR 3); rollout (ops). Co-Authored-By: Claude Opus 4.8 * UN-3559 [FIX] 9e PR 1 review — fail-closed transport coercion + doc/test fixes Address PR #2062 review (PR Review Toolkit findings): - Validation/silent-failure: add normalize_transport() (core) — fail-closed coercion of any inbound transport to a known value (unknown/None -> celery + warn). Applied at the scheduler read boundary and in WorkflowContextData __post_init__, so a garbage payload value can't reach the PR 2 fan-out read. - Comment accuracy: WorkflowContextData.transport comment now present-tense (carried in PR 1; fan-out read lands in PR 2). - Dead-code hygiene: add 'transport' to EXECUTION_EXCLUDED_PARAMS so the legacy execute_bin -> create_workflow_execution path can't TypeError. - Two-resolution-sites: documented the deliberate two-site design + the PR 3 single-chokepoint requirement at execute_workflow_async. - transport.py: drop the unused logger; leave a PR-3 fail-closed marker. - Design doc: correct anchors (transport.py / internal_api_views / workflow_ helper, not execution.py:126), class name (WorkflowContextData not WorkflowExecutionContext), scope claims (stage-1 only in PR 1), and remove the hard-coded Flipt version/date/live-state. - Tests: normalize_transport (passthrough / invalid->celery / None / logging), WorkflowContextData invalid-transport coercion. Co-Authored-By: Claude Opus 4.8 * UN-3559 [FIX] 9e PR 1 — resolve transport after execution_id guard (greptile P1) Move the normalize_transport(...) extraction below the `if not execution_id` guard in _execute_scheduled_workflow. Previously it ran before the guard, so an error response with no execution_id logged a misleading `[exec:None]` context and discarded the computed transport. Now transport is only resolved once execution_id is known non-empty. Co-Authored-By: Claude Opus 4.8 * UN-3559 [FIX] 9e PR 1 — SonarCloud: drop unused resolve_transport params + dict literal Address SonarCloud code smells on PR #2062: - python:S1172 (x3): resolve_transport() no longer declares the unused workflow_id/pipeline_id/organization_id params — the PR-1 seam is inert and needs no inputs. PR 3 reintroduces them (keyed for Flipt) when it wires the evaluation; the two call sites (internal_api_views view, execute_workflow_async) now call resolve_transport() with no args. Tests updated. - Replace dict(...) constructor with a {...} literal in test_workflow_context_transport._make_context. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .../workflow_manager/internal_api_views.py | 9 + .../workflow_v2/tests/__init__.py | 0 .../workflow_v2/tests/test_transport.py | 61 ++++ .../workflow_manager/workflow_v2/transport.py | 40 +++ .../workflow_v2/workflow_helper.py | 19 + .../core/src/unstract/core/data_models.py | 49 +++ workers/general/tasks.py | 5 + workers/queue_backend/pg_queue/9e-design.md | 329 ++++++++++++++++++ workers/scheduler/tasks.py | 19 +- workers/shared/models/execution_models.py | 20 ++ .../test_dispatch_sites_characterisation.py | 24 +- .../tests/test_workflow_context_transport.py | 51 +++ 12 files changed, 624 insertions(+), 2 deletions(-) create mode 100644 backend/workflow_manager/workflow_v2/tests/__init__.py create mode 100644 backend/workflow_manager/workflow_v2/tests/test_transport.py create mode 100644 backend/workflow_manager/workflow_v2/transport.py create mode 100644 workers/queue_backend/pg_queue/9e-design.md create mode 100644 workers/tests/test_workflow_context_transport.py diff --git a/backend/workflow_manager/internal_api_views.py b/backend/workflow_manager/internal_api_views.py index 931ba94138..ac81e31aa0 100644 --- a/backend/workflow_manager/internal_api_views.py +++ b/backend/workflow_manager/internal_api_views.py @@ -20,6 +20,7 @@ from workflow_manager.workflow_v2.enums import ExecutionStatus from workflow_manager.workflow_v2.models import Workflow, WorkflowExecution +from workflow_manager.workflow_v2.transport import resolve_transport logger = logging.getLogger(__name__) @@ -320,11 +321,19 @@ def create_workflow_execution(request): # Handle tags logic if needed pass + # Resolve the transport this execution rides (9e). Decided once here, at + # the creation chokepoint, and returned so the caller carries it in the + # dispatched task's payload — not persisted on the row. PR 1 always + # resolves "celery"; PR 3 wires Flipt (keyed on workflow/pipeline/org) in + # resolve_transport(). + transport = resolve_transport() + return Response( { "execution_id": str(execution.id), "status": execution.status, "execution_log_id": execution.execution_log_id, # Return for workers to use + "transport": transport, } ) diff --git a/backend/workflow_manager/workflow_v2/tests/__init__.py b/backend/workflow_manager/workflow_v2/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/workflow_manager/workflow_v2/tests/test_transport.py b/backend/workflow_manager/workflow_v2/tests/test_transport.py new file mode 100644 index 0000000000..328850638e --- /dev/null +++ b/backend/workflow_manager/workflow_v2/tests/test_transport.py @@ -0,0 +1,61 @@ +"""Tests for the 9e transport-resolution seam. + +PR 1 is the *inert* seam: ``resolve_transport`` always returns Celery, so the +whole pipeline is byte-identical to today. These tests pin that contract so a +future change (PR 3, Flipt wiring) can't silently flip the default. +""" + +from unittest.mock import MagicMock + +from unstract.core.data_models import ( + DEFAULT_WORKFLOW_TRANSPORT, + WorkflowTransport, + normalize_transport, +) +from workflow_manager.workflow_v2.transport import resolve_transport + + +class TestWorkflowTransportEnum: + def test_values(self): + assert WorkflowTransport.CELERY.value == "celery" + assert WorkflowTransport.PG_QUEUE.value == "pg_queue" + + def test_default_is_celery(self): + assert DEFAULT_WORKFLOW_TRANSPORT == WorkflowTransport.CELERY.value + + +class TestResolveTransport: + def test_resolves_celery_in_pr1(self): + """PR 1: always Celery (inert seam, no inputs). PR 3 adds the + workflow/pipeline/org params + Flipt evaluation.""" + assert resolve_transport() == WorkflowTransport.CELERY.value + + def test_result_is_a_valid_transport_value(self): + valid = {t.value for t in WorkflowTransport} + assert resolve_transport() in valid + + +class TestNormalizeTransport: + """The fail-closed coercion used at every untrusted read boundary.""" + + def test_recognized_values_pass_through(self): + assert normalize_transport("celery") == "celery" + assert normalize_transport("pg_queue") == "pg_queue" + + def test_unrecognized_value_falls_back_to_celery(self): + assert normalize_transport("celary") == DEFAULT_WORKFLOW_TRANSPORT + assert normalize_transport("pg-queue") == DEFAULT_WORKFLOW_TRANSPORT + + def test_none_and_empty_fall_back_to_celery(self): + assert normalize_transport(None) == DEFAULT_WORKFLOW_TRANSPORT + assert normalize_transport("") == DEFAULT_WORKFLOW_TRANSPORT + + def test_invalid_value_logs_a_warning_when_logger_given(self): + log = MagicMock() + assert normalize_transport("bogus", logger=log, context=" [exec:x]") == "celery" + log.warning.assert_called_once() + + def test_valid_value_does_not_warn(self): + log = MagicMock() + normalize_transport("pg_queue", logger=log) + log.warning.assert_not_called() diff --git a/backend/workflow_manager/workflow_v2/transport.py b/backend/workflow_manager/workflow_v2/transport.py new file mode 100644 index 0000000000..dfb22a181e --- /dev/null +++ b/backend/workflow_manager/workflow_v2/transport.py @@ -0,0 +1,40 @@ +"""Transport resolution for a workflow execution (9e). + +A workflow execution rides one transport end-to-end — legacy Celery/RabbitMQ +or the bespoke Postgres queue. The choice is resolved **once**, at the +execution-creation chokepoint, and returned to the caller so it can be carried +in the dispatched task's payload (see ``workers/queue_backend/pg_queue/ +9e-design.md``). It is deliberately **not** persisted on the ``WorkflowExecution`` +row: the payload is the single carrier, durable for PG via the queue row's +JSONB, and the giant shared table is never migrated for this work. + +PR 1 (this seam) hardwires the result to Celery, so behaviour is byte-identical +to today. PR 3 replaces the body with a Flipt evaluation (percentage rollout via +``entity_id`` hashing + per-org segment via ``context``), wrapped by an env +kill-switch and failing closed to Celery. +""" + +from __future__ import annotations + +from unstract.core.data_models import WorkflowTransport + + +def resolve_transport() -> str: + """Resolve the transport for a new workflow execution. + + Returns: + The transport value (a :class:`WorkflowTransport` value string). + + Note: + PR 1 always returns ``"celery"`` and takes no arguments (the inert seam + needs no inputs). PR 3 reintroduces ``workflow_id`` / ``pipeline_id`` / + ``organization_id`` parameters when it wires Flipt here — percentage + rollout via ``entity_id`` hashing + per-org segment via ``context`` — + and updates the two call sites (``internal_api_views`` view and + ``workflow_helper.execute_workflow_async``). PR 3 must wrap the Flipt + evaluation in ``try/except`` and fall back to + ``WorkflowTransport.CELERY.value`` (fail-closed), so a Flipt outage can + never break execution creation — mirroring ``normalize_transport`` on + the read side. + """ + return WorkflowTransport.CELERY.value diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py index abe20703e1..29a067a306 100644 --- a/backend/workflow_manager/workflow_v2/workflow_helper.py +++ b/backend/workflow_manager/workflow_v2/workflow_helper.py @@ -57,15 +57,20 @@ from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper from workflow_manager.workflow_v2.models.execution import WorkflowExecution from workflow_manager.workflow_v2.models.workflow import Workflow +from workflow_manager.workflow_v2.transport import resolve_transport logger = logging.getLogger(__name__) # Parameters to exclude when calling create_workflow_execution +# (the classmethod takes no **kwargs, so any task kwarg not a real parameter — +# incl. the 9e ``transport`` carried in the async_execute_bin payload — must be +# filtered out here before it reaches the legacy ``execute_workflow`` path). EXECUTION_EXCLUDED_PARAMS = { "llm_profile_id", "hitl_queue_name", "hitl_packet_id", "custom_data", + "transport", } @@ -503,6 +508,19 @@ def execute_workflow_async( } org_schema = UserContext.get_organization_identifier() log_events_id = StateStore.get(Common.LOG_EVENTS_ID) + # Resolve the transport this execution rides (9e) and carry it in the + # task payload — the pipeline reads it to stay on one transport + # end-to-end. PR 1 always resolves "celery" (no behaviour change). + # + # NOTE (deliberate, two resolution sites): transport is resolved here + # for the API/manual/async paths and separately in + # internal_api_views.create_workflow_execution for the scheduler + # path. These are DISTINCT entry paths — never a double-resolution of + # the same execution — so today they cannot diverge. PR 3 makes each + # an independent Flipt evaluation; before then, both must be funnelled + # through a single per-execution chokepoint (so a percentage re-roll + # can't split one execution across transports). Tracked for PR 3. + transport = resolve_transport() async_execution: AsyncResult = celery_app.send_task( "async_execute_bin", args=[ @@ -521,6 +539,7 @@ def execute_workflow_async( "hitl_queue_name": hitl_queue_name, "hitl_packet_id": hitl_packet_id, "custom_data": custom_data, + "transport": transport, }, queue=queue, ) diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py index 7e8e984a04..0336647570 100644 --- a/unstract/core/src/unstract/core/data_models.py +++ b/unstract/core/src/unstract/core/data_models.py @@ -200,6 +200,55 @@ class SourceConnectionType(str, Enum): API = "API" +class WorkflowTransport(str, Enum): + """Transport a single workflow execution rides end-to-end. + + The migration unit is the *execution*, not the task: every stage of a + coupled pipeline (async_execute → file-batch fan-out → callback) must run + on one transport, decided once at execution creation and carried in the + task payload (see ``9e-design.md``). ``CELERY`` is the legacy default; + ``PG_QUEUE`` is the bespoke Postgres queue. + """ + + CELERY = "celery" + PG_QUEUE = "pg_queue" + + +# Default transport when none is resolved/carried — keeps every pre-existing +# payload and caller on the legacy path until a transport is explicitly chosen. +DEFAULT_WORKFLOW_TRANSPORT = WorkflowTransport.CELERY.value + + +def normalize_transport(value: object, *, logger: Any = None, context: str = "") -> str: + """Coerce an inbound transport value to a known ``WorkflowTransport`` value. + + The transport crosses untrusted boundaries — a task payload, PG JSONB, an + older backend that omits the field — so a missing/garbage value must never + route an execution onto an unknown substrate. This **fails closed**: an + unrecognized value (``None``, ``"celary"``, ``""``) logs a warning (when a + ``logger`` is given) and falls back to :data:`DEFAULT_WORKFLOW_TRANSPORT` + (Celery). A recognized value passes through as its canonical string. + + Coerce-not-raise is deliberate, and differs from how + ``WorkflowContextData.workflow_type`` is validated: ``transport`` has an + explicit safe default by design (the whole point of the seam is reversible + fallback to Celery), so a bad value should degrade to Celery, not crash the + execution. ``context`` is an optional suffix for the log line (e.g. an + ``exec:`` tag) to make a version-skew warning traceable. + """ + try: + return WorkflowTransport(value).value + except ValueError: + if logger is not None: + logger.warning( + "Unrecognized workflow transport %r%s; falling back to %r", + value, + context, + DEFAULT_WORKFLOW_TRANSPORT, + ) + return DEFAULT_WORKFLOW_TRANSPORT + + class FileListingResult: """Result of listing files from a source.""" diff --git a/workers/general/tasks.py b/workers/general/tasks.py index edceacdb77..8d9e6e4bf4 100644 --- a/workers/general/tasks.py +++ b/workers/general/tasks.py @@ -51,6 +51,7 @@ # Import shared data models for type safety from unstract.core.data_models import ( + DEFAULT_WORKFLOW_TRANSPORT, ExecutionStatus, FileBatchData, FileHashData, @@ -156,6 +157,7 @@ def async_execute_bin_general( pipeline_id: str | None = None, log_events_id: str | None = None, use_file_history: bool = False, + transport: str = DEFAULT_WORKFLOW_TRANSPORT, **kwargs: dict[str, Any], ) -> dict[str, Any]: """Lightweight general workflow execution task. @@ -267,6 +269,7 @@ def async_execute_bin_general( use_file_history, scheduled, schema_name, + transport=transport, **kwargs, ) @@ -462,6 +465,7 @@ def _execute_general_workflow( use_file_history: bool, scheduled: bool, schema_name: str, + transport: str = DEFAULT_WORKFLOW_TRANSPORT, **kwargs: dict[str, Any], ) -> dict[str, Any]: """Execute general workflow specific logic for ETL/TASK workflows. @@ -522,6 +526,7 @@ def _execute_general_workflow( "execution_mode": execution_mode, }, is_scheduled=scheduled, + transport=transport, ) logger.info( diff --git a/workers/queue_backend/pg_queue/9e-design.md b/workers/queue_backend/pg_queue/9e-design.md new file mode 100644 index 0000000000..78f4393d62 --- /dev/null +++ b/workers/queue_backend/pg_queue/9e-design.md @@ -0,0 +1,329 @@ +# 9e — Coupled-Pipeline Migration (Design) + +**Status:** design locked, pre-implementation +**Epic:** UN-3445 (PG Queue) · **Phase:** 9e +**Predecessors:** 9a–9c (dispatch seam + PG consumer), 9d (leader-election lease + reaper / barrier-orphan sweep) — all merged. + +This document defines how a **whole workflow execution** is migrated from the +legacy Celery/RabbitMQ transport to the bespoke Postgres queue, and why the +transport choice is **carried in the task payload** rather than persisted on a +shared model column. + +--- + +## 1. The problem 9e solves + +Earlier slices (9a–9c) added a `dispatch()` seam that routes a task by +**name** via `select_backend(task_name)` against the `WORKER_PG_QUEUE_ENABLED_TASKS` +allow-list. That is correct for **leaf / independent** tasks, but it cannot +migrate the execution pipeline, because the pipeline is a **coupled, multi-stage +fan-out/fan-in**: + +``` +async_execute_bin → fan-out: N × process_file_batch → fan-in: callback → destination + (stage 1) (stage 2, parallel) (stage 3) +``` + +Two facts make this hard: + +1. **The worker code is shared.** The same `process_file_batch` / + orchestration code runs whether a **Celery worker** (pulled from RabbitMQ) + or a **PG consumer** (polled from `pg_queue_message` via `SELECT … FOR UPDATE + SKIP LOCKED`) picked the task up. The running task does **not** inherently + know which transport it is on. + +2. **Stages 2 and 3 are enqueued from inside the workers**, not by the backend. + The worker running `async_execute_bin` is the one that must enqueue the file + batches *and* set up the barrier/callback — so **it** has to know whether to + use a Celery `chord` or a PG enqueue + `PgBarrier`. + +> **Migration coherence rule.** An execution must run **entirely** on one +> transport, end-to-end. The migration unit is the **execution**, not the task. +> A pipeline that starts on PG must fan-out, fan-in, and recover on PG; likewise +> for Celery. Splitting a single execution across transports (e.g. a callback +> looking for a Celery chord whose counter actually lives in `pg_barrier_state`) +> is the failure mode this phase is designed to prevent. + +So `select_backend(task_name)` (per-task, name-based) is **insufficient**: the +same task name must go to PG for one execution and Celery for another. We need a +**per-execution** transport decision that every stage can read. + +--- + +## 2. Decision split: Flipt **decides**, the payload **remembers** + +These are two distinct jobs and must not be conflated. + +### Flipt = make the decision (once, at creation) + +The rollout question — *"should this **new** execution ride PG queue?"* — is +answered **once**, at the single creation chokepoint, using the existing +`@unstract/flags` Flipt infrastructure +(`check_feature_flag_status(flag_key, entity_id, context)`): + +- `entity_id` → consistent **percentage-rollout** hashing. +- `context={"organization_id": ...}` → **per-org** segment targeting. + +This is the only place Flipt is consulted. + +#### Flipt flag contract (fixed) + +PR 3's `resolve_transport` reads this exact flag. Live rollout state (whether the +flag exists in a given env, current rollout %) is tracked in the PR / ticket, not +here — this table is just the durable contract PR 3 codes against. + +| Property | Value | +|----------|-------| +| Flipt version | the `flipt` image pinned in `docker/docker-compose-dev-essentials.yaml` (single source — don't duplicate the tag here) | +| Flag type | **Boolean** (binary decision — no variant/payload needed) | +| Key | `pg_queue_execution_enabled` | +| Default value | **`false`** → legacy Celery (fails closed; returned when no rollout matches) | +| Rollouts | percentage (0%→ramp) + optional per-org segment, added in PR 3 / rollout ops | +| Helper | `check_feature_flag_status(flag_key="pg_queue_execution_enabled", entity_id=…, context={"organization_id": org_id})` → `evaluate_boolean`; already fails closed to `False` on any error / when `FLIPT_SERVICE_AVAILABLE != "true"` | +| Mapping | `True → "pg_queue"`, `False → "celery"` | +| `entity_id` (stickiness) | **TBD in PR 3** — `execution_id` (per-execution bucketing) or `organization_id` (whole orgs move together). Must be stable so an in-flight execution never re-buckets. | + +Boolean (not Variant) because the choice is binary and Flipt Boolean flags +natively carry the two knobs we need: a **percentage rollout** (the canary, +sticky via `entity_id`) **plus segment rollouts** (per-org). Variant would +require Variants + Segments + Rules all configured or it returns `match=False`. + +### Why Flipt cannot also be the per-stage source of truth + +Three hard facts (verified against `unstract/flags/src/unstract/flags/`): + +1. **The answer changes; the execution doesn't.** A percentage rollback + (10% → 0%) or a segment edit flips Flipt's answer. But an execution already + in flight on PG **must finish on PG**. Re-querying Flipt per stage would + re-route a half-done execution → split-brain. +2. **Flipt fails closed to `False`.** `FliptClient.service_available` defaults + `false`, and every error path returns `False` (`except Exception: return + False`). A transient gRPC blip during the callback stage would silently say + "Celery" and orphan a live PG execution. Per-stage coherence can never depend + on a best-effort network call. +3. **The reaper has no Flipt context.** It recovers executions it did not + dispatch; Flipt can only say "what would a *new* execution be now," never + "what *was* this one dispatched as." + +**Therefore:** evaluate Flipt once at creation → write the answer into the +**first task's payload** → never consult Flipt again for that execution. + +--- + +## 3. Chosen design — transport carried in the task payload + +The transport string (`"celery"` | `"pg_queue"`) is decided at +`create_workflow_execution` and threaded through the pipeline **in the task +payload / `ExecutionContext`** — never read from a shared DB column. + +### Why payload-carry is the right home + +- **It is not extra.** The shared worker code already *needs* the transport to + pick the next-stage topology (chord vs PG+PgBarrier). Putting it in the + payload satisfies that need directly — no separate mechanism. +- **It is durable for PG without any new column.** A PG-routed dispatch + serialises `kwargs` into the `pg_queue_message.message` JSONB column + (`TaskPayload`, see `pg_queue/task_payload.py`). If a worker crashes and the + task is redelivered on visibility-timeout, the transport comes back **with the + redelivered payload**. Recovery is covered for free. +- **The reaper needs no flag to classify executions.** The reaper only ever acts + on **PG-specific tables** (`pg_barrier_state`, `pg_queue_message`) — presence + there *is* "this is a PG execution." Celery executions never appear in those + tables, so there is nothing to disambiguate. When the reaper needs the + transport (e.g. future pipeline re-enqueue), it reads it from the payload it + already finds in the PG row. +- **All PG-specific durable state stays in PG-specific tables.** Those tables are + `DROP`-ped wholesale when the feature is retired. The huge, shared + `WorkflowExecution` table is **never altered, never migrated** for this work. + +### Flow + +``` +backend: create_workflow_execution(...) + └─ resolve_transport(workflow, pipeline_id, org) # Flipt, once + → transport ∈ {"celery", "pg_queue"} + └─ dispatch("async_execute_bin", kwargs={..., "transport": transport}) + → routed to that transport's first hop + +worker: async_execute_bin (reads transport from its own kwargs / ExecutionContext) + └─ for each batch: dispatch("process_file_batch", kwargs={..., "transport": transport}) + └─ barrier/callback set up on the SAME transport (get_barrier honours it) + +worker: process_file_batch (transport rides in kwargs → re-stamped onward) +worker: callback (transport in kwargs → finalises on the same transport) +``` + +The live worker context `WorkflowContextData` +(`workers/shared/models/execution_models.py`) gains a +`transport: str = "celery"` field, populated from the inbound task kwargs. +Default `"celery"` keeps every pre-existing payload working unchanged. In PR 1 +the field is carried but **not yet re-emitted** to the stage-2/3 dispatches; the +downstream re-stamp + fan-out read land in PR 2. (Note: the unrelated +`WorkflowExecutionContext` dataclass in the same file is dead scaffolding and is +**not** the carrier.) + +--- + +## 4. Rejected alternative — a `transport` column on `WorkflowExecution` + +A persisted `WorkflowExecution.transport` column (single queryable source of +truth) was considered and **rejected** in favour of payload-carry. + +**What it would have bought:** a single `WHERE transport='pg_queue'` query for +dashboards and post-completion history. + +**Why rejected:** +- It touches the **shared, very large** `WorkflowExecution` table. Even though + the change is cheap (see cost note), all of PG-queue's other durable state + already lives in disposable PG-specific tables; adding one field to the shared + table is the *only* migration that would outlive the feature unless explicitly + dropped. +- The transport is **already required in the payload** for next-stage routing, + so a column would be redundant with information we must carry anyway. +- The observability gap is small: in-flight PG executions are queryable directly + via `pg_queue_message` / `pg_barrier_state`; historical classification comes + from logs/metrics we already emit. + +**Cost note (for the record).** The rejection is *not* on migration-cost +grounds — on Postgres 11+ both operations are cheap: +- **`ADD COLUMN`** with a nullable / constant (empty-string) default is a + **metadata-only** catalog change — no table rewrite, near-instant regardless + of row count (brief `ACCESS EXCLUSIVE` lock; run off-peak with a short + `lock_timeout`). +- **`DROP COLUMN`** is likewise metadata-only (marks the column dropped, no + rewrite). + +So the column would have been operationally safe; payload-carry wins on +**design cleanliness** (no shared-table coupling, transport not duplicated, all +PG state confined to droppable PG tables), not on migration cost. + +--- + +## 5. Deployment topology over the rollout + +The payload pin classifies each in-flight execution; it does **not** remove the +need for both consumers to be live during migration. Work is delivered on two +physically separate channels: + +- **Celery-pinned** executions → tasks land in **RabbitMQ** → only a **Celery + worker** can pick them up. +- **PG-pinned** executions → tasks land in **`pg_queue_message`** → only a **PG + consumer** (SKIP LOCKED poller) can pick them up. + +| Rollout stage | Consumers running | +|--------------------|----------------------------------------------------------------| +| Today / 0% | Celery workers only | +| Canary (e.g. 10%) | Celery workers **and** PG consumers + reaper, side by side | +| 100% / retire | PG consumers + reaper only; RabbitMQ + Celery workers torn down | + +Both consumer sets can be **co-located in one deployment** as parallel processes +— the `pg` set (PG consumers + leader-elected singleton reaper) plus the Celery +set — which is what `run-worker.sh` already supports (`pg` / `pg-queue` set vs +the Celery sets). "One fleet, two transports in parallel," not "one loop polling +the DB." + +--- + +## 6. Gating prerequisite — per-batch idempotency key + +PG queue is **at-least-once** (visibility-timeout redelivery); the pipeline +tasks are **non-idempotent** with `max_retries=0`. Before *any* real traffic +rides PG, `process_file_batch` must carry a **per-batch idempotency key** and +guard side effects with `INSERT … ON CONFLICT (execution_id, batch_index)` so a +redelivered batch cannot double-process. This is a **hard gate** before the +canary slice, not optional hardening. + +--- + +## 7. Slice breakdown — 3 PRs + +Three PRs (each one Jira sub-task), non-regressive, dev-tested against a running +stack before any PR is opened. Ordering is forced: **PR 1 → PR 2 → PR 3** (the +seam must exist before the PG path; Flipt is pointless before the PG path +works). All land on `feat/UN-3445-pg-queue-integration`, not `main`. + +> **Why 3 and not 6 or 1.** The earlier draft over-sliced. The scheduled/ETL +> path is *not* a separate PR — the chokepoint is universal (the scheduler mints +> via the same `create_workflow_execution`), so it rides PR 1 with at most one +> extra test. Rollout is *ops*, not a code PR. The idempotency key is the safety +> belt for the PG path and ships **with** it (PR 2), never separately ahead of +> it. We stop at 3 (not 1) to keep PR 1 **inert and isolated** — it merges with +> near-zero review risk and makes the branch carry `transport` everywhere, so +> PR 2 is reviewed purely as "the PG behaviour" (the highest-blast-radius code in +> the epic). Collapsing the inert seam into the risky rewire is the one merge to +> avoid. + +### PR 1 — transport seam (inert, no routing change) +- `resolve_transport(...)` helper (`backend/workflow_manager/workflow_v2/ + transport.py`), **hardwired to `"celery"`** (Flipt wiring is PR 3), called at + the two transport-emit sites: the scheduler-path HTTP view + (`internal_api_views.py:create_workflow_execution`, returns `transport`) and + the API/manual/async path (`workflow_helper.py:execute_workflow_async`, adds + it to the `async_execute_bin` kwargs). +- Add `transport: str = "celery"` to the live `WorkflowContextData` + (`workers/shared/models/execution_models.py`); populate from inbound kwargs + (NOT `WorkflowExecutionContext`, which is dead scaffolding). +- Thread `transport` into the **stage-1** dispatch kwargs only + (`workflow_helper.py` send + `scheduler/tasks.py` dispatch) across the entry + paths (API / async / scheduled-ETL/TASK / manual). Stage-2 fan-out + (`process_file_batch`) and stage-3 callback are **not** modified in PR 1 — + they carry no `transport` yet; the downstream re-stamp + the `pg_queue` branch + land in PR 2. +- Tests: resolver returns `"celery"`; `ExecutionContext` carries/defaults the + field; payload round-trips the field through `to_payload`/decode; scheduled + path carries it through. +- **Net behaviour change: none.** Pure plumbing behind a default. + +### PR 2 — live PG pipeline + idempotency gate (ship together) +- Implement the `pg_queue` branch at each dispatch site: enqueue next stage via + `dispatch()` onto PG; barrier via `get_barrier()` → `PgBarrier`. +- Fire-and-forget self-chaining (labs §5): each task enqueues the next and does + its own in-body barrier decrement; no chord/`.link` (the PG payload carries no + Celery `.link`). +- **Per-batch idempotency key in the same PR** — `process_file_batch` carries it; + side effects guarded by `INSERT … ON CONFLICT (execution_id, batch_index)` so a + vt-redelivered batch is a no-op. The PG path is never mergeable-and-enableable + without its guard. +- Still **defaults-off** (transport still resolves to `"celery"` from PR 1). +- Dev-test end-to-end on PG via a forced `transport="pg_queue"` test workflow; + characterisation test that a redelivered batch double-fires nothing. + +### PR 3 — Flipt canary wiring (turn the knob on) +- `resolve_transport` consults Flipt (`entity_id` = execution/org for %-hashing; + `context` = org segment), replacing the hardwired `"celery"`. Env kill-switch + wraps it for instant rollback; Flipt fails-closed to `"celery"`. +- Reads the **fixed flag contract** in §2 (key `pg_queue_execution_enabled`, + Boolean, default `false`). PR 3 adds the read + decides the `entity_id` + stickiness; provisioning the flag/rollouts per env is rollout ops. + +### Rollout — ops, not a PR +- Canary %, dashboards (off `pg_queue_message` / `pg_barrier_state`), runbook, + ramp + Celery teardown criteria. + +--- + +## 8. Key files (call-graph anchors) + +| Concern | Location | +|-----------------------------|----------| +| Transport resolve + emit | `workflow_v2/transport.py` (`resolve_transport`); emitted at `internal_api_views.py` (`create_workflow_execution` view → response, scheduler path) and `workflow_helper.py` (`execute_workflow_async` → `async_execute_bin` kwargs, API/manual/async path) | +| Row-builder (no transport) | `workflow_v2/execution.py` (`WorkflowExecutionServiceHelper.create_workflow_execution` — builds the row; does NOT resolve transport) | +| Entry paths | API deploy, async, scheduled-ETL/TASK, manual UI — all reach one of the two emit sites above | +| Pipeline dispatch sites | `workflow_helper.py` (stage-1 send, stage-2 batch signatures, stage-3 chord) — only stage-1 threads transport in PR 1 | +| Worker context (carrier) | `workers/shared/models/execution_models.py` (`WorkflowContextData` — NOT the dead `WorkflowExecutionContext`) | +| Dispatch seam | `workers/queue_backend/dispatch.py`, `routing.py` (`select_backend`) | +| PG task payload (durable) | `workers/queue_backend/pg_queue/task_payload.py` (`TaskPayload` → `pg_queue_message.message` JSONB) | +| Barrier factory | `workers/queue_backend/__init__.py` (`get_barrier`) | +| Barrier impls | `barrier.py` (chord), `redis_barrier.py`, `pg_barrier.py` | +| Barrier invocation | `workers/shared/workflow/execution/orchestration_utils.py` | +| Reaper / sweep | `workers/queue_backend/pg_queue/reaper.py` | + +--- + +## 9. References + +- Labs architecture: `labs/workflow-execution-architecture/architecture-explorer-v2.html` + and `labs/workflow-execution-architecture/docs/` — §5 fire-and-forget + self-chaining is the target model for slice 3. +- 9d (merged): leader-election lease (`leader_election.py`) + reaper / + barrier-orphan sweep (`reaper.py`). diff --git a/workers/scheduler/tasks.py b/workers/scheduler/tasks.py index a152b98aed..7518a4b930 100644 --- a/workers/scheduler/tasks.py +++ b/workers/scheduler/tasks.py @@ -23,7 +23,12 @@ from shared.patterns.notification.helper import trigger_notification from shared.utils.api_client_singleton import get_singleton_api_client -from unstract.core.data_models import NotificationPayload, NotificationSource +from unstract.core.data_models import ( + DEFAULT_WORKFLOW_TRANSPORT, + NotificationPayload, + NotificationSource, + normalize_transport, +) # Import the exact backend logic to ensure consistency @@ -137,6 +142,17 @@ def _execute_scheduled_workflow( pipeline_id=context.pipeline_id, ) + # Transport this execution rides (9e), decided by the backend at creation + # and carried in the dispatched task's payload. Absent key (older backend) + # → celery default; a present-but-unrecognized value (version skew) is + # coerced to celery with a loud warning rather than dispatched onto an + # unknown substrate (fail-closed). + transport = normalize_transport( + workflow_execution.get("transport", DEFAULT_WORKFLOW_TRANSPORT), + logger=logger, + context=f" [exec:{execution_id}]", + ) + logger.info( f"[exec:{execution_id}] [pipeline:{context.pipeline_id}] Created workflow execution for scheduled pipeline {context.pipeline_name}" ) @@ -163,6 +179,7 @@ def _execute_scheduled_workflow( kwargs={ "use_file_history": context.use_file_history, "pipeline_id": context.pipeline_id, + "transport": transport, }, queue=QueueName.GENERAL, fairness=FairnessKey( diff --git a/workers/shared/models/execution_models.py b/workers/shared/models/execution_models.py index e9149b4e9a..ca5c332785 100644 --- a/workers/shared/models/execution_models.py +++ b/workers/shared/models/execution_models.py @@ -4,13 +4,16 @@ replacing fragile dictionary-based parameter passing with type-safe structures. """ +import logging from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any # Import shared domain models from core from unstract.core.data_models import ( + DEFAULT_WORKFLOW_TRANSPORT, ExecutionStatus, PreCreatedFileData, + normalize_transport, serialize_dataclass_to_dict, ) @@ -20,6 +23,8 @@ from ..enums import PipelineType +logger = logging.getLogger(__name__) + @dataclass class WorkflowExecutionContext: @@ -385,6 +390,11 @@ class WorkflowContextData: settings: dict[str, Any] | None = None metadata: dict[str, Any] | None = None is_scheduled: bool = False + # Transport this execution rides end-to-end (9e). Populated from the inbound + # task payload; the fan-out read that keeps the pipeline on one transport + # lands in PR 2. Defaults to legacy Celery; coerced fail-closed in + # __post_init__ so a garbage payload value can't reach the PR 2 read site. + transport: str = DEFAULT_WORKFLOW_TRANSPORT pre_created_file_executions: dict[str, PreCreatedFileData] = field( default_factory=dict ) @@ -414,6 +424,16 @@ def __post_init__(self): f"Must be one of: {valid_types}" ) + # Coerce transport fail-closed: a garbage payload value (version skew, + # bad PG JSONB) degrades to Celery here rather than reaching the PR 2 + # fan-out read. Unlike workflow_type (no safe default → raise), transport + # is reversible by design, so it falls back instead of crashing. + self.transport = normalize_transport( + self.transport, + logger=logger, + context=f" (execution {self.execution_id})", + ) + @property def file_count(self) -> int: """Get the number of files in this workflow context.""" diff --git a/workers/tests/test_dispatch_sites_characterisation.py b/workers/tests/test_dispatch_sites_characterisation.py index d07721bdb2..48ffe33811 100644 --- a/workers/tests/test_dispatch_sites_characterisation.py +++ b/workers/tests/test_dispatch_sites_characterisation.py @@ -219,7 +219,12 @@ def test_dispatch_positional_args_layout(self): assert args[4] is True # scheduled flag (always True here) def test_dispatch_kwargs_layout(self): - """Kwargs MUST contain use_file_history and pipeline_id.""" + """Kwargs MUST contain use_file_history, pipeline_id, and transport. + + ``transport`` (9e) is carried in the task payload so the pipeline stays + on one transport end-to-end. It defaults to ``"celery"`` when the + create-execution response omits it (older backend / inert PR 1). + """ from scheduler.tasks import _execute_scheduled_workflow ctx = self._make_context() @@ -232,8 +237,25 @@ def test_dispatch_kwargs_layout(self): assert kwargs == { "use_file_history": True, "pipeline_id": "pipe-007", + "transport": "celery", + } + + def test_dispatch_carries_backend_resolved_transport(self): + """The transport the backend returns from create-execution is threaded + verbatim into the dispatched task's payload (payload-carry, 9e).""" + from scheduler.tasks import _execute_scheduled_workflow + + api = MagicMock() + api.create_workflow_execution.return_value = { + "execution_id": "exec-123", + "transport": "pg_queue", } + with patch("scheduler.tasks.dispatch") as mock_dispatch: + _execute_scheduled_workflow(api, self._make_context()) + + assert mock_dispatch.call_args.kwargs["kwargs"]["transport"] == "pg_queue" + def test_no_dispatch_when_execution_creation_fails(self): """If api_client.create_workflow_execution returns no execution_id, the function bails out and never calls send_task.""" diff --git a/workers/tests/test_workflow_context_transport.py b/workers/tests/test_workflow_context_transport.py new file mode 100644 index 0000000000..b8db8d6434 --- /dev/null +++ b/workers/tests/test_workflow_context_transport.py @@ -0,0 +1,51 @@ +"""Tests for the 9e transport field carried on ``WorkflowContextData``. + +The transport a workflow execution rides is decided once at creation and +carried in the task payload; on the worker side it lands on the live +``WorkflowContextData`` so the fan-out (PR 2) can read it. PR 1 only adds the +field with a Celery default — these tests pin that it defaults and round-trips. +""" + +from unittest.mock import MagicMock + +from shared.models.execution_models import ( + WorkerOrganizationContext, + WorkflowContextData, +) + +from unstract.core.data_models import DEFAULT_WORKFLOW_TRANSPORT, WorkflowTransport + + +def _make_context(**overrides): + org_context = WorkerOrganizationContext( + organization_id="org-1", + api_client=MagicMock(), + ) + kwargs = { + "workflow_id": "wf-1", + "workflow_name": "wf-name", + "workflow_type": "TASK", + "execution_id": "exec-1", + "organization_context": org_context, + "files": {}, + } + kwargs.update(overrides) + return WorkflowContextData(**kwargs) + + +class TestWorkflowContextTransport: + def test_defaults_to_celery(self): + """A context built without a transport rides legacy Celery.""" + assert _make_context().transport == DEFAULT_WORKFLOW_TRANSPORT + assert _make_context().transport == WorkflowTransport.CELERY.value + + def test_carries_pg_queue_when_set(self): + ctx = _make_context(transport=WorkflowTransport.PG_QUEUE.value) + assert ctx.transport == "pg_queue" + + def test_invalid_transport_coerced_to_celery(self): + """A garbage payload value (version skew / bad PG JSONB) must fail closed + at construction, not reach the PR 2 fan-out read.""" + assert _make_context(transport="pg-queue").transport == "celery" + assert _make_context(transport="bogus").transport == "celery" + assert _make_context(transport=None).transport == "celery" From 59716f56bf39f471ab5a72035d1b8142898fb8aa Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:30:38 +0530 Subject: [PATCH 16/44] =?UTF-8?q?UN-3561=20[FEAT]=20PG=20Queue=209e=20PR?= =?UTF-8?q?=202a=20=E2=80=94=20live-PG-pipeline=20inert=20foundation=20(di?= =?UTF-8?q?spatch=20backend=20override=20+=20barrier=20decrement=20core)?= =?UTF-8?q?=20(#2067)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3561 [FEAT] PG Queue 9e PR 2a — live-PG-pipeline inert foundation (dispatch backend override + barrier decrement core) First slice of 9e PR 2 (the live PG execution pipeline), split 2a/2b/2c. 2a is the inert foundation: the two seams the live switch (2c) consumes, each with zero behaviour change on the default path. - dispatch(backend=...): per-call transport override. None (default, every call site today) keeps the env allow-list decision via select_backend — byte-identical. When set it wins over the allow-list, so 2c can route a whole execution's header/callback onto PG without opting their task names into WORKER_PG_QUEUE_ENABLED_TASKS (allow-list is for leaf tasks; the pipeline's migration unit is the execution). - Extract _barrier_pg_decrement(...) plain core out of the @worker_task barrier_pg_decr_and_check (now a thin delegator). 2c calls the core in-body on the PG-consumed path (a PG-consumed task fires no Celery .link, so the decrement runs in-body — fire-and-forget self-chaining). Inert by construction: default barrier backend is chord (Celery executions never import the PgBarrier module), and no call site passes backend=. Net behaviour change: NONE. Transport threading + live switch land in 2c; per-batch idempotency in 2b. Tests: dispatch backend-override (3) + decrement-core extraction (3, incl. real-PG in-body decrement + verbatim delegation). pg_barrier/dispatch/ barrier/routing suites green; ruff clean; worker-app bootstrap clean under WORKER_BARRIER_BACKEND=pg (both barrier tasks registered, get_barrier→PgBarrier). Co-Authored-By: Claude Opus 4.8 * UN-3561 address review (muhammad-ali-e): loud in-transaction guard, resolve_backend helper, comment/test fixes - pg_barrier: enforce the "own committed transaction" decrement contract loudly — _barrier_pg_decrement raises at entry if the shared connection is mid-transaction (was prose-only; the 2c in-body caller is the real risk). Safe for existing paths: Celery .link enters idle, tests use autocommit conns → always idle. Fix the inaccurate "one call = one _cursor() txn" parenthetical (the delete paths open a 2nd txn) and drop the rot-prone "(fire-and-forget self-chaining)" jargon. - routing: extract resolve_backend(task_name, override) — the override-wins- else-allow-list precedence now lives in one self-documenting place (2c reuses it); dispatch() calls it instead of inlining the None-means-auto rule. - dispatch: reword the backend= docstring — avoid the "payload" term collision (local to_payload var) and the stale routing.py cross-ref; point at the live carrier WorkflowContextData.transport + 9e-design.md. - tests: +fairness-reaches-row on the backend= override path; +real-row wrapper test that catches core-param drift (the mocked delegation test couldn't); +open-transaction guard test. Kept the mocked test as the keyword-forwarding pin. Deferred (LOW, reviewer-aligned): BarrierDecrementResult TypedDict union — lands in 2c when the in-body caller actually branches on the status. Co-Authored-By: Claude Opus 4.8 * UN-3561 address greptile (3× P2): docstring refs + routing-log suppression note All doc/comment-only, no logic change. greptile gave 4/5 "safe to merge"; these are the staleness its findings flagged, introduced by the resolve_backend extraction in the prior review round: - dispatch module docstring + routing "Scaffold posture" now name resolve_backend (wrapping select_backend) as the routing seam, not select_backend alone. - Note at the _pg_routing_logged log-once site that it's keyed on task name only, so an override-then-allow-list cutover won't re-announce (benign: override = pipeline headers vs allow-list = leaf tasks, no overlap expected; the allow-list config is still announced by _log_allow_list_once). Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- workers/queue_backend/dispatch.py | 34 +++++++++--- workers/queue_backend/pg_barrier.py | 68 +++++++++++++++++++---- workers/queue_backend/routing.py | 25 ++++++++- workers/tests/test_dispatch_pg.py | 83 +++++++++++++++++++++++++++++ workers/tests/test_pg_barrier.py | 75 ++++++++++++++++++++++++++ 5 files changed, 268 insertions(+), 17 deletions(-) diff --git a/workers/queue_backend/dispatch.py b/workers/queue_backend/dispatch.py index 7012b0a7e4..02b0b9636b 100644 --- a/workers/queue_backend/dispatch.py +++ b/workers/queue_backend/dispatch.py @@ -1,11 +1,12 @@ """Transport-agnostic task dispatch. -Routes each task to its transport via :func:`select_backend`: +Routes each task to its transport via :func:`resolve_backend` (which applies a +per-call ``backend`` override, else defers to :func:`select_backend`): - **Celery** (default) — a thin pass-through to ``current_app.send_task``. -- **PG Queue** — when a task is opted into ``WORKER_PG_QUEUE_ENABLED_TASKS``, - the task is serialised and enqueued to ``pg_queue_message`` (9b); the PG - consumer (9c) drains and runs it. +- **PG Queue** — when a task is opted into ``WORKER_PG_QUEUE_ENABLED_TASKS`` + (or pinned via a ``backend=`` override), the task is serialised and enqueued + to ``pg_queue_message`` (9b); the PG consumer (9c) drains and runs it. The default (empty allow-list) routes everything to Celery, so dispatch is unchanged unless an operator explicitly opts a task in. @@ -30,7 +31,7 @@ from .fairness import DEFAULT_PRIORITY, FairnessKey from .handle import DispatchHandle from .pg_queue import PgQueueClient, to_payload -from .routing import QueueBackend, select_backend +from .routing import QueueBackend, resolve_backend logger = logging.getLogger(__name__) @@ -77,14 +78,29 @@ def dispatch( kwargs: Mapping[str, Any] | None = None, queue: str | None = None, fairness: FairnessKey | None = None, + backend: QueueBackend | None = None, ) -> DispatchHandle: """Enqueue a task by name onto its selected transport. ``fairness`` is attached as the ``x-fairness-key`` header on the Celery path / serialised into the message on the PG path. Pass ``None`` for non-workflow worker tasks. + + ``backend`` is a per-call transport override (see + :func:`~queue_backend.routing.resolve_backend` for the precedence). When + ``None`` (the default, and every call site today) the transport is the env + allow-list decision via ``select_backend`` — behaviour is unchanged. When + set, it wins over the allow-list: this is the seam the execution-level PG + pipeline (9e PR 2c) uses to route a whole execution's header/callback + dispatches onto PG without opting their task *names* into + ``WORKER_PG_QUEUE_ENABLED_TASKS``. (The allow-list is for *leaf* tasks; the + coupled pipeline's migration unit is the whole execution — its transport is + resolved once at creation and travels on the execution's task kwargs onto + ``WorkflowContextData.transport``, see ``queue_backend/pg_queue/ + 9e-design.md``.) The override only forces the *transport*; it does not + bypass ``_enqueue_pg``'s no-silent-fallback contract. """ - if select_backend(task_name) is QueueBackend.PG: + if resolve_backend(task_name, backend) is QueueBackend.PG: return _enqueue_pg(task_name, args, kwargs, queue, fairness) headers = fairness.as_header() if fairness is not None else None @@ -115,6 +131,12 @@ def _enqueue_pg( # true regardless of send outcome, so a first-dispatch failure # (DB down / unmigrated) must not suppress the one announcement. # INFO so it survives a default log config; once-per-task bounds it. + # NOTE: keyed on task name only, not on *why* it routed to PG. If a name + # is first PG-routed via a ``backend=`` override and the operator later + # adds it to the allow-list, the allow-list cutover won't re-announce + # (already in the set). Benign given the usage split (override = pipeline + # headers, allow-list = leaf tasks, so no overlap expected); the + # allow-list config itself is still announced by _log_allow_list_once. _pg_routing_logged.add(task_name) logger.info( "PG-queue: routing task=%r to Postgres (queue=%r). Requires the " diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.py index 4ebc46be87..5ed2e7d021 100644 --- a/workers/queue_backend/pg_barrier.py +++ b/workers/queue_backend/pg_barrier.py @@ -65,6 +65,7 @@ from typing import TYPE_CHECKING, Any import psycopg2 +import psycopg2.extensions from .barrier import CallbackDescriptor, barrier_ttl_seconds from .decorator import worker_task @@ -246,23 +247,53 @@ def enqueue( raise -@worker_task(name="barrier_pg_decr_and_check", max_retries=0) -def barrier_pg_decr_and_check( +def _barrier_pg_decrement( result: Any, *, execution_id: str, callback_descriptor: CallbackDescriptor, ) -> dict[str, Any]: - """Per-task ``link`` callback for :class:`PgBarrier`. - - Atomically appends this task's result and decrements ``remaining``. The - single task that drives ``remaining`` to 0 dispatches the aggregating - callback (then deletes the row). ``max_retries=0`` — a Celery retry would - replay the decrement and corrupt the count; an orphaned barrier is bounded - by ``expires_at`` instead. + """Atomic barrier decrement + last-task callback fire (substrate core). + + Appends this task's result and decrements ``remaining`` in one atomic + statement; the single caller that drives ``remaining`` to 0 dispatches the + aggregating callback (then deletes the row). This is the plain, in-body- + callable core shared by two entry points: + + - the Celery ``link`` callback :func:`barrier_pg_decr_and_check` (today's + only caller, behaviour unchanged); and + - the 9e PR 2c PG-consumed pipeline path, which will call this directly in + the header task's body — a PG-consumed task fires no ``.link``, so the + decrement must run in-body. + + Callers MUST run each decrement in its own committed transaction (the + decrement ``UPDATE`` runs in its own ``_cursor()`` txn) and MUST NOT retry + it: a replay re-runs the decrement and corrupts the count. This is enforced + loudly, not just in prose — entry raises if the shared connection is already + mid-transaction (see the guard below). The Celery wrapper additionally pins + ``max_retries=0``; an orphaned barrier is bounded by ``expires_at`` instead. """ from celery import current_app + # Enforce the "own committed transaction" contract loudly. A caller that + # invokes this inside an already-open transaction on the shared thread-local + # connection would hold the row lock across the call and let the outer txn's + # commit boundary replay the decrement — corrupting ``remaining`` and + # breaking the exactly-one-sees-zero serialisation, surfacing only as a + # barrier hung until ``expires_at`` (~6h). The Celery ``.link`` path always + # enters idle (``_cursor`` commits after every use); the 2c in-body caller + # must too. (Tests inject an autocommit connection → always idle → no trip.) + if ( + _get_conn().get_transaction_status() + != psycopg2.extensions.TRANSACTION_STATUS_IDLE + ): + raise RuntimeError( + f"[exec:{execution_id}] _barrier_pg_decrement entered with an open " + f"transaction on the shared connection — each decrement MUST run in " + f"its own committed transaction (see this function's docstring). " + f"Refusing to proceed to avoid corrupting the barrier counter." + ) + try: # No default=str — a non-JSON-safe leaf must fail loudly here (it would # signal a BatchExecutionResult.to_dict() typed-boundary regression). @@ -372,6 +403,25 @@ def barrier_pg_decr_and_check( } +@worker_task(name="barrier_pg_decr_and_check", max_retries=0) +def barrier_pg_decr_and_check( + result: Any, + *, + execution_id: str, + callback_descriptor: CallbackDescriptor, +) -> dict[str, Any]: + """Per-task ``link`` callback for :class:`PgBarrier` (Celery entry point). + + Thin ``@worker_task`` wrapper around :func:`_barrier_pg_decrement` so the + decrement logic stays callable in-body (no ``.link``) on the PG-consumed + pipeline path (9e PR 2c). ``max_retries=0`` — a Celery retry would replay + the decrement and corrupt the count (see the core's contract). + """ + return _barrier_pg_decrement( + result, execution_id=execution_id, callback_descriptor=callback_descriptor + ) + + @worker_task(name="barrier_pg_abort", max_retries=0) def barrier_pg_abort( request: Any = None, diff --git a/workers/queue_backend/routing.py b/workers/queue_backend/routing.py index 82cce0d67e..228a296dfe 100644 --- a/workers/queue_backend/routing.py +++ b/workers/queue_backend/routing.py @@ -36,8 +36,9 @@ **Scaffold posture.** This module only makes the routing *decision*. In the current phase there is no PG consumer, so ``dispatch()`` still sends PG-selected tasks via Celery (the decision is observable in logs -but inert). ``select_backend()`` is the seam where the real PG dispatch -lands in a later phase. +but inert). ``resolve_backend()`` (which wraps the ``select_backend()`` +allow-list with the per-call override) is the seam where the real PG +dispatch lands in a later phase. **Observability.** Because the gate is silent-by-construction (a misrouted task still runs on Celery), the only signals are logs, emitted @@ -132,3 +133,23 @@ def select_backend(task_name: str) -> QueueBackend: if task_name in allow_list: return QueueBackend.PG return QueueBackend.CELERY + + +def resolve_backend(task_name: str, override: QueueBackend | None) -> QueueBackend: + """Resolve the transport for a dispatch, applying the per-call override. + + The single home for the override-wins-else-allow-list precedence so the + rule reads in one place (and ``dispatch()`` plus the 9e PR 2c call sites + share it): + + - ``override`` is ``None`` → defer to :func:`select_backend` (the env + allow-list) — the behaviour of every call site today. + - ``override`` is a :class:`QueueBackend` → it wins. This is how the + execution-level PG pipeline pins a whole execution's header/callback + dispatches to one transport regardless of the per-task allow-list (the + allow-list is for *leaf* tasks; the coupled pipeline's migration unit is + the execution). + + Never raises — both branches resolve to a valid :class:`QueueBackend`. + """ + return override if override is not None else select_backend(task_name) diff --git a/workers/tests/test_dispatch_pg.py b/workers/tests/test_dispatch_pg.py index e86d99193a..e9cde91824 100644 --- a/workers/tests/test_dispatch_pg.py +++ b/workers/tests/test_dispatch_pg.py @@ -19,6 +19,7 @@ from queue_backend.fairness import FairnessKey, WorkloadType from queue_backend.pg_queue import to_payload from queue_backend.routing import _ENABLED_TASKS_ENV_VAR as ENABLED_TASKS_ENV +from queue_backend.routing import QueueBackend # ``queue_backend.dispatch`` the attribute is the function (shadows the # submodule) — import the module explicitly to reach its globals. @@ -111,6 +112,88 @@ def test_celery_dispatch_unaffected(self): mock_get.assert_not_called() +class TestDispatchBackendOverride: + """The per-call ``backend=`` override (9e PR 2a inert foundation). + + When set it wins over the ``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list, so + the execution-level PG pipeline can route a whole execution's headers / + callback onto PG without opting their task *names* in. ``None`` (default) + preserves the allow-list decision exactly — every call site today. + """ + + def test_override_pg_forces_pg_without_allow_list(self, monkeypatch): + # Task name NOT in the (empty) allow-list → select_backend says Celery; + # the explicit override must still route it to PG. + captured: dict = {} + + class _Client: + def send(self, queue_name, payload, **kwargs): + captured.update(queue=queue_name, payload=payload, **kwargs) + return 11 + + monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client()) + with patch("queue_backend.dispatch.current_app") as mock_app: + handle = dispatch( + "pipeline_header", queue="general", backend=QueueBackend.PG + ) + mock_app.send_task.assert_not_called() + assert handle.id == "11" + assert captured["payload"]["task_name"] == "pipeline_header" + + def test_override_celery_forces_celery_despite_allow_list(self, monkeypatch): + # Task name IS opted into PG, but the override pins it back to Celery. + monkeypatch.setenv(ENABLED_TASKS_ENV, "pipeline_header") + with ( + patch("queue_backend.dispatch.current_app") as mock_app, + patch("queue_backend.dispatch._get_pg_client") as mock_get, + ): + dispatch("pipeline_header", queue="general", backend=QueueBackend.CELERY) + mock_app.send_task.assert_called_once() + mock_get.assert_not_called() + + def test_default_none_preserves_allow_list_decision(self, monkeypatch): + # No override → allow-list decides. Opted-in name → PG. + captured: dict = {} + + class _Client: + def send(self, queue_name, payload, **kwargs): + captured.update(queue=queue_name) + return 5 + + monkeypatch.setenv(ENABLED_TASKS_ENV, "leaf_task") + monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client()) + with patch("queue_backend.dispatch.current_app") as mock_app: + handle = dispatch("leaf_task", queue="general") + mock_app.send_task.assert_not_called() + assert handle.id == "5" + + def test_override_path_carries_fairness_to_row(self, monkeypatch): + # The override's whole purpose is to route an *execution's* dispatches — + # exactly the ones that carry a FairnessKey. The org/priority plumbing + # must work identically on the override path (not just the allow-list + # path), so a regression dropping fairness here would otherwise pass. + captured: dict = {} + + class _Client: + def send(self, queue_name, payload, **kwargs): + captured.update(queue=queue_name, **kwargs) + return 13 + + monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client()) + fairness = FairnessKey( + org_id="o", workload_type=WorkloadType.API, pipeline_priority=8 + ) + with patch("queue_backend.dispatch.current_app"): + dispatch( + "pipeline_header", + queue="general", + fairness=fairness, + backend=QueueBackend.PG, + ) + assert captured["priority"] == 8 + assert captured["org_id"] == "o" + + class TestDispatchPriorityWiring: """dispatch() carries fairness.pipeline_priority onto the PG row (mocked).""" diff --git a/workers/tests/test_pg_barrier.py b/workers/tests/test_pg_barrier.py index 0c1a635c1d..c2a9488560 100644 --- a/workers/tests/test_pg_barrier.py +++ b/workers/tests/test_pg_barrier.py @@ -23,6 +23,7 @@ from queue_backend.handle import BarrierHandle from queue_backend.pg_barrier import ( PgBarrier, + _barrier_pg_decrement, barrier_pg_abort, barrier_pg_decr_and_check, ) @@ -325,6 +326,80 @@ def test_registered_under_canonical_name(self): assert barrier_pg_decr_and_check.name == "barrier_pg_decr_and_check" +class TestDecrementCoreExtraction: + """The decrement logic lives in a plain ``_barrier_pg_decrement`` core so the + 9e PR 2c PG-consumed path can call it in-body (a PG-consumed task fires no + ``.link``). The Celery ``@worker_task`` is a thin delegator; both paths must + share one implementation (no drift). (Inert in 2a — no PG caller yet.) + """ + + def test_core_is_a_plain_callable_not_a_celery_task(self): + # A ``@worker_task`` exposes ``.name`` / ``.delay``; the in-body core + # must not, or callers could accidentally re-dispatch instead of running. + assert not hasattr(_barrier_pg_decrement, "name") + assert not hasattr(_barrier_pg_decrement, "delay") + + def test_core_runs_the_decrement_in_body(self, barrier_db): + # Called directly (no Celery), the core performs the atomic decrement. + _seed(barrier_db, "exec-core", 3) + out = _barrier_pg_decrement( + {"f": 1}, execution_id="exec-core", callback_descriptor=_CALLBACK + ) + assert out["status"] == "pending" + remaining, results = _row(barrier_db, "exec-core") + assert remaining == 2 + assert results == [{"f": 1}] + + def test_worker_task_produces_same_decrement_as_core(self, barrier_db): + # The real no-drift guard: the wrapper, run against a real seeded row, + # must produce the SAME observable decrement as a direct core call + # (mirror of test_core_runs_the_decrement_in_body). Unlike the mocked + # forwarding test below, this would catch a renamed/reordered core param + # — the core is NOT mocked away. + _seed(barrier_db, "exec-wrap", 3) + out = barrier_pg_decr_and_check( + {"f": 1}, execution_id="exec-wrap", callback_descriptor=_CALLBACK + ) + assert out["status"] == "pending" + remaining, results = _row(barrier_db, "exec-wrap") + assert remaining == 2 + assert results == [{"f": 1}] + + def test_worker_task_forwards_kwargs_verbatim(self): + # Complements the real-row test above by pinning the keyword-forwarding + # contract explicitly: the wrapper passes result + execution_id + + # callback_descriptor straight through, returning the core's result. + sentinel = {"status": "pending", "remaining": 9} + with patch.object( + pg_barrier, "_barrier_pg_decrement", return_value=sentinel + ) as core: + out = barrier_pg_decr_and_check( + {"f": 1}, execution_id="exec-d", callback_descriptor=_CALLBACK + ) + assert out is sentinel + core.assert_called_once_with( + {"f": 1}, execution_id="exec-d", callback_descriptor=_CALLBACK + ) + + def test_open_transaction_on_shared_conn_raises(self): + # The in-body contract is enforced loudly: a caller that enters with an + # already-open transaction on the shared connection is rejected before + # any decrement runs (would otherwise corrupt 'remaining' and hang the + # barrier to expiry). Guards the 2c in-body caller. + import psycopg2.extensions + + conn = MagicMock() + conn.get_transaction_status.return_value = ( + psycopg2.extensions.TRANSACTION_STATUS_INTRANS + ) + with patch.object(pg_barrier, "_get_conn", return_value=conn): + with pytest.raises(RuntimeError, match="own committed transaction"): + _barrier_pg_decrement( + {"f": 1}, execution_id="exec-tx", callback_descriptor=_CALLBACK + ) + conn.cursor.assert_not_called() # rejected before touching the DB + + class TestAbort: def test_claims_and_deletes(self, barrier_db): _seed(barrier_db, "exec-X", 2) From 5f2d18df9ce0842c2c859d621ad3a309af272cbb Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:04:26 +0530 Subject: [PATCH 17/44] =?UTF-8?q?UN-3562=20[FEAT]=20PG=20Queue=209e=20PR?= =?UTF-8?q?=202b=20=E2=80=94=20per-batch=20idempotency=20primitive=20(pg?= =?UTF-8?q?=5Fbatch=5Fdedup=20+=20claim=5Fbatch=20/=20clear=5Fexecution=5F?= =?UTF-8?q?batches)=20(#2068)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3562 [FEAT] PG Queue 9e PR 2b — per-batch idempotency primitive (pg_batch_dedup + claim_batch / clear_execution_batches) Second slice of 9e PR 2, after 2a (#2067). Inert idempotency primitive: the durable per-batch dedup marker 2c wires into the at-least-once PG path. Why: the PG queue is at-least-once, so process_file_batch can be redelivered after a crash-before-ack → re-run batch + double-decrement the barrier (non-idempotent, max_retries=0). Recon showed existing per-file protection is only partial (Redis lock released after write; WFE COMPLETED skips tool re-exec but not necessarily the destination write; FileHistory is cross-execution only), so a durable per-batch gate is needed; per-file status stays the partial-crash backstop. - backend: PgBatchDedup model + migration 0006 — table pg_batch_dedup with a UniqueConstraint(execution_id, batch_index) (the ON CONFLICT target; its execution_id-leading index also serves the cleanup DELETE). Django-managed, extension-free, same posture as the sibling pg_queue models. - workers (pg_barrier.py, reusing the barrier's _cursor() → one PG conn per worker child): claim_batch(execution_id, batch_index) -> bool (atomic INSERT ... ON CONFLICT DO NOTHING RETURNING; True=first/decrement, False=redelivery/skip) + clear_execution_batches(execution_id) -> int (barrier-teardown cleanup; reaper sweep is the backstop). No call-site wiring — claim/clear + batch_index threading + transport switch land in 2c. Inert: new table + two helpers, no callers. Net behaviour: NONE. Tests: +8 real-PG (first-claim, redelivery-rejected, distinct-batch, distinct-exec, concurrent-exactly-one-winner, clear-only-target, clear-empty-zero, reclaim-after-clear). Migration applied to dev DB, makemigrations --check clean, bootstrap clean under WORKER_BARRIER_BACKEND=pg. Co-Authored-By: Claude Opus 4.8 * UN-3562 address review (muhammad-ali-e P1-P8): fix reaper-backstop docstring claim, batch_index constraint, race test, nits - P1 (verified bug): the docstrings claimed the reaper's barrier-orphan sweep reclaims orphaned pg_batch_dedup markers — it doesn't. sweep_expired_barriers DELETEs only pg_barrier_state (no cascade), so orphaned markers leak today. Reworded both PgBatchDedup + clear_execution_batches docstrings to state the leak honestly + flag the dedup-orphan sweep as intended future work. - P4: add CheckConstraint(batch_index >= 0) (writer-proof, mirrors PgQueueMessage.priority) + a test that the DB rejects a negative index. Regenerated migration 0006 to include it. - P6: claim_batch docstring no longer says it decrements; defers the single decrement to the caller (the function only inserts the marker). - P7: generalized "partial per-file protection" → "not fully idempotent on redelivery" so the rationale can't rot. - P5: documented created_at as observability-only (future age-based sweep). - P2: REQUIRE_PG_TESTS env → skip becomes fail, so the idempotency primitive can't ship untested-green in CI where PG is expected. - P3: strengthened the race test — pre-build N=8 conns in the main thread, align claims with threading.Barrier, loop 5 trials (forces the contended ON CONFLICT path instead of a serial fast-path). - P8: hoisted import os to module top. 35 pg_barrier + 9 dedup tests green; migration applied to dev DB, makemigrations --check clean; ruff clean. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .../migrations/0006_pgbatchdedup_and_more.py | 47 +++++ backend/pg_queue/models.py | 61 +++++++ workers/queue_backend/pg_barrier.py | 48 +++++ workers/tests/test_pg_batch_dedup.py | 169 ++++++++++++++++++ 4 files changed, 325 insertions(+) create mode 100644 backend/pg_queue/migrations/0006_pgbatchdedup_and_more.py create mode 100644 workers/tests/test_pg_batch_dedup.py diff --git a/backend/pg_queue/migrations/0006_pgbatchdedup_and_more.py b/backend/pg_queue/migrations/0006_pgbatchdedup_and_more.py new file mode 100644 index 0000000000..44ad09e4cb --- /dev/null +++ b/backend/pg_queue/migrations/0006_pgbatchdedup_and_more.py @@ -0,0 +1,47 @@ +# Generated by Django 4.2.1 on 2026-06-17 09:18 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("pg_queue", "0005_pgorchestratorlock"), + ] + + operations = [ + migrations.CreateModel( + name="PgBatchDedup", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("execution_id", models.TextField()), + ("batch_index", models.IntegerField()), + ("created_at", models.DateTimeField(default=django.utils.timezone.now)), + ], + options={ + "db_table": "pg_batch_dedup", + }, + ), + migrations.AddConstraint( + model_name="pgbatchdedup", + constraint=models.UniqueConstraint( + fields=("execution_id", "batch_index"), + name="pg_batch_dedup_exec_batch_uniq", + ), + ), + migrations.AddConstraint( + model_name="pgbatchdedup", + constraint=models.CheckConstraint( + check=models.Q(("batch_index__gte", 0)), + name="pg_batch_dedup_batch_index_non_negative", + ), + ), + ] diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py index bb086c7783..079508619c 100644 --- a/backend/pg_queue/models.py +++ b/backend/pg_queue/models.py @@ -114,6 +114,67 @@ class Meta: ] +class PgBatchDedup(models.Model): + """Per-batch idempotency marker for the at-least-once PG execution pipeline. + + The PG queue is at-least-once: a ``process_file_batch`` task can be + redelivered after a crash-before-ack, which would re-run the batch and + double-decrement the barrier (``process_file_batch`` is non-idempotent — + ``max_retries=0``, and the destination push is not fully idempotent on + redelivery). This table is the durable dedup token: one row per + ``(execution_id, batch_index)``. + + The worker claims a batch by inserting its row atomically + (``INSERT … ON CONFLICT DO NOTHING RETURNING``): a fresh insert means + "first delivery — the caller proceeds and performs the single barrier + decrement"; a conflict means "already claimed — a redelivery → caller + skips". Pairing the claim with the decrement is what keeps the barrier + decremented exactly once per batch even though the queue may deliver the + task more than once. Rows for an execution are cleared when its barrier + finalises (``remaining`` → 0, via ``clear_execution_batches``). + + Orphan reclaim — executions that never finalise currently **leak** their + dedup markers: the reaper's ``sweep_expired_barriers`` deletes only the + ``pg_barrier_state`` row (there is no FK / cascade to this table), so the + matching markers are left behind. A dedup-orphan sweep (e.g. by ``created_at`` + age) is intended future work; until then the leak is bounded only by how many + executions never finalise. + + Composite uniqueness on ``(execution_id, batch_index)`` is enforced by a + ``UniqueConstraint`` (Django has no composite primary key here); the worker's + ``ON CONFLICT (execution_id, batch_index)`` targets exactly those columns, + and the constraint's index — ``execution_id`` leading — also serves the + per-execution cleanup ``DELETE``. Managed=True / generated migration, no + DB-side function, extension-free (UN-3533) — same posture as the siblings. + """ + + execution_id = models.TextField() + batch_index = models.IntegerField() + # Observability/debugging only today (when a marker was claimed); not read, + # indexed, or used for reclaim. The intended future dedup-orphan sweep (see + # the class docstring) would sweep by this column's age and add an index then. + created_at = models.DateTimeField(default=timezone.now) + + class Meta: + db_table = "pg_batch_dedup" + constraints = [ + # The claim's ON CONFLICT target and the one writer-proof invariant + # (the worker SQL can't import this model): a batch is claimed at + # most once per execution. + models.UniqueConstraint( + fields=["execution_id", "batch_index"], + name="pg_batch_dedup_exec_batch_uniq", + ), + # batch_index is a non-negative ordinal; a negative value is a + # programming error. Writer-proof, mirroring PgQueueMessage.priority's + # domain-bound CheckConstraint precedent. + models.CheckConstraint( + check=models.Q(batch_index__gte=0), + name="pg_batch_dedup_batch_index_non_negative", + ), + ] + + class PgBarrierState(models.Model): """Per-execution fan-in barrier state for ``PgBarrier`` (the Postgres ``WORKER_BARRIER_BACKEND``). diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.py index 5ed2e7d021..ba477dfd49 100644 --- a/workers/queue_backend/pg_barrier.py +++ b/workers/queue_backend/pg_barrier.py @@ -123,6 +123,54 @@ def _delete_barrier(execution_id: str) -> None: ) +def claim_batch(execution_id: str, batch_index: int) -> bool: + """Claim ``(execution_id, batch_index)`` for processing — the per-batch + idempotency gate for the at-least-once PG pipeline. + + Atomically inserts the dedup marker (``pg_batch_dedup``); returns ``True`` + if THIS call inserted the row (first delivery — the caller should proceed + and perform the single barrier decrement) and ``False`` if the row already + existed (a redelivery — the caller should skip). This function itself only + inserts the marker; pairing it with the caller's decrement is what keeps the + barrier decremented exactly once per batch. ``ON CONFLICT DO NOTHING`` makes + the check-and-set a single statement, so concurrent redeliveries of the same + batch resolve to exactly one ``True`` (the row's existence is the token) with + no race. + + Inert in 9e PR 2b — wired into ``process_file_batch`` (claim at batch start) + in PR 2c, alongside the in-body decrement. The companion + :func:`clear_execution_batches` reclaims the markers at barrier finalise. + """ + with _cursor() as cur: + cur.execute( + "INSERT INTO pg_batch_dedup (execution_id, batch_index, created_at) " + "VALUES (%s, %s, now()) " + "ON CONFLICT (execution_id, batch_index) DO NOTHING " + "RETURNING execution_id", + (execution_id, batch_index), + ) + return cur.fetchone() is not None + + +def clear_execution_batches(execution_id: str) -> int: + """Delete all per-batch dedup markers for an execution; returns the count. + + Called at barrier teardown (``remaining`` → 0) so the markers live exactly + as long as the execution needs them. Executions that never finalise + currently leak their markers — the reaper sweeps the orphaned + ``pg_barrier_state`` row but does NOT yet reclaim ``pg_batch_dedup`` (no + cascade); a dedup-orphan sweep is intended future work (see the + ``PgBatchDedup`` model docstring). Safe to call when no rows exist (returns + 0). Uses the ``(execution_id, batch_index)`` constraint index (``execution_id`` + leading) for the lookup. + + Inert in 9e PR 2b — wired into the barrier-finalise path in PR 2c. + """ + with _cursor() as cur: + cur.execute("DELETE FROM pg_batch_dedup WHERE execution_id = %s", (execution_id,)) + return cur.rowcount + + @dataclass(frozen=True, slots=True) class _PgBarrierHandle: """Minimal ``BarrierHandle`` — ``id`` is the execution id (what call sites diff --git a/workers/tests/test_pg_batch_dedup.py b/workers/tests/test_pg_batch_dedup.py new file mode 100644 index 0000000000..78b7a6e894 --- /dev/null +++ b/workers/tests/test_pg_batch_dedup.py @@ -0,0 +1,169 @@ +"""Per-batch idempotency markers (9e PR 2b) — ``claim_batch`` / +``clear_execution_batches`` against the real ``pg_batch_dedup`` table. + +The PG queue is at-least-once, so ``process_file_batch`` can be redelivered; the +marker makes the barrier decrement fire exactly once per ``(execution_id, +batch_index)``. These tests pin the claim's check-and-set semantics (incl. a +concurrent race) and the teardown cleanup. Inert in 2b — PR 2c wires them into +the batch task / barrier-finalise path. + +The fixture mirrors ``test_pg_barrier``'s ``barrier_db``: a real autocommit +connection injected into pg_barrier's thread-local (the helpers live there to +share the one connection per worker child). Skips if Postgres is unreachable or +the 0006 migration is unapplied. +""" + +from __future__ import annotations + +import os + +import psycopg2 +import pytest +from queue_backend import pg_barrier +from queue_backend.pg_barrier import claim_batch, clear_execution_batches +from queue_backend.pg_queue.connection import create_pg_connection + + +def _skip_or_fail(reason: str) -> None: + """Skip when Postgres isn't available — UNLESS ``REQUIRE_PG_TESTS`` is set + (CI where PG is expected), where a skip would silently ship the idempotency + primitive untested, so we fail loudly instead. + """ + if os.getenv("REQUIRE_PG_TESTS"): + pytest.fail(f"REQUIRE_PG_TESTS set but PG unusable: {reason}") + pytest.skip(reason) + + +@pytest.fixture +def dedup_db(): + os.environ.setdefault("TEST_DB_HOST", "127.0.0.1") + try: + conn = create_pg_connection(env_prefix="TEST_DB_") + except psycopg2.OperationalError as exc: + _skip_or_fail(f"Postgres not reachable: {exc}") + conn.autocommit = True + with conn.cursor() as cur: + cur.execute("SELECT to_regclass('pg_batch_dedup')") + if cur.fetchone()[0] is None: + conn.close() + _skip_or_fail("pg_batch_dedup migration not applied (run backend migrate)") + cur.execute("DELETE FROM pg_batch_dedup") + pg_barrier._local.conn = conn + yield conn + with conn.cursor() as cur: + cur.execute("DELETE FROM pg_batch_dedup") + conn.close() + pg_barrier._local.conn = None + + +def _count(conn, execution_id): + with conn.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s", + (execution_id,), + ) + return cur.fetchone()[0] + + +class TestClaimBatch: + def test_first_claim_succeeds(self, dedup_db): + assert claim_batch("exec-1", 0) is True + assert _count(dedup_db, "exec-1") == 1 + + def test_redelivery_of_same_batch_is_rejected(self, dedup_db): + assert claim_batch("exec-1", 0) is True + # Same (execution_id, batch_index) again = a redelivery → skip. + assert claim_batch("exec-1", 0) is False + assert _count(dedup_db, "exec-1") == 1 # no duplicate row + + def test_different_batch_index_is_a_distinct_claim(self, dedup_db): + assert claim_batch("exec-1", 0) is True + assert claim_batch("exec-1", 1) is True # sibling batch, independent + assert _count(dedup_db, "exec-1") == 2 + + def test_same_batch_index_different_execution_is_distinct(self, dedup_db): + assert claim_batch("exec-1", 0) is True + assert claim_batch("exec-2", 0) is True # the key is the (exec, idx) pair + assert _count(dedup_db, "exec-1") == 1 + assert _count(dedup_db, "exec-2") == 1 + + def test_negative_batch_index_is_rejected_by_db(self, dedup_db): + # batch_index is a non-negative ordinal; the CheckConstraint is + # writer-proof even though claim_batch's callers only pass >= 0. + with pytest.raises(psycopg2.errors.CheckViolation): + claim_batch("exec-neg", -1) + + def test_concurrent_claims_resolve_to_exactly_one_winner(self, dedup_db): + # The real at-least-once guarantee: N redeliveries racing on the SAME + # batch must produce exactly one True (the row's existence is the token). + # To actually force the contended ON CONFLICT path (not let a fast first + # thread connect+commit before the others start), pre-build all N + # connections in the main thread and release every claim simultaneously + # via a threading.Barrier. Repeat over several distinct batches so a flaky + # non-atomic check-then-insert can't pass by luck. Each thread uses its + # own connection (a libpq conn is not thread-safe across threads). + import threading + + n = 8 + conns = [create_pg_connection(env_prefix="TEST_DB_") for _ in range(n)] + for c in conns: + c.autocommit = True + try: + for trial in range(5): + batch_index = trial # a fresh, uncontended-so-far batch each round + gate = threading.Barrier(n) + results: list[bool] = [] + lock = threading.Lock() + + # Bind loop vars as defaults (ruff B023): the threads are created + # and joined within this iteration, but defaults make the capture + # explicit and lint-clean. + def run(conn, bi=batch_index, gate=gate, results=results, lock=lock): + pg_barrier._local.conn = conn + try: + gate.wait() # align all claims at the same instant + won = claim_batch("exec-race", bi) + with lock: + results.append(won) + finally: + pg_barrier._local.conn = None + + threads = [threading.Thread(target=run, args=(c,)) for c in conns] + for t in threads: + t.start() + for t in threads: + t.join() + + assert sum(results) == 1, f"trial {trial}: {sum(results)} winners" + assert _count(dedup_db, "exec-race") == trial + 1 + finally: + for c in conns: + c.close() + + +class TestClearExecutionBatches: + def test_clears_only_the_target_execution(self, dedup_db): + claim_batch("exec-keep", 0) + claim_batch("exec-drop", 0) + claim_batch("exec-drop", 1) + + removed = clear_execution_batches("exec-drop") + + assert removed == 2 # both of exec-drop's markers + assert _count(dedup_db, "exec-drop") == 0 + assert _count(dedup_db, "exec-keep") == 1 # untouched + + def test_clear_with_no_rows_returns_zero(self, dedup_db): + assert clear_execution_batches("never-claimed") == 0 + + def test_cleared_batch_can_be_reclaimed(self, dedup_db): + # After teardown clears the markers, a fresh execution reusing the same + # id (or a re-run) can claim again — the gate is per live execution. + assert claim_batch("exec-1", 0) is True + assert claim_batch("exec-1", 0) is False + clear_execution_batches("exec-1") + assert claim_batch("exec-1", 0) is True # claimable again post-cleanup + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From b3f25d9e74236f3603f5f9e271945ea8286de66e Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:56:56 +0530 Subject: [PATCH 18/44] =?UTF-8?q?UN-3563=20[FEAT]=20PG=20Queue=209e=20PR?= =?UTF-8?q?=202c=20=E2=80=94=20live=20PG=20fan-out/barrier/callback=20(fir?= =?UTF-8?q?e-and-forget)=20(#2069)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3563 [FEAT] PG Queue 9e PR 2c — live PG fan-out/barrier/callback (fire-and-forget) Wires the coupled pipeline's fan-out → barrier → callback onto the PG queue for a transport=="pg_queue" execution. Gated: resolve_transport() still returns celery (PR3 Flipt flips it), so the whole PG branch is present-but-unreachable — default path byte-identical. Orchestrator task (async_execute_bin) stays on Celery (hybrid); routing it onto PG is a 2d follow-up. - barrier.py: Barrier Protocol + CeleryChordBarrier/RedisDecrBarrier accept (and ignore) a `transport` param; CallbackDescriptor gains an optional `backend`. - orchestration_utils._barrier_for_transport: pg_queue → fresh PgBarrier() (bypasses the WORKER_BARRIER_BACKEND singleton), else the singleton. - pg_barrier.PgBarrier.enqueue(transport): pg_queue → fire-and-forget mode — _dispatch_header_pg sends each header via dispatch(backend=PG) with an injected _barrier_context {execution_id, batch_index, callback_descriptor}, no .link; descriptor marked backend=pg_queue; UPSERT block also clears pg_batch_dedup (greptile #2068 reuse-reset). _fire_barrier_callback self-chains the callback onto PG when backend==pg_queue. clear_execution_batches at finalise + abort. run_batch_with_barrier(): claim → work → in-body _barrier_pg_decrement; redelivery skips; exception → barrier_pg_abort. - file_processing.process_file_batch(_barrier_context=None): core routes None → _run_batch_stages (celery chord path), else → run_batch_with_barrier. - general/api fan-outs thread transport into create_chord_execution. Tests: +8 PgBarrier fire-and-forget + 2 orchestration routing + 2 process_file_batch routing. Each test file green alone; ruff clean. End-to-end forced-pg dev-test pending before PR. Co-Authored-By: Claude Opus 4.8 * UN-3563 fix SonarCloud S1172: drop unused task_instance from _run_batch_stages The extracted _run_batch_stages never uses task_instance — its only purpose (deriving celery_task_id) happens in _process_file_batch_core before the call. Removed the param + updated both call sites. _process_file_batch_core keeps task_instance (it reads .request.id). Routing test mocks with *a, unaffected. Co-Authored-By: Claude Opus 4.8 * UN-3563 address review (muhammad-ali-e, 15): strand-on-failure hardening + typing/dedup/docs/tests Decision (with reviewer): reaper-as-safety-net for the un-catchable strand windows + fix what's catchable + document + gate on PR3. Failure handling: - [#69 Critical] run_batch_with_barrier wraps BOTH work + decrement in the abort: a decrement-side failure (guard / DB / last-batch callback dispatch) tears the barrier down in-body instead of stranding to expiry. - [#79] extracted _abort_barrier_in_body — logs when the teardown itself fails (was silently suppressed under a misleading "torn down" message). - [#74/#81] documented the two un-catchable strand windows (hard-crash-during-work, post-commit callback-dispatch-fail) as a HARD reaper dependency for PR3. - [#86] finalise cleanup split into independent try/excepts with distinct logs. Typing / clarity: - [#01] BarrierContext(TypedDict) for _barrier_context (header fan-out, run_batch_with_barrier, process_file_batch). - [#03] renamed CallbackDescriptor "backend" -> "transport" (WorkflowTransport value; avoids the QueueBackend "pg" collision). - [#27] is_pg_transport() predicate in core; used in orchestration_utils + pg_barrier. - [#20] extracted _dispatch_pg() — single home for cycle-avoiding local import + backend=PG. - [#35] normalize_transport() at the general worker entry (parity w/ api/scheduler). - [#94] log when a header has no queue option. - [#09/#13] fixed born-stale comment + kwargs-not-args docstring. Tests (+#37/#41): last-batch self-chains callback to PG + cleans up barrier/dedup; decrement-failure aborts; PG-branch mid-loop dispatch-failure deletes row; header args/queue/pre-existing-kwargs preservation. 137 barrier/dedup/routing tests green; bootstrap clean under WORKER_BARRIER_BACKEND=pg. Co-Authored-By: Claude Opus 4.8 * UN-3563 fix run_batch_with_barrier strand-window doc inconsistency (review) The second "NOT catchable" bullet conflated two different things: it described the in-body catchable abort ("the abort here removes the row") and a *software* callback-dispatch failure — but that failure is already caught + torn down by step 3's wrap (paragraph 1), so it doesn't belong under the un-catchable heading, and on the PG path _fire_barrier_callback IS the enqueue so "committed but before the enqueue" couldn't both hold. Rewrote the bullet to the genuinely un-catchable window: a hard crash BETWEEN the decrement committing (remaining→0) and the callback enqueue completing — decrement committed (redelivery blocked by the marker), process gone before the callback enqueues or any abort runs, row survives to expiry, reaper-only recovery. Explicitly notes a software dispatch failure is the catchable case. Keeps this list an accurate spec for the PR-3 reaper-recovery dependency. Co-Authored-By: Claude Opus 4.8 * UN-3563 address greptile (#2069, 2): clear dedup on mid-loop PG failure + carry fairness on PG callback Both in the gated PG path (greptile 4/5, safe to merge). - Issue 1: PgBarrier.enqueue mid-loop dispatch-failure handler now also calls clear_execution_batches on the PG path. Earlier headers may have committed a claim_batch marker; with the barrier row deleted, their in-flight barrier_pg_abort is a no-op (already_aborted) and never reaches the clear inside it, so reclaim the markers directly here. - Issue 2: the PG callback now carries the producer's fairness. Added _fairness_from_headers() to reconstruct the FairnessKey from the stored x-fairness-key headers and pass it to _dispatch_pg, so the callback rides the same org/priority as the Celery path (was always default priority). Tests: +fairness-carried / +fairness-none-safe on _fire_barrier_callback; extended the PG mid-loop test to assert an already-claimed marker is reclaimed. 75 barrier/dedup tests green; bootstrap clean under WORKER_BARRIER_BACKEND=pg. Co-Authored-By: Claude Opus 4.8 * UN-3563 fix SonarCloud S3776: reduce PgBarrier.enqueue cognitive complexity (17→under 15) Extracted the per-header dispatch loop into PgBarrier._dispatch_headers — the deeply-nested for→try/except→if/else→if (PG-vs-celery branch + mid-loop failure teardown + PG dedup-clear) was the complexity driver. enqueue now calls the helper; behaviour identical. radon: enqueue C(11)→B(6); ruff C901 passes. 75 barrier/dedup tests green; ruff + ruff-format clean. Co-Authored-By: Claude Opus 4.8 * UN-3563 fix greptile #2069: mid-loop dedup-clear test passed for the wrong reason The pre-seeded claim_batch marker was wiped by enqueue's UPSERT block (the reuse-reset DELETE) before the dispatch loop, so the mid-loop clear_execution_batches deleted 0 rows — the count==0 assertion passed on the UPSERT, not the guard under test. Now the first dispatch side-effect claims the marker AFTER the UPSERT (simulating a fast PG consumer), so the mid-loop clear is what removes it. Verified: with the clear disabled the marker orphans (count=1). Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .../core/src/unstract/core/data_models.py | 11 + workers/api-deployment/tasks.py | 17 +- workers/file_processing/tasks.py | 77 ++- workers/general/tasks.py | 10 + workers/queue_backend/barrier.py | 35 +- workers/queue_backend/pg_barrier.py | 437 +++++++++++++++--- workers/queue_backend/redis_barrier.py | 8 +- .../workflow/execution/orchestration_utils.py | 30 +- workers/tests/test_barrier.py | 36 ++ workers/tests/test_chord_callback_boundary.py | 40 ++ workers/tests/test_pg_barrier.py | 286 ++++++++++++ 11 files changed, 894 insertions(+), 93 deletions(-) diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py index 0336647570..20e3d0c3aa 100644 --- a/unstract/core/src/unstract/core/data_models.py +++ b/unstract/core/src/unstract/core/data_models.py @@ -249,6 +249,17 @@ def normalize_transport(value: object, *, logger: Any = None, context: str = "") return DEFAULT_WORKFLOW_TRANSPORT +def is_pg_transport(transport: str | None) -> bool: + """True if ``transport`` is the Postgres-queue transport. + + Single source for "what counts as PG transport" — centralises the + ``== WorkflowTransport.PG_QUEUE.value`` comparison scattered across the + worker fan-out / barrier code, and the seam to extend if a second + PG-family transport is ever added. + """ + return transport == WorkflowTransport.PG_QUEUE.value + + class FileListingResult: """Result of listing files from a source.""" diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index f4bec94a73..1aafe70bad 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -26,7 +26,13 @@ from shared.workflow.execution.tool_validation import validate_workflow_tool_instances from worker import app -from unstract.core.data_models import ExecutionStatus, FileHashData, WorkerFileData +from unstract.core.data_models import ( + DEFAULT_WORKFLOW_TRANSPORT, + ExecutionStatus, + FileHashData, + WorkerFileData, + normalize_transport, +) from unstract.core.worker_models import ApiDeploymentResultStatus logger = WorkerLogger.get_logger(__name__) @@ -683,6 +689,14 @@ def _run_workflow_api( org_id=str(schema_name), workload_type=WorkloadType.API, ) + # Transport rides in via the dispatched task's kwargs (PR 1 seam); the + # fan-out honours it (PG path → fire-and-forget PgBarrier). Fail-closed + # to celery on any unrecognised value. + transport = normalize_transport( + kwargs.get("transport", DEFAULT_WORKFLOW_TRANSPORT), + logger=logger, + context=f" [exec:{execution_id}]", + ) result = WorkflowOrchestrationUtils.create_chord_execution( batch_tasks=batch_tasks, callback_task_name="process_batch_callback_api", @@ -694,6 +708,7 @@ def _run_workflow_api( callback_queue=file_processing_callback_queue, app_instance=app, fairness=api_fairness, + transport=transport, ) if result is None: diff --git a/workers/file_processing/tasks.py b/workers/file_processing/tasks.py index 3aa93296e7..5afb41ab54 100644 --- a/workers/file_processing/tasks.py +++ b/workers/file_processing/tasks.py @@ -11,6 +11,8 @@ from typing import Any from queue_backend import worker_task +from queue_backend.barrier import BarrierContext +from queue_backend.pg_barrier import run_batch_with_barrier # Import shared worker infrastructure from shared.api import InternalAPIClient @@ -222,25 +224,16 @@ def _enhance_batch_with_mrq_flags( ) -def _process_file_batch_core( - task_instance, file_batch_data: dict[str, Any] +def _run_batch_stages( + file_batch_data: dict[str, Any], celery_task_id: str ) -> dict[str, Any]: - """Core implementation of file batch processing. - - This function contains the actual processing logic that both the new task - and Django compatibility task will use. + """The actual batch work (validate → setup → pre-create → process → compile). - Args: - task_instance: The Celery task instance (self) - file_batch_data: Dictionary that will be converted to FileBatchData dataclass - - Returns: - Dictionary with successful_files and failed_files counts + Transport-agnostic: identical on the Celery chord path and the PG + fire-and-forget path. The task instance isn't needed here — its only use + (deriving ``celery_task_id``) happens in the caller. Returns the + JSON-serialisable batch result. """ - celery_task_id = ( - task_instance.request.id if hasattr(task_instance, "request") else "unknown" - ) - # Step 1: Validate and parse input data batch_data = _validate_and_parse_batch_data(file_batch_data) @@ -260,6 +253,44 @@ def _process_file_batch_core( return _compile_batch_result(context) +def _process_file_batch_core( + task_instance, + file_batch_data: dict[str, Any], + barrier_context: BarrierContext | None = None, +) -> dict[str, Any]: + """Core implementation of file batch processing. + + This function contains the actual processing logic that both the new task + and Django compatibility task will use. + + Args: + task_instance: The Celery task instance (self) + file_batch_data: Dictionary that will be converted to FileBatchData dataclass + barrier_context: Present only on the 9e PG fire-and-forget path — carries + ``execution_id`` / ``batch_index`` / ``callback_descriptor`` so the + batch claims its slot and runs the barrier decrement in-body (a + PG-consumed task fires no Celery ``.link``). ``None`` on the Celery + chord path, where the chord ``.link`` drives the decrement instead. + + Returns: + Dictionary with successful_files and failed_files counts + """ + celery_task_id = ( + task_instance.request.id if hasattr(task_instance, "request") else "unknown" + ) + + if barrier_context is None: + # Celery chord path — the chord's .link runs the decrement after this. + return _run_batch_stages(file_batch_data, celery_task_id) + + # PG fire-and-forget path — claim the batch (idempotent on redelivery), run + # the stages, then decrement the barrier in-body / self-chain the callback. + return run_batch_with_barrier( + barrier_context, + lambda: _run_batch_stages(file_batch_data, celery_task_id), + ) + + @worker_task( bind=True, name=TaskName.PROCESS_FILE_BATCH, @@ -272,18 +303,26 @@ def _process_file_batch_core( # Timeout inherited from global Celery config (FILE_PROCESSING_TASK_TIME_LIMIT env var) ) @monitor_performance -def process_file_batch(self, file_batch_data: dict[str, Any]) -> dict[str, Any]: - """Process a batch of files in parallel using Celery. +def process_file_batch( + self, + file_batch_data: dict[str, Any], + _barrier_context: BarrierContext | None = None, +) -> dict[str, Any]: + """Process a batch of files in parallel. This is the main task entry point for new workers. Args: file_batch_data: Dictionary that will be converted to FileBatchData dataclass + _barrier_context: Injected only when this task is dispatched onto the PG + queue (9e fire-and-forget path) by ``PgBarrier`` — carries the barrier + coordination context (``execution_id`` / ``batch_index`` / + ``callback_descriptor``). Absent on the Celery chord path. Returns: Dictionary with successful_files and failed_files counts """ - return _process_file_batch_core(self, file_batch_data) + return _process_file_batch_core(self, file_batch_data, _barrier_context) def _validate_and_parse_batch_data(file_batch_data: dict[str, Any]) -> FileBatchData: diff --git a/workers/general/tasks.py b/workers/general/tasks.py index 8d9e6e4bf4..1986a0e0cf 100644 --- a/workers/general/tasks.py +++ b/workers/general/tasks.py @@ -56,6 +56,7 @@ FileBatchData, FileHashData, WorkerFileData, + normalize_transport, ) # Import common workflow utilities @@ -488,6 +489,12 @@ def _execute_general_workflow( """ start_time = time.time() + # Fail-closed coercion, for parity with the api/scheduler workers (which run + # normalize_transport at their entry): a typo'd transport degrades to Celery + # with a warning rather than silently routing onto an unknown substrate. The + # coerced value feeds both WorkflowContextData and the fan-out below. + transport = normalize_transport(transport, logger=logger) + logger.info("Executing general workflow logic for ETL/TASK workflow") try: @@ -710,6 +717,7 @@ def _execute_general_workflow( execution_mode=execution_mode, use_file_history=use_file_history, organization_id=api_client.organization_id, + transport=transport, **kwargs, ) @@ -764,6 +772,7 @@ def _orchestrate_file_processing_general( execution_mode: tuple | None, use_file_history: bool, organization_id: str, + transport: str = DEFAULT_WORKFLOW_TRANSPORT, **kwargs: dict[str, Any], ) -> dict[str, Any]: """Orchestrate file processing for general workflows using the same pattern as API worker. @@ -957,6 +966,7 @@ def _orchestrate_file_processing_general( org_id=organization_id, workload_type=WorkloadType.NON_API, ), + transport=transport, ) if not result: diff --git a/workers/queue_backend/barrier.py b/workers/queue_backend/barrier.py index d56348ce42..0e838242ca 100644 --- a/workers/queue_backend/barrier.py +++ b/workers/queue_backend/barrier.py @@ -36,10 +36,12 @@ import logging import os -from typing import TYPE_CHECKING, Any, Protocol, TypedDict +from typing import TYPE_CHECKING, Any, NotRequired, Protocol, TypedDict from celery import chord +from unstract.core.data_models import DEFAULT_WORKFLOW_TRANSPORT + from .fairness import FairnessKey from .handle import BarrierHandle @@ -92,6 +94,29 @@ class CallbackDescriptor(TypedDict): kwargs: dict[str, Any] queue: str fairness_headers: dict[str, Any] | None + # 9e: the WorkflowTransport the aggregating callback is fired on when the + # barrier completes. Absent / ``None`` → legacy Celery dispatch + # (``current_app.apply_async`` — the ``.link`` path). ``"pg_queue"`` → the + # fire-and-forget PG path self-chains the callback via dispatch onto PG. + # Named ``transport`` (a WorkflowTransport value, e.g. ``"pg_queue"``) — NOT + # ``backend``, to avoid confusion with ``QueueBackend`` (``"pg"``). + transport: NotRequired[str | None] + + +class BarrierContext(TypedDict): + """Per-batch barrier coordination injected into a PG-dispatched header task. + + The 9e fire-and-forget sibling of :class:`CallbackDescriptor`, and typed for + the same reason: it crosses producer → PG-queue → consumer, so the contract + is pinned here to catch a typo/rename at the type layer rather than as a + remote ``KeyError`` mid-batch. Carried as the ``_barrier_context`` kwarg on + ``process_file_batch`` so the consumer can claim its slot and run the barrier + decrement in-body (a PG-consumed task fires no Celery ``.link``). + """ + + execution_id: str + batch_index: int + callback_descriptor: CallbackDescriptor class Barrier(Protocol): @@ -112,9 +137,15 @@ def enqueue( callback_queue: str, app_instance: Any, fairness: FairnessKey | None = None, + transport: str = DEFAULT_WORKFLOW_TRANSPORT, ) -> BarrierHandle | None: """Enqueue ``header_tasks`` and a single callback to fire on completion. + ``transport`` is the per-execution transport (9e). Only ``PgBarrier`` + acts on it (``pg_queue`` → fire-and-forget PG fan-out instead of Celery + ``.link``); the Celery/Redis substrates accept it for Protocol parity + and ignore it (they are only ever reached on the ``celery`` transport). + ``None`` is the **sole signal** that no work was enqueued — it is returned exclusively when ``header_tasks`` is empty. Any substrate-level failure (broker outage, serialisation error, @@ -166,8 +197,10 @@ def enqueue( callback_queue: str, app_instance: Any, fairness: FairnessKey | None = None, + transport: str = DEFAULT_WORKFLOW_TRANSPORT, ) -> BarrierHandle | None: """See :class:`Barrier.enqueue`.""" + del transport # Celery chord is only reached on the celery transport. # Empty-header guard goes FIRST so a zero-task run skips # signature construction / fairness serialisation entirely. # Any failure in those paths is now constrained to the diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.py index ba477dfd49..956ce09a05 100644 --- a/workers/queue_backend/pg_barrier.py +++ b/workers/queue_backend/pg_barrier.py @@ -60,16 +60,27 @@ import json import logging import threading -from collections.abc import Iterator +from collections.abc import Callable, Iterator from dataclasses import dataclass from typing import TYPE_CHECKING, Any import psycopg2 import psycopg2.extensions -from .barrier import CallbackDescriptor, barrier_ttl_seconds +from unstract.core.data_models import ( + DEFAULT_WORKFLOW_TRANSPORT, + WorkflowTransport, + is_pg_transport, +) + +from .barrier import BarrierContext, CallbackDescriptor, barrier_ttl_seconds from .decorator import worker_task -from .fairness import FairnessKey +from .fairness import ( + DEFAULT_PRIORITY, + FAIRNESS_HEADER_NAME, + FairnessKey, + WorkloadType, +) from .handle import BarrierHandle from .pg_queue.connection import create_pg_connection @@ -137,8 +148,8 @@ def claim_batch(execution_id: str, batch_index: int) -> bool: batch resolve to exactly one ``True`` (the row's existence is the token) with no race. - Inert in 9e PR 2b — wired into ``process_file_batch`` (claim at batch start) - in PR 2c, alongside the in-body decrement. The companion + Called on the in-body PG path from :func:`run_batch_with_barrier` (claim at + batch start), alongside the in-body decrement. The companion :func:`clear_execution_batches` reclaims the markers at barrier finalise. """ with _cursor() as cur: @@ -164,13 +175,45 @@ def clear_execution_batches(execution_id: str) -> int: 0). Uses the ``(execution_id, batch_index)`` constraint index (``execution_id`` leading) for the lookup. - Inert in 9e PR 2b — wired into the barrier-finalise path in PR 2c. + Called from the barrier-finalise + abort paths. """ with _cursor() as cur: cur.execute("DELETE FROM pg_batch_dedup WHERE execution_id = %s", (execution_id,)) return cur.rowcount +def _dispatch_pg( + task_name: str, + *, + args: list[Any] | None = None, + kwargs: dict[str, Any] | None = None, + queue: str | None, + fairness: FairnessKey | None = None, +) -> Any: + """Enqueue a task onto the PG queue (the one place that owns the cycle-avoiding + local imports + the ``backend=QueueBackend.PG`` argument). + + Both the header fan-out (:meth:`PgBarrier._dispatch_header_pg`) and the + self-chained callback (:func:`_fire_barrier_callback`) route through here. + Returns the ``dispatch`` handle. ``queue`` is required (may be ``None`` only + if the caller has already logged the fallback) — a ``None`` queue makes + ``dispatch`` fall back to its default PG queue. + """ + # Local import: dispatch/routing pull in queue plumbing that imports the + # barrier package — importing at module load would be a cycle. + from .dispatch import dispatch + from .routing import QueueBackend + + return dispatch( + task_name, + args=args, + kwargs=kwargs, + queue=queue, + fairness=fairness, + backend=QueueBackend.PG, + ) + + @dataclass(frozen=True, slots=True) class _PgBarrierHandle: """Minimal ``BarrierHandle`` — ``id`` is the execution id (what call sites @@ -197,15 +240,26 @@ def enqueue( callback_queue: str, app_instance: Any, fairness: FairnessKey | None = None, + transport: str = DEFAULT_WORKFLOW_TRANSPORT, ) -> BarrierHandle | None: """See :class:`queue_backend.barrier.Barrier.enqueue`. Empty ``header_tasks`` → ``None`` (caller owns the zero-files contract); any substrate failure raises. ``app_instance`` is accepted for Protocol - parity but unused — the callback is built inside the link task via - ``current_app`` so it runs against the worker's app. + parity but unused. + + ``transport`` selects the fan-out mode (9e): + + - ``celery`` (default, the ``WORKER_BARRIER_BACKEND=pg`` legacy path): + header tasks are Celery-dispatched with ``.link(barrier_pg_decr_and_check)`` + / ``.link_error(barrier_pg_abort)``; the link drives the decrement. + - ``pg_queue`` (the fire-and-forget path): header tasks are dispatched onto + the PG queue (``dispatch(backend=PG)``) with a ``_barrier_context`` kwarg + carrying ``execution_id`` / ``batch_index`` / ``callback_descriptor`` — + a PG-consumed task fires no ``.link``, so it claims its batch and runs + the decrement in-body, self-chaining the callback at ``remaining`` → 0. """ - del app_instance # Protocol parity; callback built in the link task. + del app_instance # Protocol parity; callback dispatched by the decrement. if not header_tasks: logger.info( f"[exec:{callback_kwargs.get('execution_id')}] " @@ -223,6 +277,7 @@ def enqueue( ) execution_id = str(execution_id) + is_pg = is_pg_transport(transport) try: fairness_headers = fairness.as_header() if fairness else None callback_descriptor: CallbackDescriptor = { @@ -231,6 +286,10 @@ def enqueue( "queue": callback_queue, "fairness_headers": fairness_headers, } + if is_pg: + # The decrement (run in-body on the PG path) reads this to + # self-chain the callback onto PG rather than Celery. + callback_descriptor["transport"] = WorkflowTransport.PG_QUEUE.value ttl_seconds = barrier_ttl_seconds() with _cursor() as cur: @@ -249,38 +308,29 @@ def enqueue( " created_at = now(), expires_at = EXCLUDED.expires_at", (execution_id, len(header_tasks), ttl_seconds), ) + # Reset per-batch dedup markers from a prior run reusing this + # execution_id, ATOMICALLY with the barrier reset. Without this a + # re-enqueue would leave stale markers → every in-body claim_batch + # returns False → all batches skip → barrier hangs to expiry. + # (Markers are written by claim_batch() on the in-body PG path.) + cur.execute( + "DELETE FROM pg_batch_dedup WHERE execution_id = %s", + (execution_id,), + ) - link_signature = barrier_pg_decr_and_check.s( + self._dispatch_headers( + header_tasks, + is_pg=is_pg, execution_id=execution_id, callback_descriptor=callback_descriptor, + fairness=fairness, + fairness_headers=fairness_headers, ) - link_error_signature = barrier_pg_abort.s(execution_id=execution_id) - for i, task in enumerate(header_tasks): - try: - cloned = task.clone() - if fairness_headers: - cloned.set(headers=fairness_headers) - cloned.link(link_signature) - cloned.link_error(link_error_signature) - cloned.apply_async() - except Exception: - # Mid-loop dispatch failure: i of N never reached the broker, - # so the counter can't reach 0. Delete the row so in-flight - # links' decrement finds no row and cleans up; re-raise so the - # caller marks the workflow ERROR. - with contextlib.suppress(Exception): - _delete_barrier(execution_id) - logger.exception( - f"[exec:{execution_id}] apply_async failed at task " - f"{i}/{len(header_tasks)}; barrier row deleted to prevent " - f"spurious callback fires from the orphan tasks' links." - ) - raise logger.info( - f"Barrier enqueued via PgBarrier — exec_id={execution_id}, " - f"header_tasks={len(header_tasks)}, callback={callback_task_name}, " - f"queue={callback_queue}" + f"Barrier enqueued via PgBarrier ({transport}) — " + f"exec_id={execution_id}, header_tasks={len(header_tasks)}, " + f"callback={callback_task_name}, queue={callback_queue}" ) return _PgBarrierHandle(id=execution_id) @@ -294,6 +344,163 @@ def enqueue( ) raise + def _dispatch_headers( + self, + header_tasks: list[Signature], + *, + is_pg: bool, + execution_id: str, + callback_descriptor: CallbackDescriptor, + fairness: FairnessKey | None, + fairness_headers: dict[str, Any] | None, + ) -> None: + """Dispatch the N header tasks, one transport or the other. + + PG path (``is_pg``) → fire-and-forget onto the PG queue via + :meth:`_dispatch_header_pg` (no ``.link``). Celery path → ``.link`` / + ``.link_error`` chord-style. On any mid-loop dispatch failure, ``i`` of N + never reached the queue so the counter can't reach 0 — delete the barrier + row (and, on the PG path, reclaim dedup markers an earlier header may have + committed, since the in-flight ``barrier_pg_abort`` is a no-op once the row + is gone) so an in-flight decrement finds nothing, then re-raise. + """ + link_signature = barrier_pg_decr_and_check.s( + execution_id=execution_id, callback_descriptor=callback_descriptor + ) + link_error_signature = barrier_pg_abort.s(execution_id=execution_id) + for i, task in enumerate(header_tasks): + try: + if is_pg: + self._dispatch_header_pg( + task, i, execution_id, callback_descriptor, fairness + ) + else: + cloned = task.clone() + if fairness_headers: + cloned.set(headers=fairness_headers) + cloned.link(link_signature) + cloned.link_error(link_error_signature) + cloned.apply_async() + except Exception: + with contextlib.suppress(Exception): + _delete_barrier(execution_id) + if is_pg: + with contextlib.suppress(Exception): + clear_execution_batches(execution_id) + logger.exception( + f"[exec:{execution_id}] header dispatch failed at task " + f"{i}/{len(header_tasks)}; barrier row deleted to prevent " + f"spurious callback fires from the orphan tasks." + ) + raise + + @staticmethod + def _dispatch_header_pg( + task: Signature, + batch_index: int, + execution_id: str, + callback_descriptor: CallbackDescriptor, + fairness: FairnessKey | None, + ) -> None: + """Dispatch one header task onto the PG queue (fire-and-forget mode). + + Unpacks the Celery ``Signature`` (the fan-out built it as + ``app.signature(name, kwargs={batch_files, batch_index, total_batches}, + queue=...)`` — the batch payload is in ``kwargs``) and re-dispatches it via + :func:`_dispatch_pg` with an added ``_barrier_context`` kwarg. The PG + consumer runs the task; it claims ``(execution_id, batch_index)`` and runs + the decrement in-body (no ``.link``). ``fairness`` carries org/priority + onto the row exactly as the bare-dispatch sites do. + """ + barrier_context: BarrierContext = { + "execution_id": execution_id, + "batch_index": batch_index, + "callback_descriptor": callback_descriptor, + } + header_kwargs = dict(task.kwargs or {}) + header_kwargs["_barrier_context"] = barrier_context + queue = task.options.get("queue") + if queue is None: + # The fan-out always sets a queue; a missing one would silently route + # to dispatch's default PG queue (off the intended file-processing + # queue), so surface it rather than let it slip by. + logger.warning( + f"[exec:{execution_id}] header task {task.task!r} (batch " + f"{batch_index}) has no queue option — falling back to the default " + f"PG queue; the consumer for the intended queue won't see it." + ) + _dispatch_pg( + task.task, + args=list(task.args or ()), + kwargs=header_kwargs, + queue=queue, + fairness=fairness, + ) + + +def _fairness_from_headers( + fairness_headers: dict[str, Any] | None, +) -> FairnessKey | None: + """Reconstruct a :class:`FairnessKey` from the stored ``x-fairness-key`` header. + + The descriptor carries the wire-shape headers (``FairnessKey.as_header()``), + but the PG dispatch path wants the ``FairnessKey`` itself (``dispatch`` writes + ``org_id`` + ``pipeline_priority`` onto the row). Reconstructing keeps the PG + callback at the producer's org/priority — parity with the Celery path, which + applies the headers directly. ``None`` when the producer attached no key. + """ + payload = (fairness_headers or {}).get(FAIRNESS_HEADER_NAME) + if not payload: + return None + return FairnessKey( + org_id=payload.get("org_id"), + workload_type=WorkloadType(payload["workload_type"]), + pipeline_priority=payload.get("pipeline_priority", DEFAULT_PRIORITY), + ) + + +def _fire_barrier_callback( + callback_descriptor: CallbackDescriptor, all_results: list[Any] +) -> str: + """Dispatch the aggregating callback when the barrier completes; return its id. + + Two transports, selected by the descriptor's ``transport`` marker: + + - ``"pg_queue"`` (9e fire-and-forget PG path): self-chain the callback onto + the PG queue via :func:`_dispatch_pg`, carrying the producer's fairness + (reconstructed from the stored headers) so the callback rides the same + org/priority as the Celery path — no Celery, so the whole execution stays + off the broker. + - absent / anything else (legacy ``.link`` path): dispatch via + ``current_app.signature(...).apply_async()`` — byte-identical to pre-9e, + preserving the ``fairness_headers`` the producer attached. + """ + if is_pg_transport(callback_descriptor.get("transport")): + handle = _dispatch_pg( + callback_descriptor["task_name"], + args=[all_results], + kwargs=callback_descriptor["kwargs"], + queue=callback_descriptor["queue"], + fairness=_fairness_from_headers(callback_descriptor.get("fairness_headers")), + ) + return str(handle.id) + + from celery import current_app + + # Build the callback-signature kwargs explicitly (headers only when truthy) — + # matching CeleryChordBarrier's idiom, no spurious headers=None. + signature_kwargs: dict[str, Any] = { + "args": [all_results], + "kwargs": callback_descriptor["kwargs"], + "queue": callback_descriptor["queue"], + } + if callback_descriptor.get("fairness_headers"): + signature_kwargs["headers"] = callback_descriptor["fairness_headers"] + callback_signature = current_app.signature( + callback_descriptor["task_name"], **signature_kwargs + ) + return str(callback_signature.apply_async().id) + def _barrier_pg_decrement( result: Any, @@ -321,8 +528,6 @@ def _barrier_pg_decrement( mid-transaction (see the guard below). The Celery wrapper additionally pins ``max_retries=0``; an orphaned barrier is bounded by ``expires_at`` instead. """ - from celery import current_app - # Enforce the "own committed transaction" contract loudly. A caller that # invokes this inside an already-open transaction on the shared thread-local # connection would hold the row lock across the call and let the outer txn's @@ -406,47 +611,46 @@ def _barrier_pg_decrement( return {"status": "abandoned", "remaining": remaining} # remaining == 0: we are the last task. psycopg2 decodes the jsonb array to - # a Python list already — no per-element json.loads needed. Build the kwargs - # explicitly (headers only when truthy) — clearer than an inline ** spread, - # matching CeleryChordBarrier's idiom. - signature_kwargs: dict[str, Any] = { - "args": [all_results], - "kwargs": callback_descriptor["kwargs"], - "queue": callback_descriptor["queue"], - } - if callback_descriptor.get("fairness_headers"): - signature_kwargs["headers"] = callback_descriptor["fairness_headers"] - callback_signature = current_app.signature( - callback_descriptor["task_name"], **signature_kwargs - ) - # Dispatch FIRST; delete the row only after dispatch succeeds, so a callback - # apply_async failure leaves the row (and its expiry) in place rather than - # stranding the execution with no state and no recovery path. - callback_result = callback_signature.apply_async() - # The post-dispatch delete must not mask the successful dispatch: the callback - # already fired, so a delete error here is logged (not raised) — the row - # lingers until expiry rather than re-running the callback. No double-fire is - # possible: this is the last decrement (remaining hit 0) and max_retries=0, so - # the link task is never replayed. + # a Python list already — no per-element json.loads needed. + # + # Dispatch the callback FIRST; delete the row only after dispatch succeeds, so + # a callback dispatch failure leaves the row (and its expiry) in place rather + # than stranding the execution with no state and no recovery path. No + # double-fire is possible: this is the last decrement (remaining hit 0) and + # max_retries=0, so the task is never replayed. + callback_id = _fire_barrier_callback(callback_descriptor, all_results) + # Post-dispatch cleanup is best-effort: the callback already fired, so a + # failure here is logged (not raised) — the rows linger until reclaim rather + # than re-running the callback. The two cleanups are independent so one + # failing doesn't skip the other and each names what failed: the barrier row + # is reclaimed by expiry, the dedup markers by the (PR-3-blocking) dedup-orphan + # sweep — neither re-runs anything (this is the last decrement, max_retries=0). try: _delete_barrier(execution_id) except Exception: logger.exception( f"[exec:{execution_id}] Barrier callback dispatched " - f"(callback_task_id={callback_result.id}) but the post-dispatch row " - f"delete failed — row will be reclaimed at expiry. Callback NOT " - f"re-run (max_retries=0)." + f"(callback_task_id={callback_id}) but deleting the pg_barrier_state " + f"row failed — reclaimed at expiry. Callback NOT re-run." + ) + try: + clear_execution_batches(execution_id) + except Exception: + logger.exception( + f"[exec:{execution_id}] Barrier callback dispatched " + f"(callback_task_id={callback_id}) but clearing pg_batch_dedup markers " + f"failed — reclaimed by the dedup-orphan sweep. Callback NOT re-run." ) logger.info( f"[exec:{execution_id}] Barrier complete — fired callback " f"{callback_descriptor['task_name']} on {callback_descriptor['queue']} " f"with {len(all_results)} aggregated results " - f"(callback_task_id={callback_result.id})" + f"(callback_task_id={callback_id})" ) return { "status": "complete", - "callback_task_id": callback_result.id, + "callback_task_id": callback_id, "aggregated_count": len(all_results), } @@ -510,8 +714,113 @@ def barrier_pg_abort( # Another failure already aborted this execution (or it's already gone). return {"status": "already_aborted", "execution_id": execution_id} + # We won the abort: reclaim the per-batch dedup markers too (best-effort — + # the barrier is already torn down). A leftover marker would otherwise linger + # until the future dedup-orphan sweep; it can never re-run anything. + with contextlib.suppress(Exception): + clear_execution_batches(execution_id) + logger.error( f"[exec:{execution_id}] PgBarrier aborted — a header task failed; " f"barrier state cleaned up (aggregating callback will not fire)." ) return {"status": "aborted", "execution_id": execution_id} + + +def _abort_barrier_in_body(execution_id: str, *, reason: str) -> None: + """Tear the barrier down from the in-body PG path (no ``.link_error`` here). + + Best-effort: a batch failure and a DB failure are correlated, so the abort + itself can fail — that's logged (not swallowed silently) so a stuck barrier + isn't masked by a misleading "torn down" message. A failed teardown bottoms + out at the barrier's ``expires_at`` and is reclaimed by the reaper. + """ + try: + barrier_pg_abort(execution_id=execution_id) + logger.error( + f"[exec:{execution_id}] {reason} — barrier teardown attempted in-body " + f"(no .link_error on the PG path); aggregating callback won't fire." + ) + except Exception: + logger.exception( + f"[exec:{execution_id}] {reason} AND the in-body barrier teardown " + f"itself failed — barrier will hang until expiry and be reclaimed by " + f"the reaper (max_retries=0, no replay)." + ) + + +def run_batch_with_barrier( + barrier_context: BarrierContext, work_fn: Callable[[], dict[str, Any]] +) -> dict[str, Any]: + """Run a fire-and-forget PG-path batch under the barrier protocol (9e PR 2c). + + A PG-consumed header task fires no Celery ``.link``, so the barrier + coordination runs in-body here, given the ``_barrier_context`` that + :meth:`PgBarrier._dispatch_header_pg` injected (``execution_id`` / + ``batch_index`` / ``callback_descriptor``): + + 1. **Claim** ``(execution_id, batch_index)``. A redelivery (at-least-once + queue) finds the marker already set → returns a no-op result WITHOUT + re-running the work or re-decrementing — exactly-once decrement. + 2. **Run** ``work_fn()`` (the batch). + 3. **Decrement** the barrier with the batch result; the task that drives + ``remaining`` → 0 self-chains the aggregating callback onto PG. + + Steps 2 *and* 3 are wrapped: any catchable failure (work error, the decrement + guard / DB error, or the last-batch callback dispatch failing) tears the + barrier down in-body via :func:`barrier_pg_abort` and re-raises, so it fails + fast instead of hanging to expiry — mirroring the chord path's ``.link_error``. + + **Strand windows the reaper must cover (NOT catchable here — hard merge + dependency for enabling the flag in PR 3, see the module / PgBatchDedup docs):** + + - A hard crash / SIGKILL / visibility-timeout expiry *during* ``work_fn()`` + leaves the dedup marker committed (claim is step 1), so redelivery returns + ``skipped_redelivery`` and never re-runs the batch — the barrier hangs to + ``expires_at``. (Claiming before the work avoids re-processing on the common + redelivery case at the cost of this crash window.) + - A hard crash *between* the decrement committing (``remaining`` → 0) and the + callback enqueue completing: the decrement is committed (so redelivery is + blocked by the marker) but the process is gone before the callback is + enqueued and before any in-body abort can run, so the ``pg_barrier_state`` + row survives to ``expires_at`` with no callback ever fired. (A *software* + callback-dispatch failure here is NOT this window — that's catchable and is + torn down by step 3's wrap above.) + + For both, the **reaper** (sweep stranded ``pg_barrier_state``, mark the + execution ERROR / re-drive, reclaim ``pg_batch_dedup``) is the recovery net; + it must be live before PR 3 flips ``pg_queue_execution_enabled``. + + ``work_fn`` must return the JSON-serialisable batch result (the same dict the + Celery chord path returns), since the decrement appends it to the barrier's + aggregated results that the callback receives. + """ + execution_id = str(barrier_context["execution_id"]) + batch_index = int(barrier_context["batch_index"]) + + if not claim_batch(execution_id, batch_index): + logger.info( + f"[exec:{execution_id}] batch {batch_index} already claimed — " + f"redelivery, skipping (barrier already decremented for this batch)." + ) + return { + "status": "skipped_redelivery", + "execution_id": execution_id, + "batch_index": batch_index, + } + + # Wrap BOTH the work and the decrement: a decrement-side failure (the + # open-transaction guard, a json/DB error, or the last-batch callback dispatch + # raising) must also tear the barrier down, else the marker is committed, + # redelivery is blocked, and the barrier strands to expiry. + try: + result = work_fn() + _barrier_pg_decrement( + result, + execution_id=execution_id, + callback_descriptor=barrier_context["callback_descriptor"], + ) + except Exception: + _abort_barrier_in_body(execution_id, reason=f"batch {batch_index} failed") + raise + return result diff --git a/workers/queue_backend/redis_barrier.py b/workers/queue_backend/redis_barrier.py index 346e44f8b7..2c1e2fb3ee 100644 --- a/workers/queue_backend/redis_barrier.py +++ b/workers/queue_backend/redis_barrier.py @@ -69,6 +69,7 @@ from celery.canvas import Signature from unstract.core.cache.redis_client import create_redis_client +from unstract.core.data_models import DEFAULT_WORKFLOW_TRANSPORT from .barrier import ( _DEFAULT_BARRIER_TTL_SECONDS, @@ -313,10 +314,13 @@ def enqueue( callback_queue: str, app_instance: Any, fairness: FairnessKey | None = None, + transport: str = DEFAULT_WORKFLOW_TRANSPORT, ) -> BarrierHandle | None: """See :class:`queue_backend.barrier.Barrier.enqueue`. - Mirrors ``CeleryChordBarrier.enqueue`` semantics: + ``transport`` is accepted for Protocol parity and ignored — the Redis + barrier is the Celery-transport fan-in substrate (the PG transport uses + ``PgBarrier``'s fire-and-forget mode). Mirrors ``CeleryChordBarrier``: - Empty ``header_tasks`` → ``None`` (caller handles the zero-files contract); no Redis keys touched. @@ -337,7 +341,7 @@ def enqueue( """ # Explicit ``del`` documents the Protocol-parity intent and # satisfies static linters' unused-parameter check. - del app_instance + del app_instance, transport if not header_tasks: execution_id = callback_kwargs.get("execution_id") pipeline_id = callback_kwargs.get("pipeline_id") diff --git a/workers/shared/workflow/execution/orchestration_utils.py b/workers/shared/workflow/execution/orchestration_utils.py index 5b7167774c..560af7b91d 100644 --- a/workers/shared/workflow/execution/orchestration_utils.py +++ b/workers/shared/workflow/execution/orchestration_utils.py @@ -10,6 +10,9 @@ from typing import TYPE_CHECKING, Any from queue_backend import BarrierHandle, FairnessKey, get_barrier +from queue_backend.pg_barrier import PgBarrier + +from unstract.core.data_models import DEFAULT_WORKFLOW_TRANSPORT, is_pg_transport from ...enums import FileDestinationType, PipelineType from ...enums.worker_enums import QueueName @@ -17,18 +20,31 @@ if TYPE_CHECKING: from celery.canvas import Signature + from queue_backend import Barrier logger = WorkerLogger.get_logger(__name__) # Single ``Barrier`` instance reused across all -# ``WorkflowOrchestrationUtils.create_chord_execution`` calls. The -# substrate is selected at module-import (worker startup) time by the -# ``WORKER_BARRIER_BACKEND`` env var (default ``chord``). Flag flips -# require a pod restart — same posture as every other ``WORKER_*`` -# env in the codebase. +# ``WorkflowOrchestrationUtils.create_chord_execution`` calls on the **celery** +# transport. The substrate is selected at module-import (worker startup) time by +# the ``WORKER_BARRIER_BACKEND`` env var (default ``chord``). Flag flips require a +# pod restart — same posture as every other ``WORKER_*`` env in the codebase. _BARRIER = get_barrier() +def _barrier_for_transport(transport: str) -> Barrier: + """Pick the fan-in substrate for a per-execution transport (9e). + + The ``pg_queue`` transport always coordinates on Postgres — it uses a fresh + :class:`PgBarrier` in its fire-and-forget mode regardless of + ``WORKER_BARRIER_BACKEND`` (the env-selected singleton is the *celery*-transport + substrate, which may be ``chord``). Every other transport uses that singleton. + """ + if is_pg_transport(transport): + return PgBarrier() + return _BARRIER + + class WorkflowOrchestrationUtils: """Centralized workflow orchestration patterns and utilities.""" @@ -41,6 +57,7 @@ def create_chord_execution( app_instance: Any, *, fairness: FairnessKey | None = None, + transport: str = DEFAULT_WORKFLOW_TRANSPORT, ) -> BarrierHandle | None: """Standardized fan-out + callback pattern (Phase 6 ``Barrier``). @@ -71,13 +88,14 @@ def create_chord_execution( parent that direct pipeline status updates should be handled instead. """ - return _BARRIER.enqueue( + return _barrier_for_transport(transport).enqueue( batch_tasks, callback_task_name=callback_task_name, callback_kwargs=callback_kwargs, callback_queue=callback_queue, app_instance=app_instance, fairness=fairness, + transport=transport, ) @staticmethod diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.py index e61bc4a182..c35b088bd8 100644 --- a/workers/tests/test_barrier.py +++ b/workers/tests/test_barrier.py @@ -392,6 +392,8 @@ def test_create_chord_execution_delegates_to_module_barrier(self): fairness=fairness, ) + # Default transport (celery) → the env-selected singleton, with the + # per-execution transport threaded through to the substrate (9e). mock_barrier.enqueue.assert_called_once_with( batch, callback_task_name="process_batch_callback_api", @@ -399,6 +401,40 @@ def test_create_chord_execution_delegates_to_module_barrier(self): callback_queue="api_file_processing_callback", app_instance=app_mock, fairness=fairness, + transport="celery", + ) + + def test_pg_queue_transport_bypasses_singleton_for_pgbarrier(self): + # 9e: a pg_queue execution must coordinate on Postgres regardless of + # WORKER_BARRIER_BACKEND, so it uses a fresh PgBarrier — NOT the (possibly + # chord) singleton. Pin that the singleton's enqueue is never called. + from shared.workflow.execution import orchestration_utils + from shared.workflow.execution.orchestration_utils import ( + WorkflowOrchestrationUtils, + ) + + app_mock = MagicMock(name="celery_app") + batch = [MagicMock(name="h1")] + + with ( + patch.object(orchestration_utils, "_BARRIER") as mock_singleton, + patch.object(orchestration_utils, "PgBarrier") as mock_pgbarrier_cls, + ): + WorkflowOrchestrationUtils.create_chord_execution( + batch_tasks=batch, + callback_task_name="process_batch_callback", + callback_kwargs={"execution_id": "exec-pg"}, + callback_queue="general", + app_instance=app_mock, + transport="pg_queue", + ) + + mock_singleton.enqueue.assert_not_called() # NOT the env singleton + mock_pgbarrier_cls.assert_called_once_with() # a fresh PgBarrier + mock_pgbarrier_cls.return_value.enqueue.assert_called_once() + assert ( + mock_pgbarrier_cls.return_value.enqueue.call_args.kwargs["transport"] + == "pg_queue" ) diff --git a/workers/tests/test_chord_callback_boundary.py b/workers/tests/test_chord_callback_boundary.py index 524708a3e5..088609420a 100644 --- a/workers/tests/test_chord_callback_boundary.py +++ b/workers/tests/test_chord_callback_boundary.py @@ -767,5 +767,45 @@ def test_aggregator_consumes_multi_batch(self): assert aggregated["batches_processed"] == 2 +class TestProcessFileBatchTransportRouting: + """9e PR 2c: ``_process_file_batch_core`` routes by ``barrier_context`` — + Celery chord path runs the stages directly (the chord ``.link`` decrements); + the PG path wraps them in ``run_batch_with_barrier`` (in-body claim + decrement). + """ + + def test_celery_path_runs_stages_directly(self, monkeypatch): + from file_processing import tasks as tasks_mod + + monkeypatch.setattr(tasks_mod, "_run_batch_stages", lambda *a: {"r": "celery"}) + barrier_calls = [] + monkeypatch.setattr( + tasks_mod, + "run_batch_with_barrier", + lambda *a, **k: barrier_calls.append(1), + ) + out = tasks_mod._process_file_batch_core(MagicMock(), {"fb": 1}, None) + assert out == {"r": "celery"} + assert barrier_calls == [] # barrier path NOT taken on the celery transport + + def test_pg_path_routes_through_barrier(self, monkeypatch): + from file_processing import tasks as tasks_mod + + monkeypatch.setattr(tasks_mod, "_run_batch_stages", lambda *a: {"r": "work"}) + captured = {} + + def fake_run_batch_with_barrier(ctx, work_fn): + captured["ctx"] = ctx + return {"via": "barrier", "work": work_fn()} + + monkeypatch.setattr( + tasks_mod, "run_batch_with_barrier", fake_run_batch_with_barrier + ) + ctx = {"execution_id": "e", "batch_index": 0, "callback_descriptor": {}} + out = tasks_mod._process_file_batch_core(MagicMock(), {"fb": 1}, ctx) + assert captured["ctx"] is ctx + assert out["via"] == "barrier" + assert out["work"] == {"r": "work"} # work_fn runs the real stages + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/workers/tests/test_pg_barrier.py b/workers/tests/test_pg_barrier.py index c2a9488560..238bbeeabc 100644 --- a/workers/tests/test_pg_barrier.py +++ b/workers/tests/test_pg_barrier.py @@ -24,10 +24,24 @@ from queue_backend.pg_barrier import ( PgBarrier, _barrier_pg_decrement, + _fire_barrier_callback, barrier_pg_abort, barrier_pg_decr_and_check, + claim_batch, + run_batch_with_barrier, ) from queue_backend.pg_queue.connection import create_pg_connection +from queue_backend.routing import QueueBackend + + +def _pg_header(task_name="process_file_batch", args=None, queue="file_processing"): + """A Celery-Signature-shaped stub for the PG-fan-out path (.task/.args/.kwargs/.options).""" + sig = MagicMock(name="pg_header_signature") + sig.task = task_name + sig.args = args if args is not None else [{"file": "f1"}] + sig.kwargs = {} + sig.options = {"queue": queue} + return sig _CALLBACK = { "task_name": "process_batch_callback_api", @@ -478,5 +492,277 @@ def test_expires_at_must_exceed_created_at(self, barrier_db): ) +class TestPgFireAndForgetMode: + """9e PR 2c — PgBarrier's ``transport="pg_queue"`` fire-and-forget path: + headers dispatched onto PG (no ``.link``), in-body claim + decrement, callback + self-chained onto PG, and dedup-marker cleanup at enqueue/finalise/abort. + """ + + def test_enqueue_pg_mode_dispatches_headers_via_pg_with_context(self, barrier_db): + with barrier_db.cursor() as cur: + cur.execute("DELETE FROM pg_batch_dedup") + # A header with a pre-existing kwarg, real args, and a queue — to prove + # the Signature→dispatch unpacking preserves all three (a dropped args or + # queue would silently process an empty/misrouted batch). + h0 = _pg_header(args=[{"file": "f0"}], queue="api_file_processing") + h0.kwargs = {"pre_existing": "keep"} + h1 = _pg_header(args=[{"file": "f1"}], queue="api_file_processing") + with patch("queue_backend.dispatch.dispatch") as mock_dispatch: + PgBarrier().enqueue( + [h0, h1], + callback_task_name="process_batch_callback", + callback_kwargs={"execution_id": "exec-pg"}, + callback_queue="general", + app_instance=None, + transport="pg_queue", + ) + assert mock_dispatch.call_count == 2 # one per header, no .link + for i, call in enumerate(mock_dispatch.call_args_list): + assert call.kwargs["backend"] is QueueBackend.PG + assert call.kwargs["args"] == [{"file": f"f{i}"}] # args preserved + assert call.kwargs["queue"] == "api_file_processing" # queue preserved + ctx = call.kwargs["kwargs"]["_barrier_context"] + assert ctx["execution_id"] == "exec-pg" + assert ctx["batch_index"] == i + assert ctx["callback_descriptor"]["transport"] == "pg_queue" + # The pre-existing kwarg on h0 survives alongside the injected context. + assert mock_dispatch.call_args_list[0].kwargs["kwargs"]["pre_existing"] == "keep" + + def test_enqueue_pg_mode_clears_stale_dedup_on_reuse(self, barrier_db): + # greptile #2068: a re-enqueue with the same execution_id must wipe prior + # dedup markers atomically with the barrier reset, else every claim_batch + # returns False and the barrier hangs. + with barrier_db.cursor() as cur: + cur.execute("DELETE FROM pg_batch_dedup") + claim_batch("exec-reuse", 0) + claim_batch("exec-reuse", 1) + with patch("queue_backend.dispatch.dispatch"): + PgBarrier().enqueue( + [_pg_header()], + callback_task_name="process_batch_callback", + callback_kwargs={"execution_id": "exec-reuse"}, + callback_queue="general", + app_instance=None, + transport="pg_queue", + ) + with barrier_db.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s", + ("exec-reuse",), + ) + assert cur.fetchone()[0] == 0 # stale markers wiped by the UPSERT block + + def test_run_batch_with_barrier_first_delivery_runs_and_decrements(self, barrier_db): + with barrier_db.cursor() as cur: + cur.execute("DELETE FROM pg_batch_dedup") + _seed(barrier_db, "exec-fd", 2) + ctx = { + "execution_id": "exec-fd", + "batch_index": 0, + "callback_descriptor": _CALLBACK, + } + work = MagicMock(return_value={"ok": 1}) + out = run_batch_with_barrier(ctx, work) + work.assert_called_once() + assert out == {"ok": 1} + remaining, results = _row(barrier_db, "exec-fd") + assert remaining == 1 + assert results == [{"ok": 1}] + + def test_run_batch_with_barrier_redelivery_skips_work_and_decrement(self, barrier_db): + with barrier_db.cursor() as cur: + cur.execute("DELETE FROM pg_batch_dedup") + _seed(barrier_db, "exec-re", 2) + claim_batch("exec-re", 0) # batch already claimed → this is a redelivery + ctx = { + "execution_id": "exec-re", + "batch_index": 0, + "callback_descriptor": _CALLBACK, + } + work = MagicMock(return_value={"ok": 1}) + out = run_batch_with_barrier(ctx, work) + work.assert_not_called() # no reprocessing + assert out["status"] == "skipped_redelivery" + remaining, _ = _row(barrier_db, "exec-re") + assert remaining == 2 # NOT decremented again + + def test_run_batch_with_barrier_exception_aborts_barrier(self, barrier_db): + with barrier_db.cursor() as cur: + cur.execute("DELETE FROM pg_batch_dedup") + _seed(barrier_db, "exec-err", 2) + ctx = { + "execution_id": "exec-err", + "batch_index": 0, + "callback_descriptor": _CALLBACK, + } + + def boom(): + raise RuntimeError("batch failed") + + with pytest.raises(RuntimeError, match="batch failed"): + run_batch_with_barrier(ctx, boom) + # No .link_error on the PG path → torn down in-body (mirror chord abort). + assert _row(barrier_db, "exec-err") is None + + def test_fire_barrier_callback_pg_self_chains_via_dispatch(self): + descriptor = {**_CALLBACK, "transport": "pg_queue"} + with patch("queue_backend.dispatch.dispatch") as mock_dispatch: + mock_dispatch.return_value = MagicMock(id="pg-cb-1") + cb_id = _fire_barrier_callback(descriptor, [{"r": 1}]) + assert cb_id == "pg-cb-1" + assert mock_dispatch.call_args.kwargs["backend"] is QueueBackend.PG + assert mock_dispatch.call_args.kwargs["args"] == [[{"r": 1}]] + + def test_fire_barrier_callback_pg_carries_fairness(self): + # greptile: the PG callback must ride the producer's org/priority (parity + # with the Celery path), reconstructed from the stored fairness_headers. + descriptor = { + **_CALLBACK, + "transport": "pg_queue", + "fairness_headers": { + FAIRNESS_HEADER_NAME: { + "org_id": "org-9", + "workload_type": "api", + "pipeline_priority": 8, + } + }, + } + with patch("queue_backend.dispatch.dispatch") as mock_dispatch: + mock_dispatch.return_value = MagicMock(id="pg-cb") + _fire_barrier_callback(descriptor, [{"r": 1}]) + fairness = mock_dispatch.call_args.kwargs["fairness"] + assert fairness is not None + assert fairness.org_id == "org-9" + assert fairness.pipeline_priority == 8 + + def test_fire_barrier_callback_pg_without_fairness_passes_none(self): + # No producer key → None (dispatch writes neutral defaults), not a crash. + descriptor = {**_CALLBACK, "transport": "pg_queue"} # fairness_headers None + with patch("queue_backend.dispatch.dispatch") as mock_dispatch: + mock_dispatch.return_value = MagicMock(id="pg-cb") + _fire_barrier_callback(descriptor, [{"r": 1}]) + assert mock_dispatch.call_args.kwargs["fairness"] is None + + def test_fire_barrier_callback_legacy_uses_celery(self): + # No backend marker → the .link-mode Celery dispatch (unchanged). + with patch("celery.current_app.signature") as sig: + sig.return_value.apply_async.return_value = MagicMock(id="celery-cb") + cb_id = _fire_barrier_callback(_CALLBACK, [{"r": 1}]) + assert cb_id == "celery-cb" + sig.assert_called_once() + + def test_abort_clears_dedup_markers(self, barrier_db): + with barrier_db.cursor() as cur: + cur.execute("DELETE FROM pg_batch_dedup") + _seed(barrier_db, "exec-ab", 2) + claim_batch("exec-ab", 0) + barrier_pg_abort(execution_id="exec-ab") + with barrier_db.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s", + ("exec-ab",), + ) + assert cur.fetchone()[0] == 0 + + def test_last_batch_self_chains_callback_to_pg_and_cleans_up(self, barrier_db): + # The PR's core promise, end to end: the batch that drives remaining → 0 + # self-chains the aggregating callback onto PG (backend=PG, args=[results]) + # and clears both the barrier row and the dedup markers. + with barrier_db.cursor() as cur: + cur.execute("DELETE FROM pg_batch_dedup") + _seed(barrier_db, "exec-last", 1) # this batch is the last → remaining → 0 + descriptor = { + "task_name": "process_batch_callback_api", + "kwargs": {"execution_id": "exec-last"}, + "queue": "api_file_processing_callback", + "fairness_headers": None, + "transport": "pg_queue", + } + ctx = { + "execution_id": "exec-last", + "batch_index": 0, + "callback_descriptor": descriptor, + } + with patch("queue_backend.dispatch.dispatch") as mock_dispatch: + mock_dispatch.return_value = MagicMock(id="pg-cb") + out = run_batch_with_barrier(ctx, lambda: {"successful_files": 1}) + assert out == {"successful_files": 1} + # callback self-chained onto PG with the aggregated results + assert mock_dispatch.call_args.kwargs["backend"] is QueueBackend.PG + assert mock_dispatch.call_args.args[0] == "process_batch_callback_api" + assert mock_dispatch.call_args.kwargs["args"] == [[{"successful_files": 1}]] + # barrier row + dedup markers cleaned up at finalise + assert _row(barrier_db, "exec-last") is None + with barrier_db.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s", + ("exec-last",), + ) + assert cur.fetchone()[0] == 0 + + def test_decrement_failure_aborts_barrier(self, barrier_db): + # #69: a decrement-side failure (after the work succeeded) must tear the + # barrier down in-body — else the dedup marker is committed, redelivery is + # blocked, and the barrier strands to expiry. + with barrier_db.cursor() as cur: + cur.execute("DELETE FROM pg_batch_dedup") + _seed(barrier_db, "exec-decfail", 2) + ctx = { + "execution_id": "exec-decfail", + "batch_index": 0, + "callback_descriptor": _CALLBACK, + } + with patch.object( + pg_barrier, "_barrier_pg_decrement", side_effect=RuntimeError("decr boom") + ): + with pytest.raises(RuntimeError, match="decr boom"): + run_batch_with_barrier(ctx, lambda: {"ok": 1}) + assert _row(barrier_db, "exec-decfail") is None # barrier torn down + + def test_pg_mid_loop_dispatch_failure_deletes_row_and_clears_dedup(self, barrier_db): + # The PG-branch counterpart of test_mid_loop_dispatch_failure_deletes_row: + # a header PG-dispatch failure mid fan-out deletes the barrier row AND + # reclaims dedup markers (greptile #2069) — an already-claimed earlier + # header's marker would otherwise orphan, since the in-flight abort is a + # no-op once the barrier row is gone. + # + # The marker must be claimed AFTER enqueue's UPSERT block (which wipes + # stale markers for this execution_id) — else the assertion would pass on + # the UPSERT, not the mid-loop clear under test. So the first dispatch + # call claims it (a fast consumer on the PG path), the second fails. + with barrier_db.cursor() as cur: + cur.execute("DELETE FROM pg_batch_dedup") + + def dispatch_side_effect(*args, **kwargs): + if not dispatch_side_effect.claimed: + dispatch_side_effect.claimed = True + claim_batch("exec-midfail", 0) # marker created post-UPSERT + return MagicMock(id="1") + raise RuntimeError("broker down") + + dispatch_side_effect.claimed = False + with patch( + "queue_backend.dispatch.dispatch", side_effect=dispatch_side_effect + ): + with pytest.raises(RuntimeError, match="broker down"): + PgBarrier().enqueue( + [_pg_header(), _pg_header()], + callback_task_name="process_batch_callback", + callback_kwargs={"execution_id": "exec-midfail"}, + callback_queue="general", + app_instance=None, + transport="pg_queue", + ) + assert _row(barrier_db, "exec-midfail") is None # row deleted on failure + with barrier_db.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s", + ("exec-midfail",), + ) + # The marker existed post-UPSERT and was removed by the mid-loop + # clear_execution_batches under test (not by the UPSERT reset). + assert cur.fetchone()[0] == 0 + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From c872b3417a403328c5e7fda227e5c289351da794 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Wed, 17 Jun 2026 22:22:26 +0530 Subject: [PATCH 19/44] =?UTF-8?q?UN-3564=20[FEAT]=20PG=20Queue=209e=20?= =?UTF-8?q?=E2=80=94=20reaper-recovery:=20mark=20stranded=20executions=20E?= =?UTF-8?q?RROR=20(gates=20PR=203)=20(#2070)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3564 [FEAT] PG Queue 9e — reaper-recovery: mark stranded executions ERROR The reaper only DELETEd expired pg_barrier_state rows — a stranded execution vanished silently (stuck in EXECUTING). This makes it RECOVER them — the hard dependency from PR 2c review for enabling the PG transport (un-catchable strand windows otherwise bottom out at the ~6h barrier expiry). Per expired barrier the leader (recover_expired_barriers), best-effort + per-exec: - Marks the execution ERROR via the internal API (the path the normal callback uses — business state never goes direct-DB; the API is functional: execution_time, error truncation, attempts, events/notifications, multi-tenant boundary). Message distinguishes remaining>0 (work incomplete) vs remaining==0 (callback never fired). Reads status first and SKIPS if already terminal (a remaining==0 row can be a COMPLETED exec whose row-delete failed; update_execution has no terminal guard) or if the row has no org. - Reclaims pg_batch_dedup + pg_barrier_state directly in PG (same boundary as the rest of queue_backend). Recover-THEN-delete: a failed mark leaves the row for the next sweep to retry (single-leader → no double-claim). - backend: organization_id column on PgBarrierState + migration 0007 (reaper needs it for the org-scoped status API). - workers: PgBarrier.enqueue stamps organization_id into the UPSERT; PgReaper holds a lazily-built InternalAPIClient; sweep_expired_barriers → recover_expired_barriers. Tests: reaper suite reworked — real-PG recovery w/ fake API client (mark-ERROR remaining>0/==0, terminal-skip no-overwrite, org-missing skip, API-failure leaves row, dedup reclaim, tick-via-real-conn). 109 reaper/barrier/dedup tests green. Dev-tested vs real InternalAPIClient+backend: PENDING stranded → ERROR + cleaned; COMPLETED → NOT overwritten + cleaned (terminal-guard verified end-to-end). Deferred: callback re-fire for remaining==0 (needs callback_descriptor on the row). Co-Authored-By: Claude Opus 4.8 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * UN-3564 fix SonarCloud S6741: avoid IndexError-prone update_calls[0] in reaper tests Replaced `api.update_calls[0]` (which Sonar can't prove non-empty → IndexError risk) with single-element unpacking `(call,) = api.update_calls` — removes the index and additionally asserts exactly one status mark was made. Behaviour identical; 52 reaper tests green. Co-Authored-By: Claude Opus 4.8 * UN-3564 address review (muhammad-ali-e, 11): fix read-failure→false-ERROR (Critical) + hardening - [Critical] _execution_status now RAISES when the status read fails (get_workflow_execution returns success=False, never raises) — a transient blip no longer reads as "non-terminal" and flips a COMPLETED execution to ERROR; the caller's except retains the row for retry. + test. - [Medium] use ExecutionStatus.is_completed (single source of truth) instead of a local _TERMINAL_STATUSES frozenset that could drift; dropped the fake in (_TERMINAL_STATUSES) parens. - [Medium] remaining is NOT-NULL int → typed int, three-way branch (>0 work-incomplete / ==0 callback-never-fired / <0 already-torn-down) with accurate messages. - [Medium] re-guard the barrier DELETE on `expires_at < now()` + only clear dedup when the barrier row was actually reclaimed — a same-id re-enqueue (UPSERT resets expires_at) is no longer torn down mid-recovery. - [Medium] org-missing now LEAVES the row (doesn't erase the only recovery handle) + logs ERROR; PgBarrier.enqueue logs ERROR at write time if a barrier is enqueued without an org (should never happen). - [Low] recover_expired_barriers emits an aggregate summary; escalates to logger.error when a non-empty sweep recovers nothing (systemic: API down). - [Low] refreshed the stale "(future) periodic sweep" comment in pg_barrier. Tests: _FakeApiClient models the real success/failure contract + records file_execution; +read-failure-retains-row, +file_execution=False, +multi-row isolation (one fails, others recover), +org-stamping x3 (stamp/default/UPSERT refresh). 115 reaper/barrier/dedup tests green; ruff + ruff-format clean. Co-Authored-By: Claude Opus 4.8 * UN-3564 fix SonarCloud S6741 (new instance): index-free get_calls access A new occurrence introduced by the prior review-round test (test_status_read_passes_file_execution_false used api.get_calls[0][2]). Replaced with single-element unpacking — [(_exec_id, _org, file_execution)] = api.get_calls — index-free and additionally pins exactly one status read. 55 reaper tests green. Co-Authored-By: Claude Opus 4.8 * UN-3564 address greptile #2070: re-arm race, status=None edge, aggregate skip-vs-fail - [main] re-arm race: the ERROR mark now fires only after a re-check that the row is STILL expired (_still_expired) immediately before marking — so a same-id re-enqueue (expires_at reset to future) between the sweep SELECT and the mark can't get its freshly-running execution flagged ERROR. The DELETE stays guarded on expires_at < now() as the second line of defence. + test (re-arm via the status-read side-effect) asserting no mark + row left intact. - status=None on a success=True read no longer falls through to mark ERROR — treated as indeterminate: skip + leave the row for the next sweep. + test. - aggregate logging distinguishes genuine failures (exceptions) from benign skips (terminal / re-armed / no-status / no-org): the systemic "recovered NONE / API down" ERROR escalates only on real failures, not on all-skipped sweeps. + caplog test that an all-org-missing sweep doesn't escalate. 58 reaper + 51 barrier tests green; ruff + ruff-format clean. Co-Authored-By: Claude Opus 4.8 * UN-3564 fix SonarCloud S125: drop code-like tuple from a test comment The comment ended with a parenthesised tuple "(exec_id, org, file_execution)" — a complete Python expression S125 reads as commented-out code. Reworded to prose (and the stale `sweep_expired_barriers`→recover rename + a couple of inline `foo()` prose mentions neutralised while here). Comment-only; tests unaffected. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../0007_pgbarrierstate_organization_id.py | 17 + backend/pg_queue/models.py | 5 + workers/queue_backend/pg_barrier.py | 21 +- workers/queue_backend/pg_queue/__init__.py | 4 +- workers/queue_backend/pg_queue/reaper.py | 312 ++++++++++++++++-- workers/tests/test_pg_barrier.py | 66 +++- workers/tests/test_pg_reaper.py | 288 +++++++++++++--- 7 files changed, 618 insertions(+), 95 deletions(-) create mode 100644 backend/pg_queue/migrations/0007_pgbarrierstate_organization_id.py diff --git a/backend/pg_queue/migrations/0007_pgbarrierstate_organization_id.py b/backend/pg_queue/migrations/0007_pgbarrierstate_organization_id.py new file mode 100644 index 0000000000..b3455e5a26 --- /dev/null +++ b/backend/pg_queue/migrations/0007_pgbarrierstate_organization_id.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.1 on 2026-06-17 14:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("pg_queue", "0006_pgbatchdedup_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="pgbarrierstate", + name="organization_id", + field=models.TextField(blank=True, default=""), + ), + ] diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py index 079508619c..e1f9f3728c 100644 --- a/backend/pg_queue/models.py +++ b/backend/pg_queue/models.py @@ -194,6 +194,11 @@ class PgBarrierState(models.Model): """ execution_id = models.TextField(primary_key=True) + # Owning org, stamped at enqueue. The reaper needs it to call the org-scoped + # internal status API when recovering a stranded (expired) barrier — it has + # only the execution_id otherwise. "" = unknown (the reaper then skips the API + # mark and just cleans up). Same no-NULL-text convention as PgQueueMessage.org_id. + organization_id = models.TextField(blank=True, default="") # Header tasks still pending. The last task to decrement it to 0 fires the # callback. A value < 0 (decrement after expiry/cleanup) means the barrier # was already torn down — the task cleans up without firing. diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.py index 956ce09a05..19e80f29a1 100644 --- a/workers/queue_backend/pg_barrier.py +++ b/workers/queue_backend/pg_barrier.py @@ -291,6 +291,19 @@ def enqueue( # self-chain the callback onto PG rather than Celery. callback_descriptor["transport"] = WorkflowTransport.PG_QUEUE.value ttl_seconds = barrier_ttl_seconds() + # Stamp the owning org so the reaper can call the org-scoped status + # API when recovering this barrier if it strands (it has only the + # execution_id off the row otherwise). + organization_id = str(callback_kwargs.get("organization_id") or "") + if not organization_id: + # Should never happen — every fan-out passes organization_id in + # callback_kwargs. Surface loudly: a barrier with no org can't be + # recovered via the org-scoped API if it strands. + logger.error( + f"[exec:{execution_id}] PgBarrier enqueued with NO " + f"organization_id in callback_kwargs — a stranded barrier " + f"could not be reaper-recovered. This is a bug in the caller." + ) with _cursor() as cur: # UPSERT clears any leftover state from a prior run with this id. @@ -300,13 +313,15 @@ def enqueue( # periodic sweep keyed on pg_barrier_expires_idx. cur.execute( "INSERT INTO pg_barrier_state " - "(execution_id, remaining, results, created_at, expires_at) " - "VALUES (%s, %s, '[]'::jsonb, now(), " + "(execution_id, organization_id, remaining, results, " + " created_at, expires_at) " + "VALUES (%s, %s, %s, '[]'::jsonb, now(), " " now() + make_interval(secs => %s)) " "ON CONFLICT (execution_id) DO UPDATE SET " + " organization_id = EXCLUDED.organization_id, " " remaining = EXCLUDED.remaining, results = '[]'::jsonb, " " created_at = now(), expires_at = EXCLUDED.expires_at", - (execution_id, len(header_tasks), ttl_seconds), + (execution_id, organization_id, len(header_tasks), ttl_seconds), ) # Reset per-batch dedup markers from a prior run reusing this # execution_id, ATOMICALLY with the barrier reset. Without this a diff --git a/workers/queue_backend/pg_queue/__init__.py b/workers/queue_backend/pg_queue/__init__.py index ec228f4617..28139cefc2 100644 --- a/workers/queue_backend/pg_queue/__init__.py +++ b/workers/queue_backend/pg_queue/__init__.py @@ -39,7 +39,7 @@ ReaperLivenessServer, TickOutcome, reaper_interval_from_env, - sweep_expired_barriers, + recover_expired_barriers, ) from .task_payload import TaskPayload, to_payload @@ -57,6 +57,6 @@ "default_worker_id", "lease_seconds_from_env", "reaper_interval_from_env", - "sweep_expired_barriers", + "recover_expired_barriers", "to_payload", ] diff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.py index ae05d8812b..6bf5d7ec2c 100644 --- a/workers/queue_backend/pg_queue/reaper.py +++ b/workers/queue_backend/pg_queue/reaper.py @@ -2,22 +2,34 @@ A singleton, guarded by :class:`LeaderLease` over ``pg_orchestrator_lock``: only the elected leader runs recovery work each cycle (several reapers would contend -and double-act). This slice ships the process *harness* (lease-maintenance loop -+ graceful shutdown) plus ONE recovery job — the **barrier-orphan sweep**. +and double-act). It ships the process *harness* (lease-maintenance loop + +graceful shutdown) plus the **barrier-orphan recovery** job. -**Barrier-orphan sweep.** Reclaims ``pg_barrier_state`` rows past their +**Barrier-orphan recovery.** Handles ``pg_barrier_state`` rows past their ``expires_at`` — a barrier whose header tasks never all completed (the documented -:class:`PgBarrier` backstop). It ``DELETE``s the orphaned row; by PgBarrier's -existing semantics a late in-flight decrement then finds no row and abandons (no -spurious callback). The owning execution is logged loudly. Marking that -execution *terminal* (ERROR) is recovery that needs the backend and the -pipeline's PG shape — that's 9e, not here; this slice is the storage/orphan -backstop only. - -**Deferred to 9e.** Pipeline recovery (counter reconstruction from -``WorkflowFileExecution``, per-stage re-enqueue of stuck file executions) is -defined against the coupled pipeline running on PG, which doesn't exist yet — so -it lands with 9e, against a real PG pipeline it can be tested on. +:class:`PgBarrier` backstop). For each, the leader (:func:`recover_expired_barriers`): + +1. **Marks the execution ERROR** via the internal API — the same path the normal + callback uses for terminal status (business state never goes direct-DB) — with + a message distinguishing ``remaining>0`` (work incomplete) from ``remaining==0`` + (all batches done, callback never fired). It reads status first and **skips the + mark if the execution is already terminal** (a ``remaining==0`` row can belong + to a COMPLETED execution whose best-effort row-delete merely failed, and the + backend status update has no terminal guard) or if the row carries no org. +2. **Reclaims the queue-infra rows** (``pg_batch_dedup`` + ``pg_barrier_state``) + directly in PG — same boundary as the rest of ``queue_backend``. + +Recovery is best-effort and per-execution: a failure (e.g. the API is +unreachable) leaves that barrier row for the next sweep to retry, and never +blocks the others. **This is the recovery net PG-queue execution rollout depends +on** — without it the un-catchable strand windows (a worker SIGKILL mid-batch, or +a crash after the final decrement but before the callback enqueues) would bottom +out silently at the ~6h barrier expiry. + +**Deferred (follow-up).** *Callback re-fire* for the ``remaining==0`` strand +(heal → COMPLETED instead of ERROR) needs the ``callback_descriptor`` stored on +the barrier row; until then those strands are marked ERROR. Per-stage re-enqueue +of stuck file executions is a larger pipeline-recovery effort beyond this net. **Lease maintenance.** Each cycle the leader renews; if ``renew()`` returns ``False`` (or raises) it lost / can't confirm the lease and steps down to @@ -41,12 +53,15 @@ import time from typing import TYPE_CHECKING, NamedTuple, Protocol +from unstract.core.data_models import ExecutionStatus + from .connection import create_pg_connection from .leader_election import LeaderLease, default_worker_id from .liveness import LivenessServer as _BaseLivenessServer if TYPE_CHECKING: from psycopg2.extensions import connection as PgConnection + from shared.api import InternalAPIClient logger = logging.getLogger(__name__) @@ -111,40 +126,249 @@ def reaper_interval_from_env() -> float: return value -def sweep_expired_barriers(conn: PgConnection) -> list[str]: - """Reclaim ``pg_barrier_state`` rows past ``expires_at``. Returns their ids. +def _execution_status( + api_client: InternalAPIClient, execution_id: str, organization_id: str +) -> str | None: + """Current execution status via the org-scoped internal API. + + **Raises** when the read fails. The client catches all errors and returns a + ``success=False`` response (it does NOT raise), so a transient blip would + yield ``status=None`` — which is not terminal, and the caller would then mark + a possibly-COMPLETED execution ERROR. Treating "couldn't read" as a hard stop + (the caller's ``except`` retains the row for retry) is what keeps the + terminal-skip guard honest. + """ + response = api_client.get_workflow_execution( + execution_id, organization_id=organization_id, file_execution=False + ) + if not getattr(response, "success", False): + raise RuntimeError( + f"status read failed for execution {execution_id} " + f"(refusing to mark ERROR on an unconfirmed status)" + ) + return getattr(response, "status", None) + - A single atomic ``DELETE … RETURNING``: concurrent sweepers would each - reclaim a disjoint subset (``RETURNING`` reports only the rows *this* - statement deleted), so it stays correct even if leadership gating ever fails — - in practice only the leader calls it. Each reclaimed barrier is logged at - WARNING: an orphaned barrier means an execution's header tasks never all - completed, worth surfacing even though deleting the row is the right backstop. +def _still_expired(conn: PgConnection, execution_id: str) -> bool: + """True iff the barrier row is still present AND still past expiry. - ``conn`` runs in manual-commit mode, so on any error we roll back before - re-raising — otherwise the connection is left in an aborted-transaction state - and every later statement on it fails with ``InFailedSqlTransaction``. + Re-checked immediately before the ERROR mark so a same-id re-enqueue (UPSERT + resets ``expires_at`` to the future) between the sweep's SELECT and the mark + doesn't get its live run flagged ERROR. + """ + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM pg_barrier_state " + "WHERE execution_id = %s AND expires_at < now()", + (execution_id,), + ) + found = cur.fetchone() is not None + conn.commit() + return found + + +def _mark_stranded_error( + api_client: InternalAPIClient, + execution_id: str, + organization_id: str, + remaining: int, +) -> None: + """Mark a confirmed-non-terminal stranded execution ERROR (message by remaining).""" + if remaining > 0: + reason = ( + f"{remaining} file batch(es) never completed before the barrier " + f"expired (worker crash / lost task)" + ) + elif remaining == 0: + reason = ( + "all file batches completed but the final aggregating callback never " + "fired before the barrier expired" + ) + else: # remaining < 0: a decrement landed after the row was torn down + reason = ( + "the barrier was already torn down (remaining < 0) yet the execution " + "was left non-terminal — inconsistent state" + ) + api_client.update_workflow_execution_status( + execution_id=execution_id, + status=ExecutionStatus.ERROR.value, + error_message=f"[reaper-recovery] Execution stranded: {reason}.", + organization_id=organization_id, + ) + logger.error( + "Reaper: marked stranded execution %s ERROR (remaining=%s).", + execution_id, + remaining, + ) + + +def _recover_one_barrier( + conn: PgConnection, + api_client: InternalAPIClient, + execution_id: str, + organization_id: str, + remaining: int, +) -> bool: + """Recover one stranded execution; return True iff its barrier row was deleted. + + Marks the execution ERROR via the **internal API** (the path the normal + callback uses for terminal status — business state never goes direct-DB), + UNLESS the status read shows it's already terminal (``ExecutionStatus + .is_completed`` — the single source of truth, so a future terminal status + can't drift from a local copy). A ``remaining==0`` expired row can belong to + a COMPLETED execution whose best-effort row-delete merely failed, and the + backend status update has no terminal guard, so the read-first skip prevents + overwriting a finished execution. Queue-infra cleanup (``pg_batch_dedup`` / + ``pg_barrier_state``) stays direct-PG. + + The barrier ``DELETE`` is re-guarded on ``expires_at < now()``: between the + sweep's SELECT and here the same ``execution_id`` could be re-enqueued + (UPSERT resets ``expires_at`` to the future), and we must not tear down a + freshly re-armed barrier. If the row was re-armed (``rowcount == 0``) we leave + it and its dedup markers (the new run owns them); the dedup delete only runs + when the barrier row was actually reclaimed. + + Returns False (no mark, no delete) when the row can't be safely recovered: + org unknown (can't call the org-scoped API → the row is LEFT, not erased, so + the only recovery handle survives for ops; should never happen), a successful + read with no status (anomalous), or the row was **re-armed** before the mark. + The mark is gated on a re-check that the row is *still* expired immediately + before it fires — so a same-id re-enqueue can't get its live run marked ERROR + (the worst outcome) — and the DELETE is additionally guarded on + ``expires_at < now()`` so a re-armed barrier is never torn down. + """ + if not organization_id: + logger.error( + "Reaper: stranded barrier for execution %s has NO organization_id — " + "cannot mark it ERROR via the org-scoped API; leaving the row (not " + "erasing the only recovery handle). A barrier was enqueued without an " + "org — investigate.", + execution_id, + ) + return False + + status = _execution_status(api_client, execution_id, organization_id) + if status is None: + # A successful read with no status is anomalous — don't mark on it; leave + # the row for the next sweep rather than risk a wrong ERROR. + logger.warning( + "Reaper: status read for execution %s returned no status — leaving the " + "row for the next sweep (not marking ERROR on an indeterminate status).", + execution_id, + ) + return False + if ExecutionStatus.is_completed(status): + logger.warning( + "Reaper: barrier for execution %s expired but the execution is already " + "%s — cleaning up the orphaned row only (no status overwrite).", + execution_id, + status, + ) + elif not _still_expired(conn, execution_id): + # Re-armed (same execution_id re-enqueued) between the read and here — + # do NOT mark a freshly-running execution ERROR; leave it for the new run. + logger.warning( + "Reaper: execution %s was re-armed during recovery — skipping the " + "ERROR mark (its new run owns the barrier).", + execution_id, + ) + return False + else: + _mark_stranded_error(api_client, execution_id, organization_id, remaining) + + # Queue-infra cleanup (direct PG), re-guarded against a concurrent re-arm. + with conn.cursor() as cur: + cur.execute( + "DELETE FROM pg_barrier_state WHERE execution_id = %s " + "AND expires_at < now()", + (execution_id,), + ) + deleted = cur.rowcount > 0 + if deleted: + cur.execute( + "DELETE FROM pg_batch_dedup WHERE execution_id = %s", (execution_id,) + ) + else: + logger.warning( + "Reaper: barrier for execution %s was re-armed during recovery " + "(no longer expired) — leaving its rows for the new run.", + execution_id, + ) + conn.commit() + return deleted + + +def recover_expired_barriers( + conn: PgConnection, api_client: InternalAPIClient +) -> list[str]: + """Recover executions stranded by an expired barrier. Returns recovered ids. + + SELECT the expired rows (the reaper is leader-elected → a single active + sweeper, so a read-then-act is safe from double-claim), then recover each one + best-effort: mark the execution ERROR if it isn't already terminal, then + delete its ``pg_batch_dedup`` + ``pg_barrier_state`` rows. One execution + failing (e.g. the API is unreachable, or a status read that can't be + confirmed) is logged and skipped — its row is left for the next sweep to + retry — so it never blocks the others. A non-empty sweep that recovers + *nothing* is escalated as a systemic failure (API down / bad migration). + + ``conn`` runs in manual-commit mode; on any error we roll back before + continuing/re-raising so the connection isn't left in an aborted-txn state. """ try: with conn.cursor() as cur: cur.execute( - "DELETE FROM pg_barrier_state WHERE expires_at < now() " - "RETURNING execution_id" + "SELECT execution_id, organization_id, remaining " + "FROM pg_barrier_state WHERE expires_at < now()" ) - reclaimed = [row[0] for row in cur.fetchall()] + rows = cur.fetchall() conn.commit() except Exception: with contextlib.suppress(Exception): conn.rollback() raise - for execution_id in reclaimed: - logger.warning( - "Reaper: reclaimed orphaned barrier for execution %s — header tasks " - "never all completed before expiry; barrier deleted (no callback " - "fired). Execution terminal-status recovery is 9e's job.", - execution_id, + + recovered: list[str] = [] + failed = 0 # genuine failures (exceptions) — NOT benign skips + for execution_id, organization_id, remaining in rows: + try: + if _recover_one_barrier( + conn, api_client, execution_id, organization_id, remaining + ): + recovered.append(execution_id) + # else: a benign skip (terminal / re-armed / no-status / no-org) — + # logged per-row inside; not a failure, not retried-as-error. + except Exception: + # Keep the connection usable for the next row, and leave THIS barrier + # row in place so the next sweep retries its recovery. + failed += 1 + with contextlib.suppress(Exception): + conn.rollback() + logger.exception( + "Reaper: failed to recover stranded barrier for execution %s — " + "leaving the row for the next sweep to retry.", + execution_id, + ) + + if rows: + skipped = len(rows) - len(recovered) - failed + summary = ( + f"recovered={len(recovered)}, failed={failed}, skipped={skipped} " + f"of {len(rows)} expired barrier(s)" ) - return reclaimed + if failed and not recovered: + # Genuine failures and nothing got through → systemic (API down / bad + # migration). Benign skips alone (terminal/re-armed/no-org) don't escalate. + logger.error( + "Reaper: %s — likely systemic (internal API down / bad migration).", + summary, + ) + elif failed: + logger.warning("Reaper: %s — failures left for the next sweep.", summary) + elif recovered: + logger.info("Reaper: %s.", summary) + # all-skipped (no recovered, no failed) is fully covered by per-row logs. + return recovered class PgReaper: @@ -156,6 +380,7 @@ def __init__( *, interval_seconds: float | None = None, sweep_conn: PgConnection | None = None, + api_client: InternalAPIClient | None = None, ) -> None: self._lease = lease self._interval = ( @@ -177,6 +402,10 @@ def __init__( ) self._sweep_conn = sweep_conn self._owns_sweep_conn = sweep_conn is None + # Lazily built so the reaper can be constructed without env/HTTP set up + # (tests inject a fake). Recovery marks execution ERROR via this client — + # business state goes through the internal API, not direct DB. + self._api_client = api_client self._running = False self._is_leader = False # Liveness heartbeat: monotonic timestamp of the last tick start. A @@ -206,6 +435,15 @@ def _get_sweep_conn(self) -> PgConnection: self._sweep_conn = create_pg_connection(env_prefix="DB_") return self._sweep_conn + def _get_api_client(self) -> InternalAPIClient: + # Lazy import + build: keeps reaper construction free of HTTP/env (an + # injected fake short-circuits this), and avoids a module-load import cycle. + if self._api_client is None: + from shared.api import InternalAPIClient + + self._api_client = InternalAPIClient() + return self._api_client + def _discard_owned_sweep_conn(self) -> None: # After a sweep error, drop an owned connection so the next tick # reconnects — covers a poisoned (aborted-txn) or dead-socket handle that @@ -244,7 +482,9 @@ def tick(self) -> TickOutcome: if not self._is_leader: return TickOutcome(was_leader=False, reclaimed=0) try: - reclaimed = len(sweep_expired_barriers(self._get_sweep_conn())) + reclaimed = len( + recover_expired_barriers(self._get_sweep_conn(), self._get_api_client()) + ) except Exception: self._discard_owned_sweep_conn() raise diff --git a/workers/tests/test_pg_barrier.py b/workers/tests/test_pg_barrier.py index 238bbeeabc..a4fca8b146 100644 --- a/workers/tests/test_pg_barrier.py +++ b/workers/tests/test_pg_barrier.py @@ -155,6 +155,16 @@ def _row(conn, execution_id): return cur.fetchone() +def _org(conn, execution_id): + with conn.cursor() as cur: + cur.execute( + "SELECT organization_id FROM pg_barrier_state WHERE execution_id = %s", + (execution_id,), + ) + row = cur.fetchone() + return row[0] if row else None + + class TestPgBarrierEnqueue: def test_upsert_creates_row_and_attaches_links(self, barrier_db): tasks = [_mock_header_task() for _ in range(3)] @@ -191,8 +201,10 @@ def test_upsert_overwrites_stale_state(self, barrier_db): with barrier_db.cursor() as cur: cur.execute( "INSERT INTO pg_barrier_state " - "(execution_id, remaining, results, created_at, expires_at) " - "VALUES ('exec-R', 1, '[1,2]'::jsonb, now(), now() + interval '1h')" + "(execution_id, organization_id, remaining, results, " + " created_at, expires_at) " + "VALUES ('exec-R', '', 1, '[1,2]'::jsonb, now(), " + " now() + interval '1h')" ) task, _ = _mock_header_task() PgBarrier().enqueue( @@ -204,6 +216,46 @@ def test_upsert_overwrites_stale_state(self, barrier_db): ) assert _row(barrier_db, "exec-R") == (2, []) + def test_enqueue_stamps_organization_id(self, barrier_db): + # The whole reason the org column + migration exist (reaper recovery). + PgBarrier().enqueue( + [_mock_header_task()[0]], + callback_task_name="cb", + callback_kwargs={"execution_id": "exec-ORG", "organization_id": "org-42"}, + callback_queue="general", + app_instance=None, + ) + assert _org(barrier_db, "exec-ORG") == "org-42" + + def test_enqueue_defaults_org_to_empty_when_absent(self, barrier_db): + PgBarrier().enqueue( + [_mock_header_task()[0]], + callback_task_name="cb", + callback_kwargs={"execution_id": "exec-NOORG"}, # no organization_id + callback_queue="general", + app_instance=None, + ) + assert _org(barrier_db, "exec-NOORG") == "" + + def test_upsert_refreshes_org_on_reenqueue(self, barrier_db): + # Exercises the ON CONFLICT DO UPDATE SET organization_id clause. + with barrier_db.cursor() as cur: + cur.execute( + "INSERT INTO pg_barrier_state " + "(execution_id, organization_id, remaining, results, " + " created_at, expires_at) " + "VALUES ('exec-REORG', 'old-org', 1, '[]'::jsonb, now(), " + " now() + interval '1h')" + ) + PgBarrier().enqueue( + [_mock_header_task()[0]], + callback_task_name="cb", + callback_kwargs={"execution_id": "exec-REORG", "organization_id": "new-org"}, + callback_queue="general", + app_instance=None, + ) + assert _org(barrier_db, "exec-REORG") == "new-org" + def test_mid_loop_dispatch_failure_deletes_row(self, barrier_db): good, _ = _mock_header_task() bad, bad_cloned = _mock_header_task() @@ -223,8 +275,9 @@ def _seed(conn, execution_id, remaining, *, results="[]"): with conn.cursor() as cur: cur.execute( "INSERT INTO pg_barrier_state " - "(execution_id, remaining, results, created_at, expires_at) " - "VALUES (%s, %s, %s::jsonb, now(), now() + interval '1h')", + "(execution_id, organization_id, remaining, results, " + " created_at, expires_at) " + "VALUES (%s, '', %s, %s::jsonb, now(), now() + interval '1h')", (execution_id, remaining, results), ) @@ -487,8 +540,9 @@ def test_expires_at_must_exceed_created_at(self, barrier_db): with barrier_db.cursor() as cur: cur.execute( "INSERT INTO pg_barrier_state " - "(execution_id, remaining, results, created_at, expires_at) " - "VALUES ('bad', 1, '[]'::jsonb, now(), now())" # expires == created + "(execution_id, organization_id, remaining, results, " + " created_at, expires_at) " + "VALUES ('bad', '', 1, '[]'::jsonb, now(), now())" # expires==created ) diff --git a/workers/tests/test_pg_reaper.py b/workers/tests/test_pg_reaper.py index a37db4494a..ed89fff2ef 100644 --- a/workers/tests/test_pg_reaper.py +++ b/workers/tests/test_pg_reaper.py @@ -18,6 +18,7 @@ import os import threading import time +from types import SimpleNamespace from unittest.mock import MagicMock, patch import psycopg2 @@ -27,7 +28,7 @@ from queue_backend.pg_queue.reaper import ( PgReaper, reaper_interval_from_env, - sweep_expired_barriers, + recover_expired_barriers, ) # --- Layer 1: env + construction (no DB) --- @@ -99,12 +100,16 @@ def test_valid_interval_accepted(self): class TestLeadershipGating: def _reaper(self, lease): - return PgReaper(lease, interval_seconds=0.01, sweep_conn=object()) + # Inject dummy sweep_conn + api_client so a tick doesn't build real ones; + # recover_expired_barriers is patched in each test, so neither is used. + return PgReaper( + lease, interval_seconds=0.01, sweep_conn=object(), api_client=object() + ) def test_sweeps_when_leader(self): reaper = self._reaper(_FakeLease(acquires=True, renews=True)) with patch.object( - reaper_mod, "sweep_expired_barriers", return_value=["x"] + reaper_mod, "recover_expired_barriers", return_value=["x"] ) as sweep: outcome = reaper.tick() # acquires leadership → sweeps assert outcome.was_leader is True @@ -114,7 +119,7 @@ def test_sweeps_when_leader(self): def test_standby_does_not_sweep(self): reaper = self._reaper(_FakeLease(acquires=False)) # can't get the lease - with patch.object(reaper_mod, "sweep_expired_barriers") as sweep: + with patch.object(reaper_mod, "recover_expired_barriers") as sweep: outcome = reaper.tick() assert outcome == (False, 0) assert reaper.is_leader is False @@ -124,7 +129,7 @@ def test_steps_down_when_renew_fails(self): # tick 1 acquires; tick 2 renew fails → step down, acquire also fails → # standby. Driven through ticks, no private-flag poking. reaper = self._reaper(_FakeLease(acquires=[True, False], renews=[False])) - with patch.object(reaper_mod, "sweep_expired_barriers", return_value=[]): + with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]): assert reaper.tick().was_leader is True assert reaper.tick().was_leader is False assert reaper.is_leader is False @@ -132,7 +137,7 @@ def test_steps_down_when_renew_fails(self): def test_steps_down_then_reacquires(self): # leader → lose the lease one cycle → re-acquire the next and resume. reaper = self._reaper(_FakeLease(acquires=[True, False, True], renews=[False])) - with patch.object(reaper_mod, "sweep_expired_barriers", return_value=[]): + with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]): assert reaper.tick().was_leader is True # acquired assert reaper.tick().was_leader is False # renew failed → standby assert reaper.tick().was_leader is True # re-acquired @@ -141,7 +146,7 @@ def test_steps_down_then_reacquires(self): def test_renew_raising_steps_down(self): lease = _FakeLease(acquires=True, renews=True) reaper = self._reaper(lease) - with patch.object(reaper_mod, "sweep_expired_barriers", return_value=[]): + with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]): reaper.tick() # becomes leader lease.renew = MagicMock(side_effect=psycopg2.OperationalError("boom")) with pytest.raises(psycopg2.OperationalError): @@ -151,7 +156,7 @@ def test_renew_raising_steps_down(self): def test_release_on_stop_when_leader(self): lease = _FakeLease(acquires=True, renews=True) reaper = self._reaper(lease) - with patch.object(reaper_mod, "sweep_expired_barriers", return_value=[]): + with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]): t = threading.Thread(target=reaper.run, kwargs={"install_signals": False}) t.start() time.sleep(0.05) @@ -179,26 +184,75 @@ def boom(): # --- Layer 3: connection handling (mocked, no DB) --- -class TestSweepConnection: - def test_sql_contract(self): +class _FakeApiClient: + """Models the real ``InternalAPIClient`` contract the reaper depends on. + + ``get_workflow_execution`` returns a response with ``success``/``status`` + (the real client returns ``success=False`` on any error instead of raising); + ``fail_read`` models that. ``fail_update`` makes the ERROR-mark raise (the + real ``update_*`` does raise). ``fail_update_for`` fails only specific ids. + """ + + def __init__( + self, status="EXECUTING", *, fail_read=False, fail_update=False, + fail_update_for=None, on_get=None, + ): + self._status = status + self._fail_read = fail_read + self._fail_update = fail_update + self._fail_update_for = set(fail_update_for or []) + self._on_get = on_get # side-effect hook (e.g. re-arm the row mid-recovery) + self.get_calls: list = [] + self.update_calls: list = [] + + def get_workflow_execution( + self, execution_id, organization_id=None, file_execution=True, **kw + ): + self.get_calls.append((execution_id, organization_id, file_execution)) + if self._on_get is not None: + self._on_get(execution_id) + if self._fail_read: + return SimpleNamespace(success=False, status=None) # real error contract + return SimpleNamespace(success=True, status=self._status) + + def update_workflow_execution_status( + self, execution_id, status, error_message=None, organization_id=None, **kw + ): + self.update_calls.append( + SimpleNamespace( + execution_id=execution_id, + status=status, + error_message=error_message, + organization_id=organization_id, + ) + ) + if self._fail_update or execution_id in self._fail_update_for: + raise RuntimeError("api down") + return {"status": "updated"} + + +class TestRecoverConnection: + def test_select_sql_contract(self): cur = MagicMock() - cur.fetchall.return_value = [("e1",), ("e2",)] + cur.fetchall.return_value = [] # nothing expired conn = MagicMock() conn.cursor.return_value.__enter__.return_value = cur - assert sweep_expired_barriers(conn) == ["e1", "e2"] + api = MagicMock() + assert recover_expired_barriers(conn, api) == [] sql = cur.execute.call_args[0][0] - assert "DELETE FROM pg_barrier_state" in sql + assert "SELECT" in sql and "pg_barrier_state" in sql + assert "organization_id" in sql and "remaining" in sql assert "expires_at < now()" in sql - assert "RETURNING execution_id" in sql conn.commit.assert_called_once() + api.update_workflow_execution_status.assert_not_called() - def test_rolls_back_on_error(self): + def test_rolls_back_on_select_error(self): cur = MagicMock() cur.execute.side_effect = psycopg2.OperationalError("dead") conn = MagicMock() conn.cursor.return_value.__enter__.return_value = cur with pytest.raises(psycopg2.OperationalError): - sweep_expired_barriers(conn) + recover_expired_barriers(conn, MagicMock()) conn.rollback.assert_called_once() conn.commit.assert_not_called() @@ -222,10 +276,14 @@ def test_failed_sweep_discards_owned_conn(self, monkeypatch): monkeypatch.setattr( reaper_mod, "create_pg_connection", MagicMock(return_value=conn) ) - reaper = PgReaper(_FakeLease(acquires=True, renews=True), interval_seconds=1) + reaper = PgReaper( + _FakeLease(acquires=True, renews=True), + interval_seconds=1, + api_client=object(), # injected so tick() doesn't build a real client + ) with patch.object( reaper_mod, - "sweep_expired_barriers", + "recover_expired_barriers", side_effect=psycopg2.OperationalError("x"), ): with pytest.raises(psycopg2.OperationalError): @@ -240,9 +298,9 @@ def test_failed_sweep_discards_owned_conn(self, monkeypatch): def _new_conn(): os.environ.setdefault("TEST_DB_HOST", "127.0.0.1") # Manual-commit — exactly as the production reaper opens it - # (create_pg_connection default). NOT autocommit: that would make - # sweep_expired_barriers' own commit() a no-op and its rollback unreachable, - # so Layer 4 would test a different mode than the real reaper runs in. + # (create_pg_connection default). NOT autocommit: that would make the + # recover_expired_barriers commit a no-op and its rollback unreachable, so + # Layer 4 would test a different mode than the real reaper runs in. return create_pg_connection(env_prefix="TEST_DB_") @@ -258,35 +316,34 @@ def barrier_conn(): conn.close() pytest.skip("pg_barrier_state migration not applied (run backend migrate)") cur.execute("DELETE FROM pg_barrier_state") + cur.execute("DELETE FROM pg_batch_dedup") conn.commit() yield conn with conn.cursor() as cur: cur.execute("DELETE FROM pg_barrier_state") + cur.execute("DELETE FROM pg_batch_dedup") conn.commit() conn.close() -def _seed(conn, execution_id, *, expired): +def _seed(conn, execution_id, *, expired, organization_id="org-1", remaining=1): # created_at must precede expires_at (CheckConstraint # pg_barrier_expires_after_created). Commit so the seed is durable like a # real barrier row (written by PgBarrier in another transaction) — and so the - # manual-commit sweep's own commit() is what persists the DELETE. + # manual-commit recovery's own commit is what persists the DELETE. + created_sql, expires_sql = ( + ("now() - interval '2 hours'", "now() - interval '1 hour'") + if expired + else ("now()", "now() + interval '6 hours'") + ) with conn.cursor() as cur: - if expired: - cur.execute( - "INSERT INTO pg_barrier_state " - "(execution_id, remaining, results, created_at, expires_at) " - "VALUES (%s, 1, '[]'::jsonb, now() - interval '2 hours', " - " now() - interval '1 hour')", - (execution_id,), - ) - else: - cur.execute( - "INSERT INTO pg_barrier_state " - "(execution_id, remaining, results, created_at, expires_at) " - "VALUES (%s, 1, '[]'::jsonb, now(), now() + interval '6 hours')", - (execution_id,), - ) + cur.execute( + "INSERT INTO pg_barrier_state " + "(execution_id, organization_id, remaining, results, " + " created_at, expires_at) " + f"VALUES (%s, %s, %s, '[]'::jsonb, {created_sql}, {expires_sql})", + (execution_id, organization_id, remaining), + ) conn.commit() @@ -298,28 +355,163 @@ def _ids(conn): return rows -class TestSweepExpiredBarriers: - def test_reclaims_only_expired(self, barrier_conn): - _seed(barrier_conn, "exp-1", expired=True) - _seed(barrier_conn, "exp-2", expired=True) +def _dedup_count(conn, execution_id): + with conn.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s", + (execution_id,), + ) + n = cur.fetchone()[0] + conn.commit() + return n + + +class TestRecoverExpiredBarriers: + def test_recovers_only_expired_marks_error_and_cleans_up(self, barrier_conn): + _seed(barrier_conn, "exp-1", expired=True, remaining=2) _seed(barrier_conn, "fresh-1", expired=False) - reclaimed = sweep_expired_barriers(barrier_conn) - assert sorted(reclaimed) == ["exp-1", "exp-2"] + api = _FakeApiClient(status="EXECUTING") + recovered = recover_expired_barriers(barrier_conn, api) + assert recovered == ["exp-1"] + (call,) = api.update_calls # exactly one execution marked + assert call.execution_id == "exp-1" + assert call.status == "ERROR" + assert call.organization_id == "org-1" + assert "never completed" in call.error_message # remaining>0 assert _ids(barrier_conn) == ["fresh-1"] # fresh barrier untouched + def test_remaining_zero_uses_callback_stranded_message(self, barrier_conn): + _seed(barrier_conn, "exp-0", expired=True, remaining=0) + api = _FakeApiClient(status="EXECUTING") + recover_expired_barriers(barrier_conn, api) + (call,) = api.update_calls + assert "callback never fired" in call.error_message + + def test_skips_already_terminal_execution(self, barrier_conn): + # A remaining==0 expired row can belong to a COMPLETED exec whose row + # delete failed — must NOT overwrite it to ERROR. + _seed(barrier_conn, "exp-done", expired=True, remaining=0) + api = _FakeApiClient(status="COMPLETED") + recovered = recover_expired_barriers(barrier_conn, api) + assert recovered == ["exp-done"] # cleaned up + assert api.update_calls == [] # no status overwrite + assert _ids(barrier_conn) == [] + + def test_org_missing_leaves_row_and_skips_mark(self, barrier_conn): + # No org → can't call the org-scoped API; LEAVE the row (don't erase the + # only recovery handle) and don't mark. + _seed(barrier_conn, "exp-noorg", expired=True, organization_id="") + api = _FakeApiClient() + recovered = recover_expired_barriers(barrier_conn, api) + assert recovered == [] # not recovered + assert api.get_calls == [] and api.update_calls == [] # can't call org API + assert _ids(barrier_conn) == ["exp-noorg"] # row preserved for ops + + def test_failed_status_read_does_not_mark_and_retains_row(self, barrier_conn): + # [Critical] the real client returns success=False (not raises) on a blip; + # a failed read must NOT fall through to ERROR (would corrupt a COMPLETED + # exec) — leave the row for the next sweep. + _seed(barrier_conn, "exp-readfail", expired=True, remaining=0) + api = _FakeApiClient(fail_read=True) + recovered = recover_expired_barriers(barrier_conn, api) + assert recovered == [] + assert api.update_calls == [] # never marked ERROR on an unconfirmed status + assert _ids(barrier_conn) == ["exp-readfail"] # retained for retry + + def test_status_read_passes_file_execution_false(self, barrier_conn): + _seed(barrier_conn, "exp-fe", expired=True) + api = _FakeApiClient(status="EXECUTING") + recover_expired_barriers(barrier_conn, api) + # Exactly one status read, recorded as exec-id / org / file_execution. + # The reaper must skip the costly file-execution fetch it doesn't need. + [(_exec_id, _org, file_execution)] = api.get_calls + assert file_execution is False + + def test_api_failure_leaves_row_for_retry(self, barrier_conn): + _seed(barrier_conn, "exp-fail", expired=True) + api = _FakeApiClient(status="EXECUTING", fail_update=True) + recovered = recover_expired_barriers(barrier_conn, api) + assert recovered == [] # not recovered + assert _ids(barrier_conn) == ["exp-fail"] # row left for next sweep + + def test_one_failing_execution_does_not_block_others(self, barrier_conn): + # Per-execution isolation: a mid-loop failure rolls back + the loop + # continues; the others still recover. + for eid in ("exp-a", "exp-bad", "exp-c"): + _seed(barrier_conn, eid, expired=True) + api = _FakeApiClient(status="EXECUTING", fail_update_for=["exp-bad"]) + recovered = recover_expired_barriers(barrier_conn, api) + assert sorted(recovered) == ["exp-a", "exp-c"] # the two good ones + assert _ids(barrier_conn) == ["exp-bad"] # only the failing row remains + + def test_rearmed_execution_is_not_marked_error(self, barrier_conn): + # greptile #2070: if the same execution_id is re-enqueued (expires_at + # reset to the future) between the sweep SELECT and the mark, the reaper + # must NOT mark the freshly-running execution ERROR. Simulate the re-arm + # via the status-read side-effect. + _seed(barrier_conn, "exp-rearm", expired=True) + + def rearm(execution_id): + with barrier_conn.cursor() as cur: + cur.execute( + "UPDATE pg_barrier_state SET expires_at = now() + interval '6 hours' " + "WHERE execution_id = %s", + (execution_id,), + ) + barrier_conn.commit() + + api = _FakeApiClient(status="EXECUTING", on_get=rearm) + recovered = recover_expired_barriers(barrier_conn, api) + assert recovered == [] # not recovered + assert api.update_calls == [] # the live re-run was NOT marked ERROR + assert _ids(barrier_conn) == ["exp-rearm"] # re-armed row left intact + + def test_status_none_on_success_does_not_mark(self, barrier_conn): + # A successful read with no status is anomalous — don't mark on it. + _seed(barrier_conn, "exp-nostatus", expired=True) + api = _FakeApiClient(status=None) # success=True, status=None + recovered = recover_expired_barriers(barrier_conn, api) + assert recovered == [] + assert api.update_calls == [] + assert _ids(barrier_conn) == ["exp-nostatus"] # left for next sweep + + def test_all_skipped_sweep_does_not_log_systemic_error(self, barrier_conn, caplog): + # org-missing rows are benign skips, not failures — they must NOT trigger + # the systemic "recovered NONE / API down" ERROR escalation. + import logging + + _seed(barrier_conn, "exp-noorg", expired=True, organization_id="") + with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.reaper"): + recover_expired_barriers(barrier_conn, _FakeApiClient()) + assert not any( + "systemic" in r.message for r in caplog.records + ), "all-skipped sweep should not escalate to a systemic-failure error" + + def test_reclaims_dedup_markers(self, barrier_conn): + _seed(barrier_conn, "exp-d", expired=True) + with barrier_conn.cursor() as cur: + cur.execute( + "INSERT INTO pg_batch_dedup (execution_id, batch_index, created_at) " + "VALUES ('exp-d', 0, now())" + ) + barrier_conn.commit() + recover_expired_barriers(barrier_conn, _FakeApiClient(status="EXECUTING")) + assert _dedup_count(barrier_conn, "exp-d") == 0 + def test_noop_when_nothing_expired(self, barrier_conn): _seed(barrier_conn, "fresh-1", expired=False) - assert sweep_expired_barriers(barrier_conn) == [] + assert recover_expired_barriers(barrier_conn, _FakeApiClient()) == [] assert _ids(barrier_conn) == ["fresh-1"] - def test_tick_sweeps_via_real_conn(self, barrier_conn): + def test_tick_recovers_via_real_conn(self, barrier_conn): _seed(barrier_conn, "exp-1", expired=True) reaper = PgReaper( _FakeLease(acquires=True, renews=True), interval_seconds=1, sweep_conn=barrier_conn, + api_client=_FakeApiClient(status="EXECUTING"), ) - outcome = reaper.tick() # became leader and reclaimed the orphan + outcome = reaper.tick() # became leader and recovered the orphan assert outcome.was_leader is True assert outcome.reclaimed == 1 assert _ids(barrier_conn) == [] @@ -396,7 +588,7 @@ def test_is_leader_reflected(self): interval_seconds=0.01, sweep_conn=object(), ) - with patch.object(reaper_mod, "sweep_expired_barriers", return_value=[]): + with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]): reaper.tick() # becomes leader server = self._server(reaper) try: From c2e6a6f8208b571f9237e0a0555a6700bbfd0e9b Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Thu, 18 Jun 2026 12:34:43 +0530 Subject: [PATCH 20/44] =?UTF-8?q?UN-3570=20[FEAT]=20PG=20Queue=209e=20PR?= =?UTF-8?q?=203=20=E2=80=94=20Flipt=20canary=20wiring=20for=20transport=20?= =?UTF-8?q?resolution=20(#2071)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3570 [FEAT] PG Queue 9e PR 3 — Flipt canary wiring for transport resolution Replace the hardwired Celery result in resolve_transport() with a real decision: an env master-gate (PG_QUEUE_TRANSPORT_ENABLED, default off) that, when on, consults the Flipt boolean flag pg_queue_execution_enabled. Fails closed to Celery on any error. - entity_id = execution_id (per-execution sticky %-rollout; resolved once, carried in the task payload so an in-flight execution never re-buckets) - context carries org/workflow/pipeline ids (str-coerced — UUIDs in the gRPC map context would be swallowed as False and silently force Celery) for segment rollouts - both creation chokepoints wired: internal_api_views.create_workflow_execution (scheduler path) and workflow_helper.execute_workflow_async (API/manual/async) Gated off by default — no behaviour change until the flag is enabled, which requires PG consumers deployed first (deploy-ordering safety). Tests: 15 cases in test_transport.py (gate off/on, flag true/false, fail-closed, entity/context shape, UUID-coercion regression). Co-Authored-By: Claude Opus 4.8 * UN-3570 address review: type honesty, fail-closed org/Flipt guards, observability - Widen resolve_transport id params to `str | UUID` (callers pass UUIDs; the body already str-coerces) — resolves the SonarCloud type-mismatch and makes the contract honest. - Fail closed to celery when organization_id is missing (str(None) must never reach the Flipt org segment) — the helper path reads it from StateStore which can be empty. - Fail closed + loud warning when the gate is ON but FLIPT_SERVICE_AVAILABLE is not true, so a blind Flipt can't masquerade as a healthy 100%-celery canary. - Log the resolved transport on the gate-ON path (deliberate celery vs pg_queue vs blind are now distinguishable); drop the inaccurate "import-time fault" clause from the fail-closed comment. - Doc: organization_id is Organization.organization_id (X-Organization-ID), not the DB pk. - sample.env: document that pg_queue needs all three (gate + FLIPT_SERVICE_AVAILABLE + flag). - +2 tests (missing-org → celery, gate-on + Flipt-unavailable → celery). Co-Authored-By: Claude Opus 4.8 * UN-3570 address greptile: organization_id None type + FLIPT_SERVICE_AVAILABLE parse parity - organization_id annotation widened to `str | UUID | None` — the helper path passes UserContext.get_organization_identifier() which can be None at runtime; the existing `if not organization_id` guard handles it, so the type should admit None rather than mislead callers/static analysis. - FLIPT_SERVICE_AVAILABLE check now parses exactly like FliptClient (`.lower()`, no `.strip()`) so the two can't disagree on a whitespaced value like " true" (which would otherwise skip the "Flipt blind" warning). Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- backend/backend/settings/base.py | 8 + backend/sample.env | 7 + .../workflow_manager/internal_api_views.py | 12 +- .../workflow_v2/tests/test_transport.py | 139 ++++++++++++++-- .../workflow_manager/workflow_v2/transport.py | 154 +++++++++++++++--- .../workflow_v2/workflow_helper.py | 17 +- 6 files changed, 296 insertions(+), 41 deletions(-) diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index 787a7603e6..4b3fe0ce90 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -151,6 +151,14 @@ def get_required_setting(setting_key: str, default: str | None = None) -> str | CELERY_BACKEND_DB_NAME = os.environ.get("CELERY_BACKEND_DB_NAME") or DB_NAME DEFAULT_ORGANIZATION = "default_org" FLIPT_BASE_URL = os.environ.get("FLIPT_BASE_URL", "http://localhost:9005") +# 9e PG-queue transport master-gate (kill-switch). When not "true", +# resolve_transport() never consults Flipt and every workflow execution rides +# Celery — the instant global rollback AND the deploy-ordering safety (stays off +# until PG consumers are running in the fleet). See +# workers/queue_backend/pg_queue/9e-design.md §2. +PG_QUEUE_TRANSPORT_ENABLED = CommonUtils.str_to_bool( + os.environ.get("PG_QUEUE_TRANSPORT_ENABLED", "False") +) PLATFORM_HOST = os.environ.get("PLATFORM_SERVICE_HOST", "http://localhost") PLATFORM_PORT = os.environ.get("PLATFORM_SERVICE_PORT", 3001) PROMPT_HOST = os.environ.get("PROMPT_HOST", "http://localhost") diff --git a/backend/sample.env b/backend/sample.env index 377016fdec..80406dfcd4 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -184,6 +184,13 @@ TOOL_REGISTRY_CONFIG_PATH="/data/tool_registry_config" # Flipt Service FLIPT_SERVICE_AVAILABLE=False +# 9e PG-queue transport master-gate (kill-switch). Routing an execution onto the +# PG queue requires ALL THREE: this gate True, FLIPT_SERVICE_AVAILABLE=True (else +# the Flipt client returns False for every flag), and the Flipt flag +# `pg_queue_execution_enabled` on. Keep False until PG queue consumers are running +# in the fleet; set False for instant rollback. +PG_QUEUE_TRANSPORT_ENABLED=False + # File System Configuration for Workflow and API Execution # Directory Prefixes for storing execution files diff --git a/backend/workflow_manager/internal_api_views.py b/backend/workflow_manager/internal_api_views.py index ac81e31aa0..bdc09a0a50 100644 --- a/backend/workflow_manager/internal_api_views.py +++ b/backend/workflow_manager/internal_api_views.py @@ -323,10 +323,14 @@ def create_workflow_execution(request): # Resolve the transport this execution rides (9e). Decided once here, at # the creation chokepoint, and returned so the caller carries it in the - # dispatched task's payload — not persisted on the row. PR 1 always - # resolves "celery"; PR 3 wires Flipt (keyed on workflow/pipeline/org) in - # resolve_transport(). - transport = resolve_transport() + # dispatched task's payload — not persisted on the row. Flipt (gated by + # the env master-switch) decides; fails closed to "celery". + transport = resolve_transport( + execution_id=str(execution.id), + organization_id=org_id, + workflow_id=str(workflow.id), + pipeline_id=data.get("pipeline_id"), + ) return Response( { diff --git a/backend/workflow_manager/workflow_v2/tests/test_transport.py b/backend/workflow_manager/workflow_v2/tests/test_transport.py index 328850638e..2b0b100f63 100644 --- a/backend/workflow_manager/workflow_v2/tests/test_transport.py +++ b/backend/workflow_manager/workflow_v2/tests/test_transport.py @@ -1,18 +1,28 @@ -"""Tests for the 9e transport-resolution seam. +"""Tests for the 9e transport-resolution seam (PR 3 — Flipt canary wiring). -PR 1 is the *inert* seam: ``resolve_transport`` always returns Celery, so the -whole pipeline is byte-identical to today. These tests pin that contract so a -future change (PR 3, Flipt wiring) can't silently flip the default. +``resolve_transport`` is the single chokepoint that decides whether a new +execution rides the legacy Celery transport or the Postgres queue. It is gated +by an env master-switch and, when that is on, by a Flipt boolean flag; it fails +closed to Celery on any problem. These tests pin that contract. """ -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch +import pytest +from django.test import override_settings from unstract.core.data_models import ( DEFAULT_WORKFLOW_TRANSPORT, WorkflowTransport, normalize_transport, ) -from workflow_manager.workflow_v2.transport import resolve_transport + +from workflow_manager.workflow_v2.transport import ( + PG_QUEUE_FLAG_KEY, + resolve_transport, +) + +# Where ``check_feature_flag_status`` is looked up (imported into the module). +_FLIPT = "workflow_manager.workflow_v2.transport.check_feature_flag_status" class TestWorkflowTransportEnum: @@ -25,14 +35,119 @@ def test_default_is_celery(self): class TestResolveTransport: - def test_resolves_celery_in_pr1(self): - """PR 1: always Celery (inert seam, no inputs). PR 3 adds the - workflow/pipeline/org params + Flipt evaluation.""" - assert resolve_transport() == WorkflowTransport.CELERY.value - + @pytest.fixture(autouse=True) + def _flipt_available(self, monkeypatch): + """The gate-ON path short-circuits to celery when Flipt is marked + unavailable; default it available so the flag-evaluation tests exercise + the real path. Individual tests override as needed.""" + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + + @override_settings(PG_QUEUE_TRANSPORT_ENABLED=False) + def test_master_gate_off_never_consults_flipt(self): + """Master-gate off → Celery, and Flipt is not even called.""" + with patch(_FLIPT) as flipt: + result = resolve_transport(execution_id="e1", organization_id="org1") + assert result == WorkflowTransport.CELERY.value + flipt.assert_not_called() + + @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) + def test_missing_organization_forces_celery(self): + """No org context → can't segment safely → fail closed; Flipt not called + (str(None)/"" must never reach the Flipt org segment).""" + with patch(_FLIPT) as flipt: + result = resolve_transport(execution_id="e1", organization_id="") + assert result == WorkflowTransport.CELERY.value + flipt.assert_not_called() + + @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) + def test_gate_on_but_flipt_unavailable_forces_celery(self, monkeypatch): + """Gate ON + Flipt service unavailable → celery, loudly — not a silent + masquerade as a healthy 100%-celery canary.""" + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false") + with patch(_FLIPT) as flipt: + result = resolve_transport(execution_id="e1", organization_id="org1") + assert result == WorkflowTransport.CELERY.value + flipt.assert_not_called() + + @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) + def test_gate_on_flipt_true_resolves_pg_queue(self): + with patch(_FLIPT, return_value=True): + result = resolve_transport(execution_id="e1", organization_id="org1") + assert result == WorkflowTransport.PG_QUEUE.value + + @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) + def test_gate_on_flipt_false_resolves_celery(self): + with patch(_FLIPT, return_value=False): + result = resolve_transport(execution_id="e1", organization_id="org1") + assert result == WorkflowTransport.CELERY.value + + @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) + def test_flipt_exception_fails_closed_to_celery(self): + """A Flipt outage must never break execution creation.""" + with patch(_FLIPT, side_effect=RuntimeError("flipt down")): + result = resolve_transport(execution_id="e1", organization_id="org1") + assert result == WorkflowTransport.CELERY.value + + @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) + def test_passes_execution_id_as_entity_and_builds_context(self): + """entity_id = execution_id (sticky bucketing); context carries the + org/workflow/pipeline for segment rules. + """ + with patch(_FLIPT, return_value=True) as flipt: + resolve_transport( + execution_id="exec-42", + organization_id="org1", + workflow_id="wf1", + pipeline_id="pl1", + ) + flipt.assert_called_once_with( + flag_key=PG_QUEUE_FLAG_KEY, + entity_id="exec-42", + context={ + "organization_id": "org1", + "workflow_id": "wf1", + "pipeline_id": "pl1", + }, + ) + + @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) + def test_non_string_ids_are_coerced_for_flipt(self): + """Callers pass UUID objects for the ids. Flipt's context is a gRPC + map and entity_id must hash stably, so every value must + reach check_feature_flag_status as a plain str — otherwise the client + swallows the serialization error as False and silently forces celery. + Regression test for that exact dev-test finding. + """ + import uuid + + ex = uuid.UUID("8c091789-9dde-45e7-bb85-06f23fe120eb") + wf = uuid.UUID("ebed2834-c9fb-4b6c-8df3-9dd841f616bb") + pl = uuid.UUID("eaca3b0e-083a-4c75-8b25-85349d54145b") + with patch(_FLIPT, return_value=True) as flipt: + result = resolve_transport( + execution_id=ex, organization_id="org1", workflow_id=wf, pipeline_id=pl + ) + assert result == WorkflowTransport.PG_QUEUE.value + _, kwargs = flipt.call_args + assert kwargs["entity_id"] == str(ex) + assert all(isinstance(v, str) for v in kwargs["context"].values()) + assert kwargs["context"]["workflow_id"] == str(wf) + assert kwargs["context"]["pipeline_id"] == str(pl) + + @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) + def test_context_omits_unset_optional_ids(self): + with patch(_FLIPT, return_value=True) as flipt: + resolve_transport(execution_id="exec-42", organization_id="org1") + _, kwargs = flipt.call_args + assert kwargs["context"] == {"organization_id": "org1"} + + @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) def test_result_is_a_valid_transport_value(self): valid = {t.value for t in WorkflowTransport} - assert resolve_transport() in valid + with patch(_FLIPT, return_value=True): + assert ( + resolve_transport(execution_id="e1", organization_id="org1") in valid + ) class TestNormalizeTransport: diff --git a/backend/workflow_manager/workflow_v2/transport.py b/backend/workflow_manager/workflow_v2/transport.py index dfb22a181e..9b0796fd06 100644 --- a/backend/workflow_manager/workflow_v2/transport.py +++ b/backend/workflow_manager/workflow_v2/transport.py @@ -8,33 +8,149 @@ row: the payload is the single carrier, durable for PG via the queue row's JSONB, and the giant shared table is never migrated for this work. -PR 1 (this seam) hardwires the result to Celery, so behaviour is byte-identical -to today. PR 3 replaces the body with a Flipt evaluation (percentage rollout via -``entity_id`` hashing + per-org segment via ``context``), wrapped by an env -kill-switch and failing closed to Celery. +PR 3 (this change) replaces PR 1's hardwired Celery with a Flipt evaluation: + + master-gate (env) → Flipt boolean (``pg_queue_execution_enabled``) → transport + +Routing onto PG needs **all three** of: the env master-gate on, Flipt reachable +(``FLIPT_SERVICE_AVAILABLE=true``), and the flag enabled for this execution. + +- **Master-gate** (``settings.PG_QUEUE_TRANSPORT_ENABLED``, default off): until + ops flips it on, Flipt is never consulted and every execution rides Celery. + This is both the instant global kill-switch *and* the deploy-ordering safety — + the flag stays inert until PG consumers are actually running in the fleet. +- **Flipt** decides per-execution: ``entity_id = execution_id`` drives the + percentage-rollout hashing (an execution resolves exactly once, so it can + never re-bucket mid-flight); ``context`` carries org/workflow/pipeline for + segment rules. The flag contract is fixed in 9e-design §2. +- **Fail-closed to Celery**: a Flipt outage must never break execution creation, + mirroring ``normalize_transport`` on the read side. The gate-ON path logs its + decision so a "gate on but still all Celery" situation (e.g. a blind Flipt) + is visible rather than silent. """ from __future__ import annotations +import logging +import os +from typing import TYPE_CHECKING + +from django.conf import settings + from unstract.core.data_models import WorkflowTransport +from unstract.flags.feature_flag import check_feature_flag_status +if TYPE_CHECKING: + from uuid import UUID -def resolve_transport() -> str: +logger = logging.getLogger(__name__) + +# The fixed Flipt flag contract (9e-design §2): Boolean, default false. +PG_QUEUE_FLAG_KEY = "pg_queue_execution_enabled" + + +def resolve_transport( + *, + execution_id: str | UUID, + organization_id: str | UUID | None, + workflow_id: str | UUID | None = None, + pipeline_id: str | UUID | None = None, +) -> str: """Resolve the transport for a new workflow execution. + The id parameters are typed ``str | UUID`` because callers pass either + (e.g. the view path pre-coerces with ``str(...)``, the helper path passes + raw UUIDs). Coercion to the wire types Flipt needs is kept internal here so + callers never have to know about it. + + Args: + execution_id: The execution's id. Used as the Flipt ``entity_id`` so the + percentage rollout buckets per execution and stays sticky (one + execution is resolved exactly once). + organization_id: The org's string identifier + (``Organization.organization_id`` — the ``X-Organization-ID`` header + value, *not* the DB pk). May be ``None`` on the helper path + (``StateStore`` not populated) → resolves to celery. Carried in the + Flipt ``context`` for per-org segment rollouts. + workflow_id: Optional, carried in ``context`` for future segment rules. + pipeline_id: Optional, carried in ``context`` for future segment rules. + Returns: - The transport value (a :class:`WorkflowTransport` value string). - - Note: - PR 1 always returns ``"celery"`` and takes no arguments (the inert seam - needs no inputs). PR 3 reintroduces ``workflow_id`` / ``pipeline_id`` / - ``organization_id`` parameters when it wires Flipt here — percentage - rollout via ``entity_id`` hashing + per-org segment via ``context`` — - and updates the two call sites (``internal_api_views`` view and - ``workflow_helper.execute_workflow_async``). PR 3 must wrap the Flipt - evaluation in ``try/except`` and fall back to - ``WorkflowTransport.CELERY.value`` (fail-closed), so a Flipt outage can - never break execution creation — mirroring ``normalize_transport`` on - the read side. + A :class:`WorkflowTransport` value string — ``"pg_queue"`` only when the + master-gate is on, Flipt is reachable, and Flipt says yes for this + execution; ``"celery"`` otherwise (including any error — fail-closed). """ - return WorkflowTransport.CELERY.value + celery = WorkflowTransport.CELERY.value + + # Master-gate: until ops sets PG_QUEUE_TRANSPORT_ENABLED=true, never consult + # Flipt — every execution rides Celery (kill-switch + deploy-ordering safety). + # Intentionally unlogged: this is the steady state for every execution while + # the gate is off, so a log here would be pure noise. + if not settings.PG_QUEUE_TRANSPORT_ENABLED: + return celery + + # Gate is ON (canary/rollout). From here the decision is logged so a + # "gate on but everything still Celery" situation cannot hide. + + # No org context → per-org segment matching can't be trusted (str(None) would + # ship a bogus "None" org into the Flipt context and mis-segment). The view + # path validates the header non-None, but the helper path reads it from + # StateStore, which can be empty. Fail closed rather than mis-bucket. + if not organization_id: + logger.warning( + "resolve_transport: no organization_id for execution %s; forcing celery", + execution_id, + ) + return celery + + # FliptClient returns False for ALL flags when the service is marked + # unavailable — indistinguishable from "rollout says no". Surface it loudly so + # a blind Flipt under an ON gate doesn't masquerade as a healthy 100%-Celery + # canary. Parse exactly as FliptClient does (``.lower()``, no ``.strip()``) + # so the two can never disagree on a value like ``" true"``. + if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true": + logger.warning( + "resolve_transport: gate ON but FLIPT_SERVICE_AVAILABLE != true " + "(Flipt is blind) for execution %s; forcing celery", + execution_id, + ) + return celery + + # Flipt's context is a gRPC map; callers pass UUID objects for + # the ids, so coerce every value to str. A non-str value makes the client's + # serialization fail and check_feature_flag_status swallow it as False — + # silently forcing celery. entity_id is str-coerced for the same reason (and + # so the %-rollout hash is stable across str/UUID call sites). + context = {"organization_id": str(organization_id)} + if workflow_id: + context["workflow_id"] = str(workflow_id) + if pipeline_id: + context["pipeline_id"] = str(pipeline_id) + + # Defense in depth: check_feature_flag_status already wraps its body in + # try/except → False, but an explicit fail-closed wrap here keeps a future + # change to that helper from ever letting a Flipt problem break execution + # creation. + try: + enabled = check_feature_flag_status( + flag_key=PG_QUEUE_FLAG_KEY, + entity_id=str(execution_id), + context=context, + ) + except Exception: + logger.warning( + "resolve_transport: Flipt evaluation failed for execution %s; " + "falling back to Celery", + execution_id, + exc_info=True, + ) + return celery + + result = WorkflowTransport.PG_QUEUE.value if enabled else celery + logger.info( + "resolve_transport: execution %s resolved to %s (flipt enabled=%s)", + execution_id, + result, + enabled, + ) + return result diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py index 29a067a306..416373c9c1 100644 --- a/backend/workflow_manager/workflow_v2/workflow_helper.py +++ b/backend/workflow_manager/workflow_v2/workflow_helper.py @@ -510,17 +510,22 @@ def execute_workflow_async( log_events_id = StateStore.get(Common.LOG_EVENTS_ID) # Resolve the transport this execution rides (9e) and carry it in the # task payload — the pipeline reads it to stay on one transport - # end-to-end. PR 1 always resolves "celery" (no behaviour change). + # end-to-end. Flipt (gated by the env master-switch) decides; fails + # closed to "celery". # # NOTE (deliberate, two resolution sites): transport is resolved here # for the API/manual/async paths and separately in # internal_api_views.create_workflow_execution for the scheduler # path. These are DISTINCT entry paths — never a double-resolution of - # the same execution — so today they cannot diverge. PR 3 makes each - # an independent Flipt evaluation; before then, both must be funnelled - # through a single per-execution chokepoint (so a percentage re-roll - # can't split one execution across transports). Tracked for PR 3. - transport = resolve_transport() + # the same execution. entity_id is execution_id, so each execution + # buckets exactly once and can't split across transports even if the + # rollout % is re-rolled between the two sites. + transport = resolve_transport( + execution_id=execution_id, + organization_id=org_schema, + workflow_id=workflow_id, + pipeline_id=pipeline_id, + ) async_execution: AsyncResult = celery_app.send_task( "async_execute_bin", args=[ From 76f39d2861aa0cbf63f9c0949475110c89d54838 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Thu, 18 Jun 2026 14:14:28 +0530 Subject: [PATCH 21/44] =?UTF-8?q?UN-3574=20[FEAT]=20PG=20Queue=209e=202d?= =?UTF-8?q?=20=E2=80=94=20orchestrator=20(async=5Fexecute=5Fbin)=20on=20PG?= =?UTF-8?q?=20+=20shared=20TaskPayload=20contract=20(#2072)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3574 [FEAT] PG Queue 9e 2d — orchestrator (async_execute_bin) on PG + shared TaskPayload Route the orchestrator task onto the PG queue when transport==pg_queue, so a pg_queue execution's orchestrate → fan-out → barrier → callback all run on PG (was hybrid: orchestrator on Celery, only fan-out/callback on PG). - Promote TaskPayload + FairnessPayload (the PG-message wire contract) to unstract.core; workers re-export them so existing imports keep working and the backend producer + worker consumer share one definition. - Backend PG producer (pg_queue/producer.py): enqueue a TaskPayload row to pg_queue_message via the ORM, mirroring the workers' PgQueueClient.send. UUIDs in args/kwargs are JSON-coerced (the message JSONField has no Django encoder). - Backend dispatch (execute_workflow_async): when transport==pg_queue, enqueue async_execute_bin to PG (general → "celery", api → "celery_api_deployments") instead of celery_app.send_task; task_id becomes "pg:". - Scheduler dispatch: pass backend=QueueBackend.PG when is_pg_transport(transport) (dispatch() already had the per-call override from 2a). Gated off by default — Celery path unchanged. Executor (tool run) + log workers stay Celery (PR B). Tests: 5 producer unit tests; workers regression suite green after the core move. Dev-tested end-to-end: a real API deployment ran async_execute_bin on the orchestrator PG consumer (not Celery), then fan-out → barrier → callback on PG to COMPLETED, clean teardown. Co-Authored-By: Claude Opus 4.8 * UN-3574 address review: reduce complexity, consolidate fairness contract, fix dispatch fork SonarCloud (cognitive complexity 18→<15) + review round on #2072: - Extract WorkflowHelper._dispatch_orchestrator_task — the PG-vs-Celery fork moves out of execute_workflow_async (complexity down) and becomes unit-testable. - HIGH: a `dispatched` flag — a post-dispatch bookkeeping failure no longer flips a running (already-enqueued) execution to ERROR; only pre-dispatch failures do. - HIGH: task_id is now bare `str(msg_id)` (was `pg:{msg_id}`) — one format across entry paths, matching the worker PgDispatchHandle.id. - Drop the dead Celery-only TimeoutError handler (manual poll never raised it; the generic handler covers any stray case). - Consolidate the fairness contract in unstract.core: WorkloadType (StrEnum) + FAIRNESS_MIN/MAX/DEFAULT_PRIORITY + FairnessPayload.workload_type Literal; workers re-export, backend producer + workflow_helper reference them (no more hand-built "api"/"non_api" literals or triplicated [1,10] bounds). - Producer: log+re-raise on enqueue failure (parity with worker _enqueue_pg); document the TaskPayload.queue field as diagnostic-only. Tests: +dispatch-fork suite (PG vs Celery, bare id, two org sentinels), +producer boundary/datetime/failure cases, +scheduler backend=PG override assertions. 34 backend + 99 workers green. Co-Authored-By: Claude Opus 4.8 * UN-3574 address sonar: use logging.exception() in the dispatch error handlers Both handlers in execute_workflow_async were `logger.error(..., exc_info=True)` inside `except Exception` → switch to `logger.exception(...)` (idiomatic, and exc_info is implicit). The post-dispatch branch drops the redundant `{error}` from the message since the traceback is now attached. Co-Authored-By: Claude Opus 4.8 * UN-3574 address greptile: drop redundant stack_info=True from logger.exception logger.exception() already attaches the active exception's traceback; stack_info additionally dumps the current call stack, producing a second overlapping stack trace per error in a direct except handler. Drop it. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- backend/pg_queue/producer.py | 111 +++++++++++++ backend/pg_queue/tests/__init__.py | 0 backend/pg_queue/tests/test_producer.py | 113 +++++++++++++ .../tests/test_dispatch_orchestrator.py | 62 +++++++ .../workflow_v2/workflow_helper.py | 155 ++++++++++++------ .../core/src/unstract/core/data_models.py | 68 +++++++- workers/queue_backend/fairness.py | 54 +++--- .../queue_backend/pg_queue/task_payload.py | 24 +-- workers/scheduler/tasks.py | 7 +- .../test_dispatch_sites_characterisation.py | 26 +++ 10 files changed, 527 insertions(+), 93 deletions(-) create mode 100644 backend/pg_queue/producer.py create mode 100644 backend/pg_queue/tests/__init__.py create mode 100644 backend/pg_queue/tests/test_producer.py create mode 100644 backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py diff --git a/backend/pg_queue/producer.py b/backend/pg_queue/producer.py new file mode 100644 index 0000000000..c4e9be5cf1 --- /dev/null +++ b/backend/pg_queue/producer.py @@ -0,0 +1,111 @@ +"""Backend-side producer for the PG queue (orchestrator dispatch — 9e PR A / 2d). + +The workers own the consumer and the worker-side ``dispatch()`` PG producer; the +backend has the ``pg_queue`` tables but, until now, no way to *enqueue* to them — +it dispatched the orchestrator (``async_execute_bin``) only via Celery. When an +execution rides the ``pg_queue`` transport, the orchestrator itself must run on +PG, so the backend enqueues it here. + +Same ``pg_queue_message`` row shape as the workers' ``PgQueueClient.send`` (a +:class:`~unstract.core.data_models.TaskPayload` JSONB), written via the +``PgQueueMessage`` ORM (whose Python-level field defaults supply ``now()`` / ``0`` +for the vt/counter columns). The ``TaskPayload`` / ``FairnessPayload`` wire +contract is shared via ``unstract.core`` so producer and consumer agree on the +keys without one codebase importing the other. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any + +from pg_queue.models import PgQueueMessage +from unstract.core.data_models import ( + FAIRNESS_DEFAULT_PRIORITY, + FAIRNESS_MAX_PRIORITY, + FAIRNESS_MIN_PRIORITY, + FairnessPayload, + TaskPayload, +) + +logger = logging.getLogger(__name__) + +# Fairness L3 priority default, re-exported under the producer's name so callers +# (workflow_helper) import one symbol. Bounds + default are the single source of +# truth in unstract.core (shared with the workers' fairness/queue client); the DB +# CheckConstraint on pg_queue_message.priority is the writer-proof backstop. +DEFAULT_PRIORITY = FAIRNESS_DEFAULT_PRIORITY + +# Default/general queue name — mirrors the workers' ``QueueName.GENERAL = "celery"`` +# (the Celery default queue), used when the caller passes no explicit queue. +DEFAULT_GENERAL_QUEUE = "celery" + + +def _json_safe(value: Any) -> Any: + """Round-trip through JSON with ``default=str`` so UUIDs / datetimes in the + task args/kwargs become strings. + + ``PgQueueMessage.message`` is a plain ``JSONField`` (no Django encoder), and + the worker consumer already receives string ids on the existing PG dispatch + path, so coercing here keeps both transports consistent. + """ + return json.loads(json.dumps(value, default=str)) + + +def enqueue_task( + *, + task_name: str, + queue: str | None, + args: list[Any] | None = None, + kwargs: dict[str, Any] | None = None, + org_id: str = "", + priority: int = DEFAULT_PRIORITY, + fairness: FairnessPayload | None = None, +) -> int: + """Enqueue a task onto the PG queue; returns the new ``msg_id``. + + Mirrors ``queue_backend.pg_queue.client.PgQueueClient.send`` so the worker + consumer can decode and run it. A PG enqueue failure propagates — the caller + decides; for the orchestrator there is no silent Celery fallback (that would + hide the failure or risk a double-dispatch). + """ + if not FAIRNESS_MIN_PRIORITY <= priority <= FAIRNESS_MAX_PRIORITY: + raise ValueError( + f"priority out of range " + f"[{FAIRNESS_MIN_PRIORITY}, {FAIRNESS_MAX_PRIORITY}]: {priority!r}" + ) + pg_queue = queue or DEFAULT_GENERAL_QUEUE + message: TaskPayload = { + "task_name": task_name, + "args": _json_safe(list(args) if args is not None else []), + "kwargs": _json_safe(dict(kwargs) if kwargs is not None else {}), + "queue": pg_queue, + "fairness": fairness, + } + # Mirror the worker _enqueue_pg path: log the failure with breadcrumbs before + # it propagates, so a DB/constraint/serialization error isn't mislabeled by + # the caller's broad handler. + try: + row = PgQueueMessage.objects.create( + queue_name=pg_queue, + message=message, + org_id=org_id or "", + priority=priority, + ) + except Exception: + logger.exception( + "PG-queue: failed to enqueue task=%r queue=%r org=%r", + task_name, + pg_queue, + org_id, + ) + raise + logger.info( + "PG-queue: enqueued task=%r queue=%r msg_id=%s org=%r", + task_name, + pg_queue, + row.msg_id, + org_id, + ) + return row.msg_id diff --git a/backend/pg_queue/tests/__init__.py b/backend/pg_queue/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/pg_queue/tests/test_producer.py b/backend/pg_queue/tests/test_producer.py new file mode 100644 index 0000000000..1d62f3b11e --- /dev/null +++ b/backend/pg_queue/tests/test_producer.py @@ -0,0 +1,113 @@ +"""Unit tests for the backend PG-queue producer (orchestrator dispatch, 9e PR A). + +DB-free: ``PgQueueMessage`` is mocked, so these pin the wire-shape contract and +the JSON-coercion logic without needing a test database. +""" + +import datetime +import uuid +from unittest.mock import MagicMock, patch + +import pytest + +from pg_queue import producer + +_MODEL = "pg_queue.producer.PgQueueMessage" + + +class TestEnqueueTask: + def test_builds_taskpayload_row(self): + with patch(_MODEL) as model: + model.objects.create.return_value = MagicMock(msg_id=4242) + msg_id = producer.enqueue_task( + task_name="async_execute_bin", + queue="celery_api_deployments", + args=["org", "wf", "exec"], + kwargs={"transport": "pg_queue"}, + org_id="org", + priority=5, + fairness={ + "org_id": "org", + "workload_type": "api", + "pipeline_priority": 5, + }, + ) + assert msg_id == 4242 + kw = model.objects.create.call_args.kwargs + assert kw["queue_name"] == "celery_api_deployments" + assert kw["org_id"] == "org" + assert kw["priority"] == 5 + msg = kw["message"] + assert msg["task_name"] == "async_execute_bin" + assert msg["queue"] == "celery_api_deployments" + assert msg["args"] == ["org", "wf", "exec"] + assert msg["kwargs"] == {"transport": "pg_queue"} + assert msg["fairness"]["workload_type"] == "api" + + def test_uuid_args_kwargs_are_json_coerced(self): + """PgQueueMessage.message is a plain JSONField → UUIDs in args/kwargs must + be coerced to str (the worker consumer receives string ids).""" + wf = uuid.UUID("ebed2834-c9fb-4b6c-8df3-9dd841f616bb") + with patch(_MODEL) as model: + model.objects.create.return_value = MagicMock(msg_id=1) + producer.enqueue_task( + task_name="async_execute_bin", + queue="celery", + args=[wf], + kwargs={"pipeline_id": wf}, + ) + msg = model.objects.create.call_args.kwargs["message"] + assert msg["args"] == [str(wf)] + assert msg["kwargs"] == {"pipeline_id": str(wf)} + assert all(isinstance(a, str) for a in msg["args"]) + + def test_none_queue_defaults_to_general(self): + with patch(_MODEL) as model: + model.objects.create.return_value = MagicMock(msg_id=1) + producer.enqueue_task(task_name="async_execute_bin", queue=None) + kw = model.objects.create.call_args.kwargs + assert kw["queue_name"] == producer.DEFAULT_GENERAL_QUEUE == "celery" + assert kw["message"]["queue"] == "celery" + + def test_empty_args_kwargs_and_no_fairness(self): + with patch(_MODEL) as model: + model.objects.create.return_value = MagicMock(msg_id=1) + producer.enqueue_task(task_name="t", queue="celery") + msg = model.objects.create.call_args.kwargs["message"] + assert msg["args"] == [] + assert msg["kwargs"] == {} + assert msg["fairness"] is None + + @pytest.mark.parametrize("priority", [1, 5, 10]) + def test_priority_boundary_values_accepted(self, priority): + with patch(_MODEL) as model: + model.objects.create.return_value = MagicMock(msg_id=1) + producer.enqueue_task(task_name="t", queue="celery", priority=priority) + assert model.objects.create.call_args.kwargs["priority"] == priority + + @pytest.mark.parametrize("priority", [0, 11, -1]) + def test_priority_out_of_range_raises(self, priority): + with pytest.raises(ValueError): + producer.enqueue_task(task_name="t", queue="celery", priority=priority) + + def test_default_priority_when_omitted(self): + with patch(_MODEL) as model: + model.objects.create.return_value = MagicMock(msg_id=1) + producer.enqueue_task(task_name="t", queue="celery") + assert model.objects.create.call_args.kwargs["priority"] == 5 # FAIRNESS_DEFAULT + + def test_json_safe_coerces_datetime(self): + dt = datetime.datetime(2026, 6, 18, 12, 0, 0) + with patch(_MODEL) as model: + model.objects.create.return_value = MagicMock(msg_id=1) + producer.enqueue_task( + task_name="t", queue="celery", kwargs={"when": dt} + ) + when = model.objects.create.call_args.kwargs["message"]["kwargs"]["when"] + assert isinstance(when, str) and "2026-06-18" in when + + def test_enqueue_failure_logs_and_propagates(self): + with patch(_MODEL) as model: + model.objects.create.side_effect = RuntimeError("db down") + with pytest.raises(RuntimeError): + producer.enqueue_task(task_name="t", queue="celery") diff --git a/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py b/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py new file mode 100644 index 0000000000..6b0a03553b --- /dev/null +++ b/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py @@ -0,0 +1,62 @@ +"""Tests for WorkflowHelper._dispatch_orchestrator_task — the PG-vs-Celery +orchestrator dispatch fork (9e 2d). + +DB-free: pg_enqueue_task and celery_app are mocked, so these pin the routing +decision (PG enqueue vs Celery send), the bare-msg_id task-id contract, the +WorkloadType value, and the two org sentinels. +""" + +from unittest.mock import MagicMock, patch + +from utils.constants import CeleryQueue +from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper + +_PG = "workflow_manager.workflow_v2.workflow_helper.pg_enqueue_task" +_CELERY = "workflow_manager.workflow_v2.workflow_helper.celery_app" + + +class TestDispatchOrchestratorTask: + def test_pg_transport_enqueues_to_pg_not_celery(self): + with patch(_PG, return_value=4242) as pg, patch(_CELERY) as celery: + task_id = WorkflowHelper._dispatch_orchestrator_task( + transport="pg_queue", + queue=CeleryQueue.CELERY_API_DEPLOYMENTS, + args=["a"], + kwargs={"transport": "pg_queue"}, + org_schema="org1", + ) + # bare msg_id, no "pg:" prefix — matches the worker PgDispatchHandle.id + assert task_id == "4242" + celery.send_task.assert_not_called() + kw = pg.call_args.kwargs + assert kw["task_name"] == "async_execute_bin" + assert kw["queue"] == CeleryQueue.CELERY_API_DEPLOYMENTS + assert kw["org_id"] == "org1" + assert kw["fairness"]["workload_type"] == "api" + + def test_pg_general_queue_uses_non_api_workload(self): + with patch(_PG, return_value=7) as pg, patch(_CELERY): + WorkflowHelper._dispatch_orchestrator_task( + transport="pg_queue", queue=None, args=[], kwargs={}, org_schema="org1" + ) + assert pg.call_args.kwargs["fairness"]["workload_type"] == "non_api" + + def test_empty_org_uses_two_sentinels(self): + """Row org_id column is NOT NULL → ""; fairness org_id is str|None → None.""" + with patch(_PG, return_value=1) as pg, patch(_CELERY): + WorkflowHelper._dispatch_orchestrator_task( + transport="pg_queue", queue=None, args=[], kwargs={}, org_schema="" + ) + kw = pg.call_args.kwargs + assert kw["org_id"] == "" + assert kw["fairness"]["org_id"] is None + + def test_celery_transport_uses_send_task_not_pg(self): + with patch(_PG) as pg, patch(_CELERY) as celery: + celery.send_task.return_value = MagicMock(id="celery-task-1") + task_id = WorkflowHelper._dispatch_orchestrator_task( + transport="celery", queue=None, args=[], kwargs={}, org_schema="org1" + ) + assert task_id == "celery-task-1" + pg.assert_not_called() + celery.send_task.assert_called_once() diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py index 416373c9c1..da868b76f2 100644 --- a/backend/workflow_manager/workflow_v2/workflow_helper.py +++ b/backend/workflow_manager/workflow_v2/workflow_helper.py @@ -8,11 +8,12 @@ from account_v2.constants import Common from api_v2.models import APIDeployment from celery import chord, current_task -from celery import exceptions as celery_exceptions from celery.result import AsyncResult from configuration.enums import ConfigKey from configuration.models import Configuration from django.db import IntegrityError +from pg_queue.producer import DEFAULT_PRIORITY as PG_DEFAULT_PRIORITY +from pg_queue.producer import enqueue_task as pg_enqueue_task from pipeline_v2.models import Pipeline from plugins import get_plugin from plugins.workflow_manager.workflow_v2.utils import WorkflowUtil @@ -26,6 +27,7 @@ from utils.user_context import UserContext from backend.celery_service import app as celery_app +from unstract.core.data_models import WorkloadType, is_pg_transport from unstract.workflow_execution.enums import LogStage from workflow_manager.endpoint_v2.destination import DestinationConnector from workflow_manager.endpoint_v2.dto import FileHash @@ -470,6 +472,51 @@ def _get_execution_status( ) return execution_cache.status + @staticmethod + def _dispatch_orchestrator_task( + *, + transport: str, + queue: str | None, + args: list[Any], + kwargs: dict[str, Any], + org_schema: str, + ) -> str | None: + """Dispatch ``async_execute_bin`` on the resolved transport; return its id. + + ``pg_queue`` → enqueue to ``pg_queue_message`` (a PG consumer runs it); + ``celery`` → ``celery_app.send_task``. The returned id is the **bare** + ``str(msg_id)`` (PG) or the Celery task id — one format across entry + paths, matching the worker PG dispatch (``PgDispatchHandle.id``). + """ + if is_pg_transport(transport): + is_api_execution = queue == CeleryQueue.CELERY_API_DEPLOYMENTS + msg_id = pg_enqueue_task( + task_name="async_execute_bin", + # None → "celery" (general); else celery_api_deployments. + queue=queue, + args=args, + kwargs=kwargs, + # Two sentinels, deliberately: the row's org_id column is NOT NULL + # (wants ""), while the fairness payload's org_id is str | None (a + # missing org means "no segment", not the literal "None"). + org_id=org_schema or "", + priority=PG_DEFAULT_PRIORITY, + fairness={ + "org_id": org_schema or None, + "workload_type": ( + WorkloadType.API.value + if is_api_execution + else WorkloadType.NON_API.value + ), + "pipeline_priority": PG_DEFAULT_PRIORITY, + }, + ) + return str(msg_id) + async_execution = celery_app.send_task( + "async_execute_bin", args=args, kwargs=kwargs, queue=queue + ) + return async_execution.id + @classmethod def execute_workflow_async( cls, @@ -502,6 +549,9 @@ def execute_workflow_async( Returns: ExecutionResponse: Existing status of execution """ + # Defined before the try so the handler can tell a pre-dispatch failure + # (mark ERROR) from a post-dispatch one (orchestrator already running). + dispatched = False try: file_hash_in_str = { key: value.to_json() for key, value in hash_values_of_files.items() @@ -526,48 +576,58 @@ def execute_workflow_async( workflow_id=workflow_id, pipeline_id=pipeline_id, ) - async_execution: AsyncResult = celery_app.send_task( - "async_execute_bin", - args=[ - org_schema, # schema_name - workflow_id, # workflow_id - execution_id, # execution_id - file_hash_in_str, # hash_values_of_files - ], - kwargs={ - "scheduled": False, - "execution_mode": None, - "pipeline_id": pipeline_id, - "log_events_id": log_events_id, - "use_file_history": use_file_history, - "llm_profile_id": llm_profile_id, - "hitl_queue_name": hitl_queue_name, - "hitl_packet_id": hitl_packet_id, - "custom_data": custom_data, - "transport": transport, - }, + dispatch_args = [ + org_schema, # schema_name + workflow_id, # workflow_id + execution_id, # execution_id + file_hash_in_str, # hash_values_of_files + ] + dispatch_kwargs = { + "scheduled": False, + "execution_mode": None, + "pipeline_id": pipeline_id, + "log_events_id": log_events_id, + "use_file_history": use_file_history, + "llm_profile_id": llm_profile_id, + "hitl_queue_name": hitl_queue_name, + "hitl_packet_id": hitl_packet_id, + "custom_data": custom_data, + "transport": transport, + } + # Orchestrator transport (9e PR A / 2d): dispatch async_execute_bin on + # the resolved transport (PG enqueue vs Celery). Extracted to a helper + # so this method stays simple and the fork is unit-testable. + task_id = cls._dispatch_orchestrator_task( + transport=transport, queue=queue, + args=dispatch_args, + kwargs=dispatch_kwargs, + org_schema=org_schema or "", ) - logger.info( - f"[{org_schema}] Job '{async_execution}' has been enqueued for " - f"execution_id '{execution_id}', '{len(hash_values_of_files)}' files" - ) + # Past this point the orchestrator is on its transport: a failure in + # the bookkeeping below must NOT flip the (now-running) row to ERROR. + dispatched = True + workflow_execution: WorkflowExecution = WorkflowExecution.objects.get( id=execution_id ) - if not async_execution.id: + if not task_id: + # PG always yields a truthy msg_id, so an empty id is Celery-only; + # keep the transport in the message so operators can tell. logger.warning( - f"[{org_schema}] Celery returned empty task_id for execution_id '{execution_id}'. " + f"[{org_schema}] Empty task_id (transport={transport}) for " + f"execution_id '{execution_id}'." ) # Continue without setting task_id - execution can still complete else: # Use existing method to handle task_id setting with validation WorkflowExecutionServiceHelper.update_execution_task( - execution_id=execution_id, task_id=async_execution.id + execution_id=execution_id, task_id=task_id ) logger.info( - f"[{org_schema}] Job '{async_execution.id}' has been enqueued for " - f"execution_id '{execution_id}', '{len(hash_values_of_files)}' files" + f"[{org_schema}] Job '{task_id}' enqueued (transport={transport}) " + f"for execution_id '{execution_id}', " + f"'{len(hash_values_of_files)}' files" ) execution_status = workflow_execution.status @@ -581,36 +641,39 @@ def execute_workflow_async( ) if ExecutionStatus.is_completed(execution_status): # Fetch the object agian to get the latest status. - workflow_execution: WorkflowExecution = WorkflowExecution.objects.get( - id=execution_id - ) + workflow_execution = WorkflowExecution.objects.get(id=execution_id) task_result = ResultCacheUtils.get_api_results( workflow_id=workflow_id, execution_id=execution_id ) cls._set_result_acknowledge(workflow_execution) else: task_result = None - execution_response = ExecutionResponse( + return ExecutionResponse( workflow_id, execution_id, execution_status, result=task_result, ) - return execution_response - except celery_exceptions.TimeoutError: - return ExecutionResponse( - workflow_id, - execution_id, - async_execution.status, - message=WorkflowMessages.CELERY_TIMEOUT_MESSAGE, - ) except Exception as error: + if dispatched: + # The orchestrator is already enqueued/running on its transport, so + # a post-dispatch bookkeeping failure (status read / poll / result + # fetch) must not mark the execution ERROR — the orchestrator owns + # the status now. (The old Celery-only TimeoutError handler is + # dropped: the wait loop is a manual poll, not AsyncResult.get, so + # it never raised; this generic handler covers any stray case.) + logger.exception( + f"[{org_schema}] Post-dispatch bookkeeping failed for execution " + f"'{execution_id}' (orchestrator already dispatched on " + f"{transport}); not marking ERROR" + ) + return ExecutionResponse( + workflow_id, execution_id, ExecutionStatus.EXECUTING.value + ) WorkflowExecutionServiceHelper.update_execution_err(execution_id, str(error)) - logger.error( + logger.exception( f"Error while enqueuing async job for WF '{workflow_id}', " - f"execution '{execution_id}': {str(error)}", - exc_info=True, - stack_info=True, + f"execution '{execution_id}'" ) return ExecutionResponse( workflow_id, diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py index 20e3d0c3aa..94e6148305 100644 --- a/unstract/core/src/unstract/core/data_models.py +++ b/unstract/core/src/unstract/core/data_models.py @@ -9,8 +9,8 @@ import uuid from dataclasses import asdict, dataclass, field from datetime import UTC, datetime -from enum import Enum -from typing import Any +from enum import Enum, StrEnum +from typing import Any, Literal, TypedDict logger = logging.getLogger(__name__) @@ -260,6 +260,70 @@ def is_pg_transport(transport: str | None) -> bool: return transport == WorkflowTransport.PG_QUEUE.value +class WorkloadType(StrEnum): + """Workflow-execution workload class (fairness L2). Binary api-vs-not. + + Shared single source of truth: the workers' ``queue_backend.fairness`` + re-exports this, and the backend producer references its values, so neither + side hand-builds the wire strings. A ``str`` Enum serialises to its value + under ``json.dumps`` / the queue payload. + """ + + API = "api" + NON_API = "non_api" + + +# Fairness L3 priority bounds (1..10, higher = claimed sooner). Single source of +# truth shared by the backend producer and the workers' fairness/queue client — +# the DB CheckConstraint on pg_queue_message.priority is the writer-proof backstop. +FAIRNESS_MIN_PRIORITY = 1 +FAIRNESS_MAX_PRIORITY = 10 +FAIRNESS_DEFAULT_PRIORITY = 5 + + +class FairnessPayload(TypedDict): + """Serialised fairness key — the wire shape of a ``FairnessKey.to_dict()``. + + The PG-queue **backend↔worker contract**: the backend (producer) and the + workers' ``queue_backend.fairness`` (consumer) both reference this exact + sub-shape so they agree on the keys instead of a loose ``dict[str, Any]``. + Lives in ``unstract.core`` because backend and workers are separate + codebases that can't import each other — this is their single source of + truth. ``workload_type`` is a :class:`WorkloadType` value; + ``pipeline_priority`` is the 1..10 fairness L3 priority. + """ + + org_id: str | None + workload_type: Literal["api", "non_api"] + pipeline_priority: int + + +class TaskPayload(TypedDict): + """The ``message`` JSONB of a task row in ``pg_queue_message``. + + The PG-queue **producer↔consumer contract**: whoever enqueues a task (the + workers' ``dispatch()`` PG path, or the backend orchestrator producer) + serialises this; the consumer poll loop decodes it and runs the task. Keep + both sides reading the same keys. Shared in ``unstract.core`` so the backend + producer and the worker consumer agree on one definition. + + ``fairness`` is a :class:`FairnessPayload` (serialised fairness key) or + ``None`` — the consumer rebuilds the ``x-fairness-key`` header from it, + mirroring the Celery dispatch path. + + ``queue`` is diagnostic only: the consumer routes by the ``queue_name`` + *column* of the row, not by this field. Producers record the logical queue + here for debugging (the backend producer always sets it; the workers' + ``to_payload`` may leave it ``None``). + """ + + task_name: str + args: list[Any] + kwargs: dict[str, Any] + queue: str | None + fairness: FairnessPayload | None + + class FileListingResult: """Result of listing files from a source.""" diff --git a/workers/queue_backend/fairness.py b/workers/queue_backend/fairness.py index 160b393e7b..29e5afdfa5 100644 --- a/workers/queue_backend/fairness.py +++ b/workers/queue_backend/fairness.py @@ -8,33 +8,33 @@ from __future__ import annotations from dataclasses import dataclass -from enum import StrEnum -from typing import Final, TypedDict - - -class FairnessPayload(TypedDict): - """Serialised :class:`FairnessKey` (``to_dict()`` / wire shape). - - Shared so producers and the PG ``TaskPayload`` wire contract agree on - the exact sub-shape instead of a loose ``dict[str, Any]``. - """ - - org_id: str | None - workload_type: str - pipeline_priority: int - - -class WorkloadType(StrEnum): - """Workflow execution type. Labs L2 check is binary api-vs-not.""" - - API = "api" - NON_API = "non_api" - - -# pipeline_priority bounds per labs schema (1..10, higher = sooner). -MIN_PRIORITY: Final[int] = 1 -MAX_PRIORITY: Final[int] = 10 -DEFAULT_PRIORITY: Final[int] = 5 +from typing import Final + +# The wire shape, the WorkloadType enum, and the priority bounds now live in +# unstract.core (shared backend↔worker single source of truth); re-exported here +# so existing ``from ..fairness import FairnessPayload/WorkloadType/...`` imports +# keep working and the backend producer references the same definitions. +from unstract.core.data_models import ( + FAIRNESS_DEFAULT_PRIORITY as DEFAULT_PRIORITY, +) +from unstract.core.data_models import ( + FAIRNESS_MAX_PRIORITY as MAX_PRIORITY, +) +from unstract.core.data_models import ( + FAIRNESS_MIN_PRIORITY as MIN_PRIORITY, +) +from unstract.core.data_models import ( + FairnessPayload, + WorkloadType, +) + +__all__ = [ + "DEFAULT_PRIORITY", + "MAX_PRIORITY", + "MIN_PRIORITY", + "FairnessPayload", + "WorkloadType", +] # Header (not kwarg) so task-body signatures without **kwargs aren't broken. FAIRNESS_HEADER_NAME: Final[str] = "x-fairness-key" diff --git a/workers/queue_backend/pg_queue/task_payload.py b/workers/queue_backend/pg_queue/task_payload.py index 3a7cc5e7fd..919914bfae 100644 --- a/workers/queue_backend/pg_queue/task_payload.py +++ b/workers/queue_backend/pg_queue/task_payload.py @@ -11,28 +11,18 @@ from __future__ import annotations from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, TypedDict +from typing import TYPE_CHECKING, Any -from ..fairness import FairnessPayload +# TaskPayload now lives in unstract.core (shared backend↔worker wire contract); +# re-exported here so existing ``from .task_payload import TaskPayload`` imports +# keep working. The ``to_payload`` builder stays worker-side (it depends on the +# worker-only ``FairnessKey``). +from unstract.core.data_models import TaskPayload if TYPE_CHECKING: from ..fairness import FairnessKey - -class TaskPayload(TypedDict): - """Everything the 9c consumer needs to run a PG-routed task. - - ``fairness`` is the serialised :class:`FairnessKey` - (:class:`~queue_backend.fairness.FairnessPayload`) or ``None`` — the - consumer rebuilds the ``x-fairness-key`` header from it, mirroring the - Celery dispatch path. - """ - - task_name: str - args: list[Any] - kwargs: dict[str, Any] - queue: str | None - fairness: FairnessPayload | None +__all__ = ["TaskPayload", "to_payload"] def to_payload( diff --git a/workers/scheduler/tasks.py b/workers/scheduler/tasks.py index 7518a4b930..789f7cac90 100644 --- a/workers/scheduler/tasks.py +++ b/workers/scheduler/tasks.py @@ -7,7 +7,7 @@ import traceback from typing import Any -from queue_backend import FairnessKey, dispatch, worker_task +from queue_backend import FairnessKey, QueueBackend, dispatch, worker_task from queue_backend.fairness import WorkloadType from shared.enums.status_enums import PipelineStatus from shared.enums.worker_enums import QueueName @@ -27,6 +27,7 @@ DEFAULT_WORKFLOW_TRANSPORT, NotificationPayload, NotificationSource, + is_pg_transport, normalize_transport, ) @@ -182,6 +183,10 @@ def _execute_scheduled_workflow( "transport": transport, }, queue=QueueName.GENERAL, + # Orchestrator transport (9e PR A / 2d): route async_execute_bin + # onto PG for a pg_queue execution (carried-marker wins over the + # allow-list); None keeps the legacy Celery dispatch. + backend=QueueBackend.PG if is_pg_transport(transport) else None, fairness=FairnessKey( org_id=context.organization_id, workload_type=WorkloadType.NON_API, diff --git a/workers/tests/test_dispatch_sites_characterisation.py b/workers/tests/test_dispatch_sites_characterisation.py index 48ffe33811..f34d3d838e 100644 --- a/workers/tests/test_dispatch_sites_characterisation.py +++ b/workers/tests/test_dispatch_sites_characterisation.py @@ -256,6 +256,32 @@ def test_dispatch_carries_backend_resolved_transport(self): assert mock_dispatch.call_args.kwargs["kwargs"]["transport"] == "pg_queue" + def test_pg_transport_routes_dispatch_to_pg_backend(self): + """The one line that actually routes the scheduled orchestrator onto PG: + transport=="pg_queue" → dispatch(backend=QueueBackend.PG) (identity, not + the allow-list).""" + from queue_backend import QueueBackend + from scheduler.tasks import _execute_scheduled_workflow + + api = MagicMock() + api.create_workflow_execution.return_value = { + "execution_id": "exec-123", + "transport": "pg_queue", + } + with patch("scheduler.tasks.dispatch") as mock_dispatch: + _execute_scheduled_workflow(api, self._make_context()) + + assert mock_dispatch.call_args.kwargs["backend"] is QueueBackend.PG + + def test_celery_transport_leaves_backend_none(self): + """transport=="celery" → backend=None (legacy Celery dispatch unchanged).""" + from scheduler.tasks import _execute_scheduled_workflow + + with patch("scheduler.tasks.dispatch") as mock_dispatch: + _execute_scheduled_workflow(self._make_api_client(), self._make_context()) + + assert mock_dispatch.call_args.kwargs["backend"] is None + def test_no_dispatch_when_execution_creation_fails(self): """If api_client.create_workflow_execution returns no execution_id, the function bails out and never calls send_task.""" From 5f4cdf089104d6eb9b9fab14992469e79602b9c2 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Thu, 18 Jun 2026 15:34:03 +0530 Subject: [PATCH 22/44] =?UTF-8?q?UN-3566=20[FEAT]=20PG=20Queue=209f=20?= =?UTF-8?q?=E2=80=94=20multi-queue=20consumer=20+=20named=20run-worker.sh?= =?UTF-8?q?=20PG=20roles=20(#2073)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3566 [FEAT] PG Queue 9f — multi-queue consumer + named run-worker.sh PG roles Make PG-queue workers first-class in run-worker.sh, runnable individually or as a set, like the Celery workers — removing the per-process-env footgun. - Multi-queue consumer: PgQueueConsumer takes queue_names: list[str] and polls them round-robin (per-queue read preserves the dequeue index's top-N; no queue starves another). WORKER_PG_QUEUE_CONSUMER_QUEUE is comma-parsed; a single value stays a one-element list (back-compat with the existing leaf consumer). - run-worker.sh named roles (PG_CONSUMER_ROLES), each a registry-bound consumer with its source worker-type + queue list baked in (no manual env): pg-orchestrator-api / pg-orchestrator-general (async_execute_bin has distinct impls per registry → split), pg-fileproc / pg-callback (multi-queue, one process drains ETL+API). The role name rides in argv (python -m pg_queue_consumer ) so pgrep (-s/-k/-r) tells co-running roles apart. - `./run-worker.sh -d pg` launches the 4 pipeline roles + reaper (mirrors `all`); each role also runs/kills/restarts/tails individually. resolve_log_file + the -L pg set + status/kill enumerate the roles. Tests: round-robin aggregation, empty-queue-no-starve, empty-list reject, comma-parse (incl single-value back-compat); existing consumer tests updated to the list arg. Dev-tested: `./run-worker.sh -d pg` → status shows 4 roles + reaper RUNNING; an API execution drains orchestrator-api → fileproc (draining both file_processing and api_file_processing) → callback → COMPLETED; `-r pg-fileproc` restarts only that role, leaving pg-callback untouched. Co-Authored-By: Claude Opus 4.8 * UN-3566 run-worker.sh: accept multiple worker types + tighten generic consumer pgrep - Multiple positional worker types: `./run-worker.sh -d all pg` starts the Celery set and the PG set in one shot (loops WORKER_TYPES). Warns if >1 type is given without -d, since a non-detached set `wait`s and would block the rest. - Tighten the generic pg_queue_consumer pgrep to end-anchored so --status/-k/-r for the generic consumer no longer also match (and aggregate/kill) the named role consumers, which run as `python -m pg_queue_consumer ` and have their own role-anchored pattern. Co-Authored-By: Claude Opus 4.8 * UN-3566 address sonar: hoist PG role-name literals into constants Define PG_ROLE_ORCH_API / _ORCH_GENERAL / _FILEPROC / _CALLBACK once and reference them in PG_CONSUMER_ROLES, PG_QUEUE_MEMBERS and WORKERS, instead of repeating each role-name string literal 4×. Co-Authored-By: Claude Opus 4.8 * UN-3566 run-worker.sh: add 'celery' run alias for the Celery set (symmetric with 'pg') `./run-worker.sh -d celery` now runs the Celery set (== `all`, excludes the PG workers), so celery-only / pg-only / both read symmetrically: -d celery (Celery only) -d pg (PG only) -d celery pg | -d all pg (both) 'celery' maps to "all" in WORKERS → dispatches to run_all_workers, is skipped by list_core_worker_dirs (no phantom dir), and falls through to the restart-all branch under -r. The pre-existing `-L celery` log-tail alias is unaffected. Co-Authored-By: Claude Opus 4.8 * UN-3566 address review: -r pg kills roles, per-queue isolation, dedup, stale comments Critical/High + cleanup from the #2073 review round: - [Critical] `-r pg` (restart set) now kills the SAME members run_pg_queue_set launches (the 4 named roles + reaper), not just the generic consumer — was leaving the roles running, so a relaunch double-polled Postgres. - [High] poll_once isolates each queue: a read/handle failure on one queue is logged and skipped so the others still run AND the work already done this cycle still counts (no false empty-queue backoff after a partial failure). - [High] fixed the stale CELERY_SET comment that claimed "no 'celery' run alias". - [Medium-High] run_pg_queue_set start-failure teardown aggregates kill_one_worker returns and surfaces a survivor (mirrors the restart path). - [Medium] de-dup + copy queue_names (list(dict.fromkeys(...))) — a duplicate would double-read a queue; the copy stops a caller mutation bypassing validation. - [Medium] _parse_queue_list warns when it drops empty entries (config typo). - [Med/Low] comment fixes: only fileproc/callback are multi-queue; "4 roles + reaper" not "consumer + reaper"; "read once per cycle in list order" not round-robin. Cleanups: drop redundant export; cwd predicate uses PG_QUEUE_MEMBERS; spell out api_file_processing_callback in help. Tests: +one-queue-failing-isolation, +batch_size qty per queue, +duplicate dedup. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- workers/queue_backend/pg_queue/consumer.py | 73 ++++-- workers/run-worker.sh | 254 +++++++++++++++------ workers/tests/test_pg_queue_consumer.py | 116 +++++++--- 3 files changed, 330 insertions(+), 113 deletions(-) diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py index 62771921c4..cdf7a20836 100644 --- a/workers/queue_backend/pg_queue/consumer.py +++ b/workers/queue_backend/pg_queue/consumer.py @@ -70,7 +70,7 @@ class PgQueueConsumer: def __init__( self, - queue_name: str, + queue_names: list[str], *, client: PgQueueClient | None = None, app: Celery | None = None, @@ -97,7 +97,17 @@ def __init__( raise ValueError( f"backoff_max ({backoff_max}) must be >= poll_interval ({poll_interval})" ) - self.queue_name = queue_name + if not queue_names: + raise ValueError("queue_names must be a non-empty list") + # One process can drain several queues (9f) — e.g. a file_processing + # consumer drains both file_processing and api_file_processing. Each is + # read once per cycle in list order (poll_once): this prevents starvation + # (every queue gets a read each cycle) but does NOT equalize throughput — + # an always-full queue_names[0] claims its full batch and runs it before + # later queues. Copy + de-dup (order-preserving): a duplicate would + # double-read a queue per cycle, and storing the caller's list by + # reference would let a later mutation bypass the non-empty validation. + self.queue_names = list(dict.fromkeys(queue_names)) self._client = client if client is not None else PgQueueClient() self._app = app if app is not None else current_app self.batch_size = batch_size @@ -114,14 +124,31 @@ def __init__( self._last_poll_monotonic = time.monotonic() def poll_once(self) -> int: - """Claim + process one batch; returns the number of messages claimed.""" + """Claim + process one batch per queue (read once each, in list order); + returns the total number of messages claimed across all queues this cycle. + + Each queue is isolated: a read/handle failure on one queue is logged and + skipped so the others still get their turn, and the work already done this + cycle still counts (so run() doesn't take the empty-queue backoff path + after a partial failure). + """ self._last_poll_monotonic = time.monotonic() - messages = self._client.read( - self.queue_name, vt_seconds=self.vt_seconds, qty=self.batch_size - ) - for message in messages: - self._handle(message) - return len(messages) + total = 0 + for queue_name in self.queue_names: + try: + messages = self._client.read( + queue_name, vt_seconds=self.vt_seconds, qty=self.batch_size + ) + for message in messages: + self._handle(message) + total += len(messages) + except Exception: + logger.exception( + "PG-queue consumer: poll failed for queue %r; " + "continuing with the other queues", + queue_name, + ) + return total def _handle(self, message: QueueMessage) -> None: payload = message.message @@ -254,9 +281,9 @@ def run(self, *, install_signals: bool = True, require_tasks: bool = True) -> No name for name in self._app.tasks if not name.startswith("celery.") ) logger.info( - "PG-queue consumer started (queue=%r, batch=%s, vt=%ss) — " + "PG-queue consumer started (queues=%r, batch=%s, vt=%ss) — " "%d application task(s) registered: %s", - self.queue_name, + self.queue_names, self.batch_size, self.vt_seconds, len(app_tasks), @@ -278,7 +305,7 @@ def run(self, *, install_signals: bool = True, require_tasks: bool = True) -> No else: time.sleep(backoff) backoff = min(backoff * 2, self.backoff_max) - logger.info("PG-queue consumer stopped (queue=%r)", self.queue_name) + logger.info("PG-queue consumer stopped (queues=%r)", self.queue_names) def stop(self, *_: object) -> None: """Request a graceful stop after the current batch.""" @@ -296,6 +323,26 @@ def _install_signal_handlers(self) -> None: ) +def _parse_queue_list(raw: str) -> list[str]: + """Comma-separated queue list (9f). A single value stays a one-element list, + so the pre-9f single-queue config remains valid. Empty entries (a doubled or + trailing comma — almost always a config typo) are dropped with a warning so a + malformed list is diagnosable from the logs, not just by eyeballing. + """ + parts = [q.strip() for q in raw.split(",")] + queues = [q for q in parts if q] + dropped = len(parts) - len(queues) + if dropped: + logger.warning( + "PG-queue consumer: dropped %d empty queue name(s) from " + "WORKER_PG_QUEUE_CONSUMER_QUEUE=%r → %r", + dropped, + raw, + queues, + ) + return queues + + def main() -> None: logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) @@ -315,7 +362,7 @@ def _env(suffix: str, default: _T, cast: Callable[[str], _T]) -> _T: raise ValueError(f"Invalid {var}={raw!r}: {exc}") from exc consumer = PgQueueConsumer( - queue_name=_env("QUEUE", _DEFAULT_QUEUE, str), + queue_names=_env("QUEUE", [_DEFAULT_QUEUE], _parse_queue_list), batch_size=_env("BATCH", _DEFAULT_BATCH, int), vt_seconds=_env("VT_SECONDS", _DEFAULT_VT_SECONDS, int), poll_interval=_env("POLL_INTERVAL", _DEFAULT_POLL_INTERVAL, float), diff --git a/workers/run-worker.sh b/workers/run-worker.sh index 68bb23aa11..fee8131aab 100755 --- a/workers/run-worker.sh +++ b/workers/run-worker.sh @@ -36,15 +36,44 @@ readonly PG_QUEUE_SET="pg-queue" # pg-queue` complement correct by construction if a third member is ever added # (callers test membership via ${PG_QUEUE_MEMBERS[$dir]:-} instead of # hand-rolling the consumer/reaper pair). +# 9f: named PG-queue consumer ROLES. Each is a multi-queue consumer (one process +# draining several queues, mirroring the Celery file_processing/callback workers +# that each cover ETL+API) with its source worker-type + queue list baked in — so +# `./run-worker.sh -d pg-fileproc` needs no per-process env. Format: +# ";". The role name is passed as argv +# to `python -m pg_queue_consumer ` so pgrep (--status/-k/-r) tells the +# co-running roles apart (they are otherwise identical processes). +# The four coupled-pipeline stages, each a registry-bound consumer (mirroring the +# Celery api-deployment / general / file_processing / callback workers). The two +# orchestrator roles are split because async_execute_bin has DISTINCT impls per +# registry (api-deployment handles it as an API deployment; general routes by +# workflow type), so one consumer can't serve both — they bind different queues. +# fileproc/callback are multi-queue (one process drains both ETL + API queues). +# Role-name constants — the literal lives in one place and is referenced across +# PG_CONSUMER_ROLES / PG_QUEUE_MEMBERS / WORKERS. +readonly PG_ROLE_ORCH_API="pg-orchestrator-api" +readonly PG_ROLE_ORCH_GENERAL="pg-orchestrator-general" +readonly PG_ROLE_FILEPROC="pg-fileproc" +readonly PG_ROLE_CALLBACK="pg-callback" +declare -rA PG_CONSUMER_ROLES=( + ["$PG_ROLE_ORCH_API"]="api_deployment;celery_api_deployments" + ["$PG_ROLE_ORCH_GENERAL"]="general;celery" + ["$PG_ROLE_FILEPROC"]="file_processing;file_processing,api_file_processing" + ["$PG_ROLE_CALLBACK"]="callback;file_processing_callback,api_file_processing_callback" +) declare -rA PG_QUEUE_MEMBERS=( ["$PG_QUEUE_CONSUMER_TYPE"]=1 ["$PG_QUEUE_REAPER_TYPE"]=1 + ["$PG_ROLE_ORCH_API"]=1 + ["$PG_ROLE_ORCH_GENERAL"]=1 + ["$PG_ROLE_FILEPROC"]=1 + ["$PG_ROLE_CALLBACK"]=1 ) -# Log-tail alias for the Celery transport: every worker EXCEPT the PG-queue -# members — the *complement* of the 'pg-queue' tail alias, so the two -# transports' logs can be tailed separately (-L celery vs -L pg-queue). -# Tail-only — there is no 'celery' run alias; this set is started via 'all', -# which by design omits the opt-in PG-queue workers. +# The Celery transport set: every worker EXCEPT the PG-queue members — the +# *complement* of the 'pg-queue' set, so the two transports' logs can be tailed +# separately (-L celery vs -L pg-queue). Since 9f, 'celery' is BOTH a -L tail +# alias AND a run alias (WORKERS maps it to "all" → run_all_workers), symmetric +# with 'pg'; 'all' remains its synonym. Either way it omits the opt-in PG workers. readonly CELERY_SET="celery" # Available workers @@ -70,6 +99,12 @@ declare -A WORKERS=( ["pg-queue-consumer"]="$PG_QUEUE_CONSUMER_TYPE" ["$PG_QUEUE_CONSUMER_TYPE"]="$PG_QUEUE_CONSUMER_TYPE" ["pg-consumer"]="$PG_QUEUE_CONSUMER_TYPE" + # PG Queue named roles (9f) — coupled-pipeline consumers, queues + source + # worker-type baked in. Run individually like any worker, or via the 'pg' set. + ["$PG_ROLE_ORCH_API"]="$PG_ROLE_ORCH_API" + ["$PG_ROLE_ORCH_GENERAL"]="$PG_ROLE_ORCH_GENERAL" + ["$PG_ROLE_FILEPROC"]="$PG_ROLE_FILEPROC" + ["$PG_ROLE_CALLBACK"]="$PG_ROLE_CALLBACK" # PG Queue reaper — leader-elected recovery loop (barrier-orphan sweep) ["reaper"]="$PG_QUEUE_REAPER_TYPE" ["pg-queue-reaper"]="$PG_QUEUE_REAPER_TYPE" @@ -78,6 +113,10 @@ declare -A WORKERS=( ["$PG_QUEUE_SET"]="$PG_QUEUE_SET" ["pg"]="$PG_QUEUE_SET" ["all"]="all" + # 'celery' is a run alias for the Celery set (== 'all', which excludes the PG + # workers) — symmetric with 'pg'. Maps to "all" so it dispatches to + # run_all_workers and list_core_worker_dirs skips it (not a phantom dir). + ["$CELERY_SET"]="all" ) # Pluggable workers will be auto-discovered at runtime @@ -149,14 +188,18 @@ WORKER_TYPE: scheduler, schedule Run scheduler worker (scheduled pipeline tasks) executor Run executor worker (extraction execution tasks) ide-callback Run IDE callback worker (Prompt Studio post-execution callbacks) - pg-queue-consumer Run PG-queue poll-loop consumer (opt-in; not part of 'all') + pg-queue-consumer Run a generic PG-queue poll-loop consumer (env-configured; opt-in) + pg-orchestrator-api Run the PG orchestrator consumer for API execs (celery_api_deployments) + pg-orchestrator-general Run the PG orchestrator consumer for ETL/general execs (celery) + pg-fileproc Run the PG fan-out consumer (file_processing + api_file_processing) + pg-callback Run the PG callback consumer (file_processing_callback + api_file_processing_callback) reaper, pg-queue-reaper Run PG-queue reaper (leader-elected recovery; opt-in) - pg, pg-queue Run the PG-queue set (consumer + reaper) together - all Run all workers (in separate processes, includes auto-discovered pluggable workers) + pg, pg-queue Run the whole PG-queue set (the 4 pipeline roles + reaper) together + all, celery Run the Celery worker set (all Celery workers; excludes the PG set) Note: Pluggable workers in pluggable_worker/ directory are automatically discovered and can be run by name. -Note: 'all' is the Celery worker set; 'pg-queue' is the PG-queue set. They are independent — - run both in parallel for a dual-transport (strangler-fig) setup. +Note: 'all' (alias 'celery') is the Celery worker set; 'pg' ('pg-queue') is the PG-queue set. They are + independent — pass both (e.g. `-d all pg`) for a dual-transport (strangler-fig) setup. Note: pg-queue-consumer overrides: WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE (source worker whose tasks to load, default notification), WORKER_PG_QUEUE_CONSUMER_QUEUE (queue to poll), and WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT (liveness server port, default 8090). @@ -194,6 +237,9 @@ EXAMPLES: # Run file processing worker in background $0 -d file + # Start BOTH transports in one shot (Celery set + PG set) — use -d so neither blocks + $0 -d all pg + # Run with custom environment file $0 -e production.env all @@ -383,8 +429,16 @@ get_worker_pids() { # `-worker@host` process, so they have no `-worker` token to anchor on. # Match the module invocation instead (covers both the `uv run python` parent # and the `python -m` child). Keeps --status / -k / -r working for them. - if [[ "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" || "$worker_type" == "$PG_QUEUE_REAPER_TYPE" ]]; then - pattern="-m[[:space:]]+${worker_type}([[:space:]]|\$)" + if [[ -n "${PG_CONSUMER_ROLES[$worker_type]:-}" ]]; then + # 9f roles run as `python -m pg_queue_consumer ` — anchor on the + # role argv so co-running roles (and the generic consumer) don't collide. + pattern="-m[[:space:]]+${PG_QUEUE_CONSUMER_TYPE}[[:space:]]+${worker_type}([[:space:]]|\$)" + elif [[ "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" || "$worker_type" == "$PG_QUEUE_REAPER_TYPE" ]]; then + # End-anchored so the GENERIC consumer doesn't also match the role + # consumers (which run as `... -m pg_queue_consumer `); they have + # their own role-anchored pattern above. The reaper takes no role arg, so + # the same end anchor matches it cleanly. + pattern="-m[[:space:]]+${worker_type}[[:space:]]*\$" else pattern="[^[:alnum:]_]${worker_type}-worker(@|-)" fi @@ -443,10 +497,15 @@ resolve_log_file() { local worker_dir=$1 local core_path="$WORKERS_DIR/$worker_dir/$worker_dir.log" local pluggable_path="$WORKERS_DIR/pluggable_worker/$worker_dir/$worker_dir.log" + # 9f roles run from (and log under) the pg_queue_consumer launcher dir as + # ".log", since they share that dir rather than having their own. + local role_path="$WORKERS_DIR/$PG_QUEUE_CONSUMER_TYPE/$worker_dir.log" if [[ -f "$core_path" ]]; then echo "$core_path" elif [[ -f "$pluggable_path" ]]; then echo "$pluggable_path" + elif [[ -n "${PG_CONSUMER_ROLES[$worker_dir]:-}" && -f "$role_path" ]]; then + echo "$role_path" fi } @@ -477,8 +536,9 @@ tail_logs() { exit 1 fi if [[ "$canonical" == "$PG_QUEUE_SET" ]]; then - # The set alias maps to no single dir — tail both member logs. - for d in "$PG_QUEUE_CONSUMER_TYPE" "$PG_QUEUE_REAPER_TYPE"; do + # The set alias maps to no single dir — tail every member's log + # (the named role consumers + the reaper). + for d in "${!PG_CONSUMER_ROLES[@]}" "$PG_QUEUE_REAPER_TYPE"; do local member_log member_log=$(resolve_log_file "$d") [[ -n "$member_log" ]] && log_files+=("$member_log") @@ -623,6 +683,10 @@ run_worker() { if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" ]]; then # Pluggable worker - use subdirectory worker_dir="$WORKERS_DIR/pluggable_worker/$worker_type" + elif [[ -n "${PG_CONSUMER_ROLES[$worker_type]:-}" ]]; then + # 9f role consumers share the pg_queue_consumer launcher dir (there is no + # per-role directory; the role only selects queues + source worker-type). + worker_dir="$WORKERS_DIR/$PG_QUEUE_CONSUMER_TYPE" else # Core worker - use root directory worker_dir="$WORKERS_DIR/$worker_type" @@ -770,17 +834,26 @@ run_worker() { # PG queue consumer is a plain Python poll-loop (polls Postgres via # SKIP LOCKED, not a Celery/RabbitMQ worker) — override the celery command # with the bootstrapping launcher and route the queue via env. - if [[ "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" ]]; then + pg_consumer_role="${PG_CONSUMER_ROLES[$worker_type]:-}" + if [[ "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" || -n "$pg_consumer_role" ]]; then + # A named 9f role bakes in its source worker-type + multi-queue list + # (";"); the generic pg-queue-consumer reads them from env + # (default source worker = notification, the first migrated leaf task). + if [[ -n "$pg_consumer_role" ]]; then + # Plain assignment — the line below is the single export point (its + # :- default/override is load-bearing for the generic consumer path). + WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE="${pg_consumer_role%%;*}" + queues="${pg_consumer_role#*;}" + fi export WORKER_PG_QUEUE_CONSUMER_QUEUE="$queues" - # The consumer registers ONE source worker's tasks (the launcher sets - # WORKER_TYPE from this before `import worker`). Default: notification — - # the worker that owns the first migrated leaf task, - # send_webhook_notification; override to drain another worker's queue. export WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE="${WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE:-notification}" - # Liveness HTTP server port (-p override wins, else the map default). - # Exported so the launcher's main() opts into the health server. - export WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT="${health_port:-${WORKER_HEALTH_PORTS[$worker_type]}}" + # Liveness HTTP server port (-p override wins, else the map default; roles + # have no map entry → empty → the launcher skips the health server). + export WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT="${health_port:-${WORKER_HEALTH_PORTS[$worker_type]:-}}" cmd_args=("uv" "run" "python" "-m" "$PG_QUEUE_CONSUMER_TYPE") + # Tag the process with the role name so pgrep (--status/-k/-r) can tell + # co-running roles apart — they are otherwise identical `python -m` procs. + [[ -n "$pg_consumer_role" ]] && cmd_args+=("$worker_type") fi # PG queue reaper — a leader-elected SQL recovery loop (no Celery, no task @@ -804,9 +877,10 @@ run_worker() { # Change to appropriate directory # For pluggable workers, stay at workers root to allow module imports # For core workers, change to worker directory - if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" || "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" || "$worker_type" == "$PG_QUEUE_REAPER_TYPE" ]]; then + if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" || -n "${PG_QUEUE_MEMBERS[$worker_type]:-}" ]]; then # Run from the workers root so `python -m pg_queue_consumer` / - # `python -m pg_queue_reaper` (and what they import) resolve. + # `python -m pg_queue_reaper` (and what they import) resolve. PG_QUEUE_MEMBERS + # is the single source of truth for "which workers are PG-queue members". cd "$WORKERS_DIR" else cd "$worker_dir" @@ -902,7 +976,7 @@ run_all_workers() { fi } -# Function to run the PG-queue worker set (consumer + reaper) together. +# Function to run the PG-queue worker set (the 4 pipeline roles + reaper) together. # A set is multiple processes, so — like 'all' — it ALWAYS runs detached # (it ignores -d/--detach; there's no foreground form). The PG-queue counterpart # to 'all' (the Celery set): run both for a dual-transport (strangler-fig) setup. @@ -915,9 +989,13 @@ run_pg_queue_set() { local concurrency=$2 local pool_type=$3 - print_status $GREEN "Starting PG-queue set (consumer + reaper)..." + # The set = the named pipeline consumer roles (9f) + the reaper. The fan-out + # and callback roles are multi-queue (ETL+API); the two orchestrator roles + # each bind a single registry-specific queue (see PG_CONSUMER_ROLES). + local members=("${!PG_CONSUMER_ROLES[@]}" "$PG_QUEUE_REAPER_TYPE") + print_status $GREEN "Starting PG-queue set (${members[*]})..." local failed=0 - for worker in "$PG_QUEUE_CONSUMER_TYPE" "$PG_QUEUE_REAPER_TYPE"; do + for worker in "${members[@]}"; do print_status $BLUE "Starting $worker in background..." # A FOREGROUND subshell: run_worker (detach=true) nohup-backgrounds the # actual worker and returns 1 on an immediate crash-on-start — so the @@ -931,11 +1009,17 @@ run_pg_queue_set() { if [[ $failed -ne 0 ]]; then print_status $RED "PG-queue set: a member failed to start — tearing down the set (see logs above)" # Don't leave a survivor: a restart-on-failure relaunch would spawn a - # second instance on top of it (the consumer would double-poll Postgres). - # Kill both members (the crashed one is already gone → no-op). Same - # all-or-nothing discipline as the restart path. - kill_one_worker "$PG_QUEUE_CONSUMER_TYPE" - kill_one_worker "$PG_QUEUE_REAPER_TYPE" + # second instance on top of it (a consumer would double-poll Postgres). + # Kill all members (a crashed one is already gone → no-op). Same + # all-or-nothing discipline as the restart path — aggregate kill failures + # so a survivor (would double-poll Postgres) is surfaced, not silent. + local teardown_failed=0 + for worker in "${members[@]}"; do + kill_one_worker "$worker" || teardown_failed=1 + done + if [[ $teardown_failed -ne 0 ]]; then + print_status $RED "PG-queue set: a member survived SIGKILL during teardown — check for a duplicate consumer double-polling Postgres" + fi show_status return 1 fi @@ -956,6 +1040,10 @@ SHOW_STATUS=false LOGS_MODE=false CLEAR_LOGS_MODE=false RESTART_MODE=false +# Multiple positional worker types may be given (e.g. `all pg` to start both the +# Celery and PG sets in one shot). WORKER_TYPE stays the first for the +# single-target paths (-r/validation); the launch path loops WORKER_TYPES. +WORKER_TYPES=() while [[ $# -gt 0 ]]; do case $1 in @@ -1021,12 +1109,16 @@ while [[ $# -gt 0 ]]; do exit 1 ;; *) - WORKER_TYPE="$1" + WORKER_TYPES+=("$1") shift ;; esac done +# First positional drives the single-target paths (-r, validation messages); the +# launch path below iterates all of WORKER_TYPES. +WORKER_TYPE="${WORKER_TYPES[0]:-}" + # Handle special actions if [[ "$KILL_WORKERS" == "true" ]]; then kill_workers @@ -1065,13 +1157,18 @@ if [[ "$RESTART_MODE" == "true" ]]; then # consumer would double-poll Postgres. Mirrors kill_workers' discipline. print_status $BLUE "Restarting PG-queue set..." restart_failed=0 - kill_one_worker "$PG_QUEUE_CONSUMER_TYPE" || restart_failed=1 - kill_one_worker "$PG_QUEUE_REAPER_TYPE" || restart_failed=1 + # Kill the SAME members run_pg_queue_set launches (the named roles + + # reaper) — not the generic consumer — or a survivor + relaunch would + # double-poll Postgres. + for pg_member in "${!PG_CONSUMER_ROLES[@]}" "$PG_QUEUE_REAPER_TYPE"; do + kill_one_worker "$pg_member" || restart_failed=1 + done if [[ $restart_failed -ne 0 ]]; then print_status $RED "Cannot restart PG-queue set: a member survived SIGKILL; aborting to avoid duplicate processes" exit 1 fi - elif [[ -n "$WORKER_TYPE" && "$WORKER_TYPE" != "all" ]]; then + elif [[ -n "$WORKER_TYPE" && "$WORKER_TYPE" != "all" && "${WORKERS[$WORKER_TYPE]:-}" != "all" ]]; then + # ('celery' maps to "all" → falls through to the restart-all branch below.) restart_target_dir="${WORKERS[$WORKER_TYPE]:-${PLUGGABLE_WORKERS[$WORKER_TYPE]:-}}" if [[ -z "$restart_target_dir" ]]; then print_status $RED "Error: Unknown worker type for restart: $WORKER_TYPE" @@ -1085,11 +1182,16 @@ if [[ "$RESTART_MODE" == "true" ]]; then fi fi -# Validate worker type -if [[ -z "$WORKER_TYPE" ]]; then - print_status $RED "Error: Worker type is required" - usage - exit 1 +# A worker type is required. (restart-all set WORKER_TYPE=all without a positional +# → seed WORKER_TYPES from it so the launch loop below has a target.) +if [[ ${#WORKER_TYPES[@]} -eq 0 ]]; then + if [[ -n "$WORKER_TYPE" ]]; then + WORKER_TYPES=("$WORKER_TYPE") + else + print_status $RED "Error: Worker type is required" + usage + exit 1 + fi fi # Load environment @@ -1098,23 +1200,25 @@ load_env "$ENV_FILE" # Discover pluggable workers discover_pluggable_workers -# Validate worker type (check both core and pluggable workers) -if [[ -z "${WORKERS[$WORKER_TYPE]}" ]] && [[ -z "${PLUGGABLE_WORKERS[$WORKER_TYPE]}" ]]; then - print_status $RED "Error: Unknown worker type: $WORKER_TYPE" - print_status $BLUE "Available core workers: ${!WORKERS[*]}" - if [[ ${#PLUGGABLE_WORKERS[@]} -gt 0 ]]; then - # Show unique pluggable worker names (not aliases) - pluggable_names="" - for key in "${!PLUGGABLE_WORKERS[@]}"; do - value="${PLUGGABLE_WORKERS[$key]}" - if [[ "$key" == "$value" ]]; then - pluggable_names="$pluggable_names $value" - fi - done - print_status $BLUE "Available pluggable workers:$pluggable_names" +# Validate every requested worker type (core or pluggable) up front. +for wt in "${WORKER_TYPES[@]}"; do + if [[ -z "${WORKERS[$wt]:-}" && -z "${PLUGGABLE_WORKERS[$wt]:-}" ]]; then + print_status $RED "Error: Unknown worker type: $wt" + print_status $BLUE "Available core workers: ${!WORKERS[*]}" + if [[ ${#PLUGGABLE_WORKERS[@]} -gt 0 ]]; then + # Show unique pluggable worker names (not aliases) + pluggable_names="" + for key in "${!PLUGGABLE_WORKERS[@]}"; do + value="${PLUGGABLE_WORKERS[$key]}" + if [[ "$key" == "$value" ]]; then + pluggable_names="$pluggable_names $value" + fi + done + print_status $BLUE "Available pluggable workers:$pluggable_names" + fi + exit 1 fi - exit 1 -fi +done # Validate environment validate_env @@ -1122,19 +1226,25 @@ validate_env # Add PYTHONPATH for imports export PYTHONPATH="$WORKERS_DIR:${PYTHONPATH:-}" -# Run the requested worker(s) -if [[ "$WORKER_TYPE" == "all" ]]; then - run_all_workers "$DETACH" "$LOG_LEVEL" "$CONCURRENCY" "$POOL_TYPE" -elif [[ "${WORKERS[$WORKER_TYPE]:-}" == "$PG_QUEUE_SET" ]]; then - # The PG-queue set (consumer + reaper). Always backgrounded (multiple procs). - # Propagate a member start-failure to the script exit code — the reaper has - # no health port, so this is the only programmatic startup signal. - run_pg_queue_set "$LOG_LEVEL" "$CONCURRENCY" "$POOL_TYPE" || exit 1 -else - # Resolve worker directory name from either WORKERS or PLUGGABLE_WORKERS - WORKER_DIR_NAME="${WORKERS[$WORKER_TYPE]}" - if [[ -z "$WORKER_DIR_NAME" ]]; then - WORKER_DIR_NAME="${PLUGGABLE_WORKERS[$WORKER_TYPE]}" - fi - run_worker "$WORKER_DIR_NAME" "$DETACH" "$LOG_LEVEL" "$CONCURRENCY" "$CUSTOM_QUEUES" "$HEALTH_PORT" "$POOL_TYPE" "$CUSTOM_HOSTNAME" +# Multi-type launches (e.g. `all pg`) must be detached: a non-detached set +# (run_all_workers) `wait`s on its background jobs and would block any later type. +if [[ ${#WORKER_TYPES[@]} -gt 1 && "$DETACH" != "true" ]]; then + print_status $YELLOW "Multiple worker types given without -d/--detach: a foreground set will block the rest. Re-run with -d to start them all." fi + +# Run each requested worker / set in turn. +for wt in "${WORKER_TYPES[@]}"; do + if [[ "${WORKERS[$wt]:-}" == "all" ]]; then + # 'all' and its synonym 'celery' (the Celery set, excludes PG workers). + run_all_workers "$DETACH" "$LOG_LEVEL" "$CONCURRENCY" "$POOL_TYPE" + elif [[ "${WORKERS[$wt]:-}" == "$PG_QUEUE_SET" ]]; then + # The PG-queue set (pipeline roles + reaper). Always backgrounded (multiple + # procs). Propagate a member start-failure to the script exit code — the + # reaper has no health port, so this is the only programmatic startup signal. + run_pg_queue_set "$LOG_LEVEL" "$CONCURRENCY" "$POOL_TYPE" || exit 1 + else + # Resolve worker directory name from either WORKERS or PLUGGABLE_WORKERS + worker_dir_name="${WORKERS[$wt]:-${PLUGGABLE_WORKERS[$wt]}}" + run_worker "$worker_dir_name" "$DETACH" "$LOG_LEVEL" "$CONCURRENCY" "$CUSTOM_QUEUES" "$HEALTH_PORT" "$POOL_TYPE" "$CUSTOM_HOSTNAME" + fi +done diff --git a/workers/tests/test_pg_queue_consumer.py b/workers/tests/test_pg_queue_consumer.py index 372252770a..c1c50a66e8 100644 --- a/workers/tests/test_pg_queue_consumer.py +++ b/workers/tests/test_pg_queue_consumer.py @@ -54,7 +54,7 @@ class TestPollOnce: def test_runs_task_and_acks(self): client = MagicMock() client.read.return_value = [_msg(1, _ok_payload(3, 4))] - assert PgQueueConsumer("q", client=client).poll_once() == 1 + assert PgQueueConsumer(["q"], client=client).poll_once() == 1 assert _calls == [(3, 4)] # task body ran client.delete.assert_called_once_with(1) # acked @@ -64,7 +64,7 @@ def test_failed_task_is_not_acked_and_logs(self, caplog): _msg(2, {"task_name": "test_pg_consumer.boom", "args": [], "kwargs": {}}) ] with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"): - PgQueueConsumer("q", client=client).poll_once() + PgQueueConsumer(["q"], client=client).poll_once() client.delete.assert_not_called() # left for vt-expiry redelivery assert "failed" in caplog.text # the cycling signal is logged @@ -74,7 +74,7 @@ def test_unknown_task_is_dropped(self, caplog): _msg(3, {"task_name": "nope.nope", "args": [], "kwargs": {}}) ] with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"): - PgQueueConsumer("q", client=client).poll_once() + PgQueueConsumer(["q"], client=client).poll_once() client.delete.assert_called_once_with(3) # dropped, not redelivered assert "unknown task" in caplog.text @@ -82,7 +82,7 @@ def test_missing_task_name_dropped_as_malformed(self, caplog): client = MagicMock() client.read.return_value = [_msg(4, {"args": [], "kwargs": {}})] # no task_name with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"): - PgQueueConsumer("q", client=client).poll_once() + PgQueueConsumer(["q"], client=client).poll_once() client.delete.assert_called_once_with(4) assert "missing task_name" in caplog.text # distinct from "unknown task" @@ -93,7 +93,7 @@ def test_poison_message_dropped_past_max_attempts(self, caplog): _msg(5, {"task_name": "test_pg_consumer.boom"}, read_ct=6) ] with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"): - PgQueueConsumer("q", client=client, max_attempts=5).poll_once() + PgQueueConsumer(["q"], client=client, max_attempts=5).poll_once() client.delete.assert_called_once_with(5) # dropped as poison assert "poison" in caplog.text @@ -107,7 +107,7 @@ def test_fairness_header_rebuilt_for_run(self): client.read.return_value = [ _msg(6, {"task_name": "t", "args": [1], "kwargs": {"k": "v"}, "fairness": fairness}) ] - PgQueueConsumer("q", client=client, app=app).poll_once() + PgQueueConsumer(["q"], client=client, app=app).poll_once() kwargs = task.apply.call_args.kwargs assert kwargs["args"] == [1] assert kwargs["kwargs"] == {"k": "v"} @@ -118,7 +118,7 @@ def test_ack_finding_no_row_warns(self, caplog): client.delete.return_value = False # row already gone (vt expired mid-run) client.read.return_value = [_msg(7, _ok_payload(1))] with caplog.at_level(logging.WARNING, logger="queue_backend.pg_queue.consumer"): - PgQueueConsumer("q", client=client).poll_once() + PgQueueConsumer(["q"], client=client).poll_once() assert "possible double-run" in caplog.text def test_multi_message_batch(self): @@ -129,7 +129,7 @@ def test_multi_message_batch(self): _msg(12, {"task_name": "nope.nope"}), _msg(13, _ok_payload(2)), ] - assert PgQueueConsumer("q", client=client).poll_once() == 4 + assert PgQueueConsumer(["q"], client=client).poll_once() == 4 assert _calls == [(1, 0), (2, 0)] # ok tasks ran in order deleted = {c.args[0] for c in client.delete.call_args_list} assert deleted == {10, 12, 13} # ok acked + unknown dropped; boom NOT acked @@ -137,7 +137,7 @@ def test_multi_message_batch(self): def test_empty_batch_acks_nothing(self): client = MagicMock() client.read.return_value = [] - assert PgQueueConsumer("q", client=client).poll_once() == 0 + assert PgQueueConsumer(["q"], client=client).poll_once() == 0 client.delete.assert_not_called() @@ -151,13 +151,13 @@ def test_rejects_non_positive_params(self): {"max_attempts": 0}, ): with pytest.raises(ValueError): - PgQueueConsumer("q", client=MagicMock(), **kw) + PgQueueConsumer(["q"], client=MagicMock(), **kw) def test_rejects_backoff_max_below_poll_interval(self): # Otherwise backoff would shrink below poll_interval instead of growing. with pytest.raises(ValueError, match="backoff_max"): PgQueueConsumer( - "q", client=MagicMock(), poll_interval=0.5, backoff_max=0.1 + ["q"], client=MagicMock(), poll_interval=0.5, backoff_max=0.1 ) @@ -165,7 +165,7 @@ class TestRunLoop: def test_run_stops_gracefully(self, monkeypatch): client = MagicMock() client.read.return_value = [] # always empty → backoff path - consumer = PgQueueConsumer("q", client=client) + consumer = PgQueueConsumer(["q"], client=client) # First empty poll → sleep (patched to request stop) → loop exits. monkeypatch.setattr( "queue_backend.pg_queue.consumer.time.sleep", lambda _s: consumer.stop() @@ -178,7 +178,7 @@ def test_backoff_grows_then_resets(self, monkeypatch): # empty, empty, one message, empty → backoff doubles, resets, doubles. client.read.side_effect = [[], [], [_msg(1, _ok_payload(1))], []] consumer = PgQueueConsumer( - "q", client=client, poll_interval=0.1, backoff_max=0.25 + ["q"], client=client, poll_interval=0.1, backoff_max=0.25 ) sleeps: list[float] = [] @@ -196,7 +196,7 @@ def test_poll_error_does_not_kill_loop(self, monkeypatch): client = MagicMock() # first poll raises (transient), then empty → loop must survive the raise. client.read.side_effect = [RuntimeError("blip"), []] - consumer = PgQueueConsumer("q", client=client) + consumer = PgQueueConsumer(["q"], client=client) sleeps: list[float] = [] def _sleep(secs): @@ -214,7 +214,7 @@ def test_run_refuses_to_start_with_empty_registry(self): from celery import Celery empty_app = Celery("empty-no-tasks") # only celery.* built-ins - consumer = PgQueueConsumer("q", client=MagicMock(), app=empty_app) + consumer = PgQueueConsumer(["q"], client=MagicMock(), app=empty_app) with pytest.raises(RuntimeError, match="no application tasks"): consumer.run(install_signals=False) @@ -223,7 +223,7 @@ def test_run_starts_with_registered_tasks(self, monkeypatch): # the module's @shared_task tasks registered, so the guard passes. client = MagicMock() client.read.return_value = [] # empty → loop sleeps → we stop it - consumer = PgQueueConsumer("q", client=client) + consumer = PgQueueConsumer(["q"], client=client) monkeypatch.setattr( "queue_backend.pg_queue.consumer.time.sleep", lambda _s: consumer.stop() ) @@ -238,7 +238,7 @@ def test_run_bypasses_guard_when_require_tasks_false(self, monkeypatch): empty_app = Celery("empty-no-tasks") client = MagicMock() client.read.return_value = [] - consumer = PgQueueConsumer("q", client=client, app=empty_app) + consumer = PgQueueConsumer(["q"], client=client, app=empty_app) monkeypatch.setattr( "queue_backend.pg_queue.consumer.time.sleep", lambda _s: consumer.stop() ) @@ -252,7 +252,7 @@ def test_registered_task_count_excludes_celery_builtins(self): empty_app = Celery("empty-no-tasks") assert ( - PgQueueConsumer("q", client=MagicMock(), app=empty_app)._registered_task_count() + PgQueueConsumer(["q"], client=MagicMock(), app=empty_app)._registered_task_count() == 0 ) @@ -261,7 +261,7 @@ def _demo(): return 1 assert ( - PgQueueConsumer("q", client=MagicMock(), app=empty_app)._registered_task_count() + PgQueueConsumer(["q"], client=MagicMock(), app=empty_app)._registered_task_count() == 1 ) @@ -273,7 +273,7 @@ class TestPollHeartbeat: def test_poll_once_refreshes_heartbeat(self): client = MagicMock() client.read.return_value = [] - consumer = PgQueueConsumer("q", client=client) + consumer = PgQueueConsumer(["q"], client=client) # Simulate a long-idle consumer, then poll → heartbeat resets to ~now. consumer._last_poll_monotonic -= 120 assert consumer.seconds_since_last_poll() > 100 @@ -286,7 +286,7 @@ def test_heartbeat_stamped_before_read(self): # the probe. A bottom-of-poll stamp would pass test_poll_once_refreshes # but fail here. client = MagicMock() - consumer = PgQueueConsumer("q", client=client) + consumer = PgQueueConsumer(["q"], client=client) before = consumer._last_poll_monotonic seen: dict[str, float] = {} client.read.side_effect = lambda *a, **k: ( @@ -297,21 +297,21 @@ def test_heartbeat_stamped_before_read(self): assert seen["during"] > before # refreshed BEFORE read ran, not after def test_is_poll_stale_threshold(self): - consumer = PgQueueConsumer("q", client=MagicMock()) + consumer = PgQueueConsumer(["q"], client=MagicMock()) consumer._last_poll_monotonic -= 120 # last poll 120s ago assert consumer.is_poll_stale(60) is True # past threshold → stale assert consumer.is_poll_stale(200) is False # within threshold → fresh def test_fresh_consumer_is_not_stale(self): # Seeded at construction, so a just-started consumer reads healthy. - consumer = PgQueueConsumer("q", client=MagicMock()) + consumer = PgQueueConsumer(["q"], client=MagicMock()) assert consumer.is_poll_stale(60) is False def test_health_server_disabled_without_port(self): # No port configured → no server bound (opt-in). from queue_backend.pg_queue.consumer import _maybe_start_health_server - consumer = PgQueueConsumer("q", client=MagicMock()) + consumer = PgQueueConsumer(["q"], client=MagicMock()) assert _maybe_start_health_server(consumer, port=None, stale_after=60) is None def test_liveness_server_reports_200_then_503(self): @@ -323,7 +323,7 @@ def test_liveness_server_reports_200_then_503(self): from queue_backend.pg_queue.consumer import LivenessServer - consumer = PgQueueConsumer("q", client=MagicMock()) + consumer = PgQueueConsumer(["q"], client=MagicMock()) server = LivenessServer(consumer, port=0, stale_after=60) server.start() try: @@ -349,7 +349,7 @@ def test_liveness_aliases_and_unknown_path(self): from queue_backend.pg_queue.consumer import LivenessServer - consumer = PgQueueConsumer("q", client=MagicMock()) + consumer = PgQueueConsumer(["q"], client=MagicMock()) server = LivenessServer(consumer, port=0, stale_after=60) server.start() try: @@ -367,7 +367,7 @@ def test_liveness_aliases_and_unknown_path(self): def test_double_start_is_rejected(self): from queue_backend.pg_queue.consumer import LivenessServer - server = LivenessServer(PgQueueConsumer("q", client=MagicMock()), port=0, stale_after=60) + server = LivenessServer(PgQueueConsumer(["q"], client=MagicMock()), port=0, stale_after=60) server.start() try: with pytest.raises(RuntimeError, match="already started"): @@ -391,7 +391,7 @@ def test_enqueue_poll_execute_ack(self, pg_client): queue_name, to_payload("test_pg_consumer.ok", args=[5], kwargs={"y": 6}), ) - claimed = PgQueueConsumer(queue_name, client=pg_client).poll_once() + claimed = PgQueueConsumer([queue_name], client=pg_client).poll_once() assert claimed == 1 assert (5, 6) in _calls # task actually executed off Postgres # Row was acked (deleted) — nothing left to claim. @@ -404,5 +404,65 @@ def test_enqueue_poll_execute_ack(self, pg_client): pg_client.conn.commit() +class TestMultiQueue: + """9f: one consumer drains several queues round-robin.""" + + def test_round_robin_reads_each_queue_and_aggregates(self): + client = MagicMock() + # qa → 2 msgs, qb → 1 msg; poll_once returns the total across both. + client.read.side_effect = [ + [_msg(1, _ok_payload(1)), _msg(2, _ok_payload(2))], + [_msg(3, _ok_payload(3))], + ] + claimed = PgQueueConsumer(["qa", "qb"], client=client).poll_once() + assert claimed == 3 + # Read once per queue, in order. + assert [c.args[0] for c in client.read.call_args_list] == ["qa", "qb"] + + def test_empty_queue_does_not_starve_the_others(self): + client = MagicMock() + client.read.side_effect = [[], [_msg(1, _ok_payload(1))]] + assert PgQueueConsumer(["empty", "busy"], client=client).poll_once() == 1 + + def test_rejects_empty_queue_list(self): + with pytest.raises(ValueError, match="queue_names"): + PgQueueConsumer([], client=MagicMock()) + + def test_one_queue_failing_does_not_abort_the_others(self): + """A read failure on one queue is isolated: the already-acked work this + cycle still counts (so run() doesn't take the empty backoff path), and + the failure doesn't propagate out of poll_once.""" + client = MagicMock() + client.read.side_effect = [[_msg(1, _ok_payload(1))], RuntimeError("boom")] + claimed = PgQueueConsumer(["qa", "qb"], client=client).poll_once() + assert claimed == 1 # qa's message counted despite qb raising + assert _calls == [(1, 0)] # qa task body ran + client.delete.assert_called_once_with(1) # qa acked + + def test_read_uses_batch_size_qty_per_queue(self): + client = MagicMock() + client.read.return_value = [] + PgQueueConsumer(["qa", "qb"], client=client, batch_size=2).poll_once() + assert [c.kwargs["qty"] for c in client.read.call_args_list] == [2, 2] + + def test_duplicate_queues_are_deduped(self): + """A duplicate (e.g. env "q,q") must not double-read the same queue.""" + client = MagicMock() + client.read.return_value = [] + consumer = PgQueueConsumer(["q", "q"], client=client) + assert consumer.queue_names == ["q"] + consumer.poll_once() + assert client.read.call_count == 1 + + +def test_parse_queue_list(): + from queue_backend.pg_queue.consumer import _parse_queue_list + + assert _parse_queue_list("a") == ["a"] # single value → one-element (back-compat) + assert _parse_queue_list("a,b,c") == ["a", "b", "c"] + assert _parse_queue_list(" a , b ,c ") == ["a", "b", "c"] # whitespace stripped + assert _parse_queue_list("a,,b") == ["a", "b"] # empties dropped + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From b2ff1b5a8f8b26f4425814741a7e620627ef3f01 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Thu, 18 Jun 2026 17:43:02 +0530 Subject: [PATCH 23/44] =?UTF-8?q?UN-3576=20[FEAT]=20PG=20Queue=209g=20?= =?UTF-8?q?=E2=80=94=20docker-compose=20PG=20consumer=20+=20reaper=20servi?= =?UTF-8?q?ces=20(gated=20profile)=20(#2074)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3576 [FEAT] PG Queue 9g — docker-compose PG consumer + reaper services (gated profile) Make the already-built PG-queue pipeline runnable under docker compose, gated OFF by default, so it runs in containers (not just the host run-worker script). Designed one-service-per-component for a mechanical K8s mapping later. - run-worker-docker.sh (image ENTRYPOINT): recognise `pg-queue-consumer` and `pg-queue-reaper` launch types -> exec the dedicated Python module instead of building a Celery worker command. Dispatched before the Celery two-path; no Dockerfile change. - docker-compose.yaml: 5 services behind the off-by-default `pg-queue` profile (4 consumers: orchestrator-api / orchestrator-general / fileproc / callback, plus a single-instance reaper). Orchestrators + reaper are broker-free; fileproc + callback keep rabbitmq (they still hand off to the Celery executor / notifications until those are migrated). - sample.env: document the new env vars + profile; the backend transport gate stays off (ramping traffic is a later step). Non-regression: default `docker compose up` starts zero PG services. Even with them running, executions route to Celery until the gate + Flipt flag are set. Co-Authored-By: Claude Opus 4.8 * UN-3576 address review: guard PG interpreter, warn on missing consumer config, reject typo'd PG commands, comment accuracy - run-worker-docker.sh: ensure_pg_interpreter() fails loudly if the venv python is missing (avoids a silent restart:unless-stopped crash-loop); WARN when WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE / _QUEUE are unset (non-compose launches); add a `pg-*|*-reaper` catch arm so a typo'd PG command fails loudly instead of silently becoming a default Celery worker (log-consumer etc. still pass through — verified by routing test). - docker-compose.yaml: soften "broker-free" wording (fail-closed-to-Celery on a missing transport); spell out the executor/notification migration steps instead of the ③/④ markers; use the exact `pg_orchestrator_lock` name. Co-Authored-By: Claude Opus 4.8 * UN-3576 address SonarCloud: add explicit default (*) arm to PG-dispatch case The PG-queue dispatch case relied on an implicit no-match fall-through to the Celery logic below. SonarCloud flags a case without a default; add an explicit `*)` no-op arm documenting the intentional fall-through. Behaviour unchanged (verified by routing test: legit worker types still reach Celery, typo'd PG commands still rejected). Co-Authored-By: Claude Opus 4.8 * UN-3576 address greptile: scope PG near-miss catch to pg- prefix only Narrow the safety-net catch arm from `pg-*|*-reaper` to `pg-*` so it can never intercept a pluggable worker whose name ends in `-reaper` (e.g. a future bulk-reaper / log-history-reaper) — those now fall through to the Celery path. The `pg-` prefix is reserved for PG-queue components; the exact reaper aliases are already matched above, and pg-prefixed typos (pg-reapr, pg-queue-reapr) are still rejected loudly. Verified by routing test. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- docker/docker-compose.yaml | 163 +++++++++++++++++++++++++++++++++++ docker/sample.env | 23 +++++ workers/run-worker-docker.sh | 87 +++++++++++++++++++ 3 files changed, 273 insertions(+) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index b7b49a7618..a85d3e6ca6 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -541,6 +541,169 @@ services: - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config - prompt_studio_data:/app/prompt-studio-data + # =========================================================================== + # PG-queue services (Postgres-backed transport) — gated behind the `pg-queue` + # compose profile, so a plain `docker compose up` starts NONE of them and + # behaviour is identical to today. Bring them up with: + # docker compose --profile pg-queue up -d + # They consume from Postgres, not the broker. The orchestrator consumers and + # the reaper are broker-free for PG-routed executions (fan-out honors the + # carried transport; a missing transport fails closed to Celery, which these + # consumers aren't expected to receive). The fan-out and callback consumers + # still hand off to the Celery executor (tool-execution RPC) / Celery + # notifications, so they keep a rabbitmq dependency until the executor and + # notifications themselves move to PG. Each maps 1:1 to a future K8s Deployment. + # Executions still route to Celery until the backend gate + # (PG_QUEUE_TRANSPORT_ENABLED) + Flipt flag are flipped — that is the ramp, + # not this change. + # =========================================================================== + + # Orchestrator consumer for API-deployment executions (async_execute_bin_api). + worker-pg-orchestrator-api: + image: unstract/worker-unified:${VERSION} + container_name: unstract-worker-pg-orchestrator-api + restart: unless-stopped + command: ["pg-queue-consumer"] + ports: + - "8093:8090" + env_file: + - ../workers/.env + - ./essentials.env + depends_on: + - db + - redis + environment: + - ENVIRONMENT=development + - APPLICATION_NAME=unstract-worker-pg-orchestrator-api + - WORKER_BARRIER_BACKEND=pg + - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=api_deployment + - WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_api_deployments + - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090 + labels: + - traefik.enable=false + volumes: + - ./workflow_data:/data + - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config + profiles: + - pg-queue + + # Orchestrator consumer for general / scheduled-ETL executions (async_execute_bin). + worker-pg-orchestrator-general: + image: unstract/worker-unified:${VERSION} + container_name: unstract-worker-pg-orchestrator-general + restart: unless-stopped + command: ["pg-queue-consumer"] + ports: + - "8094:8090" + env_file: + - ../workers/.env + - ./essentials.env + depends_on: + - db + - redis + environment: + - ENVIRONMENT=development + - APPLICATION_NAME=unstract-worker-pg-orchestrator-general + - WORKER_BARRIER_BACKEND=pg + - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=general + - WORKER_PG_QUEUE_CONSUMER_QUEUE=celery + - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090 + labels: + - traefik.enable=false + volumes: + - ./workflow_data:/data + - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config + profiles: + - pg-queue + + # Fan-out consumer (process_file_batch) — both the ETL and API file queues. + worker-pg-fileproc: + image: unstract/worker-unified:${VERSION} + container_name: unstract-worker-pg-fileproc + restart: unless-stopped + command: ["pg-queue-consumer"] + ports: + - "8095:8090" + env_file: + - ../workers/.env + - ./essentials.env + depends_on: + - db + - redis + - rabbitmq # still dispatches the tool-execution RPC to the Celery executor (until the executor/tool-RPC moves to PG) + environment: + - ENVIRONMENT=development + - APPLICATION_NAME=unstract-worker-pg-fileproc + - WORKER_BARRIER_BACKEND=pg + - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=file_processing + - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing,api_file_processing + - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090 + labels: + - traefik.enable=false + volumes: + - ./workflow_data:/data + - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config + profiles: + - pg-queue + + # Fan-in callback consumer (process_batch_callback / _api) — both callback queues. + worker-pg-callback: + image: unstract/worker-unified:${VERSION} + container_name: unstract-worker-pg-callback + restart: unless-stopped + command: ["pg-queue-consumer"] + ports: + - "8096:8090" + env_file: + - ../workers/.env + - ./essentials.env + depends_on: + - db + - redis + - rabbitmq # may dispatch Celery notifications when not PG-routed (until notifications move to PG) + environment: + - ENVIRONMENT=development + - APPLICATION_NAME=unstract-worker-pg-callback + - WORKER_BARRIER_BACKEND=pg + - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=callback + - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing_callback,api_file_processing_callback + - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090 + labels: + - traefik.enable=false + volumes: + - ./workflow_data:/data + - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config + profiles: + - pg-queue + + # Reaper — leader-elected recovery loop. Run exactly ONE instance (it elects a + # single leader via pg_orchestrator_lock; extra replicas idle as standby). + worker-pg-reaper: + image: unstract/worker-unified:${VERSION} + container_name: unstract-worker-pg-reaper + restart: unless-stopped + command: ["pg-queue-reaper"] + ports: + - "8097:8086" + env_file: + - ../workers/.env + - ./essentials.env + depends_on: + - db + - redis + environment: + - ENVIRONMENT=development + - APPLICATION_NAME=unstract-worker-pg-reaper + - WORKER_PG_REAPER_INTERVAL_SECONDS=${WORKER_PG_REAPER_INTERVAL_SECONDS:-5} + - WORKER_PG_REAPER_HEALTH_PORT=8086 + labels: + - traefik.enable=false + volumes: + - ./workflow_data:/data + - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config + profiles: + - pg-queue + volumes: prompt_studio_data: unstract_data: diff --git a/docker/sample.env b/docker/sample.env index 2feb36cf2b..7f114461bd 100644 --- a/docker/sample.env +++ b/docker/sample.env @@ -107,3 +107,26 @@ CIRCUIT_BREAKER_RECOVERY_TIMEOUT=60 HEALTH_CHECK_INTERVAL=30 HEALTH_CHECK_TIMEOUT=10 ENABLE_METRICS=true + +# ============================================================================= +# PG-Queue Services (Postgres-backed transport) — gated, opt-in +# ============================================================================= +# The PG consumer/reaper services live behind the `pg-queue` compose profile and +# are OFF by default. Bring them up with: +# docker compose --profile pg-queue up -d +# They consume from Postgres (not the broker); their worker-type/queue identity +# is set per service in docker-compose.yaml, so nothing here is required to run +# them. The values below are documented for reference / overrides only. +# +# WORKER_BARRIER_BACKEND - fan-in barrier substrate; the PG services set +# this to `pg` (default elsewhere: chord/redis). +# WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE - which worker's tasks the consumer loads. +# WORKER_PG_QUEUE_CONSUMER_QUEUE - comma-separated queues the consumer polls. +# WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT - opt-in consumer liveness port (unset = off). +# WORKER_PG_REAPER_HEALTH_PORT - opt-in reaper liveness port (unset = off). +WORKER_PG_REAPER_INTERVAL_SECONDS=5 # Reaper sweep interval (seconds) +# +# NOTE: routing executions to PG is a SEPARATE, later step. Running these +# services does NOT move any traffic — the backend gate PG_QUEUE_TRANSPORT_ENABLED +# (in backend/.env, default off) plus the Flipt flag still decide per-execution +# transport, and both stay off until the rollout ramp. diff --git a/workers/run-worker-docker.sh b/workers/run-worker-docker.sh index 1b8926a063..be5aa86789 100755 --- a/workers/run-worker-docker.sh +++ b/workers/run-worker-docker.sh @@ -25,6 +25,11 @@ ENV_FILE="/app/.env" # Worker type constant for the executor worker readonly EXECUTOR_WORKER_TYPE="executor" +# Python interpreter for PG-queue components (consumer / reaper). Unlike Celery +# workers, these launch a dedicated module rather than a `celery ... worker` +# command (see run_pg_consumer / run_pg_reaper below). +readonly PG_QUEUE_PYTHON_BIN="/app/.venv/bin/python" + # Available core workers (OSS) declare -A WORKERS=( ["api"]="api_deployment" @@ -486,6 +491,62 @@ run_worker() { exec $celery_cmd $celery_args } +# ============================================================================= +# PG-queue components (Postgres-backed transport) +# ============================================================================= +# These do NOT run a Celery worker — they exec a dedicated Python module that +# polls the Postgres queue. The consumer picks which worker's tasks to register +# and which queues to poll from the environment (set per compose service / +# K8s Deployment); the reaper needs neither. Mirrors the host launcher +# run-worker.sh (`pg-queue-consumer` / `reaper`). + +# Fail loudly if the interpreter is missing (e.g. venv path moved in an image +# refactor) rather than letting `exec` die with a terse "not found" that +# restart:unless-stopped turns into a silent crash-loop. +ensure_pg_interpreter() { + if [[ ! -x "$PG_QUEUE_PYTHON_BIN" ]]; then + print_status $RED "PG-queue interpreter not found/executable: $PG_QUEUE_PYTHON_BIN — check the image build / venv path." + exit 1 + fi +} + +run_pg_consumer() { + ensure_pg_interpreter + local source_type="${WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE:-notification}" + local queues="${WORKER_PG_QUEUE_CONSUMER_QUEUE:-}" + export WORKER_NAME="${WORKER_NAME:-pg-consumer-${source_type}}" + + # Warn (don't fail) on missing config: all compose services set these, but a + # hand-rolled docker run / K8s Deployment that forgets them would otherwise + # start a mislabelled or idle-looking consumer silently. The module logs its + # resolved worker type + queue set at startup, so these stay diagnosable. + if [[ -z "${WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE:-}" ]]; then + print_status $YELLOW "WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE unset — defaulting source worker type to '$source_type'; WORKER_NAME label may not match the module's actual role." + fi + if [[ -z "$queues" ]]; then + print_status $YELLOW "WORKER_PG_QUEUE_CONSUMER_QUEUE unset — consumer will use the module default; check the startup log for the resolved queue set." + fi + + print_status $GREEN "Starting PG-queue consumer..." + print_status $BLUE "Source worker type: $source_type" + print_status $BLUE "Queues: ${queues:-}" + print_status $BLUE "Health port: ${WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT:-}" + + # WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE / _QUEUE are read by the module itself. + exec "$PG_QUEUE_PYTHON_BIN" -m pg_queue_consumer +} + +run_pg_reaper() { + ensure_pg_interpreter + export WORKER_NAME="${WORKER_NAME:-pg-reaper}" + + print_status $GREEN "Starting PG-queue reaper (leader-elected)..." + print_status $BLUE "Interval: ${WORKER_PG_REAPER_INTERVAL_SECONDS:-5}s" + print_status $BLUE "Health port: ${WORKER_PG_REAPER_HEALTH_PORT:-}" + + exec "$PG_QUEUE_PYTHON_BIN" -m pg_queue_reaper +} + # Main execution # Load environment first for any needed variables load_env "$ENV_FILE" @@ -496,6 +557,32 @@ discover_pluggable_workers # Add PYTHONPATH for imports - include both /app and /unstract for packages export PYTHONPATH="/app:/unstract/core/src:/unstract/connectors/src:/unstract/filesystem/src:/unstract/flags/src:/unstract/tool-registry/src:/unstract/tool-sandbox/src:/unstract/workflow-execution/src:${PYTHONPATH:-}" +# PG-queue components run a dedicated module, not a Celery worker — dispatch +# them before the Celery command-building logic below. PYTHONPATH (exported +# above) is required for the module imports. +case "${1:-}" in + pg-queue-consumer|pg-consumer) + run_pg_consumer + ;; + pg-queue-reaper|pg-reaper|reaper) + run_pg_reaper + ;; + pg-*) + # Obviously-PG-intended but unrecognized (e.g. a typo'd command) — fail + # loudly instead of silently coercing it into a default Celery worker. + # Scoped to the `pg-` prefix (reserved for PG-queue components) so it + # never intercepts a pluggable worker name; the exact reaper aliases are + # already matched above. + print_status $RED "Unrecognized PG-queue command: '$1' (did you mean pg-queue-consumer / pg-queue-reaper?)." + exit 1 + ;; + *) + # Not a PG-queue command — intentionally fall through to the Celery + # command logic below (handles every existing worker type + full + # Celery commands). + ;; +esac + # Two-path logic: Full Celery command vs Traditional worker type if [[ "$1" == *"celery"* ]] || [[ "$1" == *".venv"* ]]; then # ============================================================================= From 87f9ede5b6bba185c7d863b84afddd11ddb69ba5 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Thu, 18 Jun 2026 21:50:09 +0530 Subject: [PATCH 24/44] =?UTF-8?q?UN-3581=20[FEAT]=20PG=20Queue=209h-a=20?= =?UTF-8?q?=E2=80=94=20pg=5Fperiodic=5Fschedule=20mirror=20table=20+=20dua?= =?UTF-8?q?l-write=20(inert)=20(#2080)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3581 [FEAT] PG Queue 9h-a — pg_periodic_schedule mirror table + dual-write (inert) First inert slice toward replacing Celery Beat with a PG-backed periodic scheduler (to be folded into the leader-elected reaper/orchestrator loop). This slice only mirrors each scheduled pipeline's cron definition into a new Postgres table; nothing reads it yet, so behaviour is unchanged. - backend/pg_queue/models.py: new PgPeriodicSchedule (pipeline_id PK, org_id, workflow_id, pipeline_name, cron_string, enabled; last_run_at/next_run_at left NULL — the scheduler tick owns all cron computation in the next slice). Index on (enabled, next_run_at) for the future "due schedules" query. - migration 0008 (generated; makemigrations --check clean). - backend/scheduler/tasks.py: dual-write the mirror from the four schedule choke-points (create/update, pause, resume, delete) that already manage the django_celery_beat PeriodicTask. Toggles are placed right after task.save() so a downstream pipeline-status failure can't desync the mirror. Every mirror write is best-effort (try/except + log) — a mirror failure can NEVER break the existing Beat scheduling path. - tests: 6 DB-free unit tests (field extraction, enable/disable, delete, delete-when-PeriodicTask-missing, best-effort swallow). Inert / non-regressive: nothing reads the table; the PeriodicTask + celery-beat container are untouched. Co-Authored-By: Claude Opus 4.8 * UN-3581 address review: mirror real pipeline_name from helper, harden toggles, tighten types - [High] pipeline_name now sourced from the Pipeline object in SchedulerHelper._schedule_task_job (pipeline.pipeline_name), not task_args[6] — which carries the synthetic "Pipeline job-" label, never the user name. The mirror upsert moves to the helper (clean named ids, no positional arg parsing); create_or_update_periodic_task is Beat-only again. - [Med] _mirror_periodic_schedule_set_enabled now bumps updated_at explicitly (queryset .update() does not fire auto_now) and logs when it matches 0 rows (a pre-existing/unmirrored schedule — backfill lands in ②b). - [Med] dropped the positional task_args[N] parsing entirely (the helper passes named fields), removing the silent-placeholder-on-contract-drift risk. - [Low] tightened mirror helper param types (str / str | None) — the callers already pass stringified UUIDs. - [Nit] index renamed pg_sched_due_idx -> pg_periodic_schedule_due_idx (sibling convention); migration 0008 regenerated (not yet merged). - tests: cover the standalone upsert (+enabled=False, +failure-swallow), the helper wiring proving the real pipeline_name flows, disable/enable/delete failure-swallow + still-calls-update_pipeline, and the 0-row log. 11 tests. cron_string validation: the existing 5-field split unpack in create_or_update_periodic_task already gates the Beat path before the mirror runs; full cron semantics are validated by croniter in ②b (the reader). Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .../migrations/0008_pgperiodicschedule.py | 37 ++++ backend/pg_queue/models.py | 45 ++++ backend/scheduler/helper.py | 13 ++ backend/scheduler/tasks.py | 88 ++++++++ backend/scheduler/tests/__init__.py | 0 .../tests/test_pg_periodic_schedule_mirror.py | 200 ++++++++++++++++++ 6 files changed, 383 insertions(+) create mode 100644 backend/pg_queue/migrations/0008_pgperiodicschedule.py create mode 100644 backend/scheduler/tests/__init__.py create mode 100644 backend/scheduler/tests/test_pg_periodic_schedule_mirror.py diff --git a/backend/pg_queue/migrations/0008_pgperiodicschedule.py b/backend/pg_queue/migrations/0008_pgperiodicschedule.py new file mode 100644 index 0000000000..669fe10457 --- /dev/null +++ b/backend/pg_queue/migrations/0008_pgperiodicschedule.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2.1 on 2026-06-18 16:13 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("pg_queue", "0007_pgbarrierstate_organization_id"), + ] + + operations = [ + migrations.CreateModel( + name="PgPeriodicSchedule", + fields=[ + ("pipeline_id", models.UUIDField(primary_key=True, serialize=False)), + ("organization_id", models.TextField(blank=True, default="")), + ("workflow_id", models.UUIDField(blank=True, null=True)), + ("pipeline_name", models.TextField(blank=True, default="")), + ("cron_string", models.TextField()), + ("enabled", models.BooleanField(default=True)), + ("last_run_at", models.DateTimeField(blank=True, null=True)), + ("next_run_at", models.DateTimeField(blank=True, null=True)), + ("created_at", models.DateTimeField(default=django.utils.timezone.now)), + ("updated_at", models.DateTimeField(auto_now=True)), + ], + options={ + "db_table": "pg_periodic_schedule", + "indexes": [ + models.Index( + fields=["enabled", "next_run_at"], + name="pg_periodic_schedule_due_idx", + ) + ], + }, + ), + ] diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py index e1f9f3728c..3795339a2d 100644 --- a/backend/pg_queue/models.py +++ b/backend/pg_queue/models.py @@ -227,3 +227,48 @@ class Meta: # Drives the (future) periodic expiry-sweep job. models.Index(fields=["expires_at"], name="pg_barrier_expires_idx"), ] + + +class PgPeriodicSchedule(models.Model): + """Inert mirror of a scheduled pipeline's cron definition (Phase 9, ②a). + + One row per scheduled pipeline (keyed by ``pipeline_id``), dual-written + alongside the existing ``django_celery_beat`` ``PeriodicTask`` whenever a + pipeline schedule is created/updated/paused/resumed/deleted. It exists so a + future PG-backed scheduler — folded into the leader-elected reaper loop — + can fire due schedules without Celery Beat. + + **Nothing reads this table yet.** This slice only keeps it in sync; the + scheduler tick (SELECT due → enqueue ``execute_pipeline_task`` → recompute + ``next_run_at``) lands in the next slice, which owns all cron computation. + ``last_run_at`` / ``next_run_at`` are therefore left NULL here. + + Managed=True / generated migration, extension-free (UN-3533), same posture + as ``PgBarrierState``. + """ + + # Mirrors the Pipeline PK; also the PeriodicTask ``name`` (str of this UUID). + pipeline_id = models.UUIDField(primary_key=True) + # Owning org — needed to rebuild the ``execute_pipeline_task`` args when the + # scheduler fires. "" = unknown. Same no-NULL-text convention as elsewhere. + organization_id = models.TextField(blank=True, default="") + workflow_id = models.UUIDField(null=True, blank=True) + pipeline_name = models.TextField(blank=True, default="") + cron_string = models.TextField() + # Mirrors PeriodicTask.enabled (pipeline.active / pause / resume). + enabled = models.BooleanField(default=True) + # Owned by the scheduler tick (next slice); NULL until then. + last_run_at = models.DateTimeField(null=True, blank=True) + next_run_at = models.DateTimeField(null=True, blank=True) + created_at = models.DateTimeField(default=timezone.now) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + db_table = "pg_periodic_schedule" + indexes = [ + # Drives the (future) "due schedules" dequeue in the scheduler tick. + models.Index( + fields=["enabled", "next_run_at"], + name="pg_periodic_schedule_due_idx", + ), + ] diff --git a/backend/scheduler/helper.py b/backend/scheduler/helper.py index ad265ca45e..1b8c68ecf3 100644 --- a/backend/scheduler/helper.py +++ b/backend/scheduler/helper.py @@ -15,6 +15,7 @@ delete_periodic_task, disable_task, enable_task, + mirror_periodic_schedule_upsert, ) logger = logging.getLogger(__name__) @@ -61,6 +62,18 @@ def _schedule_task_job(pipeline: Pipeline, job_data: Any) -> None: ], enabled=pipeline.active, ) + # Inert PG mirror (Phase 9, ②a): sourced from the Pipeline object here so + # it stores the real pipeline_name + ids (the PeriodicTask args carry the + # synthetic "Pipeline job-" label at index 6, not the user name). + # Best-effort — never raises, so Beat scheduling is unaffected. + mirror_periodic_schedule_upsert( + pipeline_id=str(pipeline.pk), + organization_id=organization_id or "", + workflow_id=str(workflow_id), + pipeline_name=pipeline.pipeline_name, + cron_string=cron_string, + enabled=pipeline.active, + ) @staticmethod def add_or_update_job(pipeline: Pipeline) -> None: diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.py index ad8c943069..b07da1aa12 100644 --- a/backend/scheduler/tasks.py +++ b/backend/scheduler/tasks.py @@ -4,7 +4,9 @@ from typing import Any from celery import shared_task +from django.utils import timezone from django_celery_beat.models import CrontabSchedule, PeriodicTask +from pg_queue.models import PgPeriodicSchedule from pipeline_v2.models import Pipeline from pipeline_v2.pipeline_processor import PipelineProcessor from utils.user_context import UserContext @@ -13,6 +15,83 @@ logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# pg_periodic_schedule mirror (Phase 9, ②a) — INERT. +# Dual-writes the schedule definition into pg_periodic_schedule alongside the +# django_celery_beat PeriodicTask, so a future PG-backed scheduler (folded into +# the reaper/orchestrator loop) can fire due schedules without Celery Beat. +# Nothing reads the table yet. Every write is best-effort: a mirror failure must +# NEVER break the existing Beat scheduling path. +# +# The upsert is driven from SchedulerHelper._schedule_task_job (which holds the +# Pipeline object, so it sources the real pipeline_name + clean ids — no parsing +# of the serialized PeriodicTask args). The enable/disable/delete toggles are +# keyed by pipeline_id only, so they live here next to the functions that mutate +# the PeriodicTask. +# --------------------------------------------------------------------------- + + +def mirror_periodic_schedule_upsert( + *, + pipeline_id: str, + organization_id: str, + workflow_id: str | None, + pipeline_name: str, + cron_string: str, + enabled: bool, +) -> None: + try: + PgPeriodicSchedule.objects.update_or_create( + pipeline_id=pipeline_id, + defaults={ + "organization_id": organization_id or "", + "workflow_id": workflow_id or None, + "pipeline_name": pipeline_name or "", + "cron_string": cron_string, + "enabled": enabled, + }, + ) + except Exception: + logger.exception( + f"pg_periodic_schedule mirror upsert failed for pipeline {pipeline_id} " + "(inert mirror — Beat scheduling unaffected)" + ) + + +def _mirror_periodic_schedule_set_enabled(pipeline_id: str, enabled: bool) -> None: + try: + # Bump updated_at explicitly: queryset .update() does NOT trigger the + # field's auto_now, so without this a pause/resume would change enabled + # without advancing the "last changed" timestamp. + matched = PgPeriodicSchedule.objects.filter(pipeline_id=pipeline_id).update( + enabled=enabled, updated_at=timezone.now() + ) + if matched == 0: + # No mirror row — e.g. a pipeline scheduled before this shipped, or + # whose upsert was swallowed. .update() can't self-heal (it only + # touches existing rows); the backfill of such rows lands with the + # scheduler that reads this table (②b). Log so the gap is visible. + logger.info( + f"pg_periodic_schedule mirror enabled={enabled} matched 0 rows for " + f"pipeline {pipeline_id} (not yet mirrored — backfilled in ②b)" + ) + except Exception: + logger.exception( + f"pg_periodic_schedule mirror enabled={enabled} failed for pipeline " + f"{pipeline_id} (inert mirror — Beat scheduling unaffected)" + ) + + +def _mirror_periodic_schedule_delete(pipeline_id: str) -> None: + try: + PgPeriodicSchedule.objects.filter(pipeline_id=pipeline_id).delete() + except Exception: + logger.exception( + f"pg_periodic_schedule mirror delete failed for pipeline {pipeline_id} " + "(inert mirror — Beat scheduling unaffected)" + ) + + def create_or_update_periodic_task( cron_string: str, task_name: str, @@ -49,6 +128,9 @@ def create_or_update_periodic_task( logger.info(f"Created periodic task {periodic_task}") else: logger.info(f"Updated periodic task {periodic_task}") + # The inert PG mirror upsert is driven by the caller + # (SchedulerHelper._schedule_task_job), which has the Pipeline object and so + # sources the real pipeline_name + ids directly (no positional arg parsing). # TODO: Remove unused args with a migration @@ -145,6 +227,8 @@ def delete_periodic_task(task_name: str) -> None: logger.info(f"Deleted periodic task: {task_name}") except PeriodicTask.DoesNotExist: logger.error(f"Periodic task does not exist: {task_name}") + # Clean the inert PG mirror regardless of whether the PeriodicTask existed. + _mirror_periodic_schedule_delete(task_name) def get_periodic_task(task_name: str) -> PeriodicTask | None: @@ -158,6 +242,9 @@ def disable_task(task_name: str) -> None: task = PeriodicTask.objects.get(name=task_name) task.enabled = False task.save() + # Mirror the PeriodicTask.enabled state right after save (before the pipeline + # status update, so a failure there can't desync the inert mirror). + _mirror_periodic_schedule_set_enabled(task_name, False) PipelineProcessor.update_pipeline(task_name, Pipeline.PipelineStatus.PAUSED, False) @@ -165,4 +252,5 @@ def enable_task(task_name: str) -> None: task = PeriodicTask.objects.get(name=task_name) task.enabled = True task.save() + _mirror_periodic_schedule_set_enabled(task_name, True) PipelineProcessor.update_pipeline(task_name, Pipeline.PipelineStatus.RESTARTING, True) diff --git a/backend/scheduler/tests/__init__.py b/backend/scheduler/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/scheduler/tests/test_pg_periodic_schedule_mirror.py b/backend/scheduler/tests/test_pg_periodic_schedule_mirror.py new file mode 100644 index 0000000000..e5914db655 --- /dev/null +++ b/backend/scheduler/tests/test_pg_periodic_schedule_mirror.py @@ -0,0 +1,200 @@ +"""Unit tests for the inert pg_periodic_schedule mirror (Phase 9, ②a). + +DB-free: ``PgPeriodicSchedule`` (and the django_celery_beat models / serializer) +are mocked, so these pin the dual-write contract — that every schedule mutation +mirrors the right fields, that the mirror stores the *real* pipeline name (not +the synthetic ``"Pipeline job-"`` label carried in the PeriodicTask args), +and that a mirror failure can never break the existing Celery Beat scheduling +path — without a test database. +""" + +from unittest.mock import MagicMock, patch + +from scheduler import tasks +from scheduler.helper import SchedulerHelper + +_PIPELINE_ID = "11111111-1111-1111-1111-111111111111" +_WORKFLOW_ID = "22222222-2222-2222-2222-222222222222" +_ORG = "org_abc" +_REAL_NAME = "Nightly Invoices ETL" + + +class _DoesNotExist(Exception): + """Stand-in for PeriodicTask.DoesNotExist when the model is mocked.""" + + +class TestUpsertMirror: + def test_upserts_with_given_fields(self): + with patch("scheduler.tasks.PgPeriodicSchedule") as sched: + tasks.mirror_periodic_schedule_upsert( + pipeline_id=_PIPELINE_ID, + organization_id=_ORG, + workflow_id=_WORKFLOW_ID, + pipeline_name=_REAL_NAME, + cron_string="0 9 * * *", + enabled=True, + ) + call = sched.objects.update_or_create.call_args + assert call.kwargs["pipeline_id"] == _PIPELINE_ID + defaults = call.kwargs["defaults"] + assert defaults["organization_id"] == _ORG + assert defaults["workflow_id"] == _WORKFLOW_ID + assert defaults["pipeline_name"] == _REAL_NAME + assert defaults["cron_string"] == "0 9 * * *" + assert defaults["enabled"] is True + + def test_disabled_pipeline_mirrors_enabled_false(self): + with patch("scheduler.tasks.PgPeriodicSchedule") as sched: + tasks.mirror_periodic_schedule_upsert( + pipeline_id=_PIPELINE_ID, + organization_id=_ORG, + workflow_id=_WORKFLOW_ID, + pipeline_name=_REAL_NAME, + cron_string="0 9 * * *", + enabled=False, + ) + assert sched.objects.update_or_create.call_args.kwargs["defaults"]["enabled"] is False + + def test_failure_is_swallowed(self): + with patch("scheduler.tasks.PgPeriodicSchedule") as sched: + sched.objects.update_or_create.side_effect = RuntimeError("db down") + # Must not raise. + tasks.mirror_periodic_schedule_upsert( + pipeline_id=_PIPELINE_ID, + organization_id=_ORG, + workflow_id=_WORKFLOW_ID, + pipeline_name=_REAL_NAME, + cron_string="0 9 * * *", + enabled=True, + ) + + +class TestHelperWiringSourcesRealName: + """The High contract: the mirror must store the user-facing pipeline name, + NOT the synthetic ``"Pipeline job-"`` label that the PeriodicTask args + carry at index 6.""" + + def test_schedule_task_job_passes_real_pipeline_name(self): + pipeline = MagicMock() + pipeline.pk = _PIPELINE_ID + pipeline.pipeline_name = _REAL_NAME + pipeline.active = True + pipeline.workflow.id = _WORKFLOW_ID + + serializer = MagicMock() + serializer.get_workflow_id.return_value = _WORKFLOW_ID + serializer.get_execution_action.return_value = "" + + with ( + patch("scheduler.helper.ExecuteWorkflowSerializer", return_value=serializer), + patch( + "scheduler.helper.UserContext.get_organization_identifier", + return_value=_ORG, + ), + patch("scheduler.helper.create_or_update_periodic_task"), + patch("scheduler.helper.mirror_periodic_schedule_upsert") as mirror, + ): + SchedulerHelper._schedule_task_job( + pipeline, + { + "cron_string": "0 9 * * *", + "id": _PIPELINE_ID, + "name": f"Pipeline job-{_PIPELINE_ID}", # the synthetic label + }, + ) + + mirror.assert_called_once() + kwargs = mirror.call_args.kwargs + assert kwargs["pipeline_name"] == _REAL_NAME # real name, not the label + assert kwargs["pipeline_id"] == _PIPELINE_ID + assert kwargs["organization_id"] == _ORG + assert kwargs["enabled"] is True + + +class TestEnableDisableMirror: + def test_disable_mirrors_enabled_false(self): + with ( + patch("scheduler.tasks.PeriodicTask") as pt, + patch("scheduler.tasks.PipelineProcessor"), + patch("scheduler.tasks.PgPeriodicSchedule") as sched, + ): + pt.objects.get.return_value = MagicMock() + sched.objects.filter.return_value.update.return_value = 1 + tasks.disable_task(_PIPELINE_ID) + + sched.objects.filter.assert_called_with(pipeline_id=_PIPELINE_ID) + assert sched.objects.filter.return_value.update.call_args.kwargs["enabled"] is False + + def test_enable_mirrors_enabled_true(self): + with ( + patch("scheduler.tasks.PeriodicTask") as pt, + patch("scheduler.tasks.PipelineProcessor"), + patch("scheduler.tasks.PgPeriodicSchedule") as sched, + ): + pt.objects.get.return_value = MagicMock() + sched.objects.filter.return_value.update.return_value = 1 + tasks.enable_task(_PIPELINE_ID) + + assert sched.objects.filter.return_value.update.call_args.kwargs["enabled"] is True + + def test_disable_mirror_failure_does_not_break_beat_path(self): + """A mirror failure must be swallowed AND the pipeline-status update must + still run (the central 'never break Beat' guarantee).""" + with ( + patch("scheduler.tasks.PeriodicTask") as pt, + patch("scheduler.tasks.PipelineProcessor") as pp, + patch("scheduler.tasks.PgPeriodicSchedule") as sched, + ): + pt.objects.get.return_value = MagicMock() + sched.objects.filter.return_value.update.side_effect = RuntimeError("db down") + tasks.disable_task(_PIPELINE_ID) # must not raise + + pp.update_pipeline.assert_called_once() + + def test_unmirrored_pipeline_logs_zero_match(self, caplog): + import logging + + with ( + patch("scheduler.tasks.PeriodicTask") as pt, + patch("scheduler.tasks.PipelineProcessor"), + patch("scheduler.tasks.PgPeriodicSchedule") as sched, + ): + pt.objects.get.return_value = MagicMock() + sched.objects.filter.return_value.update.return_value = 0 # no mirror row + with caplog.at_level(logging.INFO, logger="scheduler.tasks"): + tasks.disable_task(_PIPELINE_ID) + + assert any("matched 0 rows" in r.message for r in caplog.records) + + +class TestDeleteMirror: + def test_delete_removes_mirror_row(self): + with ( + patch("scheduler.tasks.PeriodicTask") as pt, + patch("scheduler.tasks.PgPeriodicSchedule") as sched, + ): + pt.objects.get.return_value = MagicMock() + tasks.delete_periodic_task(_PIPELINE_ID) + + sched.objects.filter.assert_called_with(pipeline_id=_PIPELINE_ID) + sched.objects.filter.return_value.delete.assert_called_once() + + def test_delete_cleans_mirror_even_when_periodictask_missing(self): + with ( + patch("scheduler.tasks.PeriodicTask") as pt, + patch("scheduler.tasks.PgPeriodicSchedule") as sched, + ): + pt.DoesNotExist = _DoesNotExist + pt.objects.get.side_effect = _DoesNotExist() + tasks.delete_periodic_task(_PIPELINE_ID) # must not raise + + sched.objects.filter.return_value.delete.assert_called_once() + + def test_delete_mirror_failure_is_swallowed(self): + with ( + patch("scheduler.tasks.PeriodicTask") as pt, + patch("scheduler.tasks.PgPeriodicSchedule") as sched, + ): + pt.objects.get.return_value = MagicMock() + sched.objects.filter.return_value.delete.side_effect = RuntimeError("db down") + tasks.delete_periodic_task(_PIPELINE_ID) # must not raise From e202ae0d07d90e170b667fd690e8e6622f291477 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Thu, 18 Jun 2026 23:34:18 +0530 Subject: [PATCH 25/44] =?UTF-8?q?UN-3596=20[FEAT]=20PG=20Queue=209h-b=20?= =?UTF-8?q?=E2=80=94=20PG=20scheduler=20tick=20in=20the=20reaper/orchestra?= =?UTF-8?q?tor=20loop=20(per-schedule=20ownership)=20(#2081)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3596 [FEAT] PG Queue 9h-b — PG scheduler tick in the reaper/orchestrator loop (per-schedule ownership) Adds the periodic-trigger half of the orchestrator: the leader-elected reaper now also fires due, PG-owned schedules onto the PG queue — the Celery Beat replacement — without Beat/RabbitMQ in the trigger path. Dark by default; never double-fires. - pg_scheduler.py (new): dispatch_due_schedules() scans pg_periodic_schedule for pg_owned + enabled + due rows, enqueues scheduler.tasks.execute_pipeline_task on the PG `scheduler` queue AND advances next_run_at in ONE transaction (a crash between can't re-fire). A NULL next_run_at records a baseline and does NOT fire (no burst when a schedule is handed over; matches Beat). A bad cron on one row is logged and skipped without blocking the others. croniter computes next-run; all time comparisons use the DB clock. - pg_queue/models.py + migration 0009: pg_owned flag (default False = Beat owns it; the PG scheduler fires only owned rows) + due index (pg_owned, enabled, next_run_at). Default-false keeps the table inert until a schedule is handed over, and a schedule fires from exactly one side — never both. - reaper.py: the leader tick runs the scheduler AFTER recovery (a scheduler error can't starve the recovery net). - workers deps: add croniter (already a backend dep). - run-worker.sh + docker-compose: pg-scheduler consumer role + service (profile-gated) that runs the fired execute_pipeline_task. Out of scope (next slice ②c): the ramp control that flips pg_owned by percentage + disables the matching Beat PeriodicTask atomically (reusing the existing Flipt mechanism), the one-time backfill, and retiring Beat. Non-regression: pg_owned defaults False, so the reaper fires nothing until rows are explicitly owned; recovery-only behaviour is unchanged. Tests: 10 scheduler (real-PG) + 3 reaper-wiring; full reaper suite kept green via a scheduler stub. Co-Authored-By: Claude Opus 4.8 * UN-3596 address review: per-row DB isolation, self-quiescing bad cron, typed rows, shared INSERT SQL, stronger tests - [Critical] per-row fire (INSERT+UPDATE+commit) now wrapped in try/except → rollback + log + continue, so one bad row can't poison the connection or drop the rest of the batch (mirrors recover_expired_barriers). Baseline UPDATE too. - [High] invalid cron now disables the row (enabled=FALSE) + logs once, instead of re-selecting it and emitting a traceback every ~5s tick forever. - [Med] read step (SELECT now() + due scan) wraps rollback + re-raise so the conn isn't handed back in an aborted-txn state. - [High] softened the "never double-fires" docstring + models.py comment: the guarantee is CONDITIONAL on the ②c ramp control disabling Beat; pre-ramp, safety rests on pg_owned defaulting to False. - [Med] _build_trigger_payload -> TaskPayload; workflow_id/pipeline_id typed str | uuid.UUID (| None); _DueSchedule NamedTuple binds SELECT columns to names at one site (no silent misassign on a reorder). - [Med] extracted INSERT_MESSAGE_SQL constant in client.py; send() and the scheduler share it (no verbatim SQL duplication). - [Low] comment fixes: reaper tick (ordering not isolation), execute_pipeline_task blanks vs Beat populating execution_action, models.py drop "next slice". - tests: fired == 1 (not >= 1, catches double-fire); next_run asserted at the cron's 09:00 match; baseline asserts == 0; +tz-aware next-run; +multi-row (fired == 2); +atomicity (advance UPDATE fails post-INSERT → enqueue rolls back, next_run unchanged); +reaper scheduler-error-discards-owned-conn. 75 green. Co-Authored-By: Claude Opus 4.8 * UN-3596 chore: drop accidentally-committed 9f-design.md (untracked scoping doc) Co-Authored-By: Claude Opus 4.8 * UN-3596 address greptile P1: roll back if the bad-cron disable UPDATE fails _quiesce_invalid_cron used contextlib.suppress around the cursor block, so if the enabled=FALSE UPDATE raised, commit() was skipped and the connection was left in an aborted-transaction state — poisoning the NEXT row's INSERT (caught by the outer handler and mislogged as "failed to fire"). Wrap in try/except with conn.rollback() on failure so the connection is always clean for the next row. +test: a forced disable-UPDATE failure on a bad-cron row doesn't stop a following healthy row from firing. Co-Authored-By: Claude Opus 4.8 * UN-3596 chore: remove superseded 9f-design.md (impl merged in #2073) Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- ...e_pg_periodic_schedule_due_idx_and_more.py | 28 ++ backend/pg_queue/models.py | 27 +- docker/docker-compose.yaml | 39 ++- workers/pyproject.toml | 1 + workers/queue_backend/pg_queue/client.py | 16 +- .../queue_backend/pg_queue/pg_scheduler.py | 230 +++++++++++++++ workers/queue_backend/pg_queue/reaper.py | 11 + workers/run-worker.sh | 8 + workers/tests/test_pg_reaper.py | 68 +++++ workers/tests/test_pg_scheduler.py | 276 ++++++++++++++++++ workers/uv.lock | 14 + 11 files changed, 706 insertions(+), 12 deletions(-) create mode 100644 backend/pg_queue/migrations/0009_remove_pgperiodicschedule_pg_periodic_schedule_due_idx_and_more.py create mode 100644 workers/queue_backend/pg_queue/pg_scheduler.py create mode 100644 workers/tests/test_pg_scheduler.py diff --git a/backend/pg_queue/migrations/0009_remove_pgperiodicschedule_pg_periodic_schedule_due_idx_and_more.py b/backend/pg_queue/migrations/0009_remove_pgperiodicschedule_pg_periodic_schedule_due_idx_and_more.py new file mode 100644 index 0000000000..2a37fd54b9 --- /dev/null +++ b/backend/pg_queue/migrations/0009_remove_pgperiodicschedule_pg_periodic_schedule_due_idx_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.1 on 2026-06-18 16:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("pg_queue", "0008_pgperiodicschedule"), + ] + + operations = [ + migrations.RemoveIndex( + model_name="pgperiodicschedule", + name="pg_periodic_schedule_due_idx", + ), + migrations.AddField( + model_name="pgperiodicschedule", + name="pg_owned", + field=models.BooleanField(default=False), + ), + migrations.AddIndex( + model_name="pgperiodicschedule", + index=models.Index( + fields=["pg_owned", "enabled", "next_run_at"], + name="pg_periodic_schedule_due_idx", + ), + ), + ] diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py index 3795339a2d..c079c9f498 100644 --- a/backend/pg_queue/models.py +++ b/backend/pg_queue/models.py @@ -238,10 +238,12 @@ class PgPeriodicSchedule(models.Model): future PG-backed scheduler — folded into the leader-elected reaper loop — can fire due schedules without Celery Beat. - **Nothing reads this table yet.** This slice only keeps it in sync; the - scheduler tick (SELECT due → enqueue ``execute_pipeline_task`` → recompute - ``next_run_at``) lands in the next slice, which owns all cron computation. - ``last_run_at`` / ``next_run_at`` are therefore left NULL here. + The PG scheduler tick (folded into the leader-elected reaper loop) fires + due rows where ``pg_owned`` is True; rows default to ``pg_owned=False``, so + the table stays inert until a schedule is explicitly handed over to Postgres. + ``last_run_at`` / ``next_run_at`` are owned by that tick (the dual-write + leaves them NULL; a NULL ``next_run_at`` records a baseline on first tick + rather than firing immediately). Managed=True / generated migration, extension-free (UN-3533), same posture as ``PgBarrierState``. @@ -257,7 +259,17 @@ class PgPeriodicSchedule(models.Model): cron_string = models.TextField() # Mirrors PeriodicTask.enabled (pipeline.active / pause / resume). enabled = models.BooleanField(default=True) - # Owned by the scheduler tick (next slice); NULL until then. + # Per-schedule rollout switch. The PG scheduler fires a row ONLY when this is + # True. The no-double-fire guarantee with Celery Beat is CONDITIONAL on the + # matching Beat PeriodicTask being disabled when a schedule is handed over — + # that's the ②c ramp control, which does not exist yet. Until it lands, + # safety rests on this defaulting to False: nothing is pg_owned, so the PG + # scheduler fires nothing and Beat fires everything. Flipping a row True while + # its PeriodicTask is still enabled WOULD double-fire — the ramp control must + # do both atomically. Migrating is then a reversible per-schedule flip. + pg_owned = models.BooleanField(default=False) + # Owned by the scheduler tick. NULL next_run_at = "record a baseline next + # time, don't fire this cycle" (avoids a burst when a schedule is handed over). last_run_at = models.DateTimeField(null=True, blank=True) next_run_at = models.DateTimeField(null=True, blank=True) created_at = models.DateTimeField(default=timezone.now) @@ -266,9 +278,10 @@ class PgPeriodicSchedule(models.Model): class Meta: db_table = "pg_periodic_schedule" indexes = [ - # Drives the (future) "due schedules" dequeue in the scheduler tick. + # Drives the "due schedules" scan in the scheduler tick: + # WHERE pg_owned AND enabled AND (next_run_at IS NULL OR <= now()). models.Index( - fields=["enabled", "next_run_at"], + fields=["pg_owned", "enabled", "next_run_at"], name="pg_periodic_schedule_due_idx", ), ] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index a85d3e6ca6..70579c657d 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -676,8 +676,43 @@ services: profiles: - pg-queue - # Reaper — leader-elected recovery loop. Run exactly ONE instance (it elects a - # single leader via pg_orchestrator_lock; extra replicas idle as standby). + # Scheduler consumer — runs scheduler.tasks.execute_pipeline_task fired by the + # orchestrator's PG scheduler tick (Beat replacement). Distinct from the Celery + # 'scheduler' worker; this only runs the trigger, which then creates the + # execution + dispatches the workflow exactly as today. + worker-pg-scheduler: + image: unstract/worker-unified:${VERSION} + container_name: unstract-worker-pg-scheduler + restart: unless-stopped + command: ["pg-queue-consumer"] + ports: + - "8098:8090" + env_file: + - ../workers/.env + - ./essentials.env + depends_on: + - db + - redis + - rabbitmq # the trigger dispatches async_execute_bin (Celery path when not PG-routed) + environment: + - ENVIRONMENT=development + - APPLICATION_NAME=unstract-worker-pg-scheduler + - WORKER_BARRIER_BACKEND=pg + - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=scheduler + - WORKER_PG_QUEUE_CONSUMER_QUEUE=scheduler + - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090 + labels: + - traefik.enable=false + volumes: + - ./workflow_data:/data + - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config + profiles: + - pg-queue + + # Reaper / orchestrator — leader-elected loop. Run exactly ONE instance (it + # elects a single leader via pg_orchestrator_lock; extra replicas idle as + # standby). Besides barrier-orphan recovery it runs the PG scheduler tick + # (fires due pg_owned schedules — Beat replacement); dark until rows are owned. worker-pg-reaper: image: unstract/worker-unified:${VERSION} container_name: unstract-worker-pg-reaper diff --git a/workers/pyproject.toml b/workers/pyproject.toml index e3368a4eea..618e9b72b8 100644 --- a/workers/pyproject.toml +++ b/workers/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ # PG Queue — direct Postgres access for the bespoke SKIP LOCKED queue # (reuses the backend's DB_* connection: PgBouncer in cloud, direct in OSS) "psycopg2-binary==2.9.9", # matches backend; first direct-DB worker capability + "croniter>=3.0.3", # cron next-run computation for the PG scheduler tick (matches backend) # Streaming utilities for bulk downloads (pluggable worker) "zipstream-ng>=1.7.0", # Streaming zip creation for memory-efficient bulk downloads # Note: Using dataclasses instead of pydantic for lightweight typing diff --git a/workers/queue_backend/pg_queue/client.py b/workers/queue_backend/pg_queue/client.py index 8024f11ebc..769ae0b666 100644 --- a/workers/queue_backend/pg_queue/client.py +++ b/workers/queue_backend/pg_queue/client.py @@ -119,6 +119,18 @@ class QueueMessage: read_ct: int +# The whole enqueue contract (columns + their defaults) in one place. `send()` +# appends ``RETURNING msg_id``; the PG scheduler (pg_scheduler.py) executes this +# verbatim inside its own transaction so the enqueue + next_run advance commit +# atomically (it can't call send(), which commits internally). Keep callers in +# sync by sharing this constant rather than copying the SQL. +INSERT_MESSAGE_SQL: str = ( + "INSERT INTO pg_queue_message " + "(queue_name, message, org_id, priority, enqueued_at, vt, read_ct) " + "VALUES (%s, %s::jsonb, %s, %s, now(), now(), 0)" +) + + class PgQueueClient: """``send`` / ``read`` / ``delete`` over ``pg_queue_message``. @@ -199,9 +211,7 @@ def send( ) with self._cursor() as cur: cur.execute( - "INSERT INTO pg_queue_message " - "(queue_name, message, org_id, priority, enqueued_at, vt, read_ct) " - "VALUES (%s, %s::jsonb, %s, %s, now(), now(), 0) RETURNING msg_id", + INSERT_MESSAGE_SQL + " RETURNING msg_id", # "" rather than NULL for "no org" — the column is non-null # (string fields shouldn't have two empty values; Django S6553). ( diff --git a/workers/queue_backend/pg_queue/pg_scheduler.py b/workers/queue_backend/pg_queue/pg_scheduler.py new file mode 100644 index 0000000000..42755e7b05 --- /dev/null +++ b/workers/queue_backend/pg_queue/pg_scheduler.py @@ -0,0 +1,230 @@ +"""PG scheduler tick — the periodic-trigger half of the orchestrator (Phase 9, ②b). + +Folded into the leader-elected reaper loop (the reaper becomes "the +orchestrator": recover + schedule, per the labs single-orchestrator model). +Each cycle, *only while leader*, it scans ``pg_periodic_schedule`` for rows it +owns (``pg_owned``) that are enabled and due, enqueues the existing +``scheduler.tasks.execute_pipeline_task`` onto the PG queue, and advances +``next_run_at`` — replacing what Celery Beat does, without Beat/RabbitMQ. + +Correctness properties: + +- **No re-fire on crash.** The enqueue and the ``next_run_at`` advance happen in + **one transaction**, so a crash between them can't fire twice. +- **One firer per schedule (conditional).** A ``pg_owned`` row fires here; the + no-double-fire guarantee with Beat depends on the matching Beat + ``PeriodicTask`` being disabled when a schedule is handed over — that's the + ②c ramp control, which does not exist yet. **Until it lands, safety rests on + ``pg_owned`` defaulting to False** (nothing is owned → this fires nothing → + Beat fires everything). A row manually flipped to ``pg_owned=True`` while its + PeriodicTask is still enabled *would* double-fire. +- **No burst on hand-over.** A freshly-owned row has ``next_run_at IS NULL``; the + first tick records its baseline next time and does **not** fire (matches Beat: + a new schedule fires at its next cron match, not immediately). + +Per-row isolation (mirrors :func:`recover_expired_barriers`): a bad cron or a DB +error on one row is rolled back, logged, and skipped without poisoning the +connection or blocking the other rows. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import uuid +from datetime import datetime +from typing import TYPE_CHECKING, NamedTuple + +from croniter import croniter + +from unstract.core.data_models import TaskPayload + +from ..fairness import DEFAULT_PRIORITY +from .client import INSERT_MESSAGE_SQL +from .task_payload import to_payload + +if TYPE_CHECKING: + from psycopg2.extensions import connection as PgConnection + +logger = logging.getLogger(__name__) + +# The fired task + the queue a `scheduler` PG consumer polls (QueueName.SCHEDULER). +PIPELINE_TRIGGER_TASK = "scheduler.tasks.execute_pipeline_task" +SCHEDULER_QUEUE_NAME = "scheduler" + + +class _DueSchedule(NamedTuple): + """One row from the due-schedules scan — names bound to columns at one site + so a future reorder of the SELECT can't silently misassign fields. + """ + + pipeline_id: uuid.UUID + organization_id: str + workflow_id: uuid.UUID | None + pipeline_name: str + cron_string: str + next_run_at: datetime | None + + +def compute_next_run(cron_string: str, base: datetime) -> datetime: + """Next fire time strictly after ``base`` for a 5-field cron expression.""" + return croniter(cron_string, base).get_next(datetime) + + +def _build_trigger_payload( + *, + workflow_id: str | uuid.UUID | None, + organization_id: str, + pipeline_id: str | uuid.UUID, + pipeline_name: str, +) -> TaskPayload: + """``execute_pipeline_task`` payload. Positional args match its signature: + (workflow_id, org_schema, execution_action, execution_id, pipeline_id, + with_logs, name). ``execution_action`` / ``execution_id`` are ignored by + ``execute_pipeline_task_v2``, so we send blanks even though the Beat path + populates ``execution_action`` (see SchedulerHelper._schedule_task_job). + """ + return to_payload( + PIPELINE_TRIGGER_TASK, + args=[ + str(workflow_id) if workflow_id else "", + organization_id or "", + "", # execution_action (ignored by v2) + "", # execution_id (ignored by v2) + str(pipeline_id), + False, # with_logs (ignored by v2) + pipeline_name or "", + ], + kwargs={}, + queue=SCHEDULER_QUEUE_NAME, + fairness=None, + ) + + +def _quiesce_invalid_cron(conn: PgConnection, schedule: _DueSchedule) -> None: + """Disable a row whose cron can't be parsed, so it stops being re-selected + (and re-logging a traceback) every tick. Best-effort; logged once here. + """ + logger.exception( + "PG scheduler: invalid cron %r for pipeline %s — disabling the row", + schedule.cron_string, + schedule.pipeline_id, + ) + try: + with conn.cursor() as cur: + cur.execute( + "UPDATE pg_periodic_schedule SET enabled = FALSE WHERE pipeline_id = %s", + (schedule.pipeline_id,), + ) + conn.commit() + except Exception: + # If the disable UPDATE fails, roll back so the connection isn't left in + # an aborted-transaction state that would poison the next row's INSERT. + with contextlib.suppress(Exception): + conn.rollback() + + +def dispatch_due_schedules(conn: PgConnection) -> int: + """Fire PG-owned, enabled, due schedules; return the count actually fired. + + The caller (reaper tick) gates this on leadership. All time comparisons use + the DB clock (``now()``). Each row is handled in its own transaction; a bad + cron or a DB error on one row is rolled back + logged + skipped (the others + still fire). The read step rolls back + re-raises on error so the connection + is never handed back in an aborted-transaction state (mirrors + :func:`recover_expired_barriers`). + """ + try: + with conn.cursor() as cur: + cur.execute("SELECT now()") + base = cur.fetchone()[0] + cur.execute( + """ + SELECT pipeline_id, organization_id, workflow_id, pipeline_name, + cron_string, next_run_at + FROM pg_periodic_schedule + WHERE pg_owned AND enabled + AND (next_run_at IS NULL OR next_run_at <= %s) + """, + (base,), + ) + due = [_DueSchedule(*row) for row in cur.fetchall()] + conn.commit() + except Exception: + with contextlib.suppress(Exception): + conn.rollback() + raise + + fired = 0 + for schedule in due: + try: + nxt = compute_next_run(schedule.cron_string, base) + except Exception: + _quiesce_invalid_cron(conn, schedule) + continue + + try: + if schedule.next_run_at is None: + # First observation of a freshly-owned row: record the baseline + # next time and do NOT fire (no burst when handed over). + with conn.cursor() as cur: + cur.execute( + "UPDATE pg_periodic_schedule SET next_run_at = %s " + "WHERE pipeline_id = %s", + (nxt, schedule.pipeline_id), + ) + conn.commit() + logger.info( + "PG scheduler: baselined pipeline %s (next_run_at=%s, not fired)", + schedule.pipeline_id, + nxt, + ) + continue + + payload = _build_trigger_payload( + workflow_id=schedule.workflow_id, + organization_id=schedule.organization_id, + pipeline_id=schedule.pipeline_id, + pipeline_name=schedule.pipeline_name, + ) + # Enqueue + advance in ONE transaction so a crash between them can't + # re-fire next cycle. INSERT_MESSAGE_SQL is the shared enqueue + # contract from client.py (send() uses the same constant). + with conn.cursor() as cur: + cur.execute( + INSERT_MESSAGE_SQL, + ( + SCHEDULER_QUEUE_NAME, + json.dumps(payload), + schedule.organization_id or "", + DEFAULT_PRIORITY, + ), + ) + cur.execute( + "UPDATE pg_periodic_schedule " + "SET last_run_at = %s, next_run_at = %s WHERE pipeline_id = %s", + (base, nxt, schedule.pipeline_id), + ) + conn.commit() + except Exception: + # A row-level failure (constraint, serialization, socket) must not + # poison the connection or drop the rest of the batch — roll back + + # leave the row for the next tick (next_run_at unchanged → re-fires). + with contextlib.suppress(Exception): + conn.rollback() + logger.exception( + "PG scheduler: failed to fire pipeline %s — leaving for next tick", + schedule.pipeline_id, + ) + continue + + fired += 1 + logger.info( + "PG scheduler: fired pipeline %s → %s (next_run_at=%s)", + schedule.pipeline_id, + SCHEDULER_QUEUE_NAME, + nxt, + ) + + return fired diff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.py index 6bf5d7ec2c..d07e2919e5 100644 --- a/workers/queue_backend/pg_queue/reaper.py +++ b/workers/queue_backend/pg_queue/reaper.py @@ -58,6 +58,7 @@ from .connection import create_pg_connection from .leader_election import LeaderLease, default_worker_id from .liveness import LivenessServer as _BaseLivenessServer +from .pg_scheduler import dispatch_due_schedules if TYPE_CHECKING: from psycopg2.extensions import connection as PgConnection @@ -488,6 +489,16 @@ def tick(self) -> TickOutcome: except Exception: self._discard_owned_sweep_conn() raise + # Orchestrator's second job: fire due PG-owned schedules (Beat + # replacement). Ordered AFTER recovery so this cycle's recovery has + # already completed before any scheduler error can propagate (the except + # below still re-raises + discards the conn). Dark by default — fires + # nothing until rows are pg_owned. + try: + dispatch_due_schedules(self._get_sweep_conn()) + except Exception: + self._discard_owned_sweep_conn() + raise return TickOutcome(was_leader=True, reclaimed=reclaimed) def run(self, *, install_signals: bool = True) -> None: diff --git a/workers/run-worker.sh b/workers/run-worker.sh index fee8131aab..91b18c2305 100755 --- a/workers/run-worker.sh +++ b/workers/run-worker.sh @@ -55,11 +55,16 @@ readonly PG_ROLE_ORCH_API="pg-orchestrator-api" readonly PG_ROLE_ORCH_GENERAL="pg-orchestrator-general" readonly PG_ROLE_FILEPROC="pg-fileproc" readonly PG_ROLE_CALLBACK="pg-callback" +readonly PG_ROLE_SCHEDULER="pg-scheduler" declare -rA PG_CONSUMER_ROLES=( ["$PG_ROLE_ORCH_API"]="api_deployment;celery_api_deployments" ["$PG_ROLE_ORCH_GENERAL"]="general;celery" ["$PG_ROLE_FILEPROC"]="file_processing;file_processing,api_file_processing" ["$PG_ROLE_CALLBACK"]="callback;file_processing_callback,api_file_processing_callback" + # Runs scheduler.tasks.execute_pipeline_task fired by the orchestrator's PG + # scheduler tick (Beat replacement). Distinct from the Celery 'scheduler' + # worker (which Beat fires onto RabbitMQ). + ["$PG_ROLE_SCHEDULER"]="scheduler;scheduler" ) declare -rA PG_QUEUE_MEMBERS=( ["$PG_QUEUE_CONSUMER_TYPE"]=1 @@ -68,6 +73,7 @@ declare -rA PG_QUEUE_MEMBERS=( ["$PG_ROLE_ORCH_GENERAL"]=1 ["$PG_ROLE_FILEPROC"]=1 ["$PG_ROLE_CALLBACK"]=1 + ["$PG_ROLE_SCHEDULER"]=1 ) # The Celery transport set: every worker EXCEPT the PG-queue members — the # *complement* of the 'pg-queue' set, so the two transports' logs can be tailed @@ -105,6 +111,7 @@ declare -A WORKERS=( ["$PG_ROLE_ORCH_GENERAL"]="$PG_ROLE_ORCH_GENERAL" ["$PG_ROLE_FILEPROC"]="$PG_ROLE_FILEPROC" ["$PG_ROLE_CALLBACK"]="$PG_ROLE_CALLBACK" + ["$PG_ROLE_SCHEDULER"]="$PG_ROLE_SCHEDULER" # PG Queue reaper — leader-elected recovery loop (barrier-orphan sweep) ["reaper"]="$PG_QUEUE_REAPER_TYPE" ["pg-queue-reaper"]="$PG_QUEUE_REAPER_TYPE" @@ -193,6 +200,7 @@ WORKER_TYPE: pg-orchestrator-general Run the PG orchestrator consumer for ETL/general execs (celery) pg-fileproc Run the PG fan-out consumer (file_processing + api_file_processing) pg-callback Run the PG callback consumer (file_processing_callback + api_file_processing_callback) + pg-scheduler Run the PG scheduler consumer (runs execute_pipeline_task fired by the orchestrator) reaper, pg-queue-reaper Run PG-queue reaper (leader-elected recovery; opt-in) pg, pg-queue Run the whole PG-queue set (the 4 pipeline roles + reaper) together all, celery Run the Celery worker set (all Celery workers; excludes the PG set) diff --git a/workers/tests/test_pg_reaper.py b/workers/tests/test_pg_reaper.py index ed89fff2ef..ab3b3bdf4d 100644 --- a/workers/tests/test_pg_reaper.py +++ b/workers/tests/test_pg_reaper.py @@ -31,6 +31,18 @@ recover_expired_barriers, ) +# The reaper's leader tick also runs the PG scheduler tick (②b). Its behaviour +# is covered in test_pg_scheduler.py; stub it here by default so the leadership / +# recovery / connection tests aren't coupled to a real schedule query on their +# dummy or barrier-only connections. Tests that assert the wiring opt in via the +# returned mock. +@pytest.fixture(autouse=True) +def stub_scheduler_tick(monkeypatch): + mock = MagicMock(return_value=0) + monkeypatch.setattr(reaper_mod, "dispatch_due_schedules", mock) + return mock + + # --- Layer 1: env + construction (no DB) --- @@ -181,6 +193,62 @@ def boom(): logexc.assert_called_once() +class TestSchedulerTick: + """The orchestrator's second job: the leader (and only the leader) runs the + PG scheduler tick each cycle. Scheduling behaviour itself is in + test_pg_scheduler.py; here we only assert the wiring + leader gating.""" + + def _reaper(self, lease): + return PgReaper( + lease, interval_seconds=0.01, sweep_conn=object(), api_client=object() + ) + + def test_leader_runs_scheduler(self, stub_scheduler_tick): + reaper = self._reaper(_FakeLease(acquires=True, renews=True)) + with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]): + reaper.tick() + stub_scheduler_tick.assert_called_once() + + def test_standby_does_not_run_scheduler(self, stub_scheduler_tick): + reaper = self._reaper(_FakeLease(acquires=False)) + with patch.object(reaper_mod, "recover_expired_barriers"): + reaper.tick() + stub_scheduler_tick.assert_not_called() + + def test_scheduler_runs_after_recovery(self, stub_scheduler_tick): + # Recovery is the safety net — it must run before scheduling so a + # scheduler error can't starve it. Assert ordering via a shared call log. + order = [] + reaper = self._reaper(_FakeLease(acquires=True, renews=True)) + stub_scheduler_tick.side_effect = lambda *_: order.append("schedule") + with patch.object( + reaper_mod, + "recover_expired_barriers", + side_effect=lambda *_: order.append("recover") or [], + ): + reaper.tick() + assert order == ["recover", "schedule"] + + def test_scheduler_error_discards_owned_conn(self, stub_scheduler_tick): + # A scheduler DB error must propagate (run() catches + continues) AND + # discard the owned sweep conn — same posture as a failed recovery. + # sweep_conn=None → owned; api_client=object() so _get_api_client doesn't + # build a real one (both are evaluated to call the patched recovery). + reaper = PgReaper( + _FakeLease(acquires=True, renews=True), + interval_seconds=0.01, + api_client=object(), + ) + owned = MagicMock() + owned.closed = False # so _get_sweep_conn returns it, doesn't reconnect + reaper._sweep_conn = owned + stub_scheduler_tick.side_effect = psycopg2.OperationalError("db gone") + with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]): + with pytest.raises(psycopg2.OperationalError): + reaper.tick() + assert reaper._sweep_conn is None # discarded + + # --- Layer 3: connection handling (mocked, no DB) --- diff --git a/workers/tests/test_pg_scheduler.py b/workers/tests/test_pg_scheduler.py new file mode 100644 index 0000000000..75d753b29c --- /dev/null +++ b/workers/tests/test_pg_scheduler.py @@ -0,0 +1,276 @@ +"""Tests for the PG scheduler tick (Phase 9, ②b). + +Pure tests (cron next-run, payload shape) need no DB. The dispatch behaviour is +exercised against real Postgres via the shared ``pg_conn`` fixture (skips when +unreachable/unmigrated). ``dispatch_due_schedules`` commits, so the real-PG +tests seed + clean up their own rows under a unique org marker. +""" + +import datetime +import uuid + +import psycopg2 +import pytest + +from queue_backend.pg_queue.pg_scheduler import ( + SCHEDULER_QUEUE_NAME, + _build_trigger_payload, + compute_next_run, + dispatch_due_schedules, +) + + +class TestPureHelpers: + def test_next_run_is_strictly_after_base(self): + base = datetime.datetime(2026, 6, 18, 10, 0, 0) + # 09:00 daily — already past at 10:00, so the next is tomorrow 09:00. + assert compute_next_run("0 9 * * *", base) == datetime.datetime( + 2026, 6, 19, 9, 0, 0 + ) + + def test_next_run_same_day_when_upcoming(self): + base = datetime.datetime(2026, 6, 18, 8, 0, 0) + assert compute_next_run("0 9 * * *", base) == datetime.datetime( + 2026, 6, 18, 9, 0, 0 + ) + + def test_next_run_preserves_tzaware_base(self): + # Production base is tz-aware (SELECT now()); the result must be aware too. + base = datetime.datetime(2026, 6, 18, 10, 0, 0, tzinfo=datetime.timezone.utc) + nxt = compute_next_run("0 9 * * *", base) + assert nxt.tzinfo is not None + assert nxt == datetime.datetime( + 2026, 6, 19, 9, 0, 0, tzinfo=datetime.timezone.utc + ) + + def test_invalid_cron_raises(self): + with pytest.raises(Exception): + compute_next_run("not a cron", datetime.datetime(2026, 6, 18, 10, 0, 0)) + + def test_trigger_payload_shape(self): + p = _build_trigger_payload( + workflow_id="wf-1", + organization_id="org-1", + pipeline_id="pid-1", + pipeline_name="Nightly ETL", + ) + assert p["task_name"] == "scheduler.tasks.execute_pipeline_task" + assert p["queue"] == SCHEDULER_QUEUE_NAME + # (workflow_id, org, execution_action, execution_id, pipeline_id, with_logs, name) + assert p["args"] == ["wf-1", "org-1", "", "", "pid-1", False, "Nightly ETL"] + assert p["kwargs"] == {} + assert p["fairness"] is None + + +# --- real-PG dispatch behaviour --- + +_MARKER = f"test_pgsched_{uuid.uuid4().hex[:8]}" + + +def _seed(conn, *, pg_owned, enabled, next_run_at, cron="0 9 * * *"): + """Insert one pg_periodic_schedule row; returns its pipeline_id (str).""" + pid = str(uuid.uuid4()) + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO pg_periodic_schedule + (pipeline_id, organization_id, workflow_id, pipeline_name, + cron_string, enabled, pg_owned, last_run_at, next_run_at, + created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, NULL, %s, now(), now()) + """, + (pid, _MARKER, str(uuid.uuid4()), "Test ETL", cron, enabled, pg_owned, + next_run_at), + ) + conn.commit() + return pid + + +def _row(conn, pid): + with conn.cursor() as cur: + cur.execute( + "SELECT last_run_at, next_run_at FROM pg_periodic_schedule " + "WHERE pipeline_id = %s", + (pid,), + ) + return cur.fetchone() + + +def _queued_messages(conn): + with conn.cursor() as cur: + cur.execute( + "SELECT message FROM pg_queue_message WHERE queue_name = %s AND org_id = %s", + (SCHEDULER_QUEUE_NAME, _MARKER), + ) + return [r[0] for r in cur.fetchall()] + + +@pytest.fixture +def clean(pg_conn): + """Remove any rows this test created (the tick commits, so teardown must).""" + yield pg_conn + with pg_conn.cursor() as cur: + cur.execute("DELETE FROM pg_periodic_schedule WHERE organization_id = %s", (_MARKER,)) + cur.execute("DELETE FROM pg_queue_message WHERE org_id = %s", (_MARKER,)) + pg_conn.commit() + + +class TestDispatchDueSchedules: + def test_due_owned_row_fires_and_advances(self, clean): + conn = clean + past = datetime.datetime(2020, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc) + pid = _seed(conn, pg_owned=True, enabled=True, next_run_at=past) + + fired = dispatch_due_schedules(conn) + + assert fired == 1 # fired exactly once (not zero, not double) + msgs = _queued_messages(conn) + assert len(msgs) == 1 + payload = msgs[0] + assert payload["task_name"] == "scheduler.tasks.execute_pipeline_task" + assert payload["args"][4] == pid # pipeline_id at index 4 + last_run, next_run = _row(conn, pid) + assert last_run is not None # fired → last_run stamped + # Advanced to the cron's next match (09:00 UTC), not just "something > past". + assert (next_run.hour, next_run.minute, next_run.second) == (9, 0, 0) + assert next_run > last_run + + def test_null_next_run_baselines_without_firing(self, clean): + conn = clean + pid = _seed(conn, pg_owned=True, enabled=True, next_run_at=None) + + assert dispatch_due_schedules(conn) == 0 # baseline is NOT a fire + assert _queued_messages(conn) == [] + last_run, next_run = _row(conn, pid) + assert last_run is None # never fired + assert next_run is not None # baseline recorded + + def test_two_due_rows_both_fire(self, clean): + conn = clean + past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc) + p1 = _seed(conn, pg_owned=True, enabled=True, next_run_at=past) + p2 = _seed(conn, pg_owned=True, enabled=True, next_run_at=past) + + assert dispatch_due_schedules(conn) == 2 + fired_pids = {m["args"][4] for m in _queued_messages(conn)} + assert fired_pids == {p1, p2} + + def test_not_owned_row_is_skipped(self, clean): + conn = clean + past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc) + _seed(conn, pg_owned=False, enabled=True, next_run_at=past) + + dispatch_due_schedules(conn) + + assert _queued_messages(conn) == [] + + def test_disabled_row_is_skipped(self, clean): + conn = clean + past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc) + _seed(conn, pg_owned=True, enabled=False, next_run_at=past) + + dispatch_due_schedules(conn) + + assert _queued_messages(conn) == [] + + def test_future_next_run_not_yet_due(self, clean): + conn = clean + future = datetime.datetime(2099, 1, 1, tzinfo=datetime.timezone.utc) + _seed(conn, pg_owned=True, enabled=True, next_run_at=future) + + dispatch_due_schedules(conn) + + assert _queued_messages(conn) == [] + + def test_bad_cron_skipped_and_disabled(self, clean): + conn = clean + past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc) + bad = _seed(conn, pg_owned=True, enabled=True, next_run_at=past, cron="garbage") + good = _seed(conn, pg_owned=True, enabled=True, next_run_at=past) + + dispatch_due_schedules(conn) + + msgs = _queued_messages(conn) + # Only the good row fired; the bad-cron row was skipped, not fatal. + assert len(msgs) == 1 + assert msgs[0]["args"][4] == good + # The bad-cron row is disabled so it stops being re-selected every tick. + with conn.cursor() as cur: + cur.execute( + "SELECT enabled FROM pg_periodic_schedule WHERE pipeline_id = %s", + (bad,), + ) + assert cur.fetchone()[0] is False + + def test_advance_failure_rolls_back_the_enqueue(self, clean): + """Atomicity: if the next_run_at advance fails after the INSERT, the + INSERT must roll back with it — no orphan message, next_run unchanged + (so the row simply re-fires next tick rather than double-firing).""" + conn = clean + past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc) + pid = _seed(conn, pg_owned=True, enabled=True, next_run_at=past) + + # The per-row failure is swallowed (isolation), so dispatch returns 0. + # Fail the fire-path advance (the only UPDATE setting last_run_at). + proxy = _FailingConn(conn, lambda sql: "last_run_at" in sql) + fired = dispatch_due_schedules(proxy) + + assert fired == 0 + assert _queued_messages(conn) == [] # INSERT rolled back with the UPDATE + last_run, next_run = _row(conn, pid) + assert last_run is None # not advanced + assert next_run == past # unchanged → re-fires next tick (no double-fire) + + def test_quiesce_failure_does_not_poison_next_row(self, clean): + """If disabling a bad-cron row fails, the rollback must leave the conn + clean so a following healthy row still fires (greptile P1).""" + conn = clean + past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc) + _seed(conn, pg_owned=True, enabled=True, next_run_at=past, cron="garbage") + good = _seed(conn, pg_owned=True, enabled=True, next_run_at=past) + + # Make the bad-cron disable UPDATE fail; the good row must still fire. + proxy = _FailingConn(conn, lambda sql: "enabled = FALSE" in sql) + dispatch_due_schedules(proxy) + + msgs = _queued_messages(conn) + assert len(msgs) == 1 + assert msgs[0]["args"][4] == good + + +class _FailingConn: + """Wraps a real connection so an ``execute`` whose SQL matches ``fail_when`` + raises — to prove a statement failure rolls back cleanly. Everything else + (commit/rollback/other statements) passes through to the real connection. + """ + + def __init__(self, real, fail_when): + self._real = real + self._fail_when = fail_when + + def __getattr__(self, name): + return getattr(self._real, name) # commit / rollback / etc. + + def cursor(self): + return _FailingCursor(self._real.cursor(), self._fail_when) + + +class _FailingCursor: + def __init__(self, cur, fail_when): + self._cur = cur + self._fail_when = fail_when + + def __enter__(self): + self._cur.__enter__() + return self + + def __exit__(self, *exc): + return self._cur.__exit__(*exc) + + def __getattr__(self, name): + return getattr(self._cur, name) # fetchone / fetchall / etc. + + def execute(self, sql, params=None): + if self._fail_when(sql): + raise psycopg2.OperationalError("forced failure") + return self._cur.execute(sql, params) diff --git a/workers/uv.lock b/workers/uv.lock index 6f625ec13b..2be2e6b718 100644 --- a/workers/uv.lock +++ b/workers/uv.lock @@ -875,6 +875,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0d/4a/331fe2caf6799d591109bb9c08083080f6de90a823695d412a935622abb2/coverage-7.13.4-py3-none-any.whl", hash = "sha256:1af1641e57cf7ba1bd67d677c9abdbcd6cc2ab7da3bca7fa1e2b7e50e65f2ad0", size = 211242, upload-time = "2026-02-09T12:59:02.032Z" }, ] +[[package]] +name = "croniter" +version = "6.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/de/5832661ed55107b8a09af3f0a2e71e0957226a59eb1dcf0a445cce6daf20/croniter-6.2.2.tar.gz", hash = "sha256:ba60832a5ec8e12e51b8691c3309a113d1cf6526bdf1a48150ce8ec7a532d0ab", size = 113762, upload-time = "2026-03-15T08:43:48.112Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/39/783980e78cb92c2d7bdb1fc7dbc86e94ccc6d58224d76a7f1f51b6c51e30/croniter-6.2.2-py3-none-any.whl", hash = "sha256:a5d17b1060974d36251ea4faf388233eca8acf0d09cbd92d35f4c4ac8f279960", size = 45422, upload-time = "2026-03-15T08:43:46.626Z" }, +] + [[package]] name = "cryptography" version = "48.0.0" @@ -4900,6 +4912,7 @@ source = { editable = "." } dependencies = [ { name = "boto3" }, { name = "celery" }, + { name = "croniter" }, { name = "httpx" }, { name = "prometheus-client" }, { name = "psutil" }, @@ -4949,6 +4962,7 @@ test = [ requires-dist = [ { name = "boto3", specifier = "~=1.34.0" }, { name = "celery", specifier = ">=5.5.3" }, + { name = "croniter", specifier = ">=3.0.3" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "prometheus-client", specifier = ">=0.17.0,<1.0.0" }, { name = "psutil", specifier = ">=5.9.0,<6.0.0" }, From 7d775cef49f4c1327b4f8770eb0ba2111354971e Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Fri, 19 Jun 2026 08:09:04 +0530 Subject: [PATCH 26/44] =?UTF-8?q?UN-3597=20[FEAT]=20PG=20Queue=209h-c=20?= =?UTF-8?q?=E2=80=94=20schedule=20ownership=20ramp=20control=20+=20backfil?= =?UTF-8?q?l=20(Beat=E2=86=92PG=20hand-over)=20(#2085)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3597 [FEAT] PG Queue 9h-c — schedule ownership ramp control + backfill (Beat→PG hand-over) The ramp control that hands schedules from Celery Beat to the PG scheduler, per-schedule and reversibly. Inert by default (Flipt at 0% / gate off → every schedule stays on Beat), so deploying it changes nothing until ops ramps. - scheduler/ownership.py: resolve_schedule_owner (master-gate PG_QUEUE_TRANSPORT_ ENABLED → pg_scheduler_enabled Flipt flag, keyed on pipeline_id; fail-closed to Beat) + reconcile_ownership_for (ONE transaction: set pg_owned AND PeriodicTask.enabled = active AND NOT pg_owned). Doing both atomically is what makes "never double-fires" real — a pg_owned schedule always has its Beat PeriodicTask disabled. Separate Flipt flag from pg_queue_execution_enabled so scheduling and execution ramp independently. No new env. - scheduler/helper.py: reconcile ownership after the mirror upsert on every schedule create/update (best-effort; a no-op while the rollout is off). - pg_queue management command reconcile_pg_schedules: backfills mirror rows for pre-existing Beat schedules + reconciles all ownership against the current rollout (run once + after each Flipt ramp; idempotent, --dry-run). - tests: 10 ownership (fail-closed matrix + the disable-Beat/pause invariants) + the helper-wiring assertion (21 backend tests green). Dev-tested live on the real DB: hand-over (pg_owned + Beat disabled), rollback, pause; backfill of real unmirrored schedules; and the cross-slice link — a reconciled-to-PG schedule is fired by the ②b reaper. Full container e2e (pg-scheduler consumer → real execution via the live Flipt flag) folded into the ops ramp, alongside the execution canary. Out of scope: stopping the celery-beat container / removing django_celery_beat (ops, once ramped to 100%; final decommission). Co-Authored-By: Claude Opus 4.8 * UN-3597 address review: fix resume double-fire, guard command, atomic-rollback proof, ramp robustness - [HIGH] resume double-fire: enable_task (resume path, separate from reconcile) unconditionally re-enabled Beat — on a pg_owned schedule that meant Beat AND PG both firing. Now sets PeriodicTask.enabled = NOT pg_owned (reads the mirror). - [CRITICAL] reconcile_pg_schedules: per-row json.loads guard (+ isinstance list) so one malformed PeriodicTask.args row logs+skips instead of aborting the whole command (and starving the reconcile step). - [HIGH] command failure reporting: reconcile_ownership_for returns None on a swallowed DB failure; the command tallies failed, writes an ERROR summary, and raises CommandError so automation/ops notice (was: counted failures as success). - [MED] rollback (PG→Beat) now also clears next_run_at, so a later re-hand-over re-enters the NULL baseline instead of bursting on a stale timestamp. - [MED] reconcile_ownership_for: active is keyword-only (boolean-trap guard). - [MED] docstring stale name reconcile_schedule_ownership -> reconcile_ownership_for. - [LOW] flag_key passed by keyword; Flipt-failure log downgraded to warning(exc_info=True) (expected, runs every edit); single f-string; dry-run now previews would-be pg_owned via resolve_schedule_owner (read-only). - tests: enable_task resume (both pg_owned states), next_run_at reset on rollback, None-on-failure, the full command (backfill skip / malformed-args / non-list / dry-run preview / CommandError), and a real-DB atomicity test (force the PeriodicTask update to fail → pg_owned rolls back). 29 backend tests green. Deferred [LOW]: bool-vs-enum return (pg_owned maps directly to the BooleanField; not a wire contract like WorkflowTransport) — see reply. Co-Authored-By: Claude Opus 4.8 * UN-3597 address SonarCloud: reduce reconcile_pg_schedules complexity + drop redundant except - Cognitive Complexity 29→<15: extracted handle() into _backfill_mirrors, _parse_task_args, and _reconcile_all; handle() is now a thin orchestrator. - Dropped the redundant `json.JSONDecodeError` from the except tuple — it's a ValueError subclass, so `except ValueError` already covers both the parse error and the non-array guard. Behaviour unchanged; 5 command tests + the rest green. Co-Authored-By: Claude Opus 4.8 * UN-3597 address SonarCloud: reduce _backfill_mirrors complexity 17→<15 Move the per-arg extraction (the three positional ternaries) out of the backfill loop into _mirror_fields_from_args, which returns the mirror kwargs (or None on a bad row). _backfill_mirrors now just loops + splats the fields, dropping the nested ternaries that drove the cognitive complexity. Behaviour unchanged; command tests green. Co-Authored-By: Claude Opus 4.8 * UN-3597 address greptile: race-safe resume, accurate ramp accounting, updated_at, N+1 - [P1] enable_task lost-update race: task.save() could clobber a concurrent reconcile_ownership_for that flipped pg_owned=True + PeriodicTask.enabled=False → both Beat and PG firing. Now reads pg_owned under select_for_update inside a transaction and writes only the enabled column via .update() (no stale full-row save). Preserves DoesNotExist on a bad name. - [P2] reconcile_ownership_for: when there's no mirror row (updated==0) return False, not the raw resolved pg_owned — PG can't fire without a row, so the effective owner is Beat; returning True would inflate the ramp pg_owned count. - [P2] reconcile_ownership_for now bumps updated_at in the .update() (queryset .update() doesn't fire auto_now), so an ownership flip advances the timestamp. - [P2] reconcile_pg_schedules._backfill_mirrors N+1: pre-fetch all mirrored pipeline ids in one query into a set instead of an EXISTS per PeriodicTask. - tests updated for all four (incl. select_for_update mocking, no-row→False, values_list prefetch). 29 backend tests green; handover + updated_at bump re-verified live. Co-Authored-By: Claude Opus 4.8 * UN-3597 address greptile: add missing import pytest to ownership test TestReconcileAtomicityRealDB calls pytest.skip() when the DB is unavailable, but pytest was never imported — so a DB gap would raise NameError instead of skipping (latent; local runs had a DB so the line was never hit). Add the import. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- backend/pg_queue/management/__init__.py | 0 .../pg_queue/management/commands/__init__.py | 0 .../commands/reconcile_pg_schedules.py | 148 ++++++++++++ .../test_reconcile_pg_schedules_command.py | 122 ++++++++++ backend/scheduler/helper.py | 8 + backend/scheduler/ownership.py | 145 ++++++++++++ backend/scheduler/tasks.py | 21 +- .../tests/test_pg_periodic_schedule_mirror.py | 36 ++- .../tests/test_pg_schedule_ownership.py | 216 ++++++++++++++++++ 9 files changed, 690 insertions(+), 6 deletions(-) create mode 100644 backend/pg_queue/management/__init__.py create mode 100644 backend/pg_queue/management/commands/__init__.py create mode 100644 backend/pg_queue/management/commands/reconcile_pg_schedules.py create mode 100644 backend/pg_queue/tests/test_reconcile_pg_schedules_command.py create mode 100644 backend/scheduler/ownership.py create mode 100644 backend/scheduler/tests/test_pg_schedule_ownership.py diff --git a/backend/pg_queue/management/__init__.py b/backend/pg_queue/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/pg_queue/management/commands/__init__.py b/backend/pg_queue/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/pg_queue/management/commands/reconcile_pg_schedules.py b/backend/pg_queue/management/commands/reconcile_pg_schedules.py new file mode 100644 index 0000000000..4ee7019d81 --- /dev/null +++ b/backend/pg_queue/management/commands/reconcile_pg_schedules.py @@ -0,0 +1,148 @@ +"""Backfill the pg_periodic_schedule mirror + reconcile Beat/PG schedule +ownership (Phase 9, ②c). + +Run this: +- **once** after deploying the mirror, to backfill rows for schedules created + before the mirror existed (the dual-write only covers schedules touched since); +- **after each Flipt ramp change** to ``pg_scheduler_enabled``, to apply the new + percentage — flipping ``pg_owned`` and the matching Beat ``PeriodicTask`` for + every schedule (the create/update path only reconciles the schedule it edits). + +It is idempotent and safe to run anytime: with the rollout off it leaves every +schedule on Beat. Could later be driven periodically (e.g. by the orchestrator); +kept a command here so the ramp stays an explicit, auditable ops action. +""" + +import json +from typing import Any + +from django.core.management.base import BaseCommand, CommandError +from django_celery_beat.models import CrontabSchedule, PeriodicTask +from scheduler.ownership import reconcile_ownership_for, resolve_schedule_owner +from scheduler.tasks import mirror_periodic_schedule_upsert + +from pg_queue.models import PgPeriodicSchedule + +# Only the pipeline-trigger PeriodicTasks are scheduled pipelines (other periodic +# tasks — metrics, audit — are not mirrored). +_PIPELINE_TASK_PATH = "scheduler.tasks.execute_pipeline_task" + + +def _cron_from_crontab(crontab: CrontabSchedule | None) -> str: + """Reconstruct the 5-field cron string from a CrontabSchedule row.""" + if crontab is None: + return "" + return ( + f"{crontab.minute} {crontab.hour} {crontab.day_of_month} " + f"{crontab.month_of_year} {crontab.day_of_week}" + ) + + +class Command(BaseCommand): + help = ( + "Backfill pg_periodic_schedule mirrors for pre-existing schedules and " + "reconcile Beat/PG ownership against the current pg_scheduler_enabled " + "rollout. Idempotent; with the rollout off, leaves everything on Beat." + ) + + def add_arguments(self, parser: Any) -> None: + parser.add_argument( + "--dry-run", + action="store_true", + help="Report what would change without writing.", + ) + + def handle(self, *args: Any, **options: Any) -> None: + dry_run = options["dry_run"] + backfilled = self._backfill_mirrors(dry_run) + reconciled, pg_owned, failed = self._reconcile_all(dry_run) + + prefix = "[dry-run] " if dry_run else "" + summary = ( + f"{prefix}backfilled={backfilled} reconciled={reconciled} " + f"pg_owned={pg_owned} failed={failed}" + ) + if failed: + # Surface failures where the operator looks (and to automation). + self.stderr.write(self.style.ERROR(summary)) + raise CommandError(f"{failed} schedule(s) failed to reconcile") + self.stdout.write(self.style.SUCCESS(summary)) + + def _mirror_fields_from_args(self, pt: Any, pipeline_id: str) -> dict | None: + """Extract the mirror fields from PeriodicTask.args, or None (logged) for a + malformed/non-array row — a bad row must not abort the whole command. + """ + try: + # json.JSONDecodeError is a ValueError subclass, so one except covers + # both the parse error and the non-array guard below. + task_args = json.loads(pt.args or "[]") + if not isinstance(task_args, list): + raise ValueError(f"expected JSON array, got {type(task_args).__name__}") + except ValueError as exc: + self.stderr.write( + self.style.ERROR( + f"skipping pipeline {pipeline_id}: bad PeriodicTask.args ({exc})" + ) + ) + return None + return { + "workflow_id": task_args[0] if len(task_args) > 0 else None, + "organization_id": (task_args[1] if len(task_args) > 1 else "") or "", + # args[6] is the synthetic "Pipeline job-" label; the real name + # self-heals via the dual-write on the next schedule edit. + "pipeline_name": task_args[6] if len(task_args) > 6 else "", + } + + def _backfill_mirrors(self, dry_run: bool) -> int: + """Create a mirror row for every pipeline-trigger PeriodicTask lacking one.""" + # Pre-fetch the already-mirrored ids in one query (avoid an EXISTS per row). + mirrored = { + str(pk) + for pk in PgPeriodicSchedule.objects.values_list("pipeline_id", flat=True) + } + backfilled = 0 + for pt in PeriodicTask.objects.filter(task=_PIPELINE_TASK_PATH): + pipeline_id = pt.name # = str(pipeline.pk) + if pipeline_id in mirrored: + continue + fields = self._mirror_fields_from_args(pt, pipeline_id) + if fields is None: + continue + self.stdout.write( + f"backfill mirror for pipeline {pipeline_id} (enabled={pt.enabled})" + ) + if not dry_run: + mirror_periodic_schedule_upsert( + pipeline_id=pipeline_id, + cron_string=_cron_from_crontab(pt.crontab), + enabled=pt.enabled, + **fields, + ) + backfilled += 1 + return backfilled + + def _reconcile_all(self, dry_run: bool) -> tuple[int, int, int]: + """Reconcile ownership for every mirror row against the current rollout. + Returns (reconciled, pg_owned, failed). + """ + reconciled = pg_owned = failed = 0 + for row in PgPeriodicSchedule.objects.all(): + if dry_run: + # Preview only — read the would-be owner (no DB write) so an + # operator can see how many a ramp change would hand to PG. + reconciled += 1 + if resolve_schedule_owner(str(row.pipeline_id), row.organization_id): + pg_owned += 1 + continue + # mirror.enabled tracks pipeline.active (dual-write); use it as the + # 'active' input so a paused schedule isn't re-enabled by reconcile. + result = reconcile_ownership_for( + str(row.pipeline_id), row.organization_id, active=row.enabled + ) + if result is None: # transaction failed (already logged) + failed += 1 + continue + reconciled += 1 + if result: + pg_owned += 1 + return reconciled, pg_owned, failed diff --git a/backend/pg_queue/tests/test_reconcile_pg_schedules_command.py b/backend/pg_queue/tests/test_reconcile_pg_schedules_command.py new file mode 100644 index 0000000000..88545e917e --- /dev/null +++ b/backend/pg_queue/tests/test_reconcile_pg_schedules_command.py @@ -0,0 +1,122 @@ +"""Tests for the reconcile_pg_schedules management command (Phase 9, ②c). + +DB-free: the ORM, mirror upsert, and ownership reconcile are mocked. These pin +the operator-facing contract — backfill skip, the malformed-args guard, the +dry-run preview, the counters, and the non-zero exit on failure. +""" + +from unittest.mock import MagicMock, patch + +import pytest +from django.core.management import call_command +from django.core.management.base import CommandError + +_CMD = "pg_queue.management.commands.reconcile_pg_schedules" + + +def _pt(name, args="[]", enabled=True): + m = MagicMock() + m.name = name + m.args = args + m.enabled = enabled + m.crontab.minute = "0" + m.crontab.hour = "9" + m.crontab.day_of_month = "*" + m.crontab.month_of_year = "*" + m.crontab.day_of_week = "*" + return m + + +def _row(pid, org="org", enabled=True): + m = MagicMock() + m.pipeline_id = pid + m.organization_id = org + m.enabled = enabled + return m + + +class TestReconcileCommand: + def test_backfills_only_unmirrored_and_reconciles(self): + pt_new = _pt("pid-new", args='["wf", "org", "", "", "pid-new", false, "n"]') + pt_exists = _pt("pid-exists") + with ( + patch(f"{_CMD}.PeriodicTask") as PT, + patch(f"{_CMD}.PgPeriodicSchedule") as Sched, + patch(f"{_CMD}.mirror_periodic_schedule_upsert") as upsert, + patch(f"{_CMD}.reconcile_ownership_for", return_value=False) as reconcile, + ): + PT.objects.filter.return_value = [pt_new, pt_exists] + # pid-exists already mirrored; pid-new not (one prefetch query). + Sched.objects.values_list.return_value = ["pid-exists"] + Sched.objects.all.return_value = [_row("pid-new"), _row("pid-exists")] + call_command("reconcile_pg_schedules") + + upsert.assert_called_once() # only the unmirrored one backfilled + assert upsert.call_args.kwargs["pipeline_id"] == "pid-new" + assert reconcile.call_count == 2 # both rows reconciled + + def test_malformed_args_skipped_not_fatal(self): + bad = _pt("pid-bad", args="{ this is not json") + good = _pt("pid-good", args='["wf", "org", "", "", "pid-good", false, "n"]') + with ( + patch(f"{_CMD}.PeriodicTask") as PT, + patch(f"{_CMD}.PgPeriodicSchedule") as Sched, + patch(f"{_CMD}.mirror_periodic_schedule_upsert") as upsert, + patch(f"{_CMD}.reconcile_ownership_for", return_value=False), + ): + PT.objects.filter.return_value = [bad, good] + Sched.objects.values_list.return_value = [] + Sched.objects.all.return_value = [] + # Must not raise despite the bad row. + call_command("reconcile_pg_schedules") + + # Only the good row backfilled; the bad one skipped, not fatal. + assert upsert.call_count == 1 + assert upsert.call_args.kwargs["pipeline_id"] == "pid-good" + + def test_non_list_args_skipped(self): + weird = _pt("pid-weird", args="null") # valid JSON, not a list + with ( + patch(f"{_CMD}.PeriodicTask") as PT, + patch(f"{_CMD}.PgPeriodicSchedule") as Sched, + patch(f"{_CMD}.mirror_periodic_schedule_upsert") as upsert, + patch(f"{_CMD}.reconcile_ownership_for", return_value=False), + ): + PT.objects.filter.return_value = [weird] + Sched.objects.values_list.return_value = [] + Sched.objects.all.return_value = [] + call_command("reconcile_pg_schedules") + + upsert.assert_not_called() + + def test_dry_run_writes_nothing_but_previews_owner(self): + with ( + patch(f"{_CMD}.PeriodicTask") as PT, + patch(f"{_CMD}.PgPeriodicSchedule") as Sched, + patch(f"{_CMD}.mirror_periodic_schedule_upsert") as upsert, + patch(f"{_CMD}.reconcile_ownership_for") as reconcile, + patch(f"{_CMD}.resolve_schedule_owner", return_value=True) as resolve, + ): + PT.objects.filter.return_value = [ + _pt("pid-1", args='["wf", "org", "", "", "pid-1", false, "n"]') + ] + Sched.objects.values_list.return_value = [] + Sched.objects.all.return_value = [_row("pid-1")] + call_command("reconcile_pg_schedules", "--dry-run") + + upsert.assert_not_called() # no backfill write + reconcile.assert_not_called() # no ownership write + resolve.assert_called_once() # but the would-be owner is previewed + + def test_failure_raises_command_error(self): + with ( + patch(f"{_CMD}.PeriodicTask") as PT, + patch(f"{_CMD}.PgPeriodicSchedule") as Sched, + patch(f"{_CMD}.mirror_periodic_schedule_upsert"), + patch(f"{_CMD}.reconcile_ownership_for", return_value=None), # failed + ): + PT.objects.filter.return_value = [] + Sched.objects.values_list.return_value = [] + Sched.objects.all.return_value = [_row("pid-1")] + with pytest.raises(CommandError): + call_command("reconcile_pg_schedules") diff --git a/backend/scheduler/helper.py b/backend/scheduler/helper.py index 1b8c68ecf3..d4ea878c42 100644 --- a/backend/scheduler/helper.py +++ b/backend/scheduler/helper.py @@ -9,6 +9,7 @@ from scheduler.constants import SchedulerConstants as SC from scheduler.exceptions import JobDeletionError, JobSchedulingError +from scheduler.ownership import reconcile_ownership_for from scheduler.serializer import AddJobSerializer from scheduler.tasks import ( create_or_update_periodic_task, @@ -74,6 +75,13 @@ def _schedule_task_job(pipeline: Pipeline, job_data: Any) -> None: cron_string=cron_string, enabled=pipeline.active, ) + # Ramp control (Phase 9, ②c): align this schedule's firer (Beat vs PG) + # with the rollout. Inert until the gate + pg_scheduler_enabled flag are + # on (fails closed to Beat), so this is a no-op during normal operation; + # when a schedule is owned by PG it disables the Beat PeriodicTask in the + # same transaction so the two never both fire. Best-effort (never raises). + # organization_id is the org identifier string (what the mirror stores). + reconcile_ownership_for(str(pipeline.pk), organization_id, active=pipeline.active) @staticmethod def add_or_update_job(pipeline: Pipeline) -> None: diff --git a/backend/scheduler/ownership.py b/backend/scheduler/ownership.py new file mode 100644 index 0000000000..8331cf1c2a --- /dev/null +++ b/backend/scheduler/ownership.py @@ -0,0 +1,145 @@ +"""Schedule-ownership ramp control (Phase 9, ②c) — hands a pipeline's schedule +from Celery Beat to the Postgres scheduler, per-schedule and reversibly. + +A schedule is owned by exactly one firer. ``reconcile_ownership_for`` +applies that decision atomically: + + pg_periodic_schedule.pg_owned = resolve_schedule_owner(...) # PG fires it + PeriodicTask.enabled = active AND NOT pg_owned # Beat fires it + +Doing both in one transaction is what makes "never double-fires" real (it was +*conditional* on this slice): a ``pg_owned`` row always has its Beat +``PeriodicTask`` disabled, so the two can't both fire. + +Inert by default: ``resolve_schedule_owner`` fails closed to Beat +(``pg_owned=False``) until ops turns the master gate on AND ramps the +``pg_scheduler_enabled`` Flipt flag — so reconciling on every schedule edit is a +no-op (everything stays Beat-owned) until the rollout starts. +""" + +from __future__ import annotations + +import logging +import os + +from django.conf import settings +from django.db import transaction +from django.utils import timezone +from django_celery_beat.models import PeriodicTask +from pg_queue.models import PgPeriodicSchedule + +from unstract.flags.feature_flag import check_feature_flag_status + +logger = logging.getLogger(__name__) + +# Independent of the execution-transport flag (pg_queue_execution_enabled) so +# scheduling and execution ramp separately. %-rollout keyed on pipeline_id. +SCHEDULER_FLAG_KEY = "pg_scheduler_enabled" + + +def resolve_schedule_owner(pipeline_id: str, organization_id: str | None) -> bool: + """True → the PG scheduler owns this schedule; False → Celery Beat does. + + Mirrors ``resolve_transport``: master-gated by ``PG_QUEUE_TRANSPORT_ENABLED`` + (shared PG kill-switch), then the ``pg_scheduler_enabled`` Flipt flag, keyed + on ``pipeline_id`` for a stable percentage bucket. **Fails closed to Beat** + on a closed gate, a blind Flipt, or any error — so a schedule never silently + loses its firer. + """ + # Master gate off → never consult Flipt; every schedule stays on Beat. + if not settings.PG_QUEUE_TRANSPORT_ENABLED: + return False + + if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true": + logger.warning( + "resolve_schedule_owner: gate ON but FLIPT_SERVICE_AVAILABLE != true " + "(Flipt blind) for pipeline %s; leaving on Beat", + pipeline_id, + ) + return False + + # Flipt context is a gRPC map; coerce values to str (a non-str + # makes the client swallow it as False). entity_id str-coerced too so the + # %-rollout bucket is stable across str/UUID call sites. + context = {"pipeline_id": str(pipeline_id)} + if organization_id: + context["organization_id"] = str(organization_id) + try: + owned = check_feature_flag_status( + flag_key=SCHEDULER_FLAG_KEY, entity_id=str(pipeline_id), context=context + ) + except Exception: + # Expected, recoverable (fail-closed to Beat) and runs on every schedule + # edit — warn with traceback rather than logger.exception so a persistently + # down Flipt doesn't bury real errors as a per-edit Sentry exception. + logger.warning( + "resolve_schedule_owner: Flipt check failed for pipeline %s; " + "leaving on Beat", + pipeline_id, + exc_info=True, + ) + return False + return bool(owned) + + +def reconcile_ownership_for( + pipeline_id: str, organization_id: str | None, *, active: bool +) -> bool | None: + """Align one schedule's firer (Beat vs PG) with the current rollout decision. + + In one transaction: set ``pg_owned`` from :func:`resolve_schedule_owner` and + set the Beat ``PeriodicTask.enabled = active AND NOT pg_owned`` — so a + schedule handed to PG has Beat disabled (and vice-versa on rollback), with no + window where both fire. On rollback (``pg_owned`` → False) ``next_run_at`` is + also cleared so a later re-hand-over re-enters the PG tick's NULL baseline + (no burst). ``organization_id`` is the org *identifier* string (what the + mirror stores), used for Flipt per-org segmenting. ``active`` is keyword-only + (a fire/don't-fire boolean trap otherwise). + + Best-effort: a DB failure is logged and swallowed so it can never break the + caller. Returns the resolved ``pg_owned`` on success, or **None** if the + transaction failed (so the ramp command can tally + surface failures). + """ + pg_owned = resolve_schedule_owner(pipeline_id, organization_id) + try: + with transaction.atomic(): + # queryset .update() doesn't fire auto_now, so bump updated_at + # explicitly (mirrors _mirror_periodic_schedule_set_enabled). + updates: dict = {"pg_owned": pg_owned, "updated_at": timezone.now()} + if not pg_owned: + # Back on Beat → clear the PG next-run so a future re-hand-over + # baselines instead of firing immediately on a stale timestamp. + updates["next_run_at"] = None + # The mirror row exists from the dual-write (②a) / backfill; guard + # anyway — a missing row means nothing to own yet. + updated = PgPeriodicSchedule.objects.filter(pipeline_id=pipeline_id).update( + **updates + ) + if updated == 0: + logger.info( + "reconcile_ownership_for: no mirror row for pipeline %s " + "(not yet mirrored); skipping", + pipeline_id, + ) + # Without a mirror row PG can't fire (nothing to tick) and the Beat + # PeriodicTask was never disabled → the effective owner is Beat. + # Return False so the ramp count isn't inflated past what's live. + return False + # Beat owns it only when active AND not handed to PG. + PeriodicTask.objects.filter(name=pipeline_id).update( + enabled=active and not pg_owned + ) + logger.info( + "reconcile_ownership_for: pipeline %s pg_owned=%s (beat_enabled=%s)", + pipeline_id, + pg_owned, + active and not pg_owned, + ) + return pg_owned + except Exception: + logger.exception( + "reconcile_ownership_for failed for pipeline %s — schedule stays on " + "its current firer until the next reconcile", + pipeline_id, + ) + return None diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.py index b07da1aa12..86cf8a7a16 100644 --- a/backend/scheduler/tasks.py +++ b/backend/scheduler/tasks.py @@ -4,6 +4,7 @@ from typing import Any from celery import shared_task +from django.db import transaction from django.utils import timezone from django_celery_beat.models import CrontabSchedule, PeriodicTask from pg_queue.models import PgPeriodicSchedule @@ -249,8 +250,22 @@ def disable_task(task_name: str) -> None: def enable_task(task_name: str) -> None: - task = PeriodicTask.objects.get(name=task_name) - task.enabled = True - task.save() + PeriodicTask.objects.get(name=task_name) # preserve DoesNotExist on a bad name + # Resume → the schedule is active again, but Beat must fire it ONLY when it's + # not handed to PG — else a pg_owned schedule would fire from both Beat and PG + # (the ②c ownership invariant; reconcile sets the same on create/update, but + # resume takes this path). Lock the mirror row + write only the `enabled` + # column (not a full task.save() of stale state) so a concurrent + # reconcile_ownership_for can't be clobbered into a double-fire. + with transaction.atomic(): + pg_owned = ( + PgPeriodicSchedule.objects.select_for_update() + .filter(pipeline_id=task_name) + .values_list("pg_owned", flat=True) + .first() + or False + ) + PeriodicTask.objects.filter(name=task_name).update(enabled=not pg_owned) + # mirror.enabled tracks pipeline.active (True on resume) regardless of owner. _mirror_periodic_schedule_set_enabled(task_name, True) PipelineProcessor.update_pipeline(task_name, Pipeline.PipelineStatus.RESTARTING, True) diff --git a/backend/scheduler/tests/test_pg_periodic_schedule_mirror.py b/backend/scheduler/tests/test_pg_periodic_schedule_mirror.py index e5914db655..ea56ccea5b 100644 --- a/backend/scheduler/tests/test_pg_periodic_schedule_mirror.py +++ b/backend/scheduler/tests/test_pg_periodic_schedule_mirror.py @@ -8,6 +8,7 @@ path — without a test database. """ +from contextlib import nullcontext from unittest.mock import MagicMock, patch from scheduler import tasks @@ -93,6 +94,7 @@ def test_schedule_task_job_passes_real_pipeline_name(self): ), patch("scheduler.helper.create_or_update_periodic_task"), patch("scheduler.helper.mirror_periodic_schedule_upsert") as mirror, + patch("scheduler.helper.reconcile_ownership_for") as reconcile, ): SchedulerHelper._schedule_task_job( pipeline, @@ -109,6 +111,9 @@ def test_schedule_task_job_passes_real_pipeline_name(self): assert kwargs["pipeline_id"] == _PIPELINE_ID assert kwargs["organization_id"] == _ORG assert kwargs["enabled"] is True + # ②c: ownership is reconciled after the mirror upsert (org-identifier + + # active passed through; active is keyword-only). + reconcile.assert_called_once_with(_PIPELINE_ID, _ORG, active=True) class TestEnableDisableMirror: @@ -125,17 +130,42 @@ def test_disable_mirrors_enabled_false(self): sched.objects.filter.assert_called_with(pipeline_id=_PIPELINE_ID) assert sched.objects.filter.return_value.update.call_args.kwargs["enabled"] is False - def test_enable_mirrors_enabled_true(self): + @staticmethod + def _set_pg_owned(sched, value): + # enable_task reads pg_owned via select_for_update().filter().values_list().first() + ( + sched.objects.select_for_update.return_value.filter.return_value.values_list.return_value.first.return_value + ) = value + + def test_resume_enables_beat_when_not_pg_owned(self): with ( patch("scheduler.tasks.PeriodicTask") as pt, patch("scheduler.tasks.PipelineProcessor"), patch("scheduler.tasks.PgPeriodicSchedule") as sched, + patch("scheduler.tasks.transaction.atomic", return_value=nullcontext()), ): pt.objects.get.return_value = MagicMock() - sched.objects.filter.return_value.update.return_value = 1 + self._set_pg_owned(sched, False) + tasks.enable_task(_PIPELINE_ID) + + # Beat enabled via a column-only .update() (no full task.save() to clobber) + assert pt.objects.filter.return_value.update.call_args.kwargs["enabled"] is True + + def test_resume_keeps_beat_disabled_when_pg_owned(self): + """The High bug: resuming a PG-owned schedule must NOT re-enable Beat + (both firing = double-fire).""" + with ( + patch("scheduler.tasks.PeriodicTask") as pt, + patch("scheduler.tasks.PipelineProcessor"), + patch("scheduler.tasks.PgPeriodicSchedule") as sched, + patch("scheduler.tasks.transaction.atomic", return_value=nullcontext()), + ): + pt.objects.get.return_value = MagicMock() + self._set_pg_owned(sched, True) tasks.enable_task(_PIPELINE_ID) - assert sched.objects.filter.return_value.update.call_args.kwargs["enabled"] is True + # PG owns it → Beat stays off. + assert pt.objects.filter.return_value.update.call_args.kwargs["enabled"] is False def test_disable_mirror_failure_does_not_break_beat_path(self): """A mirror failure must be swallowed AND the pipeline-status update must diff --git a/backend/scheduler/tests/test_pg_schedule_ownership.py b/backend/scheduler/tests/test_pg_schedule_ownership.py new file mode 100644 index 0000000000..ed83030a81 --- /dev/null +++ b/backend/scheduler/tests/test_pg_schedule_ownership.py @@ -0,0 +1,216 @@ +"""Unit tests for the schedule-ownership ramp control (Phase 9, ②c). + +DB-free: Flipt, settings, and the ORM (``PgPeriodicSchedule`` / ``PeriodicTask``) +are mocked. These pin the fail-closed rollout decision and — the load-bearing +property — that handing a schedule to PG disables its Beat ``PeriodicTask`` in +the same step (no double-fire), with pause state preserved. +""" + +import contextlib +from unittest.mock import MagicMock, patch + +import pytest + +from scheduler import ownership + +_PID = "11111111-1111-1111-1111-111111111111" +_ORG = "org_abc" + + +def _gate(on: bool): + s = MagicMock() + s.PG_QUEUE_TRANSPORT_ENABLED = on + return patch("scheduler.ownership.settings", s) + + +class TestResolveScheduleOwner: + def test_gate_off_is_beat(self, monkeypatch): + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with _gate(False), patch( + "scheduler.ownership.check_feature_flag_status" + ) as flag: + assert ownership.resolve_schedule_owner(_PID, _ORG) is False + flag.assert_not_called() # gate off → Flipt never consulted + + def test_flipt_unavailable_is_beat(self, monkeypatch): + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false") + with _gate(True), patch( + "scheduler.ownership.check_feature_flag_status" + ) as flag: + assert ownership.resolve_schedule_owner(_PID, _ORG) is False + flag.assert_not_called() + + def test_flag_true_is_pg(self, monkeypatch): + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with _gate(True), patch( + "scheduler.ownership.check_feature_flag_status", return_value=True + ) as flag: + assert ownership.resolve_schedule_owner(_PID, _ORG) is True + # entity_id = pipeline_id (stable %-bucket); org in context. + assert flag.call_args.kwargs["entity_id"] == _PID + assert flag.call_args.kwargs["context"]["pipeline_id"] == _PID + assert flag.call_args.kwargs["context"]["organization_id"] == _ORG + + def test_flag_false_is_beat(self, monkeypatch): + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with _gate(True), patch( + "scheduler.ownership.check_feature_flag_status", return_value=False + ): + assert ownership.resolve_schedule_owner(_PID, _ORG) is False + + def test_flipt_error_fails_closed_to_beat(self, monkeypatch): + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with _gate(True), patch( + "scheduler.ownership.check_feature_flag_status", + side_effect=RuntimeError("flipt down"), + ): + assert ownership.resolve_schedule_owner(_PID, _ORG) is False + + +class TestReconcileOwnership: + def _patches(self, *, owner: bool, rows_matched: int = 1): + sched = patch("scheduler.ownership.PgPeriodicSchedule") + pt = patch("scheduler.ownership.PeriodicTask") + resolve = patch( + "scheduler.ownership.resolve_schedule_owner", return_value=owner + ) + # transaction.atomic() as a no-op context manager. + txn = patch( + "scheduler.ownership.transaction.atomic", + return_value=contextlib.nullcontext(), + ) + return sched, pt, resolve, txn + + def test_pg_owned_disables_beat_periodictask(self): + sched, pt, resolve, txn = self._patches(owner=True) + with sched as Sched, pt as PT, resolve, txn: + Sched.objects.filter.return_value.update.return_value = 1 + result = ownership.reconcile_ownership_for(_PID, _ORG, active=True) + + assert result is True + # mirror pg_owned set True + assert ( + Sched.objects.filter.return_value.update.call_args.kwargs["pg_owned"] + is True + ) + # Beat PeriodicTask disabled (active AND NOT pg_owned == False) + PT.objects.filter.assert_called_once_with(name=_PID) + assert ( + PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is False + ) + + def test_not_pg_owned_enables_beat_and_clears_next_run(self): + sched, pt, resolve, txn = self._patches(owner=False) + with sched as Sched, pt as PT, resolve, txn: + Sched.objects.filter.return_value.update.return_value = 1 + ownership.reconcile_ownership_for(_PID, _ORG, active=True) + + update_kwargs = Sched.objects.filter.return_value.update.call_args.kwargs + assert update_kwargs["pg_owned"] is False + # Rollback to Beat clears next_run_at so a re-hand-over re-baselines. + assert update_kwargs["next_run_at"] is None + assert ( + PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is True + ) + + def test_pg_owned_does_not_clear_next_run(self): + sched, pt, resolve, txn = self._patches(owner=True) + with sched as Sched, pt, resolve, txn: + Sched.objects.filter.return_value.update.return_value = 1 + ownership.reconcile_ownership_for(_PID, _ORG, active=True) + + # An active PG-owned schedule must NOT have its next_run_at reset (that + # would re-baseline and skip a fire). + assert ( + "next_run_at" + not in Sched.objects.filter.return_value.update.call_args.kwargs + ) + + def test_paused_pipeline_keeps_beat_disabled_even_if_not_pg_owned(self): + sched, pt, resolve, txn = self._patches(owner=False) + with sched as Sched, pt as PT, resolve, txn: + Sched.objects.filter.return_value.update.return_value = 1 + ownership.reconcile_ownership_for(_PID, _ORG, active=False) + + # active=False → Beat stays disabled regardless of ownership. + assert ( + PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is False + ) + + def test_missing_mirror_row_skips_and_reports_beat(self): + sched, pt, resolve, txn = self._patches(owner=True) + with sched as Sched, pt as PT, resolve, txn: + Sched.objects.filter.return_value.update.return_value = 0 # no row + # No mirror row → PG can't fire → effective owner is Beat → returns + # False even though resolve said PG (so the ramp count isn't inflated). + assert ownership.reconcile_ownership_for(_PID, _ORG, active=True) is False + + PT.objects.filter.assert_not_called() # nothing to own yet + + def test_failure_returns_none_and_is_swallowed(self): + sched, pt, resolve, txn = self._patches(owner=True) + with sched as Sched, pt, resolve, txn: + Sched.objects.filter.return_value.update.side_effect = RuntimeError("db") + # Must not raise, and signals failure (None) so the ramp can tally it. + assert ownership.reconcile_ownership_for(_PID, _ORG, active=True) is None + + +class TestReconcileAtomicityRealDB: + """The load-bearing invariant: the pg_owned write and the PeriodicTask write + are ONE transaction — if the PeriodicTask update fails, pg_owned rolls back + (so a schedule can't end up pg_owned with Beat still enabled). Needs a real + DB (the mocked atomic() can't prove rollback); skips if unreachable.""" + + def test_periodictask_update_failure_rolls_back_pg_owned(self): + import uuid + + from django_celery_beat.models import CrontabSchedule + from django_celery_beat.models import PeriodicTask as RealPeriodicTask + from pg_queue.models import PgPeriodicSchedule + + try: + cron, _ = CrontabSchedule.objects.get_or_create( + minute="0", + hour="9", + day_of_week="*", + day_of_month="*", + month_of_year="*", + ) + except Exception as exc: # pragma: no cover - infra-dependent + pytest.skip(f"DB unavailable: {exc}") + + pid = str(uuid.uuid4()) + RealPeriodicTask.objects.create( + name=pid, + task="scheduler.tasks.execute_pipeline_task", + crontab=cron, + enabled=True, + args="[]", + ) + PgPeriodicSchedule.objects.create( + pipeline_id=pid, + organization_id="org_atomic", + cron_string="0 9 * * *", + enabled=True, + pg_owned=False, + ) + try: + # Force the second write (the Beat PeriodicTask update) to fail; the + # pg_owned write (real, before it in the same atomic) must roll back. + failing_pt = MagicMock() + failing_pt.objects.filter.return_value.update.side_effect = RuntimeError( + "beat update fail" + ) + with ( + patch("scheduler.ownership.resolve_schedule_owner", return_value=True), + patch("scheduler.ownership.PeriodicTask", failing_pt), + ): + result = ownership.reconcile_ownership_for( + pid, "org_atomic", active=True + ) + assert result is None # failure signalled + # The pg_owned=True write was rolled back with the failed PT update. + assert PgPeriodicSchedule.objects.get(pipeline_id=pid).pg_owned is False + finally: + RealPeriodicTask.objects.filter(name=pid).delete() + PgPeriodicSchedule.objects.filter(pipeline_id=pid).delete() From ca73756a6fd15a83f7214e8951816d11dd544dcd Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:47:47 +0530 Subject: [PATCH 27/44] =?UTF-8?q?UN-3602=20[FIX]=20PG=20Queue=20=E2=80=94?= =?UTF-8?q?=20restore=20API-deployment=20timeout=20sync-wait=20on=20the=20?= =?UTF-8?q?PG=20path=20(#2086)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3602 [FIX] PG Queue — restore API-deployment timeout sync-wait on the PG path On the PG transport, async_execute_bin dispatch returns the bigint pg_queue_message msg_id as the handle. Writing it into WorkflowExecution.task_id (a UUIDField) raised ValueError on save, which the broad post-dispatch handler swallowed and returned EXECUTING — silently skipping the synchronous timeout poll loop. Every PG-routed API deployment ignored `timeout` and returned immediately. - add nullable BigIntegerField queue_message_id to WorkflowExecution (migration 0020; metadata-only AddField, no table rewrite, safe on large tables) - store the PG msg_id in queue_message_id; task_id stays NULL on the PG path - new WorkflowHelper._record_dispatch_handle routes the handle by transport, called inside its own try/except so post-dispatch bookkeeping can never skip the timeout wait again (the structural root cause) - Celery path unchanged Co-Authored-By: Claude Opus 4.8 * UN-3602 [FIX] address review — defensive PG handle parse + regression test Addresses PR #2086 review (toolkit + greptile): - _record_dispatch_handle: @classmethod -> @staticmethod (never uses cls) - parse the PG msg_id defensively (try/except TypeError/ValueError) so a malformed/future handle format logs a specific cause instead of being absorbed by the caller's generic post-dispatch guard - update_execution_queue_message_id annotation int -> int | None (matches the None guard in the body; mirrors update_execution_task) - add the missing regression test (test_execute_workflow_async_wait.py): a raise during handle recording must NOT skip the timeout poll loop - add a non-numeric-PG-handle test; scope the routing-test docstring - comment precision: the ValueError was raised inside update_execution_task on save (UUID coercion), not at the call site Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .../workflow_manager/workflow_v2/execution.py | 26 ++++++ ...0020_workflowexecution_queue_message_id.py | 21 +++++ .../workflow_v2/models/execution.py | 6 ++ .../tests/test_execute_workflow_async_wait.py | 67 ++++++++++++++ .../tests/test_record_dispatch_handle.py | 92 +++++++++++++++++++ .../workflow_v2/workflow_helper.py | 87 ++++++++++++++---- 6 files changed, 282 insertions(+), 17 deletions(-) create mode 100644 backend/workflow_manager/workflow_v2/migrations/0020_workflowexecution_queue_message_id.py create mode 100644 backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.py create mode 100644 backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.py diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py index d06cb06738..83a1c24057 100644 --- a/backend/workflow_manager/workflow_v2/execution.py +++ b/backend/workflow_manager/workflow_v2/execution.py @@ -425,6 +425,32 @@ def update_execution_task(execution_id: str, task_id: str) -> None: except WorkflowExecution.DoesNotExist: logger.error(f"execution doesn't exist {execution_id}") + @staticmethod + def update_execution_queue_message_id( + execution_id: str, queue_message_id: int | None + ) -> None: + """Record the PG queue-row handle (``pg_queue_message.msg_id``) on the + execution. PG-only: ``task_id`` is a UUIDField that can't hold the bigint + msg_id, so the PG handle lives in its own ``queue_message_id`` column. + """ + try: + if queue_message_id is None: + logger.warning( + f"Skipped setting queue_message_id for execution {execution_id} " + "since it's None" + ) + return + + execution = WorkflowExecution.objects.get(pk=execution_id) + execution.queue_message_id = queue_message_id + execution.save(update_fields=["queue_message_id"]) + logger.info( + f"Successfully set queue_message_id '{queue_message_id}' for " + f"execution {execution_id}" + ) + except WorkflowExecution.DoesNotExist: + logger.error(f"execution doesn't exist {execution_id}") + @staticmethod def convert_tool_instance_model_to_data_class( tool_instance: ToolInstance, diff --git a/backend/workflow_manager/workflow_v2/migrations/0020_workflowexecution_queue_message_id.py b/backend/workflow_manager/workflow_v2/migrations/0020_workflowexecution_queue_message_id.py new file mode 100644 index 0000000000..4d1b8bf5b6 --- /dev/null +++ b/backend/workflow_manager/workflow_v2/migrations/0020_workflowexecution_queue_message_id.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.1 on 2026-06-19 05:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("workflow_v2", "0019_remove_filehistory_trigram_index"), + ] + + operations = [ + migrations.AddField( + model_name="workflowexecution", + name="queue_message_id", + field=models.BigIntegerField( + db_comment="pg_queue_message.msg_id for PG-transport executions (the queue-row handle; task_id stays NULL on the PG path)", + editable=False, + null=True, + ), + ), + ] diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.py index 262b191a2c..3d35d6f0b8 100644 --- a/backend/workflow_manager/workflow_v2/models/execution.py +++ b/backend/workflow_manager/workflow_v2/models/execution.py @@ -145,6 +145,12 @@ class Type(models.TextChoices): null=True, db_comment="task id of asynchronous execution", ) + queue_message_id = models.BigIntegerField( + editable=False, + null=True, + db_comment="pg_queue_message.msg_id for PG-transport executions " + "(the queue-row handle; task_id stays NULL on the PG path)", + ) workflow = models.ForeignKey( Workflow, on_delete=models.CASCADE, diff --git a/backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.py b/backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.py new file mode 100644 index 0000000000..74586ab533 --- /dev/null +++ b/backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.py @@ -0,0 +1,67 @@ +"""Regression test for UN-3602 — the defense-in-depth guarantee. + +The bug: post-dispatch bookkeeping (recording the dispatch handle) raised, and +because it lived inside the broad post-dispatch ``try/except``, the handler +returned ``EXECUTING`` immediately and SKIPPED the synchronous ``timeout`` poll +loop. The fix moves that bookkeeping into its own ``try/except`` so a failure +there can never abort the wait. + +This test pins that: when ``_record_dispatch_handle`` raises, +``execute_workflow_async`` must still enter the poll loop (i.e. call +``_get_execution_status``) instead of short-circuiting to an immediate +``EXECUTING``. Without the inner ``try/except``, this test fails. + +DB-free: the model, transport resolution, context, and ``time.sleep`` are mocked. +""" + +from unittest.mock import MagicMock, patch + +from workflow_manager.workflow_v2.enums import ExecutionStatus +from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper + +_MOD = "workflow_manager.workflow_v2.workflow_helper" + + +class TestTimeoutWaitSurvivesBookkeepingFailure: + def test_record_handle_raise_does_not_skip_timeout_poll(self): + exec_row = MagicMock(status=ExecutionStatus.EXECUTING.value) + + with ( + patch(f"{_MOD}.resolve_transport", return_value="pg_queue"), + patch(f"{_MOD}.UserContext") as user_ctx, + patch(f"{_MOD}.StateStore") as state_store, + patch(f"{_MOD}.time"), # no-op sleep + patch(f"{_MOD}.WorkflowExecution") as wf_exec, + patch.object( + WorkflowHelper, "_dispatch_orchestrator_task", return_value="1" + ), + patch.object( + WorkflowHelper, + "_record_dispatch_handle", + side_effect=RuntimeError("bookkeeping boom"), + ) as record_handle, + patch.object( + WorkflowHelper, + "_get_execution_status", + return_value=ExecutionStatus.EXECUTING, + ) as get_status, + ): + user_ctx.get_organization_identifier.return_value = "org1" + state_store.get.return_value = None + wf_exec.objects.get.return_value = exec_row + + response = WorkflowHelper.execute_workflow_async( + workflow_id="wf-1", + execution_id="exec-1", + hash_values_of_files={}, + timeout=2, # > -1 so the poll loop is supposed to run + ) + + # The bookkeeping was attempted and raised — and was swallowed. + record_handle.assert_called_once() + # The load-bearing assertion: the poll loop STILL ran despite the raise. + # If the recording were back outside the inner try/except, the broad + # handler would have returned EXECUTING immediately and this would be 0. + assert get_status.called + # And the call returned normally (no propagated exception). + assert response is not None diff --git a/backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.py b/backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.py new file mode 100644 index 0000000000..e21c835dc4 --- /dev/null +++ b/backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.py @@ -0,0 +1,92 @@ +"""Tests for WorkflowHelper._record_dispatch_handle — the post-dispatch +bookkeeping that records the transport's dispatch handle on the execution row. + +Regression context: the PG path returns ``str(msg_id)`` (a bigint string). It +used to be written into ``WorkflowExecution.task_id`` (a UUIDField), raising +``ValueError`` on save → the broad post-dispatch handler swallowed it and +returned EXECUTING immediately, silently skipping the ``timeout`` sync-wait. The +fix routes the PG msg_id into its own ``queue_message_id`` (BigIntegerField) and +leaves ``task_id`` NULL on the PG path, so no bigint is ever forced into a UUID. + +DB-free: ``WorkflowExecutionServiceHelper`` is mocked. These assert ROUTING +only (PG -> ``queue_message_id`` as int, Celery -> ``task_id``, malformed/empty +handle -> neither); the UUID-coercion crash and the timeout sync-wait are NOT +reproduced here (the helper is mocked). The defense-in-depth guarantee — a raise +during handle recording must not skip the wait loop — is covered separately in +``test_execute_workflow_async_wait.py``. +""" + +from unittest.mock import patch + +from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper + +_HELPER = ( + "workflow_manager.workflow_v2.workflow_helper.WorkflowExecutionServiceHelper" +) +_EXEC = "exec-123" + + +class TestRecordDispatchHandle: + def test_pg_handle_goes_to_queue_message_id_as_int_not_task_id(self): + """The PG msg_id (bigint string) must be stored in queue_message_id as an + int — never in the UUID task_id (the bug that crashed the wait loop).""" + with patch(_HELPER) as helper: + WorkflowHelper._record_dispatch_handle( + execution_id=_EXEC, + transport="pg_queue", + dispatch_handle="1527", + org_schema="org1", + file_count=3, + ) + helper.update_execution_queue_message_id.assert_called_once_with( + execution_id=_EXEC, queue_message_id=1527 + ) + assert isinstance( + helper.update_execution_queue_message_id.call_args.kwargs[ + "queue_message_id" + ], + int, + ) + helper.update_execution_task.assert_not_called() # task_id stays NULL + + def test_celery_handle_goes_to_task_id_not_queue_message_id(self): + with patch(_HELPER) as helper: + WorkflowHelper._record_dispatch_handle( + execution_id=_EXEC, + transport="celery", + dispatch_handle="b1b2c3d4-0000-0000-0000-000000000000", + org_schema="org1", + file_count=1, + ) + helper.update_execution_task.assert_called_once_with( + execution_id=_EXEC, task_id="b1b2c3d4-0000-0000-0000-000000000000" + ) + helper.update_execution_queue_message_id.assert_not_called() + + def test_empty_handle_records_nothing(self): + with patch(_HELPER) as helper: + WorkflowHelper._record_dispatch_handle( + execution_id=_EXEC, + transport="celery", + dispatch_handle=None, + org_schema="org1", + file_count=0, + ) + helper.update_execution_task.assert_not_called() + helper.update_execution_queue_message_id.assert_not_called() + + def test_non_numeric_pg_handle_records_nothing(self): + """A malformed PG handle (not a bigint) must be parsed defensively — no + ValueError out of the helper, and nothing recorded. Today this can't + happen (``_dispatch_orchestrator_task`` always returns ``str(msg_id)``), + so this pins the contract against a future handle-format change.""" + with patch(_HELPER) as helper: + WorkflowHelper._record_dispatch_handle( + execution_id=_EXEC, + transport="pg_queue", + dispatch_handle="not-a-number", + org_schema="org1", + file_count=1, + ) + helper.update_execution_queue_message_id.assert_not_called() + helper.update_execution_task.assert_not_called() diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py index da868b76f2..0e2e28aacd 100644 --- a/backend/workflow_manager/workflow_v2/workflow_helper.py +++ b/backend/workflow_manager/workflow_v2/workflow_helper.py @@ -517,6 +517,56 @@ def _dispatch_orchestrator_task( ) return async_execution.id + @staticmethod + def _record_dispatch_handle( + *, + execution_id: str, + transport: str, + dispatch_handle: str | None, + org_schema: str, + file_count: int, + ) -> None: + """Persist the transport's dispatch handle on the execution row. + + Celery → the Celery task UUID into ``task_id`` (UUIDField). PG → the + ``pg_queue_message.msg_id`` into ``queue_message_id`` (BigIntegerField); + ``task_id`` stays NULL because there is no Celery task on the PG path. + Each id lives in its own correctly-typed column so a bigint msg_id is + never forced into the UUID ``task_id``. + """ + if not dispatch_handle: + # PG always yields a truthy msg_id, so an empty handle is Celery-only. + logger.warning( + f"[{org_schema}] Empty dispatch handle (transport={transport}) " + f"for execution_id '{execution_id}'." + ) + return + if is_pg_transport(transport): + # The PG handle is the bigint msg_id as a string. Parse defensively + # so a malformed/future handle format surfaces its specific cause + # here instead of being absorbed by the caller's generic guard. + try: + msg_id = int(dispatch_handle) + except (TypeError, ValueError): + logger.error( + f"[{org_schema}] PG dispatch handle {dispatch_handle!r} is not " + f"a valid bigint msg_id for execution_id '{execution_id}'; " + "queue_message_id not recorded" + ) + return + WorkflowExecutionServiceHelper.update_execution_queue_message_id( + execution_id=execution_id, queue_message_id=msg_id + ) + else: + WorkflowExecutionServiceHelper.update_execution_task( + execution_id=execution_id, task_id=dispatch_handle + ) + logger.info( + f"[{org_schema}] Job '{dispatch_handle}' enqueued " + f"(transport={transport}) for execution_id '{execution_id}', " + f"'{file_count}' files" + ) + @classmethod def execute_workflow_async( cls, @@ -597,7 +647,7 @@ def execute_workflow_async( # Orchestrator transport (9e PR A / 2d): dispatch async_execute_bin on # the resolved transport (PG enqueue vs Celery). Extracted to a helper # so this method stays simple and the fork is unit-testable. - task_id = cls._dispatch_orchestrator_task( + dispatch_handle = cls._dispatch_orchestrator_task( transport=transport, queue=queue, args=dispatch_args, @@ -611,23 +661,26 @@ def execute_workflow_async( workflow_execution: WorkflowExecution = WorkflowExecution.objects.get( id=execution_id ) - if not task_id: - # PG always yields a truthy msg_id, so an empty id is Celery-only; - # keep the transport in the message so operators can tell. - logger.warning( - f"[{org_schema}] Empty task_id (transport={transport}) for " - f"execution_id '{execution_id}'." - ) - # Continue without setting task_id - execution can still complete - else: - # Use existing method to handle task_id setting with validation - WorkflowExecutionServiceHelper.update_execution_task( - execution_id=execution_id, task_id=task_id + # Record the dispatch handle (Celery task_id or PG msg_id) on the row. + # Best-effort, in its OWN try/except: it must NEVER abort the + # synchronous timeout wait below. (Writing a bigint msg_id into the + # UUID task_id used to raise ValueError inside update_execution_task + # (UUID coercion on save), which bubbled to the post-dispatch handler + # and silently skipped the wait — so every PG-routed API deployment + # ignored `timeout`.) + try: + cls._record_dispatch_handle( + execution_id=execution_id, + transport=transport, + dispatch_handle=dispatch_handle, + org_schema=org_schema or "", + file_count=len(hash_values_of_files), ) - logger.info( - f"[{org_schema}] Job '{task_id}' enqueued (transport={transport}) " - f"for execution_id '{execution_id}', " - f"'{len(hash_values_of_files)}' files" + except Exception: + logger.exception( + f"[{org_schema}] Failed to record dispatch handle " + f"(transport={transport}) for execution '{execution_id}'; " + "continuing — the orchestrator is already running" ) execution_status = workflow_execution.status From fd38637812144e74fc7d662a81bc23486ddb8dc9 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Fri, 19 Jun 2026 23:56:57 +0530 Subject: [PATCH 28/44] =?UTF-8?q?UN-3603=20[GATED-FEAT]=20PG=20Queue=20?= =?UTF-8?q?=E2=80=94=20blocking=20executor=20RPC=20on=20Postgres=20+=20uni?= =?UTF-8?q?fy=20gating=20to=20a=20single=20flag=20(#2094)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3603 [GATED-FEAT] PG Queue — blocking executor RPC on Postgres + unify gating to a single flag A parallel PG-native executor RPC for prompt-studio's blocking dispatch, gated by the single pg_queue_enabled flag. The SDK ExecutionDispatcher and the Celery executor worker are left completely untouched. - pg_task_result store (migration 0010) + PgResultBackend (store / poll-wait; idempotent ON CONFLICT; poll-based, PgBouncer-safe — no LISTEN/NOTIFY) - reply_key on the shared TaskPayload contract + backend producer - consumer result-write hook: request-reply messages store result/error + ack after one attempt; fire-and-forget path unchanged (guarded by reply_key) - backend executor_rpc: resolve_executor_transport gate + PgExecutionDispatcher (enqueue + poll, mirrors the SDK dispatch contract) + RoutingExecutionDispatcher; _get_dispatcher returns the routing dispatcher so all call sites stay unchanged - worker-pg-executor role (run-worker.sh + docker-compose; broker-free) - unify PG-queue gating to a SINGLE flag pg_queue_enabled: execution, scheduler, and executor all read one key (renamed from pg_queue_execution_enabled / pg_scheduler_enabled / pg_executor_enabled). One flip gates the whole feature; the PG_QUEUE_TRANSPORT_ENABLED env stays the master kill-switch. Gated off by default. Dev-tested live: on->PG / off->Celery cycle (COMPLETED both ways) for API + ETL; executor RPC round-trip through worker-pg-executor. Co-Authored-By: Claude Opus 4.8 * UN-3603 [GATED-FEAT] address SonarCloud — unused param, logging.exception, no-NULL error - PgExecutionDispatcher.dispatch: drop the unused `headers` param (PG carries fairness in the enqueue payload, not Celery headers; the routing dispatcher no longer forwards headers to the PG path) - enqueue failure handler: logger.error(exc_info=True) -> logger.exception - PgTaskResult.error: TextField(null=True) -> TextField(blank=True, default="") (no-NULL-text convention); store_result writes "" on completed; migration 0010 regenerated; result-backend test updated Co-Authored-By: Claude Opus 4.8 * UN-3603 [GATED-FEAT] address review — request-reply robustness, single-flag constant, tests, doc accuracy Toolkit review (UN-3603): - Critical: guard the success-path result store in the consumer — a store failure now logs + acks (avoids re-running the executor = LLM spend) instead of vt-redelivering; pre-execution drop branches (malformed/poison/unknown) and the task-raised branch store a definitive failure reply via a new guarded _fail_reply so the caller fails fast instead of blocking to its timeout - Med: shared PgTaskStatus StrEnum in unstract.core (writer + reader agree across the process boundary); guard ExecutionResult.from_dict so a malformed completed row can't break the never-raises contract; log timeout/failure branches - High: release the DB connection between dispatch polls (close_old_connections) + document dispatch must not run inside a transaction - Med: single source of truth for the flag key (pg_queue/flags.py PG_QUEUE_FLAG_KEY, imported by transport/ownership/executor_rpc; SCHEDULER_FLAG_KEY folded in) - Med: result-backend logs the swallowed rollback failure - doc/accuracy: soften the not-yet-wired retention-sweep claims; fix stale pg_executor_enabled -> pg_queue_enabled comments; tighten reply_key to NotRequired[str] - tests: PgExecutionDispatcher.dispatch (8 branches, DB-free) + consumer reply_key store/ack + drop-branch + store-failure-still-acks cases Co-Authored-By: Claude Opus 4.8 * UN-3603 [GATED-FEAT] address SonarCloud — logging.exception + drop comment-as-code - consumer.py: the two new request-reply error handlers use logger.exception() instead of logger.error(..., exc_info=True) - test_executor_rpc.py: reword the `# timeout=None` inline comment (Sonar read it as commented-out code) Co-Authored-By: Claude Opus 4.8 * UN-3603 [GATED-FEAT] address greptile — executor consumer VT/health tuning + guard env timeout parse greptile's load-bearing finding: worker-pg-executor inherited the consumer's 30s/60s vt / health-stale defaults, so an LLM task exceeding 30s would be re-claimed mid-run once the gate ramps — double execution + token double-spend, two runs racing the same reply_key. - docker-compose: set WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=3660 and HEALTH_STALE_SECONDS=3720 on worker-pg-executor (above the executor's hard EXECUTOR_TASK_TIME_LIMIT=3600), both overridable via env - executor_rpc.dispatch: guard the EXECUTOR_RESULT_TIMEOUT int parse so a misconfigured value can't raise out of dispatch() (never-raises) + test Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- backend/pg_queue/executor_rpc.py | 259 ++++++++++++++++++ backend/pg_queue/flags.py | 12 + .../pg_queue/migrations/0010_pgtaskresult.py | 33 +++ backend/pg_queue/models.py | 44 +++ backend/pg_queue/producer.py | 9 + backend/pg_queue/tests/test_executor_rpc.py | 231 ++++++++++++++++ .../prompt_studio_helper.py | 16 +- backend/scheduler/ownership.py | 15 +- .../workflow_manager/workflow_v2/transport.py | 8 +- docker/docker-compose.yaml | 44 +++ .../core/src/unstract/core/data_models.py | 23 +- workers/queue_backend/pg_queue/consumer.py | 92 ++++++- .../queue_backend/pg_queue/result_backend.py | 195 +++++++++++++ workers/run-worker.sh | 7 + workers/tests/test_pg_queue_consumer.py | 59 +++- workers/tests/test_pg_result_backend.py | 103 +++++++ 16 files changed, 1130 insertions(+), 20 deletions(-) create mode 100644 backend/pg_queue/executor_rpc.py create mode 100644 backend/pg_queue/flags.py create mode 100644 backend/pg_queue/migrations/0010_pgtaskresult.py create mode 100644 backend/pg_queue/tests/test_executor_rpc.py create mode 100644 workers/queue_backend/pg_queue/result_backend.py create mode 100644 workers/tests/test_pg_result_backend.py diff --git a/backend/pg_queue/executor_rpc.py b/backend/pg_queue/executor_rpc.py new file mode 100644 index 0000000000..35615671b6 --- /dev/null +++ b/backend/pg_queue/executor_rpc.py @@ -0,0 +1,259 @@ +"""Executor-RPC transport routing for the PG path (Phase 9). + +The executor "RPC" is a synchronous request-reply: a caller (prompt-studio here) +sends an ``ExecutionContext`` to the executor worker and blocks for the +``ExecutionResult``. The legacy transport is Celery — the SDK +``ExecutionDispatcher`` (``send_task`` + ``AsyncResult.get``). This module adds a +**parallel** Postgres transport that leaves Celery and the SDK completely +untouched (no SDK edit, no change to the ``execute_extraction`` task or the +Celery executor worker): + +- :class:`PgExecutionDispatcher` enqueues ``execute_extraction`` onto the PG queue + with a unique ``reply_key`` and polls ``pg_task_result`` for the reply — same + ``.dispatch()`` contract as the SDK dispatcher (never raises; failure/timeout → + ``ExecutionResult.failure``). +- :func:`resolve_executor_transport` is the gate: master + ``PG_QUEUE_TRANSPORT_ENABLED`` then the **single** Flipt flag + ``pg_queue_enabled`` — the same flag the execution path uses, so one + flip turns the whole PG-queue feature on/off (no per-subsystem flags to + maintain). Fails closed to Celery. +- :class:`RoutingExecutionDispatcher` is what callers get from + ``PromptStudioHelper._get_dispatcher()``: ``dispatch()`` picks PG-vs-Celery + **per call** (read at dispatch time → flipping the flag is an instant, + no-redeploy rollout/rollback); ``dispatch_async`` / ``dispatch_with_callback`` + always delegate to Celery (the callback path is a later slice). + +Zero-regression: gate off ⇒ every method delegates to the unchanged Celery +``ExecutionDispatcher`` and no ``pg_task_result`` row is created. +""" + +from __future__ import annotations + +import logging +import os +import time +import uuid +from typing import TYPE_CHECKING, Any + +from django.conf import settings +from django.db import close_old_connections + +from pg_queue.flags import PG_QUEUE_FLAG_KEY +from pg_queue.models import PgTaskResult +from pg_queue.producer import enqueue_task +from unstract.core.data_models import PgTaskStatus +from unstract.flags.feature_flag import check_feature_flag_status +from unstract.sdk1.execution.dispatcher import ExecutionDispatcher +from unstract.sdk1.execution.result import ExecutionResult + +if TYPE_CHECKING: + from unstract.sdk1.execution.context import ExecutionContext + +logger = logging.getLogger(__name__) + +# Gating reads the single shared PG-queue flag (pg_queue.flags.PG_QUEUE_FLAG_KEY, +# imported above) — the same key execution and the scheduler use. +_EXECUTE_TASK = "execute_extraction" +# Mirror the SDK's queue-per-executor convention so the PG executor queue name +# matches the Celery one (the queue routes by the row's queue_name column). +_QUEUE_PREFIX = "celery_executor_" +# Caller-side wait default — mirrors the SDK dispatcher (EXECUTOR_RESULT_TIMEOUT +# env, else 3600s) so a PG-routed caller waits exactly as long as a Celery one. +_DEFAULT_TIMEOUT_ENV = "EXECUTOR_RESULT_TIMEOUT" +_DEFAULT_TIMEOUT = 3600 +_POLL_INITIAL_SECONDS = 0.2 +_POLL_MAX_SECONDS = 2.0 + + +def resolve_executor_transport(context: ExecutionContext) -> bool: + """True → route this executor dispatch over PG; False → Celery (default). + + Mirrors ``resolve_transport``: master-gated by ``PG_QUEUE_TRANSPORT_ENABLED``, + then the **single** ``pg_queue_enabled`` Flipt flag (shared across + the whole PG-queue feature), bucketed per org. **Fails closed to Celery** on a + closed gate, a blind Flipt, or any error — so the executor never silently + loses its transport. + """ + if not settings.PG_QUEUE_TRANSPORT_ENABLED: + return False + if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true": + logger.warning( + "resolve_executor_transport: gate ON but FLIPT_SERVICE_AVAILABLE != " + "true (Flipt blind); using Celery" + ) + return False + org = getattr(context, "organization_id", None) + # %-bucket keyed on org (prompt-studio is org-scoped); fall back to run_id so + # a context without an org still resolves deterministically. + entity_id = str(org or getattr(context, "run_id", "") or "default") + flag_context = {"executor_name": str(context.executor_name)} + if org: + flag_context["organization_id"] = str(org) + try: + enabled = check_feature_flag_status( + flag_key=PG_QUEUE_FLAG_KEY, entity_id=entity_id, context=flag_context + ) + except Exception: + logger.warning( + "resolve_executor_transport: Flipt check failed; using Celery", + exc_info=True, + ) + return False + return bool(enabled) + + +class PgExecutionDispatcher: + """PG request-reply executor dispatch — drop-in for ``ExecutionDispatcher.dispatch``. + + Enqueues ``execute_extraction`` with a unique ``reply_key`` and blocks on + ``pg_task_result`` until the executor consumer records the result or the + timeout elapses. Honours the same contract as the SDK dispatcher: it never + raises and converts a timeout/failure into ``ExecutionResult.failure`` so + callers can branch on ``result.success`` identically on either transport. + """ + + def dispatch( + self, + context: ExecutionContext, + timeout: int | None = None, + ) -> ExecutionResult: + if timeout is None: + # Guard the env parse so a misconfigured EXECUTOR_RESULT_TIMEOUT can't + # raise out of dispatch() (the never-raises contract). + try: + timeout = int(os.environ.get(_DEFAULT_TIMEOUT_ENV, _DEFAULT_TIMEOUT)) + except (TypeError, ValueError): + timeout = _DEFAULT_TIMEOUT + reply_key = str(uuid.uuid4()) + queue = f"{_QUEUE_PREFIX}{context.executor_name}" + org = getattr(context, "organization_id", "") or "" + try: + enqueue_task( + task_name=_EXECUTE_TASK, + queue=queue, + args=[context.to_dict()], + org_id=str(org), + reply_key=reply_key, + ) + except Exception as exc: + logger.exception( + "PG executor dispatch: enqueue failed (executor=%s run_id=%s)", + context.executor_name, + context.run_id, + ) + return ExecutionResult.failure(error=f"{type(exc).__name__}: {exc}") + logger.info( + "PG executor dispatch: enqueued reply_key=%s queue=%s run_id=%s " + "timeout=%ss; waiting for result...", + reply_key, + queue, + context.run_id, + timeout, + ) + row = self._wait_for_result(reply_key, timeout) + if row is None: + logger.warning( + "PG executor dispatch: TIMEOUT after %ss (reply_key=%s run_id=%s) — " + "the executor task may still be running", + timeout, + reply_key, + context.run_id, + ) + return ExecutionResult.failure( + error=f"TimeoutError: executor reply not received within {timeout}s" + ) + if row.status == PgTaskStatus.COMPLETED.value and row.result is not None: + try: + return ExecutionResult.from_dict(row.result) + except Exception: + # Honour the never-raises contract: a malformed completed row + # becomes a failure result, not a 500 to the caller. + logger.exception( + "PG executor dispatch: malformed completed result " + "(reply_key=%s run_id=%s)", + reply_key, + context.run_id, + ) + return ExecutionResult.failure( + error=f"Malformed executor result for reply_key {reply_key}" + ) + logger.warning( + "PG executor dispatch: executor reported failure (reply_key=%s " + "run_id=%s): %s", + reply_key, + context.run_id, + row.error or "(no error)", + ) + return ExecutionResult.failure(error=row.error or "executor task failed") + + @staticmethod + def _wait_for_result(reply_key: str, timeout: float) -> PgTaskResult | None: + """Poll ``pg_task_result`` until the row appears or *timeout* elapses. + + Poll-based with capped backoff (PgBouncer-safe; no LISTEN/NOTIFY). The DB + connection is released between polls (``close_old_connections``) so a + long-running RPC does not pin a backend connection for its whole duration + and exhaust the pool. Each poll is its own autocommit query, so a row + committed by the executor consumer becomes visible — **dispatch must NOT + be called inside an open transaction** (``transaction.atomic`` / + ``ATOMIC_REQUESTS`` would pin one snapshot and never see the new row). + """ + deadline = time.monotonic() + timeout + delay = _POLL_INITIAL_SECONDS + while True: + row = PgTaskResult.objects.filter(pk=reply_key).first() + if row is not None: + return row + remaining = deadline - time.monotonic() + if remaining <= 0: + return None + # Don't hold the connection idle through the sleep. + close_old_connections() + time.sleep(min(delay, remaining)) + delay = min(delay * 2, _POLL_MAX_SECONDS) + + +class RoutingExecutionDispatcher: + """Gate-routed executor dispatcher returned by ``_get_dispatcher()``. + + ``dispatch()`` chooses PG vs Celery per call (instant rollout/rollback); + ``dispatch_async`` / ``dispatch_with_callback`` always delegate to Celery — + the async/callback path stays on Celery until a later continuation slice. + Duck-typed against the SDK ``ExecutionDispatcher`` so call sites are unchanged. + """ + + def __init__(self, celery_app: object | None = None) -> None: + self._celery = ExecutionDispatcher(celery_app=celery_app) + self._pg = PgExecutionDispatcher() + + def dispatch( + self, + context: ExecutionContext, + timeout: int | None = None, + headers: dict[str, Any] | None = None, + ) -> ExecutionResult: + if resolve_executor_transport(context): + logger.info( + "Executor RPC → PG transport (executor=%s run_id=%s)", + context.executor_name, + context.run_id, + ) + # PG carries fairness via the enqueue payload, not Celery headers, so + # the headers (fairness key) are intentionally not forwarded here. + return self._pg.dispatch(context, timeout=timeout) + return self._celery.dispatch(context, timeout=timeout, headers=headers) + + def dispatch_async( + self, context: ExecutionContext, headers: dict[str, Any] | None = None + ) -> str: + return self._celery.dispatch_async(context, headers=headers) + + def dispatch_with_callback(self, context: ExecutionContext, **kwargs: Any) -> Any: + return self._celery.dispatch_with_callback(context, **kwargs) + + +def get_executor_dispatcher( + celery_app: object | None = None, +) -> RoutingExecutionDispatcher: + """Factory: the gate-routed executor dispatcher (PG when enabled, else Celery).""" + return RoutingExecutionDispatcher(celery_app=celery_app) diff --git a/backend/pg_queue/flags.py b/backend/pg_queue/flags.py new file mode 100644 index 0000000000..f2d56d8b73 --- /dev/null +++ b/backend/pg_queue/flags.py @@ -0,0 +1,12 @@ +"""Single source of truth for the PG-queue rollout flag key. + +One Flipt flag gates the whole PG-queue feature — execution +(``workflow_v2/transport.py``), scheduler (``scheduler/ownership.py``), and +executor (``pg_queue/executor_rpc.py``) all read this one key. Kept in a neutral +leaf module so the three resolvers import a single constant instead of +duplicating the literal (a grep on ``PG_QUEUE_FLAG_KEY`` finds every use), making +"one flag" a structural guarantee. ``PG_QUEUE_TRANSPORT_ENABLED`` (env) remains +the master kill-switch consulted before this flag. +""" + +PG_QUEUE_FLAG_KEY = "pg_queue_enabled" diff --git a/backend/pg_queue/migrations/0010_pgtaskresult.py b/backend/pg_queue/migrations/0010_pgtaskresult.py new file mode 100644 index 0000000000..c66f066db6 --- /dev/null +++ b/backend/pg_queue/migrations/0010_pgtaskresult.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.1 on 2026-06-19 17:06 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ( + "pg_queue", + "0009_remove_pgperiodicschedule_pg_periodic_schedule_due_idx_and_more", + ), + ] + + operations = [ + migrations.CreateModel( + name="PgTaskResult", + fields=[ + ("task_id", models.TextField(primary_key=True, serialize=False)), + ("status", models.TextField()), + ("result", models.JSONField(blank=True, null=True)), + ("error", models.TextField(blank=True, default="")), + ("created_at", models.DateTimeField(default=django.utils.timezone.now)), + ("expires_at", models.DateTimeField(blank=True, null=True)), + ], + options={ + "db_table": "pg_task_result", + "indexes": [ + models.Index(fields=["expires_at"], name="pg_task_result_expires_idx") + ], + }, + ), + ] diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py index c079c9f498..e413bd3712 100644 --- a/backend/pg_queue/models.py +++ b/backend/pg_queue/models.py @@ -285,3 +285,47 @@ class Meta: name="pg_periodic_schedule_due_idx", ), ] + + +class PgTaskResult(models.Model): + """Request-reply result store for the executor RPC on PG (Phase 9). + + Replaces Celery's ``AsyncResult`` / result backend for the *blocking* + executor dispatch when it rides the PG transport. A row appears ONLY when the + executor task finishes: the PG executor consumer (``worker-pg-executor``) + writes the returned ``ExecutionResult`` (``status="completed"``) or the error + text if the task raised (``status="failed"``), keyed by the caller-chosen + ``task_id`` (the reply key). The blocking caller polls this table until the + row appears or its timeout elapses; a non-blocking poll just checks presence. + + The absence of a row means "not done yet" — there is deliberately no + ``pending`` state to maintain. A separate, droppable table that never touches + ``WorkflowExecution`` — same posture as ``PgBarrierState`` / ``PgBatchDedup``, + extension-free (UN-3533). ``expires_at`` is written for a **future** retention + sweep (``DELETE … WHERE expires_at <= now()``); that sweep is **not wired yet** + — until it lands the table grows with each RPC (acceptable while the feature is + gated off; tracked as follow-up before the gate ramps to 100%). + """ + + # Caller-chosen reply key (opaque text, mirrors a Celery task id's role): the + # caller waits on it, the consumer stores the result under it. + task_id = models.TextField(primary_key=True) + # "completed" = task returned (``result`` holds ExecutionResult.to_dict()); + # "failed" = task raised (``error`` holds the message). + status = models.TextField() + result = models.JSONField(null=True, blank=True) + # No-NULL text convention: "" on a completed row (no error), the message on a + # failed row — avoids a string column having two empty states (NULL vs ""). + error = models.TextField(blank=True, default="") + created_at = models.DateTimeField(default=timezone.now) + # Retention horizon for the future sweep (see class docstring). The writer + # always sets it; nullable only so the column itself imposes no NOT NULL. + expires_at = models.DateTimeField(null=True, blank=True) + + class Meta: + db_table = "pg_task_result" + indexes = [ + # Index for the future retention sweep (DELETE ... WHERE + # expires_at <= now()); no sweeper reads it yet (see class docstring). + models.Index(fields=["expires_at"], name="pg_task_result_expires_idx"), + ] diff --git a/backend/pg_queue/producer.py b/backend/pg_queue/producer.py index c4e9be5cf1..f258b8a89a 100644 --- a/backend/pg_queue/producer.py +++ b/backend/pg_queue/producer.py @@ -62,6 +62,7 @@ def enqueue_task( org_id: str = "", priority: int = DEFAULT_PRIORITY, fairness: FairnessPayload | None = None, + reply_key: str | None = None, ) -> int: """Enqueue a task onto the PG queue; returns the new ``msg_id``. @@ -69,6 +70,10 @@ def enqueue_task( consumer can decode and run it. A PG enqueue failure propagates — the caller decides; for the orchestrator there is no silent Celery fallback (that would hide the failure or risk a double-dispatch). + + ``reply_key`` marks a **request-reply** dispatch (the executor RPC on PG): + the executor consumer writes the task's result/error to ``pg_task_result`` + under it for the blocking caller to poll. Omitted = fire-and-forget. """ if not FAIRNESS_MIN_PRIORITY <= priority <= FAIRNESS_MAX_PRIORITY: raise ValueError( @@ -83,6 +88,10 @@ def enqueue_task( "queue": pg_queue, "fairness": fairness, } + # Only set for request-reply dispatches — keeps fire-and-forget rows + # byte-identical to before this field existed. + if reply_key is not None: + message["reply_key"] = reply_key # Mirror the worker _enqueue_pg path: log the failure with breadcrumbs before # it propagates, so a DB/constraint/serialization error isn't mislabeled by # the caller's broad handler. diff --git a/backend/pg_queue/tests/test_executor_rpc.py b/backend/pg_queue/tests/test_executor_rpc.py new file mode 100644 index 0000000000..b91f923377 --- /dev/null +++ b/backend/pg_queue/tests/test_executor_rpc.py @@ -0,0 +1,231 @@ +"""Tests for the executor-RPC transport routing (Phase 9). + +DB-free: settings / Flipt / the sub-dispatchers are mocked. Pins the gate's +fail-closed matrix and — the load-bearing zero-regression property — that with +the gate off ``RoutingExecutionDispatcher`` delegates EVERY mode to the unchanged +Celery ``ExecutionDispatcher`` and never touches the PG path. +""" + +from unittest.mock import MagicMock, patch + +from pg_queue.executor_rpc import ( + PgExecutionDispatcher, + RoutingExecutionDispatcher, + resolve_executor_transport, +) + +_MOD = "pg_queue.executor_rpc" + + +def _completed(result: dict) -> MagicMock: + return MagicMock(status="completed", result=result, error="") + + +def _ok_result() -> dict: + return {"success": True, "data": {"x": 1}, "metadata": {}, "error": None} + + +class TestPgExecutionDispatcherDispatch: + """The load-bearing contract: never raises; timeout/failure → failure result. + + DB-free — ``enqueue_task`` and ``_wait_for_result`` are mocked. + """ + + @staticmethod + def _ctx() -> MagicMock: + c = MagicMock() + c.executor_name = "legacy" + c.run_id = "r" + c.organization_id = "o" + c.to_dict.return_value = {"run_id": "r"} + return c + + def test_enqueue_failure_returns_failure_not_raise(self): + with patch(f"{_MOD}.enqueue_task", side_effect=RuntimeError("db down")): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is False + assert "RuntimeError" in res.error + + def test_timeout_returns_failure(self): + with ( + patch(f"{_MOD}.enqueue_task"), + patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=None), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=3) + assert res.success is False + assert "within 3s" in res.error + + def test_completed_row_returns_result(self): + with ( + patch(f"{_MOD}.enqueue_task"), + patch.object( + PgExecutionDispatcher, "_wait_for_result", + return_value=_completed(_ok_result()), + ), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is True + + def test_failed_row_returns_error(self): + row = MagicMock(status="failed", result=None, error="boom") + with ( + patch(f"{_MOD}.enqueue_task"), + patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is False + assert res.error == "boom" + + def test_failed_row_empty_error_falls_back(self): + row = MagicMock(status="failed", result=None, error="") + with ( + patch(f"{_MOD}.enqueue_task"), + patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is False + assert "executor task failed" in res.error + + def test_completed_but_result_none_is_failure(self): + row = MagicMock(status="completed", result=None, error="") + with ( + patch(f"{_MOD}.enqueue_task"), + patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is False + + def test_malformed_completed_row_is_failure_not_raise(self): + with ( + patch(f"{_MOD}.enqueue_task"), + patch.object( + PgExecutionDispatcher, "_wait_for_result", + return_value=_completed({"bad": "shape"}), + ), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is False + assert "Malformed" in res.error + + def test_timeout_none_reads_env_then_default(self, monkeypatch): + monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "42") + seen = {} + + def fake_wait(reply_key, timeout): + seen["timeout"] = timeout + return None + + with ( + patch(f"{_MOD}.enqueue_task"), + patch.object( + PgExecutionDispatcher, "_wait_for_result", side_effect=fake_wait + ), + ): + # No explicit timeout arg → falls back to the env/default. + PgExecutionDispatcher().dispatch(self._ctx()) + assert seen["timeout"] == 42 + + def test_timeout_none_bad_env_falls_back_to_default(self, monkeypatch): + monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "not-an-int") + seen = {} + + def fake_wait(reply_key, timeout): + seen["timeout"] = timeout + return None + + with ( + patch(f"{_MOD}.enqueue_task"), + patch.object( + PgExecutionDispatcher, "_wait_for_result", side_effect=fake_wait + ), + ): + PgExecutionDispatcher().dispatch(self._ctx()) # must not raise + assert seen["timeout"] == 3600 # _DEFAULT_TIMEOUT + + +def _ctx(org: str | None = "org1") -> MagicMock: + c = MagicMock() + c.executor_name = "legacy" + c.run_id = "run-1" + c.organization_id = org + return c + + +def _gate(on: bool): + s = MagicMock() + s.PG_QUEUE_TRANSPORT_ENABLED = on + return patch(f"{_MOD}.settings", s) + + +class TestResolveExecutorTransport: + def test_master_gate_off_is_celery(self, monkeypatch): + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with _gate(False), patch(f"{_MOD}.check_feature_flag_status") as flag: + assert resolve_executor_transport(_ctx()) is False + flag.assert_not_called() # gate off → Flipt never consulted + + def test_flipt_unavailable_is_celery(self, monkeypatch): + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false") + with _gate(True), patch(f"{_MOD}.check_feature_flag_status") as flag: + assert resolve_executor_transport(_ctx()) is False + flag.assert_not_called() + + def test_flag_true_is_pg_keyed_on_org(self, monkeypatch): + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with _gate(True), patch( + f"{_MOD}.check_feature_flag_status", return_value=True + ) as flag: + assert resolve_executor_transport(_ctx("orgX")) is True + assert flag.call_args.kwargs["entity_id"] == "orgX" + # The single shared PG-queue flag (not a per-subsystem flag). + assert flag.call_args.kwargs["flag_key"] == "pg_queue_enabled" + + def test_flag_false_is_celery(self, monkeypatch): + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with _gate(True), patch( + f"{_MOD}.check_feature_flag_status", return_value=False + ): + assert resolve_executor_transport(_ctx()) is False + + def test_flipt_error_fails_closed_to_celery(self, monkeypatch): + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with _gate(True), patch( + f"{_MOD}.check_feature_flag_status", side_effect=RuntimeError("down") + ): + assert resolve_executor_transport(_ctx()) is False + + +class TestRoutingZeroRegression: + @staticmethod + def _build(): + # Patch both sub-dispatchers at construction; the instances are captured + # in __init__ so they remain mocked after the context exits. + with ( + patch(f"{_MOD}.ExecutionDispatcher") as celery_cls, + patch(f"{_MOD}.PgExecutionDispatcher") as pg_cls, + ): + dispatcher = RoutingExecutionDispatcher(celery_app="app") + return dispatcher, celery_cls.return_value, pg_cls.return_value + + def test_gate_off_dispatch_uses_celery_only(self): + dispatcher, celery, pg = self._build() + with patch(f"{_MOD}.resolve_executor_transport", return_value=False): + dispatcher.dispatch(_ctx()) + celery.dispatch.assert_called_once() + pg.dispatch.assert_not_called() # the zero-regression guarantee + + def test_gate_on_dispatch_uses_pg(self): + dispatcher, celery, pg = self._build() + with patch(f"{_MOD}.resolve_executor_transport", return_value=True): + dispatcher.dispatch(_ctx()) + pg.dispatch.assert_called_once() + celery.dispatch.assert_not_called() + + def test_async_and_callback_always_celery(self): + """The callback/async path stays on Celery regardless of the gate (a later slice).""" + dispatcher, celery, pg = self._build() + dispatcher.dispatch_async(_ctx()) + dispatcher.dispatch_with_callback(_ctx(), on_success=None) + celery.dispatch_async.assert_called_once() + celery.dispatch_with_callback.assert_called_once() + pg.dispatch.assert_not_called() diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py index af6ce3a34b..17ae59ba5a 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py +++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py @@ -73,7 +73,6 @@ from unstract.sdk1.constants import LogLevel from unstract.sdk1.exceptions import IndexingError, SdkError from unstract.sdk1.execution.context import ExecutionContext -from unstract.sdk1.execution.dispatcher import ExecutionDispatcher from unstract.sdk1.file_storage.constants import StorageType from unstract.sdk1.file_storage.env_helper import EnvHelper from unstract.sdk1.utils.indexing import IndexingUtils @@ -288,9 +287,18 @@ def _publish_log( ) @staticmethod - def _get_dispatcher() -> ExecutionDispatcher: - """Get an ExecutionDispatcher for the executor worker.""" - return ExecutionDispatcher(celery_app=celery_app) + def _get_dispatcher(): + """Executor dispatcher for the executor worker. + + Gate-routed: when ``pg_queue_enabled`` is on the blocking + ``dispatch()`` rides the PG request-reply transport; otherwise — and for + all async/callback dispatches — it is the unchanged Celery + ``ExecutionDispatcher``. The decision is read per dispatch, so flipping + the flag is an instant, no-redeploy rollout/rollback. + """ + from pg_queue.executor_rpc import get_executor_dispatcher + + return get_executor_dispatcher(celery_app=celery_app) @staticmethod def _get_platform_api_key(org_id: str) -> str: diff --git a/backend/scheduler/ownership.py b/backend/scheduler/ownership.py index 8331cf1c2a..f5f3684419 100644 --- a/backend/scheduler/ownership.py +++ b/backend/scheduler/ownership.py @@ -12,8 +12,8 @@ ``PeriodicTask`` disabled, so the two can't both fire. Inert by default: ``resolve_schedule_owner`` fails closed to Beat -(``pg_owned=False``) until ops turns the master gate on AND ramps the -``pg_scheduler_enabled`` Flipt flag — so reconciling on every schedule edit is a +(``pg_owned=False``) until ops turns the master gate on AND ramps the single +``pg_queue_enabled`` Flipt flag — so reconciling on every schedule edit is a no-op (everything stays Beat-owned) until the rollout starts. """ @@ -26,22 +26,23 @@ from django.db import transaction from django.utils import timezone from django_celery_beat.models import PeriodicTask +from pg_queue.flags import PG_QUEUE_FLAG_KEY from pg_queue.models import PgPeriodicSchedule from unstract.flags.feature_flag import check_feature_flag_status logger = logging.getLogger(__name__) -# Independent of the execution-transport flag (pg_queue_execution_enabled) so -# scheduling and execution ramp separately. %-rollout keyed on pipeline_id. -SCHEDULER_FLAG_KEY = "pg_scheduler_enabled" +# Gating uses the single shared PG-queue flag (pg_queue.flags.PG_QUEUE_FLAG_KEY, +# imported above) — one flip gates execution + scheduler + executor. The scheduler +# buckets the %-rollout on pipeline_id (each subsystem keys on its own entity). def resolve_schedule_owner(pipeline_id: str, organization_id: str | None) -> bool: """True → the PG scheduler owns this schedule; False → Celery Beat does. Mirrors ``resolve_transport``: master-gated by ``PG_QUEUE_TRANSPORT_ENABLED`` - (shared PG kill-switch), then the ``pg_scheduler_enabled`` Flipt flag, keyed + (shared PG kill-switch), then the single ``pg_queue_enabled`` Flipt flag, keyed on ``pipeline_id`` for a stable percentage bucket. **Fails closed to Beat** on a closed gate, a blind Flipt, or any error — so a schedule never silently loses its firer. @@ -66,7 +67,7 @@ def resolve_schedule_owner(pipeline_id: str, organization_id: str | None) -> boo context["organization_id"] = str(organization_id) try: owned = check_feature_flag_status( - flag_key=SCHEDULER_FLAG_KEY, entity_id=str(pipeline_id), context=context + flag_key=PG_QUEUE_FLAG_KEY, entity_id=str(pipeline_id), context=context ) except Exception: # Expected, recoverable (fail-closed to Beat) and runs on every schedule diff --git a/backend/workflow_manager/workflow_v2/transport.py b/backend/workflow_manager/workflow_v2/transport.py index 9b0796fd06..337e3dfe1d 100644 --- a/backend/workflow_manager/workflow_v2/transport.py +++ b/backend/workflow_manager/workflow_v2/transport.py @@ -10,7 +10,7 @@ PR 3 (this change) replaces PR 1's hardwired Celery with a Flipt evaluation: - master-gate (env) → Flipt boolean (``pg_queue_execution_enabled``) → transport + master-gate (env) → Flipt boolean (``pg_queue_enabled``) → transport Routing onto PG needs **all three** of: the env master-gate on, Flipt reachable (``FLIPT_SERVICE_AVAILABLE=true``), and the flag enabled for this execution. @@ -36,6 +36,7 @@ from typing import TYPE_CHECKING from django.conf import settings +from pg_queue.flags import PG_QUEUE_FLAG_KEY from unstract.core.data_models import WorkflowTransport from unstract.flags.feature_flag import check_feature_flag_status @@ -45,8 +46,9 @@ logger = logging.getLogger(__name__) -# The fixed Flipt flag contract (9e-design §2): Boolean, default false. -PG_QUEUE_FLAG_KEY = "pg_queue_execution_enabled" +# The single PG-queue rollout flag (Boolean, default false) is defined once in +# pg_queue.flags and imported above; execution, scheduler, and executor all read +# that one key. Re-exported here for callers/tests that import it from this module. def resolve_transport( diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 70579c657d..f58f5a9761 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -709,6 +709,50 @@ services: profiles: - pg-queue + # Executor RPC over PG — runs execute_extraction as a request-reply: claims + # work from Postgres and writes the ExecutionResult to pg_task_result for the + # blocking caller. Same heavy executor runtime as worker-executor-v2 (tool + # execution → platform-service / adapters / file storage), but broker-free: it + # is the terminal of the RPC and dispatches nothing onward to Celery. Dark + # until the backend pg_queue_enabled gate is flipped. + worker-pg-executor: + image: unstract/worker-unified:${VERSION} + container_name: unstract-worker-pg-executor + restart: unless-stopped + command: ["pg-queue-consumer"] + ports: + - "8099:8090" + env_file: + - ../workers/.env + - ./essentials.env + depends_on: + - db + - redis + - platform-service + environment: + - ENVIRONMENT=development + - APPLICATION_NAME=unstract-worker-pg-executor + - WORKER_BARRIER_BACKEND=pg + - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=executor + - WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table + - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090 + # The executor runs LLM inference — far longer than the consumer's 30s/60s + # defaults. Set the visibility timeout AND the liveness stale-threshold + # ABOVE the executor's hard per-task limit (EXECUTOR_TASK_TIME_LIMIT=3600s), + # else a long task is re-claimed mid-run (double execution / token + # double-spend, two runs racing the same reply_key) or killed by the probe. + - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_EXECUTOR_VT_SECONDS:-3660} + - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_EXECUTOR_HEALTH_STALE_SECONDS:-3720} + - CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table} + labels: + - traefik.enable=false + volumes: + - ./workflow_data:/data + - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config + - prompt_studio_data:/app/prompt-studio-data + profiles: + - pg-queue + # Reaper / orchestrator — leader-elected loop. Run exactly ONE instance (it # elects a single leader via pg_orchestrator_lock; extra replicas idle as # standby). Besides barrier-orphan recovery it runs the PG scheduler tick diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py index 94e6148305..cfcbd027e1 100644 --- a/unstract/core/src/unstract/core/data_models.py +++ b/unstract/core/src/unstract/core/data_models.py @@ -10,7 +10,7 @@ from dataclasses import asdict, dataclass, field from datetime import UTC, datetime from enum import Enum, StrEnum -from typing import Any, Literal, TypedDict +from typing import Any, Literal, NotRequired, TypedDict logger = logging.getLogger(__name__) @@ -315,6 +315,14 @@ class TaskPayload(TypedDict): *column* of the row, not by this field. Producers record the logical queue here for debugging (the backend producer always sets it; the workers' ``to_payload`` may leave it ``None``). + + ``reply_key`` is set only for **request-reply** dispatches (the executor RPC + on PG): the caller generates a unique key, waits on it, and the + executor consumer writes the task's result/error to ``pg_task_result`` under + it. "request-reply" is exactly "key present" — it is ``NotRequired[str]`` (not + ``str | None``) so there is one meaning per representation (absent = fire-and- + forget; present = a real key). It is deliberately a dedicated key, not + ``request_id`` (a caller-supplied tracing id that may not be unique). """ task_name: str @@ -322,6 +330,19 @@ class TaskPayload(TypedDict): kwargs: dict[str, Any] queue: str | None fairness: FairnessPayload | None + reply_key: NotRequired[str] + + +class PgTaskStatus(str, Enum): + """Status vocabulary for a ``pg_task_result`` row — the executor-RPC + request-reply contract (Phase 9). Shared in ``unstract.core`` so the writer + (workers ``PgResultBackend``) and reader (backend executor-RPC dispatcher), + which live in separate trees with no shared import, agree on one source of + truth rather than matching bare literals by eye across the process boundary. + """ + + COMPLETED = "completed" # task returned; ``result`` holds ExecutionResult dict + FAILED = "failed" # task raised; ``error`` holds the message class FileListingResult: diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py index cdf7a20836..50af3efea7 100644 --- a/workers/queue_backend/pg_queue/consumer.py +++ b/workers/queue_backend/pg_queue/consumer.py @@ -32,6 +32,7 @@ from ..fairness import FAIRNESS_HEADER_NAME from .client import PgQueueClient from .liveness import LivenessServer as _BaseLivenessServer +from .result_backend import PgResultBackend if TYPE_CHECKING: from celery import Celery @@ -116,6 +117,10 @@ def __init__( self.backoff_max = backoff_max self.max_attempts = max_attempts self._running = False + # Request-reply (executor RPC) result store — lazily created the first + # time a message carries a ``reply_key``; fire-and-forget consumers + # (orchestrator/fileproc/callback/scheduler) never instantiate it. + self._result_backend: PgResultBackend | None = None # Heartbeat for the liveness probe: monotonic timestamp of the most # recent poll attempt. Seeded at construction so a just-started consumer # reads healthy. Updated at the TOP of poll_once, so a loop wedged on a @@ -153,6 +158,12 @@ def poll_once(self) -> int: def _handle(self, message: QueueMessage) -> None: payload = message.message task_name = payload.get("task_name") + # Request-reply (executor RPC): a unique key the dispatching caller is + # blocking on. Read up front so the drop branches below can store a + # definitive failure reply (the caller fails fast instead of blocking to + # its full timeout). Present → store the outcome + ack after one attempt; + # absent → fire-and-forget (the existing leaf/pipeline path). + reply_key = payload.get("reply_key") # Malformed / foreign payload: no task name → can't run; drop with a # log that points at the payload, not at task registration. @@ -163,6 +174,7 @@ def _handle(self, message: QueueMessage) -> None: message.msg_id, payload, ) + self._fail_reply(reply_key, "malformed message: missing task_name") self._client.delete(message.msg_id) return @@ -179,6 +191,9 @@ def _handle(self, message: QueueMessage) -> None: message.read_ct, payload, ) + self._fail_reply( + reply_key, f"task {task_name} exceeded max_attempts={self.max_attempts}" + ) self._client.delete(message.msg_id) return @@ -190,6 +205,7 @@ def _handle(self, message: QueueMessage) -> None: task_name, message.msg_id, ) + self._fail_reply(reply_key, f"unknown task {task_name}") self._client.delete(message.msg_id) return @@ -198,15 +214,33 @@ def _handle(self, message: QueueMessage) -> None: # header so a PG-routed run mirrors the Celery dispatch path. fairness = payload.get("fairness") headers = {FAIRNESS_HEADER_NAME: fairness} if fairness else None - task.apply( + eager = task.apply( args=payload.get("args") or [], kwargs=payload.get("kwargs") or {}, headers=headers, throw=True, ) - except Exception: - # Leave the row: its vt expires and it is redelivered (bounded by - # max_attempts above). + except Exception as exc: + if reply_key: + # Record the failure so the waiting caller gets a definitive + # result instead of blocking to its timeout, then ACK. We do NOT + # vt-redeliver: a redelivery would race a result the caller may + # already have consumed, and re-running the executor is costly + # (LLM spend). The executor task's own autoretry covers transient + # errors within this attempt; the caller re-dispatches with a + # fresh reply_key to retry the whole RPC. + self._fail_reply(reply_key, f"{type(exc).__name__}: {exc}") + self._client.delete(message.msg_id) # ack + logger.exception( + "PG-queue consumer: request-reply task %r (msg_id=%s) failed " + "— stored error + acked (reply_key=%s)", + task_name, + message.msg_id, + reply_key, + ) + return + # Fire-and-forget: leave the row — its vt expires and it is + # redelivered (bounded by max_attempts above). logger.exception( "PG-queue consumer: task %r (msg_id=%s, read_ct=%s) failed — " "leaving for vt-expiry redelivery", @@ -216,6 +250,24 @@ def _handle(self, message: QueueMessage) -> None: ) return + if reply_key: + # Persist the result for the waiting caller before ack. Guarded: a + # store failure must NOT leave the message for vt-redelivery — that + # re-runs the executor (real LLM spend) and blocks the caller to its + # full timeout. Log loudly and ack anyway; the caller degrades to a + # timeout (rare), but we never double-spend. + try: + self._store_reply(reply_key, result=eager.result) + except Exception: + logger.exception( + "PG-queue consumer: FAILED to store request-reply result " + "(task=%r msg_id=%s reply_key=%s) — acking anyway to avoid an " + "expensive re-run; caller will time out", + task_name, + message.msg_id, + reply_key, + ) + if not self._client.delete(message.msg_id): # ack logger.warning( "PG-queue consumer: ack found no row for task %r (msg_id=%s) — " @@ -224,6 +276,38 @@ def _handle(self, message: QueueMessage) -> None: message.msg_id, ) + def _store_reply( + self, reply_key: str, *, result: dict | None = None, error: str | None = None + ) -> None: + """Persist a request-reply task's outcome to ``pg_task_result``. + + Lazily opens the result-backend connection on first use (so only the + executor consumer pays for it). ``store_result`` is idempotent + (first-write-wins), so a redelivery before the original ack is harmless. + """ + if self._result_backend is None: + self._result_backend = PgResultBackend() + self._result_backend.store_result(reply_key, result=result, error=error) + + def _fail_reply(self, reply_key: str | None, error: str) -> None: + """Best-effort failure reply for a request-reply message that can't run + or whose run raised (drop / poison / unknown-task / exception). + + No-op without a ``reply_key``; never raises — a store failure here must + not wedge the drop/ack path (the message is acked regardless), it just + means the caller degrades to a timeout instead of a fast definitive error. + """ + if not reply_key: + return + try: + self._store_reply(reply_key, error=error) + except Exception: + logger.exception( + "PG-queue consumer: failed to store failure reply (reply_key=%s): %s", + reply_key, + error, + ) + def _registered_task_count(self) -> int: """Count application tasks (excluding Celery's built-ins).""" return sum(1 for name in self._app.tasks if not name.startswith("celery.")) diff --git a/workers/queue_backend/pg_queue/result_backend.py b/workers/queue_backend/pg_queue/result_backend.py new file mode 100644 index 0000000000..208d184913 --- /dev/null +++ b/workers/queue_backend/pg_queue/result_backend.py @@ -0,0 +1,195 @@ +"""Postgres request-reply result store for the executor RPC (Phase 9). + +Replaces Celery's ``AsyncResult`` / result backend for the *blocking* executor +dispatch when it rides the PG transport. The PG executor consumer +(``worker-pg-executor``) writes a finished task's outcome here, keyed by the +caller-chosen reply key; the blocking caller polls :meth:`wait_for_result` +until the row appears or its timeout elapses. + +A row appears ONLY when the task finishes — ``status="completed"`` carrying the +``ExecutionResult.to_dict()`` payload, or ``status="failed"`` carrying the error +text if the task raised. Absence of a row means "not done yet"; there is +deliberately no ``pending`` state to maintain. + +Two deliberate properties: + +- **Idempotent writes** (``INSERT … ON CONFLICT DO NOTHING``): the PG queue is + at-least-once, so a redelivered executor message must not clobber an already + recorded result. First write wins. +- **Poll-based waiting, NOT ``LISTEN``/``NOTIFY``**: ``NOTIFY`` does not survive + transaction-pooled PgBouncer (the same constraint that makes the orchestrator + lock a TTL lease rather than ``pg_advisory_lock``). Polling with backoff is + pooling-safe and needs no persistent listener connection. + +Connection discipline mirrors :class:`~queue_backend.pg_queue.client.PgQueueClient`: +an injected connection is the caller's (tests); otherwise one is created lazily +from the backend ``DB_*`` env and owned here (rolled back on error, discarded + +reconnected when it goes bad). +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import time +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any, Self + +import psycopg2 + +from unstract.core.data_models import PgTaskStatus + +from .connection import create_pg_connection + +if TYPE_CHECKING: + from psycopg2.extensions import connection as PgConnection + +logger = logging.getLogger(__name__) + +# How long a stored result lives before the reaper's retention sweep may delete +# it. Defaults to the executor caller-timeout default so a result always +# outlives any caller still waiting on it. +DEFAULT_RETENTION_SECONDS = 3600 + +# First write wins — an at-least-once redelivery of the executor message must +# not overwrite a recorded result. +_STORE_SQL = ( + "INSERT INTO pg_task_result " + "(task_id, status, result, error, created_at, expires_at) " + "VALUES (%s, %s, %s::jsonb, %s, now(), now() + make_interval(secs => %s)) " + "ON CONFLICT (task_id) DO NOTHING" +) +_GET_SQL = "SELECT status, result, error FROM pg_task_result WHERE task_id = %s" + +# Poll cadence for wait_for_result: start tight (low latency for fast tasks), +# back off to a ceiling so a long-running task doesn't hammer the DB. +_POLL_INITIAL_SECONDS = 0.2 +_POLL_MAX_SECONDS = 2.0 + +# Status vocabulary — sourced from the shared enum in unstract.core so the writer +# (here) and the backend reader agree across the process boundary. Re-exported as +# plain strings for the SQL/tests. +STATUS_COMPLETED = PgTaskStatus.COMPLETED.value +STATUS_FAILED = PgTaskStatus.FAILED.value + + +class PgResultBackend: + """``store_result`` / ``get_result`` / ``wait_for_result`` over ``pg_task_result``.""" + + def __init__(self, conn: PgConnection | None = None) -> None: + self._conn = conn + # Injected connections belong to the caller — never close/recycle them. + self._owns_conn = conn is None + + @property + def conn(self) -> PgConnection: + if self._conn is None: + self._conn = create_pg_connection() + return self._conn + + @contextlib.contextmanager + def _cursor(self) -> Iterator[Any]: + """Yield a cursor; commit on success, roll back + recover on error.""" + conn = self.conn + try: + with conn.cursor() as cur: + yield cur + conn.commit() + except Exception as exc: + conn_dead = isinstance( + exc, (psycopg2.OperationalError, psycopg2.InterfaceError) + ) + try: + conn.rollback() + except Exception: + # A failed rollback proves the connection is unusable regardless + # of the original error subclass — recycle it (and surface why, + # rather than swallowing it silently). + logger.warning( + "PgResultBackend: rollback failed; treating connection as dead", + exc_info=True, + ) + conn_dead = True + if self._owns_conn and (conn_dead or conn.closed): + with contextlib.suppress(Exception): + conn.close() + self._conn = None + raise + + def store_result( + self, + task_id: str, + *, + result: dict[str, Any] | None = None, + error: str | None = None, + retention_seconds: int = DEFAULT_RETENTION_SECONDS, + ) -> None: + """Record a finished task's outcome under *task_id* (the reply key). + + ``result`` (a dict, e.g. ``ExecutionResult.to_dict()``) → ``completed``. + Otherwise → ``failed`` with ``error`` text. Idempotent: a second write + for the same key (at-least-once redelivery) is a no-op. + """ + if result is not None: + status, result_json, error_text = STATUS_COMPLETED, json.dumps(result), "" + else: + status, result_json, error_text = STATUS_FAILED, None, error or "" + with self._cursor() as cur: + cur.execute( + _STORE_SQL, + (str(task_id), status, result_json, error_text, retention_seconds), + ) + + def get_result(self, task_id: str) -> dict[str, Any] | None: + """Return ``{status, result, error}`` if the row exists, else ``None``. + + ``result`` is the decoded JSONB dict (psycopg2 parses ``jsonb`` to a + Python ``dict``); ``None`` means the task has not finished yet. + """ + with self._cursor() as cur: + cur.execute(_GET_SQL, (str(task_id),)) + row = cur.fetchone() + if row is None: + return None + status, result, error = row + return {"status": status, "result": result, "error": error} + + def wait_for_result( + self, + task_id: str, + timeout: float, + *, + poll_interval: float = _POLL_INITIAL_SECONDS, + ) -> dict[str, Any] | None: + """Block until the result row appears or *timeout* seconds elapse. + + Returns the ``{status, result, error}`` dict, or ``None`` on timeout. + Poll-based with exponential backoff (capped) — PgBouncer-safe, no + persistent listener. The final sleep is clamped so we never overshoot + the deadline. + """ + deadline = time.monotonic() + timeout + delay = poll_interval + while True: + row = self.get_result(task_id) + if row is not None: + return row + remaining = deadline - time.monotonic() + if remaining <= 0: + return None + time.sleep(min(delay, remaining)) + delay = min(delay * 2, _POLL_MAX_SECONDS) + + def close(self) -> None: + """Close an owned connection (injected connections are the caller's).""" + if self._owns_conn and self._conn is not None and not self._conn.closed: + with contextlib.suppress(Exception): + self._conn.close() + self._conn = None + + def __enter__(self) -> Self: + return self + + def __exit__(self, *exc: object) -> None: + self.close() diff --git a/workers/run-worker.sh b/workers/run-worker.sh index 91b18c2305..0ec399deec 100755 --- a/workers/run-worker.sh +++ b/workers/run-worker.sh @@ -56,6 +56,7 @@ readonly PG_ROLE_ORCH_GENERAL="pg-orchestrator-general" readonly PG_ROLE_FILEPROC="pg-fileproc" readonly PG_ROLE_CALLBACK="pg-callback" readonly PG_ROLE_SCHEDULER="pg-scheduler" +readonly PG_ROLE_EXECUTOR="pg-executor" declare -rA PG_CONSUMER_ROLES=( ["$PG_ROLE_ORCH_API"]="api_deployment;celery_api_deployments" ["$PG_ROLE_ORCH_GENERAL"]="general;celery" @@ -65,6 +66,10 @@ declare -rA PG_CONSUMER_ROLES=( # scheduler tick (Beat replacement). Distinct from the Celery 'scheduler' # worker (which Beat fires onto RabbitMQ). ["$PG_ROLE_SCHEDULER"]="scheduler;scheduler" + # Runs execute_extraction (the executor RPC) over PG — request-reply: writes + # the result to pg_task_result for the blocking caller. Queues mirror the + # Celery executor's CELERY_QUEUES_EXECUTOR. + ["$PG_ROLE_EXECUTOR"]="executor;celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table" ) declare -rA PG_QUEUE_MEMBERS=( ["$PG_QUEUE_CONSUMER_TYPE"]=1 @@ -74,6 +79,7 @@ declare -rA PG_QUEUE_MEMBERS=( ["$PG_ROLE_FILEPROC"]=1 ["$PG_ROLE_CALLBACK"]=1 ["$PG_ROLE_SCHEDULER"]=1 + ["$PG_ROLE_EXECUTOR"]=1 ) # The Celery transport set: every worker EXCEPT the PG-queue members — the # *complement* of the 'pg-queue' set, so the two transports' logs can be tailed @@ -112,6 +118,7 @@ declare -A WORKERS=( ["$PG_ROLE_FILEPROC"]="$PG_ROLE_FILEPROC" ["$PG_ROLE_CALLBACK"]="$PG_ROLE_CALLBACK" ["$PG_ROLE_SCHEDULER"]="$PG_ROLE_SCHEDULER" + ["$PG_ROLE_EXECUTOR"]="$PG_ROLE_EXECUTOR" # PG Queue reaper — leader-elected recovery loop (barrier-orphan sweep) ["reaper"]="$PG_QUEUE_REAPER_TYPE" ["pg-queue-reaper"]="$PG_QUEUE_REAPER_TYPE" diff --git a/workers/tests/test_pg_queue_consumer.py b/workers/tests/test_pg_queue_consumer.py index c1c50a66e8..05ed6e9869 100644 --- a/workers/tests/test_pg_queue_consumer.py +++ b/workers/tests/test_pg_queue_consumer.py @@ -10,7 +10,7 @@ import logging import os -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest from celery import shared_task @@ -464,5 +464,62 @@ def test_parse_queue_list(): assert _parse_queue_list("a,,b") == ["a", "b"] # empties dropped +class TestRequestReply: + """Executor-RPC reply_key behaviour: store outcome + ack after one attempt; + drop branches store a definitive failure; a store failure still acks (no + expensive re-run). PgResultBackend is mocked so no DB is needed. + """ + + _RB = "queue_backend.pg_queue.consumer.PgResultBackend" + + def test_success_stores_result_and_acks(self): + client = MagicMock() + client.read.return_value = [_msg(1, {**_ok_payload(3, 4), "reply_key": "rk1"})] + with patch(self._RB) as rb_cls: + PgQueueConsumer(["q"], client=client).poll_once() + rb = rb_cls.return_value + rb.store_result.assert_called_once() + assert rb.store_result.call_args.args[0] == "rk1" + assert rb.store_result.call_args.kwargs["result"] == 7 # x + y + assert rb.store_result.call_args.kwargs["error"] is None + client.delete.assert_called_once_with(1) # acked + + def test_task_raise_stores_error_and_acks(self): + client = MagicMock() + client.read.return_value = [ + _msg(2, {"task_name": "test_pg_consumer.boom", "reply_key": "rk2"}) + ] + with patch(self._RB) as rb_cls: + PgQueueConsumer(["q"], client=client).poll_once() + rb = rb_cls.return_value + assert rb.store_result.call_args.kwargs["error"] is not None + assert rb.store_result.call_args.kwargs["result"] is None + client.delete.assert_called_once_with(2) # acked, NOT left for redelivery + + def test_unknown_task_stores_error_and_acks(self): + client = MagicMock() + client.read.return_value = [ + _msg(3, {"task_name": "nope.nope", "reply_key": "rk3"}) + ] + with patch(self._RB) as rb_cls: + PgQueueConsumer(["q"], client=client).poll_once() + rb = rb_cls.return_value + assert rb.store_result.call_args.kwargs["error"] is not None + client.delete.assert_called_once_with(3) + + def test_store_failure_on_success_path_still_acks(self, caplog): + client = MagicMock() + client.read.return_value = [_msg(4, {**_ok_payload(1, 1), "reply_key": "rk4"})] + with patch(self._RB) as rb_cls: + rb_cls.return_value.store_result.side_effect = RuntimeError("db down") + with caplog.at_level( + logging.ERROR, logger="queue_backend.pg_queue.consumer" + ): + PgQueueConsumer(["q"], client=client).poll_once() + # A store failure must NOT block the ack (avoids an expensive re-run). + client.delete.assert_called_once_with(4) + assert "FAILED to store request-reply result" in caplog.text + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/workers/tests/test_pg_result_backend.py b/workers/tests/test_pg_result_backend.py new file mode 100644 index 0000000000..a18ca012d0 --- /dev/null +++ b/workers/tests/test_pg_result_backend.py @@ -0,0 +1,103 @@ +"""Real-PG tests for PgResultBackend — the executor RPC result store. + +DB-gated via the ``pg_conn`` fixture (skips when Postgres is unreachable or the +``pg_queue`` migration isn't applied). Pins the request-reply contract: store/get +round-trip (completed + failed), idempotent first-write-wins, absent -> None, +wait returns when present, wait times out, and wait picks up a late write from +*another* connection (the cross-process request-reply path). +""" + +import os +import threading +import time +import uuid + +import pytest + +from queue_backend.pg_queue.connection import create_pg_connection +from queue_backend.pg_queue.result_backend import ( + STATUS_COMPLETED, + STATUS_FAILED, + PgResultBackend, +) + +_MARK = "pgtaskresult-test" + + +def _key() -> str: + return f"{_MARK}-{uuid.uuid4()}" + + +@pytest.fixture +def result_backend(pg_conn): + rb = PgResultBackend(conn=pg_conn) + yield rb + with pg_conn.cursor() as cur: + cur.execute("DELETE FROM pg_task_result WHERE task_id LIKE %s", (f"{_MARK}-%",)) + pg_conn.commit() + + +class TestStoreGet: + def test_store_completed_round_trips(self, result_backend): + k = _key() + result_backend.store_result(k, result={"success": True, "data": {"x": 1}}) + row = result_backend.get_result(k) + assert row["status"] == STATUS_COMPLETED + assert row["result"] == {"success": True, "data": {"x": 1}} + assert row["error"] == "" # no-NULL text convention + + def test_store_failed_round_trips(self, result_backend): + k = _key() + result_backend.store_result(k, error="boom") + row = result_backend.get_result(k) + assert row["status"] == STATUS_FAILED + assert row["error"] == "boom" + assert row["result"] is None + + def test_absent_returns_none(self, result_backend): + assert result_backend.get_result(_key()) is None + + def test_store_idempotent_first_write_wins(self, result_backend): + """At-least-once redelivery must not clobber a recorded result.""" + k = _key() + result_backend.store_result(k, result={"v": "first"}) + result_backend.store_result(k, error="second") # ON CONFLICT DO NOTHING + row = result_backend.get_result(k) + assert row["status"] == STATUS_COMPLETED + assert row["result"] == {"v": "first"} + + +class TestWait: + def test_wait_returns_immediately_when_present(self, result_backend): + k = _key() + result_backend.store_result(k, result={"ok": True}) + row = result_backend.wait_for_result(k, timeout=5) + assert row is not None + assert row["result"] == {"ok": True} + + def test_wait_times_out_returns_none(self, result_backend): + start = time.monotonic() + row = result_backend.wait_for_result(_key(), timeout=1, poll_interval=0.2) + assert row is None + assert time.monotonic() - start >= 0.9 # waited ~the full timeout + + def test_wait_picks_up_late_write_from_other_conn(self, result_backend): + """The real request-reply path: the waiter polls on its connection while + the result is committed from a separate connection mid-wait.""" + k = _key() + os.environ.setdefault("TEST_DB_HOST", "127.0.0.1") + writer = PgResultBackend(conn=create_pg_connection(env_prefix="TEST_DB_")) + + def write_after_delay() -> None: + time.sleep(0.6) + writer.store_result(k, result={"late": True}) + + t = threading.Thread(target=write_after_delay) + t.start() + try: + row = result_backend.wait_for_result(k, timeout=10, poll_interval=0.2) + finally: + t.join() + writer.close() + assert row is not None + assert row["result"] == {"late": True} From 5a7d3d1dbd7f1835ca1888e507d4d7eb3e3cda0f Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Sat, 20 Jun 2026 16:05:20 +0530 Subject: [PATCH 29/44] =?UTF-8?q?UN-3605=20[GATED-FEAT]=20PG=20Queue=20?= =?UTF-8?q?=E2=80=94=20structure=5Ftool=20executor=20RPC=20over=20Postgres?= =?UTF-8?q?=20(+=20register=20executors=20in=20PG=20consumer)=20(#2095)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3605 [GATED-FEAT] PG Queue — structure_tool executor RPC over Postgres (+ register executors in PG consumer) Routes the in-workflow structure_tool executor dispatch onto the Postgres executor RPC when the single pg_queue_enabled flag is on; Celery otherwise (zero-regression by construction). Next slice after UN-3603 (prompt-studio blocking path). Gated off by default. - workers queue_backend/pg_queue/executor_rpc.py (new): the workers twin of the backend module on worker primitives — resolve_executor_transport (master PG_QUEUE_TRANSPORT_ENABLED env, then the single pg_queue_enabled Flipt flag, fail-closed to Celery); PgExecutionDispatcher (enqueue execute_extraction with a unique reply_key via PgQueueClient, poll PgResultBackend; never raises — timeout/failure -> ExecutionResult.failure); RoutingExecutionDispatcher (per-call gate routing; async/callback stay on Celery); get_executor_dispatcher factory. - to_payload gains an optional reply_key (request-reply marker; fire-and-forget rows stay byte-identical). - structure_tool_task swaps its dispatcher factory to get_executor_dispatcher; the 3 blocking dispatch call sites are unchanged. - Fix (latent gap exposed by this slice): the PG executor consumer registered no executors ("No executor registered with name 'legacy'"). The @ExecutorRegistry.register side-effect import lived only in the Celery executor/worker.py, but the PG consumer bootstraps via executor/tasks.py. Import executor.executors from executor/tasks.py so any worker registering execute_extraction (Celery or PG) has the executors — also fixes the prompt-studio PG path from UN-3603. - sample.env: document the worker-side PG_QUEUE_TRANSPORT_ENABLED gate; fix a stale pg_queue_execution_enabled -> pg_queue_enabled doc reference. - Tests: workers executor_rpc unit suite (gate fail-closed matrix, zero-regression routing, never-raises dispatch); subprocess regression that importing executor.tasks alone registers the executors. Co-Authored-By: Claude Opus 4.8 * UN-3605 [GATED-FEAT] Address PR review (toolkit + greptile) Hardening + test-coverage from the review: - dispatch: row.get("status") (never-raises must not depend on producer keys); accept-and-ignore headers param for substitutability with the SDK/Routing dispatch shapes; log the EXECUTOR_RESULT_TIMEOUT parse fallback instead of swallowing it; surface the parse cause in the malformed-result error. - timeout branch: document the orphaned-task / retry-double-run risk (greptile); de-dup belongs at the file-execution layer (at-least-once + caller-timeout). - _wait_for_result: note the connection pin is bounded by file_processing prefork concurrency (vs the backend twin's close_old_connections per poll). - executor/tasks.py: reword the registration-import comment (it's executor/ worker.py, the Celery entrypoint, that the PG consumer bootstrap skips). - tests: org-less resolve bucketing (entity_id=run_id, no org in context); real _enqueue wiring (queue/org_id/payload via PgQueueClient.send); routing arg passthrough (gate-off forwards timeout+headers; gate-on drops headers); to_payload reply_key set/omitted; structure_tool factory-swap call site. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- backend/sample.env | 2 +- workers/executor/tasks.py | 10 + .../file_processing/structure_tool_task.py | 11 +- .../queue_backend/pg_queue/executor_rpc.py | 316 ++++++++++++++++++ .../queue_backend/pg_queue/task_payload.py | 15 +- workers/sample.env | 8 + workers/tests/test_dispatch_pg.py | 9 + workers/tests/test_executor_registration.py | 46 +++ workers/tests/test_executor_rpc.py | 308 +++++++++++++++++ workers/tests/test_structure_tool_task.py | 38 +++ 10 files changed, 757 insertions(+), 6 deletions(-) create mode 100644 workers/queue_backend/pg_queue/executor_rpc.py create mode 100644 workers/tests/test_executor_registration.py create mode 100644 workers/tests/test_executor_rpc.py diff --git a/backend/sample.env b/backend/sample.env index 80406dfcd4..30450faff3 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -187,7 +187,7 @@ FLIPT_SERVICE_AVAILABLE=False # 9e PG-queue transport master-gate (kill-switch). Routing an execution onto the # PG queue requires ALL THREE: this gate True, FLIPT_SERVICE_AVAILABLE=True (else # the Flipt client returns False for every flag), and the Flipt flag -# `pg_queue_execution_enabled` on. Keep False until PG queue consumers are running +# `pg_queue_enabled` on. Keep False until PG queue consumers are running # in the fleet; set False for instant rollback. PG_QUEUE_TRANSPORT_ENABLED=False diff --git a/workers/executor/tasks.py b/workers/executor/tasks.py index 97faf2bf7f..1244e60b9c 100644 --- a/workers/executor/tasks.py +++ b/workers/executor/tasks.py @@ -5,6 +5,16 @@ ExecutionOrchestrator, and returns an ExecutionResult dict. """ +# Import the executor implementations so their ``@ExecutorRegistry.register`` +# decorators run before ``execute_extraction`` can be invoked. Coupling this to +# the task module (not only the Celery ``executor/worker.py`` entrypoint) ensures +# the registry is populated wherever the task is registered — in particular the PG +# executor consumer, which bootstraps via the root-worker import that loads +# ``executor/tasks.py`` (this module) but NOT ``executor/worker.py`` where this +# import historically lived; without it the consumer hits "No executor +# registered". Import is idempotent (module cached), so the Celery entrypoint +# importing it again is harmless. +import executor.executors # noqa: E402, F401 from queue_backend import worker_task from shared.clients import UsageAPIClient from shared.enums.task_enums import TaskName diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py index cfa0e316a6..02c146b061 100644 --- a/workers/file_processing/structure_tool_task.py +++ b/workers/file_processing/structure_tool_task.py @@ -25,12 +25,15 @@ from file_processing.worker import app from queue_backend import FairnessKey, worker_task from queue_backend.fairness import WorkloadType +from queue_backend.pg_queue.executor_rpc import ( + RoutingExecutionDispatcher, + get_executor_dispatcher, +) from shared.enums.task_enums import TaskName from shared.infrastructure.context import StateStore from unstract.sdk1.constants import ToolEnv, UsageKwargs from unstract.sdk1.execution.context import ExecutionContext -from unstract.sdk1.execution.dispatcher import ExecutionDispatcher from unstract.sdk1.execution.result import ExecutionResult logger = logging.getLogger(__name__) @@ -276,7 +279,9 @@ def _execute_structure_tool_impl(params: dict) -> dict: ) platform_helper = _create_platform_helper(shim, file_execution_id) - dispatcher = ExecutionDispatcher(celery_app=app) + # Gate-routed: PG executor RPC when pg_queue_enabled is on, else the unchanged + # Celery ExecutionDispatcher (zero-regression by construction — see executor_rpc). + dispatcher = get_executor_dispatcher(celery_app=app) fs = _get_file_storage() # ---- Step 2: Fetch tool metadata ---- @@ -675,7 +680,7 @@ def _run_agentic_extraction( input_file_path: str, output_dir_path: str, tool_instance_metadata: dict, - dispatcher: ExecutionDispatcher, + dispatcher: RoutingExecutionDispatcher, shim: Any, file_execution_id: str, execution_id: str, diff --git a/workers/queue_backend/pg_queue/executor_rpc.py b/workers/queue_backend/pg_queue/executor_rpc.py new file mode 100644 index 0000000000..a68100c8a1 --- /dev/null +++ b/workers/queue_backend/pg_queue/executor_rpc.py @@ -0,0 +1,316 @@ +"""Executor-RPC transport routing for the PG path — workers side (Phase 9, ③b-2). + +The workers-side twin of ``backend/pg_queue/executor_rpc.py``. The executor "RPC" +is a synchronous request-reply: a caller (here, the ``structure_tool`` task in the +file_processing worker) sends an ``ExecutionContext`` to the executor worker and +blocks for the ``ExecutionResult``. The legacy transport is Celery — the SDK +``ExecutionDispatcher`` (``send_task`` + ``AsyncResult.get``). This module adds a +**parallel** Postgres transport that leaves Celery and the SDK completely +untouched (no SDK edit, no change to the ``execute_extraction`` task or the Celery +executor worker): + +- :class:`PgExecutionDispatcher` enqueues ``execute_extraction`` onto the PG queue + (via :class:`~queue_backend.pg_queue.client.PgQueueClient`) with a unique + ``reply_key`` and polls ``pg_task_result`` (via + :class:`~queue_backend.pg_queue.result_backend.PgResultBackend`) for the reply — + same ``.dispatch()`` contract as the SDK dispatcher (never raises; + failure/timeout → ``ExecutionResult.failure``). The already-running + ``worker-pg-executor`` consumer runs the task and writes the reply, so this side + only adds the enqueue + poll halves. +- :func:`resolve_executor_transport` is the gate: master + ``PG_QUEUE_TRANSPORT_ENABLED`` (env, the workers analogue of the backend's + Django setting) then the **single** Flipt flag ``pg_queue_enabled`` — the same + flag the execution path uses, so one flip turns the whole PG-queue feature + on/off. Fails closed to Celery. +- :class:`RoutingExecutionDispatcher` is what ``structure_tool`` gets from + :func:`get_executor_dispatcher`: ``dispatch()`` picks PG-vs-Celery **per call** + (read at dispatch time → flipping the flag is an instant, no-redeploy + rollout/rollback); ``dispatch_async`` / ``dispatch_with_callback`` always + delegate to Celery (the callback path is a later slice). + +Zero-regression: gate off ⇒ every method delegates to the unchanged Celery +``ExecutionDispatcher`` and no ``pg_task_result`` row is created. + +TODO(shared): ``resolve_executor_transport`` and the reply_key/poll orchestration +mirror the backend module almost verbatim — only the transport primitives differ +(psycopg2 here vs Django ORM there). A later slice can lift the shared logic into +``unstract.core`` so the gate/contract lives in one place. +""" + +from __future__ import annotations + +import logging +import os +import uuid +from typing import TYPE_CHECKING, Any + +from unstract.core.data_models import PgTaskStatus +from unstract.flags.feature_flag import check_feature_flag_status +from unstract.sdk1.execution.dispatcher import ExecutionDispatcher +from unstract.sdk1.execution.result import ExecutionResult + +from .client import PgQueueClient +from .result_backend import PgResultBackend +from .task_payload import to_payload + +if TYPE_CHECKING: + from unstract.sdk1.execution.context import ExecutionContext + +logger = logging.getLogger(__name__) + +# The single PG-queue rollout flag — same key the execution path and scheduler +# read. Defined in backend/pg_queue/flags.py too; kept as a literal here because +# workers can't import backend code (see TODO(shared) in the module docstring). +PG_QUEUE_FLAG_KEY = "pg_queue_enabled" +# Master kill-switch + deploy-ordering gate. The workers analogue of the backend's +# ``settings.PG_QUEUE_TRANSPORT_ENABLED`` — read straight from the env here. +_MASTER_GATE_ENV = "PG_QUEUE_TRANSPORT_ENABLED" + +_EXECUTE_TASK = "execute_extraction" +# Mirror the SDK's queue-per-executor convention so the PG executor queue name +# matches the Celery one (the worker-pg-executor consumer subscribes to these). +_QUEUE_PREFIX = "celery_executor_" +# Caller-side wait default — mirrors the SDK dispatcher (EXECUTOR_RESULT_TIMEOUT +# env, else 3600s) so a PG-routed caller waits exactly as long as a Celery one. +_DEFAULT_TIMEOUT_ENV = "EXECUTOR_RESULT_TIMEOUT" +_DEFAULT_TIMEOUT = 3600 + + +def resolve_executor_transport(context: ExecutionContext) -> bool: + """True → route this executor dispatch over PG; False → Celery (default). + + Mirrors the backend ``resolve_executor_transport``: master-gated by the + ``PG_QUEUE_TRANSPORT_ENABLED`` env, then the **single** ``pg_queue_enabled`` + Flipt flag (shared across the whole PG-queue feature), bucketed per org. + **Fails closed to Celery** on a closed gate, a blind Flipt, or any error — so + the executor never silently loses its transport. + """ + if os.environ.get(_MASTER_GATE_ENV, "false").lower() != "true": + return False + if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true": + logger.warning( + "resolve_executor_transport: gate ON but FLIPT_SERVICE_AVAILABLE != " + "true (Flipt blind); using Celery" + ) + return False + org = getattr(context, "organization_id", None) + # %-bucket keyed on org; fall back to run_id so a context without an org still + # resolves deterministically (mirrors the backend resolver). + entity_id = str(org or getattr(context, "run_id", "") or "default") + flag_context = {"executor_name": str(context.executor_name)} + if org: + flag_context["organization_id"] = str(org) + try: + enabled = check_feature_flag_status( + flag_key=PG_QUEUE_FLAG_KEY, entity_id=entity_id, context=flag_context + ) + except Exception: + logger.warning( + "resolve_executor_transport: Flipt check failed; using Celery", + exc_info=True, + ) + return False + return bool(enabled) + + +class PgExecutionDispatcher: + """PG request-reply executor dispatch — drop-in for ``ExecutionDispatcher.dispatch``. + + Enqueues ``execute_extraction`` with a unique ``reply_key`` and blocks on + ``pg_task_result`` until the executor consumer records the result or the + timeout elapses. Honours the same contract as the SDK dispatcher: it never + raises and converts a timeout/failure into ``ExecutionResult.failure`` so + callers can branch on ``result.success`` identically on either transport. + """ + + def dispatch( + self, + context: ExecutionContext, + timeout: int | None = None, + headers: dict[str, Any] | None = None, + ) -> ExecutionResult: + # ``headers`` is accepted (and ignored) for substitutability with the SDK + # ``ExecutionDispatcher.dispatch`` / ``RoutingExecutionDispatcher.dispatch`` + # shapes — the PG path carries org/routing via the enqueue payload, not + # Celery headers, so fairness headers are intentionally not forwarded. + if timeout is None: + # Guard the env parse so a misconfigured EXECUTOR_RESULT_TIMEOUT can't + # raise out of dispatch() (the never-raises contract). + try: + timeout = int(os.environ.get(_DEFAULT_TIMEOUT_ENV, _DEFAULT_TIMEOUT)) + except (TypeError, ValueError): + # Don't swallow silently — an operator who fat-fingers the value + # would otherwise wait the 3600s default with no signal. + logger.warning( + "PG executor dispatch: invalid %s=%r; falling back to %ss", + _DEFAULT_TIMEOUT_ENV, + os.environ.get(_DEFAULT_TIMEOUT_ENV), + _DEFAULT_TIMEOUT, + ) + timeout = _DEFAULT_TIMEOUT + reply_key = str(uuid.uuid4()) + queue = f"{_QUEUE_PREFIX}{context.executor_name}" + org = str(getattr(context, "organization_id", "") or "") + try: + self._enqueue(queue, context, reply_key, org) + except Exception as exc: + logger.exception( + "PG executor dispatch: enqueue failed (executor=%s run_id=%s)", + context.executor_name, + context.run_id, + ) + return ExecutionResult.failure(error=f"{type(exc).__name__}: {exc}") + logger.info( + "PG executor dispatch: enqueued reply_key=%s queue=%s run_id=%s " + "timeout=%ss; waiting for result...", + reply_key, + queue, + context.run_id, + timeout, + ) + try: + row = self._wait_for_result(reply_key, timeout) + except Exception as exc: + # Honour the never-raises contract even if the poll connection dies. + logger.exception( + "PG executor dispatch: wait failed (reply_key=%s run_id=%s)", + reply_key, + context.run_id, + ) + return ExecutionResult.failure(error=f"{type(exc).__name__}: {exc}") + if row is None: + # On timeout the executor task may still be running on the consumer; + # it will write its outcome under this reply_key, but we've already + # given up reading it (the reaper retention-sweeps the orphan row). If + # the workflow engine retries the file execution, it re-dispatches with + # a FRESH reply_key — so two executor tasks for the same file can + # overlap (double LLM spend / duplicate writes). De-duping that belongs + # at the file-execution layer, not here; this transport stays at-least- + # once + caller-timeout by design. + logger.warning( + "PG executor dispatch: TIMEOUT after %ss (reply_key=%s run_id=%s) — " + "the executor task may still be running", + timeout, + reply_key, + context.run_id, + ) + return ExecutionResult.failure( + error=f"TimeoutError: executor reply not received within {timeout}s" + ) + # ``.get`` (not ``[...]``) so a result row missing ``status`` can't raise + # out of dispatch() — the never-raises contract must not depend on the + # producer always writing every key. + if ( + row.get("status") == PgTaskStatus.COMPLETED.value + and row.get("result") is not None + ): + try: + return ExecutionResult.from_dict(row["result"]) + except Exception as exc: + # A malformed completed row becomes a failure result, not a raise. + # Surface the parse cause (like the enqueue/wait paths) so a UI + # reading result.error isn't left with an opaque message. + logger.exception( + "PG executor dispatch: malformed completed result " + "(reply_key=%s run_id=%s)", + reply_key, + context.run_id, + ) + return ExecutionResult.failure( + error=( + f"Malformed executor result ({type(exc).__name__}) " + f"for reply_key {reply_key}" + ) + ) + logger.warning( + "PG executor dispatch: executor reported failure (reply_key=%s " + "run_id=%s): %s", + reply_key, + context.run_id, + row.get("error") or "(no error)", + ) + return ExecutionResult.failure(error=row.get("error") or "executor task failed") + + @staticmethod + def _enqueue( + queue: str, context: ExecutionContext, reply_key: str, org_id: str + ) -> None: + """Enqueue the ``execute_extraction`` request-reply message. + + A short-lived client owns its connection for just the insert (which + commits internally) so the message is durably visible to the + ``worker-pg-executor`` consumer before we begin polling — and no + connection is pinned for the whole (possibly long) RPC. + """ + payload = to_payload( + _EXECUTE_TASK, + args=[context.to_dict()], + queue=queue, + reply_key=reply_key, + ) + with PgQueueClient() as client: + client.send(queue, payload, org_id=org_id) + + @staticmethod + def _wait_for_result(reply_key: str, timeout: float) -> dict[str, Any] | None: + """Poll ``pg_task_result`` until the row appears or *timeout* elapses. + + Poll-based with capped backoff (PgBouncer-safe; no LISTEN/NOTIFY). The + backend owns one connection for the duration of the wait and closes it on + exit, so a long RPC never leaks a connection. The pin is bounded: a + file_processing worker runs ``--pool=prefork`` with + ``WORKER_FILE_PROCESSING_CONCURRENCY`` (default 4) processes, and each + dispatches sequentially, so at most ~concurrency connections are held for + up to ``EXECUTOR_RESULT_TIMEOUT``. (The backend twin instead releases via + ``close_old_connections`` between polls; if file_processing concurrency is + raised materially, do the same here.) + """ + with PgResultBackend() as rb: + return rb.wait_for_result(reply_key, timeout) + + +class RoutingExecutionDispatcher: + """Gate-routed executor dispatcher returned by :func:`get_executor_dispatcher`. + + ``dispatch()`` chooses PG vs Celery per call (instant rollout/rollback); + ``dispatch_async`` / ``dispatch_with_callback`` always delegate to Celery — + the async/callback path stays on Celery until a later continuation slice. + Duck-typed against the SDK ``ExecutionDispatcher`` so call sites are unchanged. + """ + + def __init__(self, celery_app: object | None = None) -> None: + self._celery = ExecutionDispatcher(celery_app=celery_app) + self._pg = PgExecutionDispatcher() + + def dispatch( + self, + context: ExecutionContext, + timeout: int | None = None, + headers: dict[str, Any] | None = None, + ) -> ExecutionResult: + if resolve_executor_transport(context): + logger.info( + "Executor RPC → PG transport (executor=%s run_id=%s)", + context.executor_name, + context.run_id, + ) + # PG carries org/routing via the enqueue payload, not Celery headers, + # so the fairness headers are intentionally not forwarded here (parity + # with the backend executor RPC). + return self._pg.dispatch(context, timeout=timeout) + return self._celery.dispatch(context, timeout=timeout, headers=headers) + + def dispatch_async( + self, context: ExecutionContext, headers: dict[str, Any] | None = None + ) -> str: + return self._celery.dispatch_async(context, headers=headers) + + def dispatch_with_callback(self, context: ExecutionContext, **kwargs: Any) -> Any: + return self._celery.dispatch_with_callback(context, **kwargs) + + +def get_executor_dispatcher( + celery_app: object | None = None, +) -> RoutingExecutionDispatcher: + """Factory: the gate-routed executor dispatcher (PG when enabled, else Celery).""" + return RoutingExecutionDispatcher(celery_app=celery_app) diff --git a/workers/queue_backend/pg_queue/task_payload.py b/workers/queue_backend/pg_queue/task_payload.py index 919914bfae..da80f6c155 100644 --- a/workers/queue_backend/pg_queue/task_payload.py +++ b/workers/queue_backend/pg_queue/task_payload.py @@ -32,12 +32,23 @@ def to_payload( kwargs: Mapping[str, Any] | None = None, queue: str | None = None, fairness: FairnessKey | None = None, + reply_key: str | None = None, ) -> TaskPayload: - """Build the JSON-serialisable task payload for the PG queue.""" - return TaskPayload( + """Build the JSON-serialisable task payload for the PG queue. + + ``reply_key`` marks a **request-reply** dispatch (the executor RPC on PG): + the executor consumer writes the task's result/error to ``pg_task_result`` + under it for the blocking caller to poll. Omitted = fire-and-forget. + """ + payload = TaskPayload( task_name=task_name, args=list(args) if args is not None else [], kwargs=dict(kwargs) if kwargs is not None else {}, queue=queue, fairness=fairness.to_dict() if fairness is not None else None, ) + # Only set for request-reply dispatches — keeps fire-and-forget rows + # byte-identical to before this field existed (mirrors the backend producer). + if reply_key is not None: + payload["reply_key"] = reply_key + return payload diff --git a/workers/sample.env b/workers/sample.env index 8ad7df3612..0cf9fb6b2a 100644 --- a/workers/sample.env +++ b/workers/sample.env @@ -473,3 +473,11 @@ FLIPT_SERVICE_AVAILABLE=False EVALUATION_SERVER_IP=unstract-flipt EVALUATION_SERVER_PORT=9005 PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +# PG-queue transport master-gate (kill-switch), worker side. Mirrors the backend's +# PG_QUEUE_TRANSPORT_ENABLED: the file_processing worker reads it to route the +# structure_tool executor RPC onto the PG queue. Routing onto PG requires ALL +# THREE: this gate True, FLIPT_SERVICE_AVAILABLE=True, and the Flipt flag +# `pg_queue_enabled` on. Keep False until the worker-pg-executor consumer is +# running in the fleet; set False for instant rollback. +PG_QUEUE_TRANSPORT_ENABLED=False diff --git a/workers/tests/test_dispatch_pg.py b/workers/tests/test_dispatch_pg.py index e9cde91824..c97a2664fa 100644 --- a/workers/tests/test_dispatch_pg.py +++ b/workers/tests/test_dispatch_pg.py @@ -66,6 +66,15 @@ def test_json_serialisable(self): # Must round-trip through JSON (it's stored in a JSONB column). json.dumps(to_payload("t", args=[1], kwargs={"a": "b"}, fairness=fairness)) + def test_reply_key_set_when_passed(self): + # The request-reply marker (executor RPC): the consumer keys the result + # row on it; a drop/mis-key would make the caller block until timeout. + assert to_payload("t", reply_key="rk")["reply_key"] == "rk" + + def test_reply_key_omitted_when_absent(self): + # Fire-and-forget rows stay byte-identical to before the field existed. + assert "reply_key" not in to_payload("t") + # --- Integration: dispatch() lands a decodable row in pg_queue_message --- # Uses the shared ``pg_client`` fixture from conftest.py. diff --git a/workers/tests/test_executor_registration.py b/workers/tests/test_executor_registration.py new file mode 100644 index 0000000000..aec766bac2 --- /dev/null +++ b/workers/tests/test_executor_registration.py @@ -0,0 +1,46 @@ +"""Regression: the executor registry must be populated by importing the task module. + +The PG executor consumer (``pg_queue_consumer``) bootstraps a worker by importing +the source worker's ``tasks.py`` — it does NOT import ``executor/worker.py`` (the +Celery entrypoint that historically held ``import executor.executors``). So if the +``@ExecutorRegistry.register`` side-effect import lives only in ``worker.py``, the +PG executor runs ``execute_extraction`` against an EMPTY registry and every +extraction fails with ``No executor registered with name 'legacy'. Available: +(none)``. + +This is pinned in a fresh interpreter that imports ONLY ``executor.tasks`` (exactly +what the PG consumer does), so import caching from the rest of the suite can't mask +a regression. +""" + +import os +import subprocess +import sys + + +def test_importing_executor_tasks_registers_executors(): + """Importing executor.tasks alone must register the bundled executors. + + Mirrors the PG-queue consumer's bootstrap (tasks.py, not worker.py). A fresh + subprocess avoids the suite's other imports pre-populating the registry. + """ + code = ( + "import executor.tasks\n" + "from unstract.sdk1.execution.registry import ExecutorRegistry\n" + "reg = getattr(ExecutorRegistry, '_registry', {})\n" + "assert 'legacy' in reg, f'registry empty/missing legacy: {sorted(reg)}'\n" + "print('OK', sorted(reg))\n" + ) + env = {**os.environ, "WORKER_TYPE": "executor"} + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env=env, + cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + ) + assert result.returncode == 0, ( + f"executor.tasks import did not register executors.\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + assert "OK" in result.stdout diff --git a/workers/tests/test_executor_rpc.py b/workers/tests/test_executor_rpc.py new file mode 100644 index 0000000000..ebf93b5889 --- /dev/null +++ b/workers/tests/test_executor_rpc.py @@ -0,0 +1,308 @@ +"""Tests for the workers-side executor-RPC transport routing (Phase 9, ③b-2). + +DB-free: the env gate / Flipt / the enqueue + poll halves are mocked. Pins the +gate's fail-closed matrix and — the load-bearing zero-regression property — that +with the gate off ``RoutingExecutionDispatcher`` delegates EVERY mode to the +unchanged Celery ``ExecutionDispatcher`` and never touches the PG path. Mirrors +``backend/pg_queue/tests/test_executor_rpc.py`` (the backend twin) adapted to the +worker primitives: an env master-gate instead of a Django setting, and the result +row is a plain ``dict`` (``PgResultBackend``) instead of a Django model. +""" + +from unittest.mock import MagicMock, patch + +from queue_backend.pg_queue.executor_rpc import ( + PgExecutionDispatcher, + RoutingExecutionDispatcher, + resolve_executor_transport, +) + +_MOD = "queue_backend.pg_queue.executor_rpc" + + +def _completed(result: dict) -> dict: + return {"status": "completed", "result": result, "error": ""} + + +def _ok_result() -> dict: + return {"success": True, "data": {"x": 1}, "metadata": {}, "error": None} + + +class TestPgExecutionDispatcherDispatch: + """The load-bearing contract: never raises; timeout/failure → failure result. + + DB-free — the ``_enqueue`` and ``_wait_for_result`` halves are mocked. + """ + + @staticmethod + def _ctx() -> MagicMock: + c = MagicMock() + c.executor_name = "legacy" + c.run_id = "r" + c.organization_id = "o" + c.to_dict.return_value = {"run_id": "r"} + return c + + def test_enqueue_failure_returns_failure_not_raise(self): + with patch.object( + PgExecutionDispatcher, "_enqueue", side_effect=RuntimeError("db down") + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is False + assert "RuntimeError" in res.error + + def test_wait_failure_returns_failure_not_raise(self): + with ( + patch.object(PgExecutionDispatcher, "_enqueue"), + patch.object( + PgExecutionDispatcher, + "_wait_for_result", + side_effect=RuntimeError("conn died"), + ), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is False + assert "RuntimeError" in res.error + + def test_timeout_returns_failure(self): + with ( + patch.object(PgExecutionDispatcher, "_enqueue"), + patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=None), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=3) + assert res.success is False + assert "within 3s" in res.error + + def test_completed_row_returns_result(self): + with ( + patch.object(PgExecutionDispatcher, "_enqueue"), + patch.object( + PgExecutionDispatcher, + "_wait_for_result", + return_value=_completed(_ok_result()), + ), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is True + + def test_failed_row_returns_error(self): + row = {"status": "failed", "result": None, "error": "boom"} + with ( + patch.object(PgExecutionDispatcher, "_enqueue"), + patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is False + assert res.error == "boom" + + def test_failed_row_empty_error_falls_back(self): + row = {"status": "failed", "result": None, "error": ""} + with ( + patch.object(PgExecutionDispatcher, "_enqueue"), + patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is False + assert "executor task failed" in res.error + + def test_completed_but_result_none_is_failure(self): + row = {"status": "completed", "result": None, "error": ""} + with ( + patch.object(PgExecutionDispatcher, "_enqueue"), + patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is False + + def test_malformed_completed_row_is_failure_not_raise(self): + with ( + patch.object(PgExecutionDispatcher, "_enqueue"), + patch.object( + PgExecutionDispatcher, + "_wait_for_result", + return_value=_completed({"bad": "shape"}), + ), + ): + res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + assert res.success is False + assert "Malformed" in res.error + + def test_timeout_none_reads_env_then_default(self, monkeypatch): + monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "42") + seen = {} + + def fake_wait(reply_key, timeout): + seen["timeout"] = timeout + return None + + with ( + patch.object(PgExecutionDispatcher, "_enqueue"), + patch.object( + PgExecutionDispatcher, "_wait_for_result", side_effect=fake_wait + ), + ): + # No explicit timeout arg → falls back to the env/default. + PgExecutionDispatcher().dispatch(self._ctx()) + assert seen["timeout"] == 42 + + def test_timeout_none_bad_env_falls_back_to_default(self, monkeypatch): + monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "not-an-int") + seen = {} + + def fake_wait(reply_key, timeout): + seen["timeout"] = timeout + return None + + with ( + patch.object(PgExecutionDispatcher, "_enqueue"), + patch.object( + PgExecutionDispatcher, "_wait_for_result", side_effect=fake_wait + ), + ): + PgExecutionDispatcher().dispatch(self._ctx()) # must not raise + assert seen["timeout"] == 3600 # _DEFAULT_TIMEOUT + + +def _ctx(org: str | None = "org1") -> MagicMock: + c = MagicMock() + c.executor_name = "legacy" + c.run_id = "run-1" + c.organization_id = org + return c + + +class TestResolveExecutorTransport: + def test_master_gate_off_is_celery(self, monkeypatch): + monkeypatch.delenv("PG_QUEUE_TRANSPORT_ENABLED", raising=False) + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with patch(f"{_MOD}.check_feature_flag_status") as flag: + assert resolve_executor_transport(_ctx()) is False + flag.assert_not_called() # gate off → Flipt never consulted + + def test_flipt_unavailable_is_celery(self, monkeypatch): + monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false") + with patch(f"{_MOD}.check_feature_flag_status") as flag: + assert resolve_executor_transport(_ctx()) is False + flag.assert_not_called() + + def test_flag_true_is_pg_keyed_on_org(self, monkeypatch): + monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with patch( + f"{_MOD}.check_feature_flag_status", return_value=True + ) as flag: + assert resolve_executor_transport(_ctx("orgX")) is True + assert flag.call_args.kwargs["entity_id"] == "orgX" + # The single shared PG-queue flag (not a per-subsystem flag). + assert flag.call_args.kwargs["flag_key"] == "pg_queue_enabled" + + def test_flag_false_is_celery(self, monkeypatch): + monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with patch(f"{_MOD}.check_feature_flag_status", return_value=False): + assert resolve_executor_transport(_ctx()) is False + + def test_flipt_error_fails_closed_to_celery(self, monkeypatch): + monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with patch( + f"{_MOD}.check_feature_flag_status", side_effect=RuntimeError("down") + ): + assert resolve_executor_transport(_ctx()) is False + + def test_org_less_context_buckets_on_run_id(self, monkeypatch): + """No org → entity_id falls back to run_id and org is absent from context. + + Guards the org-less bucketing so cross-org/run-only contexts resolve + deterministically instead of shipping a bogus "None" org. + """ + monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") + monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") + with patch( + f"{_MOD}.check_feature_flag_status", return_value=True + ) as flag: + assert resolve_executor_transport(_ctx(org=None)) is True + assert flag.call_args.kwargs["entity_id"] == "run-1" + assert "organization_id" not in flag.call_args.kwargs["context"] + + +class TestPgExecutionDispatcherEnqueueWiring: + """The actual PG-transport wiring: queue name, payload shape, org_id. + + These are the *only* routing/identity carried on the PG path (Celery headers + are dropped), so a bug here misroutes or breaks org-fairness. ``to_payload`` + runs for real; only ``PgQueueClient`` and the wait are mocked. + """ + + @staticmethod + def _ctx(): + c = MagicMock() + c.executor_name = "legacy" + c.run_id = "r" + c.organization_id = "org9" + c.to_dict.return_value = {"run_id": "r"} + return c + + def test_enqueue_sends_queue_payload_and_org(self): + client = MagicMock() + client.__enter__.return_value = client # `with PgQueueClient() as c` → c is client + with ( + patch(f"{_MOD}.PgQueueClient", return_value=client), + patch.object( + PgExecutionDispatcher, + "_wait_for_result", + return_value=_completed(_ok_result()), + ), + ): + PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) + client.send.assert_called_once() + args, kwargs = client.send.call_args + queue_arg, payload_arg = args[0], args[1] + assert queue_arg == "celery_executor_legacy" + assert kwargs["org_id"] == "org9" + assert payload_arg["task_name"] == "execute_extraction" + assert payload_arg["args"] == [{"run_id": "r"}] + assert payload_arg["reply_key"] # request-reply marker present (a uuid) + + +class TestRoutingZeroRegression: + @staticmethod + def _build(): + # Patch both sub-dispatchers at construction; the instances are captured + # in __init__ so they remain mocked after the context exits. + with ( + patch(f"{_MOD}.ExecutionDispatcher") as celery_cls, + patch(f"{_MOD}.PgExecutionDispatcher") as pg_cls, + ): + dispatcher = RoutingExecutionDispatcher(celery_app="app") + return dispatcher, celery_cls.return_value, pg_cls.return_value + + def test_gate_off_forwards_timeout_and_headers_to_celery(self): + """Zero-regression: gate off → Celery gets timeout AND headers unchanged.""" + dispatcher, celery, pg = self._build() + ctx = _ctx() + hdrs = {"x-fairness-key": {"org_id": "o"}} + with patch(f"{_MOD}.resolve_executor_transport", return_value=False): + dispatcher.dispatch(ctx, timeout=9, headers=hdrs) + celery.dispatch.assert_called_once_with(ctx, timeout=9, headers=hdrs) + pg.dispatch.assert_not_called() # the zero-regression guarantee + + def test_gate_on_passes_timeout_to_pg_and_drops_headers(self): + """Gate on → PG gets the timeout but NOT the Celery headers (intentional).""" + dispatcher, celery, pg = self._build() + ctx = _ctx() + with patch(f"{_MOD}.resolve_executor_transport", return_value=True): + dispatcher.dispatch(ctx, timeout=7, headers={"x-fairness-key": {"o": 1}}) + pg.dispatch.assert_called_once_with(ctx, timeout=7) + assert "headers" not in pg.dispatch.call_args.kwargs + celery.dispatch.assert_not_called() + + def test_async_and_callback_always_celery(self): + """The callback/async path stays on Celery regardless of the gate (a later slice).""" + dispatcher, celery, pg = self._build() + dispatcher.dispatch_async(_ctx()) + dispatcher.dispatch_with_callback(_ctx(), on_success=None) + celery.dispatch_async.assert_called_once() + celery.dispatch_with_callback.assert_called_once() + pg.dispatch.assert_not_called() diff --git a/workers/tests/test_structure_tool_task.py b/workers/tests/test_structure_tool_task.py index 8bdb541eca..a8fb5eca65 100644 --- a/workers/tests/test_structure_tool_task.py +++ b/workers/tests/test_structure_tool_task.py @@ -17,8 +17,11 @@ from __future__ import annotations +from unittest.mock import patch + import pytest +from file_processing import structure_tool_task as st from file_processing.structure_tool_task import _fairness_headers from queue_backend.fairness import WorkloadType @@ -50,5 +53,40 @@ def test_workload_type_is_non_api_not_api(self): assert wire["x-fairness-key"]["workload_type"] != WorkloadType.API.value +class TestDispatcherFactory: + """Pin the call-site swap this PR makes: the impl builds its dispatcher via + ``get_executor_dispatcher(celery_app=app)`` (the gate-routed dispatcher), not + the raw SDK ``ExecutionDispatcher``. A mis-import or wrong arg would otherwise + silently bypass the PG routing with nothing failing. + """ + + @staticmethod + def _params() -> dict: + return { + "organization_id": "org1", + "file_execution_id": "fe1", + "tool_instance_metadata": {}, + "platform_service_api_key": "sk", + "input_file_path": "/in/f.pdf", + "output_dir_path": "/out", + "source_file_name": "f.pdf", + "execution_data_dir": "/data", + } + + def test_impl_builds_dispatcher_via_factory_with_app(self): + # Stub everything up to (and just past) the dispatcher construction, then + # raise to stop before the heavy tool-metadata work runs. + with ( + patch("executor.executor_tool_shim.ExecutorToolShim"), + patch.object(st, "_create_platform_helper"), + patch.object(st, "_get_file_storage"), + patch.object(st, "get_executor_dispatcher") as factory, + patch.object(st, "_fetch_tool_metadata", side_effect=RuntimeError("stop")), + ): + with pytest.raises(RuntimeError, match="stop"): + st._execute_structure_tool_impl(self._params()) + factory.assert_called_once_with(celery_app=st.app) + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From ab80a494bdf23c07c49db5a2614e039c234c1298 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Sat, 20 Jun 2026 20:28:51 +0530 Subject: [PATCH 30/44] =?UTF-8?q?UN-3606=20[FIX]=20PG=20Queue=20=E2=80=94?= =?UTF-8?q?=20prefork=20the=20consumer=20so=20file=20batches=20+=20executo?= =?UTF-8?q?r=20RPCs=20run=20in=20parallel=20(#2096)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3606 [FIX] PG Queue — prefork the consumer so file batches + executor RPCs run in parallel The PG file-processing path ran files SERIALLY: worker-pg-fileproc was a single batch=1 consumer, so multi-file ETL batches drained one at a time (Celery runs them in parallel via prefork --concurrency). Same single-consumer bottleneck on worker-pg-executor serialized the executor RPCs (the LLM extraction), so even parallel file fan-out gained nothing end-to-end. Fix: a prefork supervisor for the PG-queue consumer launcher — the PG analogue of Celery --pool=prefork --concurrency=N. - supervisor.py (new): when WORKER_PG_QUEUE_CONSUMER_CONCURRENCY > 1, fork N isolated consumer children (each does its own worker bootstrap, so no connections are inherited across the fork), monitor + re-fork dead children, and own a single fleet-liveness endpoint (503 when the oldest child stalls past the threshold). SKIP LOCKED distributes batches across children AND replicas; a single execution is still capped by MAX_PARALLEL_FILE_BATCHES; total live parallelism = concurrency x replicas (k8s HPA scales replicas). - The consumer code is UNCHANGED — each child is the current single-threaded PgQueueConsumer.run(); concurrency is purely a launch concern. CONCURRENCY=1 (default) keeps the byte-identical single-process path, so every other PG consumer is non-regressive. - consumer.py: extract build_consumer_from_env()/consumer_env() so the children and main() build identical consumers. - docker-compose: CONCURRENCY=4 on worker-pg-fileproc AND worker-pg-executor (mirrors WORKER_FILE_PROCESSING_CONCURRENCY). Also raise their VT_SECONDS to 3660 — process_file_batch/execute_extraction block on the executor up to 3600s, so the prior vt=30 default would re-claim a long batch mid-run (latent double-run); health-stale sits just above vt. - Tests: env-knob parse/clamp/validation, fleet-staleness calc, reap/restart matrix (dead -> re-fork, live -> left, shutdown -> not resurrected). Process model matches Celery prefork (the cloud-trusted choice): full process isolation, a crash is contained + re-forked, its in-flight message redelivers via vt (at-least-once). Dev-tested end-to-end: 4 fileproc + 4 executor children, a 2-file ETL runs fully parallel (both files + both executor RPCs overlap on separate children; ~25s vs ~42s serial), child crash auto-restarts, health green. Co-Authored-By: Claude Opus 4.8 * UN-3606 [FIX] Address PR review (toolkit + greptile) — supervisor hardening Critical/High: - Crash-loop no longer hides a wedged fleet from the probe: re-fork does NOT reseed the heartbeat (a never-polling slot ages), and a slot that dies immediately N times in a row forces freshness=inf → 503 (k8s restarts the pod). - os.fork() wrapped: initial-fleet failure fails fast with an actionable message; reap-path failure logs + leaves the slot for the next tick (no uncaught crash / no SIGTERM storm against healthy children). - Heartbeat publish loop guarded (try/except + log) so a transient error can't silently kill the thread and false-stale a healthy child. - Children reset SIGTERM/SIGINT to SIG_DFL immediately after fork — a signal in the fork→run window no longer fires the parent's _on_term against stale pids. Medium: - _Fleet class owns pid/last_fork/heartbeat/crash-count/restart-schedule with a validated slot and consistent reap (replaces three loose slot-keyed dicts). - Re-fork backoff is a scheduled not-before (non-blocking), re-checking stopping before each fork — no in-loop sleep serialising recovery, no child spawned into shutdown. - Liveness bind: EADDRINUSE logged at error (vs transient), and liveness_probe_bound surfaced in the status payload. - _join_children: per-child grace budget (one slow child can't starve the rest); _wait_for_exit helper extracted. Low: - real Callable annotations (drop noqas); HEALTH_PORT typed int|None; comment fixes (heartbeat writer reasoning, "JSON body differs" for the fleet probe); WORKER_TYPE selection deduped into pg_queue_consumer/_bootstrap.py. Tests rewritten + expanded: _Fleet bookkeeping/crash-loop/freshness, reap scheduling, restart-due gating, fork OSError + child hard-exit guard, _wait_for_exit + _join_children SIGKILL escalation, health no-port/bind-error. 32 tests. Re-validated live: 4 children, fleet health (crash_looping/liveness_probe_bound fields), crash → re-fork. Co-Authored-By: Claude Opus 4.8 * UN-3606 [FIX] Address SonarCloud — reliability + maintainability nits - test: avoid float equality (math.isinf for the crash-loop freshness; drop the 0.0 literal) — clears the only New-Code reliability bug (Rating C → A). - supervisor: clean the `# noqa: ANN201` suppression syntax (explanation moved to the docstring); narrow the child-failure catch BaseException → Exception (the realistic startup-failure surface; SystemExit/KeyboardInterrupt would exit anyway); logger.error(exc_info=True) → logger.exception() on the os.fork() guard. Co-Authored-By: Claude Opus 4.8 * UN-3606 [FIX] Address greptile — snapshot crash dict in is_crash_looping is_crash_looping() runs in the LivenessServer daemon thread (via freshness()) while the main thread mutates _consecutive_crashes in schedule_restart() — a bare .values() iteration could raise "dictionary changed size during iteration". Snapshot with tuple(...) (atomic under the GIL) before iterating. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- docker/docker-compose.yaml | 17 + workers/pg_queue_consumer/__main__.py | 27 +- workers/pg_queue_consumer/_bootstrap.py | 24 + workers/pg_queue_consumer/supervisor.py | 485 +++++++++++++++++++ workers/queue_backend/pg_queue/consumer.py | 69 ++- workers/sample.env | 9 + workers/tests/test_pg_consumer_supervisor.py | 276 +++++++++++ 7 files changed, 873 insertions(+), 34 deletions(-) create mode 100644 workers/pg_queue_consumer/_bootstrap.py create mode 100644 workers/pg_queue_consumer/supervisor.py create mode 100644 workers/tests/test_pg_consumer_supervisor.py diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index f58f5a9761..a8f64a00e9 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -638,6 +638,17 @@ services: - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=file_processing - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing,api_file_processing - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090 + # UN-3606: prefork N consumer children so file batches run in parallel (the + # PG analogue of the Celery worker's --concurrency). SKIP LOCKED distributes + # batches across children + replicas; a single execution is still capped by + # MAX_PARALLEL_FILE_BATCHES. Mirrors WORKER_FILE_PROCESSING_CONCURRENCY. + - WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_FILEPROC_CONCURRENCY:-4} + # process_file_batch blocks on the executor RPC (up to EXECUTOR_RESULT_TIMEOUT, + # default 3600s). vt must exceed ONE batch's worst case (children run in + # parallel, not serially) or a long batch gets re-claimed mid-run; health-stale + # sits just above vt so a legitimately-long batch never trips the fleet probe. + - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_FILEPROC_VT_SECONDS:-3660} + - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_FILEPROC_HEALTH_STALE_SECONDS:-3720} labels: - traefik.enable=false volumes: @@ -743,6 +754,12 @@ services: # double-spend, two runs racing the same reply_key) or killed by the probe. - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_EXECUTOR_VT_SECONDS:-3660} - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_EXECUTOR_HEALTH_STALE_SECONDS:-3720} + # UN-3606: prefork N executor children so concurrent file batches' executor + # RPCs run in parallel instead of serializing here. Without this the + # file-processing parallelism delivers no end-to-end speedup — the LLM + # extraction (the wall-clock-dominant work) is the bottleneck. Match the + # file-processing concurrency so N parallel files get N parallel executors. + - WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_EXECUTOR_CONCURRENCY:-4} - CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table} labels: - traefik.enable=false diff --git a/workers/pg_queue_consumer/__main__.py b/workers/pg_queue_consumer/__main__.py index 1be588708b..e49aee7191 100644 --- a/workers/pg_queue_consumer/__main__.py +++ b/workers/pg_queue_consumer/__main__.py @@ -20,21 +20,32 @@ Launch via ``python -m pg_queue_consumer`` or ``./run-worker.sh pg-queue-consumer``. """ -import os - def _bootstrap_and_run() -> None: - # Select the source worker whose tasks back this consumer's queue. Must run - # BEFORE `import worker`, which reads WORKER_TYPE at import time. We overwrite - # (not setdefault) because the launcher's own WORKER_TYPE owns no tasks. + # CONCURRENCY > 1 → run a prefork supervisor (UN-3606): N isolated consumer + # children, the PG analogue of Celery --pool=prefork --concurrency=N. The + # supervisor forks the children (each does its OWN worker bootstrap, so no + # connections are inherited across the fork) and owns the single health port. + # CONCURRENCY = 1 (default) keeps the plain single-process path below, + # byte-identical to before the supervisor existed. # # Kept inside this guarded function (not at module scope) so the env mutation # and the heavy `import worker` bootstrap only happen on the `python -m` # entry — never as a side-effect if this module is imported by a test, IDE, # or type-checker walking `__main__` files. - os.environ["WORKER_TYPE"] = os.environ.get( - "WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE", "notification" - ) + from pg_queue_consumer.supervisor import concurrency_from_env, run_supervised + + concurrency = concurrency_from_env() + if concurrency > 1: + run_supervised(concurrency) + return + + # Single-process path: select the source worker whose tasks back this + # consumer's queue (must run BEFORE `import worker`, which reads WORKER_TYPE at + # import time). Shared with the supervisor's children via _bootstrap. + from pg_queue_consumer._bootstrap import select_source_worker_type + + select_source_worker_type() import worker # noqa: F401 — side-effect: registers the source worker's tasks from queue_backend.pg_queue.consumer import main diff --git a/workers/pg_queue_consumer/_bootstrap.py b/workers/pg_queue_consumer/_bootstrap.py new file mode 100644 index 0000000000..a4994072a5 --- /dev/null +++ b/workers/pg_queue_consumer/_bootstrap.py @@ -0,0 +1,24 @@ +"""Shared bootstrap for the PG-queue consumer entrypoints. + +Both the single-process launcher (``__main__``) and the prefork supervisor's +children must select the source worker type *before* ``import worker`` (which +reads ``WORKER_TYPE`` at import time to register that type's tasks). Kept in a +neutral module — not ``__main__`` — so ``supervisor`` can import it without +triggering the ``python -m`` entry. +""" + +import os + + +def select_source_worker_type() -> None: + """Point ``WORKER_TYPE`` at the source worker whose tasks back this consumer. + + Overwrites (not ``setdefault``): the launcher's own ``WORKER_TYPE`` is the + ``pg_queue_consumer`` pseudo-type, which owns no tasks — a plain ``import + worker`` would fall back to the ``general`` worker and drop every message as an + unknown task. Default ``notification`` (the worker owning the first migrated + leaf task). Must run before ``import worker``. + """ + os.environ["WORKER_TYPE"] = os.environ.get( + "WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE", "notification" + ) diff --git a/workers/pg_queue_consumer/supervisor.py b/workers/pg_queue_consumer/supervisor.py new file mode 100644 index 0000000000..cc99ef091b --- /dev/null +++ b/workers/pg_queue_consumer/supervisor.py @@ -0,0 +1,485 @@ +"""Prefork supervisor for the PG-queue consumer (UN-3606). + +Forks ``WORKER_PG_QUEUE_CONSUMER_CONCURRENCY`` copies of the single-threaded +:class:`~queue_backend.pg_queue.consumer.PgQueueConsumer` so multiple file +batches run in parallel — the PG analogue of Celery's ``--pool=prefork +--concurrency=N``. ``SELECT … FOR UPDATE SKIP LOCKED`` distributes work across the +children (and across replicas): each child claims distinct rows, a single +execution is still capped by ``MAX_PARALLEL_FILE_BATCHES``, and total live +parallelism = ``concurrency × replicas`` (k8s HPA scales the replica count). + +**Process model** (matches Celery prefork — the cloud-trusted choice): each child +is a fully isolated process with its own DB connections and thread-local +``StateStore`` — no shared mutable state, no thread-safety surface. A child crash +is isolated and re-forked (rate-limited); its in-flight message redelivers via +``vt`` (at-least-once). The **consumer code is unchanged** — concurrency is purely +a launch concern. ``CONCURRENCY = 1`` keeps the plain single-process ``main()`` +path (byte-identical to before this module existed). + +**Health**: the supervisor owns the single liveness port and reports the *fleet's* +freshness — the staleness of the oldest-polling child (each child publishes its +last-poll wall-time into a shared array). A child that dies is re-forked +internally (transient); a child that **crash-loops** (dies immediately N times in +a row, never reaching a real poll) forces the probe to 503 so k8s restarts the +pod rather than the supervisor masking a wedged fleet with fresh-looking re-forks. + +**Fork safety**: the initial fleet is forked while the parent is single-threaded. +Re-forks happen after the liveness daemon thread exists; the only other thread is +that probe (idle in ``select`` between requests, and CPython 3.12 re-inits the +``logging`` locks across ``fork`` via ``os.register_at_fork``), and each child +resets inherited signal handlers + does its own ``import worker`` before touching +shared resources — so an inherited held lock or the parent's ``_on_term`` cannot +wedge or mis-signal a child. +""" + +from __future__ import annotations + +import contextlib +import logging +import multiprocessing +import os +import signal +import threading +import time +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from queue_backend.pg_queue.liveness import LivenessServer + +logger = logging.getLogger(__name__) + +_DEFAULT_CONCURRENCY = 1 +# Fork-bomb backstop for a fat-fingered env (a single machine can't usefully run +# hundreds of heavy file-processing children anyway — scale replicas instead). +_MAX_CONCURRENCY = 64 +# How often each child republishes its heartbeat, and the parent reaps + checks. +_REPORT_INTERVAL_SECONDS = 1.0 +_MONITOR_INTERVAL_SECONDS = 1.0 +# Re-fork backoff floor and ceiling — a crash-looping child must not fork-storm. +_RESTART_MIN_INTERVAL_SECONDS = 2.0 +_RESTART_MAX_BACKOFF_SECONDS = 30.0 +# A child that stays up at least this long before exiting is a normal exit, not an +# immediate crash — it resets the slot's consecutive-crash counter. +_MIN_HEALTHY_UPTIME_SECONDS = 10.0 +# Consecutive immediate crashes after which the fleet probe is forced unhealthy. +_CRASH_LOOP_THRESHOLD = 3 +# How long to wait per child for a graceful drain on shutdown before SIGKILL. +_SHUTDOWN_GRACE_SECONDS = 30.0 + + +def concurrency_from_env() -> int: + """Parse ``WORKER_PG_QUEUE_CONSUMER_CONCURRENCY`` (default 1, clamped to a sane + max). 1 → the single-process path; >1 → the prefork supervisor. + """ + raw = os.environ.get("WORKER_PG_QUEUE_CONSUMER_CONCURRENCY") + if raw is None or raw == "": + return _DEFAULT_CONCURRENCY + try: + n = int(raw) + except (TypeError, ValueError) as exc: + raise ValueError( + f"Invalid WORKER_PG_QUEUE_CONSUMER_CONCURRENCY={raw!r}: {exc}" + ) from exc + if n < 1: + raise ValueError(f"WORKER_PG_QUEUE_CONSUMER_CONCURRENCY must be >= 1, got {n}") + if n > _MAX_CONCURRENCY: + logger.warning( + "WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=%s exceeds the %s cap; clamping " + "(scale replicas for more parallelism, not one fat process)", + n, + _MAX_CONCURRENCY, + ) + n = _MAX_CONCURRENCY + return n + + +class _Fleet: + """Owns the per-slot child state — pid, last-fork, heartbeat, crash count and + pending-restart schedule — keeping them mutually consistent. Slots are + validated against ``[0, concurrency)`` so a stray key can't silently desync + the structures or ``IndexError`` the shared array. + """ + + def __init__(self, concurrency: int) -> None: + self._n = concurrency + # Shared, fork-inherited heartbeat slots (one last-poll wall-time per + # child). lock=False is safe: a slot is written either by the parent + # (seed, at construction, while no child owns it) OR by that child's + # heartbeat thread — never concurrently — and only read by the parent, so + # a torn double read just yields one stale sample that self-corrects. + self._heartbeats = multiprocessing.Array("d", concurrency, lock=False) + now = time.time() + for i in range(concurrency): + self._heartbeats[i] = now + self._pids: dict[int, int] = {} + self._last_fork: dict[int, float] = {} + self._consecutive_crashes: dict[int, int] = {} + self._restart_due: dict[int, float] = {} # slot -> monotonic not-before + + @property + def concurrency(self) -> int: + return self._n + + @property + def heartbeats(self): # noqa: ANN201 + """The shared heartbeat array (a ctypes array, passed to forked children, + which write their own slot directly). + """ + return self._heartbeats + + def _validate(self, slot: int) -> None: + if not 0 <= slot < self._n: + raise IndexError(f"slot {slot} out of range [0, {self._n})") + + def record_fork(self, slot: int, pid: int) -> None: + """Mark ``slot`` alive under ``pid``; clears any pending restart. Note the + heartbeat is deliberately NOT reseeded here — a re-forked child must earn + freshness by actually polling, so a crash-looping slot ages instead of + looking perpetually fresh. + """ + self._validate(slot) + self._pids[slot] = pid + self._last_fork[slot] = time.monotonic() + self._restart_due.pop(slot, None) + + def reap(self, slot: int) -> float: + """Drop the slot's pid + last-fork together; return the child's uptime (s).""" + forked_at = self._last_fork.pop(slot, time.monotonic()) + self._pids.pop(slot, None) + return time.monotonic() - forked_at + + def schedule_restart(self, slot: int, uptime: float) -> int: + """Record the exit and set the re-fork not-before; return the consecutive + immediate-crash count. A child that ran healthily before exiting resets the + counter; an immediate death increments it and backs the restart off + (capped) so a crash loop can't fork-storm. + """ + if uptime < _MIN_HEALTHY_UPTIME_SECONDS: + n = self._consecutive_crashes.get(slot, 0) + 1 + else: + n = 0 # ran fine, then exited — not a crash loop + self._consecutive_crashes[slot] = n + backoff = min( + _RESTART_MIN_INTERVAL_SECONDS * max(1, n), _RESTART_MAX_BACKOFF_SECONDS + ) + self._restart_due[slot] = time.monotonic() + backoff + return n + + def due_restarts(self) -> list[int]: + """Slots whose re-fork backoff has elapsed (oldest schedule first).""" + now = time.monotonic() + return sorted(s for s, due in self._restart_due.items() if due <= now) + + def consecutive_crashes(self, slot: int) -> int: + return self._consecutive_crashes.get(slot, 0) + + def alive_items(self) -> list[tuple[int, int]]: + return list(self._pids.items()) + + def alive_count(self) -> int: + return len(self._pids) + + def is_crash_looping(self) -> bool: + """True if any slot has died immediately ``_CRASH_LOOP_THRESHOLD`` times in + a row — the signal that the heartbeat alone can't be trusted fresh. + + Snapshots the values first (``tuple(...)`` is atomic under the GIL): this + runs in the liveness daemon thread (via :meth:`freshness`) while the main + thread mutates ``_consecutive_crashes`` in :meth:`schedule_restart`, so a + bare ``.values()`` iteration could raise "dictionary changed size during + iteration". + """ + return any( + n >= _CRASH_LOOP_THRESHOLD for n in tuple(self._consecutive_crashes.values()) + ) + + def oldest_age(self) -> float: + now = time.time() + return max((now - hb for hb in self._heartbeats), default=0.0) + + def freshness(self) -> float: + """Liveness verdict source: a crash-looping fleet is force-stale (``inf``) + so the probe trips 503 even if a just-constructed child briefly looked + fresh; otherwise the oldest child's staleness (catches a wedged-alive + child). + """ + return float("inf") if self.is_crash_looping() else self.oldest_age() + + +def _run_child(slot: int, heartbeats) -> None: # noqa: ANN001 (ctypes array) + """Build one consumer and run it forever, publishing its heartbeat. + + The worker import (and any connections it opens) happens HERE, in the child — + never inherited across the fork — so each process owns its own connections. + A *guarded* daemon thread publishes the consumer's last-poll wall-time into + ``heartbeats[slot]`` for the supervisor's fleet liveness. + """ + from pg_queue_consumer._bootstrap import select_source_worker_type + + select_source_worker_type() # set WORKER_TYPE before importing worker + import worker # noqa: F401 — side-effect: registers the source worker's tasks + from queue_backend.pg_queue.consumer import build_consumer_from_env + + consumer = build_consumer_from_env() + + def _publish_heartbeat() -> None: + # last-poll wall-time = now − (seconds since last poll). Frozen while a + # task runs (the consumer stamps its heartbeat at the top of poll_once), + # so a child stuck on a too-long task goes stale exactly as the single + # consumer does. Guarded so a transient error (e.g. teardown during + # shutdown) logs loudly and the loop continues instead of dying silently + # and false-staling a healthy child. + while True: + try: + heartbeats[slot] = time.time() - consumer.seconds_since_last_poll() + except Exception: + logger.exception( + "PG-queue consumer: heartbeat publish failed for slot=%s", slot + ) + time.sleep(_REPORT_INTERVAL_SECONDS) + + threading.Thread(target=_publish_heartbeat, daemon=True, name=f"pg-hb-{slot}").start() + # consumer.run() installs its own SIGTERM/SIGINT handlers → graceful stop. + consumer.run() + + +def _child_after_fork(slot: int, heartbeats) -> None: # noqa: ANN001 (ctypes array) + """Child side of the fork: reset inherited state, run, hard-exit on failure. + + Resets the supervisor's signal handlers to ``SIG_DFL`` *immediately* — until + ``consumer.run()`` installs its own, a SIGTERM arriving in the fork→run window + (which spans the slow ``import worker`` bootstrap) must NOT fire the parent's + ``_on_term`` closure in the child (it captured a stale ``children`` dict and + would signal sibling pids). ``SIG_DFL`` = terminate, the correct disposition + for a not-yet-running child. + """ + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_DFL) + try: + _run_child(slot, heartbeats) + except Exception: + # A child that can't even start must not return into the supervisor loop + # (it would fork grandchildren). Log + hard exit. Exception (not + # BaseException) is the realistic startup-failure surface here — import, + # connection, config; SystemExit/KeyboardInterrupt would exit anyway. + logger.exception("PG-queue consumer: child slot=%s failed to run", slot) + os._exit(1) + os._exit(0) + + +def _try_fork_child(fleet: _Fleet, slot: int) -> bool: + """Fork one child for ``slot``. Returns False (without raising) if ``os.fork`` + fails — EAGAIN (RLIMIT_NPROC) / ENOMEM are realistic under heavy-child load — + so the caller can fail fast (initial fleet) or leave the slot for the next + monitor tick (re-fork path) instead of an uncaught crash taking the fleet down. + """ + try: + pid = os.fork() + except OSError: + logger.exception( + "PG-queue consumer: os.fork() failed for slot=%s (process/memory " + "limit?) — will retry", + slot, + ) + return False + if pid == 0: # child — never returns + _child_after_fork(slot, fleet.heartbeats) + fleet.record_fork(slot, pid) + logger.info("PG-queue consumer: forked child slot=%s pid=%s", slot, pid) + return True + + +def _reap_dead(fleet: _Fleet, stopping: threading.Event) -> None: + """Reap exited children and schedule their re-fork (unless shutting down).""" + for slot, pid in fleet.alive_items(): + try: + reaped, _status = os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + reaped = pid # already reaped elsewhere — treat as gone + if reaped == 0: + continue # still alive + uptime = fleet.reap(slot) + if stopping.is_set(): + continue # do not resurrect during shutdown + crashes = fleet.schedule_restart(slot, uptime) + level = logging.ERROR if crashes >= _CRASH_LOOP_THRESHOLD else logging.WARNING + logger.log( + level, + "PG-queue consumer: child slot=%s pid=%s exited after %.1fs " + "(consecutive immediate crashes=%s) — re-fork scheduled", + slot, + pid, + uptime, + crashes, + ) + + +def _restart_due_children(fleet: _Fleet, stopping: threading.Event) -> None: + """Re-fork the slots whose backoff has elapsed — non-blocking (the backoff is + a scheduled not-before, not an in-loop sleep), and re-checking ``stopping`` + each iteration so a SIGTERM mid-cycle can't spawn a fresh child into shutdown. + """ + for slot in fleet.due_restarts(): + if stopping.is_set(): + return + # On success record_fork clears the pending restart; on fork failure the + # slot stays due and is retried next tick. + _try_fork_child(fleet, slot) + + +def run_supervised(concurrency: int) -> None: + """Fork ``concurrency`` consumer children and supervise them until SIGTERM.""" + logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) + + fleet = _Fleet(concurrency) + stopping = threading.Event() + + def _signal_children(sig: int) -> None: + for _slot, pid in fleet.alive_items(): + with contextlib.suppress(ProcessLookupError): + os.kill(pid, sig) + + def _on_term(signum: int, _frame: object) -> None: + logger.info( + "PG-queue consumer supervisor: signal %s — stopping %d child(ren)", + signum, + fleet.alive_count(), + ) + stopping.set() + _signal_children(signal.SIGTERM) + + signal.signal(signal.SIGTERM, _on_term) + signal.signal(signal.SIGINT, _on_term) + + # Fork the initial fleet while single-threaded (before the liveness thread). + # A fork failure here is fatal + actionable rather than a half-started fleet. + for slot in range(concurrency): + if not _try_fork_child(fleet, slot): + stopping.set() + _signal_children(signal.SIGTERM) + _join_children(fleet, _SHUTDOWN_GRACE_SECONDS) + raise RuntimeError( + f"PG-queue consumer: os.fork() failed starting child {slot}/" + f"{concurrency} — reduce WORKER_PG_QUEUE_CONSUMER_CONCURRENCY or " + "raise the process/memory limit" + ) + + health = _maybe_start_supervisor_health(fleet) + try: + while not stopping.is_set(): + _reap_dead(fleet, stopping) + _restart_due_children(fleet, stopping) + stopping.wait(_MONITOR_INTERVAL_SECONDS) # responsive to SIGTERM + finally: + stopping.set() + _signal_children(signal.SIGTERM) + _join_children(fleet, _SHUTDOWN_GRACE_SECONDS) + if health is not None: + health.stop() + logger.info("PG-queue consumer supervisor: stopped") + + +def _wait_for_exit(pid: int, deadline: float) -> bool: + """Poll ``pid`` until it exits or ``deadline`` (monotonic) passes. True if it + exited (or was already reaped). + """ + while time.monotonic() < deadline: + try: + reaped, _status = os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + return True + if reaped != 0: + return True + time.sleep(0.1) + return False + + +def _join_children(fleet: _Fleet, grace_seconds: float) -> None: + """Wait up to ``grace_seconds`` *per child* for a graceful drain; SIGKILL + + reap any straggler. The budget is per-child (not a single shared deadline) so + one slow-draining child can't starve the others of their grace. + """ + for slot, pid in fleet.alive_items(): + if _wait_for_exit(pid, time.monotonic() + grace_seconds): + continue + logger.warning( + "PG-queue consumer: child slot=%s pid=%s did not stop in %ss — SIGKILL", + slot, + pid, + grace_seconds, + ) + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + with contextlib.suppress(ChildProcessError): + os.waitpid(pid, 0) + + +def _maybe_start_supervisor_health(fleet: _Fleet) -> LivenessServer | None: + """Start the fleet liveness server when a port is configured; else None. + + Reuses the single-process consumer's env knobs (``..._HEALTH_PORT`` / + ``..._HEALTH_STALE_SECONDS``) and the same HTTP contract (``/health`` → + 200/503), so the k8s probe config is unchanged. The JSON body differs + (``check="pg_queue_fleet"``, age key ``oldest_child_seconds_since_poll``) since + the freshness source is the fleet's oldest child, not one poll loop. + + A bind failure does not abort the consumer (it must keep draining the queue), + but ``EADDRINUSE`` usually signals a real config bug, so it's logged at error; + either way ``liveness_probe_bound: false`` is surfaced in the status payload so + the degradation is observable. + """ + from queue_backend.pg_queue.consumer import ( + _DEFAULT_HEALTH_STALE_SECONDS, + consumer_env, + ) + from queue_backend.pg_queue.liveness import LivenessServer + + port: int | None = consumer_env("HEALTH_PORT", None, int) + if port is None: + logger.info("PG-queue consumer supervisor: HEALTH_PORT unset — liveness disabled") + return None + stale_after = consumer_env( + "HEALTH_STALE_SECONDS", _DEFAULT_HEALTH_STALE_SECONDS, float + ) + + def _extra_status() -> dict[str, object]: + return { + "alive_children": fleet.alive_count(), + "concurrency": fleet.concurrency, + "crash_looping": fleet.is_crash_looping(), + "liveness_probe_bound": True, + } + + server = LivenessServer( + freshness_fn=fleet.freshness, + stale_after=stale_after, + port=port, + check_name="pg_queue_fleet", + age_key="oldest_child_seconds_since_poll", + extra_status_fn=_extra_status, + thread_name="pg-supervisor-liveness", + log_label="pg-queue supervisor", + ) + try: + server.start() + except OSError as exc: + import errno + + level = logging.ERROR if exc.errno == errno.EADDRINUSE else logging.WARNING + logger.log( + level, + "PG-queue consumer supervisor: liveness could not bind :%s (%s) — " + "continuing WITHOUT a probe", + port, + exc.strerror or exc, + exc_info=True, + ) + return None + logger.info( + "PG-queue consumer supervisor: fleet liveness on :%s/health (stale after " + "%ss, %d children)", + server.bound_port, + stale_after, + fleet.concurrency, + ) + return server diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py index 50af3efea7..12081fd647 100644 --- a/workers/queue_backend/pg_queue/consumer.py +++ b/workers/queue_backend/pg_queue/consumer.py @@ -427,36 +427,53 @@ def _parse_queue_list(raw: str) -> list[str]: return queues -def main() -> None: - logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) +def consumer_env(suffix: str, default: _T, cast: Callable[[str], _T]) -> _T: + """Read ``WORKER_PG_QUEUE_CONSUMER_`` with a typed default. + + Preserves the default's type through to PgQueueConsumer's typed ``__init__`` + (a bare ``type`` would erase it). On a bad value, fail with the offending var + name instead of a context-free ``int('abc')`` ValueError. Treats empty-string + as unset (an empty HEALTH_PORT must hit the clean opt-out, not ``int("")``). + Module-level (not nested in ``main``) so the prefork supervisor reads the same + knobs the single-process path does. + """ + var = f"WORKER_PG_QUEUE_CONSUMER_{suffix}" + raw = os.getenv(var) + if raw is None or raw == "": + return default + try: + return cast(raw) + except (ValueError, TypeError) as exc: + raise ValueError(f"Invalid {var}={raw!r}: {exc}") from exc - def _env(suffix: str, default: _T, cast: Callable[[str], _T]) -> _T: - # Preserve the default's type through to PgQueueConsumer's typed - # __init__ (a bare `type` would erase it). On a bad value, fail with the - # offending var name instead of a context-free `int('abc')` ValueError. - var = f"WORKER_PG_QUEUE_CONSUMER_{suffix}" - raw = os.getenv(var) - # Treat empty-string as unset: an empty HEALTH_PORT (e.g. a run-worker.sh - # fallback resolving empty) must hit the clean opt-out, not int("") crash. - if raw is None or raw == "": - return default - try: - return cast(raw) - except (ValueError, TypeError) as exc: - raise ValueError(f"Invalid {var}={raw!r}: {exc}") from exc - - consumer = PgQueueConsumer( - queue_names=_env("QUEUE", [_DEFAULT_QUEUE], _parse_queue_list), - batch_size=_env("BATCH", _DEFAULT_BATCH, int), - vt_seconds=_env("VT_SECONDS", _DEFAULT_VT_SECONDS, int), - poll_interval=_env("POLL_INTERVAL", _DEFAULT_POLL_INTERVAL, float), - backoff_max=_env("BACKOFF_MAX", _DEFAULT_BACKOFF_MAX, float), - max_attempts=_env("MAX_ATTEMPTS", _DEFAULT_MAX_ATTEMPTS, int), + +def build_consumer_from_env() -> PgQueueConsumer: + """Construct a :class:`PgQueueConsumer` from the ``WORKER_PG_QUEUE_CONSUMER_*`` + env. Shared by ``main`` (single process) and the prefork supervisor's children, + so every consumer instance is configured identically. + """ + return PgQueueConsumer( + queue_names=consumer_env("QUEUE", [_DEFAULT_QUEUE], _parse_queue_list), + batch_size=consumer_env("BATCH", _DEFAULT_BATCH, int), + vt_seconds=consumer_env("VT_SECONDS", _DEFAULT_VT_SECONDS, int), + poll_interval=consumer_env("POLL_INTERVAL", _DEFAULT_POLL_INTERVAL, float), + backoff_max=consumer_env("BACKOFF_MAX", _DEFAULT_BACKOFF_MAX, float), + max_attempts=consumer_env("MAX_ATTEMPTS", _DEFAULT_MAX_ATTEMPTS, int), ) + + +def main() -> None: + logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) + consumer = build_consumer_from_env() + # Annotate the int|None result so the intent ("a port, or None when unset") is + # declared rather than recovered from the generic widening over default=None. + port: int | None = consumer_env("HEALTH_PORT", None, int) health_server = _maybe_start_health_server( consumer, - port=_env("HEALTH_PORT", None, int), - stale_after=_env("HEALTH_STALE_SECONDS", _DEFAULT_HEALTH_STALE_SECONDS, float), + port=port, + stale_after=consumer_env( + "HEALTH_STALE_SECONDS", _DEFAULT_HEALTH_STALE_SECONDS, float + ), ) try: consumer.run() diff --git a/workers/sample.env b/workers/sample.env index 0cf9fb6b2a..d5564536ce 100644 --- a/workers/sample.env +++ b/workers/sample.env @@ -481,3 +481,12 @@ PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python # `pg_queue_enabled` on. Keep False until the worker-pg-executor consumer is # running in the fleet; set False for instant rollback. PG_QUEUE_TRANSPORT_ENABLED=False + +# PG-queue consumer prefork concurrency (UN-3606). >1 makes the consumer launcher +# fork N isolated consumer children — the PG analogue of Celery's prefork +# --concurrency, so file batches run in parallel. SKIP LOCKED distributes work +# across children AND replicas; a single execution is still capped by +# MAX_PARALLEL_FILE_BATCHES, and total live parallelism = concurrency x replicas +# (k8s HPA scales replicas). 1 = single process (default). Set per consumer; for +# the file-processing consumer it mirrors WORKER_FILE_PROCESSING_CONCURRENCY. +# WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=1 diff --git a/workers/tests/test_pg_consumer_supervisor.py b/workers/tests/test_pg_consumer_supervisor.py new file mode 100644 index 0000000000..c5a1912ca7 --- /dev/null +++ b/workers/tests/test_pg_consumer_supervisor.py @@ -0,0 +1,276 @@ +"""Tests for the PG-queue consumer prefork supervisor (UN-3606). + +DB-free / fork-free: the env knob, the ``_Fleet`` bookkeeping, the reap/restart +scheduling, the join+SIGKILL escalation and the fork/health guards are exercised in +isolation (``os.*`` mocked) so the suite stays fast and deterministic — no real +children, no worker bootstrap, no signals installed. +""" + +import errno +import threading +import time +from unittest.mock import MagicMock, patch + +import pytest + +from pg_queue_consumer import supervisor as sup +from pg_queue_consumer.supervisor import ( + _CRASH_LOOP_THRESHOLD, + _MAX_CONCURRENCY, + _MIN_HEALTHY_UPTIME_SECONDS, + _Fleet, + _child_after_fork, + _join_children, + _reap_dead, + _restart_due_children, + _try_fork_child, + _wait_for_exit, + concurrency_from_env, +) + +_MOD = "pg_queue_consumer.supervisor" +_ENV = "WORKER_PG_QUEUE_CONSUMER_CONCURRENCY" + + +class TestConcurrencyFromEnv: + def test_unset_defaults_to_one(self, monkeypatch): + monkeypatch.delenv(_ENV, raising=False) + assert concurrency_from_env() == 1 + + def test_empty_defaults_to_one(self, monkeypatch): + monkeypatch.setenv(_ENV, "") + assert concurrency_from_env() == 1 + + def test_valid_value(self, monkeypatch): + monkeypatch.setenv(_ENV, "4") + assert concurrency_from_env() == 4 + + def test_clamped_to_max(self, monkeypatch): + monkeypatch.setenv(_ENV, str(_MAX_CONCURRENCY + 50)) + assert concurrency_from_env() == _MAX_CONCURRENCY + + @pytest.mark.parametrize("bad", ["0", "-3"]) + def test_below_one_raises(self, monkeypatch, bad): + monkeypatch.setenv(_ENV, bad) + with pytest.raises(ValueError, match=">= 1"): + concurrency_from_env() + + def test_non_int_raises(self, monkeypatch): + monkeypatch.setenv(_ENV, "abc") + with pytest.raises(ValueError, match="Invalid"): + concurrency_from_env() + + +class TestFleet: + def test_record_and_reap_keep_structures_consistent(self): + f = _Fleet(2) + f.record_fork(0, 111) + assert f.alive_items() == [(0, 111)] and f.alive_count() == 1 + uptime = f.reap(0) + assert uptime >= 0 # a non-negative monotonic duration + assert f.alive_count() == 0 # pid + last_fork dropped together + + def test_slot_out_of_range_raises(self): + f = _Fleet(2) + with pytest.raises(IndexError): + f.record_fork(5, 111) + + def test_record_fork_does_not_reseed_heartbeat(self): + # The crash-loop fix: a re-fork must NOT refresh the slot, or a child that + # never polls looks perpetually fresh. + f = _Fleet(1) + f._heartbeats[0] = time.time() - 500 # an aged slot + f.record_fork(0, 111) + assert f.oldest_age() > 400 # still aged, not reset to ~0 + + def test_immediate_crash_increments_then_loops(self): + f = _Fleet(1) + for i in range(1, _CRASH_LOOP_THRESHOLD + 1): + n = f.schedule_restart(0, uptime=0.1) # died immediately + assert n == i + assert f.is_crash_looping() is True + + def test_healthy_uptime_resets_crash_counter(self): + f = _Fleet(1) + f.schedule_restart(0, uptime=0.1) + f.schedule_restart(0, uptime=0.1) + assert f.consecutive_crashes(0) == 2 + n = f.schedule_restart(0, uptime=_MIN_HEALTHY_UPTIME_SECONDS + 1) + assert n == 0 and f.is_crash_looping() is False + + def test_freshness_is_inf_when_crash_looping(self): + import math + + f = _Fleet(1) + for _ in range(_CRASH_LOOP_THRESHOLD): + f.schedule_restart(0, uptime=0.1) + assert math.isinf(f.freshness()) + + def test_freshness_is_oldest_age_when_healthy(self): + f = _Fleet(2) + f._heartbeats[1] = time.time() - 100 + assert 99 < f.freshness() < 102 + + def test_due_restarts_respects_backoff(self, monkeypatch): + f = _Fleet(1) + clock = [1000.0] + monkeypatch.setattr(f"{_MOD}.time.monotonic", lambda: clock[0]) + f.schedule_restart(0, uptime=0.1) # backoff scheduled in the future + assert f.due_restarts() == [] # not yet due + clock[0] += 100.0 # well past any backoff + assert f.due_restarts() == [0] + + +class TestReapDead: + def test_dead_child_reaped_and_rescheduled(self): + f = _Fleet(1) + f.record_fork(0, 111) + stopping = threading.Event() + with patch(f"{_MOD}.os.waitpid", return_value=(111, 0)): # exited + _reap_dead(f, stopping) + assert f.alive_count() == 0 + assert f.due_restarts() in ([], [0]) # scheduled (maybe backed off) + assert f.consecutive_crashes(0) >= 1 # counted (immediate death) + + def test_live_child_left_alone(self): + f = _Fleet(1) + f.record_fork(0, 111) + with patch(f"{_MOD}.os.waitpid", return_value=(0, 0)): # alive + _reap_dead(f, threading.Event()) + assert f.alive_items() == [(0, 111)] + + def test_not_rescheduled_while_stopping(self): + f = _Fleet(1) + f.record_fork(0, 111) + stopping = threading.Event() + stopping.set() + with patch(f"{_MOD}.os.waitpid", return_value=(111, 0)): + _reap_dead(f, stopping) + assert f.alive_count() == 0 # reaped + assert f.due_restarts() == [] # but NOT scheduled for restart + + def test_already_reaped_is_treated_gone(self): + f = _Fleet(1) + f.record_fork(0, 111) + with patch(f"{_MOD}.os.waitpid", side_effect=ChildProcessError()): + _reap_dead(f, threading.Event()) + assert f.alive_count() == 0 + + +class TestRestartDueChildren: + def test_due_slot_is_reforked(self): + f = _Fleet(1) + f.schedule_restart(0, uptime=0.1) + with ( + patch(f"{_MOD}.time.monotonic", return_value=1e9), # everything due + patch(f"{_MOD}._try_fork_child") as fork, + ): + _restart_due_children(f, threading.Event()) + fork.assert_called_once_with(f, 0) + + def test_stopping_blocks_refork(self): + f = _Fleet(1) + f.schedule_restart(0, uptime=0.1) + stopping = threading.Event() + stopping.set() + with ( + patch(f"{_MOD}.time.monotonic", return_value=1e9), + patch(f"{_MOD}._try_fork_child") as fork, + ): + _restart_due_children(f, stopping) + fork.assert_not_called() # no fresh child spawned into shutdown + + +class TestTryForkChild: + def test_fork_oserror_returns_false_and_does_not_record(self): + f = _Fleet(1) + with patch(f"{_MOD}.os.fork", side_effect=OSError(errno.EAGAIN, "again")): + assert _try_fork_child(f, 0) is False + assert f.alive_count() == 0 # slot left for the next monitor tick + + def test_parent_records_child(self): + f = _Fleet(1) + with patch(f"{_MOD}.os.fork", return_value=222): # parent sees child pid + assert _try_fork_child(f, 0) is True + assert f.alive_items() == [(0, 222)] + + +class TestChildAfterFork: + def test_resets_signals_and_exits_zero_on_clean_run(self): + with ( + patch(f"{_MOD}.signal.signal") as sig, + patch(f"{_MOD}._run_child"), + patch(f"{_MOD}.os._exit", side_effect=SystemExit) as exit_, + ): + with pytest.raises(SystemExit): + _child_after_fork(0, MagicMock()) + # SIGTERM + SIGINT reset to default before running. + assert sig.call_count == 2 + exit_.assert_called_once_with(0) + + def test_hard_exits_one_when_run_raises(self): + with ( + patch(f"{_MOD}.signal.signal"), + patch(f"{_MOD}._run_child", side_effect=RuntimeError("boom")), + patch(f"{_MOD}.os._exit", side_effect=SystemExit) as exit_, + ): + with pytest.raises(SystemExit): + _child_after_fork(0, MagicMock()) + exit_.assert_called_once_with(1) + + +class TestWaitForExit: + def test_true_when_child_exits(self): + with patch(f"{_MOD}.os.waitpid", return_value=(111, 0)): + assert _wait_for_exit(111, time.monotonic() + 5) is True + + def test_true_when_already_reaped(self): + with patch(f"{_MOD}.os.waitpid", side_effect=ChildProcessError()): + assert _wait_for_exit(111, time.monotonic() + 5) is True + + def test_false_when_deadline_passes(self): + with patch(f"{_MOD}.os.waitpid", return_value=(0, 0)): # never exits + assert _wait_for_exit(111, time.monotonic() - 1) is False # already past + + +class TestJoinChildren: + def test_clean_exit_no_sigkill(self): + f = _Fleet(1) + f.record_fork(0, 111) + with ( + patch(f"{_MOD}._wait_for_exit", return_value=True), + patch(f"{_MOD}.os.kill") as kill, + ): + _join_children(f, grace_seconds=5) + kill.assert_not_called() + + def test_straggler_is_sigkilled_and_reaped(self): + f = _Fleet(1) + f.record_fork(0, 111) + with ( + patch(f"{_MOD}._wait_for_exit", return_value=False), # never drained + patch(f"{_MOD}.os.kill") as kill, + patch(f"{_MOD}.os.waitpid") as waitpid, + ): + _join_children(f, grace_seconds=5) + import signal as _signal + + kill.assert_called_once_with(111, _signal.SIGKILL) + waitpid.assert_called_once_with(111, 0) + + +class TestSupervisorHealth: + def test_no_port_returns_none(self, monkeypatch): + monkeypatch.delenv("WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT", raising=False) + assert sup._maybe_start_supervisor_health(_Fleet(1)) is None + + def test_bind_error_is_swallowed(self, monkeypatch): + monkeypatch.setenv("WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT", "8090") + server = MagicMock() + server.start.side_effect = OSError(errno.EADDRINUSE, "in use") + # LivenessServer is imported locally inside the function → patch its source. + with patch( + "queue_backend.pg_queue.liveness.LivenessServer", return_value=server + ): + # Must not propagate — the consumer keeps draining without a probe. + assert sup._maybe_start_supervisor_health(_Fleet(1)) is None From 46c53d187ac09a1fe0d460da847957d7113cd49a Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Sun, 21 Jun 2026 13:01:03 +0530 Subject: [PATCH 31/44] =?UTF-8?q?UN-3608=20[FEAT]=20PG=20Queue=209i=20?= =?UTF-8?q?=E2=80=94=20executor=20async/callback=20path=20on=20PG=20(self-?= =?UTF-8?q?chained=20continuations)=20(#2097)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3608 [FEAT] PG Queue 9i — executor async/callback path on PG (self-chained continuations) Migrate the executor RPC's async/callback path off Celery onto the PG queue, completing the executor-transport migration (the blocking path landed in 9h-c). - dispatch_async / dispatch_with_callback now route PG-vs-Celery per call via the single pg_queue_enabled flag (was always Celery). Backend + workers. - §5 fire-and-forget self-chaining: on_success/on_error Celery Signatures are translated to serialisable ContinuationSpecs carried in the payload; after the executor runs execute_extraction the consumer self-chains the matching continuation onto the callback queue (result prepended on success, dispatch task_id on error), acking regardless to avoid an LLM double-spend. - New gated worker-pg-ide-callback consumer drains the ide_callback queue (Prompt Studio run/index/extract, lookups). Compose profile pg-queue. - ContinuationSpec + on_success/on_error/task_id added to the shared TaskPayload wire contract (unstract.core). Zero-regression: every new path is gated and fails closed to Celery; gate OFF is byte-identical to the prior Celery behaviour. Call sites unchanged — they keep passing Celery Signatures; the dispatcher translates only on the PG branch. Tests: payload set/unset, signature->spec translation, consumer self-chain success/error + enqueue-failure guard, routing gate (backend + workers). Co-Authored-By: Claude Opus 4.8 * UN-3608 [REFACTOR] lift transport-agnostic dispatch helpers to unstract.core Cuts the new-code duplication SonarCloud flagged on the backend/workers executor_rpc.py mirror by moving the two genuinely transport-agnostic pieces of the async/callback path into unstract.core (alongside ContinuationSpec): - DispatchHandle (the .id-only AsyncResult duck-type) - signature_to_continuation (Celery Signature -> ContinuationSpec) Both mirrors now import them instead of redefining; the duplicated TestSignatureToSpec is removed from both test suites in favour of one shared test. The dispatch_async/dispatch_with_callback method bodies stay mirrored (they genuinely differ by transport — Django ORM vs psycopg2); retiring that residue is the separate shared-package follow-up (UN-3607). No behaviour change: pure code movement, gate and contract unchanged. Co-Authored-By: Claude Opus 4.8 * UN-3608 [FIX] address review — on_error on drop branches, error text, json-safety, guards Toolkit review on #2097: - Critical: chain on_error on the early-drop branches too (malformed / poison / unknown task), via a shared _fail_dispatch — a dispatch_with_callback failure always reaches its on_error callback, not only when the task body raised. - Carry the real error text to on_error via callback_kwargs['error'] (PG has no Celery AsyncResult for the callback to recover it from); error callbacks prefer it. - JSON-coerce the self-chained prepend (UUID/datetime in an executor result no longer makes client.send's plain json.dumps raise and silently drop the callback). - Enforce the reply_key XOR on_success/on_error invariant in to_payload + enqueue_task. - _json_safe the continuation specs in the backend producer. - signature_to_continuation: guard task name + reject positional-arg callbacks. - Restore ContinuationSpec/TaskPayload types at the consumer boundary; drop the over-broad except in _continuation_org; document the returned-failure->on_success parity; reword the ContinuationSpec doc and the compose "Dark" comment. Tests for every fix. Backend 35, workers 76 green. Co-Authored-By: Claude Opus 4.8 * UN-3608 [FIX] review round 2 — on_success-only redelivery, success-enqueue fallback, ide-callback vt - Correctness: broaden the failure guard to `reply_key or on_success or on_error` so an on_success-only callback dispatch that raises ACKs instead of falling through to vt-redelivery (which would re-run the executor / re-spend LLM tokens). - Silent failure: _chain_continuation now returns whether it enqueued; on a success-path enqueue failure, fall back to on_error so the HTTP-202 caller gets a terminal event instead of hanging. Failure log carries run_id/task_id/org. - ide_callback _get_task_error: `if explicit is not None` (don't discard an explicit "" error) — greptile. - worker-pg-ide-callback: set VT_SECONDS=120 / HEALTH_STALE=180 (overridable) so a slow internal-API write can't be re-claimed mid-run and double-emit — greptile. Tests: on_success-only-raise acks (no redelivery); success-enqueue-failure falls back to on_error. Workers self-chain + ide_callback green. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- backend/pg_queue/executor_rpc.py | 109 ++++++++++- backend/pg_queue/producer.py | 29 ++- backend/pg_queue/tests/test_executor_rpc.py | 81 +++++++- backend/pg_queue/tests/test_producer.py | 30 +++ docker/docker-compose.yaml | 45 +++++ .../core/src/unstract/core/data_models.py | 33 ++++ .../src/unstract/core/execution_dispatch.py | 75 ++++++++ workers/ide_callback/tasks.py | 30 ++- workers/queue_backend/pg_queue/consumer.py | 180 ++++++++++++++++-- .../queue_backend/pg_queue/executor_rpc.py | 126 ++++++++++-- .../queue_backend/pg_queue/task_payload.py | 29 ++- workers/tests/test_executor_rpc.py | 154 ++++++++++++++- workers/tests/test_pg_queue_consumer.py | 165 ++++++++++++++++ 13 files changed, 1036 insertions(+), 50 deletions(-) create mode 100644 unstract/core/src/unstract/core/execution_dispatch.py diff --git a/backend/pg_queue/executor_rpc.py b/backend/pg_queue/executor_rpc.py index 35615671b6..72c0895a35 100644 --- a/backend/pg_queue/executor_rpc.py +++ b/backend/pg_queue/executor_rpc.py @@ -42,6 +42,7 @@ from pg_queue.models import PgTaskResult from pg_queue.producer import enqueue_task from unstract.core.data_models import PgTaskStatus +from unstract.core.execution_dispatch import DispatchHandle, signature_to_continuation from unstract.flags.feature_flag import check_feature_flag_status from unstract.sdk1.execution.dispatcher import ExecutionDispatcher from unstract.sdk1.execution.result import ExecutionResult @@ -186,6 +187,80 @@ def dispatch( ) return ExecutionResult.failure(error=row.error or "executor task failed") + def dispatch_async( + self, context: ExecutionContext, headers: dict[str, Any] | None = None + ) -> str: + """Fire-and-forget enqueue of ``execute_extraction``; returns the task id. + + The PG analogue of the SDK ``dispatch_async``: no ``reply_key``, no + callback, no blocking. There is no PG ``AsyncResult`` backend, so a caller + that needs the outcome uses :meth:`dispatch_with_callback` (a self-chained + continuation), not polling on this id. ``headers`` is accepted and ignored + (PG carries routing in the payload). Enqueue failures propagate — parity + with the SDK, which lets a broker error out of ``dispatch_async``. + """ + task_id = str(uuid.uuid4()) + queue = f"{_QUEUE_PREFIX}{context.executor_name}" + org = getattr(context, "organization_id", "") or "" + enqueue_task( + task_name=_EXECUTE_TASK, + queue=queue, + args=[context.to_dict()], + org_id=str(org), + task_id=task_id, + ) + logger.info( + "PG executor dispatch_async: enqueued task_id=%s queue=%s run_id=%s", + task_id, + queue, + context.run_id, + ) + return task_id + + def dispatch_with_callback( + self, + context: ExecutionContext, + on_success: Any | None = None, + on_error: Any | None = None, + task_id: str | None = None, + headers: dict[str, Any] | None = None, + ) -> DispatchHandle: + """Fire-and-forget enqueue with self-chained callbacks (§5 model). + + The PG analogue of the SDK ``dispatch_with_callback``: instead of Celery + ``link`` / ``link_error`` (which the broker fires), the on-success / + on-error Celery ``Signature``s are translated to serialisable + :class:`ContinuationSpec`s and carried in the payload. After the executor + consumer runs ``execute_extraction`` it self-chains the matching + continuation onto the callback queue. Returns a :class:`DispatchHandle` + exposing ``.id`` (== ``task_id``) so call sites read the task id exactly + as on the Celery path. ``headers`` is accepted and ignored. + """ + task_id = task_id or str(uuid.uuid4()) + queue = f"{_QUEUE_PREFIX}{context.executor_name}" + org = getattr(context, "organization_id", "") or "" + success_spec = signature_to_continuation(on_success) + error_spec = signature_to_continuation(on_error) + enqueue_task( + task_name=_EXECUTE_TASK, + queue=queue, + args=[context.to_dict()], + org_id=str(org), + on_success=success_spec, + on_error=error_spec, + task_id=task_id, + ) + logger.info( + "PG executor dispatch_with_callback: enqueued task_id=%s queue=%s " + "run_id=%s on_success=%s on_error=%s", + task_id, + queue, + context.run_id, + success_spec["task_name"] if success_spec else None, + error_spec["task_name"] if error_spec else None, + ) + return DispatchHandle(task_id) + @staticmethod def _wait_for_result(reply_key: str, timeout: float) -> PgTaskResult | None: """Poll ``pg_task_result`` until the row appears or *timeout* elapses. @@ -216,10 +291,10 @@ def _wait_for_result(reply_key: str, timeout: float) -> PgTaskResult | None: class RoutingExecutionDispatcher: """Gate-routed executor dispatcher returned by ``_get_dispatcher()``. - ``dispatch()`` chooses PG vs Celery per call (instant rollout/rollback); - ``dispatch_async`` / ``dispatch_with_callback`` always delegate to Celery — - the async/callback path stays on Celery until a later continuation slice. - Duck-typed against the SDK ``ExecutionDispatcher`` so call sites are unchanged. + Every mode chooses PG vs Celery per call (instant rollout/rollback): + ``dispatch()`` (request-reply), ``dispatch_async`` (fire-and-forget) and + ``dispatch_with_callback`` (self-chained callbacks). Duck-typed against the SDK + ``ExecutionDispatcher`` so call sites are unchanged. """ def __init__(self, celery_app: object | None = None) -> None: @@ -246,10 +321,32 @@ def dispatch( def dispatch_async( self, context: ExecutionContext, headers: dict[str, Any] | None = None ) -> str: + if resolve_executor_transport(context): + return self._pg.dispatch_async(context) return self._celery.dispatch_async(context, headers=headers) - def dispatch_with_callback(self, context: ExecutionContext, **kwargs: Any) -> Any: - return self._celery.dispatch_with_callback(context, **kwargs) + def dispatch_with_callback( + self, + context: ExecutionContext, + on_success: Any | None = None, + on_error: Any | None = None, + task_id: str | None = None, + headers: dict[str, Any] | None = None, + ) -> Any: + if resolve_executor_transport(context): + return self._pg.dispatch_with_callback( + context, + on_success=on_success, + on_error=on_error, + task_id=task_id, + ) + return self._celery.dispatch_with_callback( + context, + on_success=on_success, + on_error=on_error, + task_id=task_id, + headers=headers, + ) def get_executor_dispatcher( diff --git a/backend/pg_queue/producer.py b/backend/pg_queue/producer.py index f258b8a89a..1056d44476 100644 --- a/backend/pg_queue/producer.py +++ b/backend/pg_queue/producer.py @@ -25,6 +25,7 @@ FAIRNESS_DEFAULT_PRIORITY, FAIRNESS_MAX_PRIORITY, FAIRNESS_MIN_PRIORITY, + ContinuationSpec, FairnessPayload, TaskPayload, ) @@ -63,6 +64,9 @@ def enqueue_task( priority: int = DEFAULT_PRIORITY, fairness: FairnessPayload | None = None, reply_key: str | None = None, + on_success: ContinuationSpec | None = None, + on_error: ContinuationSpec | None = None, + task_id: str | None = None, ) -> int: """Enqueue a task onto the PG queue; returns the new ``msg_id``. @@ -74,7 +78,19 @@ def enqueue_task( ``reply_key`` marks a **request-reply** dispatch (the executor RPC on PG): the executor consumer writes the task's result/error to ``pg_task_result`` under it for the blocking caller to poll. Omitted = fire-and-forget. + + ``on_success`` / ``on_error`` mark an **async/callback** dispatch + (``dispatch_with_callback``): the executor consumer self-chains the matching + continuation after the task runs. ``task_id`` is the dispatch id prepended to + ``on_error`` as the failed id (Celery ``link_error`` parity). Mutually + exclusive with ``reply_key`` — passing both is rejected (the consumer checks + ``reply_key`` first and would silently drop the callback). """ + if reply_key is not None and (on_success is not None or on_error is not None): + raise ValueError( + "reply_key (request-reply) and on_success/on_error (callback) are " + "mutually exclusive" + ) if not FAIRNESS_MIN_PRIORITY <= priority <= FAIRNESS_MAX_PRIORITY: raise ValueError( f"priority out of range " @@ -88,10 +104,19 @@ def enqueue_task( "queue": pg_queue, "fairness": fairness, } - # Only set for request-reply dispatches — keeps fire-and-forget rows - # byte-identical to before this field existed. + # Each optional key is set only when present — keeps fire-and-forget rows + # byte-identical to before these fields existed. if reply_key is not None: message["reply_key"] = reply_key + # Continuation specs carry a nested callback ``kwargs`` dict that may hold a + # UUID/datetime — coerce like ``args``/``kwargs`` above, else the JSONField + # insert raises at dispatch time (caller-visible). + if on_success is not None: + message["on_success"] = _json_safe(on_success) + if on_error is not None: + message["on_error"] = _json_safe(on_error) + if task_id is not None: + message["task_id"] = task_id # Mirror the worker _enqueue_pg path: log the failure with breadcrumbs before # it propagates, so a DB/constraint/serialization error isn't mislabeled by # the caller's broad handler. diff --git a/backend/pg_queue/tests/test_executor_rpc.py b/backend/pg_queue/tests/test_executor_rpc.py index b91f923377..7f5a2ed84f 100644 --- a/backend/pg_queue/tests/test_executor_rpc.py +++ b/backend/pg_queue/tests/test_executor_rpc.py @@ -221,11 +221,82 @@ def test_gate_on_dispatch_uses_pg(self): pg.dispatch.assert_called_once() celery.dispatch.assert_not_called() - def test_async_and_callback_always_celery(self): - """The callback/async path stays on Celery regardless of the gate (a later slice).""" + def test_async_and_callback_stay_celery_when_gate_off(self): + """Zero-regression: gate off → async/callback delegate to Celery unchanged.""" dispatcher, celery, pg = self._build() - dispatcher.dispatch_async(_ctx()) - dispatcher.dispatch_with_callback(_ctx(), on_success=None) + with patch(f"{_MOD}.resolve_executor_transport", return_value=False): + dispatcher.dispatch_async(_ctx(), headers={"h": 1}) + dispatcher.dispatch_with_callback(_ctx(), on_success="s", on_error="e") celery.dispatch_async.assert_called_once() celery.dispatch_with_callback.assert_called_once() - pg.dispatch.assert_not_called() + pg.dispatch_async.assert_not_called() + pg.dispatch_with_callback.assert_not_called() + + def test_async_and_callback_route_to_pg_when_gated(self): + """Gate on (③c) → async/callback take the PG self-chained path.""" + dispatcher, celery, pg = self._build() + with patch(f"{_MOD}.resolve_executor_transport", return_value=True): + dispatcher.dispatch_async(_ctx()) + dispatcher.dispatch_with_callback( + _ctx(), on_success="s", on_error="e", task_id="t" + ) + pg.dispatch_async.assert_called_once() + pg.dispatch_with_callback.assert_called_once() + assert "headers" not in pg.dispatch_with_callback.call_args.kwargs + celery.dispatch_async.assert_not_called() + celery.dispatch_with_callback.assert_not_called() + + +class TestPgAsyncCallbackWiring: + """PG fire-and-forget + self-chained-callback enqueue shapes (``enqueue_task`` + mocked). Pins that the async path carries NO reply_key and the callback path + carries the translated continuations + the tracking task_id. + """ + + @staticmethod + def _ctx(): + c = MagicMock() + c.executor_name = "legacy" + c.run_id = "r" + c.organization_id = "org9" + c.to_dict.return_value = {"run_id": "r", "organization_id": "org9"} + return c + + def test_dispatch_async_is_fire_and_forget(self): + with patch(f"{_MOD}.enqueue_task") as enq: + task_id = PgExecutionDispatcher().dispatch_async(self._ctx()) + kwargs = enq.call_args.kwargs + assert kwargs["task_name"] == "execute_extraction" + assert kwargs["queue"] == "celery_executor_legacy" + assert kwargs["org_id"] == "org9" + assert kwargs["task_id"] == task_id + assert "reply_key" not in kwargs + assert "on_success" not in kwargs + + def test_dispatch_with_callback_carries_continuations(self): + on_s = MagicMock( + task="ide_prompt_complete", + args=(), + kwargs={"callback_kwargs": {"room": "r1"}}, + options={"queue": "ide_callback"}, + ) + on_e = MagicMock( + task="ide_prompt_error", + args=(), + kwargs={"callback_kwargs": {"room": "r1"}}, + options={"queue": "ide_callback"}, + ) + with patch(f"{_MOD}.enqueue_task") as enq: + handle = PgExecutionDispatcher().dispatch_with_callback( + self._ctx(), on_success=on_s, on_error=on_e, task_id="tid-7" + ) + assert handle.id == "tid-7" # call sites read .id off the handle + kwargs = enq.call_args.kwargs + assert kwargs["on_success"] == { + "task_name": "ide_prompt_complete", + "kwargs": {"callback_kwargs": {"room": "r1"}}, + "queue": "ide_callback", + } + assert kwargs["on_error"]["task_name"] == "ide_prompt_error" + assert kwargs["task_id"] == "tid-7" + assert "reply_key" not in kwargs diff --git a/backend/pg_queue/tests/test_producer.py b/backend/pg_queue/tests/test_producer.py index 1d62f3b11e..c09466f2a6 100644 --- a/backend/pg_queue/tests/test_producer.py +++ b/backend/pg_queue/tests/test_producer.py @@ -111,3 +111,33 @@ def test_enqueue_failure_logs_and_propagates(self): model.objects.create.side_effect = RuntimeError("db down") with pytest.raises(RuntimeError): producer.enqueue_task(task_name="t", queue="celery") + + def test_reply_key_and_callback_mutually_exclusive(self): + spec = {"task_name": "cb", "kwargs": {}, "queue": "ide_callback"} + with pytest.raises(ValueError, match="mutually exclusive"): + producer.enqueue_task( + task_name="execute_extraction", + queue="celery_executor_legacy", + reply_key="rk", + on_success=spec, + ) + + def test_continuation_specs_are_json_coerced(self): + # A callback's kwargs can carry a UUID/datetime → must be coerced like + # args/kwargs, else the JSONField insert raises at dispatch (caller-visible). + uid = uuid.UUID("ebed2834-c9fb-4b6c-8df3-9dd841f616bb") + spec = { + "task_name": "ide_prompt_complete", + "kwargs": {"callback_kwargs": {"doc_id": uid}}, + "queue": "ide_callback", + } + with patch(_MODEL) as model: + model.objects.create.return_value = MagicMock(msg_id=1) + producer.enqueue_task( + task_name="execute_extraction", + queue="celery_executor_legacy", + on_success=spec, + task_id="t1", + ) + msg = model.objects.create.call_args.kwargs["message"] + assert msg["on_success"]["kwargs"]["callback_kwargs"]["doc_id"] == str(uid) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index a8f64a00e9..612e35ea8d 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -770,6 +770,51 @@ services: profiles: - pg-queue + # IDE callback consumer (③c) — drains the ``ide_callback`` queue the executor + # self-chains onto for async/callback dispatch (Prompt Studio run/index/extract, + # lookups). Registers the ide_callback worker's tasks (ide_prompt_complete, + # ide_index_complete, extraction_complete, …). Idle (receives no work) until + # dispatch_with_callback routes to PG (i.e. the gating flag is on); the Celery + # ``worker-ide-callback`` still serves the Celery path. No broker dependency: + # these callbacks only do API writes + ws emits, never an onward Celery dispatch. + # Lightweight, so a modest concurrency suffices; scale via replicas (SKIP LOCKED). + worker-pg-ide-callback: + image: unstract/worker-unified:${VERSION} + container_name: unstract-worker-pg-ide-callback + restart: unless-stopped + command: ["pg-queue-consumer"] + ports: + - "8100:8090" + env_file: + - ../workers/.env + - ./essentials.env + depends_on: + - db + - redis + - platform-service + environment: + - ENVIRONMENT=development + - APPLICATION_NAME=unstract-worker-pg-ide-callback + - WORKER_BARRIER_BACKEND=pg + - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=ide_callback + - WORKER_PG_QUEUE_CONSUMER_QUEUE=ide_callback + - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090 + - WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_IDE_CALLBACK_CONCURRENCY:-2} + # Callbacks are sub-second (ws emit + small API writes), so nowhere near the + # executor's 1h limit — but set the visibility timeout above a slow internal + # API write (backend under load) so a sibling replica can't re-claim mid-run + # and double-emit. Health-stale sits above it. Overridable per-env. + - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${PG_IDE_CALLBACK_VT_SECONDS:-120} + - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${PG_IDE_CALLBACK_HEALTH_STALE_SECONDS:-180} + labels: + - traefik.enable=false + volumes: + - ./workflow_data:/data + - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config + - prompt_studio_data:/app/prompt-studio-data + profiles: + - pg-queue + # Reaper / orchestrator — leader-elected loop. Run exactly ONE instance (it # elects a single leader via pg_orchestrator_lock; extra replicas idle as # standby). Besides barrier-orphan recovery it runs the PG scheduler tick diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py index cfcbd027e1..08ac61234f 100644 --- a/unstract/core/src/unstract/core/data_models.py +++ b/unstract/core/src/unstract/core/data_models.py @@ -298,6 +298,30 @@ class FairnessPayload(TypedDict): pipeline_priority: int +class ContinuationSpec(TypedDict): + """A self-chained continuation — the PG wire-form of a Celery ``Signature``. + + The async/callback executor path (``dispatch_with_callback``) attaches an + ``on_success`` / ``on_error`` callback. Celery carries these as ``link`` / + ``link_error`` signatures the broker fires automatically; PG has no such + broker mechanism, so the executor consumer **self-chains** instead — after it + runs ``execute_extraction`` it enqueues this continuation onto ``queue`` (the + §5 fire-and-forget model). ``task_name`` / ``kwargs`` / ``queue`` are exactly + the three fields read off a ``signature(task_name, kwargs=..., queue=...)`` so + the prompt-studio call sites keep passing Celery ``Signature``s unchanged and + the dispatcher translates them only on the PG branch. + + The consumer passes the chained value the callback expects (the executor + result dict on success, the dispatch ``task_id`` on error) as the callback's + first positional arg, alongside the spec's ``kwargs`` as a distinct mapping — + mirroring how Celery's ``link`` prepends the parent task's return value. + """ + + task_name: str + kwargs: dict[str, Any] + queue: str + + class TaskPayload(TypedDict): """The ``message`` JSONB of a task row in ``pg_queue_message``. @@ -331,6 +355,15 @@ class TaskPayload(TypedDict): queue: str | None fairness: FairnessPayload | None reply_key: NotRequired[str] + # Async/callback dispatch (``dispatch_with_callback``). Mutually exclusive with + # ``reply_key``: request-reply (blocking) sets ``reply_key``; fire-and-forget + # with callbacks sets these continuations. The executor consumer self-chains + # whichever applies after the task runs. ``task_id`` is the dispatch id the + # caller tracks (== Celery's task id); the consumer prepends it as the failed + # id to ``on_error`` (parity with Celery ``link_error``). + on_success: NotRequired[ContinuationSpec] + on_error: NotRequired[ContinuationSpec] + task_id: NotRequired[str] class PgTaskStatus(str, Enum): diff --git a/unstract/core/src/unstract/core/execution_dispatch.py b/unstract/core/src/unstract/core/execution_dispatch.py new file mode 100644 index 0000000000..57ddf02513 --- /dev/null +++ b/unstract/core/src/unstract/core/execution_dispatch.py @@ -0,0 +1,75 @@ +"""Transport-agnostic helpers for the executor-RPC dispatchers. + +The PG executor dispatch lives in two near-identical mirrors — backend (Django +ORM) and workers (psycopg2) ``pg_queue/executor_rpc.py`` — because neither +codebase can import the other and there is no shared home for the transport logic +yet (the SDK is transport-agnostic by contract, and ``unstract.sdk1`` → +``unstract.core`` would be circular). Retiring that mirror wholesale is tracked +as a follow-up (a dedicated shared executor-RPC package). + +These two helpers, however, are the genuinely **transport-agnostic** pieces of +the async/callback path: they have no Django / psycopg / SDK dependency. So they +live here in ``unstract.core`` (alongside :class:`ContinuationSpec`, the wire +type one of them produces) and BOTH mirrors import them — removing that slice of +the duplication today rather than waiting for the full package. +""" + +from __future__ import annotations + +from typing import Any + +from .data_models import ContinuationSpec + + +class DispatchHandle: + """Minimal duck-type of Celery ``AsyncResult`` for the PG callback path. + + ``dispatch_with_callback`` callers read only ``.id`` (to return the task id in + the HTTP 202 response); they must NOT call ``.get()`` — the result arrives via + the self-chained callback (WebSocket), not by polling here. Exposing just + ``.id`` lets a PG dispatch return the same shape the call sites already use. + """ + + __slots__ = ("id",) + + def __init__(self, task_id: str) -> None: + self.id = task_id + + +def signature_to_continuation(sig: Any | None) -> ContinuationSpec | None: + """Translate a Celery ``Signature`` to a serialisable continuation spec. + + Reads only the three attributes PG self-chaining needs — task name, kwargs, + target queue — so the prompt-studio call sites keep passing + ``signature(name, kwargs=..., queue=...)`` unchanged; only the PG branch + translates. ``None`` (no callback for that outcome) passes through. A signature + without a queue is a configuration error: PG routes by the row's queue and must + not silently default it, so we fail fast. + """ + if sig is None: + return None + task = getattr(sig, "task", None) + if not task: + raise ValueError( + "callback signature has no task name; PG self-chaining cannot enqueue " + "it (it would be dropped downstream as a malformed payload)" + ) + queue = (getattr(sig, "options", None) or {}).get("queue") + if not queue: + raise ValueError( + f"callback signature {task!r} has no queue; PG self-chaining routes " + "by the row's queue and cannot default it" + ) + # ContinuationSpec carries no positional args (the consumer prepends the one + # chained value). A signature with its own positional args would run with a + # different arg list on PG than on Celery — fail fast rather than drop them. + if getattr(sig, "args", None): + raise ValueError( + f"callback signature {task!r} has positional args; PG self-chaining " + "supports kwargs-only callbacks" + ) + return ContinuationSpec( + task_name=task, + kwargs=dict(getattr(sig, "kwargs", None) or {}), + queue=queue, + ) diff --git a/workers/ide_callback/tasks.py b/workers/ide_callback/tasks.py index 0902f4609b..e35b76f933 100644 --- a/workers/ide_callback/tasks.py +++ b/workers/ide_callback/tasks.py @@ -94,8 +94,20 @@ def _emit_event( ) -def _get_task_error(failed_task_id: str, default: str) -> str: - """Retrieve the error message from a failed Celery task's result backend.""" +def _get_task_error( + failed_task_id: str, default: str, explicit: str | None = None +) -> str: + """Resolve the error text for a failed-task callback. + + Prefers an ``explicit`` error when the caller already has it: the PG + self-chained path carries the real error via ``callback_kwargs['error']`` + because the executor ran eagerly (``task.apply``) and never wrote a Celery + result backend under ``failed_task_id`` — so the ``AsyncResult`` lookup below + is empty there. The Celery ``link_error`` path passes no explicit error and + falls back to the result backend, then the ``default``. + """ + if explicit is not None: + return explicit try: from celery.result import AsyncResult @@ -321,7 +333,9 @@ def ide_index_error( api = _get_api_client() try: - error_msg = _get_task_error(failed_task_id, default="Indexing failed") + error_msg = _get_task_error( + failed_task_id, default="Indexing failed", explicit=cb.get("error") + ) # Clean up the indexing-in-progress flag if doc_id_key: @@ -499,7 +513,9 @@ def ide_prompt_error( api = _get_api_client() try: - error_msg = _get_task_error(failed_task_id, default="Prompt execution failed") + error_msg = _get_task_error( + failed_task_id, default="Prompt execution failed", explicit=cb.get("error") + ) _emit_event( api, @@ -642,7 +658,11 @@ def extraction_error( # Context-manage clients to avoid per-task session leaks. with _get_extraction_client() as api, _get_api_client() as ps_api: try: - error_msg = _get_task_error(failed_task_id, default="Text extraction failed") + error_msg = _get_task_error( + failed_task_id, + default="Text extraction failed", + explicit=cb.get("error"), + ) api.mark_extraction_error( source=source, diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py index 12081fd647..5b75fe74ce 100644 --- a/workers/queue_backend/pg_queue/consumer.py +++ b/workers/queue_backend/pg_queue/consumer.py @@ -20,6 +20,7 @@ from __future__ import annotations +import json import logging import os import signal @@ -29,10 +30,13 @@ from celery import current_app +from unstract.core.data_models import ContinuationSpec, TaskPayload + from ..fairness import FAIRNESS_HEADER_NAME from .client import PgQueueClient from .liveness import LivenessServer as _BaseLivenessServer from .result_backend import PgResultBackend +from .task_payload import to_payload if TYPE_CHECKING: from celery import Celery @@ -66,6 +70,19 @@ _DEFAULT_HEALTH_STALE_SECONDS = 60.0 +def _json_safe(value: object) -> object: + """Round-trip through JSON with ``default=str`` so non-JSON-native values + (UUID / datetime) survive a self-chained enqueue. + + ``PgQueueClient.send`` serialises with a plain ``json.dumps`` (no + ``default=``), so a self-chained continuation whose prepended argument is an + executor result dict containing a UUID/datetime would raise ``TypeError`` — + swallowed by ``_chain_continuation`` and the callback (plus its user-facing + event) lost. Coercing here mirrors the backend producer's ``_json_safe``. + """ + return json.loads(json.dumps(value, default=str)) + + class PgQueueConsumer: """Polls one PG queue, runs each claimed task in-process, acks on success.""" @@ -164,6 +181,13 @@ def _handle(self, message: QueueMessage) -> None: # its full timeout). Present → store the outcome + ack after one attempt; # absent → fire-and-forget (the existing leaf/pipeline path). reply_key = payload.get("reply_key") + # Async/callback (dispatch_with_callback) self-chaining (③c): a callback + # to enqueue after the task runs (on_success after success, on_error after + # failure) — the PG analogue of Celery firing a link/link_error. Mutually + # exclusive with reply_key. ``task_id`` is the dispatch id prepended to the + # on_error callback as the failed id (Celery link_error parity). + on_success = payload.get("on_success") + on_error = payload.get("on_error") # Malformed / foreign payload: no task name → can't run; drop with a # log that points at the payload, not at task registration. @@ -174,7 +198,7 @@ def _handle(self, message: QueueMessage) -> None: message.msg_id, payload, ) - self._fail_reply(reply_key, "malformed message: missing task_name") + self._fail_dispatch(payload, error="malformed message: missing task_name") self._client.delete(message.msg_id) return @@ -191,8 +215,9 @@ def _handle(self, message: QueueMessage) -> None: message.read_ct, payload, ) - self._fail_reply( - reply_key, f"task {task_name} exceeded max_attempts={self.max_attempts}" + self._fail_dispatch( + payload, + error=f"task {task_name} exceeded max_attempts={self.max_attempts}", ) self._client.delete(message.msg_id) return @@ -205,7 +230,7 @@ def _handle(self, message: QueueMessage) -> None: task_name, message.msg_id, ) - self._fail_reply(reply_key, f"unknown task {task_name}") + self._fail_dispatch(payload, error=f"unknown task {task_name}") self._client.delete(message.msg_id) return @@ -221,22 +246,26 @@ def _handle(self, message: QueueMessage) -> None: throw=True, ) except Exception as exc: - if reply_key: - # Record the failure so the waiting caller gets a definitive - # result instead of blocking to its timeout, then ACK. We do NOT + if reply_key or on_success or on_error: + # Request-reply / async-callback dispatch: surface the failure on + # its return channel (store the error reply, or self-chain on_error + # carrying the real error text), then ACK regardless. We do NOT # vt-redeliver: a redelivery would race a result the caller may # already have consumed, and re-running the executor is costly # (LLM spend). The executor task's own autoretry covers transient - # errors within this attempt; the caller re-dispatches with a - # fresh reply_key to retry the whole RPC. - self._fail_reply(reply_key, f"{type(exc).__name__}: {exc}") + # errors within this attempt; the caller re-dispatches with a fresh + # handle to retry the whole RPC. ``on_success`` is in the guard too: + # an on_success-only callback dispatch (on_error omitted) that raises + # must still ACK — falling through to vt-redelivery would re-run the + # executor (the LLM double-spend this path exists to avoid). + # _fail_dispatch is best-effort, so the ack never wedges. + self._fail_dispatch(payload, error=f"{type(exc).__name__}: {exc}") self._client.delete(message.msg_id) # ack logger.exception( - "PG-queue consumer: request-reply task %r (msg_id=%s) failed " - "— stored error + acked (reply_key=%s)", + "PG-queue consumer: dispatch %r (msg_id=%s) failed — surfaced " + "via reply/on_error + acked", task_name, message.msg_id, - reply_key, ) return # Fire-and-forget: leave the row — its vt expires and it is @@ -267,6 +296,27 @@ def _handle(self, message: QueueMessage) -> None: message.msg_id, reply_key, ) + elif on_success: + # Async/callback: self-chain the success continuation onto the callback + # queue before the ack — the §5 hand-off (PG analogue of Celery's link). + # + # "success" == the task did not RAISE (parity with Celery `link`, which + # also fires on any non-exception return). A task that *returns* a failed + # ExecutionResult (success=False, no raise) deliberately follows this + # path — the on_success callback receives the failed result and renders + # it — exactly as on Celery. This is NOT the missing-on_error drop bug + # the early-drop branches handle. + # + # If the success hand-off can't be enqueued (transport/DB error, bad + # queue), fall back to on_error so the HTTP-202 caller still gets a + # terminal event instead of hanging after the LLM spend already + # happened. Both are best-effort + still ack (no executor re-run). + if not self._chain_continuation( + on_success, prepend=eager.result, payload=payload + ): + self._fail_dispatch( + payload, error="result delivery failed; see worker logs" + ) if not self._client.delete(message.msg_id): # ack logger.warning( @@ -289,6 +339,110 @@ def _store_reply( self._result_backend = PgResultBackend() self._result_backend.store_result(reply_key, result=result, error=error) + def _chain_continuation( + self, + spec: ContinuationSpec, + *, + prepend: object, + payload: TaskPayload, + error: str | None = None, + ) -> bool: + """Enqueue a self-chained callback continuation (best-effort, never raises). + + The PG analogue of Celery firing a ``link`` / ``link_error``: after the + executor task runs, enqueue ``spec`` (``task_name`` + ``kwargs`` + + ``queue``) onto its queue with ``prepend`` as the first positional arg — + the executor result dict on success, the dispatch ``task_id`` on error — + exactly the first parameter the callback signature expects (mirroring + Celery prepending the parent task's return value). The prepended value is + JSON-coerced so a UUID/datetime in an executor result can't make the + enqueue's ``json.dumps`` raise and silently drop the callback. + + ``error`` (failure path only): the real error text. On PG there is no + Celery ``AsyncResult`` for the on_error callback (e.g. ``ide_prompt_error``) + to recover the message from — the executor ran eagerly — so we hand it + through ``callback_kwargs['error']``, which those callbacks prefer over the + empty ``AsyncResult`` lookup. Absent on the success path. + + Returns ``True`` if the continuation was enqueued, ``False`` if it failed. + Never raises: a failure here must not wedge the executor message's ack + (which is taken regardless, to avoid an expensive re-run). The success path + uses the return value to fall back to on_error so the caller still gets a + terminal event; the failure path can't recover further (the callback — and + its user-facing WebSocket event — is lost, logged loud with the run/task/org + so the stranded session is correlatable). + """ + try: + queue = spec["queue"] + kwargs = dict(spec.get("kwargs") or {}) + if error is not None: + cb = dict(kwargs.get("callback_kwargs") or {}) + cb.setdefault("error", error) + kwargs["callback_kwargs"] = cb + self._client.send( + queue, + to_payload( + spec["task_name"], + args=[_json_safe(prepend)], + kwargs=kwargs, + queue=queue, + ), + org_id=self._continuation_org(payload), + ) + return True + except Exception: + ctx = payload.get("args") or [{}] + ctx0 = ctx[0] if ctx and isinstance(ctx[0], dict) else {} + logger.exception( + "PG-queue consumer: FAILED to self-chain continuation %r — the " + "callback (and its user-facing event) is lost " + "(run_id=%s task_id=%s org=%s)", + spec.get("task_name") if isinstance(spec, dict) else spec, + ctx0.get("run_id"), + payload.get("task_id"), + self._continuation_org(payload), + ) + return False + + @staticmethod + def _continuation_org(payload: TaskPayload) -> str: + """Best-effort org id for a chained callback (fairness/debug on its queue). + + The executor request carries it in the context dict (``args[0]``); + callbacks are not fairness-critical, so a missing/odd shape degrades to + ``""`` (no org). The guards below cover every non-dict shape, so no + try/except is needed. + """ + args = payload.get("args") or [] + if args and isinstance(args[0], dict): + return str(args[0].get("organization_id") or "") + return "" + + def _fail_dispatch(self, payload: TaskPayload, *, error: str) -> None: + """Surface a terminal dispatch failure on whichever return channel applies. + + Request-reply (``reply_key``) → store the error for the blocking caller. + Async/callback (``on_error``) → self-chain the on_error continuation, + carrying the real ``error`` text. Best-effort + never raises — the caller + acks regardless. Used by BOTH the run-raised path and the early-drop + branches (malformed / poison / unknown task), so a ``dispatch_with_callback`` + failure ALWAYS reaches its on_error callback, not only when the task body + raised (the realistic poison-executor case is exactly when the user most + needs the error surfaced). + """ + reply_key = payload.get("reply_key") + if reply_key: + self._fail_reply(reply_key, error) + return + on_error = payload.get("on_error") + if on_error: + self._chain_continuation( + on_error, + prepend=payload.get("task_id") or "", + payload=payload, + error=error, + ) + def _fail_reply(self, reply_key: str | None, error: str) -> None: """Best-effort failure reply for a request-reply message that can't run or whose run raised (drop / poison / unknown-task / exception). diff --git a/workers/queue_backend/pg_queue/executor_rpc.py b/workers/queue_backend/pg_queue/executor_rpc.py index a68100c8a1..a43754b84b 100644 --- a/workers/queue_backend/pg_queue/executor_rpc.py +++ b/workers/queue_backend/pg_queue/executor_rpc.py @@ -44,7 +44,8 @@ import uuid from typing import TYPE_CHECKING, Any -from unstract.core.data_models import PgTaskStatus +from unstract.core.data_models import ContinuationSpec, PgTaskStatus +from unstract.core.execution_dispatch import DispatchHandle, signature_to_continuation from unstract.flags.feature_flag import check_feature_flag_status from unstract.sdk1.execution.dispatcher import ExecutionDispatcher from unstract.sdk1.execution.result import ExecutionResult @@ -152,7 +153,7 @@ def dispatch( queue = f"{_QUEUE_PREFIX}{context.executor_name}" org = str(getattr(context, "organization_id", "") or "") try: - self._enqueue(queue, context, reply_key, org) + self._enqueue(queue, context, org, reply_key=reply_key) except Exception as exc: logger.exception( "PG executor dispatch: enqueue failed (executor=%s run_id=%s)", @@ -231,22 +232,103 @@ def dispatch( ) return ExecutionResult.failure(error=row.get("error") or "executor task failed") + def dispatch_async( + self, context: ExecutionContext, headers: dict[str, Any] | None = None + ) -> str: + """Fire-and-forget enqueue of ``execute_extraction``; returns the task id. + + The PG analogue of the SDK ``dispatch_async``: no ``reply_key``, no + callback, no blocking. There is no PG ``AsyncResult`` backend, so a caller + that needs the outcome uses :meth:`dispatch_with_callback` (a self-chained + continuation), not polling on this id. ``headers`` is accepted and ignored + for substitutability (PG carries routing in the payload, not Celery + headers). Enqueue failures propagate — parity with the SDK, which lets a + broker error out of ``dispatch_async``. + """ + task_id = str(uuid.uuid4()) + queue = f"{_QUEUE_PREFIX}{context.executor_name}" + org = str(getattr(context, "organization_id", "") or "") + self._enqueue(queue, context, org, task_id=task_id) + logger.info( + "PG executor dispatch_async: enqueued task_id=%s queue=%s run_id=%s", + task_id, + queue, + context.run_id, + ) + return task_id + + def dispatch_with_callback( + self, + context: ExecutionContext, + on_success: Any | None = None, + on_error: Any | None = None, + task_id: str | None = None, + headers: dict[str, Any] | None = None, + ) -> DispatchHandle: + """Fire-and-forget enqueue with self-chained callbacks (§5 model). + + The PG analogue of the SDK ``dispatch_with_callback``: instead of Celery + ``link`` / ``link_error`` (which the broker fires), the on-success / + on-error Celery ``Signature``s are translated to serialisable + :class:`ContinuationSpec`s and carried in the payload. After the executor + consumer runs ``execute_extraction`` it self-chains the matching + continuation onto the callback queue. Returns a :class:`DispatchHandle` + exposing ``.id`` (== ``task_id``) so call sites read the task id exactly + as on the Celery path. ``headers`` is accepted and ignored (see + :meth:`dispatch_async`). + """ + task_id = task_id or str(uuid.uuid4()) + queue = f"{_QUEUE_PREFIX}{context.executor_name}" + org = str(getattr(context, "organization_id", "") or "") + success_spec = signature_to_continuation(on_success) + error_spec = signature_to_continuation(on_error) + self._enqueue( + queue, + context, + org, + on_success=success_spec, + on_error=error_spec, + task_id=task_id, + ) + logger.info( + "PG executor dispatch_with_callback: enqueued task_id=%s queue=%s " + "run_id=%s on_success=%s on_error=%s", + task_id, + queue, + context.run_id, + success_spec["task_name"] if success_spec else None, + error_spec["task_name"] if error_spec else None, + ) + return DispatchHandle(task_id) + @staticmethod def _enqueue( - queue: str, context: ExecutionContext, reply_key: str, org_id: str + queue: str, + context: ExecutionContext, + org_id: str, + *, + reply_key: str | None = None, + on_success: ContinuationSpec | None = None, + on_error: ContinuationSpec | None = None, + task_id: str | None = None, ) -> None: - """Enqueue the ``execute_extraction`` request-reply message. + """Enqueue an ``execute_extraction`` message (request-reply or callback). A short-lived client owns its connection for just the insert (which commits internally) so the message is durably visible to the ``worker-pg-executor`` consumer before we begin polling — and no - connection is pinned for the whole (possibly long) RPC. + connection is pinned for the whole (possibly long) RPC. The optional keys + select the dispatch shape: ``reply_key`` → request-reply; ``on_success`` / + ``on_error`` / ``task_id`` → async/callback (self-chained). """ payload = to_payload( _EXECUTE_TASK, args=[context.to_dict()], queue=queue, reply_key=reply_key, + on_success=on_success, + on_error=on_error, + task_id=task_id, ) with PgQueueClient() as client: client.send(queue, payload, org_id=org_id) @@ -272,10 +354,10 @@ def _wait_for_result(reply_key: str, timeout: float) -> dict[str, Any] | None: class RoutingExecutionDispatcher: """Gate-routed executor dispatcher returned by :func:`get_executor_dispatcher`. - ``dispatch()`` chooses PG vs Celery per call (instant rollout/rollback); - ``dispatch_async`` / ``dispatch_with_callback`` always delegate to Celery — - the async/callback path stays on Celery until a later continuation slice. - Duck-typed against the SDK ``ExecutionDispatcher`` so call sites are unchanged. + Every mode chooses PG vs Celery per call (instant rollout/rollback): + ``dispatch()`` (request-reply), ``dispatch_async`` (fire-and-forget) and + ``dispatch_with_callback`` (self-chained callbacks). Duck-typed against the SDK + ``ExecutionDispatcher`` so call sites are unchanged. """ def __init__(self, celery_app: object | None = None) -> None: @@ -303,10 +385,32 @@ def dispatch( def dispatch_async( self, context: ExecutionContext, headers: dict[str, Any] | None = None ) -> str: + if resolve_executor_transport(context): + return self._pg.dispatch_async(context) return self._celery.dispatch_async(context, headers=headers) - def dispatch_with_callback(self, context: ExecutionContext, **kwargs: Any) -> Any: - return self._celery.dispatch_with_callback(context, **kwargs) + def dispatch_with_callback( + self, + context: ExecutionContext, + on_success: Any | None = None, + on_error: Any | None = None, + task_id: str | None = None, + headers: dict[str, Any] | None = None, + ) -> Any: + if resolve_executor_transport(context): + return self._pg.dispatch_with_callback( + context, + on_success=on_success, + on_error=on_error, + task_id=task_id, + ) + return self._celery.dispatch_with_callback( + context, + on_success=on_success, + on_error=on_error, + task_id=task_id, + headers=headers, + ) def get_executor_dispatcher( diff --git a/workers/queue_backend/pg_queue/task_payload.py b/workers/queue_backend/pg_queue/task_payload.py index da80f6c155..cefcf70e48 100644 --- a/workers/queue_backend/pg_queue/task_payload.py +++ b/workers/queue_backend/pg_queue/task_payload.py @@ -17,7 +17,7 @@ # re-exported here so existing ``from .task_payload import TaskPayload`` imports # keep working. The ``to_payload`` builder stays worker-side (it depends on the # worker-only ``FairnessKey``). -from unstract.core.data_models import TaskPayload +from unstract.core.data_models import ContinuationSpec, TaskPayload if TYPE_CHECKING: from ..fairness import FairnessKey @@ -33,13 +33,30 @@ def to_payload( queue: str | None = None, fairness: FairnessKey | None = None, reply_key: str | None = None, + on_success: ContinuationSpec | None = None, + on_error: ContinuationSpec | None = None, + task_id: str | None = None, ) -> TaskPayload: """Build the JSON-serialisable task payload for the PG queue. ``reply_key`` marks a **request-reply** dispatch (the executor RPC on PG): the executor consumer writes the task's result/error to ``pg_task_result`` under it for the blocking caller to poll. Omitted = fire-and-forget. + + ``on_success`` / ``on_error`` mark an **async/callback** dispatch + (``dispatch_with_callback``): the executor consumer self-chains the matching + continuation after the task runs (success → ``on_success``, failure → + ``on_error``). ``task_id`` is the dispatch id the consumer prepends to + ``on_error`` as the failed id (Celery ``link_error`` parity). These are + mutually exclusive with ``reply_key`` (blocking vs callback dispatch) — passing + both is rejected, since the consumer checks ``reply_key`` first and would + silently drop the callback. """ + if reply_key is not None and (on_success is not None or on_error is not None): + raise ValueError( + "reply_key (request-reply) and on_success/on_error (callback) are " + "mutually exclusive" + ) payload = TaskPayload( task_name=task_name, args=list(args) if args is not None else [], @@ -47,8 +64,14 @@ def to_payload( queue=queue, fairness=fairness.to_dict() if fairness is not None else None, ) - # Only set for request-reply dispatches — keeps fire-and-forget rows - # byte-identical to before this field existed (mirrors the backend producer). + # Each optional key is set only when present — keeps fire-and-forget rows + # byte-identical to before these fields existed (mirrors the backend producer). if reply_key is not None: payload["reply_key"] = reply_key + if on_success is not None: + payload["on_success"] = on_success + if on_error is not None: + payload["on_error"] = on_error + if task_id is not None: + payload["task_id"] = task_id return payload diff --git a/workers/tests/test_executor_rpc.py b/workers/tests/test_executor_rpc.py index ebf93b5889..823804e2a9 100644 --- a/workers/tests/test_executor_rpc.py +++ b/workers/tests/test_executor_rpc.py @@ -11,11 +11,13 @@ from unittest.mock import MagicMock, patch +import pytest from queue_backend.pg_queue.executor_rpc import ( PgExecutionDispatcher, RoutingExecutionDispatcher, resolve_executor_transport, ) +from unstract.core.execution_dispatch import DispatchHandle, signature_to_continuation _MOD = "queue_backend.pg_queue.executor_rpc" @@ -298,11 +300,153 @@ def test_gate_on_passes_timeout_to_pg_and_drops_headers(self): assert "headers" not in pg.dispatch.call_args.kwargs celery.dispatch.assert_not_called() - def test_async_and_callback_always_celery(self): - """The callback/async path stays on Celery regardless of the gate (a later slice).""" + def test_async_and_callback_stay_celery_when_gate_off(self): + """Zero-regression: gate off → async/callback delegate to Celery unchanged.""" dispatcher, celery, pg = self._build() - dispatcher.dispatch_async(_ctx()) - dispatcher.dispatch_with_callback(_ctx(), on_success=None) + with patch(f"{_MOD}.resolve_executor_transport", return_value=False): + dispatcher.dispatch_async(_ctx(), headers={"h": 1}) + dispatcher.dispatch_with_callback(_ctx(), on_success="s", on_error="e") celery.dispatch_async.assert_called_once() celery.dispatch_with_callback.assert_called_once() - pg.dispatch.assert_not_called() + pg.dispatch_async.assert_not_called() + pg.dispatch_with_callback.assert_not_called() + + def test_async_and_callback_route_to_pg_when_gated(self): + """Gate on (③c) → async/callback take the PG self-chained path.""" + dispatcher, celery, pg = self._build() + with patch(f"{_MOD}.resolve_executor_transport", return_value=True): + dispatcher.dispatch_async(_ctx()) + dispatcher.dispatch_with_callback( + _ctx(), on_success="s", on_error="e", task_id="t" + ) + pg.dispatch_async.assert_called_once() + pg.dispatch_with_callback.assert_called_once() + # PG carries callbacks in the payload, not Celery headers → no header leak. + assert "headers" not in pg.dispatch_with_callback.call_args.kwargs + celery.dispatch_async.assert_not_called() + celery.dispatch_with_callback.assert_not_called() + + +class TestPgAsyncCallbackWiring: + """PG fire-and-forget + self-chained-callback enqueue shapes. + + ``to_payload`` runs for real; only ``PgQueueClient`` is mocked. Pins that the + async path carries NO reply_key (it must not block a consumer) and the callback + path carries the translated continuations + the tracking task_id. + """ + + @staticmethod + def _ctx(): + c = MagicMock() + c.executor_name = "legacy" + c.run_id = "r" + c.organization_id = "org9" + c.to_dict.return_value = {"run_id": "r", "organization_id": "org9"} + return c + + @staticmethod + def _client(): + client = MagicMock() + client.__enter__.return_value = client + return client + + def test_dispatch_async_is_fire_and_forget(self): + client = self._client() + with patch(f"{_MOD}.PgQueueClient", return_value=client): + task_id = PgExecutionDispatcher().dispatch_async(self._ctx()) + client.send.assert_called_once() + queue_arg, payload_arg = client.send.call_args.args[:2] + assert queue_arg == "celery_executor_legacy" + assert client.send.call_args.kwargs["org_id"] == "org9" + assert payload_arg["task_name"] == "execute_extraction" + assert payload_arg["task_id"] == task_id + # No reply_key (would make a consumer try to store a reply) and no callback. + assert "reply_key" not in payload_arg + assert "on_success" not in payload_arg + assert "on_error" not in payload_arg + + def test_dispatch_with_callback_carries_continuations(self): + client = self._client() + on_s = MagicMock( + task="ide_prompt_complete", + args=(), + kwargs={"callback_kwargs": {"room": "r1"}}, + options={"queue": "ide_callback"}, + ) + on_e = MagicMock( + task="ide_prompt_error", + args=(), + kwargs={"callback_kwargs": {"room": "r1"}}, + options={"queue": "ide_callback"}, + ) + with patch(f"{_MOD}.PgQueueClient", return_value=client): + handle = PgExecutionDispatcher().dispatch_with_callback( + self._ctx(), on_success=on_s, on_error=on_e, task_id="tid-7" + ) + assert handle.id == "tid-7" # call sites read .id off the handle + payload_arg = client.send.call_args.args[1] + assert payload_arg["on_success"] == { + "task_name": "ide_prompt_complete", + "kwargs": {"callback_kwargs": {"room": "r1"}}, + "queue": "ide_callback", + } + assert payload_arg["on_error"]["task_name"] == "ide_prompt_error" + assert payload_arg["task_id"] == "tid-7" + assert "reply_key" not in payload_arg # callback, not request-reply + + def test_dispatch_with_callback_defaults_task_id(self): + client = self._client() + with patch(f"{_MOD}.PgQueueClient", return_value=client): + handle = PgExecutionDispatcher().dispatch_with_callback(self._ctx()) + # No task_id passed → a uuid is generated and echoed on the handle + payload. + assert handle.id + assert client.send.call_args.args[1]["task_id"] == handle.id + + +class TestSharedDispatchHelpers: + """The transport-agnostic helpers lifted to ``unstract.core`` (shared by the + backend + workers executor-RPC mirrors). Tested once here, not per-mirror. + """ + + def test_signature_none_passes_through(self): + assert signature_to_continuation(None) is None + + def test_signature_translates_task_kwargs_and_queue(self): + sig = MagicMock( + task="ide_prompt_complete", + args=(), # a real kwargs-only Celery Signature has empty .args + kwargs={"callback_kwargs": {"room": "r1"}}, + options={"queue": "ide_callback"}, + ) + assert signature_to_continuation(sig) == { + "task_name": "ide_prompt_complete", + "kwargs": {"callback_kwargs": {"room": "r1"}}, + "queue": "ide_callback", + } + + def test_signature_missing_queue_fails_fast(self): + sig = MagicMock(task="ide_prompt_complete", kwargs={}, options={"queue": ""}) + with pytest.raises(ValueError, match="no queue"): + signature_to_continuation(sig) + + def test_signature_missing_task_fails_fast(self): + sig = MagicMock(task=None, kwargs={}, options={"queue": "ide_callback"}) + with pytest.raises(ValueError, match="no task name"): + signature_to_continuation(sig) + + def test_signature_with_positional_args_fails_fast(self): + sig = MagicMock( + task="ide_prompt_complete", + args=("pos",), + kwargs={}, + options={"queue": "ide_callback"}, + ) + with pytest.raises(ValueError, match="positional args"): + signature_to_continuation(sig) + + def test_dispatch_handle_exposes_only_id(self): + handle = DispatchHandle("tid-1") + assert handle.id == "tid-1" + # __slots__ → no stray attributes (callers must not poke at .get()/.result). + with pytest.raises(AttributeError): + handle.result = 1 # type: ignore[attr-defined] diff --git a/workers/tests/test_pg_queue_consumer.py b/workers/tests/test_pg_queue_consumer.py index 05ed6e9869..5a731b7b9f 100644 --- a/workers/tests/test_pg_queue_consumer.py +++ b/workers/tests/test_pg_queue_consumer.py @@ -34,6 +34,15 @@ def _boom_task(): raise RuntimeError("boom") +@shared_task(name="test_pg_consumer.dt") +def _dt_task(): + # Returns a non-JSON-native value (datetime) — the self-chain must coerce it + # before enqueue, else client.send's plain json.dumps would raise. + from datetime import datetime + + return {"when": datetime(2020, 1, 1)} # noqa: DTZ001 — fixed value for the test + + @pytest.fixture(autouse=True) def _clear_calls(): _calls.clear() @@ -521,5 +530,161 @@ def test_store_failure_on_success_path_still_acks(self, caplog): assert "FAILED to store request-reply result" in caplog.text +class TestSelfChain: + """Async/callback self-chaining (③c): with no reply_key, the consumer enqueues + the on_success / on_error continuation onto its queue after the task runs, then + acks regardless (a vt-redelivery would re-run the executor = LLM double-spend). + Best-effort: a chain-enqueue failure still acks. ``client.send`` is the + self-chain enqueue (the mock client is also read/delete). + """ + + @staticmethod + def _spec(task="cb.done", queue="ide_callback"): + return { + "task_name": task, + "kwargs": {"callback_kwargs": {"room": "r1"}}, + "queue": queue, + } + + def test_success_chains_on_success_with_result_and_acks(self): + client = MagicMock() + payload = {**_ok_payload(3, 4), "on_success": self._spec(), "task_id": "tid"} + client.read.return_value = [_msg(1, payload)] + PgQueueConsumer(["q"], client=client).poll_once() + client.delete.assert_called_once_with(1) # executor msg acked + client.send.assert_called_once() # continuation enqueued + queue_arg, payload_arg = client.send.call_args.args[:2] + assert queue_arg == "ide_callback" + assert payload_arg["task_name"] == "cb.done" + assert payload_arg["args"] == [7] # x + y prepended (Celery link parity) + assert payload_arg["kwargs"] == {"callback_kwargs": {"room": "r1"}} + + def test_failure_chains_on_error_with_task_id_and_acks(self): + client = MagicMock() + payload = { + "task_name": "test_pg_consumer.boom", + "on_error": self._spec("cb.err"), + "task_id": "tid-9", + } + client.read.return_value = [_msg(2, payload)] + PgQueueConsumer(["q"], client=client).poll_once() + client.delete.assert_called_once_with(2) # acked, NOT left for redelivery + client.send.assert_called_once() + payload_arg = client.send.call_args.args[1] + assert payload_arg["task_name"] == "cb.err" + assert payload_arg["args"] == ["tid-9"] # dispatch task_id as the failed id + # The real error text rides callback_kwargs['error'] (no Celery AsyncResult + # to recover it from on PG) so on_error callbacks surface it, not a default. + assert "RuntimeError: boom" in payload_arg["kwargs"]["callback_kwargs"]["error"] + + def test_early_drop_branches_still_chain_on_error(self): + # Critical: a dispatch_with_callback that hits malformed/unknown/poison must + # still fire on_error — else the HTTP-202 caller hangs with no terminal event. + err = self._spec("cb.err") + cases = [ + # (msg_id, payload, read_ct) — malformed / unknown / poison + (10, {"on_error": err, "task_id": "t"}, 1), + (11, {"task_name": "nope.nope", "on_error": err, "task_id": "t"}, 1), + (12, {"task_name": "test_pg_consumer.boom", "on_error": err, "task_id": "t"}, 99), + ] + for msg_id, payload, read_ct in cases: + client = MagicMock() + client.read.return_value = [_msg(msg_id, payload, read_ct=read_ct)] + PgQueueConsumer(["q"], client=client).poll_once() + client.delete.assert_called_once_with(msg_id) # dropped/acked + client.send.assert_called_once() # on_error STILL chained + payload_arg = client.send.call_args.args[1] + assert payload_arg["task_name"] == "cb.err" + assert payload_arg["args"] == ["t"] + assert payload_arg["kwargs"]["callback_kwargs"]["error"] # the drop reason + + def test_non_json_safe_result_is_coerced_before_chaining(self): + client = MagicMock() + payload = { + "task_name": "test_pg_consumer.dt", + "on_success": self._spec(), + "task_id": "tid", + } + client.read.return_value = [_msg(6, payload)] + PgQueueConsumer(["q"], client=client).poll_once() + client.send.assert_called_once() # not swallowed by a json.dumps TypeError + chained = client.send.call_args.args[1]["args"][0] + assert isinstance(chained["when"], str) # datetime → str (default=str) + import json as _json + + _json.dumps(chained) # the coerced payload is plain-json serialisable + + def test_on_success_only_failure_acks_not_redelivered(self): + # An on_success-only callback dispatch (on_error omitted) whose executor + # RAISES must ACK — not fall through to vt-redelivery and re-run the + # executor (LLM double-spend). No on_error → nothing chained, but acked. + client = MagicMock() + payload = { + "task_name": "test_pg_consumer.boom", + "on_success": self._spec(), + "task_id": "tid", + } + client.read.return_value = [_msg(7, payload)] + PgQueueConsumer(["q"], client=client).poll_once() + client.delete.assert_called_once_with(7) # acked, NOT left for redelivery + client.send.assert_not_called() # success callback not fired on a raise + + def test_on_success_enqueue_failure_falls_back_to_on_error(self): + # If the success hand-off can't be enqueued, surface on_error so the caller + # still gets a terminal event instead of hanging after the LLM spend. + client = MagicMock() + client.send.side_effect = [RuntimeError("queue down"), 999] # 1st fails, 2nd ok + payload = { + **_ok_payload(2, 2), + "on_success": self._spec("cb.ok"), + "on_error": self._spec("cb.err"), + "task_id": "tid", + } + client.read.return_value = [_msg(8, payload)] + PgQueueConsumer(["q"], client=client).poll_once() + assert client.send.call_count == 2 # on_success attempt, then on_error fallback + fallback = client.send.call_args_list[1].args[1] + assert fallback["task_name"] == "cb.err" + assert "delivery failed" in fallback["kwargs"]["callback_kwargs"]["error"] + client.delete.assert_called_once_with(8) # acked + + def test_no_continuation_is_plain_fire_and_forget(self): + client = MagicMock() + client.read.return_value = [_msg(3, _ok_payload(1, 1))] + PgQueueConsumer(["q"], client=client).poll_once() + client.send.assert_not_called() # nothing self-chained + client.delete.assert_called_once_with(3) # acked + + def test_chain_failure_on_success_still_acks(self, caplog): + client = MagicMock() + client.send.side_effect = RuntimeError("queue down") + payload = {**_ok_payload(2, 2), "on_success": self._spec()} + client.read.return_value = [_msg(4, payload)] + with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"): + PgQueueConsumer(["q"], client=client).poll_once() + client.delete.assert_called_once_with(4) # acked despite the chain failure + assert "FAILED to self-chain" in caplog.text + + def test_continuation_org_extracted_from_context(self): + # The chained callback inherits the executor request's org (context dict); + # non-dict / absent args degrade to "" (callbacks aren't fairness-critical). + org_of = PgQueueConsumer._continuation_org + assert org_of({"args": [{"organization_id": "orgZ"}]}) == "orgZ" + assert org_of({"args": [42]}) == "" + assert org_of({}) == "" + + def test_reply_key_and_callback_are_mutually_exclusive(self): + # The consumer checks reply_key first and would silently drop a callback — + # so to_payload rejects the ambiguous combination at the build boundary. + spec = self._spec() + with pytest.raises(ValueError, match="mutually exclusive"): + to_payload("execute_extraction", reply_key="rk", on_success=spec) + with pytest.raises(ValueError, match="mutually exclusive"): + to_payload("execute_extraction", reply_key="rk", on_error=spec) + # Either alone is fine. + assert to_payload("execute_extraction", reply_key="rk")["reply_key"] == "rk" + assert to_payload("execute_extraction", on_success=spec)["on_success"] == spec + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 1c7723b9d6f101f40014555204c1f0f71683d808 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Mon, 22 Jun 2026 09:34:32 +0530 Subject: [PATCH 32/44] =?UTF-8?q?UN-3610=20[FEAT]=20PG=20Queue=20=E2=80=94?= =?UTF-8?q?=20reaper=20retention=20sweep=20(pg=5Ftask=5Fresult=20+=20pg=5F?= =?UTF-8?q?batch=5Fdedup=20orphans)=20(#2101)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3610 [FEAT] PG Queue — reaper retention sweep (pg_task_result + pg_batch_dedup orphans) Wire the deferred retention sweep into the PG reaper, before the gate ramps: - sweep_expired_results(conn) — DELETE FROM pg_task_result WHERE expires_at <= now(). Every result row already carries expires_at = now()+3600s (written by the consumer's store_result) and the expires index exists; this is the only reader that keeps the table from growing unbounded with each executor RPC. - sweep_orphan_dedup(conn, retention) — DELETE FROM pg_batch_dedup WHERE created_at <= now() - interval. Backstop for partial-failure executions whose per-batch markers neither clear on teardown nor get reclaimed by barrier recovery. Retention (default 24h) must exceed the longest execution. - Both run in PgReaper.tick() under leadership, after recovery + schedules, cadence-gated by WORKER_PG_REAPER_SWEEP_SECONDS (default 300s) so the ~5s loop doesn't DELETE every cycle; own try/except discards the owned conn on error. - Env knobs surfaced on the compose worker-pg-reaper service. Workers-only: no migration, no writer change, no flag. Idempotent (DELETE WHERE), dark until rows exist. Tests: env parsers, SQL contract, leader-only / standby / cadence-gated / after-recovery / error-discards-conn (27 green). Dev-tested live: the reaper deleted 9 expired pg_task_result + 1 orphaned pg_batch_dedup row. Co-Authored-By: Claude Opus 4.8 * UN-3610 [FIX] address review — independent sweeps, actionable failures, rollback signal Toolkit review on #2101: - [High] Partial-sweep starvation: the two sweeps now run INDEPENDENTLY via _run_sweep — a persistent fault in one no longer skips (and then cadence-gates out) the other. - [High] Sweep failures not actionable: _run_sweep logs each failure at the boundary with the table name + a consecutive-failure streak (distinct from the generic tick-failure log) and does NOT propagate (cleanup must not fail the tick). - [Med] Surface a rollback that itself fails (don't suppress) via _rollback_after_sweep_failure, while still re-raising the original DELETE error. - [Med] Reword the dedup-retention comment as operator-enforced (not code-coupled). - [Low] Soften the result-retention coupling claim (holds by default); note the pg_batch_dedup.created_at seq-scan; de-ambiguate the compose comment. - [type-design] _last_sweep_monotonic is now float | None ("never"), not a 0.0 sentinel. Tests (+8, 35 green): float-vs-int cast distinction, injected-knob constructor guards, sweep_orphan_dedup rollback (parametrized) + rollback-itself-fails log, one-sweep-failing-doesn't-starve-the-other, failing-sweep-still-advances-cadence, log-emission boundary (counts only when rows deleted / silent otherwise). Co-Authored-By: Claude Opus 4.8 * UN-3610 [FIX] greptile — accurate env-parse error + sweep-retry log wording - _positive_duration_from_env: include the cause ("cannot be parsed: ") instead of "is not a number" — an int knob given "1.5" IS a number, just not an integer, so the old message misled. Test asserts the message. - _run_sweep failure log: "will retry after the next sweep interval" (not "next cycle") — the retry is gated by _sweep_interval (~300s), not the ~5s tick. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- docker/docker-compose.yaml | 7 + workers/queue_backend/pg_queue/reaper.py | 208 +++++++++++++++++++- workers/tests/test_pg_reaper.py | 240 +++++++++++++++++++++++ 3 files changed, 454 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 612e35ea8d..0dca79ab4c 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -837,6 +837,13 @@ services: - APPLICATION_NAME=unstract-worker-pg-reaper - WORKER_PG_REAPER_INTERVAL_SECONDS=${WORKER_PG_REAPER_INTERVAL_SECONDS:-5} - WORKER_PG_REAPER_HEALTH_PORT=8086 + # Retention sweep (cadence-gated inside the leader's tick): drop expired + # pg_task_result rows + orphaned pg_batch_dedup markers so neither grows + # unbounded as the gate ramps. The dedup retention below (default 24h) should + # be kept above the longest possible execution, so an in-flight marker is + # never swept. + - WORKER_PG_REAPER_SWEEP_SECONDS=${WORKER_PG_REAPER_SWEEP_SECONDS:-300} + - WORKER_PG_DEDUP_RETENTION_SECONDS=${WORKER_PG_DEDUP_RETENTION_SECONDS:-86400} labels: - traefik.enable=false volumes: diff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.py index d07e2919e5..4a0c79f5f7 100644 --- a/workers/queue_backend/pg_queue/reaper.py +++ b/workers/queue_backend/pg_queue/reaper.py @@ -51,7 +51,8 @@ import signal import threading import time -from typing import TYPE_CHECKING, NamedTuple, Protocol +from collections.abc import Callable +from typing import TYPE_CHECKING, NamedTuple, Protocol, TypeVar from unstract.core.data_models import ExecutionStatus @@ -69,6 +70,17 @@ # Cadence: how often the leader renews + runs recovery. Enforced shorter than # the lease window in PgReaper.__init__. _DEFAULT_REAPER_INTERVAL_SECONDS = 5.0 +# Retention sweep cadence — far rarer than the tick (cleanup, not recovery), so +# the per-cycle DELETE doesn't run every few seconds. +_DEFAULT_SWEEP_INTERVAL_SECONDS = 300.0 +# Default age before an orphaned ``pg_batch_dedup`` marker is swept. Should be set +# (here / via WORKER_PG_DEDUP_RETENTION_SECONDS) above the longest possible +# execution, else a still-in-flight marker could be swept out from under a running +# fan-out — this is operator-enforced, not coupled in code to the actual bound. 24h +# is comfortably above any single execution. +_DEFAULT_DEDUP_RETENTION_SECONDS = 86400 + +_N = TypeVar("_N", int, float) class LeaderLeaseLike(Protocol): @@ -127,6 +139,115 @@ def reaper_interval_from_env() -> float: return value +def _positive_duration_from_env(name: str, default: _N, cast: Callable[[str], _N]) -> _N: + """Read a positive duration env var (default on unset; raise on invalid/<=0). + + Same loud-on-misconfig posture as :func:`reaper_interval_from_env`, shared by + the sweep-cadence and dedup-retention knobs. + """ + raw = os.getenv(name) + if raw is None: + return default + try: + value = cast(raw) + except ValueError as exc: + # Include the cause: an int knob given "1.5" IS a number, just not an int — + # "is not a number" would mislead. The cause spells out the real reason. + raise ValueError( + f"{name}={raw!r} cannot be parsed: {exc}. Unset it to default to {default}." + ) from exc + if value <= 0: + raise ValueError( + f"{name}={value} must be positive. Unset it to default to {default}." + ) + return value + + +def reaper_sweep_interval_from_env() -> float: + """Retention-sweep cadence from ``WORKER_PG_REAPER_SWEEP_SECONDS`` (default 300s).""" + return _positive_duration_from_env( + "WORKER_PG_REAPER_SWEEP_SECONDS", _DEFAULT_SWEEP_INTERVAL_SECONDS, float + ) + + +def dedup_retention_from_env() -> int: + """Dedup-orphan age from ``WORKER_PG_DEDUP_RETENTION_SECONDS`` (default 24h).""" + return _positive_duration_from_env( + "WORKER_PG_DEDUP_RETENTION_SECONDS", _DEFAULT_DEDUP_RETENTION_SECONDS, int + ) + + +def _rollback_after_sweep_failure(conn: PgConnection, table: str) -> None: + """Roll back after a failed sweep DELETE; surface a rollback that itself fails. + + The caller re-raises the original error regardless — but a rollback that also + raises (broken socket / admin-terminated backend) signals a dead connection, so + log it rather than swallow it silently (which would hide why the next cycle's + reconnect is needed). + """ + try: + conn.rollback() + except Exception: + logger.warning( + "Reaper: rollback after a failed %s sweep also failed " + "(connection likely dead)", + table, + exc_info=True, + ) + + +def sweep_expired_results(conn: PgConnection) -> int: + """Delete expired executor-RPC result rows; return the number deleted. + + ``pg_task_result`` rows carry ``expires_at = now() + retention`` (written by + the consumer's ``store_result``, ``DEFAULT_RETENTION_SECONDS``). Once past it no + caller is still waiting **by default** — that retention matches the caller- + timeout default (``EXECUTOR_RESULT_TIMEOUT``), so the result has outlived any + wait. (An operator who raises the caller timeout ABOVE the store retention could + in principle have a result swept mid-wait; size the retention >= the longest + caller timeout.) The table has no other reader once expired, so this is the only + thing keeping it from growing unbounded with each RPC. Idempotent + (``DELETE … WHERE``) and uses the ``pg_task_result_expires_idx`` index. Rolls + back on error so the manual-commit connection isn't left in an aborted-txn state. + """ + try: + with conn.cursor() as cur: + cur.execute("DELETE FROM pg_task_result WHERE expires_at <= now()") + deleted = cur.rowcount + conn.commit() + return deleted + except Exception: + _rollback_after_sweep_failure(conn, "pg_task_result") + raise + + +def sweep_orphan_dedup(conn: PgConnection, retention_seconds: int) -> int: + """Delete orphaned per-batch dedup markers older than *retention_seconds*. + + ``pg_batch_dedup`` markers are normally cleared on barrier teardown / reaper + barrier-recovery, but a partial-failure execution can leave them behind. A + marker only matters while its execution is in flight, so anything older than + the longest possible execution (*retention_seconds*) is a safe-to-drop orphan. + Filters on ``created_at``, which is deliberately **unindexed** (see + ``backend/pg_queue/models.py``), so each sweep seq-scans — fine at the 5-min + cadence on a normally-near-empty table; add an index if the dedup table grows. + Idempotent; rolls back on error. + """ + try: + with conn.cursor() as cur: + cur.execute( + "DELETE FROM pg_batch_dedup " + "WHERE created_at <= now() - make_interval(secs => %s)", + (retention_seconds,), + ) + deleted = cur.rowcount + conn.commit() + return deleted + except Exception: + _rollback_after_sweep_failure(conn, "pg_batch_dedup") + raise + + def _execution_status( api_client: InternalAPIClient, execution_id: str, organization_id: str ) -> str | None: @@ -380,6 +501,8 @@ def __init__( lease: LeaderLeaseLike, *, interval_seconds: float | None = None, + sweep_interval_seconds: float | None = None, + dedup_retention_seconds: int | None = None, sweep_conn: PgConnection | None = None, api_client: InternalAPIClient | None = None, ) -> None: @@ -401,6 +524,29 @@ def __init__( f"lease window {lease.lease_seconds}s, or the leader loses the " f"lease between renews" ) + # Retention-sweep cadence + dedup-orphan horizon (validated like interval: + # an injected non-positive value reaches here unvalidated by the env parser). + self._sweep_interval = ( + sweep_interval_seconds + if sweep_interval_seconds is not None + else reaper_sweep_interval_from_env() + ) + if self._sweep_interval <= 0: + raise ValueError("sweep_interval_seconds must be positive") + self._dedup_retention = ( + dedup_retention_seconds + if dedup_retention_seconds is not None + else dedup_retention_from_env() + ) + if self._dedup_retention <= 0: + raise ValueError("dedup_retention_seconds must be positive") + # None → "never swept", so the first leader tick sweeps immediately; set to + # monotonic() each sweep so the cadence holds thereafter. (A None sentinel, + # not 0.0, so the gate doesn't lean on monotonic() never returning ~0.) + self._last_sweep_monotonic: float | None = None + # Per-table consecutive-failure streak — surfaced in the failure log so a + # persistently-failing sweep (and which table) is traceable in prod. + self._sweep_fail_streak: dict[str, int] = {} self._sweep_conn = sweep_conn self._owns_sweep_conn = sweep_conn is None # Lazily built so the reaper can be constructed without env/HTTP set up @@ -499,8 +645,68 @@ def tick(self) -> TickOutcome: except Exception: self._discard_owned_sweep_conn() raise + # Orchestrator's third job: retention cleanup (cadence-gated, so it does + # NOT run every tick). Last so a sweep error can't skip recovery/schedules. + self._maybe_sweep() return TickOutcome(was_leader=True, reclaimed=reclaimed) + def _maybe_sweep(self) -> None: + """Run the retention sweep at most once per ``_sweep_interval``. + + Leader-only (called from :meth:`tick` after recovery + schedules). Deletes + expired ``pg_task_result`` rows and orphaned ``pg_batch_dedup`` markers so + neither table grows unbounded as the gate ramps. The two sweeps run + **independently** (via :meth:`_run_sweep`): they cover different tables, so + a persistent fault in one must not skip — and then cadence-gate out — the + other. The cadence is advanced BEFORE sweeping so a failure waits one + interval before retry rather than hammering the DB every tick. + """ + now = time.monotonic() + if ( + self._last_sweep_monotonic is not None + and now - self._last_sweep_monotonic < self._sweep_interval + ): + return + self._last_sweep_monotonic = now + results = self._run_sweep("pg_task_result", sweep_expired_results) + dedup = self._run_sweep( + "pg_batch_dedup", + lambda conn: sweep_orphan_dedup(conn, self._dedup_retention), + ) + if results or dedup: + logger.info( + "Reaper: retention sweep deleted %s pg_task_result + " + "%s pg_batch_dedup row(s)", + results, + dedup, + ) + + def _run_sweep(self, table: str, fn: Callable[[PgConnection], int]) -> int: + """Run one retention sweep best-effort; return its row count (0 on failure). + + A cleanup failure is NOT propagated (it must not fail the tick) and must not + starve the sibling sweep — it is logged at this boundary with the table name + and a consecutive-failure streak (so a bloated ``pg_task_result`` / + ``pg_batch_dedup`` in prod is traceable, distinct from the generic + tick-failure log), and the owned conn is discarded so the next cycle + reconnects. A clean run resets the streak. + """ + try: + count = fn(self._get_sweep_conn()) + except Exception: + streak = self._sweep_fail_streak.get(table, 0) + 1 + self._sweep_fail_streak[table] = streak + logger.exception( + "Reaper: retention sweep of %s failed (%s consecutive) — will retry " + "after the next sweep interval", + table, + streak, + ) + self._discard_owned_sweep_conn() + return 0 + self._sweep_fail_streak[table] = 0 + return count + def run(self, *, install_signals: bool = True) -> None: """Lease-maintenance + recovery loop until stopped; releases on exit.""" self._running = True diff --git a/workers/tests/test_pg_reaper.py b/workers/tests/test_pg_reaper.py index ab3b3bdf4d..01db9fda9a 100644 --- a/workers/tests/test_pg_reaper.py +++ b/workers/tests/test_pg_reaper.py @@ -15,6 +15,7 @@ from __future__ import annotations +import logging import os import threading import time @@ -27,10 +28,15 @@ from queue_backend.pg_queue.connection import create_pg_connection from queue_backend.pg_queue.reaper import ( PgReaper, + dedup_retention_from_env, reaper_interval_from_env, + reaper_sweep_interval_from_env, recover_expired_barriers, + sweep_expired_results, + sweep_orphan_dedup, ) + # The reaper's leader tick also runs the PG scheduler tick (②b). Its behaviour # is covered in test_pg_scheduler.py; stub it here by default so the leadership / # recovery / connection tests aren't coupled to a real schedule query on their @@ -43,6 +49,20 @@ def stub_scheduler_tick(monkeypatch): return mock +# The leader tick also runs the retention sweep (UN-3610). Stub the two sweep +# helpers by default so the leadership / connection tests don't hit a real DELETE +# on their dummy connections; the SQL-contract tests import the real helpers +# directly (unaffected by this module-attribute patch), and the sweep-wiring tests +# opt in via the returned mocks. +@pytest.fixture(autouse=True) +def stub_retention_sweep(monkeypatch): + results = MagicMock(return_value=0) + dedup = MagicMock(return_value=0) + monkeypatch.setattr(reaper_mod, "sweep_expired_results", results) + monkeypatch.setattr(reaper_mod, "sweep_orphan_dedup", dedup) + return SimpleNamespace(results=results, dedup=dedup) + + # --- Layer 1: env + construction (no DB) --- @@ -62,6 +82,39 @@ def test_invalid_raises(self, monkeypatch, bad): reaper_interval_from_env() +class TestSweepEnv: + def test_sweep_interval_default_and_override(self, monkeypatch): + monkeypatch.delenv("WORKER_PG_REAPER_SWEEP_SECONDS", raising=False) + assert reaper_sweep_interval_from_env() == pytest.approx(300.0) + monkeypatch.setenv("WORKER_PG_REAPER_SWEEP_SECONDS", "30") + assert reaper_sweep_interval_from_env() == pytest.approx(30.0) + + def test_dedup_retention_default_and_override(self, monkeypatch): + monkeypatch.delenv("WORKER_PG_DEDUP_RETENTION_SECONDS", raising=False) + assert dedup_retention_from_env() == 86400 + monkeypatch.setenv("WORKER_PG_DEDUP_RETENTION_SECONDS", "3600") + assert dedup_retention_from_env() == 3600 + + @pytest.mark.parametrize("bad", ["0", "-5", "abc"]) + def test_invalid_raises(self, monkeypatch, bad): + monkeypatch.setenv("WORKER_PG_REAPER_SWEEP_SECONDS", bad) + monkeypatch.setenv("WORKER_PG_DEDUP_RETENTION_SECONDS", bad) + with pytest.raises(ValueError): + reaper_sweep_interval_from_env() + with pytest.raises(ValueError): + dedup_retention_from_env() + + def test_cast_distinction_float_vs_int(self, monkeypatch): + # sweep parses as float, dedup as int — the cast is load-bearing. + monkeypatch.setenv("WORKER_PG_REAPER_SWEEP_SECONDS", "1.5") + assert reaper_sweep_interval_from_env() == pytest.approx(1.5) + monkeypatch.setenv("WORKER_PG_DEDUP_RETENTION_SECONDS", "1.5") + # int("1.5") rejects the fractional value; the message names the real cause + # ("cannot be parsed: …") rather than the misleading "is not a number". + with pytest.raises(ValueError, match="cannot be parsed"): + dedup_retention_from_env() + + class _FakeLease: """Duck-typed LeaderLease. ``acquires``/``renews`` accept a bool (constant) or a list (one outcome popped per call, then ``False``). @@ -106,6 +159,21 @@ def test_non_positive_interval_rejected(self): def test_valid_interval_accepted(self): PgReaper(_FakeLease(lease_seconds=10), interval_seconds=3, sweep_conn=object()) + def test_non_positive_sweep_interval_rejected(self): + # An injected knob bypasses the env parser's guard → re-validated in __init__. + with pytest.raises(ValueError, match="sweep_interval"): + PgReaper( + _FakeLease(), interval_seconds=1, sweep_interval_seconds=0, + sweep_conn=object(), + ) + + def test_non_positive_dedup_retention_rejected(self): + with pytest.raises(ValueError, match="dedup_retention"): + PgReaper( + _FakeLease(), interval_seconds=1, dedup_retention_seconds=0, + sweep_conn=object(), + ) + # --- Layer 2: leadership gating (fake lease + patched sweep, no DB) --- @@ -249,6 +317,178 @@ def test_scheduler_error_discards_owned_conn(self, stub_scheduler_tick): assert reaper._sweep_conn is None # discarded +class TestRetentionSweepSql: + """The sweep helpers' SQL contract (mock cursor, no DB). These call the real + helpers (imported at module load), unaffected by the autouse stub which patches + the module attribute the reaper looks up at call time. + """ + + @staticmethod + def _conn_cur(rowcount): + cur = MagicMock() + cur.rowcount = rowcount + conn = MagicMock() + conn.cursor.return_value.__enter__.return_value = cur + return conn, cur + + def test_sweep_expired_results_sql(self): + conn, cur = self._conn_cur(3) + assert sweep_expired_results(conn) == 3 + sql = cur.execute.call_args[0][0] + assert "DELETE FROM pg_task_result" in sql and "expires_at <= now()" in sql + conn.commit.assert_called_once() + + def test_sweep_orphan_dedup_sql(self): + conn, cur = self._conn_cur(2) + assert sweep_orphan_dedup(conn, 999) == 2 + args = cur.execute.call_args[0] + assert "DELETE FROM pg_batch_dedup" in args[0] + assert "created_at <= now() - make_interval" in args[0] + assert args[1] == (999,) # the retention param is bound, not interpolated + conn.commit.assert_called_once() + + @pytest.mark.parametrize( + "sweep", + [ + lambda conn: sweep_expired_results(conn), + lambda conn: sweep_orphan_dedup(conn, 60), + ], + ids=["expired_results", "orphan_dedup"], + ) + def test_sweep_rolls_back_on_error(self, sweep): + # Both helpers have their own try/except/rollback — exercise each. + cur = MagicMock() + cur.execute.side_effect = psycopg2.OperationalError("dead") + conn = MagicMock() + conn.cursor.return_value.__enter__.return_value = cur + with pytest.raises(psycopg2.OperationalError): + sweep(conn) + conn.rollback.assert_called_once() + conn.commit.assert_not_called() + + def test_rollback_failure_is_logged_and_original_error_raised(self, caplog): + # If rollback itself raises, surface it (don't swallow) but still propagate + # the ORIGINAL DELETE error, not the rollback's. + cur = MagicMock() + cur.execute.side_effect = psycopg2.OperationalError("dead") + conn = MagicMock() + conn.cursor.return_value.__enter__.return_value = cur + conn.rollback.side_effect = psycopg2.InterfaceError("conn dead") + with caplog.at_level(logging.WARNING, logger="queue_backend.pg_queue.reaper"): + with pytest.raises(psycopg2.OperationalError): + sweep_expired_results(conn) + assert "rollback after a failed pg_task_result sweep also failed" in caplog.text + + +class TestRetentionSweepTick: + """The sweep runs leader-only, after recovery + schedule, cadence-gated.""" + + def _reaper(self, lease, **kw): + kw.setdefault("sweep_interval_seconds", 300) + kw.setdefault("dedup_retention_seconds", 86400) + return PgReaper( + lease, interval_seconds=0.01, sweep_conn=object(), api_client=object(), **kw + ) + + def test_leader_sweeps(self, stub_retention_sweep): + reaper = self._reaper(_FakeLease(acquires=True, renews=True)) + conn = reaper._sweep_conn # capture once (don't re-fetch in the assertion) + with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]): + reaper.tick() + stub_retention_sweep.results.assert_called_once_with(conn) + stub_retention_sweep.dedup.assert_called_once_with(conn, 86400) + + def test_standby_does_not_sweep(self, stub_retention_sweep): + reaper = self._reaper(_FakeLease(acquires=False)) + with patch.object(reaper_mod, "recover_expired_barriers"): + reaper.tick() + stub_retention_sweep.results.assert_not_called() + stub_retention_sweep.dedup.assert_not_called() + + def test_cadence_gates_repeat_within_interval(self, stub_retention_sweep): + # Two leader ticks well within the 300s interval → swept once, not twice. + reaper = self._reaper(_FakeLease(acquires=True, renews=True)) + with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]): + reaper.tick() + reaper.tick() + assert stub_retention_sweep.results.call_count == 1 + assert stub_retention_sweep.dedup.call_count == 1 + + def test_runs_after_recovery_and_schedule( + self, stub_scheduler_tick, stub_retention_sweep + ): + order = [] + reaper = self._reaper(_FakeLease(acquires=True, renews=True)) + stub_scheduler_tick.side_effect = lambda *_: order.append("schedule") + stub_retention_sweep.results.side_effect = lambda *_: order.append("sweep") or 0 + with patch.object( + reaper_mod, + "recover_expired_barriers", + side_effect=lambda *_: order.append("recover") or [], + ): + reaper.tick() + assert order == ["recover", "schedule", "sweep"] + + def test_one_sweep_failing_does_not_starve_the_other( + self, stub_retention_sweep, caplog + ): + # [High] independence: a failing pg_task_result sweep must NOT skip the + # pg_batch_dedup sweep, must NOT propagate (cleanup mustn't fail the tick), + # and must discard the owned conn so the sibling reconnects. + reaper = PgReaper( + _FakeLease(acquires=True, renews=True), + interval_seconds=0.01, + sweep_interval_seconds=300, + api_client=object(), + ) + owned = MagicMock(closed=False) + reaper._sweep_conn = owned + stub_retention_sweep.results.side_effect = psycopg2.OperationalError("db gone") + with ( + patch.object( + reaper_mod, "create_pg_connection", return_value=MagicMock(closed=False) + ) as reconnect, + patch.object(reaper_mod, "recover_expired_barriers", return_value=[]), + caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.reaper"), + ): + reaper.tick() # must NOT raise + stub_retention_sweep.results.assert_called_once() # attempted + failed + stub_retention_sweep.dedup.assert_called_once() # sibling still ran + reconnect.assert_called_once() # reconnected after the discard + assert "retention sweep of pg_task_result failed (1 consecutive)" in caplog.text + + def test_failing_sweep_still_advances_cadence(self, stub_retention_sweep): + # The stamp is advanced before the sweep, so a failure waits one interval + # before retry (no DB hammering) — a second immediate tick is gated out. + reaper = self._reaper(_FakeLease(acquires=True, renews=True)) + stub_retention_sweep.results.side_effect = RuntimeError("boom") + with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]): + reaper.tick() + reaper.tick() + assert stub_retention_sweep.results.call_count == 1 # not retried immediately + + def test_logs_counts_only_when_rows_deleted(self, stub_retention_sweep, caplog): + reaper = self._reaper(_FakeLease(acquires=True, renews=True)) + stub_retention_sweep.results.return_value = 0 + stub_retention_sweep.dedup.return_value = 4 # one non-zero → still logs + with ( + patch.object(reaper_mod, "recover_expired_barriers", return_value=[]), + caplog.at_level(logging.INFO, logger="queue_backend.pg_queue.reaper"), + ): + reaper.tick() + assert "deleted 0 pg_task_result + 4 pg_batch_dedup row(s)" in caplog.text + + def test_no_log_when_nothing_deleted(self, stub_retention_sweep, caplog): + reaper = self._reaper(_FakeLease(acquires=True, renews=True)) + # both stubs default to return_value=0 + with ( + patch.object(reaper_mod, "recover_expired_barriers", return_value=[]), + caplog.at_level(logging.INFO, logger="queue_backend.pg_queue.reaper"), + ): + reaper.tick() + assert "retention sweep deleted" not in caplog.text + + # --- Layer 3: connection handling (mocked, no DB) --- From 20c6e0bc539af04e03d034d9a8ac0f3a728136ce Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:45:40 +0530 Subject: [PATCH 33/44] =?UTF-8?q?UN-3607=20[REFACTOR]=20retire=20the=20exe?= =?UTF-8?q?cutor=5Frpc=20mirror=20=E2=80=94=20shared=20dispatcher=20+=20in?= =?UTF-8?q?jected=20transport=20(#2102)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3607 [REFACTOR] retire the executor_rpc mirror — shared dispatcher + injected transport The backend and workers carried byte-for-byte mirrors of the executor-RPC dispatch (gate + reply_key/timeout orchestration + routing); the only real difference is the transport primitive (Django ORM vs psycopg2). Lift the shared logic into unstract.workflow_execution.executor_rpc (which both already depend on; it can't live in unstract.core because sdk1 imports core → circular) and inject the differing primitive via a QueueTransport Protocol — composition, not inheritance. - shared: PgExecutionDispatcher (concrete dispatch/async/with_callback + never-raises) calling transport.enqueue / wait_for_result; ExecResultRow (normalised result row); resolve_pg_transport (master-gate value supplied by caller, then Flipt); RoutingExecutionDispatcher (per-call PG-vs-Celery, injected celery/pg/resolve). - backend adapter: DjangoQueueTransport (enqueue_task + PgTaskResult poll) + settings master-gate + factory. - workers adapter: PgClientQueueTransport (PgQueueClient + PgResultBackend) + env master-gate + factory. ~600 duplicated lines collapse to one base + two ~40-line adapters → clears the SonarCloud duplication gate permanently. Behaviour is byte-identical (gate, routing, never-raises): zero regression. The dispatch contract + routing + gate matrix are tested ONCE against a fake transport (workers suite); each side's tests shrink to its adapter + factory wiring. Both call sites use the unchanged get_executor_dispatcher. Co-Authored-By: Claude Opus 4.8 * [FIX] vite dev proxy — forward websocket upgrades to the backend (local dev only) The Socket.IO log/result channel connects to /api/v1/socket with a websocket-only transport, but the Vite dev proxy only forwarded /api HTTP (no `ws: true`), so the upgrade never reached the backend and Prompt Studio results never streamed to the UI in local dev. ws-proxying was dropped in the CRA→Vite migration (the stale setupProxy.js comment is the leftover). Dev-server only: `server.proxy` runs solely under `vite dev`; staging/prod serve a built bundle behind nginx/Traefik (which already routes /api/v1/socket), so this has no effect on any deployed environment. Rides the UN-3607 PR. Co-Authored-By: Claude Opus 4.8 * UN-3607 [FIX] drop the unused headers param from the PG dispatcher (SonarCloud S1172) The PG path carries org/routing in the enqueue payload, not Celery headers, and the RoutingExecutionDispatcher strips fairness headers before delegating to the PG dispatcher — so headers was accepted-but-ignored on dispatch/dispatch_async/ dispatch_with_callback. SonarCloud flagged it as unused (L269/L297). Removed from all three for consistency; the RoutingExecutionDispatcher (the SDK-substitutable boundary) keeps headers on its public methods and forwards them only to Celery. Backend 7 + workers 36 green. Co-Authored-By: Claude Opus 4.8 * UN-3607 [FIX] address review — Protocol conformance, type the seam, de-dup the poll loop Toolkit review on #2102 (all 14): - Both transports now explicitly inherit QueueTransport so a type-checker verifies each implementation against the seam (not just at the construction site). - CallbackSignature Protocol in unstract.core types signature_to_continuation + dispatch_with_callback params (was Any); _CeleryDispatcher Protocol types the router's celery dependency (was Any). - Extracted the backend poll loop into a shared poll_for_row(fetch, timeout, between_polls=...) — the one duplicated logic that survived. - dispatch gets a docstring; the fire-and-forget enqueues log-before-raise (+ a note that a raised enqueue IS the failure signal since on_error can't fire); a COMPLETED row with no result is a distinct error; reworded the "strips headers" comment. - Workers master-gate warns on a non-true/false value (was a silent no-op). - Lint (I001/D209); tests: on_error translation + default task_id, dispatch_async propagates, backend multi-iteration poll. Backend 8 + workers 38 green. Co-Authored-By: Claude Opus 4.8 * UN-3607 [FIX] greptile — unify the PG poll loop (move poll_for_row to unstract.core) greptile P2: poll_for_row was backend-only — the workers PgResultBackend.wait_for_result still had its own identical backoff loop, so the constants lived in two places. Since poll_for_row is pure (no Django/psycopg/SDK deps), moved it to unstract.core.polling where BOTH PG result pollers import it: the backend DjangoQueueTransport and the workers PgResultBackend. The backoff now lives in exactly one place to tune. - new unstract.core.polling.poll_for_row (the shared backoff skeleton) - workflow_execution.executor_rpc: dropped poll_for_row (+ its time/TypeVar imports) - backend adapter + PgResultBackend.wait_for_result both delegate to it - behaviour-identical: backend 8 + workers 38 + result_backend 7 (real-PG) green Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- backend/pg_queue/executor_rpc.py | 375 +++--------- backend/pg_queue/tests/test_executor_rpc.py | 381 ++++--------- backend/uv.lock | 2 + frontend/vite.config.js | 6 + .../src/unstract/core/execution_dispatch.py | 22 +- unstract/core/src/unstract/core/polling.py | 48 ++ unstract/workflow-execution/pyproject.toml | 2 + .../workflow_execution/executor_rpc.py | 467 +++++++++++++++ .../queue_backend/pg_queue/executor_rpc.py | 430 +++----------- .../queue_backend/pg_queue/result_backend.py | 28 +- workers/tests/test_executor_rpc.py | 533 ++++++++---------- workers/uv.lock | 2 + 12 files changed, 1037 insertions(+), 1259 deletions(-) create mode 100644 unstract/core/src/unstract/core/polling.py create mode 100644 unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py diff --git a/backend/pg_queue/executor_rpc.py b/backend/pg_queue/executor_rpc.py index 72c0895a35..c50850b2e9 100644 --- a/backend/pg_queue/executor_rpc.py +++ b/backend/pg_queue/executor_rpc.py @@ -1,356 +1,117 @@ -"""Executor-RPC transport routing for the PG path (Phase 9). +"""Executor-RPC for the PG path — backend (Django) transport adapter. -The executor "RPC" is a synchronous request-reply: a caller (prompt-studio here) -sends an ``ExecutionContext`` to the executor worker and blocks for the -``ExecutionResult``. The legacy transport is Celery — the SDK -``ExecutionDispatcher`` (``send_task`` + ``AsyncResult.get``). This module adds a -**parallel** Postgres transport that leaves Celery and the SDK completely -untouched (no SDK edit, no change to the ``execute_extraction`` task or the -Celery executor worker): +The gate + reply_key/timeout orchestration + routing live ONCE in +``unstract.workflow_execution.executor_rpc`` (shared with the workers). This module +is the thin Django half: a :class:`DjangoQueueTransport` that enqueues via the ORM +(``enqueue_task``) and polls ``PgTaskResult``, plus the per-side gate (master switch = +``settings.PG_QUEUE_TRANSPORT_ENABLED``) and the :func:`get_executor_dispatcher` +factory that wires them together. -- :class:`PgExecutionDispatcher` enqueues ``execute_extraction`` onto the PG queue - with a unique ``reply_key`` and polls ``pg_task_result`` for the reply — same - ``.dispatch()`` contract as the SDK dispatcher (never raises; failure/timeout → - ``ExecutionResult.failure``). -- :func:`resolve_executor_transport` is the gate: master - ``PG_QUEUE_TRANSPORT_ENABLED`` then the **single** Flipt flag - ``pg_queue_enabled`` — the same flag the execution path uses, so one - flip turns the whole PG-queue feature on/off (no per-subsystem flags to - maintain). Fails closed to Celery. -- :class:`RoutingExecutionDispatcher` is what callers get from - ``PromptStudioHelper._get_dispatcher()``: ``dispatch()`` picks PG-vs-Celery - **per call** (read at dispatch time → flipping the flag is an instant, - no-redeploy rollout/rollback); ``dispatch_async`` / ``dispatch_with_callback`` - always delegate to Celery (the callback path is a later slice). - -Zero-regression: gate off ⇒ every method delegates to the unchanged Celery -``ExecutionDispatcher`` and no ``pg_task_result`` row is created. +Zero-regression: gate off ⇒ the routing dispatcher delegates every mode to the +unchanged Celery ``ExecutionDispatcher`` and no ``pg_task_result`` row is created. """ from __future__ import annotations -import logging -import os -import time -import uuid -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from django.conf import settings from django.db import close_old_connections -from pg_queue.flags import PG_QUEUE_FLAG_KEY from pg_queue.models import PgTaskResult from pg_queue.producer import enqueue_task -from unstract.core.data_models import PgTaskStatus -from unstract.core.execution_dispatch import DispatchHandle, signature_to_continuation -from unstract.flags.feature_flag import check_feature_flag_status +from unstract.core.polling import poll_for_row from unstract.sdk1.execution.dispatcher import ExecutionDispatcher -from unstract.sdk1.execution.result import ExecutionResult +from unstract.workflow_execution.executor_rpc import ( + EXECUTE_TASK, + ExecResultRow, + PgExecutionDispatcher, + QueueTransport, + RoutingExecutionDispatcher, + resolve_pg_transport, +) if TYPE_CHECKING: + from unstract.core.data_models import ContinuationSpec from unstract.sdk1.execution.context import ExecutionContext -logger = logging.getLogger(__name__) - -# Gating reads the single shared PG-queue flag (pg_queue.flags.PG_QUEUE_FLAG_KEY, -# imported above) — the same key execution and the scheduler use. -_EXECUTE_TASK = "execute_extraction" -# Mirror the SDK's queue-per-executor convention so the PG executor queue name -# matches the Celery one (the queue routes by the row's queue_name column). -_QUEUE_PREFIX = "celery_executor_" -# Caller-side wait default — mirrors the SDK dispatcher (EXECUTOR_RESULT_TIMEOUT -# env, else 3600s) so a PG-routed caller waits exactly as long as a Celery one. -_DEFAULT_TIMEOUT_ENV = "EXECUTOR_RESULT_TIMEOUT" -_DEFAULT_TIMEOUT = 3600 -_POLL_INITIAL_SECONDS = 0.2 -_POLL_MAX_SECONDS = 2.0 +# Re-exported so existing ``from pg_queue.executor_rpc import …`` imports keep working. +__all__ = [ + "DjangoQueueTransport", + "PgExecutionDispatcher", + "RoutingExecutionDispatcher", + "get_executor_dispatcher", + "resolve_executor_transport", +] def resolve_executor_transport(context: ExecutionContext) -> bool: """True → route this executor dispatch over PG; False → Celery (default). - Mirrors ``resolve_transport``: master-gated by ``PG_QUEUE_TRANSPORT_ENABLED``, - then the **single** ``pg_queue_enabled`` Flipt flag (shared across - the whole PG-queue feature), bucketed per org. **Fails closed to Celery** on a - closed gate, a blind Flipt, or any error — so the executor never silently - loses its transport. + The backend gate: master switch ``settings.PG_QUEUE_TRANSPORT_ENABLED``, then the + shared Flipt eval (single ``pg_queue_enabled`` flag, fail-closed). """ - if not settings.PG_QUEUE_TRANSPORT_ENABLED: - return False - if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true": - logger.warning( - "resolve_executor_transport: gate ON but FLIPT_SERVICE_AVAILABLE != " - "true (Flipt blind); using Celery" - ) - return False - org = getattr(context, "organization_id", None) - # %-bucket keyed on org (prompt-studio is org-scoped); fall back to run_id so - # a context without an org still resolves deterministically. - entity_id = str(org or getattr(context, "run_id", "") or "default") - flag_context = {"executor_name": str(context.executor_name)} - if org: - flag_context["organization_id"] = str(org) - try: - enabled = check_feature_flag_status( - flag_key=PG_QUEUE_FLAG_KEY, entity_id=entity_id, context=flag_context - ) - except Exception: - logger.warning( - "resolve_executor_transport: Flipt check failed; using Celery", - exc_info=True, - ) - return False - return bool(enabled) + return resolve_pg_transport( + context, master_gate_enabled=settings.PG_QUEUE_TRANSPORT_ENABLED + ) -class PgExecutionDispatcher: - """PG request-reply executor dispatch — drop-in for ``ExecutionDispatcher.dispatch``. +class DjangoQueueTransport(QueueTransport): + """:class:`QueueTransport` over the Django ORM (the backend half). - Enqueues ``execute_extraction`` with a unique ``reply_key`` and blocks on - ``pg_task_result`` until the executor consumer records the result or the - timeout elapses. Honours the same contract as the SDK dispatcher: it never - raises and converts a timeout/failure into ``ExecutionResult.failure`` so - callers can branch on ``result.success`` identically on either transport. + Inherits the Protocol so a type-checker verifies this implementation against the + seam independently of the ``PgExecutionDispatcher(...)`` construction site. """ - def dispatch( + def enqueue( self, + *, + queue: str, context: ExecutionContext, - timeout: int | None = None, - ) -> ExecutionResult: - if timeout is None: - # Guard the env parse so a misconfigured EXECUTOR_RESULT_TIMEOUT can't - # raise out of dispatch() (the never-raises contract). - try: - timeout = int(os.environ.get(_DEFAULT_TIMEOUT_ENV, _DEFAULT_TIMEOUT)) - except (TypeError, ValueError): - timeout = _DEFAULT_TIMEOUT - reply_key = str(uuid.uuid4()) - queue = f"{_QUEUE_PREFIX}{context.executor_name}" - org = getattr(context, "organization_id", "") or "" - try: - enqueue_task( - task_name=_EXECUTE_TASK, - queue=queue, - args=[context.to_dict()], - org_id=str(org), - reply_key=reply_key, - ) - except Exception as exc: - logger.exception( - "PG executor dispatch: enqueue failed (executor=%s run_id=%s)", - context.executor_name, - context.run_id, - ) - return ExecutionResult.failure(error=f"{type(exc).__name__}: {exc}") - logger.info( - "PG executor dispatch: enqueued reply_key=%s queue=%s run_id=%s " - "timeout=%ss; waiting for result...", - reply_key, - queue, - context.run_id, - timeout, - ) - row = self._wait_for_result(reply_key, timeout) - if row is None: - logger.warning( - "PG executor dispatch: TIMEOUT after %ss (reply_key=%s run_id=%s) — " - "the executor task may still be running", - timeout, - reply_key, - context.run_id, - ) - return ExecutionResult.failure( - error=f"TimeoutError: executor reply not received within {timeout}s" - ) - if row.status == PgTaskStatus.COMPLETED.value and row.result is not None: - try: - return ExecutionResult.from_dict(row.result) - except Exception: - # Honour the never-raises contract: a malformed completed row - # becomes a failure result, not a 500 to the caller. - logger.exception( - "PG executor dispatch: malformed completed result " - "(reply_key=%s run_id=%s)", - reply_key, - context.run_id, - ) - return ExecutionResult.failure( - error=f"Malformed executor result for reply_key {reply_key}" - ) - logger.warning( - "PG executor dispatch: executor reported failure (reply_key=%s " - "run_id=%s): %s", - reply_key, - context.run_id, - row.error or "(no error)", - ) - return ExecutionResult.failure(error=row.error or "executor task failed") - - def dispatch_async( - self, context: ExecutionContext, headers: dict[str, Any] | None = None - ) -> str: - """Fire-and-forget enqueue of ``execute_extraction``; returns the task id. - - The PG analogue of the SDK ``dispatch_async``: no ``reply_key``, no - callback, no blocking. There is no PG ``AsyncResult`` backend, so a caller - that needs the outcome uses :meth:`dispatch_with_callback` (a self-chained - continuation), not polling on this id. ``headers`` is accepted and ignored - (PG carries routing in the payload). Enqueue failures propagate — parity - with the SDK, which lets a broker error out of ``dispatch_async``. - """ - task_id = str(uuid.uuid4()) - queue = f"{_QUEUE_PREFIX}{context.executor_name}" - org = getattr(context, "organization_id", "") or "" - enqueue_task( - task_name=_EXECUTE_TASK, - queue=queue, - args=[context.to_dict()], - org_id=str(org), - task_id=task_id, - ) - logger.info( - "PG executor dispatch_async: enqueued task_id=%s queue=%s run_id=%s", - task_id, - queue, - context.run_id, - ) - return task_id - - def dispatch_with_callback( - self, - context: ExecutionContext, - on_success: Any | None = None, - on_error: Any | None = None, + org_id: str, + reply_key: str | None = None, + on_success: ContinuationSpec | None = None, + on_error: ContinuationSpec | None = None, task_id: str | None = None, - headers: dict[str, Any] | None = None, - ) -> DispatchHandle: - """Fire-and-forget enqueue with self-chained callbacks (§5 model). - - The PG analogue of the SDK ``dispatch_with_callback``: instead of Celery - ``link`` / ``link_error`` (which the broker fires), the on-success / - on-error Celery ``Signature``s are translated to serialisable - :class:`ContinuationSpec`s and carried in the payload. After the executor - consumer runs ``execute_extraction`` it self-chains the matching - continuation onto the callback queue. Returns a :class:`DispatchHandle` - exposing ``.id`` (== ``task_id``) so call sites read the task id exactly - as on the Celery path. ``headers`` is accepted and ignored. - """ - task_id = task_id or str(uuid.uuid4()) - queue = f"{_QUEUE_PREFIX}{context.executor_name}" - org = getattr(context, "organization_id", "") or "" - success_spec = signature_to_continuation(on_success) - error_spec = signature_to_continuation(on_error) + ) -> None: enqueue_task( - task_name=_EXECUTE_TASK, + task_name=EXECUTE_TASK, queue=queue, args=[context.to_dict()], - org_id=str(org), - on_success=success_spec, - on_error=error_spec, + org_id=org_id, + reply_key=reply_key, + on_success=on_success, + on_error=on_error, task_id=task_id, ) - logger.info( - "PG executor dispatch_with_callback: enqueued task_id=%s queue=%s " - "run_id=%s on_success=%s on_error=%s", - task_id, - queue, - context.run_id, - success_spec["task_name"] if success_spec else None, - error_spec["task_name"] if error_spec else None, - ) - return DispatchHandle(task_id) - @staticmethod - def _wait_for_result(reply_key: str, timeout: float) -> PgTaskResult | None: + def wait_for_result(self, reply_key: str, timeout: float) -> ExecResultRow | None: """Poll ``pg_task_result`` until the row appears or *timeout* elapses. - Poll-based with capped backoff (PgBouncer-safe; no LISTEN/NOTIFY). The DB - connection is released between polls (``close_old_connections``) so a - long-running RPC does not pin a backend connection for its whole duration - and exhaust the pool. Each poll is its own autocommit query, so a row - committed by the executor consumer becomes visible — **dispatch must NOT - be called inside an open transaction** (``transaction.atomic`` / - ``ATOMIC_REQUESTS`` would pin one snapshot and never see the new row). + Uses the shared :func:`poll_for_row` backoff skeleton, releasing the DB + connection between polls (``close_old_connections``) so a long-running RPC + does not pin a backend connection and exhaust the pool. Each poll is its own + autocommit query, so a row committed by the executor consumer becomes visible + — **dispatch must NOT be called inside an open transaction** + (``transaction.atomic`` / ``ATOMIC_REQUESTS`` would pin one snapshot and never + see the new row). """ - deadline = time.monotonic() + timeout - delay = _POLL_INITIAL_SECONDS - while True: + + def _fetch() -> ExecResultRow | None: row = PgTaskResult.objects.filter(pk=reply_key).first() - if row is not None: - return row - remaining = deadline - time.monotonic() - if remaining <= 0: + if row is None: return None - # Don't hold the connection idle through the sleep. - close_old_connections() - time.sleep(min(delay, remaining)) - delay = min(delay * 2, _POLL_MAX_SECONDS) - - -class RoutingExecutionDispatcher: - """Gate-routed executor dispatcher returned by ``_get_dispatcher()``. - - Every mode chooses PG vs Celery per call (instant rollout/rollback): - ``dispatch()`` (request-reply), ``dispatch_async`` (fire-and-forget) and - ``dispatch_with_callback`` (self-chained callbacks). Duck-typed against the SDK - ``ExecutionDispatcher`` so call sites are unchanged. - """ - - def __init__(self, celery_app: object | None = None) -> None: - self._celery = ExecutionDispatcher(celery_app=celery_app) - self._pg = PgExecutionDispatcher() - - def dispatch( - self, - context: ExecutionContext, - timeout: int | None = None, - headers: dict[str, Any] | None = None, - ) -> ExecutionResult: - if resolve_executor_transport(context): - logger.info( - "Executor RPC → PG transport (executor=%s run_id=%s)", - context.executor_name, - context.run_id, - ) - # PG carries fairness via the enqueue payload, not Celery headers, so - # the headers (fairness key) are intentionally not forwarded here. - return self._pg.dispatch(context, timeout=timeout) - return self._celery.dispatch(context, timeout=timeout, headers=headers) - - def dispatch_async( - self, context: ExecutionContext, headers: dict[str, Any] | None = None - ) -> str: - if resolve_executor_transport(context): - return self._pg.dispatch_async(context) - return self._celery.dispatch_async(context, headers=headers) + return ExecResultRow(status=row.status, result=row.result, error=row.error) - def dispatch_with_callback( - self, - context: ExecutionContext, - on_success: Any | None = None, - on_error: Any | None = None, - task_id: str | None = None, - headers: dict[str, Any] | None = None, - ) -> Any: - if resolve_executor_transport(context): - return self._pg.dispatch_with_callback( - context, - on_success=on_success, - on_error=on_error, - task_id=task_id, - ) - return self._celery.dispatch_with_callback( - context, - on_success=on_success, - on_error=on_error, - task_id=task_id, - headers=headers, - ) + return poll_for_row(_fetch, timeout, between_polls=close_old_connections) def get_executor_dispatcher( celery_app: object | None = None, ) -> RoutingExecutionDispatcher: """Factory: the gate-routed executor dispatcher (PG when enabled, else Celery).""" - return RoutingExecutionDispatcher(celery_app=celery_app) + return RoutingExecutionDispatcher( + celery=ExecutionDispatcher(celery_app=celery_app), + pg=PgExecutionDispatcher(DjangoQueueTransport()), + resolve=resolve_executor_transport, + ) diff --git a/backend/pg_queue/tests/test_executor_rpc.py b/backend/pg_queue/tests/test_executor_rpc.py index 7f5a2ed84f..1361b64f09 100644 --- a/backend/pg_queue/tests/test_executor_rpc.py +++ b/backend/pg_queue/tests/test_executor_rpc.py @@ -1,302 +1,125 @@ -"""Tests for the executor-RPC transport routing (Phase 9). - -DB-free: settings / Flipt / the sub-dispatchers are mocked. Pins the gate's -fail-closed matrix and — the load-bearing zero-regression property — that with -the gate off ``RoutingExecutionDispatcher`` delegates EVERY mode to the unchanged -Celery ``ExecutionDispatcher`` and never touches the PG path. +"""Tests for the backend executor-RPC adapter (UN-3607). + +The dispatch contract (never-raises / result interpretation), the routing, and the +Flipt gate matrix live ONCE in the shared module and are covered in +``workers/tests/test_executor_rpc.py`` against a fake transport. This suite pins the +**backend half**: the :class:`DjangoQueueTransport` (enqueue via the ORM, poll +``PgTaskResult`` → ``ExecResultRow``), the per-side gate (master switch = +``settings.PG_QUEUE_TRANSPORT_ENABLED``), and the factory wiring. """ from unittest.mock import MagicMock, patch +from unstract.workflow_execution.executor_rpc import ExecResultRow + from pg_queue.executor_rpc import ( - PgExecutionDispatcher, + DjangoQueueTransport, RoutingExecutionDispatcher, + get_executor_dispatcher, resolve_executor_transport, ) _MOD = "pg_queue.executor_rpc" -def _completed(result: dict) -> MagicMock: - return MagicMock(status="completed", result=result, error="") - - -def _ok_result() -> dict: - return {"success": True, "data": {"x": 1}, "metadata": {}, "error": None} - - -class TestPgExecutionDispatcherDispatch: - """The load-bearing contract: never raises; timeout/failure → failure result. - - DB-free — ``enqueue_task`` and ``_wait_for_result`` are mocked. - """ - - @staticmethod - def _ctx() -> MagicMock: - c = MagicMock() - c.executor_name = "legacy" - c.run_id = "r" - c.organization_id = "o" - c.to_dict.return_value = {"run_id": "r"} - return c - - def test_enqueue_failure_returns_failure_not_raise(self): - with patch(f"{_MOD}.enqueue_task", side_effect=RuntimeError("db down")): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is False - assert "RuntimeError" in res.error - - def test_timeout_returns_failure(self): - with ( - patch(f"{_MOD}.enqueue_task"), - patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=None), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=3) - assert res.success is False - assert "within 3s" in res.error - - def test_completed_row_returns_result(self): - with ( - patch(f"{_MOD}.enqueue_task"), - patch.object( - PgExecutionDispatcher, "_wait_for_result", - return_value=_completed(_ok_result()), - ), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is True - - def test_failed_row_returns_error(self): - row = MagicMock(status="failed", result=None, error="boom") - with ( - patch(f"{_MOD}.enqueue_task"), - patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is False - assert res.error == "boom" - - def test_failed_row_empty_error_falls_back(self): - row = MagicMock(status="failed", result=None, error="") - with ( - patch(f"{_MOD}.enqueue_task"), - patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is False - assert "executor task failed" in res.error - - def test_completed_but_result_none_is_failure(self): - row = MagicMock(status="completed", result=None, error="") - with ( - patch(f"{_MOD}.enqueue_task"), - patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is False - - def test_malformed_completed_row_is_failure_not_raise(self): - with ( - patch(f"{_MOD}.enqueue_task"), - patch.object( - PgExecutionDispatcher, "_wait_for_result", - return_value=_completed({"bad": "shape"}), - ), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is False - assert "Malformed" in res.error - - def test_timeout_none_reads_env_then_default(self, monkeypatch): - monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "42") - seen = {} - - def fake_wait(reply_key, timeout): - seen["timeout"] = timeout - return None - - with ( - patch(f"{_MOD}.enqueue_task"), - patch.object( - PgExecutionDispatcher, "_wait_for_result", side_effect=fake_wait - ), - ): - # No explicit timeout arg → falls back to the env/default. - PgExecutionDispatcher().dispatch(self._ctx()) - assert seen["timeout"] == 42 - - def test_timeout_none_bad_env_falls_back_to_default(self, monkeypatch): - monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "not-an-int") - seen = {} - - def fake_wait(reply_key, timeout): - seen["timeout"] = timeout - return None - - with ( - patch(f"{_MOD}.enqueue_task"), - patch.object( - PgExecutionDispatcher, "_wait_for_result", side_effect=fake_wait - ), - ): - PgExecutionDispatcher().dispatch(self._ctx()) # must not raise - assert seen["timeout"] == 3600 # _DEFAULT_TIMEOUT - - -def _ctx(org: str | None = "org1") -> MagicMock: +def _ctx() -> MagicMock: c = MagicMock() c.executor_name = "legacy" - c.run_id = "run-1" - c.organization_id = org + c.run_id = "r" + c.to_dict.return_value = {"run_id": "r"} return c -def _gate(on: bool): - s = MagicMock() - s.PG_QUEUE_TRANSPORT_ENABLED = on - return patch(f"{_MOD}.settings", s) - - -class TestResolveExecutorTransport: - def test_master_gate_off_is_celery(self, monkeypatch): - monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with _gate(False), patch(f"{_MOD}.check_feature_flag_status") as flag: - assert resolve_executor_transport(_ctx()) is False - flag.assert_not_called() # gate off → Flipt never consulted - - def test_flipt_unavailable_is_celery(self, monkeypatch): - monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false") - with _gate(True), patch(f"{_MOD}.check_feature_flag_status") as flag: - assert resolve_executor_transport(_ctx()) is False - flag.assert_not_called() - - def test_flag_true_is_pg_keyed_on_org(self, monkeypatch): - monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with _gate(True), patch( - f"{_MOD}.check_feature_flag_status", return_value=True - ) as flag: - assert resolve_executor_transport(_ctx("orgX")) is True - assert flag.call_args.kwargs["entity_id"] == "orgX" - # The single shared PG-queue flag (not a per-subsystem flag). - assert flag.call_args.kwargs["flag_key"] == "pg_queue_enabled" - - def test_flag_false_is_celery(self, monkeypatch): - monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with _gate(True), patch( - f"{_MOD}.check_feature_flag_status", return_value=False - ): - assert resolve_executor_transport(_ctx()) is False - - def test_flipt_error_fails_closed_to_celery(self, monkeypatch): - monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with _gate(True), patch( - f"{_MOD}.check_feature_flag_status", side_effect=RuntimeError("down") +class TestDjangoQueueTransportEnqueue: + def test_enqueue_calls_enqueue_task_with_request_reply_fields(self): + with patch(f"{_MOD}.enqueue_task") as enq: + DjangoQueueTransport().enqueue( + queue="celery_executor_legacy", context=_ctx(), org_id="org9", + reply_key="rk1", + ) + kw = enq.call_args.kwargs + assert kw["task_name"] == "execute_extraction" + assert kw["queue"] == "celery_executor_legacy" + assert kw["args"] == [{"run_id": "r"}] + assert kw["org_id"] == "org9" + assert kw["reply_key"] == "rk1" + assert kw["task_id"] is None and kw["on_success"] is None + + def test_enqueue_carries_continuations(self): + spec = {"task_name": "ide_prompt_complete", "kwargs": {}, "queue": "ide_callback"} + with patch(f"{_MOD}.enqueue_task") as enq: + DjangoQueueTransport().enqueue( + queue="celery_executor_legacy", context=_ctx(), org_id="o", + on_success=spec, task_id="tid-7", + ) + kw = enq.call_args.kwargs + assert kw["on_success"] == spec and kw["task_id"] == "tid-7" + assert kw["reply_key"] is None # callback dispatch, not request-reply + + +class TestDjangoQueueTransportWait: + def test_present_row_folds_to_exec_result_row(self): + row = MagicMock(status="completed", result={"a": 1}, error="") + qs = MagicMock() + qs.filter.return_value.first.return_value = row + with patch(f"{_MOD}.PgTaskResult", MagicMock(objects=qs)): + out = DjangoQueueTransport().wait_for_result("rk", timeout=5) + assert isinstance(out, ExecResultRow) + assert out.status == "completed" and out.result == {"a": 1} + + def test_timeout_returns_none(self): + qs = MagicMock() + qs.filter.return_value.first.return_value = None # never appears + with ( + patch(f"{_MOD}.PgTaskResult", MagicMock(objects=qs)), + patch(f"{_MOD}.close_old_connections"), ): - assert resolve_executor_transport(_ctx()) is False - - -class TestRoutingZeroRegression: - @staticmethod - def _build(): - # Patch both sub-dispatchers at construction; the instances are captured - # in __init__ so they remain mocked after the context exits. + # timeout=0 → first poll misses, remaining <= 0 → None (no sleep). + assert DjangoQueueTransport().wait_for_result("rk", timeout=0) is None + + def test_multi_iteration_poll_misses_then_hits(self): + # The load-bearing backoff path: a miss, then a hit on the next poll — the + # connection is released (close_old_connections) and it sleeps once between. + row = MagicMock(status="completed", result={"a": 1}, error="") + qs = MagicMock() + qs.filter.return_value.first.side_effect = [None, row] with ( - patch(f"{_MOD}.ExecutionDispatcher") as celery_cls, - patch(f"{_MOD}.PgExecutionDispatcher") as pg_cls, + patch(f"{_MOD}.PgTaskResult", MagicMock(objects=qs)), + patch(f"{_MOD}.close_old_connections") as cl, + patch("unstract.core.polling.time.sleep") as slp, ): - dispatcher = RoutingExecutionDispatcher(celery_app="app") - return dispatcher, celery_cls.return_value, pg_cls.return_value + out = DjangoQueueTransport().wait_for_result("rk", timeout=5) + assert isinstance(out, ExecResultRow) and out.result == {"a": 1} + cl.assert_called_once() # between_polls released the conn on the miss + slp.assert_called_once() # slept once between the two polls - def test_gate_off_dispatch_uses_celery_only(self): - dispatcher, celery, pg = self._build() - with patch(f"{_MOD}.resolve_executor_transport", return_value=False): - dispatcher.dispatch(_ctx()) - celery.dispatch.assert_called_once() - pg.dispatch.assert_not_called() # the zero-regression guarantee - - def test_gate_on_dispatch_uses_pg(self): - dispatcher, celery, pg = self._build() - with patch(f"{_MOD}.resolve_executor_transport", return_value=True): - dispatcher.dispatch(_ctx()) - pg.dispatch.assert_called_once() - celery.dispatch.assert_not_called() - - def test_async_and_callback_stay_celery_when_gate_off(self): - """Zero-regression: gate off → async/callback delegate to Celery unchanged.""" - dispatcher, celery, pg = self._build() - with patch(f"{_MOD}.resolve_executor_transport", return_value=False): - dispatcher.dispatch_async(_ctx(), headers={"h": 1}) - dispatcher.dispatch_with_callback(_ctx(), on_success="s", on_error="e") - celery.dispatch_async.assert_called_once() - celery.dispatch_with_callback.assert_called_once() - pg.dispatch_async.assert_not_called() - pg.dispatch_with_callback.assert_not_called() - - def test_async_and_callback_route_to_pg_when_gated(self): - """Gate on (③c) → async/callback take the PG self-chained path.""" - dispatcher, celery, pg = self._build() - with patch(f"{_MOD}.resolve_executor_transport", return_value=True): - dispatcher.dispatch_async(_ctx()) - dispatcher.dispatch_with_callback( - _ctx(), on_success="s", on_error="e", task_id="t" - ) - pg.dispatch_async.assert_called_once() - pg.dispatch_with_callback.assert_called_once() - assert "headers" not in pg.dispatch_with_callback.call_args.kwargs - celery.dispatch_async.assert_not_called() - celery.dispatch_with_callback.assert_not_called() - - -class TestPgAsyncCallbackWiring: - """PG fire-and-forget + self-chained-callback enqueue shapes (``enqueue_task`` - mocked). Pins that the async path carries NO reply_key and the callback path - carries the translated continuations + the tracking task_id. - """ +class TestResolveExecutorTransport: @staticmethod - def _ctx(): - c = MagicMock() - c.executor_name = "legacy" - c.run_id = "r" - c.organization_id = "org9" - c.to_dict.return_value = {"run_id": "r", "organization_id": "org9"} - return c - - def test_dispatch_async_is_fire_and_forget(self): - with patch(f"{_MOD}.enqueue_task") as enq: - task_id = PgExecutionDispatcher().dispatch_async(self._ctx()) - kwargs = enq.call_args.kwargs - assert kwargs["task_name"] == "execute_extraction" - assert kwargs["queue"] == "celery_executor_legacy" - assert kwargs["org_id"] == "org9" - assert kwargs["task_id"] == task_id - assert "reply_key" not in kwargs - assert "on_success" not in kwargs - - def test_dispatch_with_callback_carries_continuations(self): - on_s = MagicMock( - task="ide_prompt_complete", - args=(), - kwargs={"callback_kwargs": {"room": "r1"}}, - options={"queue": "ide_callback"}, - ) - on_e = MagicMock( - task="ide_prompt_error", - args=(), - kwargs={"callback_kwargs": {"room": "r1"}}, - options={"queue": "ide_callback"}, - ) - with patch(f"{_MOD}.enqueue_task") as enq: - handle = PgExecutionDispatcher().dispatch_with_callback( - self._ctx(), on_success=on_s, on_error=on_e, task_id="tid-7" - ) - assert handle.id == "tid-7" # call sites read .id off the handle - kwargs = enq.call_args.kwargs - assert kwargs["on_success"] == { - "task_name": "ide_prompt_complete", - "kwargs": {"callback_kwargs": {"room": "r1"}}, - "queue": "ide_callback", - } - assert kwargs["on_error"]["task_name"] == "ide_prompt_error" - assert kwargs["task_id"] == "tid-7" - assert "reply_key" not in kwargs + def _gate(on: bool): + s = MagicMock() + s.PG_QUEUE_TRANSPORT_ENABLED = on + return patch(f"{_MOD}.settings", s) + + def test_reads_settings_master_gate_on(self): + with self._gate(True), patch( + f"{_MOD}.resolve_pg_transport", return_value=True + ) as r: + assert resolve_executor_transport(_ctx()) is True + assert r.call_args.kwargs["master_gate_enabled"] is True + + def test_reads_settings_master_gate_off(self): + with self._gate(False), patch( + f"{_MOD}.resolve_pg_transport", return_value=False + ) as r: + resolve_executor_transport(_ctx()) + assert r.call_args.kwargs["master_gate_enabled"] is False + + +class TestFactoryWiring: + def test_factory_wires_routing_with_django_transport(self): + d = get_executor_dispatcher(celery_app="app") + assert isinstance(d, RoutingExecutionDispatcher) + # PG dispatcher uses the backend ORM transport; gate = the settings resolver. + assert isinstance(d._pg._transport, DjangoQueueTransport) + assert d._resolve is resolve_executor_transport diff --git a/backend/uv.lock b/backend/uv.lock index 6997c3b82a..10a787b064 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -4037,6 +4037,7 @@ dependencies = [ { name = "unstract-core" }, { name = "unstract-filesystem" }, { name = "unstract-flags" }, + { name = "unstract-sdk1" }, { name = "unstract-tool-registry" }, { name = "unstract-tool-sandbox" }, ] @@ -4046,6 +4047,7 @@ requires-dist = [ { name = "unstract-core", editable = "../unstract/core" }, { name = "unstract-filesystem", editable = "../unstract/filesystem" }, { name = "unstract-flags", editable = "../unstract/flags" }, + { name = "unstract-sdk1", editable = "../unstract/sdk1" }, { name = "unstract-tool-registry", editable = "../unstract/tool-registry" }, { name = "unstract-tool-sandbox", editable = "../unstract/tool-sandbox" }, ] diff --git a/frontend/vite.config.js b/frontend/vite.config.js index 397f001312..e00202c8f3 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -133,6 +133,12 @@ export default defineConfig(({ mode }) => { target: env.VITE_BACKEND_URL, changeOrigin: true, secure: false, + // Forward WebSocket upgrades too — the Socket.IO log/result + // channel connects to `/api/v1/socket` with a websocket-only + // transport. Without this the upgrade is never proxied to the + // backend and Prompt Studio results never stream to the UI in + // dev. (Prod is unaffected: Traefik routes /api/v1/socket.) + ws: true, }, } : undefined, diff --git a/unstract/core/src/unstract/core/execution_dispatch.py b/unstract/core/src/unstract/core/execution_dispatch.py index 57ddf02513..133f85a8dd 100644 --- a/unstract/core/src/unstract/core/execution_dispatch.py +++ b/unstract/core/src/unstract/core/execution_dispatch.py @@ -16,11 +16,27 @@ from __future__ import annotations -from typing import Any +from typing import Any, Protocol from .data_models import ContinuationSpec +class CallbackSignature(Protocol): + """Structural type of the Celery ``Signature`` the callback path accepts. + + Documents (and lets a type-checker enforce) the precondition + :func:`signature_to_continuation` reads: a task name, a ``queue`` in + ``options``, and kwargs-only (no positional ``args``). Defined here as a + Protocol — not imported from ``celery`` — so ``unstract.core`` stays + celery-free; a real ``celery.Signature`` conforms structurally. + """ + + task: str + options: dict[str, Any] + args: tuple[Any, ...] + kwargs: dict[str, Any] + + class DispatchHandle: """Minimal duck-type of Celery ``AsyncResult`` for the PG callback path. @@ -36,7 +52,9 @@ def __init__(self, task_id: str) -> None: self.id = task_id -def signature_to_continuation(sig: Any | None) -> ContinuationSpec | None: +def signature_to_continuation( + sig: CallbackSignature | None, +) -> ContinuationSpec | None: """Translate a Celery ``Signature`` to a serialisable continuation spec. Reads only the three attributes PG self-chaining needs — task name, kwargs, diff --git a/unstract/core/src/unstract/core/polling.py b/unstract/core/src/unstract/core/polling.py new file mode 100644 index 0000000000..c48379f568 --- /dev/null +++ b/unstract/core/src/unstract/core/polling.py @@ -0,0 +1,48 @@ +"""Shared polling helper for the PG result backends. + +A capped-exponential-backoff poll loop (PgBouncer-safe; no LISTEN/NOTIFY) used by +*both* PG result pollers — the backend's ``DjangoQueueTransport.wait_for_result`` +(Django ORM) and the workers' ``PgResultBackend.wait_for_result`` (psycopg2). It +has no Django / psycopg / SDK dependency, so it lives in ``unstract.core`` where +both trees import it — the backoff lives in exactly one place to tune. +""" + +from __future__ import annotations + +import time +from typing import TYPE_CHECKING, TypeVar + +if TYPE_CHECKING: + from collections.abc import Callable + +_T = TypeVar("_T") + + +def poll_for_row( + fetch: Callable[[], _T | None], + timeout: float, + *, + between_polls: Callable[[], None] | None = None, + initial: float = 0.2, + maximum: float = 2.0, +) -> _T | None: + """Poll ``fetch()`` until it returns non-``None`` or *timeout* elapses. + + Capped exponential backoff; the final sleep is clamped so we never overshoot the + deadline. ``between_polls`` runs once before each sleep — the backend poller + passes ``close_old_connections`` to release the DB connection between polls. + Returns the fetched row, or ``None`` on timeout. + """ + deadline = time.monotonic() + timeout + delay = initial + while True: + row = fetch() + if row is not None: + return row + remaining = deadline - time.monotonic() + if remaining <= 0: + return None + if between_polls is not None: + between_polls() + time.sleep(min(delay, remaining)) + delay = min(delay * 2, maximum) diff --git a/unstract/workflow-execution/pyproject.toml b/unstract/workflow-execution/pyproject.toml index a49c7b45a1..3680e82f4d 100644 --- a/unstract/workflow-execution/pyproject.toml +++ b/unstract/workflow-execution/pyproject.toml @@ -9,6 +9,7 @@ readme = "README.md" # dependencies = [ "unstract-core", + "unstract-sdk1", "unstract-tool-registry", "unstract-tool-sandbox", "unstract-flags", @@ -21,6 +22,7 @@ unstract-tool-sandbox = { path = "../tool-sandbox", editable = true } unstract-tool-registry = { path = "../tool-registry", editable = true } unstract-flags = { path = "../flags", editable = true } unstract-core = { path = "../core", editable = true } +unstract-sdk1 = { path = "../sdk1", editable = true } # [build-system] # requires = ["hatchling"] diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py new file mode 100644 index 0000000000..5d256a9132 --- /dev/null +++ b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py @@ -0,0 +1,467 @@ +"""Shared executor-RPC dispatch for the PG path — the gate + reply_key orchestration. + +The executor "RPC" is a synchronous request-reply: a caller sends an +``ExecutionContext`` to the executor worker and blocks for the ``ExecutionResult``. +The legacy transport is Celery (the SDK ``ExecutionDispatcher``); the PG path adds a +parallel Postgres transport. Backend (Django/prompt-studio) and workers +(``structure_tool``) both need it, and used to carry **byte-for-byte mirrors** of +this logic — the only thing that genuinely differs between them is the *transport +primitive*: the backend enqueues via the Django ORM (``enqueue_task`` + +``PgTaskResult``), the workers via psycopg2 (``PgQueueClient`` + ``PgResultBackend``). + +So this module owns everything transport-agnostic exactly once, and the differing +primitive is **injected** (composition, not inheritance) via :class:`QueueTransport`: + +- :class:`PgExecutionDispatcher` — concrete; ``dispatch`` / ``dispatch_async`` / + ``dispatch_with_callback`` + the reply_key/timeout orchestration and the + never-raises contract (timeout/failure → ``ExecutionResult.failure``). It calls + ``transport.enqueue(...)`` and ``transport.wait_for_result(...)``. +- :func:`resolve_pg_transport` — the gate: a master kill-switch (its boolean value + supplied by the caller — a Django setting on the backend, an env var on the + workers) then the single ``pg_queue_enabled`` Flipt flag, bucketed per org. Fails + closed to Celery. +- :class:`RoutingExecutionDispatcher` — picks PG-vs-Celery per call (instant + rollout/rollback) for every mode; the Celery dispatcher, the PG dispatcher and the + per-side ``resolve`` are all injected. + +It lives in ``unstract-workflow-execution`` (which both backend and workers already +depend on) rather than ``unstract.core`` because it needs ``unstract.sdk1`` and +``sdk1`` imports ``core`` — hosting it in ``core`` would be circular. It has no +Django/psycopg2 dependency: those live entirely in the injected transport. +""" + +from __future__ import annotations + +import logging +import os +import uuid +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Protocol + +from unstract.core.data_models import PgTaskStatus +from unstract.core.execution_dispatch import DispatchHandle, signature_to_continuation +from unstract.flags.feature_flag import check_feature_flag_status +from unstract.sdk1.execution.result import ExecutionResult + +if TYPE_CHECKING: + from collections.abc import Callable + + from unstract.core.data_models import ContinuationSpec + from unstract.core.execution_dispatch import CallbackSignature + from unstract.sdk1.execution.context import ExecutionContext + +logger = logging.getLogger(__name__) + +# The single PG-queue rollout flag — the same key execution and the scheduler read, +# so one flip turns the whole PG-queue feature on/off. +PG_QUEUE_FLAG_KEY = "pg_queue_enabled" +EXECUTE_TASK = "execute_extraction" +# Mirror the SDK's queue-per-executor convention so the PG executor queue name +# matches the Celery one (the worker-pg-executor consumer subscribes to these). +QUEUE_PREFIX = "celery_executor_" +# Caller-side wait default — mirrors the SDK dispatcher (EXECUTOR_RESULT_TIMEOUT env, +# else 3600s) so a PG-routed caller waits exactly as long as a Celery one. +DEFAULT_TIMEOUT_ENV = "EXECUTOR_RESULT_TIMEOUT" +DEFAULT_TIMEOUT = 3600 + + +class _CeleryDispatcher(Protocol): + """The subset of the SDK ``ExecutionDispatcher`` the router delegates to on the + Celery path — so ``RoutingExecutionDispatcher`` holds it typed, not as ``Any``. + """ + + def dispatch( + self, + context: ExecutionContext, + timeout: int | None = ..., + headers: dict[str, Any] | None = ..., + ) -> ExecutionResult: ... + + def dispatch_async( + self, context: ExecutionContext, headers: dict[str, Any] | None = ... + ) -> str: ... + + def dispatch_with_callback( + self, + context: ExecutionContext, + on_success: CallbackSignature | None = ..., + on_error: CallbackSignature | None = ..., + task_id: str | None = ..., + headers: dict[str, Any] | None = ..., + ) -> Any: ... + + +@dataclass +class ExecResultRow: + """Normalised executor-RPC result row — the transport-agnostic shape + :meth:`PgExecutionDispatcher.dispatch` interprets. + + The backend's result row is a Django model (attribute access) and the workers' + is a ``dict``; both fold to this so ``dispatch`` has one code path. Every field + defaults to ``None`` — the never-raises contract must not depend on the producer + having written every key. + """ + + status: str | None = None + result: dict | None = None + error: str | None = None + + +class QueueTransport(Protocol): + """The per-side primitive the shared dispatcher needs — the ONLY thing that + differs between backend and workers. + + ``enqueue`` writes one ``execute_extraction`` request-row (the optional keys + select the dispatch shape: ``reply_key`` → request-reply; ``on_success`` / + ``on_error`` / ``task_id`` → async/callback). ``wait_for_result`` polls for the + reply and returns a normalised :class:`ExecResultRow` (or ``None`` on timeout). + """ + + def enqueue( + self, + *, + queue: str, + context: ExecutionContext, + org_id: str, + reply_key: str | None = None, + on_success: ContinuationSpec | None = None, + on_error: ContinuationSpec | None = None, + task_id: str | None = None, + ) -> None: ... + + def wait_for_result(self, reply_key: str, timeout: float) -> ExecResultRow | None: ... + + +def resolve_pg_transport( + context: ExecutionContext, + *, + master_gate_enabled: bool, + flag_key: str = PG_QUEUE_FLAG_KEY, +) -> bool: + """True → route this executor dispatch over PG; False → Celery (default). + + Master-gated by ``master_gate_enabled`` (the caller supplies its value — a Django + setting on the backend, an env var on the workers), then the single + ``pg_queue_enabled`` Flipt flag, bucketed per org. **Fails closed to Celery** on a + closed gate, a blind Flipt, or any error — so the executor never silently loses + its transport. + """ + if not master_gate_enabled: + return False + if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true": + logger.warning( + "resolve_pg_transport: gate ON but FLIPT_SERVICE_AVAILABLE != true " + "(Flipt blind); using Celery" + ) + return False + org = getattr(context, "organization_id", None) + # %-bucket keyed on org; fall back to run_id so a context without an org still + # resolves deterministically. + entity_id = str(org or getattr(context, "run_id", "") or "default") + flag_context = {"executor_name": str(context.executor_name)} + if org: + flag_context["organization_id"] = str(org) + try: + enabled = check_feature_flag_status( + flag_key=flag_key, entity_id=entity_id, context=flag_context + ) + except Exception: + logger.warning( + "resolve_pg_transport: Flipt check failed; using Celery", exc_info=True + ) + return False + return bool(enabled) + + +def _resolve_timeout(timeout: int | None) -> int: + """Caller timeout, defaulting to ``EXECUTOR_RESULT_TIMEOUT`` env then 3600s. + + Guarded so a misconfigured env value can't raise out of ``dispatch`` (the + never-raises contract) — it logs and falls back instead of silently waiting the + full default with no signal. + """ + if timeout is not None: + return timeout + try: + return int(os.environ.get(DEFAULT_TIMEOUT_ENV, DEFAULT_TIMEOUT)) + except (TypeError, ValueError): + logger.warning( + "PG executor dispatch: invalid %s=%r; falling back to %ss", + DEFAULT_TIMEOUT_ENV, + os.environ.get(DEFAULT_TIMEOUT_ENV), + DEFAULT_TIMEOUT, + ) + return DEFAULT_TIMEOUT + + +class PgExecutionDispatcher: + """PG request-reply executor dispatch — drop-in for the SDK dispatch contract. + + Concrete + transport-injected: enqueues ``execute_extraction`` with a unique + ``reply_key`` and blocks on the result row until the executor consumer records it + or the timeout elapses. Honours the SDK dispatcher contract: it never raises and + converts a timeout/failure into ``ExecutionResult.failure`` so callers branch on + ``result.success`` identically on either transport. + """ + + def __init__(self, transport: QueueTransport) -> None: + self._transport = transport + + def dispatch( + self, + context: ExecutionContext, + timeout: int | None = None, + ) -> ExecutionResult: + """Send ``execute_extraction`` and block for the result (request-reply). + + Enqueues with a unique ``reply_key``, polls the result row until it appears + or *timeout* elapses, and converts the outcome to an ``ExecutionResult``. + Never raises (the SDK dispatch contract): an enqueue/poll failure, a timeout, + or a malformed/failed/empty result all become ``ExecutionResult.failure`` so + callers branch on ``result.success`` identically on either transport. + + No ``headers`` on any PG dispatch method: the PG path carries org/routing in + the enqueue payload, not Celery headers, so the ``RoutingExecutionDispatcher`` + does not forward fairness headers to the PG path. + """ + timeout = _resolve_timeout(timeout) + reply_key = str(uuid.uuid4()) + queue = f"{QUEUE_PREFIX}{context.executor_name}" + org = str(getattr(context, "organization_id", "") or "") + try: + self._transport.enqueue( + queue=queue, context=context, org_id=org, reply_key=reply_key + ) + except Exception as exc: + logger.exception( + "PG executor dispatch: enqueue failed (executor=%s run_id=%s)", + context.executor_name, + context.run_id, + ) + return ExecutionResult.failure(error=f"{type(exc).__name__}: {exc}") + logger.info( + "PG executor dispatch: enqueued reply_key=%s queue=%s run_id=%s " + "timeout=%ss; waiting for result...", + reply_key, + queue, + context.run_id, + timeout, + ) + try: + row = self._transport.wait_for_result(reply_key, timeout) + except Exception as exc: + # Honour the never-raises contract even if the poll connection dies. + logger.exception( + "PG executor dispatch: wait failed (reply_key=%s run_id=%s)", + reply_key, + context.run_id, + ) + return ExecutionResult.failure(error=f"{type(exc).__name__}: {exc}") + if row is None: + # On timeout the executor task may still be running on the consumer; it + # writes its outcome under this reply_key, but we've given up reading it + # (the reaper retention-sweeps the orphan row). A retry re-dispatches with + # a FRESH reply_key — at-least-once + caller-timeout by design. + logger.warning( + "PG executor dispatch: TIMEOUT after %ss (reply_key=%s run_id=%s) — " + "the executor task may still be running", + timeout, + reply_key, + context.run_id, + ) + return ExecutionResult.failure( + error=f"TimeoutError: executor reply not received within {timeout}s" + ) + if row.status == PgTaskStatus.COMPLETED.value and row.result is not None: + try: + return ExecutionResult.from_dict(row.result) + except Exception as exc: + # A malformed completed row becomes a failure result, not a raise. + # Surface the parse cause so a UI reading result.error isn't left with + # an opaque message. + logger.exception( + "PG executor dispatch: malformed completed result " + "(reply_key=%s run_id=%s)", + reply_key, + context.run_id, + ) + return ExecutionResult.failure( + error=( + f"Malformed executor result ({type(exc).__name__}) " + f"for reply_key {reply_key}" + ) + ) + if row.status == PgTaskStatus.COMPLETED.value: + # COMPLETED but result is None — a producer-side anomaly (the consumer + # recorded success yet wrote no payload). Distinguish it from a real task + # failure so it isn't mislabelled "executor task failed". + logger.warning( + "PG executor dispatch: completed row has no result " + "(reply_key=%s run_id=%s)", + reply_key, + context.run_id, + ) + return ExecutionResult.failure( + error=f"Executor reported completion with no result (reply_key {reply_key})" + ) + logger.warning( + "PG executor dispatch: executor reported failure (reply_key=%s " + "run_id=%s): %s", + reply_key, + context.run_id, + row.error or "(no error)", + ) + return ExecutionResult.failure(error=row.error or "executor task failed") + + def dispatch_async(self, context: ExecutionContext) -> str: + """Fire-and-forget enqueue of ``execute_extraction``; returns the task id. + + No ``reply_key``, no callback, no blocking. A caller that needs the outcome + uses :meth:`dispatch_with_callback` (a self-chained continuation), not polling + on this id. Enqueue failures **propagate** — parity with the SDK, which lets a + broker error out of ``dispatch_async`` — but are logged here first so the + failure is observable even if the caller swallows it. + """ + task_id = str(uuid.uuid4()) + queue = f"{QUEUE_PREFIX}{context.executor_name}" + org = str(getattr(context, "organization_id", "") or "") + try: + self._transport.enqueue( + queue=queue, context=context, org_id=org, task_id=task_id + ) + except Exception: + # The enqueue is the only fallible step (fire-and-forget). Log before the + # re-raise so a swallowed error is still observable. + logger.exception( + "PG executor dispatch_async: enqueue failed (executor=%s run_id=%s)", + context.executor_name, + context.run_id, + ) + raise + logger.info( + "PG executor dispatch_async: enqueued task_id=%s queue=%s run_id=%s", + task_id, + queue, + context.run_id, + ) + return task_id + + def dispatch_with_callback( + self, + context: ExecutionContext, + on_success: CallbackSignature | None = None, + on_error: CallbackSignature | None = None, + task_id: str | None = None, + ) -> DispatchHandle: + """Fire-and-forget enqueue with self-chained callbacks (§5 model). + + Instead of Celery ``link`` / ``link_error``, the on-success / on-error Celery + ``Signature``s are translated to serialisable ``ContinuationSpec``s and + carried in the payload; after the executor runs, the consumer self-chains the + matching continuation. Returns a :class:`DispatchHandle` exposing ``.id`` + (== ``task_id``) so call sites read the task id exactly as on the Celery path. + + Enqueue failures **propagate** (parity with :meth:`dispatch_async`), logged + first. NOTE: because the continuations are carried *in the payload*, a failed + enqueue means the executor never runs and ``on_error`` never fires — so a + caller MUST treat a raised enqueue as the failure signal itself (the + prompt-studio views do, in their own try/except). + """ + task_id = task_id or str(uuid.uuid4()) + queue = f"{QUEUE_PREFIX}{context.executor_name}" + org = str(getattr(context, "organization_id", "") or "") + success_spec = signature_to_continuation(on_success) + error_spec = signature_to_continuation(on_error) + try: + self._transport.enqueue( + queue=queue, + context=context, + org_id=org, + on_success=success_spec, + on_error=error_spec, + task_id=task_id, + ) + except Exception: + logger.exception( + "PG executor dispatch_with_callback: enqueue failed — on_error will " + "NOT fire (executor=%s run_id=%s)", + context.executor_name, + context.run_id, + ) + raise + logger.info( + "PG executor dispatch_with_callback: enqueued task_id=%s queue=%s " + "run_id=%s on_success=%s on_error=%s", + task_id, + queue, + context.run_id, + success_spec["task_name"] if success_spec else None, + error_spec["task_name"] if error_spec else None, + ) + return DispatchHandle(task_id) + + +class RoutingExecutionDispatcher: + """Gate-routed executor dispatcher: every mode picks PG vs Celery per call + (read at dispatch time → flipping the flag is an instant, no-redeploy + rollout/rollback). Duck-typed against the SDK ``ExecutionDispatcher`` so call + sites are unchanged. + + Composition-injected: the Celery dispatcher (``celery``), the PG dispatcher + (``pg``) and the per-side gate (``resolve(context) -> bool``). + """ + + def __init__( + self, + *, + celery: _CeleryDispatcher, + pg: PgExecutionDispatcher, + resolve: Callable[[ExecutionContext], bool], + ) -> None: + self._celery = celery + self._pg = pg + self._resolve = resolve + + def dispatch( + self, + context: ExecutionContext, + timeout: int | None = None, + headers: dict[str, Any] | None = None, + ) -> ExecutionResult: + if self._resolve(context): + logger.info( + "Executor RPC → PG transport (executor=%s run_id=%s)", + context.executor_name, + context.run_id, + ) + # PG carries org/routing via the enqueue payload, not Celery headers, so + # the fairness headers are intentionally not forwarded here. + return self._pg.dispatch(context, timeout=timeout) + return self._celery.dispatch(context, timeout=timeout, headers=headers) + + def dispatch_async( + self, context: ExecutionContext, headers: dict[str, Any] | None = None + ) -> str: + if self._resolve(context): + return self._pg.dispatch_async(context) + return self._celery.dispatch_async(context, headers=headers) + + def dispatch_with_callback( + self, + context: ExecutionContext, + on_success: CallbackSignature | None = None, + on_error: CallbackSignature | None = None, + task_id: str | None = None, + headers: dict[str, Any] | None = None, + ) -> Any: + if self._resolve(context): + return self._pg.dispatch_with_callback( + context, on_success=on_success, on_error=on_error, task_id=task_id + ) + return self._celery.dispatch_with_callback( + context, + on_success=on_success, + on_error=on_error, + task_id=task_id, + headers=headers, + ) diff --git a/workers/queue_backend/pg_queue/executor_rpc.py b/workers/queue_backend/pg_queue/executor_rpc.py index a43754b84b..1b3367bb03 100644 --- a/workers/queue_backend/pg_queue/executor_rpc.py +++ b/workers/queue_backend/pg_queue/executor_rpc.py @@ -1,328 +1,104 @@ -"""Executor-RPC transport routing for the PG path — workers side (Phase 9, ③b-2). - -The workers-side twin of ``backend/pg_queue/executor_rpc.py``. The executor "RPC" -is a synchronous request-reply: a caller (here, the ``structure_tool`` task in the -file_processing worker) sends an ``ExecutionContext`` to the executor worker and -blocks for the ``ExecutionResult``. The legacy transport is Celery — the SDK -``ExecutionDispatcher`` (``send_task`` + ``AsyncResult.get``). This module adds a -**parallel** Postgres transport that leaves Celery and the SDK completely -untouched (no SDK edit, no change to the ``execute_extraction`` task or the Celery -executor worker): - -- :class:`PgExecutionDispatcher` enqueues ``execute_extraction`` onto the PG queue - (via :class:`~queue_backend.pg_queue.client.PgQueueClient`) with a unique - ``reply_key`` and polls ``pg_task_result`` (via - :class:`~queue_backend.pg_queue.result_backend.PgResultBackend`) for the reply — - same ``.dispatch()`` contract as the SDK dispatcher (never raises; - failure/timeout → ``ExecutionResult.failure``). The already-running - ``worker-pg-executor`` consumer runs the task and writes the reply, so this side - only adds the enqueue + poll halves. -- :func:`resolve_executor_transport` is the gate: master - ``PG_QUEUE_TRANSPORT_ENABLED`` (env, the workers analogue of the backend's - Django setting) then the **single** Flipt flag ``pg_queue_enabled`` — the same - flag the execution path uses, so one flip turns the whole PG-queue feature - on/off. Fails closed to Celery. -- :class:`RoutingExecutionDispatcher` is what ``structure_tool`` gets from - :func:`get_executor_dispatcher`: ``dispatch()`` picks PG-vs-Celery **per call** - (read at dispatch time → flipping the flag is an instant, no-redeploy - rollout/rollback); ``dispatch_async`` / ``dispatch_with_callback`` always - delegate to Celery (the callback path is a later slice). - -Zero-regression: gate off ⇒ every method delegates to the unchanged Celery -``ExecutionDispatcher`` and no ``pg_task_result`` row is created. - -TODO(shared): ``resolve_executor_transport`` and the reply_key/poll orchestration -mirror the backend module almost verbatim — only the transport primitives differ -(psycopg2 here vs Django ORM there). A later slice can lift the shared logic into -``unstract.core`` so the gate/contract lives in one place. +"""Executor-RPC for the PG path — workers (psycopg2) transport adapter. + +The gate + reply_key/timeout orchestration + routing live ONCE in +``unstract.workflow_execution.executor_rpc`` (shared with the backend). This module +is the thin psycopg2 half: a :class:`PgClientQueueTransport` that enqueues via +:class:`~queue_backend.pg_queue.client.PgQueueClient` and polls via +:class:`~queue_backend.pg_queue.result_backend.PgResultBackend`, plus the per-side +gate (master switch = the ``PG_QUEUE_TRANSPORT_ENABLED`` env, the workers analogue of +the backend's Django setting) and the :func:`get_executor_dispatcher` factory. + +Zero-regression: gate off ⇒ the routing dispatcher delegates every mode to the +unchanged Celery ``ExecutionDispatcher`` and no ``pg_task_result`` row is created. """ from __future__ import annotations import logging import os -import uuid -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING -from unstract.core.data_models import ContinuationSpec, PgTaskStatus -from unstract.core.execution_dispatch import DispatchHandle, signature_to_continuation -from unstract.flags.feature_flag import check_feature_flag_status from unstract.sdk1.execution.dispatcher import ExecutionDispatcher -from unstract.sdk1.execution.result import ExecutionResult +from unstract.workflow_execution.executor_rpc import ( + EXECUTE_TASK, + ExecResultRow, + PgExecutionDispatcher, + QueueTransport, + RoutingExecutionDispatcher, + resolve_pg_transport, +) from .client import PgQueueClient from .result_backend import PgResultBackend from .task_payload import to_payload if TYPE_CHECKING: + from unstract.core.data_models import ContinuationSpec from unstract.sdk1.execution.context import ExecutionContext +# Re-exported so existing ``from queue_backend.pg_queue.executor_rpc import …`` +# imports keep working. +__all__ = [ + "PgClientQueueTransport", + "PgExecutionDispatcher", + "RoutingExecutionDispatcher", + "get_executor_dispatcher", + "resolve_executor_transport", +] + logger = logging.getLogger(__name__) -# The single PG-queue rollout flag — same key the execution path and scheduler -# read. Defined in backend/pg_queue/flags.py too; kept as a literal here because -# workers can't import backend code (see TODO(shared) in the module docstring). -PG_QUEUE_FLAG_KEY = "pg_queue_enabled" -# Master kill-switch + deploy-ordering gate. The workers analogue of the backend's -# ``settings.PG_QUEUE_TRANSPORT_ENABLED`` — read straight from the env here. +# Master kill-switch + deploy-ordering gate — the workers analogue of the backend's +# ``settings.PG_QUEUE_TRANSPORT_ENABLED``, read straight from the env here. _MASTER_GATE_ENV = "PG_QUEUE_TRANSPORT_ENABLED" - -_EXECUTE_TASK = "execute_extraction" -# Mirror the SDK's queue-per-executor convention so the PG executor queue name -# matches the Celery one (the worker-pg-executor consumer subscribes to these). -_QUEUE_PREFIX = "celery_executor_" -# Caller-side wait default — mirrors the SDK dispatcher (EXECUTOR_RESULT_TIMEOUT -# env, else 3600s) so a PG-routed caller waits exactly as long as a Celery one. -_DEFAULT_TIMEOUT_ENV = "EXECUTOR_RESULT_TIMEOUT" -_DEFAULT_TIMEOUT = 3600 +_TRUE = "true" +_FALSE = "false" def resolve_executor_transport(context: ExecutionContext) -> bool: """True → route this executor dispatch over PG; False → Celery (default). - Mirrors the backend ``resolve_executor_transport``: master-gated by the - ``PG_QUEUE_TRANSPORT_ENABLED`` env, then the **single** ``pg_queue_enabled`` - Flipt flag (shared across the whole PG-queue feature), bucketed per org. - **Fails closed to Celery** on a closed gate, a blind Flipt, or any error — so - the executor never silently loses its transport. + The workers gate: master switch = the ``PG_QUEUE_TRANSPORT_ENABLED`` env, then the + shared Flipt eval (single ``pg_queue_enabled`` flag, fail-closed). """ - if os.environ.get(_MASTER_GATE_ENV, "false").lower() != "true": - return False - if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true": + raw = os.environ.get(_MASTER_GATE_ENV, _FALSE) + master = raw.strip().lower() == _TRUE + if not master and raw.strip().lower() != _FALSE: + # A fat-fingered value ("1"/"yes"/"on"/" True ") parses to OFF — warn so it + # isn't a silent no-op for an operator who expected it to enable PG. logger.warning( - "resolve_executor_transport: gate ON but FLIPT_SERVICE_AVAILABLE != " - "true (Flipt blind); using Celery" - ) - return False - org = getattr(context, "organization_id", None) - # %-bucket keyed on org; fall back to run_id so a context without an org still - # resolves deterministically (mirrors the backend resolver). - entity_id = str(org or getattr(context, "run_id", "") or "default") - flag_context = {"executor_name": str(context.executor_name)} - if org: - flag_context["organization_id"] = str(org) - try: - enabled = check_feature_flag_status( - flag_key=PG_QUEUE_FLAG_KEY, entity_id=entity_id, context=flag_context + "resolve_executor_transport: %s=%r is not 'true'/'false' — treating as " + "OFF (PG transport disabled); use exactly 'true' to enable", + _MASTER_GATE_ENV, + raw, ) - except Exception: - logger.warning( - "resolve_executor_transport: Flipt check failed; using Celery", - exc_info=True, - ) - return False - return bool(enabled) + return resolve_pg_transport(context, master_gate_enabled=master) -class PgExecutionDispatcher: - """PG request-reply executor dispatch — drop-in for ``ExecutionDispatcher.dispatch``. +class PgClientQueueTransport(QueueTransport): + """:class:`QueueTransport` over psycopg2 (the workers half). - Enqueues ``execute_extraction`` with a unique ``reply_key`` and blocks on - ``pg_task_result`` until the executor consumer records the result or the - timeout elapses. Honours the same contract as the SDK dispatcher: it never - raises and converts a timeout/failure into ``ExecutionResult.failure`` so - callers can branch on ``result.success`` identically on either transport. + Inherits the Protocol so a type-checker verifies this implementation against the + seam independently of the ``PgExecutionDispatcher(...)`` construction site. """ - def dispatch( + def enqueue( self, - context: ExecutionContext, - timeout: int | None = None, - headers: dict[str, Any] | None = None, - ) -> ExecutionResult: - # ``headers`` is accepted (and ignored) for substitutability with the SDK - # ``ExecutionDispatcher.dispatch`` / ``RoutingExecutionDispatcher.dispatch`` - # shapes — the PG path carries org/routing via the enqueue payload, not - # Celery headers, so fairness headers are intentionally not forwarded. - if timeout is None: - # Guard the env parse so a misconfigured EXECUTOR_RESULT_TIMEOUT can't - # raise out of dispatch() (the never-raises contract). - try: - timeout = int(os.environ.get(_DEFAULT_TIMEOUT_ENV, _DEFAULT_TIMEOUT)) - except (TypeError, ValueError): - # Don't swallow silently — an operator who fat-fingers the value - # would otherwise wait the 3600s default with no signal. - logger.warning( - "PG executor dispatch: invalid %s=%r; falling back to %ss", - _DEFAULT_TIMEOUT_ENV, - os.environ.get(_DEFAULT_TIMEOUT_ENV), - _DEFAULT_TIMEOUT, - ) - timeout = _DEFAULT_TIMEOUT - reply_key = str(uuid.uuid4()) - queue = f"{_QUEUE_PREFIX}{context.executor_name}" - org = str(getattr(context, "organization_id", "") or "") - try: - self._enqueue(queue, context, org, reply_key=reply_key) - except Exception as exc: - logger.exception( - "PG executor dispatch: enqueue failed (executor=%s run_id=%s)", - context.executor_name, - context.run_id, - ) - return ExecutionResult.failure(error=f"{type(exc).__name__}: {exc}") - logger.info( - "PG executor dispatch: enqueued reply_key=%s queue=%s run_id=%s " - "timeout=%ss; waiting for result...", - reply_key, - queue, - context.run_id, - timeout, - ) - try: - row = self._wait_for_result(reply_key, timeout) - except Exception as exc: - # Honour the never-raises contract even if the poll connection dies. - logger.exception( - "PG executor dispatch: wait failed (reply_key=%s run_id=%s)", - reply_key, - context.run_id, - ) - return ExecutionResult.failure(error=f"{type(exc).__name__}: {exc}") - if row is None: - # On timeout the executor task may still be running on the consumer; - # it will write its outcome under this reply_key, but we've already - # given up reading it (the reaper retention-sweeps the orphan row). If - # the workflow engine retries the file execution, it re-dispatches with - # a FRESH reply_key — so two executor tasks for the same file can - # overlap (double LLM spend / duplicate writes). De-duping that belongs - # at the file-execution layer, not here; this transport stays at-least- - # once + caller-timeout by design. - logger.warning( - "PG executor dispatch: TIMEOUT after %ss (reply_key=%s run_id=%s) — " - "the executor task may still be running", - timeout, - reply_key, - context.run_id, - ) - return ExecutionResult.failure( - error=f"TimeoutError: executor reply not received within {timeout}s" - ) - # ``.get`` (not ``[...]``) so a result row missing ``status`` can't raise - # out of dispatch() — the never-raises contract must not depend on the - # producer always writing every key. - if ( - row.get("status") == PgTaskStatus.COMPLETED.value - and row.get("result") is not None - ): - try: - return ExecutionResult.from_dict(row["result"]) - except Exception as exc: - # A malformed completed row becomes a failure result, not a raise. - # Surface the parse cause (like the enqueue/wait paths) so a UI - # reading result.error isn't left with an opaque message. - logger.exception( - "PG executor dispatch: malformed completed result " - "(reply_key=%s run_id=%s)", - reply_key, - context.run_id, - ) - return ExecutionResult.failure( - error=( - f"Malformed executor result ({type(exc).__name__}) " - f"for reply_key {reply_key}" - ) - ) - logger.warning( - "PG executor dispatch: executor reported failure (reply_key=%s " - "run_id=%s): %s", - reply_key, - context.run_id, - row.get("error") or "(no error)", - ) - return ExecutionResult.failure(error=row.get("error") or "executor task failed") - - def dispatch_async( - self, context: ExecutionContext, headers: dict[str, Any] | None = None - ) -> str: - """Fire-and-forget enqueue of ``execute_extraction``; returns the task id. - - The PG analogue of the SDK ``dispatch_async``: no ``reply_key``, no - callback, no blocking. There is no PG ``AsyncResult`` backend, so a caller - that needs the outcome uses :meth:`dispatch_with_callback` (a self-chained - continuation), not polling on this id. ``headers`` is accepted and ignored - for substitutability (PG carries routing in the payload, not Celery - headers). Enqueue failures propagate — parity with the SDK, which lets a - broker error out of ``dispatch_async``. - """ - task_id = str(uuid.uuid4()) - queue = f"{_QUEUE_PREFIX}{context.executor_name}" - org = str(getattr(context, "organization_id", "") or "") - self._enqueue(queue, context, org, task_id=task_id) - logger.info( - "PG executor dispatch_async: enqueued task_id=%s queue=%s run_id=%s", - task_id, - queue, - context.run_id, - ) - return task_id - - def dispatch_with_callback( - self, - context: ExecutionContext, - on_success: Any | None = None, - on_error: Any | None = None, - task_id: str | None = None, - headers: dict[str, Any] | None = None, - ) -> DispatchHandle: - """Fire-and-forget enqueue with self-chained callbacks (§5 model). - - The PG analogue of the SDK ``dispatch_with_callback``: instead of Celery - ``link`` / ``link_error`` (which the broker fires), the on-success / - on-error Celery ``Signature``s are translated to serialisable - :class:`ContinuationSpec`s and carried in the payload. After the executor - consumer runs ``execute_extraction`` it self-chains the matching - continuation onto the callback queue. Returns a :class:`DispatchHandle` - exposing ``.id`` (== ``task_id``) so call sites read the task id exactly - as on the Celery path. ``headers`` is accepted and ignored (see - :meth:`dispatch_async`). - """ - task_id = task_id or str(uuid.uuid4()) - queue = f"{_QUEUE_PREFIX}{context.executor_name}" - org = str(getattr(context, "organization_id", "") or "") - success_spec = signature_to_continuation(on_success) - error_spec = signature_to_continuation(on_error) - self._enqueue( - queue, - context, - org, - on_success=success_spec, - on_error=error_spec, - task_id=task_id, - ) - logger.info( - "PG executor dispatch_with_callback: enqueued task_id=%s queue=%s " - "run_id=%s on_success=%s on_error=%s", - task_id, - queue, - context.run_id, - success_spec["task_name"] if success_spec else None, - error_spec["task_name"] if error_spec else None, - ) - return DispatchHandle(task_id) - - @staticmethod - def _enqueue( + *, queue: str, context: ExecutionContext, org_id: str, - *, reply_key: str | None = None, on_success: ContinuationSpec | None = None, on_error: ContinuationSpec | None = None, task_id: str | None = None, ) -> None: - """Enqueue an ``execute_extraction`` message (request-reply or callback). - - A short-lived client owns its connection for just the insert (which - commits internally) so the message is durably visible to the - ``worker-pg-executor`` consumer before we begin polling — and no - connection is pinned for the whole (possibly long) RPC. The optional keys - select the dispatch shape: ``reply_key`` → request-reply; ``on_success`` / - ``on_error`` / ``task_id`` → async/callback (self-chained). - """ + # A short-lived client owns its connection for just the insert (which commits + # internally) so the message is durably visible to the worker-pg-executor + # consumer before we begin polling — and no connection is pinned for the whole + # (possibly long) RPC. payload = to_payload( - _EXECUTE_TASK, + EXECUTE_TASK, args=[context.to_dict()], queue=queue, reply_key=reply_key, @@ -333,83 +109,19 @@ def _enqueue( with PgQueueClient() as client: client.send(queue, payload, org_id=org_id) - @staticmethod - def _wait_for_result(reply_key: str, timeout: float) -> dict[str, Any] | None: + def wait_for_result(self, reply_key: str, timeout: float) -> ExecResultRow | None: """Poll ``pg_task_result`` until the row appears or *timeout* elapses. - Poll-based with capped backoff (PgBouncer-safe; no LISTEN/NOTIFY). The - backend owns one connection for the duration of the wait and closes it on - exit, so a long RPC never leaks a connection. The pin is bounded: a - file_processing worker runs ``--pool=prefork`` with - ``WORKER_FILE_PROCESSING_CONCURRENCY`` (default 4) processes, and each - dispatches sequentially, so at most ~concurrency connections are held for - up to ``EXECUTOR_RESULT_TIMEOUT``. (The backend twin instead releases via - ``close_old_connections`` between polls; if file_processing concurrency is - raised materially, do the same here.) + ``PgResultBackend`` owns one connection for the duration of the wait and + closes it on exit, so a long RPC never leaks a connection. The result row is a + ``{status, result, error}`` dict; fold it to the shared :class:`ExecResultRow`. """ with PgResultBackend() as rb: - return rb.wait_for_result(reply_key, timeout) - - -class RoutingExecutionDispatcher: - """Gate-routed executor dispatcher returned by :func:`get_executor_dispatcher`. - - Every mode chooses PG vs Celery per call (instant rollout/rollback): - ``dispatch()`` (request-reply), ``dispatch_async`` (fire-and-forget) and - ``dispatch_with_callback`` (self-chained callbacks). Duck-typed against the SDK - ``ExecutionDispatcher`` so call sites are unchanged. - """ - - def __init__(self, celery_app: object | None = None) -> None: - self._celery = ExecutionDispatcher(celery_app=celery_app) - self._pg = PgExecutionDispatcher() - - def dispatch( - self, - context: ExecutionContext, - timeout: int | None = None, - headers: dict[str, Any] | None = None, - ) -> ExecutionResult: - if resolve_executor_transport(context): - logger.info( - "Executor RPC → PG transport (executor=%s run_id=%s)", - context.executor_name, - context.run_id, - ) - # PG carries org/routing via the enqueue payload, not Celery headers, - # so the fairness headers are intentionally not forwarded here (parity - # with the backend executor RPC). - return self._pg.dispatch(context, timeout=timeout) - return self._celery.dispatch(context, timeout=timeout, headers=headers) - - def dispatch_async( - self, context: ExecutionContext, headers: dict[str, Any] | None = None - ) -> str: - if resolve_executor_transport(context): - return self._pg.dispatch_async(context) - return self._celery.dispatch_async(context, headers=headers) - - def dispatch_with_callback( - self, - context: ExecutionContext, - on_success: Any | None = None, - on_error: Any | None = None, - task_id: str | None = None, - headers: dict[str, Any] | None = None, - ) -> Any: - if resolve_executor_transport(context): - return self._pg.dispatch_with_callback( - context, - on_success=on_success, - on_error=on_error, - task_id=task_id, - ) - return self._celery.dispatch_with_callback( - context, - on_success=on_success, - on_error=on_error, - task_id=task_id, - headers=headers, + row = rb.wait_for_result(reply_key, timeout) + if row is None: + return None + return ExecResultRow( + status=row.get("status"), result=row.get("result"), error=row.get("error") ) @@ -417,4 +129,8 @@ def get_executor_dispatcher( celery_app: object | None = None, ) -> RoutingExecutionDispatcher: """Factory: the gate-routed executor dispatcher (PG when enabled, else Celery).""" - return RoutingExecutionDispatcher(celery_app=celery_app) + return RoutingExecutionDispatcher( + celery=ExecutionDispatcher(celery_app=celery_app), + pg=PgExecutionDispatcher(PgClientQueueTransport()), + resolve=resolve_executor_transport, + ) diff --git a/workers/queue_backend/pg_queue/result_backend.py b/workers/queue_backend/pg_queue/result_backend.py index 208d184913..e6544afc49 100644 --- a/workers/queue_backend/pg_queue/result_backend.py +++ b/workers/queue_backend/pg_queue/result_backend.py @@ -32,13 +32,13 @@ import contextlib import json import logging -import time from collections.abc import Iterator from typing import TYPE_CHECKING, Any, Self import psycopg2 from unstract.core.data_models import PgTaskStatus +from unstract.core.polling import poll_for_row from .connection import create_pg_connection @@ -164,22 +164,18 @@ def wait_for_result( ) -> dict[str, Any] | None: """Block until the result row appears or *timeout* seconds elapse. - Returns the ``{status, result, error}`` dict, or ``None`` on timeout. - Poll-based with exponential backoff (capped) — PgBouncer-safe, no - persistent listener. The final sleep is clamped so we never overshoot - the deadline. + Returns the ``{status, result, error}`` dict, or ``None`` on timeout. Uses + the shared :func:`~unstract.core.polling.poll_for_row` backoff (capped + exponential, PgBouncer-safe; the final sleep is clamped to the deadline) — the + same skeleton the backend's ``DjangoQueueTransport`` poller uses, so the + backoff lives in one place. """ - deadline = time.monotonic() + timeout - delay = poll_interval - while True: - row = self.get_result(task_id) - if row is not None: - return row - remaining = deadline - time.monotonic() - if remaining <= 0: - return None - time.sleep(min(delay, remaining)) - delay = min(delay * 2, _POLL_MAX_SECONDS) + return poll_for_row( + lambda: self.get_result(task_id), + timeout, + initial=poll_interval, + maximum=_POLL_MAX_SECONDS, + ) def close(self) -> None: """Close an owned connection (injected connections are the caller's).""" diff --git a/workers/tests/test_executor_rpc.py b/workers/tests/test_executor_rpc.py index 823804e2a9..75ed262a58 100644 --- a/workers/tests/test_executor_rpc.py +++ b/workers/tests/test_executor_rpc.py @@ -1,423 +1,365 @@ -"""Tests for the workers-side executor-RPC transport routing (Phase 9, ③b-2). - -DB-free: the env gate / Flipt / the enqueue + poll halves are mocked. Pins the -gate's fail-closed matrix and — the load-bearing zero-regression property — that -with the gate off ``RoutingExecutionDispatcher`` delegates EVERY mode to the -unchanged Celery ``ExecutionDispatcher`` and never touches the PG path. Mirrors -``backend/pg_queue/tests/test_executor_rpc.py`` (the backend twin) adapted to the -worker primitives: an env master-gate instead of a Django setting, and the result -row is a plain ``dict`` (``PgResultBackend``) instead of a Django model. +"""Tests for the executor-RPC dispatch (UN-3607 shared module + the workers adapter). + +The gate + reply_key/timeout orchestration + routing now live ONCE in +``unstract.workflow_execution.executor_rpc`` (shared by backend + workers). This +suite is the home for the **shared contract** (tested against a fake transport, so +the dispatch never-raises / result-interpretation / routing logic is verified a +single time, not per-mirror) PLUS the **workers adapter** (``PgClientQueueTransport`` ++ the env-master-gate ``resolve_executor_transport`` + the ``get_executor_dispatcher`` +wiring). The backend adapter is tested in ``backend/pg_queue/tests/test_executor_rpc.py``. """ from unittest.mock import MagicMock, patch import pytest from queue_backend.pg_queue.executor_rpc import ( - PgExecutionDispatcher, + PgClientQueueTransport, RoutingExecutionDispatcher, + get_executor_dispatcher, resolve_executor_transport, ) from unstract.core.execution_dispatch import DispatchHandle, signature_to_continuation +from unstract.workflow_execution.executor_rpc import ( + ExecResultRow, + PgExecutionDispatcher, + resolve_pg_transport, +) -_MOD = "queue_backend.pg_queue.executor_rpc" +_WMOD = "queue_backend.pg_queue.executor_rpc" +_SMOD = "unstract.workflow_execution.executor_rpc" -def _completed(result: dict) -> dict: - return {"status": "completed", "result": result, "error": ""} +def _ctx(org: str | None = "org1") -> MagicMock: + c = MagicMock() + c.executor_name = "legacy" + c.run_id = "run-1" + c.organization_id = org + c.to_dict.return_value = {"run_id": "run-1"} + return c def _ok_result() -> dict: return {"success": True, "data": {"x": 1}, "metadata": {}, "error": None} -class TestPgExecutionDispatcherDispatch: - """The load-bearing contract: never raises; timeout/failure → failure result. +def _completed(result: dict) -> ExecResultRow: + return ExecResultRow(status="completed", result=result, error="") + - DB-free — the ``_enqueue`` and ``_wait_for_result`` halves are mocked. +class _FakeTransport: + """Records ``enqueue`` calls and returns a configured result for the poll. + + Lets the shared dispatcher's logic be exercised without any DB. """ - @staticmethod - def _ctx() -> MagicMock: - c = MagicMock() - c.executor_name = "legacy" - c.run_id = "r" - c.organization_id = "o" - c.to_dict.return_value = {"run_id": "r"} - return c + def __init__(self, *, wait_return=None, wait_raises=None, enqueue_raises=None): + self.enqueue_calls: list[dict] = [] + self.wait_timeouts: list[float] = [] + self._wait_return = wait_return + self._wait_raises = wait_raises + self._enqueue_raises = enqueue_raises + + def enqueue(self, **kwargs) -> None: + self.enqueue_calls.append(kwargs) + if self._enqueue_raises is not None: + raise self._enqueue_raises + + def wait_for_result(self, reply_key, timeout): + self.wait_timeouts.append(timeout) + if self._wait_raises is not None: + raise self._wait_raises + return self._wait_return + +# --- Shared contract: PgExecutionDispatcher never-raises + result interpretation --- + + +class TestSharedDispatchContract: def test_enqueue_failure_returns_failure_not_raise(self): - with patch.object( - PgExecutionDispatcher, "_enqueue", side_effect=RuntimeError("db down") - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is False - assert "RuntimeError" in res.error + d = PgExecutionDispatcher(_FakeTransport(enqueue_raises=RuntimeError("db down"))) + res = d.dispatch(_ctx(), timeout=5) + assert res.success is False and "RuntimeError" in res.error def test_wait_failure_returns_failure_not_raise(self): - with ( - patch.object(PgExecutionDispatcher, "_enqueue"), - patch.object( - PgExecutionDispatcher, - "_wait_for_result", - side_effect=RuntimeError("conn died"), - ), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is False - assert "RuntimeError" in res.error + d = PgExecutionDispatcher(_FakeTransport(wait_raises=RuntimeError("conn died"))) + res = d.dispatch(_ctx(), timeout=5) + assert res.success is False and "RuntimeError" in res.error def test_timeout_returns_failure(self): - with ( - patch.object(PgExecutionDispatcher, "_enqueue"), - patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=None), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=3) - assert res.success is False - assert "within 3s" in res.error + d = PgExecutionDispatcher(_FakeTransport(wait_return=None)) + res = d.dispatch(_ctx(), timeout=3) + assert res.success is False and "within 3s" in res.error def test_completed_row_returns_result(self): - with ( - patch.object(PgExecutionDispatcher, "_enqueue"), - patch.object( - PgExecutionDispatcher, - "_wait_for_result", - return_value=_completed(_ok_result()), - ), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is True + d = PgExecutionDispatcher(_FakeTransport(wait_return=_completed(_ok_result()))) + assert d.dispatch(_ctx(), timeout=5).success is True def test_failed_row_returns_error(self): - row = {"status": "failed", "result": None, "error": "boom"} - with ( - patch.object(PgExecutionDispatcher, "_enqueue"), - patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is False - assert res.error == "boom" + row = ExecResultRow(status="failed", result=None, error="boom") + res = PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(_ctx(), timeout=5) + assert res.success is False and res.error == "boom" def test_failed_row_empty_error_falls_back(self): - row = {"status": "failed", "result": None, "error": ""} - with ( - patch.object(PgExecutionDispatcher, "_enqueue"), - patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is False - assert "executor task failed" in res.error + row = ExecResultRow(status="failed", result=None, error="") + res = PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(_ctx(), timeout=5) + assert res.success is False and "executor task failed" in res.error def test_completed_but_result_none_is_failure(self): - row = {"status": "completed", "result": None, "error": ""} - with ( - patch.object(PgExecutionDispatcher, "_enqueue"), - patch.object(PgExecutionDispatcher, "_wait_for_result", return_value=row), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is False + row = ExecResultRow(status="completed", result=None, error="") + assert PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(_ctx(), timeout=5).success is False def test_malformed_completed_row_is_failure_not_raise(self): - with ( - patch.object(PgExecutionDispatcher, "_enqueue"), - patch.object( - PgExecutionDispatcher, - "_wait_for_result", - return_value=_completed({"bad": "shape"}), - ), - ): - res = PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - assert res.success is False - assert "Malformed" in res.error + d = PgExecutionDispatcher(_FakeTransport(wait_return=_completed({"bad": "shape"}))) + res = d.dispatch(_ctx(), timeout=5) + assert res.success is False and "Malformed" in res.error def test_timeout_none_reads_env_then_default(self, monkeypatch): monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "42") - seen = {} - - def fake_wait(reply_key, timeout): - seen["timeout"] = timeout - return None - - with ( - patch.object(PgExecutionDispatcher, "_enqueue"), - patch.object( - PgExecutionDispatcher, "_wait_for_result", side_effect=fake_wait - ), - ): - # No explicit timeout arg → falls back to the env/default. - PgExecutionDispatcher().dispatch(self._ctx()) - assert seen["timeout"] == 42 + t = _FakeTransport(wait_return=None) + PgExecutionDispatcher(t).dispatch(_ctx()) # no explicit timeout + assert t.wait_timeouts == [42] def test_timeout_none_bad_env_falls_back_to_default(self, monkeypatch): monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "not-an-int") - seen = {} + t = _FakeTransport(wait_return=None) + PgExecutionDispatcher(t).dispatch(_ctx()) # must not raise + assert t.wait_timeouts == [3600] + + def test_dispatch_request_reply_enqueues_reply_key_only(self): + t = _FakeTransport(wait_return=_completed(_ok_result())) + PgExecutionDispatcher(t).dispatch(_ctx(), timeout=5) + (kw,) = t.enqueue_calls + assert kw["queue"] == "celery_executor_legacy" and kw["org_id"] == "org1" + assert kw["reply_key"] # request-reply marker + assert kw.get("task_id") is None and kw.get("on_success") is None - def fake_wait(reply_key, timeout): - seen["timeout"] = timeout - return None + def test_dispatch_async_is_fire_and_forget(self): + t = _FakeTransport() + task_id = PgExecutionDispatcher(t).dispatch_async(_ctx()) + (kw,) = t.enqueue_calls + assert kw["task_id"] == task_id and kw.get("reply_key") is None + assert kw.get("on_success") is None and kw.get("on_error") is None + + def test_dispatch_async_propagates_enqueue_failure(self): + # Documented asymmetry vs the never-raises dispatch: a fire-and-forget enqueue + # error propagates (the caller has no result object to fail into). + t = _FakeTransport(enqueue_raises=RuntimeError("db down")) + with pytest.raises(RuntimeError, match="db down"): + PgExecutionDispatcher(t).dispatch_async(_ctx()) - with ( - patch.object(PgExecutionDispatcher, "_enqueue"), - patch.object( - PgExecutionDispatcher, "_wait_for_result", side_effect=fake_wait - ), - ): - PgExecutionDispatcher().dispatch(self._ctx()) # must not raise - assert seen["timeout"] == 3600 # _DEFAULT_TIMEOUT + @staticmethod + def _sig(task: str): + return MagicMock( + task=task, args=(), kwargs={"callback_kwargs": {"room": "r1"}}, + options={"queue": "ide_callback"}, + ) + def test_dispatch_with_callback_translates_both_signatures(self): + t = _FakeTransport() + handle = PgExecutionDispatcher(t).dispatch_with_callback( + _ctx(), on_success=self._sig("ide_prompt_complete"), + on_error=self._sig("ide_prompt_error"), task_id="tid-7", + ) + assert handle.id == "tid-7" + (kw,) = t.enqueue_calls + assert kw["on_success"] == { + "task_name": "ide_prompt_complete", + "kwargs": {"callback_kwargs": {"room": "r1"}}, + "queue": "ide_callback", + } + assert kw["on_error"]["task_name"] == "ide_prompt_error" # on_error translated + assert kw["task_id"] == "tid-7" and kw.get("reply_key") is None -def _ctx(org: str | None = "org1") -> MagicMock: - c = MagicMock() - c.executor_name = "legacy" - c.run_id = "run-1" - c.organization_id = org - return c + def test_dispatch_with_callback_defaults_task_id(self): + t = _FakeTransport() + handle = PgExecutionDispatcher(t).dispatch_with_callback( + _ctx(), on_success=self._sig("ide_prompt_complete") + ) + # No task_id passed → a uuid is generated, echoed on the handle AND the payload. + assert handle.id + assert t.enqueue_calls[0]["task_id"] == handle.id -class TestResolveExecutorTransport: - def test_master_gate_off_is_celery(self, monkeypatch): - monkeypatch.delenv("PG_QUEUE_TRANSPORT_ENABLED", raising=False) - monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with patch(f"{_MOD}.check_feature_flag_status") as flag: - assert resolve_executor_transport(_ctx()) is False - flag.assert_not_called() # gate off → Flipt never consulted +# --- Shared gate: resolve_pg_transport (master-gated, then Flipt, fail-closed) --- + + +class TestResolvePgTransport: + def test_master_gate_off_is_celery(self): + with patch(f"{_SMOD}.check_feature_flag_status") as flag: + assert resolve_pg_transport(_ctx(), master_gate_enabled=False) is False + flag.assert_not_called() def test_flipt_unavailable_is_celery(self, monkeypatch): - monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false") - with patch(f"{_MOD}.check_feature_flag_status") as flag: - assert resolve_executor_transport(_ctx()) is False + with patch(f"{_SMOD}.check_feature_flag_status") as flag: + assert resolve_pg_transport(_ctx(), master_gate_enabled=True) is False flag.assert_not_called() def test_flag_true_is_pg_keyed_on_org(self, monkeypatch): - monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with patch( - f"{_MOD}.check_feature_flag_status", return_value=True - ) as flag: - assert resolve_executor_transport(_ctx("orgX")) is True + with patch(f"{_SMOD}.check_feature_flag_status", return_value=True) as flag: + assert resolve_pg_transport(_ctx("orgX"), master_gate_enabled=True) is True assert flag.call_args.kwargs["entity_id"] == "orgX" - # The single shared PG-queue flag (not a per-subsystem flag). assert flag.call_args.kwargs["flag_key"] == "pg_queue_enabled" def test_flag_false_is_celery(self, monkeypatch): - monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with patch(f"{_MOD}.check_feature_flag_status", return_value=False): - assert resolve_executor_transport(_ctx()) is False + with patch(f"{_SMOD}.check_feature_flag_status", return_value=False): + assert resolve_pg_transport(_ctx(), master_gate_enabled=True) is False - def test_flipt_error_fails_closed_to_celery(self, monkeypatch): - monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") + def test_flipt_error_fails_closed(self, monkeypatch): monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with patch( - f"{_MOD}.check_feature_flag_status", side_effect=RuntimeError("down") - ): - assert resolve_executor_transport(_ctx()) is False + with patch(f"{_SMOD}.check_feature_flag_status", side_effect=RuntimeError("x")): + assert resolve_pg_transport(_ctx(), master_gate_enabled=True) is False def test_org_less_context_buckets_on_run_id(self, monkeypatch): - """No org → entity_id falls back to run_id and org is absent from context. - - Guards the org-less bucketing so cross-org/run-only contexts resolve - deterministically instead of shipping a bogus "None" org. - """ - monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with patch( - f"{_MOD}.check_feature_flag_status", return_value=True - ) as flag: - assert resolve_executor_transport(_ctx(org=None)) is True + with patch(f"{_SMOD}.check_feature_flag_status", return_value=True) as flag: + assert resolve_pg_transport(_ctx(org=None), master_gate_enabled=True) is True assert flag.call_args.kwargs["entity_id"] == "run-1" assert "organization_id" not in flag.call_args.kwargs["context"] -class TestPgExecutionDispatcherEnqueueWiring: - """The actual PG-transport wiring: queue name, payload shape, org_id. +# --- Shared routing: RoutingExecutionDispatcher (zero-regression) --- - These are the *only* routing/identity carried on the PG path (Celery headers - are dropped), so a bug here misroutes or breaks org-fairness. ``to_payload`` - runs for real; only ``PgQueueClient`` and the wait are mocked. - """ +class TestRoutingDispatch: @staticmethod - def _ctx(): - c = MagicMock() - c.executor_name = "legacy" - c.run_id = "r" - c.organization_id = "org9" - c.to_dict.return_value = {"run_id": "r"} - return c - - def test_enqueue_sends_queue_payload_and_org(self): - client = MagicMock() - client.__enter__.return_value = client # `with PgQueueClient() as c` → c is client - with ( - patch(f"{_MOD}.PgQueueClient", return_value=client), - patch.object( - PgExecutionDispatcher, - "_wait_for_result", - return_value=_completed(_ok_result()), - ), - ): - PgExecutionDispatcher().dispatch(self._ctx(), timeout=5) - client.send.assert_called_once() - args, kwargs = client.send.call_args - queue_arg, payload_arg = args[0], args[1] - assert queue_arg == "celery_executor_legacy" - assert kwargs["org_id"] == "org9" - assert payload_arg["task_name"] == "execute_extraction" - assert payload_arg["args"] == [{"run_id": "r"}] - assert payload_arg["reply_key"] # request-reply marker present (a uuid) - - -class TestRoutingZeroRegression: - @staticmethod - def _build(): - # Patch both sub-dispatchers at construction; the instances are captured - # in __init__ so they remain mocked after the context exits. - with ( - patch(f"{_MOD}.ExecutionDispatcher") as celery_cls, - patch(f"{_MOD}.PgExecutionDispatcher") as pg_cls, - ): - dispatcher = RoutingExecutionDispatcher(celery_app="app") - return dispatcher, celery_cls.return_value, pg_cls.return_value + def _build(route_to_pg: bool): + celery, pg = MagicMock(), MagicMock() + d = RoutingExecutionDispatcher( + celery=celery, pg=pg, resolve=lambda _ctx: route_to_pg + ) + return d, celery, pg def test_gate_off_forwards_timeout_and_headers_to_celery(self): - """Zero-regression: gate off → Celery gets timeout AND headers unchanged.""" - dispatcher, celery, pg = self._build() + d, celery, pg = self._build(route_to_pg=False) ctx = _ctx() hdrs = {"x-fairness-key": {"org_id": "o"}} - with patch(f"{_MOD}.resolve_executor_transport", return_value=False): - dispatcher.dispatch(ctx, timeout=9, headers=hdrs) + d.dispatch(ctx, timeout=9, headers=hdrs) celery.dispatch.assert_called_once_with(ctx, timeout=9, headers=hdrs) - pg.dispatch.assert_not_called() # the zero-regression guarantee + pg.dispatch.assert_not_called() def test_gate_on_passes_timeout_to_pg_and_drops_headers(self): - """Gate on → PG gets the timeout but NOT the Celery headers (intentional).""" - dispatcher, celery, pg = self._build() + d, celery, pg = self._build(route_to_pg=True) ctx = _ctx() - with patch(f"{_MOD}.resolve_executor_transport", return_value=True): - dispatcher.dispatch(ctx, timeout=7, headers={"x-fairness-key": {"o": 1}}) - pg.dispatch.assert_called_once_with(ctx, timeout=7) - assert "headers" not in pg.dispatch.call_args.kwargs + d.dispatch(ctx, timeout=7, headers={"x-fairness-key": {"o": 1}}) + pg.dispatch.assert_called_once_with(ctx, timeout=7) # headers dropped celery.dispatch.assert_not_called() def test_async_and_callback_stay_celery_when_gate_off(self): - """Zero-regression: gate off → async/callback delegate to Celery unchanged.""" - dispatcher, celery, pg = self._build() - with patch(f"{_MOD}.resolve_executor_transport", return_value=False): - dispatcher.dispatch_async(_ctx(), headers={"h": 1}) - dispatcher.dispatch_with_callback(_ctx(), on_success="s", on_error="e") + d, celery, pg = self._build(route_to_pg=False) + d.dispatch_async(_ctx(), headers={"h": 1}) + d.dispatch_with_callback(_ctx(), on_success="s", on_error="e") celery.dispatch_async.assert_called_once() celery.dispatch_with_callback.assert_called_once() pg.dispatch_async.assert_not_called() pg.dispatch_with_callback.assert_not_called() def test_async_and_callback_route_to_pg_when_gated(self): - """Gate on (③c) → async/callback take the PG self-chained path.""" - dispatcher, celery, pg = self._build() - with patch(f"{_MOD}.resolve_executor_transport", return_value=True): - dispatcher.dispatch_async(_ctx()) - dispatcher.dispatch_with_callback( - _ctx(), on_success="s", on_error="e", task_id="t" - ) + d, celery, pg = self._build(route_to_pg=True) + d.dispatch_async(_ctx()) + d.dispatch_with_callback(_ctx(), on_success="s", on_error="e", task_id="t") pg.dispatch_async.assert_called_once() pg.dispatch_with_callback.assert_called_once() - # PG carries callbacks in the payload, not Celery headers → no header leak. assert "headers" not in pg.dispatch_with_callback.call_args.kwargs celery.dispatch_async.assert_not_called() celery.dispatch_with_callback.assert_not_called() -class TestPgAsyncCallbackWiring: - """PG fire-and-forget + self-chained-callback enqueue shapes. +# --- Workers adapter: PgClientQueueTransport + env gate + factory wiring --- - ``to_payload`` runs for real; only ``PgQueueClient`` is mocked. Pins that the - async path carries NO reply_key (it must not block a consumer) and the callback - path carries the translated continuations + the tracking task_id. - """ +class TestWorkersAdapter: @staticmethod def _ctx(): c = MagicMock() c.executor_name = "legacy" c.run_id = "r" - c.organization_id = "org9" - c.to_dict.return_value = {"run_id": "r", "organization_id": "org9"} + c.to_dict.return_value = {"run_id": "r"} return c @staticmethod def _client(): client = MagicMock() - client.__enter__.return_value = client + client.__enter__.return_value = client # `with PgQueueClient() as c` return client - def test_dispatch_async_is_fire_and_forget(self): + def test_enqueue_sends_queue_payload_and_org(self): client = self._client() - with patch(f"{_MOD}.PgQueueClient", return_value=client): - task_id = PgExecutionDispatcher().dispatch_async(self._ctx()) + with patch(f"{_WMOD}.PgQueueClient", return_value=client): + PgClientQueueTransport().enqueue( + queue="celery_executor_legacy", context=self._ctx(), + org_id="org9", reply_key="rk1", + ) client.send.assert_called_once() queue_arg, payload_arg = client.send.call_args.args[:2] assert queue_arg == "celery_executor_legacy" assert client.send.call_args.kwargs["org_id"] == "org9" assert payload_arg["task_name"] == "execute_extraction" - assert payload_arg["task_id"] == task_id - # No reply_key (would make a consumer try to store a reply) and no callback. - assert "reply_key" not in payload_arg - assert "on_success" not in payload_arg - assert "on_error" not in payload_arg + assert payload_arg["args"] == [{"run_id": "r"}] + assert payload_arg["reply_key"] == "rk1" - def test_dispatch_with_callback_carries_continuations(self): + def test_enqueue_carries_continuations(self): client = self._client() - on_s = MagicMock( - task="ide_prompt_complete", - args=(), - kwargs={"callback_kwargs": {"room": "r1"}}, - options={"queue": "ide_callback"}, - ) - on_e = MagicMock( - task="ide_prompt_error", - args=(), - kwargs={"callback_kwargs": {"room": "r1"}}, - options={"queue": "ide_callback"}, - ) - with patch(f"{_MOD}.PgQueueClient", return_value=client): - handle = PgExecutionDispatcher().dispatch_with_callback( - self._ctx(), on_success=on_s, on_error=on_e, task_id="tid-7" + spec = {"task_name": "ide_prompt_complete", "kwargs": {}, "queue": "ide_callback"} + with patch(f"{_WMOD}.PgQueueClient", return_value=client): + PgClientQueueTransport().enqueue( + queue="celery_executor_legacy", context=self._ctx(), org_id="o", + on_success=spec, task_id="tid-7", ) - assert handle.id == "tid-7" # call sites read .id off the handle - payload_arg = client.send.call_args.args[1] - assert payload_arg["on_success"] == { - "task_name": "ide_prompt_complete", - "kwargs": {"callback_kwargs": {"room": "r1"}}, - "queue": "ide_callback", - } - assert payload_arg["on_error"]["task_name"] == "ide_prompt_error" - assert payload_arg["task_id"] == "tid-7" - assert "reply_key" not in payload_arg # callback, not request-reply + payload = client.send.call_args.args[1] + assert payload["on_success"] == spec and payload["task_id"] == "tid-7" + assert "reply_key" not in payload # callback dispatch, not request-reply + + def test_wait_for_result_folds_dict_to_row(self): + rb = MagicMock() + rb.__enter__.return_value = rb + rb.wait_for_result.return_value = {"status": "completed", "result": {"a": 1}, "error": ""} + with patch(f"{_WMOD}.PgResultBackend", return_value=rb): + row = PgClientQueueTransport().wait_for_result("rk", 5) + assert isinstance(row, ExecResultRow) + assert row.status == "completed" and row.result == {"a": 1} + + def test_wait_for_result_none_passes_through(self): + rb = MagicMock() + rb.__enter__.return_value = rb + rb.wait_for_result.return_value = None + with patch(f"{_WMOD}.PgResultBackend", return_value=rb): + assert PgClientQueueTransport().wait_for_result("rk", 5) is None + + def test_resolve_reads_env_master_gate(self, monkeypatch): + monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") + with patch(f"{_WMOD}.resolve_pg_transport", return_value=True) as r: + assert resolve_executor_transport(self._ctx()) is True + assert r.call_args.kwargs["master_gate_enabled"] is True - def test_dispatch_with_callback_defaults_task_id(self): - client = self._client() - with patch(f"{_MOD}.PgQueueClient", return_value=client): - handle = PgExecutionDispatcher().dispatch_with_callback(self._ctx()) - # No task_id passed → a uuid is generated and echoed on the handle + payload. - assert handle.id - assert client.send.call_args.args[1]["task_id"] == handle.id + def test_resolve_env_off_is_false(self, monkeypatch): + monkeypatch.delenv("PG_QUEUE_TRANSPORT_ENABLED", raising=False) + with patch(f"{_WMOD}.resolve_pg_transport", return_value=False) as r: + resolve_executor_transport(self._ctx()) + assert r.call_args.kwargs["master_gate_enabled"] is False + def test_factory_wires_routing_with_workers_transport(self): + d = get_executor_dispatcher(celery_app="app") + assert isinstance(d, RoutingExecutionDispatcher) + # The PG dispatcher is wired with the workers psycopg2 transport, and the gate + # is the workers env-master-gate resolver. + assert isinstance(d._pg._transport, PgClientQueueTransport) + assert d._resolve is resolve_executor_transport -class TestSharedDispatchHelpers: - """The transport-agnostic helpers lifted to ``unstract.core`` (shared by the - backend + workers executor-RPC mirrors). Tested once here, not per-mirror. - """ +# --- Core helpers (unchanged; the shared signature/handle primitives) --- + + +class TestSharedDispatchHelpers: def test_signature_none_passes_through(self): assert signature_to_continuation(None) is None def test_signature_translates_task_kwargs_and_queue(self): - sig = MagicMock( - task="ide_prompt_complete", - args=(), # a real kwargs-only Celery Signature has empty .args - kwargs={"callback_kwargs": {"room": "r1"}}, - options={"queue": "ide_callback"}, - ) + sig = MagicMock(task="ide_prompt_complete", args=(), + kwargs={"callback_kwargs": {"room": "r1"}}, + options={"queue": "ide_callback"}) assert signature_to_continuation(sig) == { "task_name": "ide_prompt_complete", "kwargs": {"callback_kwargs": {"room": "r1"}}, @@ -435,18 +377,13 @@ def test_signature_missing_task_fails_fast(self): signature_to_continuation(sig) def test_signature_with_positional_args_fails_fast(self): - sig = MagicMock( - task="ide_prompt_complete", - args=("pos",), - kwargs={}, - options={"queue": "ide_callback"}, - ) + sig = MagicMock(task="ide_prompt_complete", args=("pos",), kwargs={}, + options={"queue": "ide_callback"}) with pytest.raises(ValueError, match="positional args"): signature_to_continuation(sig) def test_dispatch_handle_exposes_only_id(self): handle = DispatchHandle("tid-1") assert handle.id == "tid-1" - # __slots__ → no stray attributes (callers must not poke at .get()/.result). with pytest.raises(AttributeError): handle.result = 1 # type: ignore[attr-defined] diff --git a/workers/uv.lock b/workers/uv.lock index 2be2e6b718..96eb092d57 100644 --- a/workers/uv.lock +++ b/workers/uv.lock @@ -5016,6 +5016,7 @@ dependencies = [ { name = "unstract-core" }, { name = "unstract-filesystem" }, { name = "unstract-flags" }, + { name = "unstract-sdk1" }, { name = "unstract-tool-registry" }, { name = "unstract-tool-sandbox" }, ] @@ -5025,6 +5026,7 @@ requires-dist = [ { name = "unstract-core", editable = "../unstract/core" }, { name = "unstract-filesystem", editable = "../unstract/filesystem" }, { name = "unstract-flags", editable = "../unstract/flags" }, + { name = "unstract-sdk1", editable = "../unstract/sdk1" }, { name = "unstract-tool-registry", editable = "../unstract/tool-registry" }, { name = "unstract-tool-sandbox", editable = "../unstract/tool-sandbox" }, ] From ea7a97b3da7ac3b9f6732db0dec35b2eff400d79 Mon Sep 17 00:00:00 2001 From: ali Date: Mon, 22 Jun 2026 13:21:27 +0530 Subject: [PATCH 34/44] =?UTF-8?q?UN-3445=20[TOOL]=20pg=5Fbenchmark=20?= =?UTF-8?q?=E2=80=94=20PG-vs-Celery=20execution=20benchmark=20harness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dev-phase validation tooling (not product code; no PR ceremony) to prove PG >= Celery before any Celery decommission. Self-contained top-level package, no imports into backend/workers — non-regressive by construction. S1 (measurement spine): `report` + `queue-depth` read the backend DB directly and compare executions per transport. Transport classified post-hoc from the persistent queue_message_id/task_id columns; headline signal is server-measured execution_time + per-file parallelism (sum(file_times)/execution_time). S2 (load generation): `run` drives N executions at concurrency C against a running stack, polls each to terminal, and reports wall-clock / server / overhead / http latency per transport plus throughput. Transport observed per-execution so PG and Celery batches bucket correctly even mid-rollout. 26 unit tests (pure stats + classification + probe timing with injected clock/faked I/O). S1 dev-tested against the live dev DB. Co-Authored-By: Claude Opus 4.8 --- pg_benchmark/.gitignore | 5 + pg_benchmark/README.md | 67 ++++++++ pg_benchmark/pg_benchmark/__init__.py | 1 + pg_benchmark/pg_benchmark/__main__.py | 10 ++ pg_benchmark/pg_benchmark/cli.py | 177 +++++++++++++++++++++ pg_benchmark/pg_benchmark/config.py | 53 +++++++ pg_benchmark/pg_benchmark/db.py | 210 +++++++++++++++++++++++++ pg_benchmark/pg_benchmark/probe.py | 121 +++++++++++++++ pg_benchmark/pg_benchmark/report.py | 156 +++++++++++++++++++ pg_benchmark/pg_benchmark/runner.py | 76 +++++++++ pg_benchmark/pg_benchmark/stats.py | 74 +++++++++ pg_benchmark/pg_benchmark/trigger.py | 128 ++++++++++++++++ pg_benchmark/pyproject.toml | 14 ++ pg_benchmark/requirements.txt | 5 + pg_benchmark/tests/test_load.py | 212 ++++++++++++++++++++++++++ pg_benchmark/tests/test_stats.py | 113 ++++++++++++++ 16 files changed, 1422 insertions(+) create mode 100644 pg_benchmark/.gitignore create mode 100644 pg_benchmark/README.md create mode 100644 pg_benchmark/pg_benchmark/__init__.py create mode 100644 pg_benchmark/pg_benchmark/__main__.py create mode 100644 pg_benchmark/pg_benchmark/cli.py create mode 100644 pg_benchmark/pg_benchmark/config.py create mode 100644 pg_benchmark/pg_benchmark/db.py create mode 100644 pg_benchmark/pg_benchmark/probe.py create mode 100644 pg_benchmark/pg_benchmark/report.py create mode 100644 pg_benchmark/pg_benchmark/runner.py create mode 100644 pg_benchmark/pg_benchmark/stats.py create mode 100644 pg_benchmark/pg_benchmark/trigger.py create mode 100644 pg_benchmark/pyproject.toml create mode 100644 pg_benchmark/requirements.txt create mode 100644 pg_benchmark/tests/test_load.py create mode 100644 pg_benchmark/tests/test_stats.py diff --git a/pg_benchmark/.gitignore b/pg_benchmark/.gitignore new file mode 100644 index 0000000000..6fecb2bdb8 --- /dev/null +++ b/pg_benchmark/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +.pytest_cache/ +*.egg-info/ +.venv/ diff --git a/pg_benchmark/README.md b/pg_benchmark/README.md new file mode 100644 index 0000000000..6ba438dd1a --- /dev/null +++ b/pg_benchmark/README.md @@ -0,0 +1,67 @@ +# pg_benchmark — PG-queue vs Celery execution benchmark harness + +A self-contained harness to compare the **PG-queue transport** against the +**Celery/RabbitMQ transport** for Unstract workflow executions — functional +parity, performance, and (later) load/soak. This is the **gate** that must show +`PG ≥ Celery` before any Celery decommission (epic UN-3445). + +It is dev/ops tooling, not product code: a new top-level directory, no imports +into backend/workers, non-regressive by construction. + +## Latency model (why it measures what it measures) + +The harness deliberately leans on **persistent, server-measured** signals: + +| Signal | Source | Persistent? | +|---|---|---| +| `execution_time` (server) | `workflow_execution.execution_time` | ✅ truest cross-transport number — excludes harness HTTP/poll overhead | +| per-file `execution_time` | `workflow_file_execution.execution_time` | ✅ exposes **parallelism** (the key fan-out signal) | +| transport class | `queue_message_id` / `task_id` columns | ✅ survive on the row after the queue message is deleted | +| wall-clock E2E | harness client clock (trigger → terminal) | ✅ (only when the harness triggers the run) | +| enqueue→pickup | `pg_queue_message.enqueued_at` / `vt` | ❌ **ephemeral** — row deleted on ack; only observable by *live sampling* during a run | + +**Parallelism** `= sum(file_times) / execution_time`: `≈ N` means all N files +overlapped (ideal fan-out), `≈ 1` means they ran serially. This directly +measures the serial-fileproc concern on the PG path. + +> Transport micro-latency (enqueue→pickup, the SKIP-LOCKED-poll-vs-push KPI) is +> **not** readable post-hoc — those queue rows are deleted on ack. It needs a +> live sampler that watches `pg_queue_message` while a run is in flight; that is +> a later slice. + +## Status — slices + +- **S1 (this slice): measurement spine.** `report` + `queue-depth` read the DB + directly and print a per-transport latency comparison. Read-only. +- **S2: load generation.** `run` — trigger N executions at concurrency C against + a running stack, flip the transport, collect wall-clock + server latency. +- **S3: live transport sampler.** Capture enqueue→pickup + queue depth during a + run (PG-only). +- **S4: functional parity matrix.** Flag OFF vs ON across workflow shapes. + +## Usage + +```bash +cd pg_benchmark +pip install -r requirements.txt # or use the backend .venv (has psycopg2) + +# Compare the last 200 finished executions, all transports side by side: +python -m pg_benchmark report --last 200 + +# Just the PG path: +python -m pg_benchmark report --last 200 --transport pg + +# Current live queue depth per queue: +python -m pg_benchmark queue-depth +``` + +DB connection defaults mirror the local docker-compose stack +(`localhost:5432/unstract_db`, schema `unstract`). Override via `DB_HOST`, +`DB_PORT`, `DB_NAME`, `DB_USER`, `DB_PASSWORD` env vars or the `--db-*` flags. + +## Tests + +```bash +cd pg_benchmark +pytest # pure stats + classification + parallelism (no DB needed) +``` diff --git a/pg_benchmark/pg_benchmark/__init__.py b/pg_benchmark/pg_benchmark/__init__.py new file mode 100644 index 0000000000..118b9b3b67 --- /dev/null +++ b/pg_benchmark/pg_benchmark/__init__.py @@ -0,0 +1 @@ +"""PG-queue vs Celery execution benchmark harness.""" diff --git a/pg_benchmark/pg_benchmark/__main__.py b/pg_benchmark/pg_benchmark/__main__.py new file mode 100644 index 0000000000..3b2648627a --- /dev/null +++ b/pg_benchmark/pg_benchmark/__main__.py @@ -0,0 +1,10 @@ +"""``python -m pg_benchmark`` entrypoint.""" + +from __future__ import annotations + +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pg_benchmark/pg_benchmark/cli.py b/pg_benchmark/pg_benchmark/cli.py new file mode 100644 index 0000000000..08757b6cc1 --- /dev/null +++ b/pg_benchmark/pg_benchmark/cli.py @@ -0,0 +1,177 @@ +"""CLI entrypoint for the PG-vs-Celery benchmark harness. + +Slice 1 ships the measurement spine: ``report`` reads finished executions +straight from the DB and prints a per-transport latency comparison. Load +generation (``run``) and live transport sampling are later slices and plug into +these same readers. +""" + +from __future__ import annotations + +import argparse +import os +import sys + +from .config import DbConfig +from .db import Transport, connect, fetch_recent, queue_depth +from .probe import RunResult +from .report import build_reports, render, render_load +from .runner import run_load +from .trigger import TriggerConfig + +_TRANSPORT_CHOICES = {t.value: t for t in Transport} + + +def _add_db_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--db-host", help="override DB_HOST") + parser.add_argument("--db-port", type=int, help="override DB_PORT") + parser.add_argument("--db-name", help="override DB_NAME") + parser.add_argument("--db-user", help="override DB_USER") + parser.add_argument("--db-password", help="override DB_PASSWORD") + + +def _db_config(args: argparse.Namespace) -> DbConfig: + base = DbConfig.from_env() + return DbConfig( + host=args.db_host or base.host, + port=args.db_port or base.port, + name=args.db_name or base.name, + user=args.db_user or base.user, + password=args.db_password or base.password, + schema=base.schema, + ) + + +def _cmd_report(args: argparse.Namespace) -> int: + transport = _TRANSPORT_CHOICES.get(args.transport) if args.transport else None + conn = connect(_db_config(args)) + try: + executions = fetch_recent(conn, limit=args.last, transport=transport) + print( + f"Sampled {len(executions)} terminal executions " + f"(last {args.last}{', ' + args.transport if args.transport else ''}).\n" + ) + print(render(build_reports(executions))) + finally: + conn.close() + return 0 + + +def _cmd_queue_depth(args: argparse.Namespace) -> int: + conn = connect(_db_config(args)) + try: + depth = queue_depth(conn) + if not depth: + print("pg_queue_message is empty.") + else: + for name, count in sorted(depth.items(), key=lambda kv: -kv[1]): + print(f" {name:<28} {count}") + finally: + conn.close() + return 0 + + +def _cmd_run(args: argparse.Namespace) -> int: + api_key = args.api_key or os.environ.get("PGBENCH_API_KEY", "") + if not args.path: + print("error: --path (API deployment execute path) is required", file=sys.stderr) + return 2 + if not api_key: + print("error: --api-key or PGBENCH_API_KEY is required", file=sys.stderr) + return 2 + trigger_cfg = TriggerConfig( + base_url=args.base_url, + path=args.path, + api_key=api_key, + files=args.file or [], + auth_header=args.auth_header, + auth_prefix=args.auth_prefix, + ) + db_cfg = _db_config(args) + done = {"n": 0} + + def _progress(result: RunResult) -> None: + done["n"] += 1 + tag = (result.transport.value if result.transport else "?").upper() + state = "ok" if result.ok else f"FAIL({result.error})" + wc = f"{result.wall_clock_e2e:.1f}s" if result.wall_clock_e2e else "-" + print(f" [{done['n']}/{args.n}] {tag} {state} wall={wc}", file=sys.stderr) + + print( + f"Driving {args.n} executions at concurrency {args.concurrency} " + f"against {trigger_cfg.url} ...\n", + file=sys.stderr, + ) + outcome = run_load( + trigger_cfg, + db_cfg, + n=args.n, + concurrency=args.concurrency, + poll_interval=args.poll_interval, + timeout=args.timeout, + on_result=_progress, + ) + print() + print(render_load(outcome)) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="pg_benchmark", + description="PG-queue vs Celery execution benchmark harness.", + ) + sub = parser.add_subparsers(dest="command", required=True) + + report = sub.add_parser( + "report", help="compare recent executions by transport (read-only)" + ) + report.add_argument( + "--last", type=int, default=100, help="how many recent executions to sample" + ) + report.add_argument( + "--transport", + choices=sorted(_TRANSPORT_CHOICES), + help="restrict to one transport (default: all, side by side)", + ) + _add_db_args(report) + report.set_defaults(func=_cmd_report) + + qd = sub.add_parser("queue-depth", help="current pg_queue_message depth per queue") + _add_db_args(qd) + qd.set_defaults(func=_cmd_queue_depth) + + run = sub.add_parser( + "run", help="drive N executions at concurrency C and measure latency" + ) + run.add_argument("--n", type=int, default=10, help="total executions to trigger") + run.add_argument( + "--concurrency", type=int, default=4, help="max executions in flight at once" + ) + run.add_argument( + "--base-url", default=os.environ.get("PGBENCH_BASE_URL", "http://localhost:8000") + ) + run.add_argument( + "--path", + default=os.environ.get("PGBENCH_DEPLOY_PATH", ""), + help="API deployment execute path, e.g. /deployment/api///", + ) + run.add_argument("--api-key", help="API key (or set PGBENCH_API_KEY)") + run.add_argument("--file", action="append", help="local file to upload (repeatable)") + run.add_argument("--auth-header", default="Authorization") + run.add_argument("--auth-prefix", default="Bearer ") + run.add_argument("--poll-interval", type=float, default=0.5) + run.add_argument("--timeout", type=float, default=600.0) + _add_db_args(run) + run.set_defaults(func=_cmd_run) + + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return int(args.func(args)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pg_benchmark/pg_benchmark/config.py b/pg_benchmark/pg_benchmark/config.py new file mode 100644 index 0000000000..1e65ff2af9 --- /dev/null +++ b/pg_benchmark/pg_benchmark/config.py @@ -0,0 +1,53 @@ +"""Benchmark configuration — DB connection + backend endpoint. + +Defaults mirror the local docker-compose stack (``backend/backend/settings/base.py``) +so the harness runs against a dev stack with zero flags, and every value is +overridable by env or CLI for staging/load hosts. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class DbConfig: + """Read-only Postgres connection to the backend DB. + + The harness only ever SELECTs (latency readers + queue depth); it never + writes to these tables. ``schema`` is set on the search_path at connect time + so the PG-queue tables (which live in the ``unstract`` schema) resolve + unqualified. + """ + + host: str = "localhost" + port: int = 5432 + name: str = "unstract_db" + user: str = "unstract_dev" + password: str = "unstract_pass" + schema: str = "unstract" + + @classmethod + def from_env(cls) -> DbConfig: + # Read defaults off a default instance, not the class: on a slots + # dataclass ``cls.port`` is the slot descriptor, not the default value. + d = cls() + return cls( + host=os.environ.get("DB_HOST", d.host), + port=int(os.environ.get("DB_PORT", d.port)), + name=os.environ.get("DB_NAME", d.name), + user=os.environ.get("DB_USER", d.user), + password=os.environ.get("DB_PASSWORD", d.password), + schema=os.environ.get("DB_SCHEMA", d.schema), + ) + + def dsn_kwargs(self) -> dict[str, object]: + return { + "host": self.host, + "port": self.port, + "dbname": self.name, + "user": self.user, + "password": self.password, + "options": f"-c search_path={self.schema},public", + } diff --git a/pg_benchmark/pg_benchmark/db.py b/pg_benchmark/pg_benchmark/db.py new file mode 100644 index 0000000000..55f36b1296 --- /dev/null +++ b/pg_benchmark/pg_benchmark/db.py @@ -0,0 +1,210 @@ +"""Read-only latency readers over the backend DB. + +Why DB-side and not just client wall-clock: the server-measured +``execution_time`` is the truest cross-transport number (it excludes the +harness's own HTTP/poll overhead), and per-file ``execution_time`` exposes +*parallelism* — the single most important PG-vs-Celery signal for fan-out work. + +Transport is classified post-hoc from columns that survive on the execution row +even after the (ephemeral) queue message is deleted: + +- ``queue_message_id IS NOT NULL`` → PG transport +- ``task_id IS NOT NULL`` → Celery transport +- neither → inline / synchronous (no async dispatch) + +Deliberately NOT read here: ``pg_queue_message.enqueued_at`` / ``vt`` and +``pg_task_result`` — those rows are deleted on ack / swept on expiry, so +enqueue→pickup latency is only observable by *live sampling* during a run (see +``sampler`` — a later slice), never post-hoc. +""" + +from __future__ import annotations + +import enum +from dataclasses import dataclass, field + +import psycopg2 + +from .config import DbConfig + + +class Transport(enum.Enum): + """Which transport carried an execution, inferred from persistent columns.""" + + PG = "pg" + CELERY = "celery" + INLINE = "inline" + + @classmethod + def classify(cls, *, has_queue_message_id: bool, has_task_id: bool) -> Transport: + if has_queue_message_id: + return cls.PG + if has_task_id: + return cls.CELERY + return cls.INLINE + + +@dataclass(frozen=True, slots=True) +class ExecutionLatency: + """One execution's measured latencies (server-side, persistent).""" + + execution_id: str + transport: Transport + status: str + total_files: int + server_execution_time: float | None + file_times: list[float] = field(default_factory=list) + + @property + def is_terminal(self) -> bool: + return self.status in ("COMPLETED", "ERROR", "STOPPED") + + @property + def parallelism(self) -> float | None: + """Effective parallelism = sum(file_times) / server_execution_time. + + ``≈ N`` means all N files overlapped (ideal fan-out); ``≈ 1`` means they + ran serially. ``None`` when it can't be computed (no files / no timing). + """ + if not self.file_times or not self.server_execution_time: + return None + return sum(self.file_times) / self.server_execution_time + + +def connect(config: DbConfig) -> psycopg2.extensions.connection: + """Open a read-only-intent connection (autocommit; we only SELECT).""" + conn = psycopg2.connect(**config.dsn_kwargs()) + conn.autocommit = True + return conn + + +_RECENT_SQL = """ +SELECT + e.id::text, + (e.queue_message_id IS NOT NULL) AS has_qmid, + (e.task_id IS NOT NULL) AS has_taskid, + e.status, + e.total_files, + e.execution_time, + COALESCE( + ARRAY( + SELECT f.execution_time + FROM workflow_file_execution f + WHERE f.workflow_execution_id = e.id + AND f.execution_time IS NOT NULL + ORDER BY f.created_at + ), + ARRAY[]::double precision[] + ) AS file_times +FROM workflow_execution e +{where} +ORDER BY e.created_at DESC +LIMIT %(limit)s +""" + + +def fetch_recent( + conn: psycopg2.extensions.connection, + *, + limit: int = 100, + transport: Transport | None = None, + terminal_only: bool = True, +) -> list[ExecutionLatency]: + """Return the most recent executions as ``ExecutionLatency`` records. + + ``transport`` filters to one transport (server-side); ``terminal_only`` + restricts to finished runs so ``execution_time`` is populated. + """ + clauses: list[str] = [] + params: dict[str, object] = {"limit": limit} + if terminal_only: + clauses.append("e.status IN ('COMPLETED', 'ERROR', 'STOPPED')") + if transport is Transport.PG: + clauses.append("e.queue_message_id IS NOT NULL") + elif transport is Transport.CELERY: + clauses.append("e.queue_message_id IS NULL AND e.task_id IS NOT NULL") + elif transport is Transport.INLINE: + clauses.append("e.queue_message_id IS NULL AND e.task_id IS NULL") + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" + sql = _RECENT_SQL.format(where=where) + + rows: list[ExecutionLatency] = [] + with conn.cursor() as cur: + cur.execute(sql, params) + for ( + execution_id, + has_qmid, + has_taskid, + status, + total_files, + exec_time, + file_times, + ) in cur: + rows.append( + ExecutionLatency( + execution_id=execution_id, + transport=Transport.classify( + has_queue_message_id=has_qmid, has_task_id=has_taskid + ), + status=status, + total_files=total_files or 0, + server_execution_time=exec_time, + file_times=list(file_times or []), + ) + ) + return rows + + +_TERMINAL_STATUSES = ("COMPLETED", "ERROR", "STOPPED") + +_ONE_SQL = _RECENT_SQL.format(where="WHERE e.id = %(execution_id)s::uuid") + + +def fetch_status(conn: psycopg2.extensions.connection, execution_id: str) -> str | None: + """Return an execution's current status, or ``None`` if the row is absent. + + Used by the load probe to poll for terminality straight from the DB (no auth, + cheaper than the REST status endpoint). + """ + with conn.cursor() as cur: + cur.execute( + "SELECT status FROM workflow_execution WHERE id = %s::uuid", + (execution_id,), + ) + row = cur.fetchone() + return row[0] if row else None + + +def fetch_one( + conn: psycopg2.extensions.connection, execution_id: str +) -> ExecutionLatency | None: + """Return the full ``ExecutionLatency`` for one execution, or ``None``.""" + with conn.cursor() as cur: + cur.execute(_ONE_SQL, {"execution_id": execution_id, "limit": 1}) + row = cur.fetchone() + if row is None: + return None + execution_id_, has_qmid, has_taskid, status, total_files, exec_time, file_times = row + return ExecutionLatency( + execution_id=execution_id_, + transport=Transport.classify( + has_queue_message_id=has_qmid, has_task_id=has_taskid + ), + status=status, + total_files=total_files or 0, + server_execution_time=exec_time, + file_times=list(file_times or []), + ) + + +def is_terminal_status(status: str | None) -> bool: + return status in _TERMINAL_STATUSES + + +def queue_depth(conn: psycopg2.extensions.connection) -> dict[str, int]: + """Current live ``pg_queue_message`` count per queue (load-monitoring aid).""" + with conn.cursor() as cur: + cur.execute( + "SELECT queue_name, count(*) FROM pg_queue_message GROUP BY queue_name" + ) + return {name: count for name, count in cur} diff --git a/pg_benchmark/pg_benchmark/probe.py b/pg_benchmark/pg_benchmark/probe.py new file mode 100644 index 0000000000..0e5ade192a --- /dev/null +++ b/pg_benchmark/pg_benchmark/probe.py @@ -0,0 +1,121 @@ +"""One full execution probe: trigger → poll to terminal → read server latency. + +Combines the two clocks the benchmark cares about: + +- **client wall-clock** (``http_latency``, ``wall_clock_e2e``) — what a caller + feels, including queue wait + dispatch + poll granularity +- **server-measured** (``server_execution_time``, ``parallelism``) — the + transport-fair number, read from the DB by ``execution_id`` + +``overhead = wall_clock_e2e - server_execution_time`` is the part the transport +is responsible for (admission + queue wait + result delivery); it is where a +polling vs push transport would diverge. +""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from dataclasses import dataclass + +import requests + +from .config import DbConfig +from .db import Transport, connect, fetch_one, fetch_status, is_terminal_status +from .trigger import TriggerConfig, trigger_execution + + +@dataclass(frozen=True, slots=True) +class RunResult: + """Everything measured for one execution probe.""" + + execution_id: str | None + transport: Transport | None + status: str | None + ok: bool + http_latency: float + wall_clock_e2e: float | None + server_execution_time: float | None + parallelism: float | None + overhead: float | None + error: str | None = None + + +def run_probe( + trigger_cfg: TriggerConfig, + db_cfg: DbConfig, + *, + poll_interval: float = 0.5, + timeout: float = 600.0, + session: requests.Session | None = None, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> RunResult: + """Trigger one execution and measure it end to end. + + ``clock``/``sleep`` are injected so the poll loop is unit-testable without + real time. Each probe owns its own DB connection (psycopg2 connections are + not safe to share across the load runner's threads). + """ + start = clock() + trig = trigger_execution(trigger_cfg, session) + if trig.execution_id is None: + return RunResult( + execution_id=None, + transport=None, + status=None, + ok=False, + http_latency=trig.http_latency, + wall_clock_e2e=None, + server_execution_time=None, + parallelism=None, + overhead=None, + error=trig.error or "trigger returned no execution_id", + ) + + conn = connect(db_cfg) + try: + deadline = start + timeout + status: str | None = None + while clock() < deadline: + status = fetch_status(conn, trig.execution_id) + if is_terminal_status(status): + break + sleep(poll_interval) + wall_clock_e2e = clock() - start + latency = fetch_one(conn, trig.execution_id) + finally: + conn.close() + + if not is_terminal_status(status): + return RunResult( + execution_id=trig.execution_id, + transport=latency.transport if latency else None, + status=status, + ok=False, + http_latency=trig.http_latency, + wall_clock_e2e=wall_clock_e2e, + server_execution_time=latency.server_execution_time if latency else None, + parallelism=latency.parallelism if latency else None, + overhead=None, + error=f"timed out after {timeout:.0f}s (last status={status})", + ) + + server_time = latency.server_execution_time if latency else None + overhead = ( + wall_clock_e2e - server_time + if server_time is not None and wall_clock_e2e is not None + else None + ) + return RunResult( + execution_id=trig.execution_id, + transport=latency.transport if latency else None, + status=status, + ok=status == "COMPLETED", + http_latency=trig.http_latency, + wall_clock_e2e=wall_clock_e2e, + server_execution_time=server_time, + parallelism=latency.parallelism if latency else None, + overhead=overhead, + error=None if status == "COMPLETED" else f"terminal status={status}", + ) diff --git a/pg_benchmark/pg_benchmark/report.py b/pg_benchmark/pg_benchmark/report.py new file mode 100644 index 0000000000..4264572698 --- /dev/null +++ b/pg_benchmark/pg_benchmark/report.py @@ -0,0 +1,156 @@ +"""Aggregate executions by transport and render a PG-vs-Celery comparison. + +The comparison is the point: same metric, side by side, per transport, so a +regression or win is visible at a glance rather than buried in two separate runs. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .db import ExecutionLatency, Transport +from .probe import RunResult +from .runner import LoadOutcome +from .stats import Summary, summarize + + +@dataclass(frozen=True, slots=True) +class TransportReport: + """Per-transport rollup of a sample of executions.""" + + transport: Transport + count: int + execution_time: Summary + parallelism: Summary + error_rate: float + + +def build_reports(executions: list[ExecutionLatency]) -> list[TransportReport]: + """Group by transport and summarise execution-time + parallelism + errors.""" + reports: list[TransportReport] = [] + for transport in Transport: + bucket = [e for e in executions if e.transport is transport] + if not bucket: + continue + exec_times = [ + e.server_execution_time for e in bucket if e.server_execution_time is not None + ] + parallelisms = [e.parallelism for e in bucket if e.parallelism is not None] + errors = sum(1 for e in bucket if e.status == "ERROR") + reports.append( + TransportReport( + transport=transport, + count=len(bucket), + execution_time=summarize(exec_times), + parallelism=summarize(parallelisms), + error_rate=errors / len(bucket), + ) + ) + return reports + + +@dataclass(frozen=True, slots=True) +class LoadReport: + """Per-transport rollup of a controlled load run.""" + + transport: Transport + triggered: int + completed: int + failed: int + wall_clock_e2e: Summary + server_execution_time: Summary + overhead: Summary + http_latency: Summary + + +def build_load_reports(results: list[RunResult]) -> list[LoadReport]: + """Group probe results by observed transport and summarise each metric.""" + reports: list[LoadReport] = [] + # ``None`` transport bucket = runs that never produced a classifiable row. + transports: list[Transport | None] = [*Transport, None] + for transport in transports: + bucket = [r for r in results if r.transport is transport] + if not bucket: + continue + completed = [r for r in bucket if r.ok] + reports.append( + LoadReport( + transport=transport or Transport.INLINE, + triggered=len(bucket), + completed=len(completed), + failed=len(bucket) - len(completed), + wall_clock_e2e=summarize( + [r.wall_clock_e2e for r in bucket if r.wall_clock_e2e is not None] + ), + server_execution_time=summarize( + [ + r.server_execution_time + for r in bucket + if r.server_execution_time is not None + ] + ), + overhead=summarize( + [r.overhead for r in bucket if r.overhead is not None] + ), + http_latency=summarize([r.http_latency for r in bucket]), + ) + ) + return reports + + +def render_load(outcome: LoadOutcome) -> str: + """Render a load run: throughput headline + per-transport latency tables.""" + reports = build_load_reports(outcome.results) + lines = [ + f"Load run: {len(outcome.results)} triggered, " + f"{len(outcome.completed)} completed in {outcome.wall_clock:.1f}s " + f"→ {outcome.throughput:.2f} completed/s", + "", + ] + if not reports: + lines.append("No results.") + return "\n".join(lines) + for r in reports: + lines.append( + f"── {r.transport.value.upper()} " + f"({r.completed}/{r.triggered} ok, {r.failed} failed) ".ljust(78, "─") + ) + lines.append(f" wall-clock e2e (s): {_fmt(r.wall_clock_e2e)}") + lines.append(f" server exec (s) : {_fmt(r.server_execution_time)}") + lines.append(f" overhead (s) : {_fmt(r.overhead)}") + lines.append(f" http trigger (s) : {_fmt(r.http_latency)}") + lines.append("") + return "\n".join(lines) + + +def _fmt(summary: Summary) -> str: + if summary.empty: + return " (no samples)" + return ( + f"n={summary.n:<4} mean={summary.mean:7.2f} p50={summary.p50:7.2f} " + f"p95={summary.p95:7.2f} p99={summary.p99:7.2f} max={summary.maximum:7.2f}" + ) + + +def render(reports: list[TransportReport]) -> str: + """Render reports as a human-readable text block.""" + if not reports: + return "No executions matched the query." + lines: list[str] = [] + for r in reports: + lines.append( + f"── {r.transport.value.upper()} ({r.count} executions, " + f"error_rate={r.error_rate:.1%}) ".ljust(78, "─") + ) + lines.append(f" execution_time (s): {_fmt(r.execution_time)}") + para = r.parallelism + if para.empty: + lines.append(" parallelism : (single-file or untimed)") + else: + lines.append( + f" parallelism (x) : mean={para.mean:5.2f} p50={para.p50:5.2f} " + f"min={para.minimum:5.2f} max={para.maximum:5.2f} " + f"(≈1 serial, ≈N fully parallel)" + ) + lines.append("") + return "\n".join(lines) diff --git a/pg_benchmark/pg_benchmark/runner.py b/pg_benchmark/pg_benchmark/runner.py new file mode 100644 index 0000000000..4348d5fa76 --- /dev/null +++ b/pg_benchmark/pg_benchmark/runner.py @@ -0,0 +1,76 @@ +"""Concurrent load runner: drive N probes at concurrency C. + +A fixed-size thread pool keeps exactly ``concurrency`` executions in flight at +once (the probes are I/O-bound — HTTP + DB polling — so threads are the right +tool). Each probe is independent and owns its own connections, so failures are +isolated: one bad run becomes one ``RunResult`` with ``ok=False``, never a dead +batch. +""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass + +from .config import DbConfig +from .probe import RunResult, run_probe +from .trigger import TriggerConfig + + +@dataclass(frozen=True, slots=True) +class LoadOutcome: + """The full result set of a load run plus its wall-clock + throughput.""" + + results: list[RunResult] + wall_clock: float # total seconds for the whole batch + + @property + def completed(self) -> list[RunResult]: + return [r for r in self.results if r.ok] + + @property + def throughput(self) -> float: + """Completed executions per second over the batch wall-clock.""" + if self.wall_clock <= 0: + return 0.0 + return len(self.completed) / self.wall_clock + + +def run_load( + trigger_cfg: TriggerConfig, + db_cfg: DbConfig, + *, + n: int, + concurrency: int, + poll_interval: float = 0.5, + timeout: float = 600.0, + on_result: Callable[[RunResult], None] | None = None, + probe: Callable[..., RunResult] = run_probe, +) -> LoadOutcome: + """Run ``n`` probes, at most ``concurrency`` at a time. + + ``on_result`` (optional) is called as each probe finishes — used by the CLI + to stream progress. ``probe`` is injectable so the runner is unit-testable + without HTTP/DB. + """ + results: list[RunResult] = [] + start = time.monotonic() + with ThreadPoolExecutor(max_workers=concurrency) as pool: + futures = [ + pool.submit( + probe, + trigger_cfg, + db_cfg, + poll_interval=poll_interval, + timeout=timeout, + ) + for _ in range(n) + ] + for future in as_completed(futures): + result = future.result() + results.append(result) + if on_result is not None: + on_result(result) + return LoadOutcome(results=results, wall_clock=time.monotonic() - start) diff --git a/pg_benchmark/pg_benchmark/stats.py b/pg_benchmark/pg_benchmark/stats.py new file mode 100644 index 0000000000..83e3cd190f --- /dev/null +++ b/pg_benchmark/pg_benchmark/stats.py @@ -0,0 +1,74 @@ +"""Pure latency statistics — no I/O, fully unit-testable. + +The benchmark's headline output is a distribution, not a single number: a mean +hides the tail, and the tail is exactly where a polling transport (PG +SKIP-LOCKED) is suspected to differ from a push transport (RabbitMQ). So every +metric is summarised as ``n / mean / p50 / p95 / p99 / min / max / stdev``. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class Summary: + """A latency distribution summary over a sample of measurements.""" + + n: int + mean: float + p50: float + p95: float + p99: float + minimum: float + maximum: float + stdev: float + + @property + def empty(self) -> bool: + return self.n == 0 + + +def percentile(values: list[float], pct: float) -> float: + """Linear-interpolated percentile (``pct`` in 0..100). + + Uses the same "linear interpolation between closest ranks" method as + ``numpy.percentile`` / ``statistics.quantiles(method="inclusive")`` so the + numbers line up with what an analyst expects, without pulling numpy in. + """ + if not values: + raise ValueError("percentile() of an empty sample") + if pct <= 0: + return min(values) + if pct >= 100: + return max(values) + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + rank = (pct / 100.0) * (len(ordered) - 1) + low = math.floor(rank) + high = math.ceil(rank) + if low == high: + return ordered[low] + frac = rank - low + return ordered[low] * (1.0 - frac) + ordered[high] * frac + + +def summarize(values: list[float]) -> Summary: + """Summarise a sample. An empty sample yields an all-zero ``Summary``.""" + n = len(values) + if n == 0: + return Summary(0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0) + mean = sum(values) / n + variance = sum((v - mean) ** 2 for v in values) / n if n > 1 else 0.0 + return Summary( + n=n, + mean=mean, + p50=percentile(values, 50), + p95=percentile(values, 95), + p99=percentile(values, 99), + minimum=min(values), + maximum=max(values), + stdev=math.sqrt(variance), + ) diff --git a/pg_benchmark/pg_benchmark/trigger.py b/pg_benchmark/pg_benchmark/trigger.py new file mode 100644 index 0000000000..0fa19c8350 --- /dev/null +++ b/pg_benchmark/pg_benchmark/trigger.py @@ -0,0 +1,128 @@ +"""Trigger a workflow execution over HTTP against a running stack. + +Targets an Unstract **API deployment** (`POST .../execute/` with file uploads) — +the realistic load path that exercises the full dispatch → fan-out → executor +pipeline. The endpoint path and auth header are config so the same harness drives +local, staging, or any deployment without code changes. + +The response carries the ``execution_id`` (async deployments return it +immediately; sync ones return it alongside the finished status) — that id is the +join key the probe uses to read server-side latency from the DB. +""" + +from __future__ import annotations + +import os +import time +from dataclasses import dataclass, field + +import requests + + +@dataclass(frozen=True, slots=True) +class TriggerConfig: + """How to POST one execution to an API deployment.""" + + base_url: str # e.g. http://localhost:8000 + path: str # e.g. /deployment/api/// + api_key: str + files: list[str] = field(default_factory=list) # local paths to upload + auth_header: str = "Authorization" + auth_prefix: str = "Bearer " + request_timeout: float = 600.0 + extra_fields: dict[str, str] = field(default_factory=dict) + + @property + def url(self) -> str: + return self.base_url.rstrip("/") + "/" + self.path.lstrip("/") + + @classmethod + def from_env(cls, **overrides: object) -> TriggerConfig: + base = { + "base_url": os.environ.get("PGBENCH_BASE_URL", "http://localhost:8000"), + "path": os.environ.get("PGBENCH_DEPLOY_PATH", ""), + "api_key": os.environ.get("PGBENCH_API_KEY", ""), + } + base.update({k: v for k, v in overrides.items() if v is not None}) + return cls(**base) # type: ignore[arg-type] + + +@dataclass(frozen=True, slots=True) +class TriggerResult: + """Outcome of a single trigger POST.""" + + execution_id: str | None + http_status: int + http_latency: float # seconds, request → response + error: str | None = None + + +def _extract_execution_id(payload: object) -> str | None: + """Pull the execution id out of common Unstract response shapes.""" + if not isinstance(payload, dict): + return None + for key in ("execution_id", "id"): + value = payload.get(key) + if isinstance(value, str) and value: + return value + # Some responses nest under "execution" or "data". + for key in ("execution", "data"): + nested = payload.get(key) + found = _extract_execution_id(nested) + if found: + return found + return None + + +def trigger_execution( + cfg: TriggerConfig, session: requests.Session | None = None +) -> TriggerResult: + """POST one execution; return the execution id + HTTP latency. + + Opens each upload file fresh (so the same config is reusable across many + concurrent triggers) and always closes them. + """ + session = session or requests.Session() + headers = {cfg.auth_header: f"{cfg.auth_prefix}{cfg.api_key}"} + handles = [open(path, "rb") for path in cfg.files] # noqa: SIM115 — closed below + try: + files = [ + ("files", (os.path.basename(path), fh, "application/octet-stream")) + for path, fh in zip(cfg.files, handles, strict=True) + ] + start = time.monotonic() + try: + resp = session.post( + cfg.url, + headers=headers, + files=files or None, + data=cfg.extra_fields or None, + timeout=cfg.request_timeout, + ) + except requests.RequestException as exc: + return TriggerResult( + execution_id=None, + http_status=0, + http_latency=time.monotonic() - start, + error=f"request failed: {exc}", + ) + latency = time.monotonic() - start + try: + payload = resp.json() + except ValueError: + payload = None + execution_id = _extract_execution_id(payload) + error = None + if resp.status_code >= 400: + error = f"HTTP {resp.status_code}: {resp.text[:200]}" + elif execution_id is None: + error = f"no execution_id in response: {str(payload)[:200]}" + return TriggerResult( + execution_id=execution_id, + http_status=resp.status_code, + http_latency=latency, + error=error, + ) + finally: + for fh in handles: + fh.close() diff --git a/pg_benchmark/pyproject.toml b/pg_benchmark/pyproject.toml new file mode 100644 index 0000000000..c4d66184a7 --- /dev/null +++ b/pg_benchmark/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "pg_benchmark" +version = "0.1.0" +description = "PG-queue vs Celery execution benchmark harness for Unstract" +requires-python = ">=3.10" +dependencies = ["psycopg2-binary>=2.9", "requests>=2.31"] + +[tool.pytest.ini_options] +# Root pytest here so it does not inherit the backend's Django pytest config. +testpaths = ["tests"] +pythonpath = ["."] + +[tool.ruff] +line-length = 90 diff --git a/pg_benchmark/requirements.txt b/pg_benchmark/requirements.txt new file mode 100644 index 0000000000..e8be2e4ce9 --- /dev/null +++ b/pg_benchmark/requirements.txt @@ -0,0 +1,5 @@ +# Runtime deps for the PG-vs-Celery benchmark harness. +# Kept minimal + stdlib-first; psycopg2 for DB readers, requests for triggering. +psycopg2-binary>=2.9 +requests>=2.31 +# test-only: pytest diff --git a/pg_benchmark/tests/test_load.py b/pg_benchmark/tests/test_load.py new file mode 100644 index 0000000000..f87680c916 --- /dev/null +++ b/pg_benchmark/tests/test_load.py @@ -0,0 +1,212 @@ +"""Unit tests for the load layer (probe / runner / load report) with fakes. + +No HTTP or DB: ``run_probe``'s collaborators (trigger + DB readers) are +monkeypatched, and the poll loop's clock/sleep are injected, so the timing logic +is exercised deterministically. +""" + +from __future__ import annotations + +import pytest + +import pg_benchmark.probe as probe_mod +from pg_benchmark.config import DbConfig +from pg_benchmark.db import ExecutionLatency, Transport +from pg_benchmark.probe import RunResult, run_probe +from pg_benchmark.report import build_load_reports, render_load +from pg_benchmark.runner import LoadOutcome, run_load +from pg_benchmark.trigger import TriggerConfig, TriggerResult + + +class FakeClock: + """Returns successive values, repeating the last once exhausted.""" + + def __init__(self, values): + self._values = list(values) + self._i = 0 + + def __call__(self): + v = self._values[min(self._i, len(self._values) - 1)] + self._i += 1 + return v + + +def _trigger_cfg(): + return TriggerConfig(base_url="http://x", path="/p/", api_key="k") + + +def _db_cfg(): + return DbConfig() + + +def _latency(transport=Transport.PG, status="COMPLETED", exec_time=30.0, files=(29.0,)): + return ExecutionLatency( + execution_id="e1", + transport=transport, + status=status, + total_files=len(files), + server_execution_time=exec_time, + file_times=list(files), + ) + + +def _patch_probe(monkeypatch, *, trigger, statuses, latency): + monkeypatch.setattr(probe_mod, "trigger_execution", lambda cfg, session=None: trigger) + monkeypatch.setattr(probe_mod, "connect", lambda cfg: object()) + status_iter = iter(statuses) + monkeypatch.setattr( + probe_mod, "fetch_status", lambda conn, eid: next(status_iter, statuses[-1]) + ) + monkeypatch.setattr(probe_mod, "fetch_one", lambda conn, eid: latency) + # connect() returns a bare object(); ensure .close() is a no-op + monkeypatch.setattr( + probe_mod, "connect", lambda cfg: type("C", (), {"close": lambda s: None})() + ) + + +class TestRunProbe: + def test_success_computes_overhead(self, monkeypatch): + trig = TriggerResult(execution_id="e1", http_status=200, http_latency=0.05) + _patch_probe( + monkeypatch, + trigger=trig, + statuses=["EXECUTING", "COMPLETED"], + latency=_latency(exec_time=30.0), + ) + clock = FakeClock([100.0, 100.1, 100.2, 130.6]) # wall = 30.6s + result = run_probe( + _trigger_cfg(), + _db_cfg(), + poll_interval=0.0, + clock=clock, + sleep=lambda s: None, + ) + assert result.ok + assert result.transport is Transport.PG + assert result.wall_clock_e2e == pytest.approx(30.6) + assert result.server_execution_time == 30.0 + assert result.overhead == pytest.approx(0.6) + assert result.http_latency == 0.05 + + def test_trigger_without_execution_id_is_failure(self, monkeypatch): + trig = TriggerResult( + execution_id=None, http_status=401, http_latency=0.02, error="HTTP 401" + ) + _patch_probe(monkeypatch, trigger=trig, statuses=["x"], latency=None) + result = run_probe(_trigger_cfg(), _db_cfg(), clock=FakeClock([1.0])) + assert not result.ok + assert result.execution_id is None + assert "401" in result.error + + def test_timeout_when_never_terminal(self, monkeypatch): + trig = TriggerResult(execution_id="e1", http_status=200, http_latency=0.05) + _patch_probe( + monkeypatch, + trigger=trig, + statuses=["EXECUTING"], + latency=_latency(status="EXECUTING", exec_time=None, files=()), + ) + # deadline = start + timeout = 100 + 1 = 101; clock crosses it immediately. + clock = FakeClock([100.0, 102.0, 102.0]) + result = run_probe( + _trigger_cfg(), + _db_cfg(), + timeout=1.0, + clock=clock, + sleep=lambda s: None, + ) + assert not result.ok + assert "timed out" in result.error + assert result.overhead is None + + def test_error_status_is_not_ok(self, monkeypatch): + trig = TriggerResult(execution_id="e1", http_status=200, http_latency=0.05) + _patch_probe( + monkeypatch, + trigger=trig, + statuses=["ERROR"], + latency=_latency(status="ERROR"), + ) + clock = FakeClock([100.0, 100.1, 130.0]) + result = run_probe( + _trigger_cfg(), + _db_cfg(), + clock=clock, + sleep=lambda s: None, + ) + assert not result.ok + assert result.status == "ERROR" + + +class TestRunner: + def _result(self, ok=True, transport=Transport.PG, wall=10.0): + return RunResult( + execution_id="e", + transport=transport, + status="COMPLETED" if ok else "ERROR", + ok=ok, + http_latency=0.05, + wall_clock_e2e=wall, + server_execution_time=9.0, + parallelism=1.8, + overhead=wall - 9.0, + error=None if ok else "err", + ) + + def test_run_load_collects_all_and_computes_throughput(self): + calls = {"n": 0} + + def fake_probe(tcfg, dcfg, **kw): + calls["n"] += 1 + return self._result(ok=calls["n"] % 2 == 1) + + outcome = run_load( + _trigger_cfg(), + _db_cfg(), + n=4, + concurrency=2, + probe=fake_probe, + ) + assert len(outcome.results) == 4 + assert len(outcome.completed) == 2 + assert outcome.throughput >= 0.0 + + def test_load_outcome_zero_walltime_is_safe(self): + outcome = LoadOutcome(results=[self._result()], wall_clock=0.0) + assert outcome.throughput == 0.0 + + +class TestLoadReport: + def _result(self, transport, ok=True, wall=10.0, server=9.0): + return RunResult( + execution_id="e", + transport=transport, + status="COMPLETED" if ok else "ERROR", + ok=ok, + http_latency=0.05, + wall_clock_e2e=wall, + server_execution_time=server, + parallelism=1.8, + overhead=wall - server, + error=None, + ) + + def test_groups_by_transport(self): + results = [ + self._result(Transport.PG), + self._result(Transport.PG, ok=False), + self._result(Transport.CELERY), + ] + reports = build_load_reports(results) + by_t = {r.transport: r for r in reports} + assert by_t[Transport.PG].triggered == 2 + assert by_t[Transport.PG].completed == 1 + assert by_t[Transport.PG].failed == 1 + assert by_t[Transport.CELERY].triggered == 1 + + def test_render_load_smoke(self): + outcome = LoadOutcome(results=[self._result(Transport.PG)], wall_clock=10.0) + text = render_load(outcome) + assert "Load run" in text + assert "PG" in text + assert "wall-clock e2e" in text diff --git a/pg_benchmark/tests/test_stats.py b/pg_benchmark/tests/test_stats.py new file mode 100644 index 0000000000..d8f62811c1 --- /dev/null +++ b/pg_benchmark/tests/test_stats.py @@ -0,0 +1,113 @@ +"""Unit tests for the pure statistics + transport-classification layer.""" + +from __future__ import annotations + +import math + +import pytest + +from pg_benchmark.db import ExecutionLatency, Transport +from pg_benchmark.stats import percentile, summarize + + +class TestPercentile: + def test_endpoints_are_min_and_max(self): + values = [1.0, 2.0, 3.0, 4.0, 5.0] + assert percentile(values, 0) == 1.0 + assert percentile(values, 100) == 5.0 + + def test_median_of_odd_sample(self): + assert percentile([3.0, 1.0, 2.0], 50) == 2.0 + + def test_median_of_even_sample_interpolates(self): + assert percentile([1.0, 2.0, 3.0, 4.0], 50) == 2.5 + + def test_p95_interpolates_between_ranks(self): + # 0..100 step 1 → p95 lands exactly on 95 (rank 95 of 100). + values = [float(i) for i in range(101)] + assert percentile(values, 95) == pytest.approx(95.0) + + def test_single_element(self): + assert percentile([7.0], 50) == 7.0 + assert percentile([7.0], 99) == 7.0 + + def test_empty_raises(self): + with pytest.raises(ValueError): + percentile([], 50) + + +class TestSummarize: + def test_empty_sample_is_all_zero(self): + s = summarize([]) + assert s.empty + assert s.n == 0 and s.mean == 0.0 and s.p99 == 0.0 + + def test_basic_summary(self): + s = summarize([2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0]) + assert s.n == 8 + assert s.mean == pytest.approx(5.0) + assert s.minimum == 2.0 and s.maximum == 9.0 + assert s.stdev == pytest.approx(2.0) # population stdev of this classic sample + + def test_single_value_has_zero_stdev(self): + s = summarize([3.3]) + assert s.n == 1 and s.stdev == 0.0 and s.p50 == 3.3 + + +class TestTransportClassify: + def test_pg_wins_when_queue_message_present(self): + assert ( + Transport.classify(has_queue_message_id=True, has_task_id=True) + is Transport.PG + ) + + def test_celery_when_only_task_id(self): + assert ( + Transport.classify(has_queue_message_id=False, has_task_id=True) + is Transport.CELERY + ) + + def test_inline_when_neither(self): + assert ( + Transport.classify(has_queue_message_id=False, has_task_id=False) + is Transport.INLINE + ) + + +class TestParallelism: + def _exec(self, *, exec_time, file_times): + return ExecutionLatency( + execution_id="x", + transport=Transport.PG, + status="COMPLETED", + total_files=len(file_times), + server_execution_time=exec_time, + file_times=file_times, + ) + + def test_fully_parallel_two_files_is_near_two(self): + # Two 30s files that overlapped → execution ~34s → ratio ≈ 1.76. + e = self._exec(exec_time=34.0, file_times=[29.9, 29.8]) + assert e.parallelism == pytest.approx(59.7 / 34.0) + assert e.parallelism > 1.5 + + def test_serial_two_files_is_near_one(self): + e = self._exec(exec_time=60.0, file_times=[30.0, 30.0]) + assert e.parallelism == pytest.approx(1.0) + + def test_none_when_no_files(self): + assert self._exec(exec_time=10.0, file_times=[]).parallelism is None + + def test_none_when_no_exec_time(self): + e = self._exec(exec_time=None, file_times=[1.0]) + assert e.parallelism is None + + def test_is_terminal(self): + assert self._exec(exec_time=1.0, file_times=[1.0]).is_terminal + + +def test_stdev_matches_math(): + values = [1.0, 2.0, 3.0] + s = summarize(values) + expected = math.sqrt(((1 - 2) ** 2 + 0 + (3 - 2) ** 2) / 3) + assert s.stdev == pytest.approx(expected) From bc33c8c83492533f5dddfca0934e3f6733fb1534 Mon Sep 17 00:00:00 2001 From: ali Date: Mon, 22 Jun 2026 17:06:36 +0530 Subject: [PATCH 35/44] =?UTF-8?q?UN-3445=20[TOOL]=20pg=5Fbenchmark=20?= =?UTF-8?q?=E2=80=94=20parse=20real=20API-deployment=20response=20+=20extr?= =?UTF-8?q?a=20headers/form=20fields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validated against a live API deployment: the execute response nests the execution_id inside message.status_api as a ?execution_id= query param (not a top-level field). Fix _extract_execution_id to handle it (+ status_api/message wrappers), and add --header KEY:VALUE / --form KEY=VALUE so the harness can send the subscription headers + tags/timeout fields a real deployment expects. One real execution confirmed end-to-end: PG transport, COMPLETED, 24.08s server time / 19.9s file — measurement path verified. test_trigger.py pins the response shape + the header/form parsing. Co-Authored-By: Claude Opus 4.8 --- pg_benchmark/pg_benchmark/cli.py | 25 +++++++++++++ pg_benchmark/pg_benchmark/trigger.py | 36 ++++++++++++++---- pg_benchmark/tests/test_trigger.py | 56 ++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 pg_benchmark/tests/test_trigger.py diff --git a/pg_benchmark/pg_benchmark/cli.py b/pg_benchmark/pg_benchmark/cli.py index 08757b6cc1..6fb18231de 100644 --- a/pg_benchmark/pg_benchmark/cli.py +++ b/pg_benchmark/pg_benchmark/cli.py @@ -71,6 +71,17 @@ def _cmd_queue_depth(args: argparse.Namespace) -> int: return 0 +def _parse_kv(items: list[str], *, sep: str, label: str) -> dict[str, str]: + """Parse repeated ``KEYVALUE`` CLI args into a dict (split on first sep only).""" + out: dict[str, str] = {} + for item in items: + key, found, value = item.partition(sep) + if not found: + raise SystemExit(f"error: {label} expects KEY{sep}VALUE, got {item!r}") + out[key.strip()] = value.strip() + return out + + def _cmd_run(args: argparse.Namespace) -> int: api_key = args.api_key or os.environ.get("PGBENCH_API_KEY", "") if not args.path: @@ -79,6 +90,8 @@ def _cmd_run(args: argparse.Namespace) -> int: if not api_key: print("error: --api-key or PGBENCH_API_KEY is required", file=sys.stderr) return 2 + extra_headers = _parse_kv(args.header or [], sep=":", label="--header") + extra_fields = _parse_kv(args.form or [], sep="=", label="--form") trigger_cfg = TriggerConfig( base_url=args.base_url, path=args.path, @@ -86,6 +99,8 @@ def _cmd_run(args: argparse.Namespace) -> int: files=args.file or [], auth_header=args.auth_header, auth_prefix=args.auth_prefix, + extra_headers=extra_headers, + extra_fields=extra_fields, ) db_cfg = _db_config(args) done = {"n": 0} @@ -158,6 +173,16 @@ def build_parser() -> argparse.ArgumentParser: ) run.add_argument("--api-key", help="API key (or set PGBENCH_API_KEY)") run.add_argument("--file", action="append", help="local file to upload (repeatable)") + run.add_argument( + "--header", + action="append", + help="extra request header KEY:VALUE (repeatable), e.g. X-subscription-id:sub-1", + ) + run.add_argument( + "--form", + action="append", + help="extra multipart form field KEY=VALUE (repeatable), e.g. tags=ali1", + ) run.add_argument("--auth-header", default="Authorization") run.add_argument("--auth-prefix", default="Bearer ") run.add_argument("--poll-interval", type=float, default=0.5) diff --git a/pg_benchmark/pg_benchmark/trigger.py b/pg_benchmark/pg_benchmark/trigger.py index 0fa19c8350..fc9f320913 100644 --- a/pg_benchmark/pg_benchmark/trigger.py +++ b/pg_benchmark/pg_benchmark/trigger.py @@ -15,6 +15,7 @@ import os import time from dataclasses import dataclass, field +from urllib.parse import parse_qs, urlparse import requests @@ -30,7 +31,8 @@ class TriggerConfig: auth_header: str = "Authorization" auth_prefix: str = "Bearer " request_timeout: float = 600.0 - extra_fields: dict[str, str] = field(default_factory=dict) + extra_fields: dict[str, str] = field(default_factory=dict) # multipart form fields + extra_headers: dict[str, str] = field(default_factory=dict) # e.g. X-subscription-id @property def url(self) -> str: @@ -57,18 +59,38 @@ class TriggerResult: error: str | None = None +def _id_from_url(url: str) -> str | None: + """Extract the ``execution_id`` query param from a status URL, if present.""" + try: + values = parse_qs(urlparse(url).query).get("execution_id") + except ValueError: + return None + return values[0] if values else None + + def _extract_execution_id(payload: object) -> str | None: - """Pull the execution id out of common Unstract response shapes.""" + """Pull the execution id out of Unstract API-deployment response shapes. + + The async/sync execute response nests the id inside ``message.status_api`` as + a ``?execution_id=...`` query param (not a top-level field), so probe the + direct keys, then ``status_api``, then recurse into the common wrappers. + """ + if isinstance(payload, str): + return _id_from_url(payload) if not isinstance(payload, dict): return None for key in ("execution_id", "id"): value = payload.get(key) if isinstance(value, str) and value: return value - # Some responses nest under "execution" or "data". - for key in ("execution", "data"): - nested = payload.get(key) - found = _extract_execution_id(nested) + status_api = payload.get("status_api") + if isinstance(status_api, str): + found = _id_from_url(status_api) + if found: + return found + # Real shape wraps everything under "message"; older shapes used execution/data. + for key in ("message", "execution", "data"): + found = _extract_execution_id(payload.get(key)) if found: return found return None @@ -83,7 +105,7 @@ def trigger_execution( concurrent triggers) and always closes them. """ session = session or requests.Session() - headers = {cfg.auth_header: f"{cfg.auth_prefix}{cfg.api_key}"} + headers = {cfg.auth_header: f"{cfg.auth_prefix}{cfg.api_key}", **cfg.extra_headers} handles = [open(path, "rb") for path in cfg.files] # noqa: SIM115 — closed below try: files = [ diff --git a/pg_benchmark/tests/test_trigger.py b/pg_benchmark/tests/test_trigger.py new file mode 100644 index 0000000000..46e5bc21e7 --- /dev/null +++ b/pg_benchmark/tests/test_trigger.py @@ -0,0 +1,56 @@ +"""Unit tests for trigger response parsing — pins the real Unstract response shape.""" + +from __future__ import annotations + +from pg_benchmark.cli import _parse_kv +from pg_benchmark.trigger import _extract_execution_id, _id_from_url + +EID = "b1f16024-45f2-4e39-8756-d40e24148e30" + + +class TestExtractExecutionId: + def test_real_api_deployment_shape(self): + # The exact shape returned by POST /deployment/api/// — id is a + # query param inside message.status_api, not a top-level field. + payload = { + "message": { + "execution_status": "COMPLETED", + "status_api": f"/deployment/api/org_X/api-test/?execution_id={EID}", + "error": None, + "result": [{"file": "dec-2025.pdf"}], + } + } + assert _extract_execution_id(payload) == EID + + def test_top_level_execution_id(self): + assert _extract_execution_id({"execution_id": EID}) == EID + + def test_nested_under_data(self): + assert _extract_execution_id({"data": {"id": EID}}) == EID + + def test_none_when_absent(self): + assert _extract_execution_id({"message": {"status": "ok"}}) is None + assert _extract_execution_id("not-json") is None + + def test_id_from_url(self): + assert _id_from_url(f"/x/?execution_id={EID}&foo=1") == EID + assert _id_from_url("/x/?foo=1") is None + + +class TestParseKv: + def test_header_split_on_first_colon(self): + # A header value can itself contain a colon (e.g. a URL). + out = _parse_kv(["X-Sub:sub-1", "X-Url:http://x:8000"], sep=":", label="--header") + assert out == {"X-Sub": "sub-1", "X-Url": "http://x:8000"} + + def test_form_split_on_equals(self): + assert _parse_kv(["tags=ali1", "timeout=300"], sep="=", label="--form") == { + "tags": "ali1", + "timeout": "300", + } + + def test_missing_separator_raises(self): + import pytest + + with pytest.raises(SystemExit): + _parse_kv(["bad"], sep=":", label="--header") From d9b2e95d9b372acb3e2161099b4cb1196d6ebce8 Mon Sep 17 00:00:00 2001 From: ali Date: Mon, 22 Jun 2026 17:41:16 +0530 Subject: [PATCH 36/44] =?UTF-8?q?UN-3445=20[TOOL]=20pg=5Fbenchmark=20?= =?UTF-8?q?=E2=80=94=20mock=20OpenAI=20server=20for=20zero-cost=20load=20t?= =?UTF-8?q?esting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instant OpenAI-compatible mock LLM+embedding responder (stdlib http.server, threaded, dependency-free). Point an OpenAI-compatible adapter's api_base at it + pair with Unstract's built-in noOpX2text / noOpVectorDb adapters → a full execution runs the whole queue -> fan-out -> executor path at $0, so the transport (not the LLM) is the measured variable. python -m pg_benchmark.mock_server --port 8901 Serves /v1/chat/completions, /v1/embeddings (batch), /v1/models, /health; --content / --embedding-dim / --latency-ms knobs. 6 tests over real HTTP on an ephemeral port. README documents the zero-cost load-test setup. Co-Authored-By: Claude Opus 4.8 --- pg_benchmark/README.md | 39 ++++ pg_benchmark/pg_benchmark/mock_server.py | 223 +++++++++++++++++++++++ pg_benchmark/tests/test_mock_server.py | 84 +++++++++ 3 files changed, 346 insertions(+) create mode 100644 pg_benchmark/pg_benchmark/mock_server.py create mode 100644 pg_benchmark/tests/test_mock_server.py diff --git a/pg_benchmark/README.md b/pg_benchmark/README.md index 6ba438dd1a..8b4286239b 100644 --- a/pg_benchmark/README.md +++ b/pg_benchmark/README.md @@ -59,6 +59,45 @@ DB connection defaults mirror the local docker-compose stack (`localhost:5432/unstract_db`, schema `unstract`). Override via `DB_HOST`, `DB_PORT`, `DB_NAME`, `DB_USER`, `DB_PASSWORD` env vars or the `--db-*` flags. +### Triggering a real deployment + +`run` POSTs to an API deployment. The execute response nests the `execution_id` +inside `message.status_api` (a `?execution_id=` query param) — the harness parses +that automatically. Subscription headers + form fields the deployment expects are +passed through: + +```bash +python -m pg_benchmark run --n 1 --concurrency 1 \ + --path /deployment/api/// --api-key \ + --file /path/to/doc.pdf \ + --header X-subscription-id:sub-1 --form tags=bench --form timeout=300 +``` + +## Zero-cost load testing (mock adapters) + +Real LLM/extraction calls cost money **and** are a poor transport signal — a 20s +LLM call swamps the millisecond queue differences. For a true PG-vs-Celery +**transport** benchmark, mock the adapters so cost = $0 and execution time ≈ +queue + dispatch + fan-out: + +1. **LLM + embedding** → the bundled mock server (instant, OpenAI-compatible): + + ```bash + python -m pg_benchmark.mock_server --port 8901 # POST /v1/chat/completions + /v1/embeddings + ``` + + Then set an **OpenAI-compatible** LLM and embedding adapter's `api_base` to + `http://:8901/v1`. `--latency-ms` can simulate adapter latency; `--content` + sets the canned answer. + +2. **Extraction + vector DB** → Unstract's built-in **`noOpX2text`** and + **`noOpVectorDb`** adapters (instant, free, active by default). + +Wire a "benchmark" workflow to those four adapters, expose it as an API +deployment, then fire load through the **same** transport-comparison protocol: +run a batch with the flag/gate **off** (Celery) and another **on** (PG) against +the same deployment — the runner buckets each execution by its observed transport. + ## Tests ```bash diff --git a/pg_benchmark/pg_benchmark/mock_server.py b/pg_benchmark/pg_benchmark/mock_server.py new file mode 100644 index 0000000000..3531f96240 --- /dev/null +++ b/pg_benchmark/pg_benchmark/mock_server.py @@ -0,0 +1,223 @@ +"""Instant OpenAI-compatible mock server for zero-cost load testing. + +Stands in for the LLM + embedding adapters during a PG-vs-Celery transport +benchmark: point an OpenAI-compatible adapter's ``api_base`` at this server and +every chat/embedding call returns a canned response in ~milliseconds — so an +execution exercises the full queue → fan-out → executor path with **no real +inference cost**, and the transport (not the LLM) becomes the measured variable. +Pair with Unstract's built-in ``noOpX2text`` + ``noOpVectorDb`` adapters for a +fully free pipeline. + +Dependency-free (stdlib ``http.server``, threaded so concurrent executions don't +serialise) — runs anywhere the harness runs. NOT for production: no auth, canned +data only. + +Run: ``python -m pg_benchmark.mock_server --port 8901`` +Then set the adapter ``api_base`` to ``http://:8901/v1``. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Lock + + +@dataclass(frozen=True, slots=True) +class MockConfig: + """Canned-response knobs for the mock server.""" + + content: str = '{"result": "mock"}' # chat completion message content + embedding_dim: int = 1536 + latency_ms: float = 0.0 # artificial per-request delay (0 = instant) + + @property + def latency_s(self) -> float: + return self.latency_ms / 1000.0 + + +class _Counter: + """Thread-safe request tallies (so a load run can confirm the server was hit).""" + + def __init__(self) -> None: + self._lock = Lock() + self.chat = 0 + self.embeddings = 0 + + def bump(self, kind: str) -> int: + with self._lock: + value = getattr(self, kind) + 1 + setattr(self, kind, value) + return value + + +def chat_completion(model: str, content: str) -> dict: + return { + "id": "chatcmpl-mock", + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + +def text_completion(model: str, content: str) -> dict: + return { + "id": "cmpl-mock", + "object": "text_completion", + "created": int(time.time()), + "model": model, + "choices": [{"index": 0, "text": content, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + +def embeddings(model: str, count: int, dim: int) -> dict: + vector = [0.0] * dim + return { + "object": "list", + "data": [ + {"object": "embedding", "index": i, "embedding": vector} + for i in range(max(1, count)) + ], + "model": model, + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + } + + +def _make_handler(config: MockConfig, counter: _Counter) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + # HTTP/1.1 + explicit Content-Length → keep-alive works (LiteLLM reuses + # connections under load). + protocol_version = "HTTP/1.1" + + def log_message(self, *args: object) -> None: # silence per-request logging + pass + + def _send_json(self, payload: dict, status: int = 200) -> None: + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _read_json(self) -> dict: + length = int(self.headers.get("Content-Length", 0) or 0) + raw = self.rfile.read(length) if length else b"" + try: + parsed = json.loads(raw or b"{}") + except ValueError: + return {} + return parsed if isinstance(parsed, dict) else {} + + def do_GET(self) -> None: # noqa: N802 — http.server API + path = self.path.rstrip("/") + if path.endswith("/models"): + self._send_json( + {"object": "list", "data": [{"id": "mock-model", "object": "model"}]} + ) + elif path.endswith("/health"): + self._send_json( + { + "status": "ok", + "chat": counter.chat, + "embeddings": counter.embeddings, + } + ) + else: + self._send_json({"error": "not found"}, status=404) + + def do_POST(self) -> None: # noqa: N802 — http.server API + if config.latency_s: + time.sleep(config.latency_s) + req = self._read_json() + model = req.get("model", "mock-model") + path = self.path + # Order matters: chat/completions must be matched before the bare + # /completions suffix. + if path.endswith("/chat/completions"): + counter.bump("chat") + self._send_json(chat_completion(model, config.content)) + elif path.endswith("/embeddings"): + counter.bump("embeddings") + inputs = req.get("input", "") + count = len(inputs) if isinstance(inputs, list) else 1 + self._send_json(embeddings(model, count, config.embedding_dim)) + elif path.endswith("/completions"): + counter.bump("chat") + self._send_json(text_completion(model, config.content)) + else: + self._send_json({"error": "not found"}, status=404) + + return Handler + + +def make_server( + config: MockConfig, host: str = "0.0.0.0", port: int = 8901 +) -> tuple[ThreadingHTTPServer, _Counter]: + """Build (but don't start) a threaded mock server. ``port=0`` picks a free port.""" + counter = _Counter() + server = ThreadingHTTPServer((host, port), _make_handler(config, counter)) + return server, counter + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="pg_benchmark.mock_server", + description="Instant OpenAI-compatible mock LLM+embedding server for load testing.", + ) + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=8901) + parser.add_argument("--embedding-dim", type=int, default=1536) + parser.add_argument( + "--content", + default='{"result": "mock"}', + help="canned chat-completion message content", + ) + parser.add_argument( + "--latency-ms", + type=float, + default=0.0, + help="artificial per-request latency (0 = instant; use to simulate adapters)", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + config = MockConfig( + content=args.content, + embedding_dim=args.embedding_dim, + latency_ms=args.latency_ms, + ) + server, _counter = make_server(config, host=args.host, port=args.port) + print( + f"mock OpenAI server on http://{args.host}:{args.port}/v1 " + f"(chat/completions + embeddings; dim={args.embedding_dim}, " + f"latency={args.latency_ms}ms) — Ctrl-C to stop", + flush=True, + ) + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nstopping mock server", flush=True) + finally: + server.shutdown() + server.server_close() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pg_benchmark/tests/test_mock_server.py b/pg_benchmark/tests/test_mock_server.py new file mode 100644 index 0000000000..c680629c96 --- /dev/null +++ b/pg_benchmark/tests/test_mock_server.py @@ -0,0 +1,84 @@ +"""Unit tests for the mock OpenAI server — real HTTP on an ephemeral port.""" + +from __future__ import annotations + +import threading +from collections.abc import Iterator + +import pytest +import requests + +from pg_benchmark.mock_server import MockConfig, make_server + + +@pytest.fixture +def server() -> Iterator[tuple[str, object]]: + config = MockConfig(content='{"x": 1}', embedding_dim=8, latency_ms=0) + srv, counter = make_server(config, host="127.0.0.1", port=0) # 0 = free port + port = srv.server_address[1] + thread = threading.Thread(target=srv.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{port}", counter + finally: + srv.shutdown() + srv.server_close() + + +def test_chat_completion_shape(server): + base, counter = server + resp = requests.post( + f"{base}/v1/chat/completions", + json={"model": "m", "messages": [{"role": "user", "content": "hi"}]}, + timeout=5, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["object"] == "chat.completion" + assert body["choices"][0]["message"]["content"] == '{"x": 1}' + assert body["choices"][0]["finish_reason"] == "stop" + assert counter.chat == 1 + + +def test_embeddings_batch_returns_one_vector_per_input(server): + base, _ = server + resp = requests.post( + f"{base}/v1/embeddings", + json={"model": "m", "input": ["a", "b", "c"]}, + timeout=5, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["object"] == "list" + assert len(body["data"]) == 3 + assert len(body["data"][0]["embedding"]) == 8 # configured dim + + +def test_embeddings_single_string_input(server): + base, _ = server + resp = requests.post( + f"{base}/v1/embeddings", json={"model": "m", "input": "solo"}, timeout=5 + ) + assert len(resp.json()["data"]) == 1 + + +def test_path_without_v1_prefix_still_matches(server): + # LiteLLM may call /chat/completions directly if api_base already has no /v1. + base, _ = server + resp = requests.post( + f"{base}/chat/completions", json={"model": "m", "messages": []}, timeout=5 + ) + assert resp.status_code == 200 + assert resp.json()["object"] == "chat.completion" + + +def test_health_and_models(server): + base, _ = server + assert requests.get(f"{base}/health", timeout=5).json()["status"] == "ok" + models = requests.get(f"{base}/v1/models", timeout=5).json() + assert models["data"][0]["id"] == "mock-model" + + +def test_unknown_path_404(server): + base, _ = server + assert requests.post(f"{base}/v1/nonsense", json={}, timeout=5).status_code == 404 From 556e27e6860ba5296a40c2c7a2f423482a054e68 Mon Sep 17 00:00:00 2001 From: ali Date: Tue, 23 Jun 2026 16:15:54 +0530 Subject: [PATCH 37/44] =?UTF-8?q?UN-3445=20[TOOL]=20pg=5Fbenchmark=20?= =?UTF-8?q?=E2=80=94=20address=20SonarCloud=20findings=20(reflection=20+?= =?UTF-8?q?=20float-eq=20+=20http)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - mock_server: respond with a fixed MOCK_MODEL instead of echoing the request's model field — do not reflect untrusted client input into the response body. - tests: route float comparisons through a math.isclose _close() helper instead of ==/pytest.approx (S1244: no equality checks on floats). - tests: https dummy URLs in fixtures (S5332 clear-text hotspot). Dev tooling; 40 tests green. Co-Authored-By: Claude Opus 4.8 --- pg_benchmark/pg_benchmark/mock_server.py | 14 ++++++--- pg_benchmark/tests/test_load.py | 20 ++++++++----- pg_benchmark/tests/test_stats.py | 37 ++++++++++++++---------- pg_benchmark/tests/test_trigger.py | 8 +++-- 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/pg_benchmark/pg_benchmark/mock_server.py b/pg_benchmark/pg_benchmark/mock_server.py index 3531f96240..9b0936823c 100644 --- a/pg_benchmark/pg_benchmark/mock_server.py +++ b/pg_benchmark/pg_benchmark/mock_server.py @@ -40,6 +40,11 @@ def latency_s(self) -> float: return self.latency_ms / 1000.0 +# Fixed model name returned in every response. Deliberately NOT echoed from the +# request body — never reflect untrusted client input back into the response. +MOCK_MODEL = "mock-model" + + class _Counter: """Thread-safe request tallies (so a load run can confirm the server was hit).""" @@ -143,21 +148,22 @@ def do_POST(self) -> None: # noqa: N802 — http.server API if config.latency_s: time.sleep(config.latency_s) req = self._read_json() - model = req.get("model", "mock-model") + # Always respond with MOCK_MODEL — never echo the request's "model" + # (untrusted input) back into the response body. path = self.path # Order matters: chat/completions must be matched before the bare # /completions suffix. if path.endswith("/chat/completions"): counter.bump("chat") - self._send_json(chat_completion(model, config.content)) + self._send_json(chat_completion(MOCK_MODEL, config.content)) elif path.endswith("/embeddings"): counter.bump("embeddings") inputs = req.get("input", "") count = len(inputs) if isinstance(inputs, list) else 1 - self._send_json(embeddings(model, count, config.embedding_dim)) + self._send_json(embeddings(MOCK_MODEL, count, config.embedding_dim)) elif path.endswith("/completions"): counter.bump("chat") - self._send_json(text_completion(model, config.content)) + self._send_json(text_completion(MOCK_MODEL, config.content)) else: self._send_json({"error": "not found"}, status=404) diff --git a/pg_benchmark/tests/test_load.py b/pg_benchmark/tests/test_load.py index f87680c916..4d87054022 100644 --- a/pg_benchmark/tests/test_load.py +++ b/pg_benchmark/tests/test_load.py @@ -7,7 +7,8 @@ from __future__ import annotations -import pytest +import math + import pg_benchmark.probe as probe_mod from pg_benchmark.config import DbConfig @@ -18,6 +19,11 @@ from pg_benchmark.trigger import TriggerConfig, TriggerResult +def _close(a: float, b: float) -> bool: + """Float comparison helper — avoids ``==`` on floats (SonarCloud S1244).""" + return math.isclose(a, b, rel_tol=1e-9, abs_tol=1e-12) + + class FakeClock: """Returns successive values, repeating the last once exhausted.""" @@ -32,7 +38,7 @@ def __call__(self): def _trigger_cfg(): - return TriggerConfig(base_url="http://x", path="/p/", api_key="k") + return TriggerConfig(base_url="https://x", path="/p/", api_key="k") def _db_cfg(): @@ -83,10 +89,10 @@ def test_success_computes_overhead(self, monkeypatch): ) assert result.ok assert result.transport is Transport.PG - assert result.wall_clock_e2e == pytest.approx(30.6) - assert result.server_execution_time == 30.0 - assert result.overhead == pytest.approx(0.6) - assert result.http_latency == 0.05 + assert _close(result.wall_clock_e2e, 30.6) + assert _close(result.server_execution_time, 30.0) + assert _close(result.overhead, 0.6) + assert _close(result.http_latency, 0.05) def test_trigger_without_execution_id_is_failure(self, monkeypatch): trig = TriggerResult( @@ -173,7 +179,7 @@ def fake_probe(tcfg, dcfg, **kw): def test_load_outcome_zero_walltime_is_safe(self): outcome = LoadOutcome(results=[self._result()], wall_clock=0.0) - assert outcome.throughput == 0.0 + assert _close(outcome.throughput, 0.0) class TestLoadReport: diff --git a/pg_benchmark/tests/test_stats.py b/pg_benchmark/tests/test_stats.py index d8f62811c1..0fcfa778d5 100644 --- a/pg_benchmark/tests/test_stats.py +++ b/pg_benchmark/tests/test_stats.py @@ -10,26 +10,31 @@ from pg_benchmark.stats import percentile, summarize +def _close(a: float, b: float) -> bool: + """Float comparison helper — avoids ``==`` on floats (SonarCloud S1244).""" + return math.isclose(a, b, rel_tol=1e-9, abs_tol=1e-12) + + class TestPercentile: def test_endpoints_are_min_and_max(self): values = [1.0, 2.0, 3.0, 4.0, 5.0] - assert percentile(values, 0) == 1.0 - assert percentile(values, 100) == 5.0 + assert _close(percentile(values, 0), 1.0) + assert _close(percentile(values, 100), 5.0) def test_median_of_odd_sample(self): - assert percentile([3.0, 1.0, 2.0], 50) == 2.0 + assert _close(percentile([3.0, 1.0, 2.0], 50), 2.0) def test_median_of_even_sample_interpolates(self): - assert percentile([1.0, 2.0, 3.0, 4.0], 50) == 2.5 + assert _close(percentile([1.0, 2.0, 3.0, 4.0], 50), 2.5) def test_p95_interpolates_between_ranks(self): # 0..100 step 1 → p95 lands exactly on 95 (rank 95 of 100). values = [float(i) for i in range(101)] - assert percentile(values, 95) == pytest.approx(95.0) + assert _close(percentile(values, 95), 95.0) def test_single_element(self): - assert percentile([7.0], 50) == 7.0 - assert percentile([7.0], 99) == 7.0 + assert _close(percentile([7.0], 50), 7.0) + assert _close(percentile([7.0], 99), 7.0) def test_empty_raises(self): with pytest.raises(ValueError): @@ -40,18 +45,20 @@ class TestSummarize: def test_empty_sample_is_all_zero(self): s = summarize([]) assert s.empty - assert s.n == 0 and s.mean == 0.0 and s.p99 == 0.0 + assert s.n == 0 + assert _close(s.mean, 0.0) and _close(s.p99, 0.0) def test_basic_summary(self): s = summarize([2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0]) assert s.n == 8 - assert s.mean == pytest.approx(5.0) - assert s.minimum == 2.0 and s.maximum == 9.0 - assert s.stdev == pytest.approx(2.0) # population stdev of this classic sample + assert _close(s.mean, 5.0) + assert _close(s.minimum, 2.0) and _close(s.maximum, 9.0) + assert _close(s.stdev, 2.0) # population stdev of this classic sample def test_single_value_has_zero_stdev(self): s = summarize([3.3]) - assert s.n == 1 and s.stdev == 0.0 and s.p50 == 3.3 + assert s.n == 1 + assert _close(s.stdev, 0.0) and _close(s.p50, 3.3) class TestTransportClassify: @@ -88,12 +95,12 @@ def _exec(self, *, exec_time, file_times): def test_fully_parallel_two_files_is_near_two(self): # Two 30s files that overlapped → execution ~34s → ratio ≈ 1.76. e = self._exec(exec_time=34.0, file_times=[29.9, 29.8]) - assert e.parallelism == pytest.approx(59.7 / 34.0) + assert _close(e.parallelism, 59.7 / 34.0) assert e.parallelism > 1.5 def test_serial_two_files_is_near_one(self): e = self._exec(exec_time=60.0, file_times=[30.0, 30.0]) - assert e.parallelism == pytest.approx(1.0) + assert _close(e.parallelism, 1.0) def test_none_when_no_files(self): assert self._exec(exec_time=10.0, file_times=[]).parallelism is None @@ -110,4 +117,4 @@ def test_stdev_matches_math(): values = [1.0, 2.0, 3.0] s = summarize(values) expected = math.sqrt(((1 - 2) ** 2 + 0 + (3 - 2) ** 2) / 3) - assert s.stdev == pytest.approx(expected) + assert _close(s.stdev, expected) diff --git a/pg_benchmark/tests/test_trigger.py b/pg_benchmark/tests/test_trigger.py index 46e5bc21e7..d919d1fde5 100644 --- a/pg_benchmark/tests/test_trigger.py +++ b/pg_benchmark/tests/test_trigger.py @@ -39,9 +39,11 @@ def test_id_from_url(self): class TestParseKv: def test_header_split_on_first_colon(self): - # A header value can itself contain a colon (e.g. a URL). - out = _parse_kv(["X-Sub:sub-1", "X-Url:http://x:8000"], sep=":", label="--header") - assert out == {"X-Sub": "sub-1", "X-Url": "http://x:8000"} + # A header value can itself contain a colon (e.g. a URL with a port). + out = _parse_kv( + ["X-Sub:sub-1", "X-Url:https://x:8000"], sep=":", label="--header" + ) + assert out == {"X-Sub": "sub-1", "X-Url": "https://x:8000"} def test_form_split_on_equals(self): assert _parse_kv(["tags=ali1", "timeout=300"], sep="=", label="--form") == { From 4fee3638d17a0a65d7b50b5e00c8e8dfd4b9987d Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Tue, 23 Jun 2026 17:01:40 +0530 Subject: [PATCH 38/44] =?UTF-8?q?UN-3616=20[FEAT]=20PG=20Queue=20=E2=80=94?= =?UTF-8?q?=20route=20API-triggered=20pipeline=20trigger=20through=20the?= =?UTF-8?q?=20transport=20flag=20(#2106)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3616 [FEAT] PG Queue — route API-triggered pipeline trigger through the transport flag The API-triggered pipeline view dispatched scheduler.tasks.execute_pipeline_task on Celery unconditionally — so when the flag was on, the trigger stayed on Celery while the execution it spawns rode PG, breaking uniform flag control. The SCHEDULED path already routes this task to PG (pg_scheduler.dispatch_due_schedules → scheduler queue, run by worker-pg-scheduler); only the API-trigger view bypassed it. Route the trigger through resolve_transport (same flag as everything else) via a new dispatch_pipeline_trigger helper: PG → enqueue to the scheduler queue via pg_queue.producer.enqueue_task with the exact arg shape pg_scheduler sends; else Celery send_task (unchanged). Fail-closed — master gate off (prod default) → Celery → byte-identical to before, zero regression. Routing logic lives in the helper, not the view. 4 unit tests (PG/Celery routing, pipeline-id entity bucketing, identical args on both paths). Dev-tested against the live flag: gate ON → PG scheduler queue, gate OFF → Celery. Co-Authored-By: Claude Opus 4.8 * UN-3616 [REVIEW] address PR feedback on the pipeline-trigger dispatcher - Compare the WorkflowTransport enum member instead of its .value string, so a literal typo can't silently fall through to the Celery branch. - Coerce org_id to str at args[1] for consistency with the str(pipeline_id) / org_id kwarg coercion. - Add tests: PG enqueue failure propagates with NO Celery fallback (no double-dispatch); UUID pipeline_id is str-coerced in args while resolve_transport receives the raw UUID entity. - Docstring: lead with current behavior + correct the 'byte-identical args' claim (PG JSON-normalizes via enqueue_task). Trim the duplicated view comment. 6 unit tests green. Co-Authored-By: Claude Opus 4.8 * UN-3616 [REVIEW] address 2 SonarCloud findings on the dispatcher - S3516 (Blocker): dispatch_pipeline_trigger returned the same 'transport' var on both branches (invariant return the view ignored). Make it a None-returning side-effect (if/else, no return); tests assert on the dispatch (enqueue vs send_task) instead of the return. - Typing (High): widen org_id / pipeline_id to 'str | UUID' — what resolve_transport accepts and what the UUID test passes; they're str-coerced into the task args. 6 unit tests green; ruff clean. Co-Authored-By: Claude Opus 4.8 * UN-3616 [REVIEW] use shared is_pg_transport() helper (greptile) Replace the hand-rolled 'transport == WorkflowTransport.PG_QUEUE' check with unstract.core.data_models.is_pg_transport() — the single source for 'what counts as PG transport' — so this dispatcher doesn't open a second comparison site, consistent with workers/scheduler/tasks.py. 6 unit tests green; ruff clean. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .../piepline_api_execution_views.py | 19 ++- backend/pipeline_v2/pipeline_dispatch.py | 75 ++++++++++++ backend/pipeline_v2/tests/__init__.py | 0 .../tests/test_pipeline_dispatch.py | 109 ++++++++++++++++++ 4 files changed, 192 insertions(+), 11 deletions(-) create mode 100644 backend/pipeline_v2/pipeline_dispatch.py create mode 100644 backend/pipeline_v2/tests/__init__.py create mode 100644 backend/pipeline_v2/tests/test_pipeline_dispatch.py diff --git a/backend/pipeline_v2/piepline_api_execution_views.py b/backend/pipeline_v2/piepline_api_execution_views.py index a2b88a13df..ca0ba8fa35 100644 --- a/backend/pipeline_v2/piepline_api_execution_views.py +++ b/backend/pipeline_v2/piepline_api_execution_views.py @@ -8,6 +8,7 @@ from backend.celery_service import app as celery_app from pipeline_v2.deployment_helper import DeploymentHelper from pipeline_v2.models import Pipeline +from pipeline_v2.pipeline_dispatch import dispatch_pipeline_trigger logger = logging.getLogger(__name__) @@ -29,17 +30,13 @@ def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Req def post( self, request: Request, org_name: str, pipeline_id: str, pipeline: Pipeline ) -> Response: - celery_app.send_task( - "scheduler.tasks.execute_pipeline_task", - args=[ - "", # workflow_id - org_name, # org_schema - "", # execution_action - "", # execution_id - pipeline_id, # pipepline_id - True, # with_logs - pipeline.pipeline_name, # name - ], + # Route through the transport flag instead of hardcoding Celery + # (see dispatch_pipeline_trigger). + dispatch_pipeline_trigger( + celery_app=celery_app, + org_id=org_name, + pipeline_id=pipeline_id, + pipeline_name=pipeline.pipeline_name, ) logger.info(f"Triggered {pipeline} by API") return Response({"message": f"Triggered {pipeline}"}, status=status.HTTP_200_OK) diff --git a/backend/pipeline_v2/pipeline_dispatch.py b/backend/pipeline_v2/pipeline_dispatch.py new file mode 100644 index 0000000000..c8d8002d09 --- /dev/null +++ b/backend/pipeline_v2/pipeline_dispatch.py @@ -0,0 +1,75 @@ +"""Transport-routed dispatch for the API-triggered pipeline execution trigger. + +Routes ``scheduler.tasks.execute_pipeline_task`` through the same +:func:`resolve_transport` flag as the rest of the execution path: the PG queue +when enabled for this pipeline/org, else Celery. **Fail-closed** — with the +master gate off (the production default) it resolves to Celery, behaving exactly +like the prior unconditional ``send_task`` (zero regression). The +scheduled-pipeline path already enqueues this task onto the PG ``scheduler`` +queue (``pg_scheduler.dispatch_due_schedules``, run by ``worker-pg-scheduler``); +this brings the API-trigger view to parity. + +The same ``args`` are sent on both paths; on PG they're JSON-normalized by +``enqueue_task`` (UUIDs/datetimes → str), a no-op for these string/bool values, +so the consumer sees the same payload. +""" + +from __future__ import annotations + +import logging +from typing import Any +from uuid import UUID + +from pg_queue.producer import enqueue_task +from workflow_manager.workflow_v2.transport import resolve_transport + +from unstract.core.data_models import is_pg_transport + +logger = logging.getLogger(__name__) + +# The fired task + the PG queue a ``scheduler`` consumer polls. Mirrors +# ``pg_scheduler.PIPELINE_TRIGGER_TASK`` / ``SCHEDULER_QUEUE_NAME`` — kept as local +# constants so the backend doesn't import the workers package. +PIPELINE_TRIGGER_TASK = "scheduler.tasks.execute_pipeline_task" +SCHEDULER_QUEUE = "scheduler" + + +def dispatch_pipeline_trigger( + *, + celery_app: Any, + org_id: str | UUID, + pipeline_id: str | UUID, + pipeline_name: str, +) -> None: + """Dispatch the pipeline-trigger task on the resolved transport. + + The positional args match ``execute_pipeline_task``'s signature on **both** + paths: ``(workflow_id, org_schema, execution_action, execution_id, + pipeline_id, with_logs, name)``. ``org_id`` / ``pipeline_id`` accept ``UUID`` + (what ``resolve_transport`` takes) and are str-coerced into the task args. + """ + args = ["", str(org_id), "", "", str(pipeline_id), True, pipeline_name] + # No execution exists yet (it's created inside the task), so the trigger + # buckets the flag by pipeline_id as the sticky entity. + transport = resolve_transport( + execution_id=pipeline_id, + organization_id=org_id, + pipeline_id=pipeline_id, + ) + # Use the shared is_pg_transport() — the single source for "what counts as PG + # transport" — rather than opening a second comparison site. + if is_pg_transport(transport): + msg_id = enqueue_task( + task_name=PIPELINE_TRIGGER_TASK, + queue=SCHEDULER_QUEUE, + args=args, + org_id=str(org_id or ""), + ) + logger.info( + "Pipeline %s trigger enqueued on PG scheduler queue (msg_id=%s)", + pipeline_id, + msg_id, + ) + else: + celery_app.send_task(PIPELINE_TRIGGER_TASK, args=args) + logger.info("Pipeline %s trigger dispatched on Celery", pipeline_id) diff --git a/backend/pipeline_v2/tests/__init__.py b/backend/pipeline_v2/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/pipeline_v2/tests/test_pipeline_dispatch.py b/backend/pipeline_v2/tests/test_pipeline_dispatch.py new file mode 100644 index 0000000000..ac8bd63e0a --- /dev/null +++ b/backend/pipeline_v2/tests/test_pipeline_dispatch.py @@ -0,0 +1,109 @@ +"""Unit tests for the API-triggered pipeline-trigger transport routing (UN-3616). + +`resolve_transport` + `enqueue_task` are patched on the module, so no Flipt / DB +is needed — these pin the routing contract: PG when the flag resolves PG, Celery +otherwise (fail-closed), with byte-identical args on both paths. +""" + +from __future__ import annotations + +import uuid +from unittest.mock import MagicMock, patch + +import pytest + +import pipeline_v2.pipeline_dispatch as pd + +_EXPECTED_ARGS = ["", "org_x", "", "", "pid-1", True, "My Pipeline"] + + +def _dispatch(celery_app): + return pd.dispatch_pipeline_trigger( + celery_app=celery_app, + org_id="org_x", + pipeline_id="pid-1", + pipeline_name="My Pipeline", + ) + + +class TestDispatchPipelineTrigger: + def test_routes_to_pg_when_flag_resolves_pg(self): + celery = MagicMock() + with ( + patch.object(pd, "resolve_transport", return_value="pg_queue"), + patch.object(pd, "enqueue_task", return_value=42) as enqueue, + ): + _dispatch(celery) + enqueue.assert_called_once() + kwargs = enqueue.call_args.kwargs + assert kwargs["task_name"] == "scheduler.tasks.execute_pipeline_task" + assert kwargs["queue"] == "scheduler" + assert kwargs["args"] == _EXPECTED_ARGS + assert kwargs["org_id"] == "org_x" + celery.send_task.assert_not_called() + + def test_pg_enqueue_failure_propagates_with_no_celery_fallback(self): + # The dispatcher has no try/except → a PG enqueue failure must surface + # (no silent Celery fallback, which would risk a double-dispatch). + celery = MagicMock() + with ( + patch.object(pd, "resolve_transport", return_value="pg_queue"), + patch.object(pd, "enqueue_task", side_effect=RuntimeError("pg down")), + ): + with pytest.raises(RuntimeError, match="pg down"): + _dispatch(celery) + celery.send_task.assert_not_called() + + def test_uuid_pipeline_id_is_coerced_to_str_in_args(self): + # resolve_transport accepts a UUID, but the task args must carry strings. + pid = uuid.UUID("b1f16024-45f2-4e39-8756-d40e24148e30") + celery = MagicMock() + with ( + patch.object(pd, "resolve_transport", return_value="pg_queue") as resolve, + patch.object(pd, "enqueue_task", return_value=1) as enqueue, + ): + pd.dispatch_pipeline_trigger( + celery_app=celery, org_id="org_x", pipeline_id=pid, pipeline_name="P" + ) + assert enqueue.call_args.kwargs["args"][4] == str(pid) + # The raw UUID is passed to resolve_transport as the sticky entity. + assert resolve.call_args.kwargs["execution_id"] == pid + + def test_routes_to_celery_when_flag_resolves_celery(self): + celery = MagicMock() + with ( + patch.object(pd, "resolve_transport", return_value="celery"), + patch.object(pd, "enqueue_task") as enqueue, + ): + _dispatch(celery) + celery.send_task.assert_called_once_with( + "scheduler.tasks.execute_pipeline_task", args=_EXPECTED_ARGS + ) + enqueue.assert_not_called() + + def test_resolve_transport_called_with_pipeline_as_entity(self): + celery = MagicMock() + with ( + patch.object(pd, "resolve_transport", return_value="celery") as resolve, + patch.object(pd, "enqueue_task"), + ): + _dispatch(celery) + kwargs = resolve.call_args.kwargs + assert kwargs["execution_id"] == "pid-1" # buckets by pipeline_id + assert kwargs["organization_id"] == "org_x" + assert kwargs["pipeline_id"] == "pid-1" + + def test_args_identical_on_both_paths(self): + # The consumer must behave the same regardless of transport — assert the + # PG-path args equal the Celery-path args. + celery = MagicMock() + with patch.object(pd, "resolve_transport", return_value="celery"): + _dispatch(celery) + celery_args = celery.send_task.call_args.kwargs["args"] + celery2 = MagicMock() + with ( + patch.object(pd, "resolve_transport", return_value="pg_queue"), + patch.object(pd, "enqueue_task", return_value=1) as enqueue, + ): + _dispatch(celery2) + assert enqueue.call_args.kwargs["args"] == celery_args From edc78bb34702793672a381513a1b5de3bae5bf0a Mon Sep 17 00:00:00 2001 From: muhammad-ali-e <117142933+muhammad-ali-e@users.noreply.github.com> Date: Wed, 24 Jun 2026 04:22:18 +0000 Subject: [PATCH 39/44] Commit uv.lock changes --- uv.lock | 2222 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 1120 insertions(+), 1102 deletions(-) diff --git a/uv.lock b/uv.lock index 1beb7a74de..8c1258f524 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 1 requires-python = "==3.12.*" [[package]] @@ -14,9 +14,9 @@ dependencies = [ { name = "azure-storage-blob" }, { name = "fsspec" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b4/1e/6d5146676044247af566fa5843b335b1a647e6446070cec9c8b61c31b369/adlfs-2024.7.0.tar.gz", hash = "sha256:106995b91f0eb5e775bcd5957d180d9a14faef3271a063b1f65c66fd5ab05ddf", size = 48588, upload-time = "2024-07-22T12:10:33.849Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/1e/6d5146676044247af566fa5843b335b1a647e6446070cec9c8b61c31b369/adlfs-2024.7.0.tar.gz", hash = "sha256:106995b91f0eb5e775bcd5957d180d9a14faef3271a063b1f65c66fd5ab05ddf", size = 48588 } wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/51/a71c457bd0bc8af3e522b6999ff300852c7c446e384fd9904b0794f875df/adlfs-2024.7.0-py3-none-any.whl", hash = "sha256:2005c8e124fda3948f2a6abb2dbebb2c936d2d821acaca6afd61932edfa9bc07", size = 41349, upload-time = "2024-07-22T12:10:32.226Z" }, + { url = "https://files.pythonhosted.org/packages/6f/51/a71c457bd0bc8af3e522b6999ff300852c7c446e384fd9904b0794f875df/adlfs-2024.7.0-py3-none-any.whl", hash = "sha256:2005c8e124fda3948f2a6abb2dbebb2c936d2d821acaca6afd61932edfa9bc07", size = 41349 }, ] [[package]] @@ -29,9 +29,9 @@ dependencies = [ { name = "botocore" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ea/17/2f6305cc52976dea8156b56badc3602f162f86693a6cc8badc20d2c5cfe6/aiobotocore-2.13.3.tar.gz", hash = "sha256:ac5620f93cc3e7c2aef7c67ba2bb74035ff8d49ee2325821daed13b3dd82a473", size = 106736, upload-time = "2024-08-22T20:42:13.086Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/17/2f6305cc52976dea8156b56badc3602f162f86693a6cc8badc20d2c5cfe6/aiobotocore-2.13.3.tar.gz", hash = "sha256:ac5620f93cc3e7c2aef7c67ba2bb74035ff8d49ee2325821daed13b3dd82a473", size = 106736 } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/98/1c1a1ba5d74f87cbabbf207d36a98551b082b8b187dc35f7808b43f4cf1a/aiobotocore-2.13.3-py3-none-any.whl", hash = "sha256:1272f765fd9414e1a68f8add71978367db94e17e36c3bf629cf1153eb5141fb9", size = 77194, upload-time = "2024-08-22T20:42:09.784Z" }, + { url = "https://files.pythonhosted.org/packages/81/98/1c1a1ba5d74f87cbabbf207d36a98551b082b8b187dc35f7808b43f4cf1a/aiobotocore-2.13.3-py3-none-any.whl", hash = "sha256:1272f765fd9414e1a68f8add71978367db94e17e36c3bf629cf1153eb5141fb9", size = 77194 }, ] [package.optional-dependencies] @@ -43,9 +43,9 @@ boto3 = [ name = "aiohappyeyeballs" version = "2.6.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } +sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760 } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, + { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265 }, ] [[package]] @@ -62,35 +62,35 @@ dependencies = [ { name = "typing-extensions" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ee/ab/93ce242f899b68c51b0578c027aafa791ab3614cb9345fa5d37b5f5c8e3e/aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b", size = 7940674, upload-time = "2026-06-01T19:41:02.763Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/97/2b6889bfb6b6847520d50d95eb8c4307a45e28aaca39faf4a9454b3d1b2f/aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e", size = 750194, upload-time = "2026-06-01T19:37:48.164Z" }, - { url = "https://files.pythonhosted.org/packages/21/e2/62634b7fff918ed98c3c6b2f0e70d520f7f28846cb412d451b04354c6459/aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c", size = 506966, upload-time = "2026-06-01T19:37:50.014Z" }, - { url = "https://files.pythonhosted.org/packages/dd/fb/5ce075150828c797a5106f1c2fb26034e709d4289b9d2bf8b07f1e59fac6/aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff", size = 507527, upload-time = "2026-06-01T19:37:51.96Z" }, - { url = "https://files.pythonhosted.org/packages/01/d5/405a0ae4e6b081754a3609c1c97c63a950e000a2def16046f1e736933a0e/aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108", size = 1762420, upload-time = "2026-06-01T19:37:53.839Z" }, - { url = "https://files.pythonhosted.org/packages/ae/1d/e05a7c896b15a6bc6fb8fc5319eb437861c2c49c34559ef928add6590315/aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a", size = 1733672, upload-time = "2026-06-01T19:37:55.791Z" }, - { url = "https://files.pythonhosted.org/packages/cc/22/a72f7c459e195fa41bf4f7abd1f925b91fe91f8097e51c654229ba144a33/aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500", size = 1805064, upload-time = "2026-06-01T19:37:57.931Z" }, - { url = "https://files.pythonhosted.org/packages/80/50/e85bdaba0be59ca4838005ebfef4048fcdd5f35a02b07057a9a123394440/aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955", size = 1902125, upload-time = "2026-06-01T19:38:00.225Z" }, - { url = "https://files.pythonhosted.org/packages/19/d8/51de5c6b971c27bb1ef620293b8d1ca611ec78736b34b3f6ccf68e4c8785/aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2", size = 1783112, upload-time = "2026-06-01T19:38:02.641Z" }, - { url = "https://files.pythonhosted.org/packages/73/ae/b4402bfde77e43dfb1b6ccff83c7b7ab63ed06b50c4754f0c5423fb374fe/aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159", size = 1586356, upload-time = "2026-06-01T19:38:04.637Z" }, - { url = "https://files.pythonhosted.org/packages/bc/05/750a3265ca4dc54a460bd0cb1121a8f2ce9171fce4a135fb47ea7fd594d2/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02", size = 1723119, upload-time = "2026-06-01T19:38:06.713Z" }, - { url = "https://files.pythonhosted.org/packages/37/01/8c0812c50b3b1b1c37b323bf170d6be8847a8f234060485b7d1e71953f60/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd", size = 1757216, upload-time = "2026-06-01T19:38:08.736Z" }, - { url = "https://files.pythonhosted.org/packages/47/2a/50fb98028a26887cbe48dcc1df92a90825615bc73b5584301304090cded8/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef", size = 1770500, upload-time = "2026-06-01T19:38:11.111Z" }, - { url = "https://files.pythonhosted.org/packages/bd/32/0ffd598a2fa2b9a423daf242e700cfdabda35d6e602394ad9ae58972c1c7/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e", size = 1576224, upload-time = "2026-06-01T19:38:13.391Z" }, - { url = "https://files.pythonhosted.org/packages/0b/f9/b9fc381dd9b66afb33f2634c40e229d106467be0afcabe79648631ab6712/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae", size = 1794252, upload-time = "2026-06-01T19:38:15.498Z" }, - { url = "https://files.pythonhosted.org/packages/a8/fb/05d9214c975f23225a8cd5c439325e338c7c377b315480ef3871db51f54e/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066", size = 1760193, upload-time = "2026-06-01T19:38:17.624Z" }, - { url = "https://files.pythonhosted.org/packages/d9/4b/02992fc4fb9e1b6673ee3f888a8e587a6447afda1f6f4aca776c148c2876/aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430", size = 448650, upload-time = "2026-06-01T19:38:19.545Z" }, - { url = "https://files.pythonhosted.org/packages/39/e9/246532214c3abda518477cbaaf16d420295ad8effa5233844cbb38f299ab/aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a", size = 476145, upload-time = "2026-06-01T19:38:21.505Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c3/63f8c20090048915711598b0adf475b149216d736157961de06480a45b15/aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370", size = 444250, upload-time = "2026-06-01T19:38:24.027Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/ee/ab/93ce242f899b68c51b0578c027aafa791ab3614cb9345fa5d37b5f5c8e3e/aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b", size = 7940674 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/97/2b6889bfb6b6847520d50d95eb8c4307a45e28aaca39faf4a9454b3d1b2f/aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e", size = 750194 }, + { url = "https://files.pythonhosted.org/packages/21/e2/62634b7fff918ed98c3c6b2f0e70d520f7f28846cb412d451b04354c6459/aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c", size = 506966 }, + { url = "https://files.pythonhosted.org/packages/dd/fb/5ce075150828c797a5106f1c2fb26034e709d4289b9d2bf8b07f1e59fac6/aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff", size = 507527 }, + { url = "https://files.pythonhosted.org/packages/01/d5/405a0ae4e6b081754a3609c1c97c63a950e000a2def16046f1e736933a0e/aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108", size = 1762420 }, + { url = "https://files.pythonhosted.org/packages/ae/1d/e05a7c896b15a6bc6fb8fc5319eb437861c2c49c34559ef928add6590315/aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a", size = 1733672 }, + { url = "https://files.pythonhosted.org/packages/cc/22/a72f7c459e195fa41bf4f7abd1f925b91fe91f8097e51c654229ba144a33/aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500", size = 1805064 }, + { url = "https://files.pythonhosted.org/packages/80/50/e85bdaba0be59ca4838005ebfef4048fcdd5f35a02b07057a9a123394440/aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955", size = 1902125 }, + { url = "https://files.pythonhosted.org/packages/19/d8/51de5c6b971c27bb1ef620293b8d1ca611ec78736b34b3f6ccf68e4c8785/aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2", size = 1783112 }, + { url = "https://files.pythonhosted.org/packages/73/ae/b4402bfde77e43dfb1b6ccff83c7b7ab63ed06b50c4754f0c5423fb374fe/aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159", size = 1586356 }, + { url = "https://files.pythonhosted.org/packages/bc/05/750a3265ca4dc54a460bd0cb1121a8f2ce9171fce4a135fb47ea7fd594d2/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02", size = 1723119 }, + { url = "https://files.pythonhosted.org/packages/37/01/8c0812c50b3b1b1c37b323bf170d6be8847a8f234060485b7d1e71953f60/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd", size = 1757216 }, + { url = "https://files.pythonhosted.org/packages/47/2a/50fb98028a26887cbe48dcc1df92a90825615bc73b5584301304090cded8/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef", size = 1770500 }, + { url = "https://files.pythonhosted.org/packages/bd/32/0ffd598a2fa2b9a423daf242e700cfdabda35d6e602394ad9ae58972c1c7/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e", size = 1576224 }, + { url = "https://files.pythonhosted.org/packages/0b/f9/b9fc381dd9b66afb33f2634c40e229d106467be0afcabe79648631ab6712/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae", size = 1794252 }, + { url = "https://files.pythonhosted.org/packages/a8/fb/05d9214c975f23225a8cd5c439325e338c7c377b315480ef3871db51f54e/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066", size = 1760193 }, + { url = "https://files.pythonhosted.org/packages/d9/4b/02992fc4fb9e1b6673ee3f888a8e587a6447afda1f6f4aca776c148c2876/aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430", size = 448650 }, + { url = "https://files.pythonhosted.org/packages/39/e9/246532214c3abda518477cbaaf16d420295ad8effa5233844cbb38f299ab/aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a", size = 476145 }, + { url = "https://files.pythonhosted.org/packages/2b/c3/63f8c20090048915711598b0adf475b149216d736157961de06480a45b15/aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370", size = 444250 }, ] [[package]] name = "aioitertools" version = "0.13.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fd/3c/53c4a17a05fb9ea2313ee1777ff53f5e001aefd5cc85aa2f4c2d982e1e38/aioitertools-0.13.0.tar.gz", hash = "sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c", size = 19322, upload-time = "2025-11-06T22:17:07.609Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/3c/53c4a17a05fb9ea2313ee1777ff53f5e001aefd5cc85aa2f4c2d982e1e38/aioitertools-0.13.0.tar.gz", hash = "sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c", size = 19322 } wheels = [ - { url = "https://files.pythonhosted.org/packages/10/a1/510b0a7fadc6f43a6ce50152e69dbd86415240835868bb0bd9b5b88b1e06/aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be", size = 24182, upload-time = "2025-11-06T22:17:06.502Z" }, + { url = "https://files.pythonhosted.org/packages/10/a1/510b0a7fadc6f43a6ce50152e69dbd86415240835868bb0bd9b5b88b1e06/aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be", size = 24182 }, ] [[package]] @@ -101,18 +101,18 @@ dependencies = [ { name = "frozenlist" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007 } wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490 }, ] [[package]] name = "aiosqlite" version = "0.22.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/8a/64761f4005f17809769d23e518d915db74e6310474e733e3593cfc854ef1/aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650", size = 14821, upload-time = "2025-12-23T19:25:43.997Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/8a/64761f4005f17809769d23e518d915db74e6310474e733e3593cfc854ef1/aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650", size = 14821 } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/b7/e3bf5133d697a08128598c8d0abc5e16377b51465a33756de24fa7dee953/aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb", size = 17405, upload-time = "2025-12-23T19:25:42.139Z" }, + { url = "https://files.pythonhosted.org/packages/00/b7/e3bf5133d697a08128598c8d0abc5e16377b51465a33756de24fa7dee953/aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb", size = 17405 }, ] [[package]] @@ -122,27 +122,27 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "vine" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } +sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013 } wheels = [ - { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, + { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944 }, ] [[package]] name = "annotated-doc" version = "0.0.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288 } wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303 }, ] [[package]] name = "annotated-types" version = "0.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081 } wheels = [ - { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643 }, ] [[package]] @@ -153,18 +153,18 @@ dependencies = [ { name = "idna" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622 } wheels = [ - { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353 }, ] [[package]] name = "appdirs" version = "1.4.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/d8/05696357e0311f5b5c316d7b95f46c669dd9c15aaeecbb48c7d0aeb88c40/appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", size = 13470, upload-time = "2020-05-11T07:59:51.037Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/d8/05696357e0311f5b5c316d7b95f46c669dd9c15aaeecbb48c7d0aeb88c40/appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", size = 13470 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128", size = 9566, upload-time = "2020-05-11T07:59:49.499Z" }, + { url = "https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128", size = 9566 }, ] [[package]] @@ -174,9 +174,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "argon2-cffi-bindings" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657 }, ] [[package]] @@ -186,61 +186,61 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441 } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/57/96b8b9f93166147826da5f90376e784a10582dd39a393c99bb62cfcf52f0/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:aecba1723ae35330a008418a91ea6cfcedf6d31e5fbaa056a166462ff066d500", size = 54121, upload-time = "2025-07-30T10:01:50.815Z" }, - { url = "https://files.pythonhosted.org/packages/0a/08/a9bebdb2e0e602dde230bdde8021b29f71f7841bd54801bcfd514acb5dcf/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2630b6240b495dfab90aebe159ff784d08ea999aa4b0d17efa734055a07d2f44", size = 29177, upload-time = "2025-07-30T10:01:51.681Z" }, - { url = "https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0", size = 31090, upload-time = "2025-07-30T10:01:53.184Z" }, - { url = "https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6", size = 81246, upload-time = "2025-07-30T10:01:54.145Z" }, - { url = "https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a", size = 87126, upload-time = "2025-07-30T10:01:55.074Z" }, - { url = "https://files.pythonhosted.org/packages/72/70/7a2993a12b0ffa2a9271259b79cc616e2389ed1a4d93842fac5a1f923ffd/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d", size = 80343, upload-time = "2025-07-30T10:01:56.007Z" }, - { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777, upload-time = "2025-07-30T10:01:56.943Z" }, - { url = "https://files.pythonhosted.org/packages/74/cd/15777dfde1c29d96de7f18edf4cc94c385646852e7c7b0320aa91ccca583/argon2_cffi_bindings-25.1.0-cp39-abi3-win32.whl", hash = "sha256:473bcb5f82924b1becbb637b63303ec8d10e84c8d241119419897a26116515d2", size = 27180, upload-time = "2025-07-30T10:01:57.759Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c6/a759ece8f1829d1f162261226fbfd2c6832b3ff7657384045286d2afa384/argon2_cffi_bindings-25.1.0-cp39-abi3-win_amd64.whl", hash = "sha256:a98cd7d17e9f7ce244c0803cad3c23a7d379c301ba618a5fa76a67d116618b98", size = 31715, upload-time = "2025-07-30T10:01:58.56Z" }, - { url = "https://files.pythonhosted.org/packages/42/b9/f8d6fa329ab25128b7e98fd83a3cb34d9db5b059a9847eddb840a0af45dd/argon2_cffi_bindings-25.1.0-cp39-abi3-win_arm64.whl", hash = "sha256:b0fdbcf513833809c882823f98dc2f931cf659d9a1429616ac3adebb49f5db94", size = 27149, upload-time = "2025-07-30T10:01:59.329Z" }, + { url = "https://files.pythonhosted.org/packages/1d/57/96b8b9f93166147826da5f90376e784a10582dd39a393c99bb62cfcf52f0/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:aecba1723ae35330a008418a91ea6cfcedf6d31e5fbaa056a166462ff066d500", size = 54121 }, + { url = "https://files.pythonhosted.org/packages/0a/08/a9bebdb2e0e602dde230bdde8021b29f71f7841bd54801bcfd514acb5dcf/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2630b6240b495dfab90aebe159ff784d08ea999aa4b0d17efa734055a07d2f44", size = 29177 }, + { url = "https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0", size = 31090 }, + { url = "https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6", size = 81246 }, + { url = "https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a", size = 87126 }, + { url = "https://files.pythonhosted.org/packages/72/70/7a2993a12b0ffa2a9271259b79cc616e2389ed1a4d93842fac5a1f923ffd/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d", size = 80343 }, + { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777 }, + { url = "https://files.pythonhosted.org/packages/74/cd/15777dfde1c29d96de7f18edf4cc94c385646852e7c7b0320aa91ccca583/argon2_cffi_bindings-25.1.0-cp39-abi3-win32.whl", hash = "sha256:473bcb5f82924b1becbb637b63303ec8d10e84c8d241119419897a26116515d2", size = 27180 }, + { url = "https://files.pythonhosted.org/packages/e2/c6/a759ece8f1829d1f162261226fbfd2c6832b3ff7657384045286d2afa384/argon2_cffi_bindings-25.1.0-cp39-abi3-win_amd64.whl", hash = "sha256:a98cd7d17e9f7ce244c0803cad3c23a7d379c301ba618a5fa76a67d116618b98", size = 31715 }, + { url = "https://files.pythonhosted.org/packages/42/b9/f8d6fa329ab25128b7e98fd83a3cb34d9db5b059a9847eddb840a0af45dd/argon2_cffi_bindings-25.1.0-cp39-abi3-win_arm64.whl", hash = "sha256:b0fdbcf513833809c882823f98dc2f931cf659d9a1429616ac3adebb49f5db94", size = 27149 }, ] [[package]] name = "asgiref" version = "3.11.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/63/40/f03da1264ae8f7cfdbf9146542e5e7e8100a4c66ab48e791df9a03d3f6c0/asgiref-3.11.1.tar.gz", hash = "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", size = 38550, upload-time = "2026-02-03T13:30:14.33Z" } +sdist = { url = "https://files.pythonhosted.org/packages/63/40/f03da1264ae8f7cfdbf9146542e5e7e8100a4c66ab48e791df9a03d3f6c0/asgiref-3.11.1.tar.gz", hash = "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", size = 38550 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/0a/a72d10ed65068e115044937873362e6e32fab1b7dce0046aeb224682c989/asgiref-3.11.1-py3-none-any.whl", hash = "sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133", size = 24345, upload-time = "2026-02-03T13:30:13.039Z" }, + { url = "https://files.pythonhosted.org/packages/5c/0a/a72d10ed65068e115044937873362e6e32fab1b7dce0046aeb224682c989/asgiref-3.11.1-py3-none-any.whl", hash = "sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133", size = 24345 }, ] [[package]] name = "asn1crypto" version = "1.5.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/cf/d547feed25b5244fcb9392e288ff9fdc3280b10260362fc45d37a798a6ee/asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", size = 121080, upload-time = "2022-03-15T14:46:52.889Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/cf/d547feed25b5244fcb9392e288ff9fdc3280b10260362fc45d37a798a6ee/asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", size = 121080 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" }, + { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045 }, ] [[package]] name = "asyncpg" version = "0.31.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/cc/d18065ce2380d80b1bcce927c24a2642efd38918e33fd724bc4bca904877/asyncpg-0.31.0.tar.gz", hash = "sha256:c989386c83940bfbd787180f2b1519415e2d3d6277a70d9d0f0145ac73500735", size = 993667, upload-time = "2025-11-24T23:27:00.812Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cc/d18065ce2380d80b1bcce927c24a2642efd38918e33fd724bc4bca904877/asyncpg-0.31.0.tar.gz", hash = "sha256:c989386c83940bfbd787180f2b1519415e2d3d6277a70d9d0f0145ac73500735", size = 993667 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/a6/59d0a146e61d20e18db7396583242e32e0f120693b67a8de43f1557033e2/asyncpg-0.31.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b44c31e1efc1c15188ef183f287c728e2046abb1d26af4d20858215d50d91fad", size = 662042, upload-time = "2025-11-24T23:25:49.578Z" }, - { url = "https://files.pythonhosted.org/packages/36/01/ffaa189dcb63a2471720615e60185c3f6327716fdc0fc04334436fbb7c65/asyncpg-0.31.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c89ccf741c067614c9b5fc7f1fc6f3b61ab05ae4aaa966e6fd6b93097c7d20d", size = 638504, upload-time = "2025-11-24T23:25:51.501Z" }, - { url = "https://files.pythonhosted.org/packages/9f/62/3f699ba45d8bd24c5d65392190d19656d74ff0185f42e19d0bbd973bb371/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:12b3b2e39dc5470abd5e98c8d3373e4b1d1234d9fbdedf538798b2c13c64460a", size = 3426241, upload-time = "2025-11-24T23:25:53.278Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d1/a867c2150f9c6e7af6462637f613ba67f78a314b00db220cd26ff559d532/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:aad7a33913fb8bcb5454313377cc330fbb19a0cd5faa7272407d8a0c4257b671", size = 3520321, upload-time = "2025-11-24T23:25:54.982Z" }, - { url = "https://files.pythonhosted.org/packages/7a/1a/cce4c3f246805ecd285a3591222a2611141f1669d002163abef999b60f98/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3df118d94f46d85b2e434fd62c84cb66d5834d5a890725fe625f498e72e4d5ec", size = 3316685, upload-time = "2025-11-24T23:25:57.43Z" }, - { url = "https://files.pythonhosted.org/packages/40/ae/0fc961179e78cc579e138fad6eb580448ecae64908f95b8cb8ee2f241f67/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd5b6efff3c17c3202d4b37189969acf8927438a238c6257f66be3c426beba20", size = 3471858, upload-time = "2025-11-24T23:25:59.636Z" }, - { url = "https://files.pythonhosted.org/packages/52/b2/b20e09670be031afa4cbfabd645caece7f85ec62d69c312239de568e058e/asyncpg-0.31.0-cp312-cp312-win32.whl", hash = "sha256:027eaa61361ec735926566f995d959ade4796f6a49d3bde17e5134b9964f9ba8", size = 527852, upload-time = "2025-11-24T23:26:01.084Z" }, - { url = "https://files.pythonhosted.org/packages/b5/f0/f2ed1de154e15b107dc692262395b3c17fc34eafe2a78fc2115931561730/asyncpg-0.31.0-cp312-cp312-win_amd64.whl", hash = "sha256:72d6bdcbc93d608a1158f17932de2321f68b1a967a13e014998db87a72ed3186", size = 597175, upload-time = "2025-11-24T23:26:02.564Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a6/59d0a146e61d20e18db7396583242e32e0f120693b67a8de43f1557033e2/asyncpg-0.31.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b44c31e1efc1c15188ef183f287c728e2046abb1d26af4d20858215d50d91fad", size = 662042 }, + { url = "https://files.pythonhosted.org/packages/36/01/ffaa189dcb63a2471720615e60185c3f6327716fdc0fc04334436fbb7c65/asyncpg-0.31.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c89ccf741c067614c9b5fc7f1fc6f3b61ab05ae4aaa966e6fd6b93097c7d20d", size = 638504 }, + { url = "https://files.pythonhosted.org/packages/9f/62/3f699ba45d8bd24c5d65392190d19656d74ff0185f42e19d0bbd973bb371/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:12b3b2e39dc5470abd5e98c8d3373e4b1d1234d9fbdedf538798b2c13c64460a", size = 3426241 }, + { url = "https://files.pythonhosted.org/packages/8c/d1/a867c2150f9c6e7af6462637f613ba67f78a314b00db220cd26ff559d532/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:aad7a33913fb8bcb5454313377cc330fbb19a0cd5faa7272407d8a0c4257b671", size = 3520321 }, + { url = "https://files.pythonhosted.org/packages/7a/1a/cce4c3f246805ecd285a3591222a2611141f1669d002163abef999b60f98/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3df118d94f46d85b2e434fd62c84cb66d5834d5a890725fe625f498e72e4d5ec", size = 3316685 }, + { url = "https://files.pythonhosted.org/packages/40/ae/0fc961179e78cc579e138fad6eb580448ecae64908f95b8cb8ee2f241f67/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd5b6efff3c17c3202d4b37189969acf8927438a238c6257f66be3c426beba20", size = 3471858 }, + { url = "https://files.pythonhosted.org/packages/52/b2/b20e09670be031afa4cbfabd645caece7f85ec62d69c312239de568e058e/asyncpg-0.31.0-cp312-cp312-win32.whl", hash = "sha256:027eaa61361ec735926566f995d959ade4796f6a49d3bde17e5134b9964f9ba8", size = 527852 }, + { url = "https://files.pythonhosted.org/packages/b5/f0/f2ed1de154e15b107dc692262395b3c17fc34eafe2a78fc2115931561730/asyncpg-0.31.0-cp312-cp312-win_amd64.whl", hash = "sha256:72d6bdcbc93d608a1158f17932de2321f68b1a967a13e014998db87a72ed3186", size = 597175 }, ] [[package]] name = "attrs" version = "26.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055 } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548 }, ] [[package]] @@ -251,9 +251,9 @@ dependencies = [ { name = "cryptography" }, { name = "joserfc" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/36/98/7d93f30d029643c0275dbc0bd6d5a6f670661ee6c9a94d93af7ab4887600/authlib-1.7.2.tar.gz", hash = "sha256:2cea25fefcd4e7173bdf1372c0afc265c8034b23a8cd5dcb6a9164b826c64231", size = 176511, upload-time = "2026-05-06T08:10:23.116Z" } +sdist = { url = "https://files.pythonhosted.org/packages/36/98/7d93f30d029643c0275dbc0bd6d5a6f670661ee6c9a94d93af7ab4887600/authlib-1.7.2.tar.gz", hash = "sha256:2cea25fefcd4e7173bdf1372c0afc265c8034b23a8cd5dcb6a9164b826c64231", size = 176511 } wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/95/adcb68e20c34162e9135f370d6e31737719c2b6f94bc953fe7ed1f10fe21/authlib-1.7.2-py2.py3-none-any.whl", hash = "sha256:3e1faedc9d87e7d56a164eca3ccb6ace0d61b94abe83e92242f8dc8bba9b4a9f", size = 259548, upload-time = "2026-05-06T08:10:21.436Z" }, + { url = "https://files.pythonhosted.org/packages/fb/95/adcb68e20c34162e9135f370d6e31737719c2b6f94bc953fe7ed1f10fe21/authlib-1.7.2-py2.py3-none-any.whl", hash = "sha256:3e1faedc9d87e7d56a164eca3ccb6ace0d61b94abe83e92242f8dc8bba9b4a9f", size = 259548 }, ] [[package]] @@ -264,9 +264,9 @@ dependencies = [ { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920, upload-time = "2026-05-07T23:30:56.357Z" }, + { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920 }, ] [[package]] @@ -278,9 +278,9 @@ dependencies = [ { name = "msal" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/22/ff/61369d06422b5ac48067215ff404841342651b14a89b46c8d8e1507c8f17/azure-datalake-store-0.0.53.tar.gz", hash = "sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393", size = 71430, upload-time = "2023-05-10T21:17:05.665Z" } +sdist = { url = "https://files.pythonhosted.org/packages/22/ff/61369d06422b5ac48067215ff404841342651b14a89b46c8d8e1507c8f17/azure-datalake-store-0.0.53.tar.gz", hash = "sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393", size = 71430 } wheels = [ - { url = "https://files.pythonhosted.org/packages/88/2a/75f56b14f115189155cf12e46b366ad1fe3357af5a1a7c09f7446662d617/azure_datalake_store-0.0.53-py2.py3-none-any.whl", hash = "sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b", size = 55308, upload-time = "2023-05-10T21:17:02.629Z" }, + { url = "https://files.pythonhosted.org/packages/88/2a/75f56b14f115189155cf12e46b366ad1fe3357af5a1a7c09f7446662d617/azure_datalake_store-0.0.53-py2.py3-none-any.whl", hash = "sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b", size = 55308 }, ] [[package]] @@ -294,9 +294,9 @@ dependencies = [ { name = "msal-extensions" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c5/0e/3a63efb48aa4a5ae2cfca61ee152fbcb668092134d3eb8bfda472dd5c617/azure_identity-1.25.3.tar.gz", hash = "sha256:ab23c0d63015f50b630ef6c6cf395e7262f439ce06e5d07a64e874c724f8d9e6", size = 286304, upload-time = "2026-03-13T01:12:20.892Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c5/0e/3a63efb48aa4a5ae2cfca61ee152fbcb668092134d3eb8bfda472dd5c617/azure_identity-1.25.3.tar.gz", hash = "sha256:ab23c0d63015f50b630ef6c6cf395e7262f439ce06e5d07a64e874c724f8d9e6", size = 286304 } wheels = [ - { url = "https://files.pythonhosted.org/packages/49/9a/417b3a533e01953a7c618884df2cb05a71e7b68bdbce4fbdb62349d2a2e8/azure_identity-1.25.3-py3-none-any.whl", hash = "sha256:f4d0b956a8146f30333e071374171f3cfa7bdb8073adb8c3814b65567aa7447c", size = 192138, upload-time = "2026-03-13T01:12:22.951Z" }, + { url = "https://files.pythonhosted.org/packages/49/9a/417b3a533e01953a7c618884df2cb05a71e7b68bdbce4fbdb62349d2a2e8/azure_identity-1.25.3-py3-none-any.whl", hash = "sha256:f4d0b956a8146f30333e071374171f3cfa7bdb8073adb8c3814b65567aa7447c", size = 192138 }, ] [[package]] @@ -309,9 +309,9 @@ dependencies = [ { name = "isodate" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/59/25/fdcf1e381922dbab8ba23d6fd78d397fe6cbac6b480310218834b7bc91fe/azure_storage_blob-12.29.0.tar.gz", hash = "sha256:2824ddd7ebc9056034ebc76b17971a38e9aa5835abb0d565b9700493f2a6c657", size = 611359, upload-time = "2026-05-15T03:34:59.865Z" } +sdist = { url = "https://files.pythonhosted.org/packages/59/25/fdcf1e381922dbab8ba23d6fd78d397fe6cbac6b480310218834b7bc91fe/azure_storage_blob-12.29.0.tar.gz", hash = "sha256:2824ddd7ebc9056034ebc76b17971a38e9aa5835abb0d565b9700493f2a6c657", size = 611359 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/2c/6ddee6a3e42d0236ba9259e4df7fa97fdc415ff0802b736c634baaf4b285/azure_storage_blob-12.29.0-py3-none-any.whl", hash = "sha256:ccf8a1bcd5e49df83ab85aab793b579e5ba2eeea2ad8900b2f62ca3a37dc391f", size = 434823, upload-time = "2026-05-15T03:35:01.837Z" }, + { url = "https://files.pythonhosted.org/packages/c2/2c/6ddee6a3e42d0236ba9259e4df7fa97fdc415ff0802b736c634baaf4b285/azure_storage_blob-12.29.0-py3-none-any.whl", hash = "sha256:ccf8a1bcd5e49df83ab85aab793b579e5ba2eeea2ad8900b2f62ca3a37dc391f", size = 434823 }, ] [[package]] @@ -326,65 +326,65 @@ dependencies = [ { name = "platformdirs" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bd/51/08fb68d23f4b0f6256fe85dc86e9576941550f890b079352fba719e07b39/banks-2.4.2.tar.gz", hash = "sha256:cda6013bd377ea7b701933578bfb9370fc21ad70bc13cedfc3f5cb2c034ca3dc", size = 188633, upload-time = "2026-04-27T12:15:22.021Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/51/08fb68d23f4b0f6256fe85dc86e9576941550f890b079352fba719e07b39/banks-2.4.2.tar.gz", hash = "sha256:cda6013bd377ea7b701933578bfb9370fc21ad70bc13cedfc3f5cb2c034ca3dc", size = 188633 } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/b6/8dc5477681b782e2f99de703e7a99828883364b9e03a60d3e2c47053d56a/banks-2.4.2-py3-none-any.whl", hash = "sha256:5fe407cc48c101f3e13d1cf732b83b8246003337612f13c0705d2e81f6faffb7", size = 35050, upload-time = "2026-04-27T12:15:20.785Z" }, + { url = "https://files.pythonhosted.org/packages/00/b6/8dc5477681b782e2f99de703e7a99828883364b9e03a60d3e2c47053d56a/banks-2.4.2-py3-none-any.whl", hash = "sha256:5fe407cc48c101f3e13d1cf732b83b8246003337612f13c0705d2e81f6faffb7", size = 35050 }, ] [[package]] name = "bcrypt" version = "5.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d4/36/3329e2518d70ad8e2e5817d5a4cac6bba05a47767ec416c7d020a965f408/bcrypt-5.0.0.tar.gz", hash = "sha256:f748f7c2d6fd375cc93d3fba7ef4a9e3a092421b8dbf34d8d4dc06be9492dfdd", size = 25386, upload-time = "2025-09-25T19:50:47.829Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/29/6237f151fbfe295fe3e074ecc6d44228faa1e842a81f6d34a02937ee1736/bcrypt-5.0.0-cp38-abi3-macosx_10_12_universal2.whl", hash = "sha256:fc746432b951e92b58317af8e0ca746efe93e66555f1b40888865ef5bf56446b", size = 494553, upload-time = "2025-09-25T19:49:49.006Z" }, - { url = "https://files.pythonhosted.org/packages/45/b6/4c1205dde5e464ea3bd88e8742e19f899c16fa8916fb8510a851fae985b5/bcrypt-5.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c2388ca94ffee269b6038d48747f4ce8df0ffbea43f31abfa18ac72f0218effb", size = 275009, upload-time = "2025-09-25T19:49:50.581Z" }, - { url = "https://files.pythonhosted.org/packages/3b/71/427945e6ead72ccffe77894b2655b695ccf14ae1866cd977e185d606dd2f/bcrypt-5.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:560ddb6ec730386e7b3b26b8b4c88197aaed924430e7b74666a586ac997249ef", size = 278029, upload-time = "2025-09-25T19:49:52.533Z" }, - { url = "https://files.pythonhosted.org/packages/17/72/c344825e3b83c5389a369c8a8e58ffe1480b8a699f46c127c34580c4666b/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d79e5c65dcc9af213594d6f7f1fa2c98ad3fc10431e7aa53c176b441943efbdd", size = 275907, upload-time = "2025-09-25T19:49:54.709Z" }, - { url = "https://files.pythonhosted.org/packages/0b/7e/d4e47d2df1641a36d1212e5c0514f5291e1a956a7749f1e595c07a972038/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2b732e7d388fa22d48920baa267ba5d97cca38070b69c0e2d37087b381c681fd", size = 296500, upload-time = "2025-09-25T19:49:56.013Z" }, - { url = "https://files.pythonhosted.org/packages/0f/c3/0ae57a68be2039287ec28bc463b82e4b8dc23f9d12c0be331f4782e19108/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0c8e093ea2532601a6f686edbc2c6b2ec24131ff5c52f7610dd64fa4553b5464", size = 278412, upload-time = "2025-09-25T19:49:57.356Z" }, - { url = "https://files.pythonhosted.org/packages/45/2b/77424511adb11e6a99e3a00dcc7745034bee89036ad7d7e255a7e47be7d8/bcrypt-5.0.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5b1589f4839a0899c146e8892efe320c0fa096568abd9b95593efac50a87cb75", size = 275486, upload-time = "2025-09-25T19:49:59.116Z" }, - { url = "https://files.pythonhosted.org/packages/43/0a/405c753f6158e0f3f14b00b462d8bca31296f7ecfc8fc8bc7919c0c7d73a/bcrypt-5.0.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:89042e61b5e808b67daf24a434d89bab164d4de1746b37a8d173b6b14f3db9ff", size = 277940, upload-time = "2025-09-25T19:50:00.869Z" }, - { url = "https://files.pythonhosted.org/packages/62/83/b3efc285d4aadc1fa83db385ec64dcfa1707e890eb42f03b127d66ac1b7b/bcrypt-5.0.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:e3cf5b2560c7b5a142286f69bde914494b6d8f901aaa71e453078388a50881c4", size = 310776, upload-time = "2025-09-25T19:50:02.393Z" }, - { url = "https://files.pythonhosted.org/packages/95/7d/47ee337dacecde6d234890fe929936cb03ebc4c3a7460854bbd9c97780b8/bcrypt-5.0.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f632fd56fc4e61564f78b46a2269153122db34988e78b6be8b32d28507b7eaeb", size = 312922, upload-time = "2025-09-25T19:50:04.232Z" }, - { url = "https://files.pythonhosted.org/packages/d6/3a/43d494dfb728f55f4e1cf8fd435d50c16a2d75493225b54c8d06122523c6/bcrypt-5.0.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:801cad5ccb6b87d1b430f183269b94c24f248dddbbc5c1f78b6ed231743e001c", size = 341367, upload-time = "2025-09-25T19:50:05.559Z" }, - { url = "https://files.pythonhosted.org/packages/55/ab/a0727a4547e383e2e22a630e0f908113db37904f58719dc48d4622139b5c/bcrypt-5.0.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3cf67a804fc66fc217e6914a5635000259fbbbb12e78a99488e4d5ba445a71eb", size = 359187, upload-time = "2025-09-25T19:50:06.916Z" }, - { url = "https://files.pythonhosted.org/packages/1b/bb/461f352fdca663524b4643d8b09e8435b4990f17fbf4fea6bc2a90aa0cc7/bcrypt-5.0.0-cp38-abi3-win32.whl", hash = "sha256:3abeb543874b2c0524ff40c57a4e14e5d3a66ff33fb423529c88f180fd756538", size = 153752, upload-time = "2025-09-25T19:50:08.515Z" }, - { url = "https://files.pythonhosted.org/packages/41/aa/4190e60921927b7056820291f56fc57d00d04757c8b316b2d3c0d1d6da2c/bcrypt-5.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:35a77ec55b541e5e583eb3436ffbbf53b0ffa1fa16ca6782279daf95d146dcd9", size = 150881, upload-time = "2025-09-25T19:50:09.742Z" }, - { url = "https://files.pythonhosted.org/packages/54/12/cd77221719d0b39ac0b55dbd39358db1cd1246e0282e104366ebbfb8266a/bcrypt-5.0.0-cp38-abi3-win_arm64.whl", hash = "sha256:cde08734f12c6a4e28dc6755cd11d3bdfea608d93d958fffbe95a7026ebe4980", size = 144931, upload-time = "2025-09-25T19:50:11.016Z" }, - { url = "https://files.pythonhosted.org/packages/5d/ba/2af136406e1c3839aea9ecadc2f6be2bcd1eff255bd451dd39bcf302c47a/bcrypt-5.0.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0c418ca99fd47e9c59a301744d63328f17798b5947b0f791e9af3c1c499c2d0a", size = 495313, upload-time = "2025-09-25T19:50:12.309Z" }, - { url = "https://files.pythonhosted.org/packages/ac/ee/2f4985dbad090ace5ad1f7dd8ff94477fe089b5fab2040bd784a3d5f187b/bcrypt-5.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb4e1500f6efdd402218ffe34d040a1196c072e07929b9820f363a1fd1f4191", size = 275290, upload-time = "2025-09-25T19:50:13.673Z" }, - { url = "https://files.pythonhosted.org/packages/e4/6e/b77ade812672d15cf50842e167eead80ac3514f3beacac8902915417f8b7/bcrypt-5.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7aeef54b60ceddb6f30ee3db090351ecf0d40ec6e2abf41430997407a46d2254", size = 278253, upload-time = "2025-09-25T19:50:15.089Z" }, - { url = "https://files.pythonhosted.org/packages/36/c4/ed00ed32f1040f7990dac7115f82273e3c03da1e1a1587a778d8cea496d8/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f0ce778135f60799d89c9693b9b398819d15f1921ba15fe719acb3178215a7db", size = 276084, upload-time = "2025-09-25T19:50:16.699Z" }, - { url = "https://files.pythonhosted.org/packages/e7/c4/fa6e16145e145e87f1fa351bbd54b429354fd72145cd3d4e0c5157cf4c70/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a71f70ee269671460b37a449f5ff26982a6f2ba493b3eabdd687b4bf35f875ac", size = 297185, upload-time = "2025-09-25T19:50:18.525Z" }, - { url = "https://files.pythonhosted.org/packages/24/b4/11f8a31d8b67cca3371e046db49baa7c0594d71eb40ac8121e2fc0888db0/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f8429e1c410b4073944f03bd778a9e066e7fad723564a52ff91841d278dfc822", size = 278656, upload-time = "2025-09-25T19:50:19.809Z" }, - { url = "https://files.pythonhosted.org/packages/ac/31/79f11865f8078e192847d2cb526e3fa27c200933c982c5b2869720fa5fce/bcrypt-5.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:edfcdcedd0d0f05850c52ba3127b1fce70b9f89e0fe5ff16517df7e81fa3cbb8", size = 275662, upload-time = "2025-09-25T19:50:21.567Z" }, - { url = "https://files.pythonhosted.org/packages/d4/8d/5e43d9584b3b3591a6f9b68f755a4da879a59712981ef5ad2a0ac1379f7a/bcrypt-5.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:611f0a17aa4a25a69362dcc299fda5c8a3d4f160e2abb3831041feb77393a14a", size = 278240, upload-time = "2025-09-25T19:50:23.305Z" }, - { url = "https://files.pythonhosted.org/packages/89/48/44590e3fc158620f680a978aafe8f87a4c4320da81ed11552f0323aa9a57/bcrypt-5.0.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:db99dca3b1fdc3db87d7c57eac0c82281242d1eabf19dcb8a6b10eb29a2e72d1", size = 311152, upload-time = "2025-09-25T19:50:24.597Z" }, - { url = "https://files.pythonhosted.org/packages/5f/85/e4fbfc46f14f47b0d20493669a625da5827d07e8a88ee460af6cd9768b44/bcrypt-5.0.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:5feebf85a9cefda32966d8171f5db7e3ba964b77fdfe31919622256f80f9cf42", size = 313284, upload-time = "2025-09-25T19:50:26.268Z" }, - { url = "https://files.pythonhosted.org/packages/25/ae/479f81d3f4594456a01ea2f05b132a519eff9ab5768a70430fa1132384b1/bcrypt-5.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3ca8a166b1140436e058298a34d88032ab62f15aae1c598580333dc21d27ef10", size = 341643, upload-time = "2025-09-25T19:50:28.02Z" }, - { url = "https://files.pythonhosted.org/packages/df/d2/36a086dee1473b14276cd6ea7f61aef3b2648710b5d7f1c9e032c29b859f/bcrypt-5.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:61afc381250c3182d9078551e3ac3a41da14154fbff647ddf52a769f588c4172", size = 359698, upload-time = "2025-09-25T19:50:31.347Z" }, - { url = "https://files.pythonhosted.org/packages/c0/f6/688d2cd64bfd0b14d805ddb8a565e11ca1fb0fd6817175d58b10052b6d88/bcrypt-5.0.0-cp39-abi3-win32.whl", hash = "sha256:64d7ce196203e468c457c37ec22390f1a61c85c6f0b8160fd752940ccfb3a683", size = 153725, upload-time = "2025-09-25T19:50:34.384Z" }, - { url = "https://files.pythonhosted.org/packages/9f/b9/9d9a641194a730bda138b3dfe53f584d61c58cd5230e37566e83ec2ffa0d/bcrypt-5.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:64ee8434b0da054d830fa8e89e1c8bf30061d539044a39524ff7dec90481e5c2", size = 150912, upload-time = "2025-09-25T19:50:35.69Z" }, - { url = "https://files.pythonhosted.org/packages/27/44/d2ef5e87509158ad2187f4dd0852df80695bb1ee0cfe0a684727b01a69e0/bcrypt-5.0.0-cp39-abi3-win_arm64.whl", hash = "sha256:f2347d3534e76bf50bca5500989d6c1d05ed64b440408057a37673282c654927", size = 144953, upload-time = "2025-09-25T19:50:37.32Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/d4/36/3329e2518d70ad8e2e5817d5a4cac6bba05a47767ec416c7d020a965f408/bcrypt-5.0.0.tar.gz", hash = "sha256:f748f7c2d6fd375cc93d3fba7ef4a9e3a092421b8dbf34d8d4dc06be9492dfdd", size = 25386 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/29/6237f151fbfe295fe3e074ecc6d44228faa1e842a81f6d34a02937ee1736/bcrypt-5.0.0-cp38-abi3-macosx_10_12_universal2.whl", hash = "sha256:fc746432b951e92b58317af8e0ca746efe93e66555f1b40888865ef5bf56446b", size = 494553 }, + { url = "https://files.pythonhosted.org/packages/45/b6/4c1205dde5e464ea3bd88e8742e19f899c16fa8916fb8510a851fae985b5/bcrypt-5.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c2388ca94ffee269b6038d48747f4ce8df0ffbea43f31abfa18ac72f0218effb", size = 275009 }, + { url = "https://files.pythonhosted.org/packages/3b/71/427945e6ead72ccffe77894b2655b695ccf14ae1866cd977e185d606dd2f/bcrypt-5.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:560ddb6ec730386e7b3b26b8b4c88197aaed924430e7b74666a586ac997249ef", size = 278029 }, + { url = "https://files.pythonhosted.org/packages/17/72/c344825e3b83c5389a369c8a8e58ffe1480b8a699f46c127c34580c4666b/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d79e5c65dcc9af213594d6f7f1fa2c98ad3fc10431e7aa53c176b441943efbdd", size = 275907 }, + { url = "https://files.pythonhosted.org/packages/0b/7e/d4e47d2df1641a36d1212e5c0514f5291e1a956a7749f1e595c07a972038/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2b732e7d388fa22d48920baa267ba5d97cca38070b69c0e2d37087b381c681fd", size = 296500 }, + { url = "https://files.pythonhosted.org/packages/0f/c3/0ae57a68be2039287ec28bc463b82e4b8dc23f9d12c0be331f4782e19108/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0c8e093ea2532601a6f686edbc2c6b2ec24131ff5c52f7610dd64fa4553b5464", size = 278412 }, + { url = "https://files.pythonhosted.org/packages/45/2b/77424511adb11e6a99e3a00dcc7745034bee89036ad7d7e255a7e47be7d8/bcrypt-5.0.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5b1589f4839a0899c146e8892efe320c0fa096568abd9b95593efac50a87cb75", size = 275486 }, + { url = "https://files.pythonhosted.org/packages/43/0a/405c753f6158e0f3f14b00b462d8bca31296f7ecfc8fc8bc7919c0c7d73a/bcrypt-5.0.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:89042e61b5e808b67daf24a434d89bab164d4de1746b37a8d173b6b14f3db9ff", size = 277940 }, + { url = "https://files.pythonhosted.org/packages/62/83/b3efc285d4aadc1fa83db385ec64dcfa1707e890eb42f03b127d66ac1b7b/bcrypt-5.0.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:e3cf5b2560c7b5a142286f69bde914494b6d8f901aaa71e453078388a50881c4", size = 310776 }, + { url = "https://files.pythonhosted.org/packages/95/7d/47ee337dacecde6d234890fe929936cb03ebc4c3a7460854bbd9c97780b8/bcrypt-5.0.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f632fd56fc4e61564f78b46a2269153122db34988e78b6be8b32d28507b7eaeb", size = 312922 }, + { url = "https://files.pythonhosted.org/packages/d6/3a/43d494dfb728f55f4e1cf8fd435d50c16a2d75493225b54c8d06122523c6/bcrypt-5.0.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:801cad5ccb6b87d1b430f183269b94c24f248dddbbc5c1f78b6ed231743e001c", size = 341367 }, + { url = "https://files.pythonhosted.org/packages/55/ab/a0727a4547e383e2e22a630e0f908113db37904f58719dc48d4622139b5c/bcrypt-5.0.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3cf67a804fc66fc217e6914a5635000259fbbbb12e78a99488e4d5ba445a71eb", size = 359187 }, + { url = "https://files.pythonhosted.org/packages/1b/bb/461f352fdca663524b4643d8b09e8435b4990f17fbf4fea6bc2a90aa0cc7/bcrypt-5.0.0-cp38-abi3-win32.whl", hash = "sha256:3abeb543874b2c0524ff40c57a4e14e5d3a66ff33fb423529c88f180fd756538", size = 153752 }, + { url = "https://files.pythonhosted.org/packages/41/aa/4190e60921927b7056820291f56fc57d00d04757c8b316b2d3c0d1d6da2c/bcrypt-5.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:35a77ec55b541e5e583eb3436ffbbf53b0ffa1fa16ca6782279daf95d146dcd9", size = 150881 }, + { url = "https://files.pythonhosted.org/packages/54/12/cd77221719d0b39ac0b55dbd39358db1cd1246e0282e104366ebbfb8266a/bcrypt-5.0.0-cp38-abi3-win_arm64.whl", hash = "sha256:cde08734f12c6a4e28dc6755cd11d3bdfea608d93d958fffbe95a7026ebe4980", size = 144931 }, + { url = "https://files.pythonhosted.org/packages/5d/ba/2af136406e1c3839aea9ecadc2f6be2bcd1eff255bd451dd39bcf302c47a/bcrypt-5.0.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0c418ca99fd47e9c59a301744d63328f17798b5947b0f791e9af3c1c499c2d0a", size = 495313 }, + { url = "https://files.pythonhosted.org/packages/ac/ee/2f4985dbad090ace5ad1f7dd8ff94477fe089b5fab2040bd784a3d5f187b/bcrypt-5.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb4e1500f6efdd402218ffe34d040a1196c072e07929b9820f363a1fd1f4191", size = 275290 }, + { url = "https://files.pythonhosted.org/packages/e4/6e/b77ade812672d15cf50842e167eead80ac3514f3beacac8902915417f8b7/bcrypt-5.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7aeef54b60ceddb6f30ee3db090351ecf0d40ec6e2abf41430997407a46d2254", size = 278253 }, + { url = "https://files.pythonhosted.org/packages/36/c4/ed00ed32f1040f7990dac7115f82273e3c03da1e1a1587a778d8cea496d8/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f0ce778135f60799d89c9693b9b398819d15f1921ba15fe719acb3178215a7db", size = 276084 }, + { url = "https://files.pythonhosted.org/packages/e7/c4/fa6e16145e145e87f1fa351bbd54b429354fd72145cd3d4e0c5157cf4c70/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a71f70ee269671460b37a449f5ff26982a6f2ba493b3eabdd687b4bf35f875ac", size = 297185 }, + { url = "https://files.pythonhosted.org/packages/24/b4/11f8a31d8b67cca3371e046db49baa7c0594d71eb40ac8121e2fc0888db0/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f8429e1c410b4073944f03bd778a9e066e7fad723564a52ff91841d278dfc822", size = 278656 }, + { url = "https://files.pythonhosted.org/packages/ac/31/79f11865f8078e192847d2cb526e3fa27c200933c982c5b2869720fa5fce/bcrypt-5.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:edfcdcedd0d0f05850c52ba3127b1fce70b9f89e0fe5ff16517df7e81fa3cbb8", size = 275662 }, + { url = "https://files.pythonhosted.org/packages/d4/8d/5e43d9584b3b3591a6f9b68f755a4da879a59712981ef5ad2a0ac1379f7a/bcrypt-5.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:611f0a17aa4a25a69362dcc299fda5c8a3d4f160e2abb3831041feb77393a14a", size = 278240 }, + { url = "https://files.pythonhosted.org/packages/89/48/44590e3fc158620f680a978aafe8f87a4c4320da81ed11552f0323aa9a57/bcrypt-5.0.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:db99dca3b1fdc3db87d7c57eac0c82281242d1eabf19dcb8a6b10eb29a2e72d1", size = 311152 }, + { url = "https://files.pythonhosted.org/packages/5f/85/e4fbfc46f14f47b0d20493669a625da5827d07e8a88ee460af6cd9768b44/bcrypt-5.0.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:5feebf85a9cefda32966d8171f5db7e3ba964b77fdfe31919622256f80f9cf42", size = 313284 }, + { url = "https://files.pythonhosted.org/packages/25/ae/479f81d3f4594456a01ea2f05b132a519eff9ab5768a70430fa1132384b1/bcrypt-5.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3ca8a166b1140436e058298a34d88032ab62f15aae1c598580333dc21d27ef10", size = 341643 }, + { url = "https://files.pythonhosted.org/packages/df/d2/36a086dee1473b14276cd6ea7f61aef3b2648710b5d7f1c9e032c29b859f/bcrypt-5.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:61afc381250c3182d9078551e3ac3a41da14154fbff647ddf52a769f588c4172", size = 359698 }, + { url = "https://files.pythonhosted.org/packages/c0/f6/688d2cd64bfd0b14d805ddb8a565e11ca1fb0fd6817175d58b10052b6d88/bcrypt-5.0.0-cp39-abi3-win32.whl", hash = "sha256:64d7ce196203e468c457c37ec22390f1a61c85c6f0b8160fd752940ccfb3a683", size = 153725 }, + { url = "https://files.pythonhosted.org/packages/9f/b9/9d9a641194a730bda138b3dfe53f584d61c58cd5230e37566e83ec2ffa0d/bcrypt-5.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:64ee8434b0da054d830fa8e89e1c8bf30061d539044a39524ff7dec90481e5c2", size = 150912 }, + { url = "https://files.pythonhosted.org/packages/27/44/d2ef5e87509158ad2187f4dd0852df80695bb1ee0cfe0a684727b01a69e0/bcrypt-5.0.0-cp39-abi3-win_arm64.whl", hash = "sha256:f2347d3534e76bf50bca5500989d6c1d05ed64b440408057a37673282c654927", size = 144953 }, ] [[package]] name = "bidict" version = "0.23.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/6e/026678aa5a830e07cd9498a05d3e7e650a4f56a42f267a53d22bcda1bdc9/bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71", size = 29093, upload-time = "2024-02-18T19:09:05.748Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/6e/026678aa5a830e07cd9498a05d3e7e650a4f56a42f267a53d22bcda1bdc9/bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71", size = 29093 } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/37/e8730c3587a65eb5645d4aba2d27aae48e8003614d6aaf15dda67f702f1f/bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5", size = 32764, upload-time = "2024-02-18T19:09:04.156Z" }, + { url = "https://files.pythonhosted.org/packages/99/37/e8730c3587a65eb5645d4aba2d27aae48e8003614d6aaf15dda67f702f1f/bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5", size = 32764 }, ] [[package]] name = "billiard" version = "4.2.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537 } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, + { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070 }, ] [[package]] @@ -396,9 +396,9 @@ dependencies = [ { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/74/c1/f80cfbe564c89cdb080cd9ac2079ce05a2fac869bf8fbc45929ed3190da9/boto3-1.34.162.tar.gz", hash = "sha256:873f8f5d2f6f85f1018cbb0535b03cceddc7b655b61f66a0a56995238804f41f", size = 108585, upload-time = "2024-08-15T19:25:38.714Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/c1/f80cfbe564c89cdb080cd9ac2079ce05a2fac869bf8fbc45929ed3190da9/boto3-1.34.162.tar.gz", hash = "sha256:873f8f5d2f6f85f1018cbb0535b03cceddc7b655b61f66a0a56995238804f41f", size = 108585 } wheels = [ - { url = "https://files.pythonhosted.org/packages/65/41/faa5081761be3bac3999f912996c14c4dc9d06eab86c234bd6441f54bd64/boto3-1.34.162-py3-none-any.whl", hash = "sha256:d6f6096bdab35a0c0deff469563b87d184a28df7689790f7fe7be98502b7c590", size = 139174, upload-time = "2024-08-15T19:25:35.384Z" }, + { url = "https://files.pythonhosted.org/packages/65/41/faa5081761be3bac3999f912996c14c4dc9d06eab86c234bd6441f54bd64/boto3-1.34.162-py3-none-any.whl", hash = "sha256:d6f6096bdab35a0c0deff469563b87d184a28df7689790f7fe7be98502b7c590", size = 139174 }, ] [[package]] @@ -410,9 +410,9 @@ dependencies = [ { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/22/de/17d672eac6725da49bd5832e3bd2f74c4d212311cd393fd56b59f51a4e86/botocore-1.34.162.tar.gz", hash = "sha256:adc23be4fb99ad31961236342b7cbf3c0bfc62532cd02852196032e8c0d682f3", size = 12676693, upload-time = "2024-08-15T19:25:25.162Z" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/17d672eac6725da49bd5832e3bd2f74c4d212311cd393fd56b59f51a4e86/botocore-1.34.162.tar.gz", hash = "sha256:adc23be4fb99ad31961236342b7cbf3c0bfc62532cd02852196032e8c0d682f3", size = 12676693 } wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/47/e35f788047c91110f48703a6254e5c84e33111b3291f7b57a653ca00accf/botocore-1.34.162-py3-none-any.whl", hash = "sha256:2d918b02db88d27a75b48275e6fb2506e9adaaddbec1ffa6a8a0898b34e769be", size = 12468049, upload-time = "2024-08-15T19:25:18.301Z" }, + { url = "https://files.pythonhosted.org/packages/bc/47/e35f788047c91110f48703a6254e5c84e33111b3291f7b57a653ca00accf/botocore-1.34.162-py3-none-any.whl", hash = "sha256:2d918b02db88d27a75b48275e6fb2506e9adaaddbec1ffa6a8a0898b34e769be", size = 12468049 }, ] [[package]] @@ -423,9 +423,9 @@ dependencies = [ { name = "boxsdk", extra = ["jwt"] }, { name = "fsspec" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/be/de/1c5e0faec600538f6a1d41c7ce7834cacddb2237923e30ddb225254b74b9/boxfs-0.2.1.tar.gz", hash = "sha256:c1889e12f53be3216b44f088237ac0f367a7a759a53b01b0c0edf2b3d694e50f", size = 9523, upload-time = "2023-08-23T19:24:35.233Z" } +sdist = { url = "https://files.pythonhosted.org/packages/be/de/1c5e0faec600538f6a1d41c7ce7834cacddb2237923e30ddb225254b74b9/boxfs-0.2.1.tar.gz", hash = "sha256:c1889e12f53be3216b44f088237ac0f367a7a759a53b01b0c0edf2b3d694e50f", size = 9523 } wheels = [ - { url = "https://files.pythonhosted.org/packages/86/bb/243d10169c8397051bad6bdd10beb2407fa490bfe01216f5fad09e066191/boxfs-0.2.1-py3-none-any.whl", hash = "sha256:ae796c30309bd5a02654fff9eddf1ed320356225568fad0e109e1942beaef72a", size = 9358, upload-time = "2023-08-23T19:24:34.066Z" }, + { url = "https://files.pythonhosted.org/packages/86/bb/243d10169c8397051bad6bdd10beb2407fa490bfe01216f5fad09e066191/boxfs-0.2.1-py3-none-any.whl", hash = "sha256:ae796c30309bd5a02654fff9eddf1ed320356225568fad0e109e1942beaef72a", size = 9358 }, ] [[package]] @@ -439,9 +439,9 @@ dependencies = [ { name = "requests-toolbelt" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bf/d7/c1a95bb602d7f90a85a68d8e6f11954e50c255110d39e2167c7796252622/boxsdk-3.14.0.tar.gz", hash = "sha256:7918b1929368724662474fffa417fa0457a523d089b8185260efbedd28c4f9b1", size = 232630, upload-time = "2025-04-09T15:07:15.181Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/d7/c1a95bb602d7f90a85a68d8e6f11954e50c255110d39e2167c7796252622/boxsdk-3.14.0.tar.gz", hash = "sha256:7918b1929368724662474fffa417fa0457a523d089b8185260efbedd28c4f9b1", size = 232630 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/5d/4e15511e0f4f2f9fbbf4646a8d0e138e5c53a3d428f1724e7dc3c8acf556/boxsdk-3.14.0-py2.py3-none-any.whl", hash = "sha256:0314e2f172b050e98489955f2e9001263de79c3dd751e6feee19f2195fdf7c01", size = 141329, upload-time = "2025-04-09T15:07:13.295Z" }, + { url = "https://files.pythonhosted.org/packages/4d/5d/4e15511e0f4f2f9fbbf4646a8d0e138e5c53a3d428f1724e7dc3c8acf556/boxsdk-3.14.0-py2.py3-none-any.whl", hash = "sha256:0314e2f172b050e98489955f2e9001263de79c3dd751e6feee19f2195fdf7c01", size = 141329 }, ] [package.optional-dependencies] @@ -454,9 +454,9 @@ jwt = [ name = "cachetools" version = "5.5.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/81/3747dad6b14fa2cf53fcf10548cf5aea6913e96fab41a3c198676f8948a5/cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", size = 28380, upload-time = "2025-02-20T21:01:19.524Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/81/3747dad6b14fa2cf53fcf10548cf5aea6913e96fab41a3c198676f8948a5/cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", size = 28380 } wheels = [ - { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, + { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080 }, ] [[package]] @@ -473,18 +473,18 @@ dependencies = [ { name = "python-dateutil" }, { name = "vine" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/7d/6c289f407d219ba36d8b384b42489ebdd0c84ce9c413875a8aae0c85f35b/celery-5.5.3.tar.gz", hash = "sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5", size = 1667144, upload-time = "2025-06-01T11:08:12.563Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/7d/6c289f407d219ba36d8b384b42489ebdd0c84ce9c413875a8aae0c85f35b/celery-5.5.3.tar.gz", hash = "sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5", size = 1667144 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/af/0dcccc7fdcdf170f9a1585e5e96b6fb0ba1749ef6be8c89a6202284759bd/celery-5.5.3-py3-none-any.whl", hash = "sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525", size = 438775, upload-time = "2025-06-01T11:08:09.94Z" }, + { url = "https://files.pythonhosted.org/packages/c9/af/0dcccc7fdcdf170f9a1585e5e96b6fb0ba1749ef6be8c89a6202284759bd/celery-5.5.3-py3-none-any.whl", hash = "sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525", size = 438775 }, ] [[package]] name = "certifi" version = "2026.4.22" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/ee/6caf7a40c36a1220410afe15a1cc64993a1f864871f698c0f93acb72842a/certifi-2026.4.22.tar.gz", hash = "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580", size = 137077, upload-time = "2026-04-22T11:26:11.191Z" } +sdist = { url = "https://files.pythonhosted.org/packages/25/ee/6caf7a40c36a1220410afe15a1cc64993a1f864871f698c0f93acb72842a/certifi-2026.4.22.tar.gz", hash = "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580", size = 137077 } wheels = [ - { url = "https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", size = 135707, upload-time = "2026-04-22T11:26:09.372Z" }, + { url = "https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", size = 135707 }, ] [[package]] @@ -494,63 +494,63 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, - { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, - { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, - { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, - { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, - { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, - { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, - { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, - { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, - { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, - { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271 }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048 }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529 }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097 }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983 }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519 }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572 }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963 }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361 }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932 }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557 }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762 }, ] [[package]] name = "cfgv" version = "3.5.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334 } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445 }, ] [[package]] name = "chardet" version = "6.0.0.post1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7f/42/fb9436c103a881a377e34b9f58d77b5f503461c702ff654ebe86151bcfe9/chardet-6.0.0.post1.tar.gz", hash = "sha256:6b78048c3c97c7b2ed1fbad7a18f76f5a6547f7d34dbab536cc13887c9a92fa4", size = 12521798, upload-time = "2026-02-22T15:09:17.925Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7f/42/fb9436c103a881a377e34b9f58d77b5f503461c702ff654ebe86151bcfe9/chardet-6.0.0.post1.tar.gz", hash = "sha256:6b78048c3c97c7b2ed1fbad7a18f76f5a6547f7d34dbab536cc13887c9a92fa4", size = 12521798 } wheels = [ - { url = "https://files.pythonhosted.org/packages/66/42/5de54f632c2de53cd3415b3703383d5fff43a94cbc0567ef362515261a21/chardet-6.0.0.post1-py3-none-any.whl", hash = "sha256:c894a36800549adf7bb5f2af47033281b75fdfcd2aa0f0243be0ad22a52e2dcb", size = 627245, upload-time = "2026-02-22T15:09:15.876Z" }, + { url = "https://files.pythonhosted.org/packages/66/42/5de54f632c2de53cd3415b3703383d5fff43a94cbc0567ef362515261a21/chardet-6.0.0.post1-py3-none-any.whl", hash = "sha256:c894a36800549adf7bb5f2af47033281b75fdfcd2aa0f0243be0ad22a52e2dcb", size = 627245 }, ] [[package]] name = "charset-normalizer" version = "3.4.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271 } wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, - { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, - { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, - { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, - { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, - { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, - { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, - { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, - { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, - { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, - { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, - { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, - { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, - { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328 }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061 }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031 }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239 }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589 }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733 }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652 }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229 }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552 }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806 }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316 }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274 }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468 }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460 }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330 }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828 }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958 }, ] [[package]] @@ -560,9 +560,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/23/e4/796662cd90cf80e3a363c99db2b88e0e394b988a575f60a17e16440cd011/click-8.4.0.tar.gz", hash = "sha256:638f1338fe1235c8f4e008e4a8a254fb5c5fbdcbb40ece3c9142ebb78e792973", size = 350843, upload-time = "2026-05-17T00:47:58.425Z" } +sdist = { url = "https://files.pythonhosted.org/packages/23/e4/796662cd90cf80e3a363c99db2b88e0e394b988a575f60a17e16440cd011/click-8.4.0.tar.gz", hash = "sha256:638f1338fe1235c8f4e008e4a8a254fb5c5fbdcbb40ece3c9142ebb78e792973", size = 350843 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/ae/8e92f8058baf87f6c7d86ee7e457668690195cc77efedb8d3797a06e3940/click-8.4.0-py3-none-any.whl", hash = "sha256:40c50b7c6c6adac2823d411041ec84f3f103f1b280d5e9ce0d7f998995832f81", size = 116147, upload-time = "2026-05-17T00:47:56.842Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ae/8e92f8058baf87f6c7d86ee7e457668690195cc77efedb8d3797a06e3940/click-8.4.0-py3-none-any.whl", hash = "sha256:40c50b7c6c6adac2823d411041ec84f3f103f1b280d5e9ce0d7f998995832f81", size = 116147 }, ] [[package]] @@ -572,9 +572,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } +sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089 } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, + { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631 }, ] [[package]] @@ -584,9 +584,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" }, + { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051 }, ] [[package]] @@ -597,49 +597,61 @@ dependencies = [ { name = "click" }, { name = "prompt-toolkit" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449 } wheels = [ - { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, + { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289 }, ] [[package]] name = "colorama" version = "0.4.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, ] [[package]] name = "coverage" version = "7.14.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/23/7f/d0720730a397a999ffc0fd3f5bebef347338e3a47b727da66fbb228e2ff2/coverage-7.14.0.tar.gz", hash = "sha256:057a6af2f160a85384cde4ab36f0d2777bae1057bae255f95413cdd382aa5c74", size = 919489, upload-time = "2026-05-10T18:02:31.397Z" } +sdist = { url = "https://files.pythonhosted.org/packages/23/7f/d0720730a397a999ffc0fd3f5bebef347338e3a47b727da66fbb228e2ff2/coverage-7.14.0.tar.gz", hash = "sha256:057a6af2f160a85384cde4ab36f0d2777bae1057bae255f95413cdd382aa5c74", size = 919489 } wheels = [ - { url = "https://files.pythonhosted.org/packages/09/1e/2f996b2c8415cbb6f54b0f5ec1ee850c96d7911961afb4fc05f4a89d8c58/coverage-7.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7ffd19fc8aed057fd686a17a4935eef5f9859d69208f96310e893e64b9b6ccf5", size = 219967, upload-time = "2026-05-10T18:00:13.756Z" }, - { url = "https://files.pythonhosted.org/packages/34/23/35c7aea1274aef7525bdd2dc92f710bdde6d11652239d71d1ec450067939/coverage-7.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:829994cfe1aeb773ca27bf246d4badc1e764893e3bfb98fff820fcecd1ca4662", size = 220329, upload-time = "2026-05-10T18:00:15.264Z" }, - { url = "https://files.pythonhosted.org/packages/75/cf/a8f4b43a16e194b0261257ad28ded5853ec052570afef4a84e1d81189f3b/coverage-7.14.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b4f07cf7edcb7ec39431a5074d7ea83b29a9f71fcfc494f0f40af4e65180420f", size = 251839, upload-time = "2026-05-10T18:00:17.16Z" }, - { url = "https://files.pythonhosted.org/packages/69/ff/6699e7b71e60d3049eb2bdcbc95ee3f35707b2b0e48f32e9e63d3ce30c08/coverage-7.14.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ca3d9cf2c32b521bd9518385608787fa86f38daf993695307531822c3430ed67", size = 254576, upload-time = "2026-05-10T18:00:18.829Z" }, - { url = "https://files.pythonhosted.org/packages/22/ec/c936d495fcd67f48f03a9c4ad3297ff80d1f222a5df3980f15b34c186c21/coverage-7.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92af52828e7f29d827346b0294e5a0853fa206db77db0395b282918d41e28db9", size = 255690, upload-time = "2026-05-10T18:00:20.648Z" }, - { url = "https://files.pythonhosted.org/packages/5c/42/5af63f636cc62a4a2b1b3ba9146f6ee6f53a35a50d5cefc54d5670f60999/coverage-7.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7b2bb6c9d7e769360d0f20a0f219603fd64f0c8f97de17ab25853261602be0fb", size = 257949, upload-time = "2026-05-10T18:00:22.28Z" }, - { url = "https://files.pythonhosted.org/packages/26/d3/a225317bd2012132a27e1176d51660b826f99bb975876463c44ea0d7ee5a/coverage-7.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1c9ed6ef99f88fb8c14aa8e2bf8eb0fe55fa2edfea68f8675d78741df1a5ac0e", size = 252242, upload-time = "2026-05-10T18:00:24.076Z" }, - { url = "https://files.pythonhosted.org/packages/f1/7f/9e65495298c3ea414742998539c37d048b5e81cc818fb1828cc6b51d10bf/coverage-7.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8231ade007f37959fbf58acc677f26b922c02eda6f0428ea307da0fd39681bf3", size = 253608, upload-time = "2026-05-10T18:00:25.588Z" }, - { url = "https://files.pythonhosted.org/packages/94/46/1522b524a35bdad22b2b8c4f9d32d0a104b524726ec380b2db68db1746f5/coverage-7.14.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d8b013632cc1ce1d09dbe4f32667b4d320ec2f54fc326ebeffcd0b0bcc2bb6c4", size = 251753, upload-time = "2026-05-10T18:00:27.104Z" }, - { url = "https://files.pythonhosted.org/packages/f3/e9/cdf00d38817742c541ade405e115a3f7bf36e6f2a8b99d4f209861b85a2d/coverage-7.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1733198802d71ec4c524f322e2867ee05c62e9e75df86bdca545407a221827d1", size = 255823, upload-time = "2026-05-10T18:00:29.038Z" }, - { url = "https://files.pythonhosted.org/packages/38/fc/5e7877cf5f902d08a17ff1c532511476d87e1bea355bd5028cb97f902e79/coverage-7.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:72a305291fa8ee01332f1aaf38b348ca34097f6aa0b0ef627eef2837e57bbba5", size = 251323, upload-time = "2026-05-10T18:00:30.647Z" }, - { url = "https://files.pythonhosted.org/packages/18/9d/50f05a72dff8487464fdd4178dda5daed642a060e60afb644e3d45123559/coverage-7.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fcaba850dd317c65423a9d63d88f9573c53b00354d6dd95724576cc98a131595", size = 253197, upload-time = "2026-05-10T18:00:32.211Z" }, - { url = "https://files.pythonhosted.org/packages/00/3f/6f61ffe6439df266c3cf60f5c99cfaa21103d0210d706a42fc6c30683ff8/coverage-7.14.0-cp312-cp312-win32.whl", hash = "sha256:5ac83957a80d0701310e96d8bec68cdcf4f90a7674b7d13f15a344315b41ab27", size = 222515, upload-time = "2026-05-10T18:00:33.717Z" }, - { url = "https://files.pythonhosted.org/packages/85/19/93853133df2cb371083285ef6a93982a0173e7a233b0f61373ba9fd30eb2/coverage-7.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:70390b0da32cb90b501953716302906e8bcce087cb283e70d8c97729f22e92b2", size = 223324, upload-time = "2026-05-10T18:00:35.172Z" }, - { url = "https://files.pythonhosted.org/packages/74/18/9f7fe62f659f24b7a82a0be56bf94c1bd0a89e0ae7ab4c668f6e82404294/coverage-7.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:91b993743d959b8be85b4abf9d5478216a69329c321efe5be0433c1a841d691d", size = 221944, upload-time = "2026-05-10T18:00:37.014Z" }, - { url = "https://files.pythonhosted.org/packages/61/e8/cb8e80d6f9f55b99588625062822bf946cf03ed06315df4bd8397f5632a1/coverage-7.14.0-py3-none-any.whl", hash = "sha256:8de5b61163aee3d05c8a2beab6f47913df7981dad1baf82c414d99158c286ab1", size = 211764, upload-time = "2026-05-10T18:02:29.538Z" }, + { url = "https://files.pythonhosted.org/packages/09/1e/2f996b2c8415cbb6f54b0f5ec1ee850c96d7911961afb4fc05f4a89d8c58/coverage-7.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7ffd19fc8aed057fd686a17a4935eef5f9859d69208f96310e893e64b9b6ccf5", size = 219967 }, + { url = "https://files.pythonhosted.org/packages/34/23/35c7aea1274aef7525bdd2dc92f710bdde6d11652239d71d1ec450067939/coverage-7.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:829994cfe1aeb773ca27bf246d4badc1e764893e3bfb98fff820fcecd1ca4662", size = 220329 }, + { url = "https://files.pythonhosted.org/packages/75/cf/a8f4b43a16e194b0261257ad28ded5853ec052570afef4a84e1d81189f3b/coverage-7.14.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b4f07cf7edcb7ec39431a5074d7ea83b29a9f71fcfc494f0f40af4e65180420f", size = 251839 }, + { url = "https://files.pythonhosted.org/packages/69/ff/6699e7b71e60d3049eb2bdcbc95ee3f35707b2b0e48f32e9e63d3ce30c08/coverage-7.14.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ca3d9cf2c32b521bd9518385608787fa86f38daf993695307531822c3430ed67", size = 254576 }, + { url = "https://files.pythonhosted.org/packages/22/ec/c936d495fcd67f48f03a9c4ad3297ff80d1f222a5df3980f15b34c186c21/coverage-7.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92af52828e7f29d827346b0294e5a0853fa206db77db0395b282918d41e28db9", size = 255690 }, + { url = "https://files.pythonhosted.org/packages/5c/42/5af63f636cc62a4a2b1b3ba9146f6ee6f53a35a50d5cefc54d5670f60999/coverage-7.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7b2bb6c9d7e769360d0f20a0f219603fd64f0c8f97de17ab25853261602be0fb", size = 257949 }, + { url = "https://files.pythonhosted.org/packages/26/d3/a225317bd2012132a27e1176d51660b826f99bb975876463c44ea0d7ee5a/coverage-7.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1c9ed6ef99f88fb8c14aa8e2bf8eb0fe55fa2edfea68f8675d78741df1a5ac0e", size = 252242 }, + { url = "https://files.pythonhosted.org/packages/f1/7f/9e65495298c3ea414742998539c37d048b5e81cc818fb1828cc6b51d10bf/coverage-7.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8231ade007f37959fbf58acc677f26b922c02eda6f0428ea307da0fd39681bf3", size = 253608 }, + { url = "https://files.pythonhosted.org/packages/94/46/1522b524a35bdad22b2b8c4f9d32d0a104b524726ec380b2db68db1746f5/coverage-7.14.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d8b013632cc1ce1d09dbe4f32667b4d320ec2f54fc326ebeffcd0b0bcc2bb6c4", size = 251753 }, + { url = "https://files.pythonhosted.org/packages/f3/e9/cdf00d38817742c541ade405e115a3f7bf36e6f2a8b99d4f209861b85a2d/coverage-7.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1733198802d71ec4c524f322e2867ee05c62e9e75df86bdca545407a221827d1", size = 255823 }, + { url = "https://files.pythonhosted.org/packages/38/fc/5e7877cf5f902d08a17ff1c532511476d87e1bea355bd5028cb97f902e79/coverage-7.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:72a305291fa8ee01332f1aaf38b348ca34097f6aa0b0ef627eef2837e57bbba5", size = 251323 }, + { url = "https://files.pythonhosted.org/packages/18/9d/50f05a72dff8487464fdd4178dda5daed642a060e60afb644e3d45123559/coverage-7.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fcaba850dd317c65423a9d63d88f9573c53b00354d6dd95724576cc98a131595", size = 253197 }, + { url = "https://files.pythonhosted.org/packages/00/3f/6f61ffe6439df266c3cf60f5c99cfaa21103d0210d706a42fc6c30683ff8/coverage-7.14.0-cp312-cp312-win32.whl", hash = "sha256:5ac83957a80d0701310e96d8bec68cdcf4f90a7674b7d13f15a344315b41ab27", size = 222515 }, + { url = "https://files.pythonhosted.org/packages/85/19/93853133df2cb371083285ef6a93982a0173e7a233b0f61373ba9fd30eb2/coverage-7.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:70390b0da32cb90b501953716302906e8bcce087cb283e70d8c97729f22e92b2", size = 223324 }, + { url = "https://files.pythonhosted.org/packages/74/18/9f7fe62f659f24b7a82a0be56bf94c1bd0a89e0ae7ab4c668f6e82404294/coverage-7.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:91b993743d959b8be85b4abf9d5478216a69329c321efe5be0433c1a841d691d", size = 221944 }, + { url = "https://files.pythonhosted.org/packages/61/e8/cb8e80d6f9f55b99588625062822bf946cf03ed06315df4bd8397f5632a1/coverage-7.14.0-py3-none-any.whl", hash = "sha256:8de5b61163aee3d05c8a2beab6f47913df7981dad1baf82c414d99158c286ab1", size = 211764 }, ] [[package]] name = "cron-descriptor" version = "1.4.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/24/a0/455f5a0181cf9a0d2e84d3a66c88de019dce5644ad9680825d1c8a403335/cron_descriptor-1.4.0.tar.gz", hash = "sha256:b6ff4e3a988d7ca04a4ab150248e9f166fb7a5c828a85090e75bcc25aa93b4dd", size = 29922, upload-time = "2023-05-19T07:46:16.992Z" } +sdist = { url = "https://files.pythonhosted.org/packages/24/a0/455f5a0181cf9a0d2e84d3a66c88de019dce5644ad9680825d1c8a403335/cron_descriptor-1.4.0.tar.gz", hash = "sha256:b6ff4e3a988d7ca04a4ab150248e9f166fb7a5c828a85090e75bcc25aa93b4dd", size = 29922 } + +[[package]] +name = "croniter" +version = "6.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/de/5832661ed55107b8a09af3f0a2e71e0957226a59eb1dcf0a445cce6daf20/croniter-6.2.2.tar.gz", hash = "sha256:ba60832a5ec8e12e51b8691c3309a113d1cf6526bdf1a48150ce8ec7a532d0ab", size = 113762 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/39/783980e78cb92c2d7bdb1fc7dbc86e94ccc6d58224d76a7f1f51b6c51e30/croniter-6.2.2-py3-none-any.whl", hash = "sha256:a5d17b1060974d36251ea4faf388233eca8acf0d09cbd92d35f4c4ac8f279960", size = 45422 }, +] [[package]] name = "cryptography" @@ -648,36 +660,36 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", size = 832984, upload-time = "2026-05-04T22:59:38.133Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", size = 8001587, upload-time = "2026-05-04T22:57:36.803Z" }, - { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", size = 4690433, upload-time = "2026-05-04T22:57:40.373Z" }, - { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", size = 4710620, upload-time = "2026-05-04T22:57:42.935Z" }, - { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", size = 4696283, upload-time = "2026-05-04T22:57:45.294Z" }, - { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", size = 5296573, upload-time = "2026-05-04T22:57:47.933Z" }, - { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", size = 4743677, upload-time = "2026-05-04T22:57:50.067Z" }, - { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", size = 4330808, upload-time = "2026-05-04T22:57:52.301Z" }, - { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", size = 4695941, upload-time = "2026-05-04T22:57:54.603Z" }, - { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", size = 5252579, upload-time = "2026-05-04T22:57:57.207Z" }, - { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", size = 4743326, upload-time = "2026-05-04T22:57:59.535Z" }, - { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", size = 4826672, upload-time = "2026-05-04T22:58:01.996Z" }, - { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", size = 4972574, upload-time = "2026-05-04T22:58:03.968Z" }, - { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", size = 3294868, upload-time = "2026-05-04T22:58:06.467Z" }, - { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", size = 3817107, upload-time = "2026-05-04T22:58:08.845Z" }, - { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", size = 7983482, upload-time = "2026-05-04T22:58:43.769Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", size = 4683266, upload-time = "2026-05-04T22:58:46.064Z" }, - { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", size = 4696228, upload-time = "2026-05-04T22:58:48.22Z" }, - { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", size = 4689097, upload-time = "2026-05-04T22:58:50.9Z" }, - { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", size = 5283582, upload-time = "2026-05-04T22:58:53.017Z" }, - { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", size = 4730479, upload-time = "2026-05-04T22:58:55.611Z" }, - { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", size = 4326481, upload-time = "2026-05-04T22:58:57.607Z" }, - { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", size = 4688713, upload-time = "2026-05-04T22:59:00.077Z" }, - { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", size = 5238165, upload-time = "2026-05-04T22:59:02.317Z" }, - { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", size = 4729947, upload-time = "2026-05-04T22:59:05.255Z" }, - { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", size = 4822059, upload-time = "2026-05-04T22:59:07.802Z" }, - { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", size = 4960575, upload-time = "2026-05-04T22:59:09.851Z" }, - { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", size = 3279117, upload-time = "2026-05-04T22:59:12.019Z" }, - { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", size = 3792100, upload-time = "2026-05-04T22:59:14.884Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", size = 832984 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", size = 8001587 }, + { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", size = 4690433 }, + { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", size = 4710620 }, + { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", size = 4696283 }, + { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", size = 5296573 }, + { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", size = 4743677 }, + { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", size = 4330808 }, + { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", size = 4695941 }, + { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", size = 5252579 }, + { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", size = 4743326 }, + { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", size = 4826672 }, + { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", size = 4972574 }, + { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", size = 3294868 }, + { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", size = 3817107 }, + { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", size = 7983482 }, + { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", size = 4683266 }, + { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", size = 4696228 }, + { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", size = 4689097 }, + { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", size = 5283582 }, + { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", size = 4730479 }, + { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", size = 4326481 }, + { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", size = 4688713 }, + { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", size = 5238165 }, + { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", size = 4729947 }, + { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", size = 4822059 }, + { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", size = 4960575 }, + { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", size = 3279117 }, + { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", size = 3792100 }, ] [[package]] @@ -688,9 +700,9 @@ dependencies = [ { name = "marshmallow" }, { name = "typing-inspect" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/64/a4/f71d9cf3a5ac257c993b5ca3f93df5f7fb395c725e7f1e6479d2514173c3/dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0", size = 32227, upload-time = "2024-06-09T16:20:19.103Z" } +sdist = { url = "https://files.pythonhosted.org/packages/64/a4/f71d9cf3a5ac257c993b5ca3f93df5f7fb395c725e7f1e6479d2514173c3/dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0", size = 32227 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, + { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686 }, ] [[package]] @@ -701,27 +713,27 @@ dependencies = [ { name = "mbstrdecoder" }, { name = "typepy", extra = ["datetime"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f1/6f/a801320bb388d965be9c370ec753cc33120e6cbe0069fa05644f05821975/dataproperty-1.1.1.tar.gz", hash = "sha256:a83af82a234edda5378a36fb092bc90dd554646c5e58202a310acf468ae81bc8", size = 42954, upload-time = "2026-05-09T10:33:42.212Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/6f/a801320bb388d965be9c370ec753cc33120e6cbe0069fa05644f05821975/dataproperty-1.1.1.tar.gz", hash = "sha256:a83af82a234edda5378a36fb092bc90dd554646c5e58202a310acf468ae81bc8", size = 42954 } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/41/eab7fe313820578b341a2a1d6aeeedd2c38ec1e3f3d51e57e2735b5beac0/dataproperty-1.1.1-py3-none-any.whl", hash = "sha256:cf026aa002dbd6c57c619ec6741ffd61ae7bf2f20481951d8af2dff44480340e", size = 27691, upload-time = "2026-05-09T10:33:40.468Z" }, + { url = "https://files.pythonhosted.org/packages/03/41/eab7fe313820578b341a2a1d6aeeedd2c38ec1e3f3d51e57e2735b5beac0/dataproperty-1.1.1-py3-none-any.whl", hash = "sha256:cf026aa002dbd6c57c619ec6741ffd61ae7bf2f20481951d8af2dff44480340e", size = 27691 }, ] [[package]] name = "decorator" version = "5.3.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084, upload-time = "2026-05-18T06:03:28.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084 } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, + { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365 }, ] [[package]] name = "defusedxml" version = "0.7.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520 } wheels = [ - { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604 }, ] [[package]] @@ -731,36 +743,36 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523 } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298 }, ] [[package]] name = "dirtyjson" version = "1.0.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/db/04/d24f6e645ad82ba0ef092fa17d9ef7a21953781663648a01c9371d9e8e98/dirtyjson-1.0.8.tar.gz", hash = "sha256:90ca4a18f3ff30ce849d100dcf4a003953c79d3a2348ef056f1d9c22231a25fd", size = 30782, upload-time = "2022-11-28T23:32:33.319Z" } +sdist = { url = "https://files.pythonhosted.org/packages/db/04/d24f6e645ad82ba0ef092fa17d9ef7a21953781663648a01c9371d9e8e98/dirtyjson-1.0.8.tar.gz", hash = "sha256:90ca4a18f3ff30ce849d100dcf4a003953c79d3a2348ef056f1d9c22231a25fd", size = 30782 } wheels = [ - { url = "https://files.pythonhosted.org/packages/68/69/1bcf70f81de1b4a9f21b3a62ec0c83bdff991c88d6cc2267d02408457e88/dirtyjson-1.0.8-py3-none-any.whl", hash = "sha256:125e27248435a58acace26d5c2c4c11a1c0de0a9c5124c5a94ba78e517d74f53", size = 25197, upload-time = "2022-11-28T23:32:31.219Z" }, + { url = "https://files.pythonhosted.org/packages/68/69/1bcf70f81de1b4a9f21b3a62ec0c83bdff991c88d6cc2267d02408457e88/dirtyjson-1.0.8-py3-none-any.whl", hash = "sha256:125e27248435a58acace26d5c2c4c11a1c0de0a9c5124c5a94ba78e517d74f53", size = 25197 }, ] [[package]] name = "distlib" version = "0.4.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605 } wheels = [ - { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047 }, ] [[package]] name = "distro" version = "1.9.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722 } wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277 }, ] [[package]] @@ -772,9 +784,9 @@ dependencies = [ { name = "sqlparse" }, { name = "tzdata", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/89/76/23ee9b9d2bd4119e930eb19164732b79c0a4f6259ca198209b0fe36551ea/Django-4.2.1.tar.gz", hash = "sha256:7efa6b1f781a6119a10ac94b4794ded90db8accbe7802281cd26f8664ffed59c", size = 10420051, upload-time = "2023-05-03T12:58:41.313Z" } +sdist = { url = "https://files.pythonhosted.org/packages/89/76/23ee9b9d2bd4119e930eb19164732b79c0a4f6259ca198209b0fe36551ea/Django-4.2.1.tar.gz", hash = "sha256:7efa6b1f781a6119a10ac94b4794ded90db8accbe7802281cd26f8664ffed59c", size = 10420051 } wheels = [ - { url = "https://files.pythonhosted.org/packages/12/13/78e8622180f101e95297965045ff1325ea7301c1b80f756debbeaa84c3be/Django-4.2.1-py3-none-any.whl", hash = "sha256:066b6debb5ac335458d2a713ed995570536c8b59a580005acb0732378d5eb1ee", size = 7988496, upload-time = "2023-05-03T12:58:27.208Z" }, + { url = "https://files.pythonhosted.org/packages/12/13/78e8622180f101e95297965045ff1325ea7301c1b80f756debbeaa84c3be/Django-4.2.1-py3-none-any.whl", hash = "sha256:066b6debb5ac335458d2a713ed995570536c8b59a580005acb0732378d5eb1ee", size = 7988496 }, ] [[package]] @@ -789,9 +801,9 @@ dependencies = [ { name = "python-crontab" }, { name = "tzdata" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0b/97/ca63898f76dd43fc91f4791b05dbbecb60dc99215f16b270e9b1e29af974/django-celery-beat-2.5.0.tar.gz", hash = "sha256:cd0a47f5958402f51ac0c715bc942ae33d7b50b4e48cba91bc3f2712be505df1", size = 159635, upload-time = "2023-03-14T10:02:10.9Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/97/ca63898f76dd43fc91f4791b05dbbecb60dc99215f16b270e9b1e29af974/django-celery-beat-2.5.0.tar.gz", hash = "sha256:cd0a47f5958402f51ac0c715bc942ae33d7b50b4e48cba91bc3f2712be505df1", size = 159635 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/92/fa53396870566276357bb81e3fece5b7f8a00f99c91689ff777c481d40e0/django_celery_beat-2.5.0-py3-none-any.whl", hash = "sha256:ae460faa5ea142fba0875409095d22f6bd7bcc7377889b85e8cab5c0dfb781fe", size = 97223, upload-time = "2023-03-14T10:02:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/c5/92/fa53396870566276357bb81e3fece5b7f8a00f99c91689ff777c481d40e0/django_celery_beat-2.5.0-py3-none-any.whl", hash = "sha256:ae460faa5ea142fba0875409095d22f6bd7bcc7377889b85e8cab5c0dfb781fe", size = 97223 }, ] [[package]] @@ -802,9 +814,9 @@ dependencies = [ { name = "asgiref" }, { name = "django" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/21/39/55822b15b7ec87410f34cd16ce04065ff390e50f9e29f31d6d116fc80456/django_cors_headers-4.9.0.tar.gz", hash = "sha256:fe5d7cb59fdc2c8c646ce84b727ac2bca8912a247e6e68e1fb507372178e59e8", size = 21458, upload-time = "2025-09-18T10:40:52.326Z" } +sdist = { url = "https://files.pythonhosted.org/packages/21/39/55822b15b7ec87410f34cd16ce04065ff390e50f9e29f31d6d116fc80456/django_cors_headers-4.9.0.tar.gz", hash = "sha256:fe5d7cb59fdc2c8c646ce84b727ac2bca8912a247e6e68e1fb507372178e59e8", size = 21458 } wheels = [ - { url = "https://files.pythonhosted.org/packages/30/d8/19ed1e47badf477d17fb177c1c19b5a21da0fd2d9f093f23be3fb86c5fab/django_cors_headers-4.9.0-py3-none-any.whl", hash = "sha256:15c7f20727f90044dcee2216a9fd7303741a864865f0c3657e28b7056f61b449", size = 12809, upload-time = "2025-09-18T10:40:50.843Z" }, + { url = "https://files.pythonhosted.org/packages/30/d8/19ed1e47badf477d17fb177c1c19b5a21da0fd2d9f093f23be3fb86c5fab/django_cors_headers-4.9.0-py3-none-any.whl", hash = "sha256:15c7f20727f90044dcee2216a9fd7303741a864865f0c3657e28b7056f61b449", size = 12809 }, ] [[package]] @@ -815,9 +827,9 @@ dependencies = [ { name = "django" }, { name = "redis" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/83/9d/2272742fdd9d0a9f0b28cd995b0539430c9467a2192e4de2cea9ea6ad38c/django-redis-5.4.0.tar.gz", hash = "sha256:6a02abaa34b0fea8bf9b707d2c363ab6adc7409950b2db93602e6cb292818c42", size = 52567, upload-time = "2023-10-01T20:22:01.221Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/9d/2272742fdd9d0a9f0b28cd995b0539430c9467a2192e4de2cea9ea6ad38c/django-redis-5.4.0.tar.gz", hash = "sha256:6a02abaa34b0fea8bf9b707d2c363ab6adc7409950b2db93602e6cb292818c42", size = 52567 } wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/f1/63caad7c9222c26a62082f4f777de26389233b7574629996098bf6d25a4d/django_redis-5.4.0-py3-none-any.whl", hash = "sha256:ebc88df7da810732e2af9987f7f426c96204bf89319df4c6da6ca9a2942edd5b", size = 31119, upload-time = "2023-10-01T20:21:33.009Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f1/63caad7c9222c26a62082f4f777de26389233b7574629996098bf6d25a4d/django_redis-5.4.0-py3-none-any.whl", hash = "sha256:ebc88df7da810732e2af9987f7f426c96204bf89319df4c6da6ca9a2942edd5b", size = 31119 }, ] [[package]] @@ -827,7 +839,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "django" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9a/2a/da4db7649ac516fc4b89b86d697edb92362c4f6b0ab2d2fe20d1e0f6ab10/django-tenants-3.5.0.tar.gz", hash = "sha256:bed426108e1bd4f962afa38c1e0fd985a3e8c4c902ded60bd57dbf4fcc92d2cc", size = 117503, upload-time = "2023-05-11T14:10:26.045Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/2a/da4db7649ac516fc4b89b86d697edb92362c4f6b0ab2d2fe20d1e0f6ab10/django-tenants-3.5.0.tar.gz", hash = "sha256:bed426108e1bd4f962afa38c1e0fd985a3e8c4c902ded60bd57dbf4fcc92d2cc", size = 117503 } [[package]] name = "django-timezone-field" @@ -836,9 +848,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "django" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/05/9b93a66452cdb8a08ab26f08d5766d2332673e659a8b2aeb73f2a904d421/django_timezone_field-7.2.1.tar.gz", hash = "sha256:def846f9e7200b7b8f2a28fcce2b78fb2d470f6a9f272b07c4e014f6ba4c6d2e", size = 13096, upload-time = "2025-12-06T23:50:44.591Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/05/9b93a66452cdb8a08ab26f08d5766d2332673e659a8b2aeb73f2a904d421/django_timezone_field-7.2.1.tar.gz", hash = "sha256:def846f9e7200b7b8f2a28fcce2b78fb2d470f6a9f272b07c4e014f6ba4c6d2e", size = 13096 } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/7f/d885667401515b467f84569c56075bc9add72c9fd425fca51a25f4c997e1/django_timezone_field-7.2.1-py3-none-any.whl", hash = "sha256:276915b72c5816f57c3baf9e43f816c695ef940d1b21f91ebf6203c09bf4ad44", size = 13284, upload-time = "2025-12-06T23:50:43.302Z" }, + { url = "https://files.pythonhosted.org/packages/41/7f/d885667401515b467f84569c56075bc9add72c9fd425fca51a25f4c997e1/django_timezone_field-7.2.1-py3-none-any.whl", hash = "sha256:276915b72c5816f57c3baf9e43f816c695ef940d1b21f91ebf6203c09bf4ad44", size = 13284 }, ] [[package]] @@ -849,9 +861,9 @@ dependencies = [ { name = "django" }, { name = "pytz" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8e/53/5b2a002c5ebafd60dff1e1945a7d63dee40155830997439a9ba324f0fd50/djangorestframework-3.14.0.tar.gz", hash = "sha256:579a333e6256b09489cbe0a067e66abe55c6595d8926be6b99423786334350c8", size = 1055343, upload-time = "2022-09-22T11:38:44.245Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/53/5b2a002c5ebafd60dff1e1945a7d63dee40155830997439a9ba324f0fd50/djangorestframework-3.14.0.tar.gz", hash = "sha256:579a333e6256b09489cbe0a067e66abe55c6595d8926be6b99423786334350c8", size = 1055343 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/4b/3b46c0914ba4b7546a758c35fdfa8e7f017fcbe7f23c878239e93623337a/djangorestframework-3.14.0-py3-none-any.whl", hash = "sha256:eb63f58c9f218e1a7d064d17a70751f528ed4e1d35547fdade9aaf4cd103fd08", size = 1062761, upload-time = "2022-09-22T11:38:41.825Z" }, + { url = "https://files.pythonhosted.org/packages/ff/4b/3b46c0914ba4b7546a758c35fdfa8e7f017fcbe7f23c878239e93623337a/djangorestframework-3.14.0-py3-none-any.whl", hash = "sha256:eb63f58c9f218e1a7d064d17a70751f528ed4e1d35547fdade9aaf4cd103fd08", size = 1062761 }, ] [[package]] @@ -865,18 +877,18 @@ dependencies = [ { name = "urllib3" }, { name = "websocket-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f0/73/f7c9a14e88e769f38cb7fb45aa88dfd795faa8e18aea11bababf6e068d5e/docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20", size = 259301, upload-time = "2023-06-01T14:24:49.268Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/73/f7c9a14e88e769f38cb7fb45aa88dfd795faa8e18aea11bababf6e068d5e/docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20", size = 259301 } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/be/3032490fa33b36ddc8c4b1da3252c6f974e7133f1a50de00c6b85cca203a/docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9", size = 148096, upload-time = "2023-06-01T14:24:47.769Z" }, + { url = "https://files.pythonhosted.org/packages/db/be/3032490fa33b36ddc8c4b1da3252c6f974e7133f1a50de00c6b85cca203a/docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9", size = 148096 }, ] [[package]] name = "docutils" version = "0.20.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1f/53/a5da4f2c5739cf66290fac1431ee52aff6851c7c8ffd8264f13affd7bcdd/docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b", size = 2058365, upload-time = "2023-05-16T23:39:19.748Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/53/a5da4f2c5739cf66290fac1431ee52aff6851c7c8ffd8264f13affd7bcdd/docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b", size = 2058365 } wheels = [ - { url = "https://files.pythonhosted.org/packages/26/87/f238c0670b94533ac0353a4e2a1a771a0cc73277b88bff23d3ae35a256c1/docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6", size = 572666, upload-time = "2023-05-16T23:39:15.976Z" }, + { url = "https://files.pythonhosted.org/packages/26/87/f238c0670b94533ac0353a4e2a1a771a0cc73277b88bff23d3ae35a256c1/docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6", size = 572666 }, ] [[package]] @@ -892,9 +904,9 @@ dependencies = [ { name = "pyyaml" }, { name = "uritemplate" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/89/e4/8f619b63bd8095f3797d41da186c707dd9add86b86341d1f350f1d15b2dd/drf-yasg-1.21.7.tar.gz", hash = "sha256:4c3b93068b3dfca6969ab111155e4dd6f7b2d680b98778de8fd460b7837bdb0d", size = 4512723, upload-time = "2023-07-20T13:47:34.308Z" } +sdist = { url = "https://files.pythonhosted.org/packages/89/e4/8f619b63bd8095f3797d41da186c707dd9add86b86341d1f350f1d15b2dd/drf-yasg-1.21.7.tar.gz", hash = "sha256:4c3b93068b3dfca6969ab111155e4dd6f7b2d680b98778de8fd460b7837bdb0d", size = 4512723 } wheels = [ - { url = "https://files.pythonhosted.org/packages/26/a5/9fedcd955821ec3b4d26b8a723081eb0f400b7f0bc51f1f49136648423ff/drf_yasg-1.21.7-py3-none-any.whl", hash = "sha256:f85642072c35e684356475781b7ecf5d218fff2c6185c040664dd49f0a4be181", size = 4289125, upload-time = "2023-07-20T13:47:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/26/a5/9fedcd955821ec3b4d26b8a723081eb0f400b7f0bc51f1f49136648423ff/drf_yasg-1.21.7-py3-none-any.whl", hash = "sha256:f85642072c35e684356475781b7ecf5d218fff2c6185c040664dd49f0a4be181", size = 4289125 }, ] [[package]] @@ -906,9 +918,9 @@ dependencies = [ { name = "six" }, { name = "stone" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9e/56/ac085f58e8e0d0bcafdf98c2605e454ac946e3d0c72679669ae112dc30be/dropbox-12.0.2.tar.gz", hash = "sha256:50057fd5ad5fcf047f542dfc6747a896e7ef982f1b5f8500daf51f3abd609962", size = 560236, upload-time = "2024-06-03T16:45:30.448Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/56/ac085f58e8e0d0bcafdf98c2605e454ac946e3d0c72679669ae112dc30be/dropbox-12.0.2.tar.gz", hash = "sha256:50057fd5ad5fcf047f542dfc6747a896e7ef982f1b5f8500daf51f3abd609962", size = 560236 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/de/95d8204d9a20fbdb353c5f8e4229b0fcb90f22b96f8246ff1f47c8a45fd5/dropbox-12.0.2-py3-none-any.whl", hash = "sha256:c5b7e9c2668adb6b12dcecd84342565dc50f7d35ab6a748d155cb79040979d1c", size = 572076, upload-time = "2024-06-03T16:45:28.153Z" }, + { url = "https://files.pythonhosted.org/packages/2d/de/95d8204d9a20fbdb353c5f8e4229b0fcb90f22b96f8246ff1f47c8a45fd5/dropbox-12.0.2-py3-none-any.whl", hash = "sha256:c5b7e9c2668adb6b12dcecd84342565dc50f7d35ab6a748d155cb79040979d1c", size = 572076 }, ] [[package]] @@ -920,15 +932,15 @@ dependencies = [ { name = "fsspec" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/97/15/6d8f4c3033ad2bc364b8bb613c52c96653f2268f32ecff4f3ab5f1d7c19b/dropboxdrivefs-1.4.1.tar.gz", hash = "sha256:6f3c6061d045813553ce91ed0e2b682f1d70bec74011943c92b3181faacefd34", size = 7413, upload-time = "2024-05-27T14:04:37.648Z" } +sdist = { url = "https://files.pythonhosted.org/packages/97/15/6d8f4c3033ad2bc364b8bb613c52c96653f2268f32ecff4f3ab5f1d7c19b/dropboxdrivefs-1.4.1.tar.gz", hash = "sha256:6f3c6061d045813553ce91ed0e2b682f1d70bec74011943c92b3181faacefd34", size = 7413 } [[package]] name = "execnet" version = "2.1.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708 }, ] [[package]] @@ -940,83 +952,83 @@ dependencies = [ { name = "packaging" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/07/c9/671f66f6b31ec48e5825d36435f0cb91189fa8bb6b50724029dbff4ca83c/faiss_cpu-1.13.2-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9064eb34f8f64438dd5b95c8f03a780b1a3f0b99c46eeacb1f0b5d15fc02dc1", size = 3452776, upload-time = "2025-12-24T10:27:01.419Z" }, - { url = "https://files.pythonhosted.org/packages/5a/4a/97150aa1582fb9c2bca95bd8fc37f27d3b470acec6f0a6833844b21e4b40/faiss_cpu-1.13.2-cp310-abi3-macosx_14_0_x86_64.whl", hash = "sha256:c8d097884521e1ecaea6467aeebbf1aa56ee4a36350b48b2ca6b39366565c317", size = 7896434, upload-time = "2025-12-24T10:27:03.592Z" }, - { url = "https://files.pythonhosted.org/packages/0b/d0/0940575f059591ca31b63a881058adb16a387020af1709dcb7669460115c/faiss_cpu-1.13.2-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ee330a284042c2480f2e90450a10378fd95655d62220159b1408f59ee83ebf1", size = 11485825, upload-time = "2025-12-24T10:27:05.681Z" }, - { url = "https://files.pythonhosted.org/packages/e7/e1/a5acac02aa593809f0123539afe7b4aff61d1db149e7093239888c9053e1/faiss_cpu-1.13.2-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab88ee287c25a119213153d033f7dd64c3ccec466ace267395872f554b648cd7", size = 23845772, upload-time = "2025-12-24T10:27:08.194Z" }, - { url = "https://files.pythonhosted.org/packages/9c/7b/49dcaf354834ec457e85ca769d50bc9b5f3003fab7c94a9dcf08cf742793/faiss_cpu-1.13.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:85511129b34f890d19c98b82a0cd5ffb27d89d1cec2ee41d2621ee9f9ef8cf3f", size = 13477567, upload-time = "2025-12-24T10:27:10.822Z" }, - { url = "https://files.pythonhosted.org/packages/f7/6b/12bb4037921c38bb2c0b4cfc213ca7e04bbbebbfea89b0b5746248ce446e/faiss_cpu-1.13.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b32eb4065bac352b52a9f5ae07223567fab0a976c7d05017c01c45a1c24264f", size = 25102239, upload-time = "2025-12-24T10:27:13.476Z" }, + { url = "https://files.pythonhosted.org/packages/07/c9/671f66f6b31ec48e5825d36435f0cb91189fa8bb6b50724029dbff4ca83c/faiss_cpu-1.13.2-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9064eb34f8f64438dd5b95c8f03a780b1a3f0b99c46eeacb1f0b5d15fc02dc1", size = 3452776 }, + { url = "https://files.pythonhosted.org/packages/5a/4a/97150aa1582fb9c2bca95bd8fc37f27d3b470acec6f0a6833844b21e4b40/faiss_cpu-1.13.2-cp310-abi3-macosx_14_0_x86_64.whl", hash = "sha256:c8d097884521e1ecaea6467aeebbf1aa56ee4a36350b48b2ca6b39366565c317", size = 7896434 }, + { url = "https://files.pythonhosted.org/packages/0b/d0/0940575f059591ca31b63a881058adb16a387020af1709dcb7669460115c/faiss_cpu-1.13.2-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ee330a284042c2480f2e90450a10378fd95655d62220159b1408f59ee83ebf1", size = 11485825 }, + { url = "https://files.pythonhosted.org/packages/e7/e1/a5acac02aa593809f0123539afe7b4aff61d1db149e7093239888c9053e1/faiss_cpu-1.13.2-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab88ee287c25a119213153d033f7dd64c3ccec466ace267395872f554b648cd7", size = 23845772 }, + { url = "https://files.pythonhosted.org/packages/9c/7b/49dcaf354834ec457e85ca769d50bc9b5f3003fab7c94a9dcf08cf742793/faiss_cpu-1.13.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:85511129b34f890d19c98b82a0cd5ffb27d89d1cec2ee41d2621ee9f9ef8cf3f", size = 13477567 }, + { url = "https://files.pythonhosted.org/packages/f7/6b/12bb4037921c38bb2c0b4cfc213ca7e04bbbebbfea89b0b5746248ce446e/faiss_cpu-1.13.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b32eb4065bac352b52a9f5ae07223567fab0a976c7d05017c01c45a1c24264f", size = 25102239 }, ] [[package]] name = "fastuuid" version = "0.14.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/7d/d9daedf0f2ebcacd20d599928f8913e9d2aea1d56d2d355a93bfa2b611d7/fastuuid-0.14.0.tar.gz", hash = "sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26", size = 18232, upload-time = "2025-10-19T22:19:22.402Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/7d/d9daedf0f2ebcacd20d599928f8913e9d2aea1d56d2d355a93bfa2b611d7/fastuuid-0.14.0.tar.gz", hash = "sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26", size = 18232 } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/a2/e78fcc5df65467f0d207661b7ef86c5b7ac62eea337c0c0fcedbeee6fb13/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77e94728324b63660ebf8adb27055e92d2e4611645bf12ed9d88d30486471d0a", size = 510164, upload-time = "2025-10-19T22:31:45.635Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b3/c846f933f22f581f558ee63f81f29fa924acd971ce903dab1a9b6701816e/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:caa1f14d2102cb8d353096bc6ef6c13b2c81f347e6ab9d6fbd48b9dea41c153d", size = 261837, upload-time = "2025-10-19T22:38:38.53Z" }, - { url = "https://files.pythonhosted.org/packages/54/ea/682551030f8c4fa9a769d9825570ad28c0c71e30cf34020b85c1f7ee7382/fastuuid-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d23ef06f9e67163be38cece704170486715b177f6baae338110983f99a72c070", size = 251370, upload-time = "2025-10-19T22:40:26.07Z" }, - { url = "https://files.pythonhosted.org/packages/14/dd/5927f0a523d8e6a76b70968e6004966ee7df30322f5fc9b6cdfb0276646a/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c9ec605ace243b6dbe3bd27ebdd5d33b00d8d1d3f580b39fdd15cd96fd71796", size = 277766, upload-time = "2025-10-19T22:37:23.779Z" }, - { url = "https://files.pythonhosted.org/packages/16/6e/c0fb547eef61293153348f12e0f75a06abb322664b34a1573a7760501336/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:808527f2407f58a76c916d6aa15d58692a4a019fdf8d4c32ac7ff303b7d7af09", size = 278105, upload-time = "2025-10-19T22:26:56.821Z" }, - { url = "https://files.pythonhosted.org/packages/2d/b1/b9c75e03b768f61cf2e84ee193dc18601aeaf89a4684b20f2f0e9f52b62c/fastuuid-0.14.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fb3c0d7fef6674bbeacdd6dbd386924a7b60b26de849266d1ff6602937675c8", size = 301564, upload-time = "2025-10-19T22:30:31.604Z" }, - { url = "https://files.pythonhosted.org/packages/fc/fa/f7395fdac07c7a54f18f801744573707321ca0cee082e638e36452355a9d/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab3f5d36e4393e628a4df337c2c039069344db5f4b9d2a3c9cea48284f1dd741", size = 459659, upload-time = "2025-10-19T22:31:32.341Z" }, - { url = "https://files.pythonhosted.org/packages/66/49/c9fd06a4a0b1f0f048aacb6599e7d96e5d6bc6fa680ed0d46bf111929d1b/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b9a0ca4f03b7e0b01425281ffd44e99d360e15c895f1907ca105854ed85e2057", size = 478430, upload-time = "2025-10-19T22:26:22.962Z" }, - { url = "https://files.pythonhosted.org/packages/be/9c/909e8c95b494e8e140e8be6165d5fc3f61fdc46198c1554df7b3e1764471/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3acdf655684cc09e60fb7e4cf524e8f42ea760031945aa8086c7eae2eeeabeb8", size = 450894, upload-time = "2025-10-19T22:27:01.647Z" }, - { url = "https://files.pythonhosted.org/packages/90/eb/d29d17521976e673c55ef7f210d4cdd72091a9ec6755d0fd4710d9b3c871/fastuuid-0.14.0-cp312-cp312-win32.whl", hash = "sha256:9579618be6280700ae36ac42c3efd157049fe4dd40ca49b021280481c78c3176", size = 154374, upload-time = "2025-10-19T22:29:19.879Z" }, - { url = "https://files.pythonhosted.org/packages/cc/fc/f5c799a6ea6d877faec0472d0b27c079b47c86b1cdc577720a5386483b36/fastuuid-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:d9e4332dc4ba054434a9594cbfaf7823b57993d7d8e7267831c3e059857cf397", size = 156550, upload-time = "2025-10-19T22:27:49.658Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/e78fcc5df65467f0d207661b7ef86c5b7ac62eea337c0c0fcedbeee6fb13/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77e94728324b63660ebf8adb27055e92d2e4611645bf12ed9d88d30486471d0a", size = 510164 }, + { url = "https://files.pythonhosted.org/packages/2b/b3/c846f933f22f581f558ee63f81f29fa924acd971ce903dab1a9b6701816e/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:caa1f14d2102cb8d353096bc6ef6c13b2c81f347e6ab9d6fbd48b9dea41c153d", size = 261837 }, + { url = "https://files.pythonhosted.org/packages/54/ea/682551030f8c4fa9a769d9825570ad28c0c71e30cf34020b85c1f7ee7382/fastuuid-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d23ef06f9e67163be38cece704170486715b177f6baae338110983f99a72c070", size = 251370 }, + { url = "https://files.pythonhosted.org/packages/14/dd/5927f0a523d8e6a76b70968e6004966ee7df30322f5fc9b6cdfb0276646a/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c9ec605ace243b6dbe3bd27ebdd5d33b00d8d1d3f580b39fdd15cd96fd71796", size = 277766 }, + { url = "https://files.pythonhosted.org/packages/16/6e/c0fb547eef61293153348f12e0f75a06abb322664b34a1573a7760501336/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:808527f2407f58a76c916d6aa15d58692a4a019fdf8d4c32ac7ff303b7d7af09", size = 278105 }, + { url = "https://files.pythonhosted.org/packages/2d/b1/b9c75e03b768f61cf2e84ee193dc18601aeaf89a4684b20f2f0e9f52b62c/fastuuid-0.14.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fb3c0d7fef6674bbeacdd6dbd386924a7b60b26de849266d1ff6602937675c8", size = 301564 }, + { url = "https://files.pythonhosted.org/packages/fc/fa/f7395fdac07c7a54f18f801744573707321ca0cee082e638e36452355a9d/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab3f5d36e4393e628a4df337c2c039069344db5f4b9d2a3c9cea48284f1dd741", size = 459659 }, + { url = "https://files.pythonhosted.org/packages/66/49/c9fd06a4a0b1f0f048aacb6599e7d96e5d6bc6fa680ed0d46bf111929d1b/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b9a0ca4f03b7e0b01425281ffd44e99d360e15c895f1907ca105854ed85e2057", size = 478430 }, + { url = "https://files.pythonhosted.org/packages/be/9c/909e8c95b494e8e140e8be6165d5fc3f61fdc46198c1554df7b3e1764471/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3acdf655684cc09e60fb7e4cf524e8f42ea760031945aa8086c7eae2eeeabeb8", size = 450894 }, + { url = "https://files.pythonhosted.org/packages/90/eb/d29d17521976e673c55ef7f210d4cdd72091a9ec6755d0fd4710d9b3c871/fastuuid-0.14.0-cp312-cp312-win32.whl", hash = "sha256:9579618be6280700ae36ac42c3efd157049fe4dd40ca49b021280481c78c3176", size = 154374 }, + { url = "https://files.pythonhosted.org/packages/cc/fc/f5c799a6ea6d877faec0472d0b27c079b47c86b1cdc577720a5386483b36/fastuuid-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:d9e4332dc4ba054434a9594cbfaf7823b57993d7d8e7267831c3e059857cf397", size = 156550 }, ] [[package]] name = "filelock" version = "3.29.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571 } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, + { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812 }, ] [[package]] name = "filetype" version = "1.2.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bb/29/745f7d30d47fe0f251d3ad3dc2978a23141917661998763bebb6da007eb1/filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb", size = 998020, upload-time = "2022-11-02T17:34:04.141Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/29/745f7d30d47fe0f251d3ad3dc2978a23141917661998763bebb6da007eb1/filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb", size = 998020 } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970, upload-time = "2022-11-02T17:34:01.425Z" }, + { url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970 }, ] [[package]] name = "frozenlist" version = "1.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875 } wheels = [ - { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, - { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, - { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, - { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, - { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, - { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, - { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, - { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, - { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, - { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, - { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, - { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, - { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, - { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, - { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, - { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782 }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594 }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448 }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411 }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014 }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909 }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049 }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485 }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619 }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320 }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820 }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518 }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096 }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985 }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591 }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102 }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409 }, ] [[package]] name = "fsspec" version = "2024.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a0/52/f16a068ebadae42526484c31f4398e62962504e5724a8ba5dc3409483df2/fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493", size = 286853, upload-time = "2024-10-21T01:21:16.969Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a0/52/f16a068ebadae42526484c31f4398e62962504e5724a8ba5dc3409483df2/fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493", size = 286853 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/b2/454d6e7f0158951d8a78c2e1eb4f69ae81beb8dca5fee9809c6c99e9d0d0/fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871", size = 179641, upload-time = "2024-10-21T01:21:14.793Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b2/454d6e7f0158951d8a78c2e1eb4f69ae81beb8dca5fee9809c6c99e9d0d0/fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871", size = 179641 }, ] [package.optional-dependencies] @@ -1028,9 +1040,9 @@ sftp = [ name = "funcy" version = "2.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/70/b8/c6081521ff70afdff55cd9512b2220bbf4fa88804dae51d1b57b4b58ef32/funcy-2.0.tar.gz", hash = "sha256:3963315d59d41c6f30c04bc910e10ab50a3ac4a225868bfa96feed133df075cb", size = 537931, upload-time = "2023-03-28T06:22:46.764Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/b8/c6081521ff70afdff55cd9512b2220bbf4fa88804dae51d1b57b4b58ef32/funcy-2.0.tar.gz", hash = "sha256:3963315d59d41c6f30c04bc910e10ab50a3ac4a225868bfa96feed133df075cb", size = 537931 } wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/08/c2409cb01d5368dcfedcbaffa7d044cc8957d57a9d0855244a5eb4709d30/funcy-2.0-py2.py3-none-any.whl", hash = "sha256:53df23c8bb1651b12f095df764bfb057935d49537a56de211b098f4c79614bb0", size = 30891, upload-time = "2023-03-28T06:22:42.576Z" }, + { url = "https://files.pythonhosted.org/packages/d5/08/c2409cb01d5368dcfedcbaffa7d044cc8957d57a9d0855244a5eb4709d30/funcy-2.0-py2.py3-none-any.whl", hash = "sha256:53df23c8bb1651b12f095df764bfb057935d49537a56de211b098f4c79614bb0", size = 30891 }, ] [[package]] @@ -1046,9 +1058,9 @@ dependencies = [ { name = "google-cloud-storage" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e5/1e/1d8c4593d9e2eb04918fec43253ab152823d67ad51ad9e3ab6b3a78c431a/gcsfs-2024.10.0.tar.gz", hash = "sha256:5df54cfe568e8fdeea5aafa7fed695cdc69a9a674e991ca8c1ce634f5df1d314", size = 79588, upload-time = "2024-10-21T13:43:26.163Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/1e/1d8c4593d9e2eb04918fec43253ab152823d67ad51ad9e3ab6b3a78c431a/gcsfs-2024.10.0.tar.gz", hash = "sha256:5df54cfe568e8fdeea5aafa7fed695cdc69a9a674e991ca8c1ce634f5df1d314", size = 79588 } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/96/d60e835fb7d10166c77aef0c1fa30e634153c03a0f486786977b95f88fde/gcsfs-2024.10.0-py2.py3-none-any.whl", hash = "sha256:bb2d23547e61203ea2dda5fa6c4b91a0c34b74ebe8bb6ab1926f6c33381bceb2", size = 34953, upload-time = "2024-10-21T13:43:24.951Z" }, + { url = "https://files.pythonhosted.org/packages/dc/96/d60e835fb7d10166c77aef0c1fa30e634153c03a0f486786977b95f88fde/gcsfs-2024.10.0-py2.py3-none-any.whl", hash = "sha256:bb2d23547e61203ea2dda5fa6c4b91a0c34b74ebe8bb6ab1926f6c33381bceb2", size = 34953 }, ] [[package]] @@ -1062,9 +1074,9 @@ dependencies = [ { name = "protobuf" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/16/ce/502a57fb0ec752026d24df1280b162294b22a0afb98a326084f9a979138b/google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b", size = 177001, upload-time = "2026-04-10T00:41:28.035Z" } +sdist = { url = "https://files.pythonhosted.org/packages/16/ce/502a57fb0ec752026d24df1280b162294b22a0afb98a326084f9a979138b/google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b", size = 177001 } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/15/e56f351cf6ef1cfea58e6ac226a7318ed1deb2218c4b3cc9bd9e4b786c5a/google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8", size = 173274, upload-time = "2026-04-09T22:57:16.198Z" }, + { url = "https://files.pythonhosted.org/packages/03/15/e56f351cf6ef1cfea58e6ac226a7318ed1deb2218c4b3cc9bd9e4b786c5a/google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8", size = 173274 }, ] [package.optional-dependencies] @@ -1084,9 +1096,9 @@ dependencies = [ { name = "httplib2" }, { name = "uritemplate" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6d/f3/34ef8aca7909675fe327f96c1ed927f0520e7acf68af19157e96acc05e76/google_api_python_client-2.196.0.tar.gz", hash = "sha256:9f335d38f6caaa2747bcf64335ed1a9a19047d53e86538eda6a1b17d37f1743d", size = 14628129, upload-time = "2026-05-06T23:47:35.655Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/f3/34ef8aca7909675fe327f96c1ed927f0520e7acf68af19157e96acc05e76/google_api_python_client-2.196.0.tar.gz", hash = "sha256:9f335d38f6caaa2747bcf64335ed1a9a19047d53e86538eda6a1b17d37f1743d", size = 14628129 } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/c7/1817b4edf966d5afcac1c0781ca36d621bc0cb58104c4e7c2a475ab185f7/google_api_python_client-2.196.0-py3-none-any.whl", hash = "sha256:2591e9b47dcb17e4e62a09370aaee3bcf323af8f28ccecdabcd0a42a23ca4db5", size = 15206663, upload-time = "2026-05-06T23:47:32.886Z" }, + { url = "https://files.pythonhosted.org/packages/99/c7/1817b4edf966d5afcac1c0781ca36d621bc0cb58104c4e7c2a475ab185f7/google_api_python_client-2.196.0-py3-none-any.whl", hash = "sha256:2591e9b47dcb17e4e62a09370aaee3bcf323af8f28ccecdabcd0a42a23ca4db5", size = 15206663 }, ] [[package]] @@ -1100,9 +1112,9 @@ dependencies = [ { name = "six" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4b/e0/d2c96098280f17eb626d4da0b7e553b8e5648d57514c8cefec851c16920c/google-auth-2.20.0.tar.gz", hash = "sha256:030af34138909ccde0fbce611afc178f1d65d32fbff281f25738b1fe1c6f3eaa", size = 229669, upload-time = "2023-06-13T17:50:38.754Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/e0/d2c96098280f17eb626d4da0b7e553b8e5648d57514c8cefec851c16920c/google-auth-2.20.0.tar.gz", hash = "sha256:030af34138909ccde0fbce611afc178f1d65d32fbff281f25738b1fe1c6f3eaa", size = 229669 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/1a/5866a7c6e16abc1df395e6d2b9808984d0905c747d75f5e20f1a052421d1/google_auth-2.20.0-py2.py3-none-any.whl", hash = "sha256:23b7b0950fcda519bfb6692bf0d5289d2ea49fc143717cc7188458ec620e63fa", size = 181456, upload-time = "2023-06-13T17:50:36.408Z" }, + { url = "https://files.pythonhosted.org/packages/9a/1a/5866a7c6e16abc1df395e6d2b9808984d0905c747d75f5e20f1a052421d1/google_auth-2.20.0-py2.py3-none-any.whl", hash = "sha256:23b7b0950fcda519bfb6692bf0d5289d2ea49fc143717cc7188458ec620e63fa", size = 181456 }, ] [[package]] @@ -1113,9 +1125,9 @@ dependencies = [ { name = "google-auth" }, { name = "httplib2" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/b3/f192c8bc7e41e0ebdbd95afcae4783417a34b6a6af62d22daf22c3fd38fc/google_auth_httplib2-0.4.0.tar.gz", hash = "sha256:d5b030a204b7a4b4d553ba9ca701b62481ee2b74419325580be70f7d85ffed35", size = 11161, upload-time = "2026-05-07T08:03:46.878Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/b3/f192c8bc7e41e0ebdbd95afcae4783417a34b6a6af62d22daf22c3fd38fc/google_auth_httplib2-0.4.0.tar.gz", hash = "sha256:d5b030a204b7a4b4d553ba9ca701b62481ee2b74419325580be70f7d85ffed35", size = 11161 } wheels = [ - { url = "https://files.pythonhosted.org/packages/97/be/954c35a62b9e31de66b0a43c225c9b6bb9e0f98d6b1dc110a2308e3644f5/google_auth_httplib2-0.4.0-py3-none-any.whl", hash = "sha256:8e55cfafa3358cba85f6cad4a886138e88e158d71e7e5c9ee5936a5c1507fb91", size = 9529, upload-time = "2026-05-07T08:02:12.375Z" }, + { url = "https://files.pythonhosted.org/packages/97/be/954c35a62b9e31de66b0a43c225c9b6bb9e0f98d6b1dc110a2308e3644f5/google_auth_httplib2-0.4.0-py3-none-any.whl", hash = "sha256:8e55cfafa3358cba85f6cad4a886138e88e158d71e7e5c9ee5936a5c1507fb91", size = 9529 }, ] [[package]] @@ -1126,9 +1138,9 @@ dependencies = [ { name = "google-auth" }, { name = "requests-oauthlib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/70/18/90c7fac516e63cf2058166fce0c88c353647c677b51cc036c09c49bb5cbb/google_auth_oauthlib-1.4.0.tar.gz", hash = "sha256:18b5e28880eb8eba9065c436becdc0ee8e4b59117a73a510679c82f70cd363d2", size = 21675, upload-time = "2026-05-07T08:03:47.816Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/18/90c7fac516e63cf2058166fce0c88c353647c677b51cc036c09c49bb5cbb/google_auth_oauthlib-1.4.0.tar.gz", hash = "sha256:18b5e28880eb8eba9065c436becdc0ee8e4b59117a73a510679c82f70cd363d2", size = 21675 } wheels = [ - { url = "https://files.pythonhosted.org/packages/37/d3/d7dff0d58a9e9244b48044bfb6a898bfcc8ecc42e0031d1bebc695344725/google_auth_oauthlib-1.4.0-py3-none-any.whl", hash = "sha256:251314f213a9ee46a5ae73988e84fd7cca8bb68e7ecf4bfd45940f9e7f51d070", size = 19261, upload-time = "2026-05-07T08:02:13.798Z" }, + { url = "https://files.pythonhosted.org/packages/37/d3/d7dff0d58a9e9244b48044bfb6a898bfcc8ecc42e0031d1bebc695344725/google_auth_oauthlib-1.4.0-py3-none-any.whl", hash = "sha256:251314f213a9ee46a5ae73988e84fd7cca8bb68e7ecf4bfd45940f9e7f51d070", size = 19261 }, ] [[package]] @@ -1146,9 +1158,9 @@ dependencies = [ { name = "python-dateutil" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/ff/2c520952db184dec31e2ee988cfa37fa9e7776935a3f2eccc44252ecab5f/google-cloud-bigquery-3.11.4.tar.gz", hash = "sha256:697df117241a2283bcbb93b21e10badc14e51c9a90800d2a7e1a3e1c7d842974", size = 410777, upload-time = "2023-07-19T23:12:12.7Z" } +sdist = { url = "https://files.pythonhosted.org/packages/62/ff/2c520952db184dec31e2ee988cfa37fa9e7776935a3f2eccc44252ecab5f/google-cloud-bigquery-3.11.4.tar.gz", hash = "sha256:697df117241a2283bcbb93b21e10badc14e51c9a90800d2a7e1a3e1c7d842974", size = 410777 } wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/6a/d0ef792288f2fa2cfea80899a82de302b3332dfda41984fe114e2cfbf700/google_cloud_bigquery-3.11.4-py2.py3-none-any.whl", hash = "sha256:5fa7897743a0ed949ade25a0942fc9e7557d8fce307c6f8a76d1b604cf27f1b1", size = 219607, upload-time = "2023-07-19T23:12:09.449Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/d0ef792288f2fa2cfea80899a82de302b3332dfda41984fe114e2cfbf700/google_cloud_bigquery-3.11.4-py2.py3-none-any.whl", hash = "sha256:5fa7897743a0ed949ade25a0942fc9e7557d8fce307c6f8a76d1b604cf27f1b1", size = 219607 }, ] [[package]] @@ -1159,9 +1171,9 @@ dependencies = [ { name = "google-api-core" }, { name = "google-auth" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a8/dd/1eef226e470369b26824a505c34482c0b493bc35fe8e0c6b003b5feca21a/google_cloud_core-2.6.0.tar.gz", hash = "sha256:e76149739f90fac1fc6757c09f47eaccb3145b54adbd7759b0f7c4b235f46c83", size = 36001, upload-time = "2026-05-07T08:04:04.124Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/dd/1eef226e470369b26824a505c34482c0b493bc35fe8e0c6b003b5feca21a/google_cloud_core-2.6.0.tar.gz", hash = "sha256:e76149739f90fac1fc6757c09f47eaccb3145b54adbd7759b0f7c4b235f46c83", size = 36001 } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/4a/98da8930ab109c73d9a5d13782a9ebb81ea8c111f6d534a567b71d23e52b/google_cloud_core-2.6.0-py3-none-any.whl", hash = "sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e", size = 29390, upload-time = "2026-05-07T08:02:34.672Z" }, + { url = "https://files.pythonhosted.org/packages/84/4a/98da8930ab109c73d9a5d13782a9ebb81ea8c111f6d534a567b71d23e52b/google_cloud_core-2.6.0-py3-none-any.whl", hash = "sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e", size = 29390 }, ] [[package]] @@ -1174,9 +1186,9 @@ dependencies = [ { name = "proto-plus" }, { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/48/6b/92b705f408c1d928526b65d1259be4254ef1f45e620f01f8665156b4d781/google-cloud-secret-manager-2.16.1.tar.gz", hash = "sha256:149d11ce9be7ea81d4ac3544d3fcd4c716a9edb2cb775d9c075231570b079fbb", size = 128884, upload-time = "2023-03-27T14:51:09.684Z" } +sdist = { url = "https://files.pythonhosted.org/packages/48/6b/92b705f408c1d928526b65d1259be4254ef1f45e620f01f8665156b4d781/google-cloud-secret-manager-2.16.1.tar.gz", hash = "sha256:149d11ce9be7ea81d4ac3544d3fcd4c716a9edb2cb775d9c075231570b079fbb", size = 128884 } wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/e3/c3aade516eaf544bd7d86694178de9c2da8eff8fc40326d0265acc65991d/google_cloud_secret_manager-2.16.1-py2.py3-none-any.whl", hash = "sha256:dad28c24921fb62961aafe808be0e7935a99096f03ac29eeeefa04b85534c1f3", size = 116749, upload-time = "2023-03-27T14:51:07.661Z" }, + { url = "https://files.pythonhosted.org/packages/6c/e3/c3aade516eaf544bd7d86694178de9c2da8eff8fc40326d0265acc65991d/google_cloud_secret_manager-2.16.1-py2.py3-none-any.whl", hash = "sha256:dad28c24921fb62961aafe808be0e7935a99096f03ac29eeeefa04b85534c1f3", size = 116749 }, ] [[package]] @@ -1190,22 +1202,22 @@ dependencies = [ { name = "google-resumable-media" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fc/50/c9998f84fd8ce8799d7f8020466bbc5c9e3b1126b04a09fdb02378d451b0/google-cloud-storage-2.9.0.tar.gz", hash = "sha256:9b6ae7b509fc294bdacb84d0f3ea8e20e2c54a8b4bbe39c5707635fec214eff3", size = 5498811, upload-time = "2023-05-04T17:56:46.265Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/50/c9998f84fd8ce8799d7f8020466bbc5c9e3b1126b04a09fdb02378d451b0/google-cloud-storage-2.9.0.tar.gz", hash = "sha256:9b6ae7b509fc294bdacb84d0f3ea8e20e2c54a8b4bbe39c5707635fec214eff3", size = 5498811 } wheels = [ - { url = "https://files.pythonhosted.org/packages/74/fb/3770e7f44cf6133f502e1b8503b6739351b53272cf8313b47f1de6cf4960/google_cloud_storage-2.9.0-py2.py3-none-any.whl", hash = "sha256:83a90447f23d5edd045e0037982c270302e3aeb45fc1288d2c2ca713d27bad94", size = 113512, upload-time = "2023-05-04T17:56:43.929Z" }, + { url = "https://files.pythonhosted.org/packages/74/fb/3770e7f44cf6133f502e1b8503b6739351b53272cf8313b47f1de6cf4960/google_cloud_storage-2.9.0-py2.py3-none-any.whl", hash = "sha256:83a90447f23d5edd045e0037982c270302e3aeb45fc1288d2c2ca713d27bad94", size = 113512 }, ] [[package]] name = "google-crc32c" version = "1.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } +sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" }, - { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" }, - { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, - { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300 }, + { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867 }, + { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364 }, + { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740 }, + { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437 }, ] [[package]] @@ -1215,9 +1227,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-crc32c" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/00/4b/0b235beccc310d0a48adbc7246b719d173cca6c88c572dfa4b090e39143c/google_resumable_media-2.9.0.tar.gz", hash = "sha256:f7cfb224846a9dd444d125115dfbe8ef02a2b893e78f087762fe716a255a734b", size = 2164534, upload-time = "2026-05-07T08:04:44.236Z" } +sdist = { url = "https://files.pythonhosted.org/packages/00/4b/0b235beccc310d0a48adbc7246b719d173cca6c88c572dfa4b090e39143c/google_resumable_media-2.9.0.tar.gz", hash = "sha256:f7cfb224846a9dd444d125115dfbe8ef02a2b893e78f087762fe716a255a734b", size = 2164534 } wheels = [ - { url = "https://files.pythonhosted.org/packages/07/73/3518e63deb1667c5409a4579e28daf5e84479a87a72c547e0487f7883dcd/google_resumable_media-2.9.0-py3-none-any.whl", hash = "sha256:c8901e88e389af8bed64d9696c74d8bad961865eb2236e13e0bfca9bb0a65ca3", size = 81507, upload-time = "2026-05-07T08:03:23.809Z" }, + { url = "https://files.pythonhosted.org/packages/07/73/3518e63deb1667c5409a4579e28daf5e84479a87a72c547e0487f7883dcd/google_resumable_media-2.9.0-py3-none-any.whl", hash = "sha256:c8901e88e389af8bed64d9696c74d8bad961865eb2236e13e0bfca9bb0a65ca3", size = 81507 }, ] [[package]] @@ -1227,9 +1239,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631 }, ] [package.optional-dependencies] @@ -1241,18 +1253,18 @@ grpc = [ name = "greenlet" version = "3.5.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3c/3f/dbf99fb14bfeb88c28f16729215478c0e265cacd6dc22270c8f31bb6892f/greenlet-3.5.0.tar.gz", hash = "sha256:d419647372241bc68e957bf38d5c1f98852155e4146bd1e4121adea81f4f01e4", size = 196995, upload-time = "2026-04-27T13:37:15.544Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3c/3f/dbf99fb14bfeb88c28f16729215478c0e265cacd6dc22270c8f31bb6892f/greenlet-3.5.0.tar.gz", hash = "sha256:d419647372241bc68e957bf38d5c1f98852155e4146bd1e4121adea81f4f01e4", size = 196995 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/32/f2ce6d4cac3e55bc6173f92dbe627e782e1850f89d986c3606feb63aafa7/greenlet-3.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:db2910d3c809444e0a20147361f343fe2798e106af8d9d8506f5305302655a9f", size = 286228, upload-time = "2026-04-27T12:20:34.421Z" }, - { url = "https://files.pythonhosted.org/packages/b7/aa/caed9e5adf742315fc7be2a84196373aab4816e540e38ba0d76cb7584d68/greenlet-3.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec9ea74e7268ace7f9aab1b1a4e730193fc661b39a993cd91c606c32d4a3628", size = 601775, upload-time = "2026-04-27T12:52:41.045Z" }, - { url = "https://files.pythonhosted.org/packages/c7/af/90ae08497400a941595d12774447f752d3dfe0fbb012e35b76bc5c0ff37e/greenlet-3.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54d243512da35485fc7a6bf3c178fdda6327a9d6506fcdd62b1abd1e41b2927b", size = 614436, upload-time = "2026-04-27T12:59:41.595Z" }, - { url = "https://files.pythonhosted.org/packages/3f/e9/4eeadf8cb3403ac274245ba75f07844abc7fa5f6787583fc9156ba741e0f/greenlet-3.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:41353ec2ecedf7aa8f682753a41919f8718031a6edac46b8d3dc7ed9e1ceb136", size = 620610, upload-time = "2026-04-27T13:02:39.194Z" }, - { url = "https://files.pythonhosted.org/packages/2b/e0/2e13df68f367e2f9960616927d60857dd7e56aaadd59a47c644216b2f920/greenlet-3.5.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d280a7f5c331622c69f97eb167f33577ff2d1df282c41cd15907fc0a3ca198c", size = 611388, upload-time = "2026-04-27T12:25:28.008Z" }, - { url = "https://files.pythonhosted.org/packages/ee/ef/f913b3c0eb7d26d86a2401c5e1546c9d46b657efee724b06f6f4ac5d8824/greenlet-3.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:58c1c374fe2b3d852f9b6b11a7dff4c85404e51b9a596fd9e89cf904eb09866d", size = 422775, upload-time = "2026-04-27T13:05:14.261Z" }, - { url = "https://files.pythonhosted.org/packages/82/f7/393c64055132ac0d488ef6be549253b7e6274194863967ddc0bc8f5b87b8/greenlet-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1eb67d5adefb5bd2e182d42678a328979a209e4e82eb93575708185d31d1f588", size = 1570768, upload-time = "2026-04-27T12:53:28.099Z" }, - { url = "https://files.pythonhosted.org/packages/b8/4b/eaf7735253522cf56d1b74d672a58f54fc114702ceaf05def59aae72f6e1/greenlet-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2628d6c86f6cb0cb45e0c3c54058bbec559f57eaae699447748cb3928150577e", size = 1635983, upload-time = "2026-04-27T12:25:26.903Z" }, - { url = "https://files.pythonhosted.org/packages/4c/fe/4fb3a0805bd5165da5ebf858da7cc01cce8061674106d2cf5bdab32cbfde/greenlet-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:d4d9f0624c775f2dfc56ba54d515a8c771044346852a918b405914f6b19d7fd8", size = 238840, upload-time = "2026-04-27T12:23:54.806Z" }, - { url = "https://files.pythonhosted.org/packages/cb/cb/baa584cb00532126ffe12d9787db0a60c5a4f55c27bfe2666df5d4c30a32/greenlet-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:83ed9f27f1680b50e89f40f6df348a290ea234b249a4003d366663a12eab94f2", size = 235615, upload-time = "2026-04-27T12:21:38.57Z" }, + { url = "https://files.pythonhosted.org/packages/ef/32/f2ce6d4cac3e55bc6173f92dbe627e782e1850f89d986c3606feb63aafa7/greenlet-3.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:db2910d3c809444e0a20147361f343fe2798e106af8d9d8506f5305302655a9f", size = 286228 }, + { url = "https://files.pythonhosted.org/packages/b7/aa/caed9e5adf742315fc7be2a84196373aab4816e540e38ba0d76cb7584d68/greenlet-3.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec9ea74e7268ace7f9aab1b1a4e730193fc661b39a993cd91c606c32d4a3628", size = 601775 }, + { url = "https://files.pythonhosted.org/packages/c7/af/90ae08497400a941595d12774447f752d3dfe0fbb012e35b76bc5c0ff37e/greenlet-3.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54d243512da35485fc7a6bf3c178fdda6327a9d6506fcdd62b1abd1e41b2927b", size = 614436 }, + { url = "https://files.pythonhosted.org/packages/3f/e9/4eeadf8cb3403ac274245ba75f07844abc7fa5f6787583fc9156ba741e0f/greenlet-3.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:41353ec2ecedf7aa8f682753a41919f8718031a6edac46b8d3dc7ed9e1ceb136", size = 620610 }, + { url = "https://files.pythonhosted.org/packages/2b/e0/2e13df68f367e2f9960616927d60857dd7e56aaadd59a47c644216b2f920/greenlet-3.5.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d280a7f5c331622c69f97eb167f33577ff2d1df282c41cd15907fc0a3ca198c", size = 611388 }, + { url = "https://files.pythonhosted.org/packages/ee/ef/f913b3c0eb7d26d86a2401c5e1546c9d46b657efee724b06f6f4ac5d8824/greenlet-3.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:58c1c374fe2b3d852f9b6b11a7dff4c85404e51b9a596fd9e89cf904eb09866d", size = 422775 }, + { url = "https://files.pythonhosted.org/packages/82/f7/393c64055132ac0d488ef6be549253b7e6274194863967ddc0bc8f5b87b8/greenlet-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1eb67d5adefb5bd2e182d42678a328979a209e4e82eb93575708185d31d1f588", size = 1570768 }, + { url = "https://files.pythonhosted.org/packages/b8/4b/eaf7735253522cf56d1b74d672a58f54fc114702ceaf05def59aae72f6e1/greenlet-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2628d6c86f6cb0cb45e0c3c54058bbec559f57eaae699447748cb3928150577e", size = 1635983 }, + { url = "https://files.pythonhosted.org/packages/4c/fe/4fb3a0805bd5165da5ebf858da7cc01cce8061674106d2cf5bdab32cbfde/greenlet-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:d4d9f0624c775f2dfc56ba54d515a8c771044346852a918b405914f6b19d7fd8", size = 238840 }, + { url = "https://files.pythonhosted.org/packages/cb/cb/baa584cb00532126ffe12d9787db0a60c5a4f55c27bfe2666df5d4c30a32/greenlet-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:83ed9f27f1680b50e89f40f6df348a290ea234b249a4003d366663a12eab94f2", size = 235615 }, ] [[package]] @@ -1263,9 +1275,9 @@ dependencies = [ { name = "griffecli" }, { name = "griffelib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4a/49/eb6d2935e27883af92c930ed40cc4c69bcd32c402be43b8ca4ab20510f67/griffe-2.0.2.tar.gz", hash = "sha256:c5d56326d159f274492e9bf93a9895cec101155d944caa66d0fc4e0c13751b92", size = 293757, upload-time = "2026-03-27T11:34:52.205Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/49/eb6d2935e27883af92c930ed40cc4c69bcd32c402be43b8ca4ab20510f67/griffe-2.0.2.tar.gz", hash = "sha256:c5d56326d159f274492e9bf93a9895cec101155d944caa66d0fc4e0c13751b92", size = 293757 } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/c0/2bb018eecf9a83c68db9cd9fffd9dab25f102ad30ed869451046e46d1187/griffe-2.0.2-py3-none-any.whl", hash = "sha256:2b31816460aee1996af26050a1fc6927a2e5936486856707f55508e4c9b5960b", size = 5141, upload-time = "2026-03-27T11:34:47.721Z" }, + { url = "https://files.pythonhosted.org/packages/94/c0/2bb018eecf9a83c68db9cd9fffd9dab25f102ad30ed869451046e46d1187/griffe-2.0.2-py3-none-any.whl", hash = "sha256:2b31816460aee1996af26050a1fc6927a2e5936486856707f55508e4c9b5960b", size = 5141 }, ] [[package]] @@ -1276,18 +1288,18 @@ dependencies = [ { name = "colorama" }, { name = "griffelib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/79/e0/6a7d661d71bb043656a109b91d84a42b5342752542074ec83b16a6eb97f0/griffecli-2.0.2.tar.gz", hash = "sha256:40a1ad4181fc39685d025e119ae2c5b669acdc1f19b705fb9bf971f4e6f6dffb", size = 56281, upload-time = "2026-03-27T11:34:50.087Z" } +sdist = { url = "https://files.pythonhosted.org/packages/79/e0/6a7d661d71bb043656a109b91d84a42b5342752542074ec83b16a6eb97f0/griffecli-2.0.2.tar.gz", hash = "sha256:40a1ad4181fc39685d025e119ae2c5b669acdc1f19b705fb9bf971f4e6f6dffb", size = 56281 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/e8/90d93356c88ac34c20cb5edffca68138df55ca9bbd1a06eccfbcec8fdbe5/griffecli-2.0.2-py3-none-any.whl", hash = "sha256:0d44d39e59afa81e288a3e1c3bf352cc4fa537483326ac06b8bb6a51fd8303a0", size = 9500, upload-time = "2026-03-27T11:34:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e8/90d93356c88ac34c20cb5edffca68138df55ca9bbd1a06eccfbcec8fdbe5/griffecli-2.0.2-py3-none-any.whl", hash = "sha256:0d44d39e59afa81e288a3e1c3bf352cc4fa537483326ac06b8bb6a51fd8303a0", size = 9500 }, ] [[package]] name = "griffelib" version = "2.0.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/82/74f4a3310cdabfbb10da554c3a672847f1ed33c6f61dd472681ce7f1fe67/griffelib-2.0.2.tar.gz", hash = "sha256:3cf20b3bc470e83763ffbf236e0076b1211bac1bc67de13daf494640f2de707e", size = 166461, upload-time = "2026-03-27T11:34:51.091Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/82/74f4a3310cdabfbb10da554c3a672847f1ed33c6f61dd472681ce7f1fe67/griffelib-2.0.2.tar.gz", hash = "sha256:3cf20b3bc470e83763ffbf236e0076b1211bac1bc67de13daf494640f2de707e", size = 166461 } wheels = [ - { url = "https://files.pythonhosted.org/packages/11/8c/c9138d881c79aa0ea9ed83cbd58d5ca75624378b38cee225dcf5c42cc91f/griffelib-2.0.2-py3-none-any.whl", hash = "sha256:925c857658fb1ba40c0772c37acbc2ab650bd794d9c1b9726922e36ea4117ea1", size = 142357, upload-time = "2026-03-27T11:34:46.275Z" }, + { url = "https://files.pythonhosted.org/packages/11/8c/c9138d881c79aa0ea9ed83cbd58d5ca75624378b38cee225dcf5c42cc91f/griffelib-2.0.2-py3-none-any.whl", hash = "sha256:925c857658fb1ba40c0772c37acbc2ab650bd794d9c1b9726922e36ea4117ea1", size = 142357 }, ] [[package]] @@ -1299,9 +1311,9 @@ dependencies = [ { name = "grpcio" }, { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/44/4f/d098419ad0bfc06c9ce440575f05aa22d8973b6c276e86ac7890093d3c37/grpc_google_iam_v1-0.14.4.tar.gz", hash = "sha256:392b3796947ed6334e61171d9ab06bf7eb357f554e5fc7556ad7aab6d0e17038", size = 23706, upload-time = "2026-04-01T01:57:49.813Z" } +sdist = { url = "https://files.pythonhosted.org/packages/44/4f/d098419ad0bfc06c9ce440575f05aa22d8973b6c276e86ac7890093d3c37/grpc_google_iam_v1-0.14.4.tar.gz", hash = "sha256:392b3796947ed6334e61171d9ab06bf7eb357f554e5fc7556ad7aab6d0e17038", size = 23706 } wheels = [ - { url = "https://files.pythonhosted.org/packages/89/22/c2dd50c09bf679bd38173656cd4402d2511e563b33bc88f90009cf50613c/grpc_google_iam_v1-0.14.4-py3-none-any.whl", hash = "sha256:412facc320fcbd94034b4df3d557662051d4d8adfa86e0ddb4dca70a3f739964", size = 32675, upload-time = "2026-04-01T01:57:47.69Z" }, + { url = "https://files.pythonhosted.org/packages/89/22/c2dd50c09bf679bd38173656cd4402d2511e563b33bc88f90009cf50613c/grpc_google_iam_v1-0.14.4-py3-none-any.whl", hash = "sha256:412facc320fcbd94034b4df3d557662051d4d8adfa86e0ddb4dca70a3f739964", size = 32675 }, ] [[package]] @@ -1311,18 +1323,18 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/8a/3d098f35c143a89520e568e6539cc098fcd294495910e359889ce8741c84/grpcio-1.78.0.tar.gz", hash = "sha256:7382b95189546f375c174f53a5fa873cef91c4b8005faa05cc5b3beea9c4f1c5", size = 12852416, upload-time = "2026-02-06T09:57:18.093Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/8a/3d098f35c143a89520e568e6539cc098fcd294495910e359889ce8741c84/grpcio-1.78.0.tar.gz", hash = "sha256:7382b95189546f375c174f53a5fa873cef91c4b8005faa05cc5b3beea9c4f1c5", size = 12852416 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/f4/7384ed0178203d6074446b3c4f46c90a22ddf7ae0b3aee521627f54cfc2a/grpcio-1.78.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:f9ab915a267fc47c7e88c387a3a28325b58c898e23d4995f765728f4e3dedb97", size = 5913985, upload-time = "2026-02-06T09:55:26.832Z" }, - { url = "https://files.pythonhosted.org/packages/81/ed/be1caa25f06594463f685b3790b320f18aea49b33166f4141bfdc2bfb236/grpcio-1.78.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3f8904a8165ab21e07e58bf3e30a73f4dffc7a1e0dbc32d51c61b5360d26f43e", size = 11811853, upload-time = "2026-02-06T09:55:29.224Z" }, - { url = "https://files.pythonhosted.org/packages/24/a7/f06d151afc4e64b7e3cc3e872d331d011c279aaab02831e40a81c691fb65/grpcio-1.78.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:859b13906ce098c0b493af92142ad051bf64c7870fa58a123911c88606714996", size = 6475766, upload-time = "2026-02-06T09:55:31.825Z" }, - { url = "https://files.pythonhosted.org/packages/8a/a8/4482922da832ec0082d0f2cc3a10976d84a7424707f25780b82814aafc0a/grpcio-1.78.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b2342d87af32790f934a79c3112641e7b27d63c261b8b4395350dad43eff1dc7", size = 7170027, upload-time = "2026-02-06T09:55:34.7Z" }, - { url = "https://files.pythonhosted.org/packages/54/bf/f4a3b9693e35d25b24b0b39fa46d7d8a3c439e0a3036c3451764678fec20/grpcio-1.78.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:12a771591ae40bc65ba67048fa52ef4f0e6db8279e595fd349f9dfddeef571f9", size = 6690766, upload-time = "2026-02-06T09:55:36.902Z" }, - { url = "https://files.pythonhosted.org/packages/c7/b9/521875265cc99fe5ad4c5a17010018085cae2810a928bf15ebe7d8bcd9cc/grpcio-1.78.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:185dea0d5260cbb2d224c507bf2a5444d5abbb1fa3594c1ed7e4c709d5eb8383", size = 7266161, upload-time = "2026-02-06T09:55:39.824Z" }, - { url = "https://files.pythonhosted.org/packages/05/86/296a82844fd40a4ad4a95f100b55044b4f817dece732bf686aea1a284147/grpcio-1.78.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:51b13f9aed9d59ee389ad666b8c2214cc87b5de258fa712f9ab05f922e3896c6", size = 8253303, upload-time = "2026-02-06T09:55:42.353Z" }, - { url = "https://files.pythonhosted.org/packages/f3/e4/ea3c0caf5468537f27ad5aab92b681ed7cc0ef5f8c9196d3fd42c8c2286b/grpcio-1.78.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fd5f135b1bd58ab088930b3c613455796dfa0393626a6972663ccdda5b4ac6ce", size = 7698222, upload-time = "2026-02-06T09:55:44.629Z" }, - { url = "https://files.pythonhosted.org/packages/d7/47/7f05f81e4bb6b831e93271fb12fd52ba7b319b5402cbc101d588f435df00/grpcio-1.78.0-cp312-cp312-win32.whl", hash = "sha256:94309f498bcc07e5a7d16089ab984d42ad96af1d94b5a4eb966a266d9fcabf68", size = 4066123, upload-time = "2026-02-06T09:55:47.644Z" }, - { url = "https://files.pythonhosted.org/packages/ad/e7/d6914822c88aa2974dbbd10903d801a28a19ce9cd8bad7e694cbbcf61528/grpcio-1.78.0-cp312-cp312-win_amd64.whl", hash = "sha256:9566fe4ababbb2610c39190791e5b829869351d14369603702e890ef3ad2d06e", size = 4797657, upload-time = "2026-02-06T09:55:49.86Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f4/7384ed0178203d6074446b3c4f46c90a22ddf7ae0b3aee521627f54cfc2a/grpcio-1.78.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:f9ab915a267fc47c7e88c387a3a28325b58c898e23d4995f765728f4e3dedb97", size = 5913985 }, + { url = "https://files.pythonhosted.org/packages/81/ed/be1caa25f06594463f685b3790b320f18aea49b33166f4141bfdc2bfb236/grpcio-1.78.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3f8904a8165ab21e07e58bf3e30a73f4dffc7a1e0dbc32d51c61b5360d26f43e", size = 11811853 }, + { url = "https://files.pythonhosted.org/packages/24/a7/f06d151afc4e64b7e3cc3e872d331d011c279aaab02831e40a81c691fb65/grpcio-1.78.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:859b13906ce098c0b493af92142ad051bf64c7870fa58a123911c88606714996", size = 6475766 }, + { url = "https://files.pythonhosted.org/packages/8a/a8/4482922da832ec0082d0f2cc3a10976d84a7424707f25780b82814aafc0a/grpcio-1.78.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b2342d87af32790f934a79c3112641e7b27d63c261b8b4395350dad43eff1dc7", size = 7170027 }, + { url = "https://files.pythonhosted.org/packages/54/bf/f4a3b9693e35d25b24b0b39fa46d7d8a3c439e0a3036c3451764678fec20/grpcio-1.78.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:12a771591ae40bc65ba67048fa52ef4f0e6db8279e595fd349f9dfddeef571f9", size = 6690766 }, + { url = "https://files.pythonhosted.org/packages/c7/b9/521875265cc99fe5ad4c5a17010018085cae2810a928bf15ebe7d8bcd9cc/grpcio-1.78.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:185dea0d5260cbb2d224c507bf2a5444d5abbb1fa3594c1ed7e4c709d5eb8383", size = 7266161 }, + { url = "https://files.pythonhosted.org/packages/05/86/296a82844fd40a4ad4a95f100b55044b4f817dece732bf686aea1a284147/grpcio-1.78.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:51b13f9aed9d59ee389ad666b8c2214cc87b5de258fa712f9ab05f922e3896c6", size = 8253303 }, + { url = "https://files.pythonhosted.org/packages/f3/e4/ea3c0caf5468537f27ad5aab92b681ed7cc0ef5f8c9196d3fd42c8c2286b/grpcio-1.78.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fd5f135b1bd58ab088930b3c613455796dfa0393626a6972663ccdda5b4ac6ce", size = 7698222 }, + { url = "https://files.pythonhosted.org/packages/d7/47/7f05f81e4bb6b831e93271fb12fd52ba7b319b5402cbc101d588f435df00/grpcio-1.78.0-cp312-cp312-win32.whl", hash = "sha256:94309f498bcc07e5a7d16089ab984d42ad96af1d94b5a4eb966a266d9fcabf68", size = 4066123 }, + { url = "https://files.pythonhosted.org/packages/ad/e7/d6914822c88aa2974dbbd10903d801a28a19ce9cd8bad7e694cbbcf61528/grpcio-1.78.0-cp312-cp312-win_amd64.whl", hash = "sha256:9566fe4ababbb2610c39190791e5b829869351d14369603702e890ef3ad2d06e", size = 4797657 }, ] [[package]] @@ -1334,9 +1346,9 @@ dependencies = [ { name = "grpcio" }, { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7c/d7/013ef01c5a1c2fd0932c27c904934162f69f41ca0f28396d3ffe4d386123/grpcio-status-1.62.3.tar.gz", hash = "sha256:289bdd7b2459794a12cf95dc0cb727bd4a1742c37bd823f760236c937e53a485", size = 13063, upload-time = "2024-08-06T00:37:08.003Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/d7/013ef01c5a1c2fd0932c27c904934162f69f41ca0f28396d3ffe4d386123/grpcio-status-1.62.3.tar.gz", hash = "sha256:289bdd7b2459794a12cf95dc0cb727bd4a1742c37bd823f760236c937e53a485", size = 13063 } wheels = [ - { url = "https://files.pythonhosted.org/packages/90/40/972271de05f9315c0d69f9f7ebbcadd83bc85322f538637d11bb8c67803d/grpcio_status-1.62.3-py3-none-any.whl", hash = "sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8", size = 14448, upload-time = "2024-08-06T00:30:15.702Z" }, + { url = "https://files.pythonhosted.org/packages/90/40/972271de05f9315c0d69f9f7ebbcadd83bc85322f538637d11bb8c67803d/grpcio_status-1.62.3-py3-none-any.whl", hash = "sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8", size = 14448 }, ] [[package]] @@ -1348,25 +1360,25 @@ dependencies = [ { name = "protobuf" }, { name = "setuptools" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/54/fa/b69bd8040eafc09b88bb0ec0fea59e8aacd1a801e688af087cead213b0d0/grpcio-tools-1.62.3.tar.gz", hash = "sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833", size = 4538520, upload-time = "2024-08-06T00:37:11.035Z" } +sdist = { url = "https://files.pythonhosted.org/packages/54/fa/b69bd8040eafc09b88bb0ec0fea59e8aacd1a801e688af087cead213b0d0/grpcio-tools-1.62.3.tar.gz", hash = "sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833", size = 4538520 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/a5/d6887eba415ce318ae5005e8dfac3fa74892400b54b6d37b79e8b4f14f5e/grpcio_tools-1.62.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5", size = 5147690, upload-time = "2024-08-06T00:31:16.436Z" }, - { url = "https://files.pythonhosted.org/packages/8a/7c/3cde447a045e83ceb4b570af8afe67ffc86896a2fe7f59594dc8e5d0a645/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133", size = 2720538, upload-time = "2024-08-06T00:31:18.905Z" }, - { url = "https://files.pythonhosted.org/packages/88/07/f83f2750d44ac4f06c07c37395b9c1383ef5c994745f73c6bfaf767f0944/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa", size = 3071571, upload-time = "2024-08-06T00:31:21.684Z" }, - { url = "https://files.pythonhosted.org/packages/37/74/40175897deb61e54aca716bc2e8919155b48f33aafec8043dda9592d8768/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0", size = 2806207, upload-time = "2024-08-06T00:31:24.208Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ee/d8de915105a217cbcb9084d684abdc032030dcd887277f2ef167372287fe/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d", size = 3685815, upload-time = "2024-08-06T00:31:26.917Z" }, - { url = "https://files.pythonhosted.org/packages/fd/d9/4360a6c12be3d7521b0b8c39e5d3801d622fbb81cc2721dbd3eee31e28c8/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc", size = 3298378, upload-time = "2024-08-06T00:31:30.401Z" }, - { url = "https://files.pythonhosted.org/packages/29/3b/7cdf4a9e5a3e0a35a528b48b111355cd14da601413a4f887aa99b6da468f/grpcio_tools-1.62.3-cp312-cp312-win32.whl", hash = "sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b", size = 910416, upload-time = "2024-08-06T00:31:33.118Z" }, - { url = "https://files.pythonhosted.org/packages/6c/66/dd3ec249e44c1cc15e902e783747819ed41ead1336fcba72bf841f72c6e9/grpcio_tools-1.62.3-cp312-cp312-win_amd64.whl", hash = "sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7", size = 1052856, upload-time = "2024-08-06T00:31:36.519Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a5/d6887eba415ce318ae5005e8dfac3fa74892400b54b6d37b79e8b4f14f5e/grpcio_tools-1.62.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5", size = 5147690 }, + { url = "https://files.pythonhosted.org/packages/8a/7c/3cde447a045e83ceb4b570af8afe67ffc86896a2fe7f59594dc8e5d0a645/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133", size = 2720538 }, + { url = "https://files.pythonhosted.org/packages/88/07/f83f2750d44ac4f06c07c37395b9c1383ef5c994745f73c6bfaf767f0944/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa", size = 3071571 }, + { url = "https://files.pythonhosted.org/packages/37/74/40175897deb61e54aca716bc2e8919155b48f33aafec8043dda9592d8768/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0", size = 2806207 }, + { url = "https://files.pythonhosted.org/packages/ec/ee/d8de915105a217cbcb9084d684abdc032030dcd887277f2ef167372287fe/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d", size = 3685815 }, + { url = "https://files.pythonhosted.org/packages/fd/d9/4360a6c12be3d7521b0b8c39e5d3801d622fbb81cc2721dbd3eee31e28c8/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc", size = 3298378 }, + { url = "https://files.pythonhosted.org/packages/29/3b/7cdf4a9e5a3e0a35a528b48b111355cd14da601413a4f887aa99b6da468f/grpcio_tools-1.62.3-cp312-cp312-win32.whl", hash = "sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b", size = 910416 }, + { url = "https://files.pythonhosted.org/packages/6c/66/dd3ec249e44c1cc15e902e783747819ed41ead1336fcba72bf841f72c6e9/grpcio_tools-1.62.3-cp312-cp312-win_amd64.whl", hash = "sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7", size = 1052856 }, ] [[package]] name = "h11" version = "0.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250 } wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515 }, ] [[package]] @@ -1377,34 +1389,34 @@ dependencies = [ { name = "hpack" }, { name = "hyperframe" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026 } wheels = [ - { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, + { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779 }, ] [[package]] name = "hf-xet" version = "1.5.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/d8/5c06fc76461418326a7decf8367480c35be11a41fd938633929c60a9ec6b/hf_xet-1.5.0.tar.gz", hash = "sha256:e0fb0a34d9f406eed88233e829a67ec016bec5af19e480eac65a233ea289a948", size = 837196, upload-time = "2026-05-06T06:18:15.583Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/d8/5c06fc76461418326a7decf8367480c35be11a41fd938633929c60a9ec6b/hf_xet-1.5.0.tar.gz", hash = "sha256:e0fb0a34d9f406eed88233e829a67ec016bec5af19e480eac65a233ea289a948", size = 837196 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/fb/69ff198a82cae7eb1a69fb84d93b3a3e4816564d76817fe541ddc96874eb/hf_xet-1.5.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dad0dc84e941b8ba3c860659fe1fdc35c049d47cce293f003287757e971a8f56", size = 4030814, upload-time = "2026-05-06T06:17:57.933Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ff/edcc2b40162bef3ff78e14ab637e5f3b89243d6aee72f5949d3bb6a5af83/hf_xet-1.5.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a", size = 3798444, upload-time = "2026-05-06T06:17:55.79Z" }, - { url = "https://files.pythonhosted.org/packages/49/4d/103f76b04310e5e57656696cc184690d20c466af0bca3ca88f8c8ea5d4f3/hf_xet-1.5.0-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3531b1823a0e6d77d80f9ed15ca0e00f0d115094f8ac033d5cae88f4564cc949", size = 4465986, upload-time = "2026-05-06T06:17:44.886Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a2/546f47f464737b3edbab6f8ddb57f2599b93d2cbb66f06abb475ccb48651/hf_xet-1.5.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9a0ee58cd18d5ea799f7ed11290bbccbe56bdd8b1d97ca74b9cc49a3945d7a3b", size = 4259865, upload-time = "2026-05-06T06:17:42.639Z" }, - { url = "https://files.pythonhosted.org/packages/95/7f/1be593c1f28613be2e196473481cd81bfc5910795e30a34e8f744f6cac4f/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18", size = 4459835, upload-time = "2026-05-06T06:18:08.026Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b2/703569fc881f3284487e68cda7b42179978480da3c438042a6bbbb4a671c/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b35549ce62601b84da4ff9b24d970032ace3d4430f52d91bcbb26c901d6c690", size = 4672414, upload-time = "2026-05-06T06:18:09.864Z" }, - { url = "https://files.pythonhosted.org/packages/af/37/1b6def445c567286b50aa3b33828158e135b1be44938dde59f11382a500c/hf_xet-1.5.0-cp37-abi3-win_amd64.whl", hash = "sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4", size = 3977238, upload-time = "2026-05-06T06:18:23.621Z" }, - { url = "https://files.pythonhosted.org/packages/62/94/3b66b148778ee100dcfd69c2ca22b57b41b44d3063ceec934f209e9184ce/hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be", size = 3806916, upload-time = "2026-05-06T06:18:21.7Z" }, + { url = "https://files.pythonhosted.org/packages/3d/fb/69ff198a82cae7eb1a69fb84d93b3a3e4816564d76817fe541ddc96874eb/hf_xet-1.5.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dad0dc84e941b8ba3c860659fe1fdc35c049d47cce293f003287757e971a8f56", size = 4030814 }, + { url = "https://files.pythonhosted.org/packages/9b/ff/edcc2b40162bef3ff78e14ab637e5f3b89243d6aee72f5949d3bb6a5af83/hf_xet-1.5.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a", size = 3798444 }, + { url = "https://files.pythonhosted.org/packages/49/4d/103f76b04310e5e57656696cc184690d20c466af0bca3ca88f8c8ea5d4f3/hf_xet-1.5.0-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3531b1823a0e6d77d80f9ed15ca0e00f0d115094f8ac033d5cae88f4564cc949", size = 4465986 }, + { url = "https://files.pythonhosted.org/packages/c4/a2/546f47f464737b3edbab6f8ddb57f2599b93d2cbb66f06abb475ccb48651/hf_xet-1.5.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9a0ee58cd18d5ea799f7ed11290bbccbe56bdd8b1d97ca74b9cc49a3945d7a3b", size = 4259865 }, + { url = "https://files.pythonhosted.org/packages/95/7f/1be593c1f28613be2e196473481cd81bfc5910795e30a34e8f744f6cac4f/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18", size = 4459835 }, + { url = "https://files.pythonhosted.org/packages/aa/b2/703569fc881f3284487e68cda7b42179978480da3c438042a6bbbb4a671c/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b35549ce62601b84da4ff9b24d970032ace3d4430f52d91bcbb26c901d6c690", size = 4672414 }, + { url = "https://files.pythonhosted.org/packages/af/37/1b6def445c567286b50aa3b33828158e135b1be44938dde59f11382a500c/hf_xet-1.5.0-cp37-abi3-win_amd64.whl", hash = "sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4", size = 3977238 }, + { url = "https://files.pythonhosted.org/packages/62/94/3b66b148778ee100dcfd69c2ca22b57b41b44d3063ceec934f209e9184ce/hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be", size = 3806916 }, ] [[package]] name = "hpack" version = "4.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276 } wheels = [ - { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" }, + { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357 }, ] [[package]] @@ -1415,9 +1427,9 @@ dependencies = [ { name = "certifi" }, { name = "h11" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484 } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784 }, ] [[package]] @@ -1427,9 +1439,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyparsing" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c1/1f/e86365613582c027dda5ddb64e1010e57a3d53e99ab8a72093fa13d565ec/httplib2-0.31.2.tar.gz", hash = "sha256:385e0869d7397484f4eab426197a4c020b606edd43372492337c0b4010ae5d24", size = 250800, upload-time = "2026-01-23T11:04:44.165Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c1/1f/e86365613582c027dda5ddb64e1010e57a3d53e99ab8a72093fa13d565ec/httplib2-0.31.2.tar.gz", hash = "sha256:385e0869d7397484f4eab426197a4c020b606edd43372492337c0b4010ae5d24", size = 250800 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/90/fd509079dfcab01102c0fdd87f3a9506894bc70afcf9e9785ef6b2b3aff6/httplib2-0.31.2-py3-none-any.whl", hash = "sha256:dbf0c2fa3862acf3c55c078ea9c0bc4481d7dc5117cae71be9514912cf9f8349", size = 91099, upload-time = "2026-01-23T11:04:42.78Z" }, + { url = "https://files.pythonhosted.org/packages/2f/90/fd509079dfcab01102c0fdd87f3a9506894bc70afcf9e9785ef6b2b3aff6/httplib2-0.31.2-py3-none-any.whl", hash = "sha256:dbf0c2fa3862acf3c55c078ea9c0bc4481d7dc5117cae71be9514912cf9f8349", size = 91099 }, ] [[package]] @@ -1442,9 +1454,9 @@ dependencies = [ { name = "httpcore" }, { name = "idna" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 }, ] [package.optional-dependencies] @@ -1467,36 +1479,36 @@ dependencies = [ { name = "typer" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/b6/e22bd20a25299c34b8c5922c1545a6320825b13906eb0f7298edfd034a0b/huggingface_hub-1.15.0.tar.gz", hash = "sha256:28abfdddda3927fd4de6a63cf26ab012498a2c24dae52baf150c5c6edf98a1d5", size = 784100, upload-time = "2026-05-15T11:42:52.149Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/b6/e22bd20a25299c34b8c5922c1545a6320825b13906eb0f7298edfd034a0b/huggingface_hub-1.15.0.tar.gz", hash = "sha256:28abfdddda3927fd4de6a63cf26ab012498a2c24dae52baf150c5c6edf98a1d5", size = 784100 } wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/11/0b64cc9024329b76d7547c19a67604a61d21d3ba678a69d1b220c29d5112/huggingface_hub-1.15.0-py3-none-any.whl", hash = "sha256:a4a59af04cbc41a3fe3fec429b171ef994ef8c971eda10136746f408dd4e3744", size = 663602, upload-time = "2026-05-15T11:42:50.487Z" }, + { url = "https://files.pythonhosted.org/packages/6e/11/0b64cc9024329b76d7547c19a67604a61d21d3ba678a69d1b220c29d5112/huggingface_hub-1.15.0-py3-none-any.whl", hash = "sha256:a4a59af04cbc41a3fe3fec429b171ef994ef8c971eda10136746f408dd4e3744", size = 663602 }, ] [[package]] name = "hyperframe" version = "6.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566 } wheels = [ - { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007 }, ] [[package]] name = "identify" version = "2.6.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/63/51723b5f116cc04b061cb6f5a561790abf249d25931d515cd375e063e0f4/identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842", size = 99567, upload-time = "2026-04-17T18:39:50.265Z" } +sdist = { url = "https://files.pythonhosted.org/packages/52/63/51723b5f116cc04b061cb6f5a561790abf249d25931d515cd375e063e0f4/identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842", size = 99567 } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a", size = 99397, upload-time = "2026-04-17T18:39:49.221Z" }, + { url = "https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a", size = 99397 }, ] [[package]] name = "idna" version = "3.15" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245 } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, + { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340 }, ] [[package]] @@ -1506,45 +1518,45 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "zipp" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", size = 55304, upload-time = "2024-09-11T14:56:08.937Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", size = 55304 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514, upload-time = "2024-09-11T14:56:07.019Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514 }, ] [[package]] name = "inflection" version = "0.5.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e1/7e/691d061b7329bc8d54edbf0ec22fbfb2afe61facb681f9aaa9bff7a27d04/inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417", size = 15091, upload-time = "2020-08-22T08:16:29.139Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/7e/691d061b7329bc8d54edbf0ec22fbfb2afe61facb681f9aaa9bff7a27d04/inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417", size = 15091 } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/91/aa6bde563e0085a02a435aa99b49ef75b0a4b062635e606dab23ce18d720/inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2", size = 9454, upload-time = "2020-08-22T08:16:27.816Z" }, + { url = "https://files.pythonhosted.org/packages/59/91/aa6bde563e0085a02a435aa99b49ef75b0a4b062635e606dab23ce18d720/inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2", size = 9454 }, ] [[package]] name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503 } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484 }, ] [[package]] name = "invoke" version = "3.0.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/33/f6/227c48c5fe47fa178ccf1fda8f047d16c97ba926567b661e9ce2045c600c/invoke-3.0.3.tar.gz", hash = "sha256:437b6a622223824380bfb4e64f612711a6b648c795f565efc8625af66fb57f0c", size = 343419, upload-time = "2026-04-07T15:17:48.307Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/f6/227c48c5fe47fa178ccf1fda8f047d16c97ba926567b661e9ce2045c600c/invoke-3.0.3.tar.gz", hash = "sha256:437b6a622223824380bfb4e64f612711a6b648c795f565efc8625af66fb57f0c", size = 343419 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/de/bbc12563bbf979618d17625a4e753ff7a078523e28d870d3626daa97261a/invoke-3.0.3-py3-none-any.whl", hash = "sha256:f11327165e5cbb89b2ad1d88d3292b5113332c43b8553b494da435d6ec6f5053", size = 160958, upload-time = "2026-04-07T15:17:46.875Z" }, + { url = "https://files.pythonhosted.org/packages/5a/de/bbc12563bbf979618d17625a4e753ff7a078523e28d870d3626daa97261a/invoke-3.0.3-py3-none-any.whl", hash = "sha256:f11327165e5cbb89b2ad1d88d3292b5113332c43b8553b494da435d6ec6f5053", size = 160958 }, ] [[package]] name = "isodate" version = "0.7.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } +sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705 } wheels = [ - { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, + { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320 }, ] [[package]] @@ -1554,53 +1566,53 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markupsafe" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115 } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899 }, ] [[package]] name = "jiter" version = "0.14.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6e/c1/0cddc6eb17d4c53a99840953f95dd3accdc5cfc7a337b0e9b26476276be9/jiter-0.14.0.tar.gz", hash = "sha256:e8a39e66dac7153cf3f964a12aad515afa8d74938ec5cc0018adcdae5367c79e", size = 165725, upload-time = "2026-04-10T14:28:42.01Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/68/7390a418f10897da93b158f2d5a8bd0bcd73a0f9ec3bb36917085bb759ef/jiter-0.14.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:2fb2ce3a7bc331256dfb14cefc34832366bb28a9aca81deaf43bbf2a5659e607", size = 316295, upload-time = "2026-04-10T14:26:24.887Z" }, - { url = "https://files.pythonhosted.org/packages/60/a0/5854ac00ff63551c52c6c89534ec6aba4b93474e7924d64e860b1c94165b/jiter-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5252a7ca23785cef5d02d4ece6077a1b556a410c591b379f82091c3001e14844", size = 315898, upload-time = "2026-04-10T14:26:26.601Z" }, - { url = "https://files.pythonhosted.org/packages/41/a1/4f44832650a16b18e8391f1bf1d6ca4909bc738351826bcc198bba4357f4/jiter-0.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c409578cbd77c338975670ada777add4efd53379667edf0aceea730cabede6fb", size = 343730, upload-time = "2026-04-10T14:26:28.326Z" }, - { url = "https://files.pythonhosted.org/packages/48/64/a329e9d469f86307203594b1707e11ae51c3348d03bfd514a5f997870012/jiter-0.14.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7ede4331a1899d604463369c730dbb961ffdc5312bc7f16c41c2896415b1304a", size = 370102, upload-time = "2026-04-10T14:26:30.089Z" }, - { url = "https://files.pythonhosted.org/packages/94/c1/5e3dfc59635aa4d4c7bd20a820ac1d09b8ed851568356802cf1c08edb3cf/jiter-0.14.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92cd8b6025981a041f5310430310b55b25ca593972c16407af8837d3d7d2ca01", size = 461335, upload-time = "2026-04-10T14:26:31.911Z" }, - { url = "https://files.pythonhosted.org/packages/e3/1b/dd157009dbc058f7b00108f545ccb72a2d56461395c4fc7b9cfdccb00af4/jiter-0.14.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:351bf6eda4e3a7ceb876377840c702e9a3e4ecc4624dbfb2d6463c67ae52637d", size = 378536, upload-time = "2026-04-10T14:26:33.595Z" }, - { url = "https://files.pythonhosted.org/packages/91/78/256013667b7c10b8834f8e6e54cd3e562d4c6e34227a1596addccc05e38c/jiter-0.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1dcfbeb93d9ecd9ca128bbf8910120367777973fa193fb9a39c31237d8df165", size = 353859, upload-time = "2026-04-10T14:26:35.098Z" }, - { url = "https://files.pythonhosted.org/packages/de/d9/137d65ade9093a409fe80955ce60b12bb753722c986467aeda47faf450ad/jiter-0.14.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:ae039aaef8de3f8157ecc1fdd4d85043ac4f57538c245a0afaecb8321ec951c3", size = 357626, upload-time = "2026-04-10T14:26:36.685Z" }, - { url = "https://files.pythonhosted.org/packages/2e/48/76750835b87029342727c1a268bea8878ab988caf81ee4e7b880900eeb5a/jiter-0.14.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7d9d51eb96c82a9652933bd769fe6de66877d6eb2b2440e281f2938c51b5643e", size = 393172, upload-time = "2026-04-10T14:26:38.097Z" }, - { url = "https://files.pythonhosted.org/packages/a6/60/456c4e81d5c8045279aefe60e9e483be08793828800a4e64add8fdde7f2a/jiter-0.14.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d824ca4148b705970bf4e120924a212fdfca9859a73e42bd7889a63a4ea6bb98", size = 520300, upload-time = "2026-04-10T14:26:39.532Z" }, - { url = "https://files.pythonhosted.org/packages/a8/9f/2020e0984c235f678dced38fe4eec3058cf528e6af36ebf969b410305941/jiter-0.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff3a6465b3a0f54b1a430f45c3c0ba7d61ceb45cbc3e33f9e1a7f638d690baf3", size = 553059, upload-time = "2026-04-10T14:26:40.991Z" }, - { url = "https://files.pythonhosted.org/packages/ef/32/e2d298e1a22a4bbe6062136d1c7192db7dba003a6975e51d9a9eecabc4c2/jiter-0.14.0-cp312-cp312-win32.whl", hash = "sha256:5dec7c0a3e98d2a3f8a2e67382d0d7c3ac60c69103a4b271da889b4e8bb1e129", size = 206030, upload-time = "2026-04-10T14:26:42.517Z" }, - { url = "https://files.pythonhosted.org/packages/36/ac/96369141b3d8a4a8e4590e983085efe1c436f35c0cda940dd76d942e3e40/jiter-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:fc7e37b4b8bc7e80a63ad6cfa5fc11fab27dbfea4cc4ae644b1ab3f273dc348f", size = 201603, upload-time = "2026-04-10T14:26:44.328Z" }, - { url = "https://files.pythonhosted.org/packages/01/c3/75d847f264647017d7e3052bbcc8b1e24b95fa139c320c5f5066fa7a0bdd/jiter-0.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:ee4a72f12847ef29b072aee9ad5474041ab2924106bdca9fcf5d7d965853e057", size = 191525, upload-time = "2026-04-10T14:26:46Z" }, - { url = "https://files.pythonhosted.org/packages/21/42/9042c3f3019de4adcb8c16591c325ec7255beea9fcd33a42a43f3b0b1000/jiter-0.14.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:fbd9e482663ca9d005d051330e4d2d8150bb208a209409c10f7e7dfdf7c49da9", size = 308810, upload-time = "2026-04-10T14:28:34.673Z" }, - { url = "https://files.pythonhosted.org/packages/60/cf/a7e19b308bd86bb04776803b1f01a5f9a287a4c55205f4708827ee487fbf/jiter-0.14.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:33a20d838b91ef376b3a56896d5b04e725c7df5bc4864cc6569cf046a8d73b6d", size = 308443, upload-time = "2026-04-10T14:28:36.658Z" }, - { url = "https://files.pythonhosted.org/packages/ca/44/e26ede3f0caeff93f222559cb0cc4ca68579f07d009d7b6010c5b586f9b1/jiter-0.14.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:432c4db5255d86a259efde91e55cb4c8d18c0521d844c9e2e7efcce3899fb016", size = 343039, upload-time = "2026-04-10T14:28:38.356Z" }, - { url = "https://files.pythonhosted.org/packages/da/e9/1f9ada30cef7b05e74bb06f52127e7a724976c225f46adb65c37b1dadfb6/jiter-0.14.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67f00d94b281174144d6532a04b66a12cb866cbdc47c3af3bfe2973677f9861a", size = 349613, upload-time = "2026-04-10T14:28:40.066Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/6e/c1/0cddc6eb17d4c53a99840953f95dd3accdc5cfc7a337b0e9b26476276be9/jiter-0.14.0.tar.gz", hash = "sha256:e8a39e66dac7153cf3f964a12aad515afa8d74938ec5cc0018adcdae5367c79e", size = 165725 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/68/7390a418f10897da93b158f2d5a8bd0bcd73a0f9ec3bb36917085bb759ef/jiter-0.14.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:2fb2ce3a7bc331256dfb14cefc34832366bb28a9aca81deaf43bbf2a5659e607", size = 316295 }, + { url = "https://files.pythonhosted.org/packages/60/a0/5854ac00ff63551c52c6c89534ec6aba4b93474e7924d64e860b1c94165b/jiter-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5252a7ca23785cef5d02d4ece6077a1b556a410c591b379f82091c3001e14844", size = 315898 }, + { url = "https://files.pythonhosted.org/packages/41/a1/4f44832650a16b18e8391f1bf1d6ca4909bc738351826bcc198bba4357f4/jiter-0.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c409578cbd77c338975670ada777add4efd53379667edf0aceea730cabede6fb", size = 343730 }, + { url = "https://files.pythonhosted.org/packages/48/64/a329e9d469f86307203594b1707e11ae51c3348d03bfd514a5f997870012/jiter-0.14.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7ede4331a1899d604463369c730dbb961ffdc5312bc7f16c41c2896415b1304a", size = 370102 }, + { url = "https://files.pythonhosted.org/packages/94/c1/5e3dfc59635aa4d4c7bd20a820ac1d09b8ed851568356802cf1c08edb3cf/jiter-0.14.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92cd8b6025981a041f5310430310b55b25ca593972c16407af8837d3d7d2ca01", size = 461335 }, + { url = "https://files.pythonhosted.org/packages/e3/1b/dd157009dbc058f7b00108f545ccb72a2d56461395c4fc7b9cfdccb00af4/jiter-0.14.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:351bf6eda4e3a7ceb876377840c702e9a3e4ecc4624dbfb2d6463c67ae52637d", size = 378536 }, + { url = "https://files.pythonhosted.org/packages/91/78/256013667b7c10b8834f8e6e54cd3e562d4c6e34227a1596addccc05e38c/jiter-0.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1dcfbeb93d9ecd9ca128bbf8910120367777973fa193fb9a39c31237d8df165", size = 353859 }, + { url = "https://files.pythonhosted.org/packages/de/d9/137d65ade9093a409fe80955ce60b12bb753722c986467aeda47faf450ad/jiter-0.14.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:ae039aaef8de3f8157ecc1fdd4d85043ac4f57538c245a0afaecb8321ec951c3", size = 357626 }, + { url = "https://files.pythonhosted.org/packages/2e/48/76750835b87029342727c1a268bea8878ab988caf81ee4e7b880900eeb5a/jiter-0.14.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7d9d51eb96c82a9652933bd769fe6de66877d6eb2b2440e281f2938c51b5643e", size = 393172 }, + { url = "https://files.pythonhosted.org/packages/a6/60/456c4e81d5c8045279aefe60e9e483be08793828800a4e64add8fdde7f2a/jiter-0.14.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d824ca4148b705970bf4e120924a212fdfca9859a73e42bd7889a63a4ea6bb98", size = 520300 }, + { url = "https://files.pythonhosted.org/packages/a8/9f/2020e0984c235f678dced38fe4eec3058cf528e6af36ebf969b410305941/jiter-0.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff3a6465b3a0f54b1a430f45c3c0ba7d61ceb45cbc3e33f9e1a7f638d690baf3", size = 553059 }, + { url = "https://files.pythonhosted.org/packages/ef/32/e2d298e1a22a4bbe6062136d1c7192db7dba003a6975e51d9a9eecabc4c2/jiter-0.14.0-cp312-cp312-win32.whl", hash = "sha256:5dec7c0a3e98d2a3f8a2e67382d0d7c3ac60c69103a4b271da889b4e8bb1e129", size = 206030 }, + { url = "https://files.pythonhosted.org/packages/36/ac/96369141b3d8a4a8e4590e983085efe1c436f35c0cda940dd76d942e3e40/jiter-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:fc7e37b4b8bc7e80a63ad6cfa5fc11fab27dbfea4cc4ae644b1ab3f273dc348f", size = 201603 }, + { url = "https://files.pythonhosted.org/packages/01/c3/75d847f264647017d7e3052bbcc8b1e24b95fa139c320c5f5066fa7a0bdd/jiter-0.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:ee4a72f12847ef29b072aee9ad5474041ab2924106bdca9fcf5d7d965853e057", size = 191525 }, + { url = "https://files.pythonhosted.org/packages/21/42/9042c3f3019de4adcb8c16591c325ec7255beea9fcd33a42a43f3b0b1000/jiter-0.14.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:fbd9e482663ca9d005d051330e4d2d8150bb208a209409c10f7e7dfdf7c49da9", size = 308810 }, + { url = "https://files.pythonhosted.org/packages/60/cf/a7e19b308bd86bb04776803b1f01a5f9a287a4c55205f4708827ee487fbf/jiter-0.14.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:33a20d838b91ef376b3a56896d5b04e725c7df5bc4864cc6569cf046a8d73b6d", size = 308443 }, + { url = "https://files.pythonhosted.org/packages/ca/44/e26ede3f0caeff93f222559cb0cc4ca68579f07d009d7b6010c5b586f9b1/jiter-0.14.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:432c4db5255d86a259efde91e55cb4c8d18c0521d844c9e2e7efcce3899fb016", size = 343039 }, + { url = "https://files.pythonhosted.org/packages/da/e9/1f9ada30cef7b05e74bb06f52127e7a724976c225f46adb65c37b1dadfb6/jiter-0.14.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67f00d94b281174144d6532a04b66a12cb866cbdc47c3af3bfe2973677f9861a", size = 349613 }, ] [[package]] name = "jmespath" version = "1.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377 } wheels = [ - { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419 }, ] [[package]] name = "joblib" version = "1.5.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603 } wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071 }, ] [[package]] @@ -1610,9 +1622,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3b/dc/5f768c2e391e9afabe5d18e3221346deb5fb6338565f1ccc9e7c6d7befdd/joserfc-1.6.5.tar.gz", hash = "sha256:1482a7db78fb4602e44ed89e51b599d052e091288c7c532c5b694e20149dec48", size = 231881, upload-time = "2026-05-06T04:58:13.408Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/dc/5f768c2e391e9afabe5d18e3221346deb5fb6338565f1ccc9e7c6d7befdd/joserfc-1.6.5.tar.gz", hash = "sha256:1482a7db78fb4602e44ed89e51b599d052e091288c7c532c5b694e20149dec48", size = 231881 } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/3b/ad1cb22e75c963b1f07c8a2329bf47227ce7e4361df5eb2fb101b2ce33ef/joserfc-1.6.5-py3-none-any.whl", hash = "sha256:e9878a0f8243fe7b95e11fdda81374ca9f7a689e302751579d3dfdeec559675e", size = 70464, upload-time = "2026-05-06T04:58:11.668Z" }, + { url = "https://files.pythonhosted.org/packages/54/3b/ad1cb22e75c963b1f07c8a2329bf47227ce7e4361df5eb2fb101b2ce33ef/joserfc-1.6.5-py3-none-any.whl", hash = "sha256:e9878a0f8243fe7b95e11fdda81374ca9f7a689e302751579d3dfdeec559675e", size = 70464 }, ] [[package]] @@ -1625,9 +1637,9 @@ dependencies = [ { name = "referencing" }, { name = "rpds-py" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/2e/03362ee4034a4c917f697890ccd4aec0800ccf9ded7f511971c75451deec/jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4", size = 325778, upload-time = "2024-07-08T18:40:05.546Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/2e/03362ee4034a4c917f697890ccd4aec0800ccf9ded7f511971c75451deec/jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4", size = 325778 } wheels = [ - { url = "https://files.pythonhosted.org/packages/69/4a/4f9dbeb84e8850557c02365a0eee0649abe5eb1d84af92a25731c6c0f922/jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566", size = 88462, upload-time = "2024-07-08T18:40:00.165Z" }, + { url = "https://files.pythonhosted.org/packages/69/4a/4f9dbeb84e8850557c02365a0eee0649abe5eb1d84af92a25731c6c0f922/jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566", size = 88462 }, ] [[package]] @@ -1637,9 +1649,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "referencing" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855 } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437 }, ] [[package]] @@ -1652,9 +1664,9 @@ dependencies = [ { name = "tzdata" }, { name = "vine" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0f/d3/5ff936d8319ac86b9c409f1501b07c426e6ad41966fedace9ef1b966e23f/kombu-5.5.4.tar.gz", hash = "sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363", size = 461992, upload-time = "2025-06-01T10:19:22.281Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d3/5ff936d8319ac86b9c409f1501b07c426e6ad41966fedace9ef1b966e23f/kombu-5.5.4.tar.gz", hash = "sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363", size = 461992 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/70/a07dcf4f62598c8ad579df241af55ced65bed76e42e45d3c368a6d82dbc1/kombu-5.5.4-py3-none-any.whl", hash = "sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8", size = 210034, upload-time = "2025-06-01T10:19:20.436Z" }, + { url = "https://files.pythonhosted.org/packages/ef/70/a07dcf4f62598c8ad579df241af55ced65bed76e42e45d3c368a6d82dbc1/kombu-5.5.4-py3-none-any.whl", hash = "sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8", size = 210034 }, ] [[package]] @@ -1664,16 +1676,16 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/de/cd/337df968b38d94c5aabd3e1b10630f047a2b345f6e1d4456bd9fe7417537/libcst-1.8.6.tar.gz", hash = "sha256:f729c37c9317126da9475bdd06a7208eb52fcbd180a6341648b45a56b4ba708b", size = 891354, upload-time = "2025-11-03T22:33:30.621Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/cd/337df968b38d94c5aabd3e1b10630f047a2b345f6e1d4456bd9fe7417537/libcst-1.8.6.tar.gz", hash = "sha256:f729c37c9317126da9475bdd06a7208eb52fcbd180a6341648b45a56b4ba708b", size = 891354 } wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/3c/93365c17da3d42b055a8edb0e1e99f1c60c776471db6c9b7f1ddf6a44b28/libcst-1.8.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0c13d5bd3d8414a129e9dccaf0e5785108a4441e9b266e1e5e9d1f82d1b943c9", size = 2206166, upload-time = "2025-11-03T22:32:16.012Z" }, - { url = "https://files.pythonhosted.org/packages/1d/cb/7530940e6ac50c6dd6022349721074e19309eb6aa296e942ede2213c1a19/libcst-1.8.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1472eeafd67cdb22544e59cf3bfc25d23dc94058a68cf41f6654ff4fcb92e09", size = 2083726, upload-time = "2025-11-03T22:32:17.312Z" }, - { url = "https://files.pythonhosted.org/packages/1b/cf/7e5eaa8c8f2c54913160671575351d129170db757bb5e4b7faffed022271/libcst-1.8.6-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:089c58e75cb142ec33738a1a4ea7760a28b40c078ab2fd26b270dac7d2633a4d", size = 2235755, upload-time = "2025-11-03T22:32:18.859Z" }, - { url = "https://files.pythonhosted.org/packages/55/54/570ec2b0e9a3de0af9922e3bb1b69a5429beefbc753a7ea770a27ad308bd/libcst-1.8.6-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c9d7aeafb1b07d25a964b148c0dda9451efb47bbbf67756e16eeae65004b0eb5", size = 2301473, upload-time = "2025-11-03T22:32:20.499Z" }, - { url = "https://files.pythonhosted.org/packages/11/4c/163457d1717cd12181c421a4cca493454bcabd143fc7e53313bc6a4ad82a/libcst-1.8.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:207481197afd328aa91d02670c15b48d0256e676ce1ad4bafb6dc2b593cc58f1", size = 2298899, upload-time = "2025-11-03T22:32:21.765Z" }, - { url = "https://files.pythonhosted.org/packages/35/1d/317ddef3669883619ef3d3395ea583305f353ef4ad87d7a5ac1c39be38e3/libcst-1.8.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:375965f34cc6f09f5f809244d3ff9bd4f6cb6699f571121cebce53622e7e0b86", size = 2408239, upload-time = "2025-11-03T22:32:23.275Z" }, - { url = "https://files.pythonhosted.org/packages/9a/a1/f47d8cccf74e212dd6044b9d6dbc223636508da99acff1d54786653196bc/libcst-1.8.6-cp312-cp312-win_amd64.whl", hash = "sha256:da95b38693b989eaa8d32e452e8261cfa77fe5babfef1d8d2ac25af8c4aa7e6d", size = 2119660, upload-time = "2025-11-03T22:32:24.822Z" }, - { url = "https://files.pythonhosted.org/packages/19/d0/dd313bf6a7942cdf951828f07ecc1a7695263f385065edc75ef3016a3cb5/libcst-1.8.6-cp312-cp312-win_arm64.whl", hash = "sha256:bff00e1c766658adbd09a175267f8b2f7616e5ee70ce45db3d7c4ce6d9f6bec7", size = 1999824, upload-time = "2025-11-03T22:32:26.131Z" }, + { url = "https://files.pythonhosted.org/packages/0c/3c/93365c17da3d42b055a8edb0e1e99f1c60c776471db6c9b7f1ddf6a44b28/libcst-1.8.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0c13d5bd3d8414a129e9dccaf0e5785108a4441e9b266e1e5e9d1f82d1b943c9", size = 2206166 }, + { url = "https://files.pythonhosted.org/packages/1d/cb/7530940e6ac50c6dd6022349721074e19309eb6aa296e942ede2213c1a19/libcst-1.8.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1472eeafd67cdb22544e59cf3bfc25d23dc94058a68cf41f6654ff4fcb92e09", size = 2083726 }, + { url = "https://files.pythonhosted.org/packages/1b/cf/7e5eaa8c8f2c54913160671575351d129170db757bb5e4b7faffed022271/libcst-1.8.6-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:089c58e75cb142ec33738a1a4ea7760a28b40c078ab2fd26b270dac7d2633a4d", size = 2235755 }, + { url = "https://files.pythonhosted.org/packages/55/54/570ec2b0e9a3de0af9922e3bb1b69a5429beefbc753a7ea770a27ad308bd/libcst-1.8.6-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c9d7aeafb1b07d25a964b148c0dda9451efb47bbbf67756e16eeae65004b0eb5", size = 2301473 }, + { url = "https://files.pythonhosted.org/packages/11/4c/163457d1717cd12181c421a4cca493454bcabd143fc7e53313bc6a4ad82a/libcst-1.8.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:207481197afd328aa91d02670c15b48d0256e676ce1ad4bafb6dc2b593cc58f1", size = 2298899 }, + { url = "https://files.pythonhosted.org/packages/35/1d/317ddef3669883619ef3d3395ea583305f353ef4ad87d7a5ac1c39be38e3/libcst-1.8.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:375965f34cc6f09f5f809244d3ff9bd4f6cb6699f571121cebce53622e7e0b86", size = 2408239 }, + { url = "https://files.pythonhosted.org/packages/9a/a1/f47d8cccf74e212dd6044b9d6dbc223636508da99acff1d54786653196bc/libcst-1.8.6-cp312-cp312-win_amd64.whl", hash = "sha256:da95b38693b989eaa8d32e452e8261cfa77fe5babfef1d8d2ac25af8c4aa7e6d", size = 2119660 }, + { url = "https://files.pythonhosted.org/packages/19/d0/dd313bf6a7942cdf951828f07ecc1a7695263f385065edc75ef3016a3cb5/libcst-1.8.6-cp312-cp312-win_arm64.whl", hash = "sha256:bff00e1c766658adbd09a175267f8b2f7616e5ee70ce45db3d7c4ce6d9f6bec7", size = 1999824 }, ] [[package]] @@ -1694,9 +1706,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324, upload-time = "2026-05-21T02:30:38.185Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324 } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080, upload-time = "2026-05-21T02:30:35.096Z" }, + { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080 }, ] [[package]] @@ -1708,9 +1720,9 @@ dependencies = [ { name = "httpx" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/f3/f4d6520f8d546e6c5a02f6ebeed5c09774a074b8d2c24ad559ace97a56a6/llama_cloud-0.1.46.tar.gz", hash = "sha256:e86f8791c053590d70cc59e0fc13ce72f9b681a8e658bc61df86d0285288d8ee", size = 127752, upload-time = "2026-01-21T18:40:57.103Z" } +sdist = { url = "https://files.pythonhosted.org/packages/40/f3/f4d6520f8d546e6c5a02f6ebeed5c09774a074b8d2c24ad559ace97a56a6/llama_cloud-0.1.46.tar.gz", hash = "sha256:e86f8791c053590d70cc59e0fc13ce72f9b681a8e658bc61df86d0285288d8ee", size = 127752 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/3a/6caaea28c8c804add33c91d356ed7d5a5412d6c9598e1450af95a15e0bcd/llama_cloud-0.1.46-py3-none-any.whl", hash = "sha256:6c6546c09c04a038c86d84d42f00eae8fd3bff49991ad3aab844bd866ecdf352", size = 361989, upload-time = "2026-01-21T18:40:54.863Z" }, + { url = "https://files.pythonhosted.org/packages/c4/3a/6caaea28c8c804add33c91d356ed7d5a5412d6c9598e1450af95a15e0bcd/llama_cloud-0.1.46-py3-none-any.whl", hash = "sha256:6c6546c09c04a038c86d84d42f00eae8fd3bff49991ad3aab844bd866ecdf352", size = 361989 }, ] [[package]] @@ -1727,9 +1739,9 @@ dependencies = [ { name = "python-dotenv" }, { name = "tenacity" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d3/91/c3c94a58c44d0a12e0df2d5038b188fc283877f56cf2f6c41c60f43258e6/llama_cloud_services-0.6.94.tar.gz", hash = "sha256:127b8440d3d3a964d0c4b3f5fe7fcac3ead482f7645971cc8ae30768dcf63306", size = 64114, upload-time = "2026-02-13T23:29:40.454Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/91/c3c94a58c44d0a12e0df2d5038b188fc283877f56cf2f6c41c60f43258e6/llama_cloud_services-0.6.94.tar.gz", hash = "sha256:127b8440d3d3a964d0c4b3f5fe7fcac3ead482f7645971cc8ae30768dcf63306", size = 64114 } wheels = [ - { url = "https://files.pythonhosted.org/packages/14/ab/876486e4f1c137cfeca8f876abd18eeec35a66a0fd8adb15afba7b28aa8c/llama_cloud_services-0.6.94-py3-none-any.whl", hash = "sha256:ac89785f3689d71298511f751bcf4ca16952a616bd75ff06e0ff164f04b0775b", size = 77098, upload-time = "2026-02-13T23:29:38.958Z" }, + { url = "https://files.pythonhosted.org/packages/14/ab/876486e4f1c137cfeca8f876abd18eeec35a66a0fd8adb15afba7b28aa8c/llama_cloud_services-0.6.94-py3-none-any.whl", hash = "sha256:ac89785f3689d71298511f751bcf4ca16952a616bd75ff06e0ff164f04b0775b", size = 77098 }, ] [[package]] @@ -1742,9 +1754,9 @@ dependencies = [ { name = "llama-index-llms-openai" }, { name = "nltk" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/28/89/3b6f3318ea2249158daab3ff22777ef5ffa87a63c011659a6cfc55e54c35/llama_index-0.14.22.tar.gz", hash = "sha256:c2c9b31f50d2815abdc191085db4acaf96b7c01851ac66b2e4cc82be8cde589e", size = 8565, upload-time = "2026-05-14T20:22:21.006Z" } +sdist = { url = "https://files.pythonhosted.org/packages/28/89/3b6f3318ea2249158daab3ff22777ef5ffa87a63c011659a6cfc55e54c35/llama_index-0.14.22.tar.gz", hash = "sha256:c2c9b31f50d2815abdc191085db4acaf96b7c01851ac66b2e4cc82be8cde589e", size = 8565 } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/fd/f0837c4ce049d8ece7525bbf64564e93e3f16333856c2a0b47fecb58f317/llama_index-0.14.22-py3-none-any.whl", hash = "sha256:14b4bdd799112062e38288eab6aa16643f29d7532505ab174b0b6d5b0817fe94", size = 7115, upload-time = "2026-05-14T20:22:19.611Z" }, + { url = "https://files.pythonhosted.org/packages/59/fd/f0837c4ce049d8ece7525bbf64564e93e3f16333856c2a0b47fecb58f317/llama_index-0.14.22-py3-none-any.whl", hash = "sha256:14b4bdd799112062e38288eab6aa16643f29d7532505ab174b0b6d5b0817fe94", size = 7115 }, ] [[package]] @@ -1781,9 +1793,9 @@ dependencies = [ { name = "typing-inspect" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/96/7f/94a4b940ef0d069840df0fd6d361a2aa832a2dd73b4cecdf86e8f8c353c8/llama_index_core-0.14.22.tar.gz", hash = "sha256:1384410f89bdbd32349aab444ef4f5c828c338787bc65bd1ffd8e86dfb44ac41", size = 11584786, upload-time = "2026-05-14T20:21:37.271Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/7f/94a4b940ef0d069840df0fd6d361a2aa832a2dd73b4cecdf86e8f8c353c8/llama_index_core-0.14.22.tar.gz", hash = "sha256:1384410f89bdbd32349aab444ef4f5c828c338787bc65bd1ffd8e86dfb44ac41", size = 11584786 } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/15/e1a26d8d56aa55fa07587a3e9c7e85294d2df5af6c2229193019bc549ef6/llama_index_core-0.14.22-py3-none-any.whl", hash = "sha256:9cfffde46fd5b7937101e1c0c9bb5c21bd7ff8c8a56937810b87ba3542f31225", size = 11920774, upload-time = "2026-05-14T20:21:40.409Z" }, + { url = "https://files.pythonhosted.org/packages/39/15/e1a26d8d56aa55fa07587a3e9c7e85294d2df5af6c2229193019bc549ef6/llama_index_core-0.14.22-py3-none-any.whl", hash = "sha256:9cfffde46fd5b7937101e1c0c9bb5c21bd7ff8c8a56937810b87ba3542f31225", size = 11920774 }, ] [[package]] @@ -1794,9 +1806,9 @@ dependencies = [ { name = "llama-index-core" }, { name = "openai" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/52/eb56a4887501651fb17400f7f571c1878109ff698efbe0bbac9165a5603d/llama_index_embeddings_openai-0.6.0.tar.gz", hash = "sha256:eb3e6606be81cb89125073e23c97c0a6119dabb4827adbd14697c2029ad73f29", size = 7629, upload-time = "2026-03-12T20:21:27.234Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/52/eb56a4887501651fb17400f7f571c1878109ff698efbe0bbac9165a5603d/llama_index_embeddings_openai-0.6.0.tar.gz", hash = "sha256:eb3e6606be81cb89125073e23c97c0a6119dabb4827adbd14697c2029ad73f29", size = 7629 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/d1/4bb0b80f4057903110060f617ef519197194b3ff5dd6153d850c8f5676fa/llama_index_embeddings_openai-0.6.0-py3-none-any.whl", hash = "sha256:039bb1007ad4267e25ddb89a206dfdab862bfb87d58da4271a3919e4f9df4d61", size = 7666, upload-time = "2026-03-12T20:21:28.079Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d1/4bb0b80f4057903110060f617ef519197194b3ff5dd6153d850c8f5676fa/llama_index_embeddings_openai-0.6.0-py3-none-any.whl", hash = "sha256:039bb1007ad4267e25ddb89a206dfdab862bfb87d58da4271a3919e4f9df4d61", size = 7666 }, ] [[package]] @@ -1807,9 +1819,9 @@ dependencies = [ { name = "deprecated" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4e/d0/671b23ccff255c9bce132a84ffd5a6f4541ceefdeab9c1786b08c9722f2e/llama_index_instrumentation-0.5.0.tar.gz", hash = "sha256:eeb724648b25d149de882a5ac9e21c5acb1ce780da214bda2b075341af29ad8e", size = 43831, upload-time = "2026-03-12T20:17:06.742Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/d0/671b23ccff255c9bce132a84ffd5a6f4541ceefdeab9c1786b08c9722f2e/llama_index_instrumentation-0.5.0.tar.gz", hash = "sha256:eeb724648b25d149de882a5ac9e21c5acb1ce780da214bda2b075341af29ad8e", size = 43831 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/45/6dcaccef44e541ffa138e4b45e33e0d40ab2a7d845338483954fcf77bc75/llama_index_instrumentation-0.5.0-py3-none-any.whl", hash = "sha256:aaab83cddd9dd434278891012d8995f47a3bc7ed1736a371db90965348c56a21", size = 16444, upload-time = "2026-03-12T20:17:05.957Z" }, + { url = "https://files.pythonhosted.org/packages/c3/45/6dcaccef44e541ffa138e4b45e33e0d40ab2a7d845338483954fcf77bc75/llama_index_instrumentation-0.5.0-py3-none-any.whl", hash = "sha256:aaab83cddd9dd434278891012d8995f47a3bc7ed1736a371db90965348c56a21", size = 16444 }, ] [[package]] @@ -1820,9 +1832,9 @@ dependencies = [ { name = "llama-index-core" }, { name = "openai" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/00/d5/2de9c05f1f1d21eb678a6044c59e943063e70099ac39b8b6f835e6e39511/llama_index_llms_openai-0.7.8.tar.gz", hash = "sha256:3352aed617ee5b7aefeb12719609ff84b4b590a1f49aa1e2e9c383d67ea88b0e", size = 27539, upload-time = "2026-05-08T20:02:09.42Z" } +sdist = { url = "https://files.pythonhosted.org/packages/00/d5/2de9c05f1f1d21eb678a6044c59e943063e70099ac39b8b6f835e6e39511/llama_index_llms_openai-0.7.8.tar.gz", hash = "sha256:3352aed617ee5b7aefeb12719609ff84b4b590a1f49aa1e2e9c383d67ea88b0e", size = 27539 } wheels = [ - { url = "https://files.pythonhosted.org/packages/32/49/4250108a76f4f7622109ecb9c57861829f508aba0ffdc502b27134378505/llama_index_llms_openai-0.7.8-py3-none-any.whl", hash = "sha256:967aac1f4ceff99185b2cc425c2757d4fefaf3fac0a35ace247f87a212a29359", size = 28617, upload-time = "2026-05-08T20:02:10.583Z" }, + { url = "https://files.pythonhosted.org/packages/32/49/4250108a76f4f7622109ecb9c57861829f508aba0ffdc502b27134378505/llama_index_llms_openai-0.7.8-py3-none-any.whl", hash = "sha256:967aac1f4ceff99185b2cc425c2757d4fefaf3fac0a35ace247f87a212a29359", size = 28617 }, ] [[package]] @@ -1833,9 +1845,9 @@ dependencies = [ { name = "llama-index-core" }, { name = "pymilvus" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ef/79/72bf70fac3b77770a5c2b1b7d441aa1998bb522d50fe88b0f9c4071854e5/llama_index_vector_stores_milvus-0.9.6.tar.gz", hash = "sha256:6d38ac5939a570e0240687f54fbee4e1ff6c5faa2d28d25377a3f38d2ca07e2b", size = 15584, upload-time = "2026-01-13T11:46:41.394Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/79/72bf70fac3b77770a5c2b1b7d441aa1998bb522d50fe88b0f9c4071854e5/llama_index_vector_stores_milvus-0.9.6.tar.gz", hash = "sha256:6d38ac5939a570e0240687f54fbee4e1ff6c5faa2d28d25377a3f38d2ca07e2b", size = 15584 } wheels = [ - { url = "https://files.pythonhosted.org/packages/51/04/98359849c095b5a3eb06f0649865a30b5a733b9fdbfeac3c6951253f804c/llama_index_vector_stores_milvus-0.9.6-py3-none-any.whl", hash = "sha256:916cbd9b07035ec137905970ef6a49dd77d3ece6e0a79271db35705cca5f5f84", size = 15792, upload-time = "2026-01-13T11:46:39.708Z" }, + { url = "https://files.pythonhosted.org/packages/51/04/98359849c095b5a3eb06f0649865a30b5a733b9fdbfeac3c6951253f804c/llama_index_vector_stores_milvus-0.9.6-py3-none-any.whl", hash = "sha256:916cbd9b07035ec137905970ef6a49dd77d3ece6e0a79271db35705cca5f5f84", size = 15792 }, ] [[package]] @@ -1846,9 +1858,9 @@ dependencies = [ { name = "llama-index-core" }, { name = "pinecone" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1e/69/f51173b041d7273e0486634f14a6b437d147d40bbd1f767c4293c19f42c0/llama_index_vector_stores_pinecone-0.8.0.tar.gz", hash = "sha256:19e3ef74f638f4b693619d797d914fbcd55973ffa9f0ba28b3eb40ac3ff2da89", size = 7852, upload-time = "2026-03-12T20:47:52.405Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/69/f51173b041d7273e0486634f14a6b437d147d40bbd1f767c4293c19f42c0/llama_index_vector_stores_pinecone-0.8.0.tar.gz", hash = "sha256:19e3ef74f638f4b693619d797d914fbcd55973ffa9f0ba28b3eb40ac3ff2da89", size = 7852 } wheels = [ - { url = "https://files.pythonhosted.org/packages/82/5c/4f3560524f97e89318350e78ba81af69fd435debf47d77740df4a3066444/llama_index_vector_stores_pinecone-0.8.0-py3-none-any.whl", hash = "sha256:49f324299ae06cae61231c1ed902d69a75c06083efb37087a484294c34344a3e", size = 8039, upload-time = "2026-03-12T20:47:51.419Z" }, + { url = "https://files.pythonhosted.org/packages/82/5c/4f3560524f97e89318350e78ba81af69fd435debf47d77740df4a3066444/llama_index_vector_stores_pinecone-0.8.0-py3-none-any.whl", hash = "sha256:49f324299ae06cae61231c1ed902d69a75c06083efb37087a484294c34344a3e", size = 8039 }, ] [[package]] @@ -1862,9 +1874,9 @@ dependencies = [ { name = "psycopg2-binary" }, { name = "sqlalchemy", extra = ["asyncio"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5d/88/e89e75d7bd90b870e091157245c65ff89edddf5b6bedae83dbec4bdee28a/llama_index_vector_stores_postgres-0.8.1.tar.gz", hash = "sha256:e3f72f16f0a8776b610b44625b5fcab55a5977ce2fa5a7d3b162306a10d9b4e8", size = 14531, upload-time = "2026-03-13T15:21:52.084Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/88/e89e75d7bd90b870e091157245c65ff89edddf5b6bedae83dbec4bdee28a/llama_index_vector_stores_postgres-0.8.1.tar.gz", hash = "sha256:e3f72f16f0a8776b610b44625b5fcab55a5977ce2fa5a7d3b162306a10d9b4e8", size = 14531 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/60/e2b938d79cd7d0c4e58f33d1c62e5d1edc2b9a4ac8c4f7e613dd7c4e17a1/llama_index_vector_stores_postgres-0.8.1-py3-none-any.whl", hash = "sha256:832a79f2276b51cbb249e34cdb27911b2369c1b9a29e9d39359879bfe789b196", size = 14417, upload-time = "2026-03-13T15:21:51.064Z" }, + { url = "https://files.pythonhosted.org/packages/5d/60/e2b938d79cd7d0c4e58f33d1c62e5d1edc2b9a4ac8c4f7e613dd7c4e17a1/llama_index_vector_stores_postgres-0.8.1-py3-none-any.whl", hash = "sha256:832a79f2276b51cbb249e34cdb27911b2369c1b9a29e9d39359879bfe789b196", size = 14417 }, ] [[package]] @@ -1876,9 +1888,9 @@ dependencies = [ { name = "llama-index-core" }, { name = "qdrant-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ec/9c/7df54b10e6f7d7b3fde3853d25f0c0ec6e701e9dea35359d316e5ea6d942/llama_index_vector_stores_qdrant-0.10.1.tar.gz", hash = "sha256:fef4ca8411c3e33636aabcf883941fde7a9d6deaa45254ee80627f2d0ffbf551", size = 14731, upload-time = "2026-05-04T15:17:50.368Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/9c/7df54b10e6f7d7b3fde3853d25f0c0ec6e701e9dea35359d316e5ea6d942/llama_index_vector_stores_qdrant-0.10.1.tar.gz", hash = "sha256:fef4ca8411c3e33636aabcf883941fde7a9d6deaa45254ee80627f2d0ffbf551", size = 14731 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/9d/32ac29c621db5e635cd248ef824cf49ea934c52085d19c409d6aea4d4167/llama_index_vector_stores_qdrant-0.10.1-py3-none-any.whl", hash = "sha256:90e8c0c0cd96309a9dd84a29cfe77aa6b3ef2f2596f0a0360eaaa8e3a004f1ff", size = 15000, upload-time = "2026-05-04T15:17:51.117Z" }, + { url = "https://files.pythonhosted.org/packages/ab/9d/32ac29c621db5e635cd248ef824cf49ea934c52085d19c409d6aea4d4167/llama_index_vector_stores_qdrant-0.10.1-py3-none-any.whl", hash = "sha256:90e8c0c0cd96309a9dd84a29cfe77aa6b3ef2f2596f0a0360eaaa8e3a004f1ff", size = 15000 }, ] [[package]] @@ -1889,9 +1901,9 @@ dependencies = [ { name = "llama-index-core" }, { name = "weaviate-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/16/de/4c6e4aa714558fd61e76bbf2b3a46bf6e4b24553b7302fb07bc05c7e84f9/llama_index_vector_stores_weaviate-1.6.0.tar.gz", hash = "sha256:d88256f94f9e55d8e0a2f87e936f1481d016bdd3c871f142367b64ff179fcafa", size = 9678, upload-time = "2026-03-12T20:48:59.692Z" } +sdist = { url = "https://files.pythonhosted.org/packages/16/de/4c6e4aa714558fd61e76bbf2b3a46bf6e4b24553b7302fb07bc05c7e84f9/llama_index_vector_stores_weaviate-1.6.0.tar.gz", hash = "sha256:d88256f94f9e55d8e0a2f87e936f1481d016bdd3c871f142367b64ff179fcafa", size = 9678 } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/01/069f641c7d8775392f8e2fc4f8a701b29d8818463207088a29f9ef1da04c/llama_index_vector_stores_weaviate-1.6.0-py3-none-any.whl", hash = "sha256:702bef9254ee68587084ba974c65c18cbd64223c24015499cf03c392aea6ba2b", size = 10438, upload-time = "2026-03-12T20:48:58.967Z" }, + { url = "https://files.pythonhosted.org/packages/fc/01/069f641c7d8775392f8e2fc4f8a701b29d8818463207088a29f9ef1da04c/llama_index_vector_stores_weaviate-1.6.0-py3-none-any.whl", hash = "sha256:702bef9254ee68587084ba974c65c18cbd64223c24015499cf03c392aea6ba2b", size = 10438 }, ] [[package]] @@ -1903,9 +1915,9 @@ dependencies = [ { name = "pydantic" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c4/ec/05f3db99a2e6e252e3939e7751cad2fb1322dc6d32f4cf5c795cf7ddcad3/llama_index_workflows-2.20.0.tar.gz", hash = "sha256:df2760fea9e100c97a4e919d255461e344413acac4382d17d8217337806e4772", size = 97410, upload-time = "2026-04-24T14:54:41.524Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c4/ec/05f3db99a2e6e252e3939e7751cad2fb1322dc6d32f4cf5c795cf7ddcad3/llama_index_workflows-2.20.0.tar.gz", hash = "sha256:df2760fea9e100c97a4e919d255461e344413acac4382d17d8217337806e4772", size = 97410 } wheels = [ - { url = "https://files.pythonhosted.org/packages/71/5f/385231406d777cb4b608fd8ebe3577dbd90962770717181e6b91b44fb1b8/llama_index_workflows-2.20.0-py3-none-any.whl", hash = "sha256:36f6b6ace77f837d9907078aea7e830251afe96a58daecff5ed090c88c55095d", size = 121238, upload-time = "2026-04-24T14:54:40.455Z" }, + { url = "https://files.pythonhosted.org/packages/71/5f/385231406d777cb4b608fd8ebe3577dbd90962770717181e6b91b44fb1b8/llama_index_workflows-2.20.0-py3-none-any.whl", hash = "sha256:36f6b6ace77f837d9907078aea7e830251afe96a58daecff5ed090c88c55095d", size = 121238 }, ] [[package]] @@ -1915,9 +1927,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "llama-cloud-services" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/97/55/0d089a90fe98027853dd41dead1f094458df1a964e0d7c379b6a44208761/llama_parse-0.6.94.tar.gz", hash = "sha256:d9e4347ec6caa1e9d5266cc5d4d8b29a29aa0d0948d921a26c73d3e4aaf5ba72", size = 3996, upload-time = "2026-02-13T23:29:34.14Z" } +sdist = { url = "https://files.pythonhosted.org/packages/97/55/0d089a90fe98027853dd41dead1f094458df1a964e0d7c379b6a44208761/llama_parse-0.6.94.tar.gz", hash = "sha256:d9e4347ec6caa1e9d5266cc5d4d8b29a29aa0d0948d921a26c73d3e4aaf5ba72", size = 3996 } wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/70/3d4cc14f99c80491401d4ab514f3ebe3113e38c8017cd384a73dc67b3ae4/llama_parse-0.6.94-py3-none-any.whl", hash = "sha256:48f21d909696597bf992acf5017685d88a6b25604bc1e98775021f39fe66649f", size = 5362, upload-time = "2026-02-13T23:29:35.432Z" }, + { url = "https://files.pythonhosted.org/packages/bd/70/3d4cc14f99c80491401d4ab514f3ebe3113e38c8017cd384a73dc67b3ae4/llama_parse-0.6.94-py3-none-any.whl", hash = "sha256:48f21d909696597bf992acf5017685d88a6b25604bc1e98775021f39fe66649f", size = 5362 }, ] [[package]] @@ -1928,9 +1940,9 @@ dependencies = [ { name = "requests" }, { name = "tenacity" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c8/f1/ac02558536cb97db15c2e6047f7943869f260299c700c3b5bce6821982e6/llmwhisperer_client-2.7.0.tar.gz", hash = "sha256:bebf3ca95b4a9a0802d290d88a834539e8054f68b1013298f1730dbcaafc69ea", size = 3268816, upload-time = "2026-03-16T10:44:42.234Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/f1/ac02558536cb97db15c2e6047f7943869f260299c700c3b5bce6821982e6/llmwhisperer_client-2.7.0.tar.gz", hash = "sha256:bebf3ca95b4a9a0802d290d88a834539e8054f68b1013298f1730dbcaafc69ea", size = 3268816 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/d9/b481433b644541a745c1d7de4fb2f53df088110ac34e4a1e89dc3ff9e28f/llmwhisperer_client-2.7.0-py3-none-any.whl", hash = "sha256:72903da9b7f07a2584792cb59094b703a31fc48ad33b6dee27a1ec14d84e0d18", size = 11093, upload-time = "2026-03-16T10:44:40.924Z" }, + { url = "https://files.pythonhosted.org/packages/c4/d9/b481433b644541a745c1d7de4fb2f53df088110ac34e4a1e89dc3ff9e28f/llmwhisperer_client-2.7.0-py3-none-any.whl", hash = "sha256:72903da9b7f07a2584792cb59094b703a31fc48ad33b6dee27a1ec14d84e0d18", size = 11093 }, ] [[package]] @@ -1940,28 +1952,28 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mdurl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454 } wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687 }, ] [[package]] name = "markupsafe" version = "3.0.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, - { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, - { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, - { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, - { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, - { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, - { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, - { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, - { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615 }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020 }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332 }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947 }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962 }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760 }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529 }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015 }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540 }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105 }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906 }, ] [[package]] @@ -1971,9 +1983,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/55/79/de6c16cc902f4fc372236926b0ce2ab7845268dcc30fb2fbb7f71b418631/marshmallow-3.26.2.tar.gz", hash = "sha256:bbe2adb5a03e6e3571b573f42527c6fe926e17467833660bebd11593ab8dfd57", size = 222095, upload-time = "2025-12-22T06:53:53.309Z" } +sdist = { url = "https://files.pythonhosted.org/packages/55/79/de6c16cc902f4fc372236926b0ce2ab7845268dcc30fb2fbb7f71b418631/marshmallow-3.26.2.tar.gz", hash = "sha256:bbe2adb5a03e6e3571b573f42527c6fe926e17467833660bebd11593ab8dfd57", size = 222095 } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/2f/5108cb3ee4ba6501748c4908b908e55f42a5b66245b4cfe0c99326e1ef6e/marshmallow-3.26.2-py3-none-any.whl", hash = "sha256:013fa8a3c4c276c24d26d84ce934dc964e2aa794345a0f8c7e5a7191482c8a73", size = 50964, upload-time = "2025-12-22T06:53:51.801Z" }, + { url = "https://files.pythonhosted.org/packages/be/2f/5108cb3ee4ba6501748c4908b908e55f42a5b66245b4cfe0c99326e1ef6e/marshmallow-3.26.2-py3-none-any.whl", hash = "sha256:013fa8a3c4c276c24d26d84ce934dc964e2aa794345a0f8c7e5a7191482c8a73", size = 50964 }, ] [[package]] @@ -1983,18 +1995,18 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "chardet" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4b/9c/dd6e38d747a62ead27f9abef32f4ca4311d4e40ac28e76bcc9ffb5dd0329/mbstrdecoder-1.1.5.tar.gz", hash = "sha256:8cbfba26938befd8a35e3cc06ca0632f61320b7b2be7df32550b895e1725b1ce", size = 14529, upload-time = "2026-05-05T04:17:58.23Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/9c/dd6e38d747a62ead27f9abef32f4ca4311d4e40ac28e76bcc9ffb5dd0329/mbstrdecoder-1.1.5.tar.gz", hash = "sha256:8cbfba26938befd8a35e3cc06ca0632f61320b7b2be7df32550b895e1725b1ce", size = 14529 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/eb/711270faab7b7df702339a2c68b31fd3ed4fffc68b0e99e5bdf49b1e87e4/mbstrdecoder-1.1.5-py3-none-any.whl", hash = "sha256:4a50fe113d4abecfd86e8f716b2e413cce03d63af83ec3c7cdbe81dec0e519ed", size = 7966, upload-time = "2026-05-05T04:17:56.78Z" }, + { url = "https://files.pythonhosted.org/packages/4c/eb/711270faab7b7df702339a2c68b31fd3ed4fffc68b0e99e5bdf49b1e87e4/mbstrdecoder-1.1.5-py3-none-any.whl", hash = "sha256:4a50fe113d4abecfd86e8f716b2e413cce03d63af83ec3c7cdbe81dec0e519ed", size = 7966 }, ] [[package]] name = "mdurl" version = "0.1.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729 } wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979 }, ] [[package]] @@ -2007,9 +2019,9 @@ dependencies = [ { name = "numpy" }, { name = "pyarrow" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a3/9a/d80d260e6fe1246818a8ef782c374ba9c6ca46ca3b987c14eabe914ef805/milvus_lite-3.0.tar.gz", hash = "sha256:2c35d0d046b1faae3402cde1fb73d65f51ee8c6aba65f54de1dda46f7bb18b9b", size = 589749, upload-time = "2026-05-13T07:14:05.827Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/9a/d80d260e6fe1246818a8ef782c374ba9c6ca46ca3b987c14eabe914ef805/milvus_lite-3.0.tar.gz", hash = "sha256:2c35d0d046b1faae3402cde1fb73d65f51ee8c6aba65f54de1dda46f7bb18b9b", size = 589749 } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/42/dee08ae3bfc1731572f193d00b248c9370b0b9dff12becb0ffd8b2ee8d56/milvus_lite-3.0-py3-none-any.whl", hash = "sha256:d9a094eab84bdaa4253da3721482282c939da1cce6f4e1759f947e8d3e53406e", size = 230490, upload-time = "2026-05-13T07:14:00.816Z" }, + { url = "https://files.pythonhosted.org/packages/59/42/dee08ae3bfc1731572f193d00b248c9370b0b9dff12becb0ffd8b2ee8d56/milvus_lite-3.0-py3-none-any.whl", hash = "sha256:d9a094eab84bdaa4253da3721482282c939da1cce6f4e1759f947e8d3e53406e", size = 230490 }, ] [[package]] @@ -2023,9 +2035,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/df/6dfc6540f96a74125a11653cce717603fd5b7d0001a8e847b3e54e72d238/minio-7.2.20.tar.gz", hash = "sha256:95898b7a023fbbfde375985aa77e2cd6a0762268db79cf886f002a9ea8e68598", size = 136113, upload-time = "2025-11-27T00:37:15.569Z" } +sdist = { url = "https://files.pythonhosted.org/packages/40/df/6dfc6540f96a74125a11653cce717603fd5b7d0001a8e847b3e54e72d238/minio-7.2.20.tar.gz", hash = "sha256:95898b7a023fbbfde375985aa77e2cd6a0762268db79cf886f002a9ea8e68598", size = 136113 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/9a/b697530a882588a84db616580f2ba5d1d515c815e11c30d219145afeec87/minio-7.2.20-py3-none-any.whl", hash = "sha256:eb33dd2fb80e04c3726a76b13241c6be3c4c46f8d81e1d58e757786f6501897e", size = 93751, upload-time = "2025-11-27T00:37:13.993Z" }, + { url = "https://files.pythonhosted.org/packages/3e/9a/b697530a882588a84db616580f2ba5d1d515c815e11c30d219145afeec87/minio-7.2.20-py3-none-any.whl", hash = "sha256:eb33dd2fb80e04c3726a76b13241c6be3c4c46f8d81e1d58e757786f6501897e", size = 93751 }, ] [[package]] @@ -2037,9 +2049,9 @@ dependencies = [ { name = "pyjwt", extra = ["crypto"] }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/de/cb/b02b0f748ac668922364ccb3c3bff5b71628a05f5adfec2ba2a5c3031483/msal-1.36.0.tar.gz", hash = "sha256:3f6a4af2b036b476a4215111c4297b4e6e236ed186cd804faefba23e4990978b", size = 174217, upload-time = "2026-04-09T10:20:33.525Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/cb/b02b0f748ac668922364ccb3c3bff5b71628a05f5adfec2ba2a5c3031483/msal-1.36.0.tar.gz", hash = "sha256:3f6a4af2b036b476a4215111c4297b4e6e236ed186cd804faefba23e4990978b", size = 174217 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/d3/414d1f0a5f6f4fe5313c2b002c54e78a3332970feb3f5fed14237aa17064/msal-1.36.0-py3-none-any.whl", hash = "sha256:36ecac30e2ff4322d956029aabce3c82301c29f0acb1ad89b94edcabb0e58ec4", size = 121547, upload-time = "2026-04-09T10:20:32.336Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d3/414d1f0a5f6f4fe5313c2b002c54e78a3332970feb3f5fed14237aa17064/msal-1.36.0-py3-none-any.whl", hash = "sha256:36ecac30e2ff4322d956029aabce3c82301c29f0acb1ad89b94edcabb0e58ec4", size = 121547 }, ] [[package]] @@ -2049,36 +2061,36 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "msal" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } +sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, + { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583 }, ] [[package]] name = "multidict" version = "6.7.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" }, - { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" }, - { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" }, - { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" }, - { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" }, - { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" }, - { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" }, - { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" }, - { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, - { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" }, - { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" }, - { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" }, - { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" }, - { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" }, - { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, - { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" }, - { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" }, - { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" }, - { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893 }, + { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456 }, + { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872 }, + { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018 }, + { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883 }, + { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413 }, + { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404 }, + { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456 }, + { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322 }, + { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955 }, + { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254 }, + { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059 }, + { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588 }, + { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642 }, + { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377 }, + { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887 }, + { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053 }, + { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307 }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319 }, ] [[package]] @@ -2089,41 +2101,41 @@ dependencies = [ { name = "mypy-extensions" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/1e/a587a862c766a755a58b62d8c00aed11b74a15dc415c1bf5da7b607b0efd/mypy-1.9.0.tar.gz", hash = "sha256:3cc5da0127e6a478cddd906068496a97a7618a21ce9b54bde5bf7e539c7af974", size = 2995901, upload-time = "2024-03-08T16:10:12.412Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/1e/a587a862c766a755a58b62d8c00aed11b74a15dc415c1bf5da7b607b0efd/mypy-1.9.0.tar.gz", hash = "sha256:3cc5da0127e6a478cddd906068496a97a7618a21ce9b54bde5bf7e539c7af974", size = 2995901 } wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/96/40f0f605b1d4e2ad1fb11d21988ce3a3e205886c0fcbd35c9789a214de9a/mypy-1.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aceb1db093b04db5cd390821464504111b8ec3e351eb85afd1433490163d60cd", size = 10725390, upload-time = "2024-03-08T16:10:01.099Z" }, - { url = "https://files.pythonhosted.org/packages/d7/d2/072e40384b53051106b4fcf03537fb88e2a6ad0757d2ab7f6c8c2f188a69/mypy-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0235391f1c6f6ce487b23b9dbd1327b4ec33bb93934aa986efe8a9563d9349e6", size = 9731292, upload-time = "2024-03-08T16:08:48.463Z" }, - { url = "https://files.pythonhosted.org/packages/85/a5/b7dc7eb69eda899fd07e71403b51b598a1f4df0f452d1da5844374082bcd/mypy-1.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4d5ddc13421ba3e2e082a6c2d74c2ddb3979c39b582dacd53dd5d9431237185", size = 12455450, upload-time = "2024-03-08T16:08:57.375Z" }, - { url = "https://files.pythonhosted.org/packages/1c/1b/3e962a201d2f0f57c9fa1990e0dd6076f4f2f94954ab56e4a701ec3cc070/mypy-1.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:190da1ee69b427d7efa8aa0d5e5ccd67a4fb04038c380237a0d96829cb157913", size = 12530368, upload-time = "2024-03-08T16:09:17.061Z" }, - { url = "https://files.pythonhosted.org/packages/72/1f/8b214b69d08cc5e4bd8c3769ac55a43318f3529362ea55e5957774b69924/mypy-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe28657de3bfec596bbeef01cb219833ad9d38dd5393fc649f4b366840baefe6", size = 9319112, upload-time = "2024-03-08T16:09:07.961Z" }, - { url = "https://files.pythonhosted.org/packages/60/db/0ba2eaedca52bf5276275e8489951c26206030b3d31bf06f00875ae75d5d/mypy-1.9.0-py3-none-any.whl", hash = "sha256:a260627a570559181a9ea5de61ac6297aa5af202f06fd7ab093ce74e7181e43e", size = 2555887, upload-time = "2024-03-08T16:09:48.584Z" }, + { url = "https://files.pythonhosted.org/packages/6e/96/40f0f605b1d4e2ad1fb11d21988ce3a3e205886c0fcbd35c9789a214de9a/mypy-1.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aceb1db093b04db5cd390821464504111b8ec3e351eb85afd1433490163d60cd", size = 10725390 }, + { url = "https://files.pythonhosted.org/packages/d7/d2/072e40384b53051106b4fcf03537fb88e2a6ad0757d2ab7f6c8c2f188a69/mypy-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0235391f1c6f6ce487b23b9dbd1327b4ec33bb93934aa986efe8a9563d9349e6", size = 9731292 }, + { url = "https://files.pythonhosted.org/packages/85/a5/b7dc7eb69eda899fd07e71403b51b598a1f4df0f452d1da5844374082bcd/mypy-1.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4d5ddc13421ba3e2e082a6c2d74c2ddb3979c39b582dacd53dd5d9431237185", size = 12455450 }, + { url = "https://files.pythonhosted.org/packages/1c/1b/3e962a201d2f0f57c9fa1990e0dd6076f4f2f94954ab56e4a701ec3cc070/mypy-1.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:190da1ee69b427d7efa8aa0d5e5ccd67a4fb04038c380237a0d96829cb157913", size = 12530368 }, + { url = "https://files.pythonhosted.org/packages/72/1f/8b214b69d08cc5e4bd8c3769ac55a43318f3529362ea55e5957774b69924/mypy-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe28657de3bfec596bbeef01cb219833ad9d38dd5393fc649f4b366840baefe6", size = 9319112 }, + { url = "https://files.pythonhosted.org/packages/60/db/0ba2eaedca52bf5276275e8489951c26206030b3d31bf06f00875ae75d5d/mypy-1.9.0-py3-none-any.whl", hash = "sha256:a260627a570559181a9ea5de61ac6297aa5af202f06fd7ab093ce74e7181e43e", size = 2555887 }, ] [[package]] name = "mypy-extensions" version = "1.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343 } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963 }, ] [[package]] name = "nest-asyncio" version = "1.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195 }, ] [[package]] name = "networkx" version = "3.6.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504 }, ] [[package]] @@ -2136,37 +2148,37 @@ dependencies = [ { name = "regex" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" }, + { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087 }, ] [[package]] name = "nodeenv" version = "1.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611 } wheels = [ - { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438 }, ] [[package]] name = "numpy" version = "2.4.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807 } wheels = [ - { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, - { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, - { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, - { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, - { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, - { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, - { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, - { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, - { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, + { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119 }, + { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246 }, + { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410 }, + { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240 }, + { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012 }, + { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538 }, + { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706 }, + { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541 }, + { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825 }, + { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687 }, + { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482 }, ] [[package]] @@ -2180,18 +2192,18 @@ dependencies = [ { name = "rsa" }, { name = "six" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/7b/17244b1083e8e604bf154cf9b716aecd6388acd656dd01893d0d244c94d9/oauth2client-4.1.3.tar.gz", hash = "sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6", size = 155910, upload-time = "2018-09-07T21:38:18.036Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/7b/17244b1083e8e604bf154cf9b716aecd6388acd656dd01893d0d244c94d9/oauth2client-4.1.3.tar.gz", hash = "sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6", size = 155910 } wheels = [ - { url = "https://files.pythonhosted.org/packages/95/a9/4f25a14d23f0786b64875b91784607c2277eff25d48f915e39ff0cff505a/oauth2client-4.1.3-py2.py3-none-any.whl", hash = "sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac", size = 98206, upload-time = "2018-09-07T21:38:16.742Z" }, + { url = "https://files.pythonhosted.org/packages/95/a9/4f25a14d23f0786b64875b91784607c2277eff25d48f915e39ff0cff505a/oauth2client-4.1.3-py2.py3-none-any.whl", hash = "sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac", size = 98206 }, ] [[package]] name = "oauthlib" version = "3.3.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918 } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065 }, ] [[package]] @@ -2204,9 +2216,9 @@ dependencies = [ { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bc/04/6dce2d581c54a8e55a3b128cf79a93821a68a62bb9a956e65476c5bb247e/office365_rest_python_client-2.6.2.tar.gz", hash = "sha256:ce27f5a1c0cc3ff97041ccd9b386145692be4c64739f243f7d6ac3edbe0a3c46", size = 659460, upload-time = "2025-05-11T10:24:21.895Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bc/04/6dce2d581c54a8e55a3b128cf79a93821a68a62bb9a956e65476c5bb247e/office365_rest_python_client-2.6.2.tar.gz", hash = "sha256:ce27f5a1c0cc3ff97041ccd9b386145692be4c64739f243f7d6ac3edbe0a3c46", size = 659460 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/a4/611155711f8af347875c15b8b83f5fd9e978bd4de45f90085b9a583b684d/Office365_REST_Python_Client-2.6.2-py3-none-any.whl", hash = "sha256:06fc6829c39b503897caa9d881db419d7f97a8e4f1c95c4c2d12db36ea6c955d", size = 1337139, upload-time = "2025-05-11T10:24:18.926Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a4/611155711f8af347875c15b8b83f5fd9e978bd4de45f90085b9a583b684d/Office365_REST_Python_Client-2.6.2-py3-none-any.whl", hash = "sha256:06fc6829c39b503897caa9d881db419d7f97a8e4f1c95c4c2d12db36ea6c955d", size = 1337139 }, ] [[package]] @@ -2223,9 +2235,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/32/50/5901f01ef14e6c27788beb91e54fef5d6204fb5fb9e97402fc8a14de2e32/openai-2.37.0.tar.gz", hash = "sha256:f4bc562cc5f3a43d40d678105572d9d44765f6e0f50c125f63055419b72f4bd9", size = 754706, upload-time = "2026-05-15T22:30:35.428Z" } +sdist = { url = "https://files.pythonhosted.org/packages/32/50/5901f01ef14e6c27788beb91e54fef5d6204fb5fb9e97402fc8a14de2e32/openai-2.37.0.tar.gz", hash = "sha256:f4bc562cc5f3a43d40d678105572d9d44765f6e0f50c125f63055419b72f4bd9", size = 754706 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/4c/bce61680d0699a78a405fd9a67989b175ba020590428831aab2ab1d2be7c/openai-2.37.0-py3-none-any.whl", hash = "sha256:814633888b8f3b1ffd6615697c6e4ef93632d08b7c2e28c8c5ef3556e5a10107", size = 1303238, upload-time = "2026-05-15T22:30:32.767Z" }, + { url = "https://files.pythonhosted.org/packages/ed/4c/bce61680d0699a78a405fd9a67989b175ba020590428831aab2ab1d2be7c/openai-2.37.0-py3-none-any.whl", hash = "sha256:814633888b8f3b1ffd6615697c6e4ef93632d08b7c2e28c8c5ef3556e5a10107", size = 1303238 }, ] [[package]] @@ -2235,22 +2247,22 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ed/c7/12632c03022aa5059ce9b6738397cda682dfda9d9afe7008b8a4f98c6ee5/oracledb-2.4.0.tar.gz", hash = "sha256:bdd61a9d5077448b5f1c58af6a14accc287bf8032846c351a3cdde5cf64fe95b", size = 614809, upload-time = "2024-08-20T21:02:35.362Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/c7/12632c03022aa5059ce9b6738397cda682dfda9d9afe7008b8a4f98c6ee5/oracledb-2.4.0.tar.gz", hash = "sha256:bdd61a9d5077448b5f1c58af6a14accc287bf8032846c351a3cdde5cf64fe95b", size = 614809 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/5b/5125e0a74a58717ac094d953ddaa4c61cfefcd926850c0ecc081e0c209f3/oracledb-2.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:03d1072db83e3f95a8792b8452c78940141902ef97f31223f1d96bfeb8ff830b", size = 3769983, upload-time = "2024-08-20T21:03:08.186Z" }, - { url = "https://files.pythonhosted.org/packages/17/22/81eb81e15a86989acd21220480a87a3891a27b3f2d64b249098e09e002eb/oracledb-2.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fda77ace54379ad70187627ed02329f9ef4f35c1cc1052e4d27fe4ec68d38fc", size = 2081340, upload-time = "2024-08-20T21:03:10.988Z" }, - { url = "https://files.pythonhosted.org/packages/6f/56/9cd84f67a573cc6066589d8264ab13f710a128197977205b9c4b177ee85e/oracledb-2.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bed34cdd5037277424bb5a38987e00cbb6eea3670ce9c4fcc3cab5971fab5348", size = 2234827, upload-time = "2024-08-20T21:03:13.716Z" }, - { url = "https://files.pythonhosted.org/packages/f5/ca/4406cfe3400735bf4a1eee951eb174c6cd8573e74d43c1aba9448066a3d2/oracledb-2.4.0-cp312-cp312-win32.whl", hash = "sha256:02e1eea36de371d7719ca02d20a8900fab767e5db71aa59be101405060cf2cfa", size = 1373933, upload-time = "2024-08-20T21:03:15.514Z" }, - { url = "https://files.pythonhosted.org/packages/a8/e9/1a8afdbe4aaba030476c91284d7599f54fce2879232d28797a4a71d5cfe2/oracledb-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:0b81ec1e20d4d20b0f95a673bb73923d24673e8739d3a25a746113519612c057", size = 1681666, upload-time = "2024-08-20T21:03:17.366Z" }, + { url = "https://files.pythonhosted.org/packages/9e/5b/5125e0a74a58717ac094d953ddaa4c61cfefcd926850c0ecc081e0c209f3/oracledb-2.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:03d1072db83e3f95a8792b8452c78940141902ef97f31223f1d96bfeb8ff830b", size = 3769983 }, + { url = "https://files.pythonhosted.org/packages/17/22/81eb81e15a86989acd21220480a87a3891a27b3f2d64b249098e09e002eb/oracledb-2.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fda77ace54379ad70187627ed02329f9ef4f35c1cc1052e4d27fe4ec68d38fc", size = 2081340 }, + { url = "https://files.pythonhosted.org/packages/6f/56/9cd84f67a573cc6066589d8264ab13f710a128197977205b9c4b177ee85e/oracledb-2.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bed34cdd5037277424bb5a38987e00cbb6eea3670ce9c4fcc3cab5971fab5348", size = 2234827 }, + { url = "https://files.pythonhosted.org/packages/f5/ca/4406cfe3400735bf4a1eee951eb174c6cd8573e74d43c1aba9448066a3d2/oracledb-2.4.0-cp312-cp312-win32.whl", hash = "sha256:02e1eea36de371d7719ca02d20a8900fab767e5db71aa59be101405060cf2cfa", size = 1373933 }, + { url = "https://files.pythonhosted.org/packages/a8/e9/1a8afdbe4aaba030476c91284d7599f54fce2879232d28797a4a71d5cfe2/oracledb-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:0b81ec1e20d4d20b0f95a673bb73923d24673e8739d3a25a746113519612c057", size = 1681666 }, ] [[package]] name = "packaging" version = "26.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134 } wheels = [ - { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195 }, ] [[package]] @@ -2263,15 +2275,15 @@ dependencies = [ { name = "pytz" }, { name = "tzdata" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53", size = 11597846, upload-time = "2025-09-29T23:19:48.856Z" }, - { url = "https://files.pythonhosted.org/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35", size = 10729618, upload-time = "2025-09-29T23:39:08.659Z" }, - { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212, upload-time = "2025-09-29T23:19:59.765Z" }, - { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693, upload-time = "2025-09-29T23:20:14.098Z" }, - { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002, upload-time = "2025-09-29T23:20:26.76Z" }, - { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971, upload-time = "2025-09-29T23:20:41.344Z" }, - { url = "https://files.pythonhosted.org/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b", size = 10992722, upload-time = "2025-09-29T23:20:54.139Z" }, + { url = "https://files.pythonhosted.org/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53", size = 11597846 }, + { url = "https://files.pythonhosted.org/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35", size = 10729618 }, + { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212 }, + { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693 }, + { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002 }, + { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971 }, + { url = "https://files.pythonhosted.org/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b", size = 10992722 }, ] [[package]] @@ -2284,27 +2296,27 @@ dependencies = [ { name = "invoke" }, { name = "pynacl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/93/dcc25d52f49022ae6175d15e6bd751f1acc99b98bc61fc55e5155a7be2e7/paramiko-5.0.0.tar.gz", hash = "sha256:36763b5b95c2a0dcfdf1abc48e48156ee425b21efe2f0e787c2dd5a95c0e5e79", size = 1548586, upload-time = "2026-05-09T18:28:52.256Z" } +sdist = { url = "https://files.pythonhosted.org/packages/62/93/dcc25d52f49022ae6175d15e6bd751f1acc99b98bc61fc55e5155a7be2e7/paramiko-5.0.0.tar.gz", hash = "sha256:36763b5b95c2a0dcfdf1abc48e48156ee425b21efe2f0e787c2dd5a95c0e5e79", size = 1548586 } wheels = [ - { url = "https://files.pythonhosted.org/packages/82/5b/eadf6d45de38d30ab603f49393b6cd2cbe7e233af8cf90197e32782b68a9/paramiko-5.0.0-py3-none-any.whl", hash = "sha256:b7044611c30140d9a75261653210e2002977b71a0497ff3ba0d98d7edbf62f7c", size = 208919, upload-time = "2026-05-09T18:28:50.295Z" }, + { url = "https://files.pythonhosted.org/packages/82/5b/eadf6d45de38d30ab603f49393b6cd2cbe7e233af8cf90197e32782b68a9/paramiko-5.0.0-py3-none-any.whl", hash = "sha256:b7044611c30140d9a75261653210e2002977b71a0497ff3ba0d98d7edbf62f7c", size = 208919 }, ] [[package]] name = "pathspec" version = "1.1.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328 }, ] [[package]] name = "pathvalidate" version = "3.3.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/2a/52a8da6fe965dea6192eb716b357558e103aea0a1e9a8352ad575a8406ca/pathvalidate-3.3.1.tar.gz", hash = "sha256:b18c07212bfead624345bb8e1d6141cdcf15a39736994ea0b94035ad2b1ba177", size = 63262, upload-time = "2025-06-15T09:07:20.736Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/2a/52a8da6fe965dea6192eb716b357558e103aea0a1e9a8352ad575a8406ca/pathvalidate-3.3.1.tar.gz", hash = "sha256:b18c07212bfead624345bb8e1d6141cdcf15a39736994ea0b94035ad2b1ba177", size = 63262 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/70/875f4a23bfc4731703a5835487d0d2fb999031bd415e7d17c0ae615c18b7/pathvalidate-3.3.1-py3-none-any.whl", hash = "sha256:5263baab691f8e1af96092fa5137ee17df5bdfbd6cff1fcac4d6ef4bc2e1735f", size = 24305, upload-time = "2025-06-15T09:07:19.117Z" }, + { url = "https://files.pythonhosted.org/packages/9a/70/875f4a23bfc4731703a5835487d0d2fb999031bd415e7d17c0ae615c18b7/pathvalidate-3.3.1-py3-none-any.whl", hash = "sha256:5263baab691f8e1af96092fa5137ee17df5bdfbd6cff1fcac4d6ef4bc2e1735f", size = 24305 }, ] [[package]] @@ -2315,9 +2327,9 @@ dependencies = [ { name = "charset-normalizer" }, { name = "cryptography" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/46/9a/d79d8fa6d47a0338846bb558b39b9963b8eb2dfedec61867c138c1b17eeb/pdfminer_six-20251230.tar.gz", hash = "sha256:e8f68a14c57e00c2d7276d26519ea64be1b48f91db1cdc776faa80528ca06c1e", size = 8511285, upload-time = "2025-12-30T15:49:13.104Z" } +sdist = { url = "https://files.pythonhosted.org/packages/46/9a/d79d8fa6d47a0338846bb558b39b9963b8eb2dfedec61867c138c1b17eeb/pdfminer_six-20251230.tar.gz", hash = "sha256:e8f68a14c57e00c2d7276d26519ea64be1b48f91db1cdc776faa80528ca06c1e", size = 8511285 } wheels = [ - { url = "https://files.pythonhosted.org/packages/65/d7/b288ea32deb752a09aab73c75e1e7572ab2a2b56c3124a5d1eb24c62ceb3/pdfminer_six-20251230-py3-none-any.whl", hash = "sha256:9ff2e3466a7dfc6de6fd779478850b6b7c2d9e9405aa2a5869376a822771f485", size = 6591909, upload-time = "2025-12-30T15:49:10.76Z" }, + { url = "https://files.pythonhosted.org/packages/65/d7/b288ea32deb752a09aab73c75e1e7572ab2a2b56c3124a5d1eb24c62ceb3/pdfminer_six-20251230-py3-none-any.whl", hash = "sha256:9ff2e3466a7dfc6de6fd779478850b6b7c2d9e9405aa2a5869376a822771f485", size = 6591909 }, ] [[package]] @@ -2329,9 +2341,9 @@ dependencies = [ { name = "pillow" }, { name = "pypdfium2" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/37/9ca3519e92a8434eb93be570b131476cc0a4e840bb39c62ddb7813a39d53/pdfplumber-0.11.9.tar.gz", hash = "sha256:481224b678b2bbdbf376e2c39bf914144eef7c3d301b4a28eebf0f7f6109d6dc", size = 102768, upload-time = "2026-01-05T08:10:29.072Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/37/9ca3519e92a8434eb93be570b131476cc0a4e840bb39c62ddb7813a39d53/pdfplumber-0.11.9.tar.gz", hash = "sha256:481224b678b2bbdbf376e2c39bf914144eef7c3d301b4a28eebf0f7f6109d6dc", size = 102768 } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/c8/cdbc975f5b634e249cfa6597e37c50f3078412474f21c015e508bfbfe3c3/pdfplumber-0.11.9-py3-none-any.whl", hash = "sha256:33ec5580959ba524e9100138746e090879504c42955df1b8a997604dd326c443", size = 60045, upload-time = "2026-01-05T08:10:27.512Z" }, + { url = "https://files.pythonhosted.org/packages/8b/c8/cdbc975f5b634e249cfa6597e37c50f3078412474f21c015e508bfbfe3c3/pdfplumber-0.11.9-py3-none-any.whl", hash = "sha256:33ec5580959ba524e9100138746e090879504c42955df1b8a997604dd326c443", size = 60045 }, ] [[package]] @@ -2341,37 +2353,37 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/25/6c/6d8b4b03b958c02fa8687ec6063c49d952a189f8c91ebbe51e877dfab8f7/pgvector-0.4.2.tar.gz", hash = "sha256:322cac0c1dc5d41c9ecf782bd9991b7966685dee3a00bc873631391ed949513a", size = 31354, upload-time = "2025-12-05T01:07:17.87Z" } +sdist = { url = "https://files.pythonhosted.org/packages/25/6c/6d8b4b03b958c02fa8687ec6063c49d952a189f8c91ebbe51e877dfab8f7/pgvector-0.4.2.tar.gz", hash = "sha256:322cac0c1dc5d41c9ecf782bd9991b7966685dee3a00bc873631391ed949513a", size = 31354 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/26/6cee8a1ce8c43625ec561aff19df07f9776b7525d9002c86bceb3e0ac970/pgvector-0.4.2-py3-none-any.whl", hash = "sha256:549d45f7a18593783d5eec609ea1684a724ba8405c4cb182a0b2b08aeff04e08", size = 27441, upload-time = "2025-12-05T01:07:16.536Z" }, + { url = "https://files.pythonhosted.org/packages/5a/26/6cee8a1ce8c43625ec561aff19df07f9776b7525d9002c86bceb3e0ac970/pgvector-0.4.2-py3-none-any.whl", hash = "sha256:549d45f7a18593783d5eec609ea1684a724ba8405c4cb182a0b2b08aeff04e08", size = 27441 }, ] [[package]] name = "pika" version = "1.4.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4b/15/b6a706f1e886335aedc1e21d23913fe1a0fdaadd597d7721b26f11fe306a/pika-1.4.0.tar.gz", hash = "sha256:84aa6d0cf60bbdb79d5780544a4a4e1799392760127bf9de2a03d3c3b92f5f1a", size = 154264, upload-time = "2026-05-06T18:04:32.569Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/15/b6a706f1e886335aedc1e21d23913fe1a0fdaadd597d7721b26f11fe306a/pika-1.4.0.tar.gz", hash = "sha256:84aa6d0cf60bbdb79d5780544a4a4e1799392760127bf9de2a03d3c3b92f5f1a", size = 154264 } wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/07/b7269c5367995648897a40ae37054780fc68b71afcc51c169a739c653d4b/pika-1.4.0-py3-none-any.whl", hash = "sha256:937d8576f92a1ce3673d442161fefef614ac557583e10b9d84c14a6e228ed6a7", size = 164964, upload-time = "2026-05-06T18:04:31.343Z" }, + { url = "https://files.pythonhosted.org/packages/dd/07/b7269c5367995648897a40ae37054780fc68b71afcc51c169a739c653d4b/pika-1.4.0-py3-none-any.whl", hash = "sha256:937d8576f92a1ce3673d442161fefef614ac557583e10b9d84c14a6e228ed6a7", size = 164964 }, ] [[package]] name = "pillow" version = "12.2.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819 } wheels = [ - { url = "https://files.pythonhosted.org/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5", size = 5308279, upload-time = "2026-04-01T14:43:13.246Z" }, - { url = "https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421", size = 4695490, upload-time = "2026-04-01T14:43:15.584Z" }, - { url = "https://files.pythonhosted.org/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987", size = 6284462, upload-time = "2026-04-01T14:43:18.268Z" }, - { url = "https://files.pythonhosted.org/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76", size = 8094744, upload-time = "2026-04-01T14:43:20.716Z" }, - { url = "https://files.pythonhosted.org/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005", size = 6398371, upload-time = "2026-04-01T14:43:23.443Z" }, - { url = "https://files.pythonhosted.org/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780", size = 7087215, upload-time = "2026-04-01T14:43:26.758Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f8/2f6825e441d5b1959d2ca5adec984210f1ec086435b0ed5f52c19b3b8a6e/pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5", size = 6509783, upload-time = "2026-04-01T14:43:29.56Z" }, - { url = "https://files.pythonhosted.org/packages/67/f9/029a27095ad20f854f9dba026b3ea6428548316e057e6fc3545409e86651/pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5", size = 7212112, upload-time = "2026-04-01T14:43:32.091Z" }, - { url = "https://files.pythonhosted.org/packages/be/42/025cfe05d1be22dbfdb4f264fe9de1ccda83f66e4fc3aac94748e784af04/pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940", size = 6378489, upload-time = "2026-04-01T14:43:34.601Z" }, - { url = "https://files.pythonhosted.org/packages/5d/7b/25a221d2c761c6a8ae21bfa3874988ff2583e19cf8a27bf2fee358df7942/pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5", size = 7084129, upload-time = "2026-04-01T14:43:37.213Z" }, - { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612, upload-time = "2026-04-01T14:43:39.421Z" }, + { url = "https://files.pythonhosted.org/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5", size = 5308279 }, + { url = "https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421", size = 4695490 }, + { url = "https://files.pythonhosted.org/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987", size = 6284462 }, + { url = "https://files.pythonhosted.org/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76", size = 8094744 }, + { url = "https://files.pythonhosted.org/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005", size = 6398371 }, + { url = "https://files.pythonhosted.org/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780", size = 7087215 }, + { url = "https://files.pythonhosted.org/packages/8b/f8/2f6825e441d5b1959d2ca5adec984210f1ec086435b0ed5f52c19b3b8a6e/pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5", size = 6509783 }, + { url = "https://files.pythonhosted.org/packages/67/f9/029a27095ad20f854f9dba026b3ea6428548316e057e6fc3545409e86651/pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5", size = 7212112 }, + { url = "https://files.pythonhosted.org/packages/be/42/025cfe05d1be22dbfdb4f264fe9de1ccda83f66e4fc3aac94748e784af04/pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940", size = 6378489 }, + { url = "https://files.pythonhosted.org/packages/5d/7b/25a221d2c761c6a8ae21bfa3874988ff2583e19cf8a27bf2fee358df7942/pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5", size = 7084129 }, + { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612 }, ] [[package]] @@ -2385,45 +2397,45 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bc/9d/07a7f2136ce04cabd21d69c057dc2915867082b0047e6873e424388d4475/pinecone-7.0.1.tar.gz", hash = "sha256:49ff7b0f5be4a2ddec5aaa709758a9f2df56baa58ad46507d081409e246a81ec", size = 207930, upload-time = "2025-05-21T19:39:01.218Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bc/9d/07a7f2136ce04cabd21d69c057dc2915867082b0047e6873e424388d4475/pinecone-7.0.1.tar.gz", hash = "sha256:49ff7b0f5be4a2ddec5aaa709758a9f2df56baa58ad46507d081409e246a81ec", size = 207930 } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/88/896221e991077d353e61991b759f46d75f3b4298eb5a4aa6534c1371f4b0/pinecone-7.0.1-py3-none-any.whl", hash = "sha256:ce7b0dab3c9f7d81e75b24c13fcbca4a51371e08021faaecaf0cd9a45ca1be6c", size = 516590, upload-time = "2025-05-21T19:38:59.117Z" }, + { url = "https://files.pythonhosted.org/packages/81/88/896221e991077d353e61991b759f46d75f3b4298eb5a4aa6534c1371f4b0/pinecone-7.0.1-py3-none-any.whl", hash = "sha256:ce7b0dab3c9f7d81e75b24c13fcbca4a51371e08021faaecaf0cd9a45ca1be6c", size = 516590 }, ] [[package]] name = "pinecone-plugin-interface" version = "0.0.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f4/fb/e8a4063264953ead9e2b24d9b390152c60f042c951c47f4592e9996e57ff/pinecone_plugin_interface-0.0.7.tar.gz", hash = "sha256:b8e6675e41847333aa13923cc44daa3f85676d7157324682dc1640588a982846", size = 3370, upload-time = "2024-06-05T01:57:52.093Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/fb/e8a4063264953ead9e2b24d9b390152c60f042c951c47f4592e9996e57ff/pinecone_plugin_interface-0.0.7.tar.gz", hash = "sha256:b8e6675e41847333aa13923cc44daa3f85676d7157324682dc1640588a982846", size = 3370 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/1d/a21fdfcd6d022cb64cef5c2a29ee6691c6c103c4566b41646b080b7536a5/pinecone_plugin_interface-0.0.7-py3-none-any.whl", hash = "sha256:875857ad9c9fc8bbc074dbe780d187a2afd21f5bfe0f3b08601924a61ef1bba8", size = 6249, upload-time = "2024-06-05T01:57:50.583Z" }, + { url = "https://files.pythonhosted.org/packages/3b/1d/a21fdfcd6d022cb64cef5c2a29ee6691c6c103c4566b41646b080b7536a5/pinecone_plugin_interface-0.0.7-py3-none-any.whl", hash = "sha256:875857ad9c9fc8bbc074dbe780d187a2afd21f5bfe0f3b08601924a61ef1bba8", size = 6249 }, ] [[package]] name = "platformdirs" version = "4.9.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400 } wheels = [ - { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, + { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348 }, ] [[package]] name = "pluggy" version = "1.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412 } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 }, ] [[package]] name = "ply" version = "3.11" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e5/69/882ee5c9d017149285cab114ebeab373308ef0f874fcdac9beb90e0ac4da/ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3", size = 159130, upload-time = "2018-02-15T19:01:31.097Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/69/882ee5c9d017149285cab114ebeab373308ef0f874fcdac9beb90e0ac4da/ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3", size = 159130 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567, upload-time = "2018-02-15T19:01:27.172Z" }, + { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567 }, ] [[package]] @@ -2433,9 +2445,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pywin32", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5e/77/65b857a69ed876e1951e88aaba60f5ce6120c33703f7cb61a3c894b8c1b6/portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac", size = 95644, upload-time = "2025-06-14T13:20:40.03Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/77/65b857a69ed876e1951e88aaba60f5ce6120c33703f7cb61a3c894b8c1b6/portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac", size = 95644 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/a6/38c8e2f318bf67d338f4d629e93b0b4b9af331f455f0390ea8ce4a099b26/portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968", size = 22424, upload-time = "2025-06-14T13:20:38.083Z" }, + { url = "https://files.pythonhosted.org/packages/4b/a6/38c8e2f318bf67d338f4d629e93b0b4b9af331f455f0390ea8ce4a099b26/portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968", size = 22424 }, ] [[package]] @@ -2449,18 +2461,18 @@ dependencies = [ { name = "pyyaml" }, { name = "virtualenv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/04/bd/8a672a86e68f542c3f2ae17a9a8fa63babf16d1107be2f5290e5aa4369ba/pre_commit-3.6.2.tar.gz", hash = "sha256:c3ef34f463045c88658c5b99f38c1e297abdcc0ff13f98d3370055fbbfabc67e", size = 177293, upload-time = "2024-02-18T18:19:41.431Z" } +sdist = { url = "https://files.pythonhosted.org/packages/04/bd/8a672a86e68f542c3f2ae17a9a8fa63babf16d1107be2f5290e5aa4369ba/pre_commit-3.6.2.tar.gz", hash = "sha256:c3ef34f463045c88658c5b99f38c1e297abdcc0ff13f98d3370055fbbfabc67e", size = 177293 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/7c/f7a50d07ae9fa86d2149d4acb2daf61e7c0257b56bc1a24a7fb09c1b70df/pre_commit-3.6.2-py2.py3-none-any.whl", hash = "sha256:ba637c2d7a670c10daedc059f5c49b5bd0aadbccfcd7ec15592cf9665117532c", size = 204185, upload-time = "2024-02-18T18:19:38.953Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7c/f7a50d07ae9fa86d2149d4acb2daf61e7c0257b56bc1a24a7fb09c1b70df/pre_commit-3.6.2-py2.py3-none-any.whl", hash = "sha256:ba637c2d7a670c10daedc059f5c49b5bd0aadbccfcd7ec15592cf9665117532c", size = 204185 }, ] [[package]] name = "prometheus-client" version = "0.25.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/fb/d9aa83ffe43ce1f19e557c0971d04b90561b0cfd50762aafb01968285553/prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28", size = 86035, upload-time = "2026-04-09T19:53:42.359Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/fb/d9aa83ffe43ce1f19e557c0971d04b90561b0cfd50762aafb01968285553/prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28", size = 86035 } wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/9b/d4b1e644385499c8346fa9b622a3f030dce14cd6ef8a1871c221a17a67e7/prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1", size = 64154, upload-time = "2026-04-09T19:53:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/8d/9b/d4b1e644385499c8346fa9b622a3f030dce14cd6ef8a1871c221a17a67e7/prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1", size = 64154 }, ] [[package]] @@ -2470,35 +2482,35 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "wcwidth" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198 } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431 }, ] [[package]] name = "propcache" version = "0.5.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, - { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, - { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, - { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, - { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, - { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, - { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, - { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, - { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, - { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, - { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, - { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, - { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, - { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, - { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, - { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, - { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, - { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887 }, + { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654 }, + { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190 }, + { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995 }, + { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422 }, + { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342 }, + { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639 }, + { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588 }, + { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029 }, + { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774 }, + { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532 }, + { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592 }, + { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788 }, + { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514 }, + { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018 }, + { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322 }, + { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172 }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036 }, ] [[package]] @@ -2508,81 +2520,81 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c9/56/e647b0c675392d2da368da7b6f158f7368b18542fd6f7d7400a2f39de000/proto_plus-1.28.0.tar.gz", hash = "sha256:38e5696342835b08fc116f30a25665b29531cda9d5d5643e9b81fc312385abd9", size = 57221, upload-time = "2026-05-07T08:04:50.811Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/56/e647b0c675392d2da368da7b6f158f7368b18542fd6f7d7400a2f39de000/proto_plus-1.28.0.tar.gz", hash = "sha256:38e5696342835b08fc116f30a25665b29531cda9d5d5643e9b81fc312385abd9", size = 57221 } wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/20/b122d4626976acb81132036d2ad1bb35a1a8775fceb837ec30964622516a/proto_plus-1.28.0-py3-none-any.whl", hash = "sha256:a630604310899e73c59ec302e5765c058d412b2f090b9c79c8822589f14955b8", size = 50410, upload-time = "2026-05-07T08:03:31.962Z" }, + { url = "https://files.pythonhosted.org/packages/7c/20/b122d4626976acb81132036d2ad1bb35a1a8775fceb837ec30964622516a/proto_plus-1.28.0-py3-none-any.whl", hash = "sha256:a630604310899e73c59ec302e5765c058d412b2f090b9c79c8822589f14955b8", size = 50410 }, ] [[package]] name = "protobuf" version = "4.25.9" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d9/8e/d08c41a8c004e1d437ef467e7c4f9c3295cd784eba48ed5d1d01f94b1dad/protobuf-4.25.9.tar.gz", hash = "sha256:b0dc7e7c68de8b1ce831dacb12fb407e838edbb8b6cc0dc3a2a6b4cbf6de9cff", size = 381040, upload-time = "2026-03-25T23:09:36.423Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/8e/d08c41a8c004e1d437ef467e7c4f9c3295cd784eba48ed5d1d01f94b1dad/protobuf-4.25.9.tar.gz", hash = "sha256:b0dc7e7c68de8b1ce831dacb12fb407e838edbb8b6cc0dc3a2a6b4cbf6de9cff", size = 381040 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/e9/59435bd04bdd46cb38c42a336b22f9843e8e586ff83c35a5423f8b14704e/protobuf-4.25.9-cp310-abi3-win32.whl", hash = "sha256:bde396f568b0b46fc8fbfe9f02facf25b6755b2578a3b8ac61e74b9d69499e03", size = 392879, upload-time = "2026-03-25T23:09:21.32Z" }, - { url = "https://files.pythonhosted.org/packages/f3/16/42a5c7f1001783d2b5bfcecde10127f09010f78982c86ae409122ce3ece6/protobuf-4.25.9-cp310-abi3-win_amd64.whl", hash = "sha256:3683c05154252206f7cb2d371626514b3708199d9bcf683b503dabf3a2e38e06", size = 413900, upload-time = "2026-03-25T23:09:23.589Z" }, - { url = "https://files.pythonhosted.org/packages/56/5b/0074a0a9eb01f3d1c4648ca5e81b22090c811b210b61df9018ac6d6c5cda/protobuf-4.25.9-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:9560813560e6ee72c11ca8873878bdb7ee003c96a57ebb013245fe84e2540904", size = 394826, upload-time = "2026-03-25T23:09:25.194Z" }, - { url = "https://files.pythonhosted.org/packages/54/aa/b2dba856f64c36b2a06c67be1472de98cca07a2322d0f0cbf03279a40e5b/protobuf-4.25.9-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:999146ef02e7fa6a692477badd1528bcd7268df211852a3df2d834ba2b480791", size = 294191, upload-time = "2026-03-25T23:09:26.613Z" }, - { url = "https://files.pythonhosted.org/packages/a8/5c/53f18822017b8bda6bd8bb4e02048e911fdc79a3dafdc83ab994fe922a84/protobuf-4.25.9-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:438c636de8fb706a0de94a12a268ef1ae8f5ba5ae655a7671fcda5968ba3c9be", size = 295178, upload-time = "2026-03-25T23:09:27.839Z" }, - { url = "https://files.pythonhosted.org/packages/16/28/d5065b212685875d3924bcdb3201cbf467cb4d58a18aa19a8dfd99ea80a9/protobuf-4.25.9-py3-none-any.whl", hash = "sha256:d49b615e7c935194ac161f0965699ac84df6112c378e05ec53da65d2e4cbb6d4", size = 156822, upload-time = "2026-03-25T23:09:34.957Z" }, + { url = "https://files.pythonhosted.org/packages/a8/e9/59435bd04bdd46cb38c42a336b22f9843e8e586ff83c35a5423f8b14704e/protobuf-4.25.9-cp310-abi3-win32.whl", hash = "sha256:bde396f568b0b46fc8fbfe9f02facf25b6755b2578a3b8ac61e74b9d69499e03", size = 392879 }, + { url = "https://files.pythonhosted.org/packages/f3/16/42a5c7f1001783d2b5bfcecde10127f09010f78982c86ae409122ce3ece6/protobuf-4.25.9-cp310-abi3-win_amd64.whl", hash = "sha256:3683c05154252206f7cb2d371626514b3708199d9bcf683b503dabf3a2e38e06", size = 413900 }, + { url = "https://files.pythonhosted.org/packages/56/5b/0074a0a9eb01f3d1c4648ca5e81b22090c811b210b61df9018ac6d6c5cda/protobuf-4.25.9-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:9560813560e6ee72c11ca8873878bdb7ee003c96a57ebb013245fe84e2540904", size = 394826 }, + { url = "https://files.pythonhosted.org/packages/54/aa/b2dba856f64c36b2a06c67be1472de98cca07a2322d0f0cbf03279a40e5b/protobuf-4.25.9-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:999146ef02e7fa6a692477badd1528bcd7268df211852a3df2d834ba2b480791", size = 294191 }, + { url = "https://files.pythonhosted.org/packages/a8/5c/53f18822017b8bda6bd8bb4e02048e911fdc79a3dafdc83ab994fe922a84/protobuf-4.25.9-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:438c636de8fb706a0de94a12a268ef1ae8f5ba5ae655a7671fcda5968ba3c9be", size = 295178 }, + { url = "https://files.pythonhosted.org/packages/16/28/d5065b212685875d3924bcdb3201cbf467cb4d58a18aa19a8dfd99ea80a9/protobuf-4.25.9-py3-none-any.whl", hash = "sha256:d49b615e7c935194ac161f0965699ac84df6112c378e05ec53da65d2e4cbb6d4", size = 156822 }, ] [[package]] name = "psutil" version = "5.9.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/90/c7/6dc0a455d111f68ee43f27793971cf03fe29b6ef972042549db29eec39a2/psutil-5.9.8.tar.gz", hash = "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c", size = 503247, upload-time = "2024-01-19T20:47:09.517Z" } +sdist = { url = "https://files.pythonhosted.org/packages/90/c7/6dc0a455d111f68ee43f27793971cf03fe29b6ef972042549db29eec39a2/psutil-5.9.8.tar.gz", hash = "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c", size = 503247 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/e3/07ae864a636d70a8a6f58da27cb1179192f1140d5d1da10886ade9405797/psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81", size = 248702, upload-time = "2024-01-19T20:47:36.303Z" }, - { url = "https://files.pythonhosted.org/packages/b3/bd/28c5f553667116b2598b9cc55908ec435cb7f77a34f2bff3e3ca765b0f78/psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421", size = 285242, upload-time = "2024-01-19T20:47:39.65Z" }, - { url = "https://files.pythonhosted.org/packages/c5/4f/0e22aaa246f96d6ac87fe5ebb9c5a693fbe8877f537a1022527c47ca43c5/psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4", size = 288191, upload-time = "2024-01-19T20:47:43.078Z" }, - { url = "https://files.pythonhosted.org/packages/6e/f5/2aa3a4acdc1e5940b59d421742356f133185667dd190b166dbcfcf5d7b43/psutil-5.9.8-cp37-abi3-win32.whl", hash = "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0", size = 251252, upload-time = "2024-01-19T20:47:52.88Z" }, - { url = "https://files.pythonhosted.org/packages/93/52/3e39d26feae7df0aa0fd510b14012c3678b36ed068f7d78b8d8784d61f0e/psutil-5.9.8-cp37-abi3-win_amd64.whl", hash = "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf", size = 255090, upload-time = "2024-01-19T20:47:56.019Z" }, - { url = "https://files.pythonhosted.org/packages/05/33/2d74d588408caedd065c2497bdb5ef83ce6082db01289a1e1147f6639802/psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8", size = 249898, upload-time = "2024-01-19T20:47:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/e7/e3/07ae864a636d70a8a6f58da27cb1179192f1140d5d1da10886ade9405797/psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81", size = 248702 }, + { url = "https://files.pythonhosted.org/packages/b3/bd/28c5f553667116b2598b9cc55908ec435cb7f77a34f2bff3e3ca765b0f78/psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421", size = 285242 }, + { url = "https://files.pythonhosted.org/packages/c5/4f/0e22aaa246f96d6ac87fe5ebb9c5a693fbe8877f537a1022527c47ca43c5/psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4", size = 288191 }, + { url = "https://files.pythonhosted.org/packages/6e/f5/2aa3a4acdc1e5940b59d421742356f133185667dd190b166dbcfcf5d7b43/psutil-5.9.8-cp37-abi3-win32.whl", hash = "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0", size = 251252 }, + { url = "https://files.pythonhosted.org/packages/93/52/3e39d26feae7df0aa0fd510b14012c3678b36ed068f7d78b8d8784d61f0e/psutil-5.9.8-cp37-abi3-win_amd64.whl", hash = "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf", size = 255090 }, + { url = "https://files.pythonhosted.org/packages/05/33/2d74d588408caedd065c2497bdb5ef83ce6082db01289a1e1147f6639802/psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8", size = 249898 }, ] [[package]] name = "psycopg2-binary" version = "2.9.9" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/07/e720e53bfab016ebcc34241695ccc06a9e3d91ba19b40ca81317afbdc440/psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c", size = 384973, upload-time = "2023-10-03T12:48:55.128Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/07/e720e53bfab016ebcc34241695ccc06a9e3d91ba19b40ca81317afbdc440/psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c", size = 384973 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/d0/5f2db14e7b53552276ab613399a83f83f85b173a862d3f20580bc7231139/psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf", size = 2823784, upload-time = "2023-10-03T12:47:00.404Z" }, - { url = "https://files.pythonhosted.org/packages/18/ca/da384fd47233e300e3e485c90e7aab5d7def896d1281239f75901faf87d4/psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d", size = 2553308, upload-time = "2023-11-01T10:40:33.984Z" }, - { url = "https://files.pythonhosted.org/packages/50/66/fa53d2d3d92f6e1ef469d92afc6a4fe3f6e8a9a04b687aa28fb1f1d954ee/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212", size = 2851283, upload-time = "2023-10-03T12:47:02.736Z" }, - { url = "https://files.pythonhosted.org/packages/04/37/2429360ac5547378202db14eec0dde76edbe1f6627df5a43c7e164922859/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493", size = 3081839, upload-time = "2023-10-03T12:47:05.027Z" }, - { url = "https://files.pythonhosted.org/packages/62/2a/c0530b59d7e0d09824bc2102ecdcec0456b8ca4d47c0caa82e86fce3ed4c/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996", size = 3264488, upload-time = "2023-10-03T12:47:08.962Z" }, - { url = "https://files.pythonhosted.org/packages/19/57/9f172b900795ea37246c78b5f52e00f4779984370855b3e161600156906d/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119", size = 3020700, upload-time = "2023-10-03T12:47:12.23Z" }, - { url = "https://files.pythonhosted.org/packages/94/68/1176fc14ea76861b7b8360be5176e87fb20d5091b137c76570eb4e237324/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba", size = 2355968, upload-time = "2023-10-03T12:47:14.817Z" }, - { url = "https://files.pythonhosted.org/packages/70/bb/aec2646a705a09079d008ce88073401cd61fc9b04f92af3eb282caa3a2ec/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07", size = 2536101, upload-time = "2023-10-03T12:47:17.454Z" }, - { url = "https://files.pythonhosted.org/packages/14/33/12818c157e333cb9d9e6753d1b2463b6f60dbc1fade115f8e4dc5c52cac4/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb", size = 2487064, upload-time = "2023-10-03T12:47:20.717Z" }, - { url = "https://files.pythonhosted.org/packages/56/a2/7851c68fe8768f3c9c246198b6356ee3e4a8a7f6820cc798443faada3400/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe", size = 2456257, upload-time = "2023-10-03T12:47:23.004Z" }, - { url = "https://files.pythonhosted.org/packages/6f/ee/3ba07c6dc7c3294e717e94720da1597aedc82a10b1b180203ce183d4631a/psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93", size = 1024709, upload-time = "2023-10-28T09:37:24.991Z" }, - { url = "https://files.pythonhosted.org/packages/7b/08/9c66c269b0d417a0af9fb969535f0371b8c538633535a7a6a5ca3f9231e2/psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab", size = 1163864, upload-time = "2023-10-28T09:37:28.155Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d0/5f2db14e7b53552276ab613399a83f83f85b173a862d3f20580bc7231139/psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf", size = 2823784 }, + { url = "https://files.pythonhosted.org/packages/18/ca/da384fd47233e300e3e485c90e7aab5d7def896d1281239f75901faf87d4/psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d", size = 2553308 }, + { url = "https://files.pythonhosted.org/packages/50/66/fa53d2d3d92f6e1ef469d92afc6a4fe3f6e8a9a04b687aa28fb1f1d954ee/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212", size = 2851283 }, + { url = "https://files.pythonhosted.org/packages/04/37/2429360ac5547378202db14eec0dde76edbe1f6627df5a43c7e164922859/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493", size = 3081839 }, + { url = "https://files.pythonhosted.org/packages/62/2a/c0530b59d7e0d09824bc2102ecdcec0456b8ca4d47c0caa82e86fce3ed4c/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996", size = 3264488 }, + { url = "https://files.pythonhosted.org/packages/19/57/9f172b900795ea37246c78b5f52e00f4779984370855b3e161600156906d/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119", size = 3020700 }, + { url = "https://files.pythonhosted.org/packages/94/68/1176fc14ea76861b7b8360be5176e87fb20d5091b137c76570eb4e237324/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba", size = 2355968 }, + { url = "https://files.pythonhosted.org/packages/70/bb/aec2646a705a09079d008ce88073401cd61fc9b04f92af3eb282caa3a2ec/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07", size = 2536101 }, + { url = "https://files.pythonhosted.org/packages/14/33/12818c157e333cb9d9e6753d1b2463b6f60dbc1fade115f8e4dc5c52cac4/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb", size = 2487064 }, + { url = "https://files.pythonhosted.org/packages/56/a2/7851c68fe8768f3c9c246198b6356ee3e4a8a7f6820cc798443faada3400/psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe", size = 2456257 }, + { url = "https://files.pythonhosted.org/packages/6f/ee/3ba07c6dc7c3294e717e94720da1597aedc82a10b1b180203ce183d4631a/psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93", size = 1024709 }, + { url = "https://files.pythonhosted.org/packages/7b/08/9c66c269b0d417a0af9fb969535f0371b8c538633535a7a6a5ca3f9231e2/psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab", size = 1163864 }, ] [[package]] name = "pyarrow" version = "18.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7f/7b/640785a9062bb00314caa8a387abce547d2a420cf09bd6c715fe659ccffb/pyarrow-18.1.0.tar.gz", hash = "sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73", size = 1118671, upload-time = "2024-11-26T02:01:48.62Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7f/7b/640785a9062bb00314caa8a387abce547d2a420cf09bd6c715fe659ccffb/pyarrow-18.1.0.tar.gz", hash = "sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73", size = 1118671 } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/50/12829e7111b932581e51dda51d5cb39207a056c30fe31ef43f14c63c4d7e/pyarrow-18.1.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d", size = 29514620, upload-time = "2024-11-26T01:59:39.797Z" }, - { url = "https://files.pythonhosted.org/packages/d1/41/468c944eab157702e96abab3d07b48b8424927d4933541ab43788bb6964d/pyarrow-18.1.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee", size = 30856494, upload-time = "2024-11-26T01:59:44.725Z" }, - { url = "https://files.pythonhosted.org/packages/68/f9/29fb659b390312a7345aeb858a9d9c157552a8852522f2c8bad437c29c0a/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992", size = 39203624, upload-time = "2024-11-26T01:59:49.189Z" }, - { url = "https://files.pythonhosted.org/packages/6e/f6/19360dae44200e35753c5c2889dc478154cd78e61b1f738514c9f131734d/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54", size = 40139341, upload-time = "2024-11-26T01:59:54.849Z" }, - { url = "https://files.pythonhosted.org/packages/bb/e6/9b3afbbcf10cc724312e824af94a2e993d8ace22994d823f5c35324cebf5/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33", size = 38618629, upload-time = "2024-11-26T01:59:59.966Z" }, - { url = "https://files.pythonhosted.org/packages/3a/2e/3b99f8a3d9e0ccae0e961978a0d0089b25fb46ebbcfb5ebae3cca179a5b3/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30", size = 40078661, upload-time = "2024-11-26T02:00:04.55Z" }, - { url = "https://files.pythonhosted.org/packages/76/52/f8da04195000099d394012b8d42c503d7041b79f778d854f410e5f05049a/pyarrow-18.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99", size = 25092330, upload-time = "2024-11-26T02:00:09.576Z" }, + { url = "https://files.pythonhosted.org/packages/6a/50/12829e7111b932581e51dda51d5cb39207a056c30fe31ef43f14c63c4d7e/pyarrow-18.1.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d", size = 29514620 }, + { url = "https://files.pythonhosted.org/packages/d1/41/468c944eab157702e96abab3d07b48b8424927d4933541ab43788bb6964d/pyarrow-18.1.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee", size = 30856494 }, + { url = "https://files.pythonhosted.org/packages/68/f9/29fb659b390312a7345aeb858a9d9c157552a8852522f2c8bad437c29c0a/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992", size = 39203624 }, + { url = "https://files.pythonhosted.org/packages/6e/f6/19360dae44200e35753c5c2889dc478154cd78e61b1f738514c9f131734d/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54", size = 40139341 }, + { url = "https://files.pythonhosted.org/packages/bb/e6/9b3afbbcf10cc724312e824af94a2e993d8ace22994d823f5c35324cebf5/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33", size = 38618629 }, + { url = "https://files.pythonhosted.org/packages/3a/2e/3b99f8a3d9e0ccae0e961978a0d0089b25fb46ebbcfb5ebae3cca179a5b3/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30", size = 40078661 }, + { url = "https://files.pythonhosted.org/packages/76/52/f8da04195000099d394012b8d42c503d7041b79f778d854f410e5f05049a/pyarrow-18.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99", size = 25092330 }, ] [[package]] name = "pyasn1" version = "0.6.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997 }, ] [[package]] @@ -2592,9 +2604,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyasn1" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892 } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259 }, ] [[package]] @@ -2608,37 +2620,37 @@ dependencies = [ { name = "tomlkit" }, { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a8/1b/be7329dcde1bf5126c7367ddc3b87945e3c9a208eb161b732bebad014eb9/pycln-2.6.0.tar.gz", hash = "sha256:758cfd07f0d9711f69989bd4cef7e451e3d7166073b0c0f77d4a712dbac8b3fe", size = 33427, upload-time = "2025-10-21T20:06:17.902Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/1b/be7329dcde1bf5126c7367ddc3b87945e3c9a208eb161b732bebad014eb9/pycln-2.6.0.tar.gz", hash = "sha256:758cfd07f0d9711f69989bd4cef7e451e3d7166073b0c0f77d4a712dbac8b3fe", size = 33427 } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/67/5aa8a63e1af76d29c67ff85d0c2f44dd294044fadacb19beb4bae7eafa01/pycln-2.6.0-py3-none-any.whl", hash = "sha256:32faabf77daf2a8995238f7522399d9e6f2dc40bcfe90e619299512cdb5bd8f2", size = 37745, upload-time = "2025-10-21T20:06:16.92Z" }, + { url = "https://files.pythonhosted.org/packages/db/67/5aa8a63e1af76d29c67ff85d0c2f44dd294044fadacb19beb4bae7eafa01/pycln-2.6.0-py3-none-any.whl", hash = "sha256:32faabf77daf2a8995238f7522399d9e6f2dc40bcfe90e619299512cdb5bd8f2", size = 37745 }, ] [[package]] name = "pycparser" version = "3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492 } wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172 }, ] [[package]] name = "pycryptodome" version = "3.23.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276 } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" }, - { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" }, - { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" }, - { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" }, - { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" }, - { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" }, - { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" }, - { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" }, - { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" }, - { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, + { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627 }, + { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362 }, + { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625 }, + { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954 }, + { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534 }, + { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853 }, + { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465 }, + { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414 }, + { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484 }, + { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636 }, + { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675 }, ] [[package]] @@ -2651,9 +2663,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775 } wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262 }, ] [[package]] @@ -2663,27 +2675,27 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, - { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, - { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, - { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, - { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, - { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, - { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, - { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, - { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, - { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, - { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, - { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, - { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, - { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, - { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, - { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, - { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158 }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724 }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742 }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418 }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274 }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940 }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516 }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854 }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306 }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044 }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133 }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464 }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823 }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919 }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604 }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527 }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024 }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696 }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590 }, ] [[package]] @@ -2696,9 +2708,9 @@ dependencies = [ { name = "pyopenssl" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/45/74/b591079fa588351cec61861b85ba26f7deb96f3b445556c100e17db5572b/PyDrive2-1.15.4.tar.gz", hash = "sha256:0c011b74ebc24f3c6ca72820626b77f1dfe0ae88f5740c5a5cf96e83dd79ba99", size = 60514, upload-time = "2023-05-21T02:25:57.217Z" } +sdist = { url = "https://files.pythonhosted.org/packages/45/74/b591079fa588351cec61861b85ba26f7deb96f3b445556c100e17db5572b/PyDrive2-1.15.4.tar.gz", hash = "sha256:0c011b74ebc24f3c6ca72820626b77f1dfe0ae88f5740c5a5cf96e83dd79ba99", size = 60514 } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/f4/d0b40ee1c703304e8cc737e53516f834c0fbad4fe9b27aed7680d9fdf344/PyDrive2-1.15.4-py3-none-any.whl", hash = "sha256:91fe28e5f094a6dfff834495c4aee0041cbef979467ad27cd0d4b1f91afa8869", size = 45011, upload-time = "2023-05-21T02:25:55.265Z" }, + { url = "https://files.pythonhosted.org/packages/18/f4/d0b40ee1c703304e8cc737e53516f834c0fbad4fe9b27aed7680d9fdf344/PyDrive2-1.15.4-py3-none-any.whl", hash = "sha256:91fe28e5f094a6dfff834495c4aee0041cbef979467ad27cd0d4b1f91afa8869", size = 45011 }, ] [package.optional-dependencies] @@ -2713,18 +2725,18 @@ fsspec = [ name = "pygments" version = "2.20.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151 }, ] [[package]] name = "pyjwt" version = "2.12.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c2/27/a3b6e5bf6ff856d2509292e95c8f57f0df7017cf5394921fc4e4ef40308a/pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b", size = 102564, upload-time = "2026-03-13T19:27:37.25Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/27/a3b6e5bf6ff856d2509292e95c8f57f0df7017cf5394921fc4e4ef40308a/pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b", size = 102564 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/7a/8dd906bd22e79e47397a61742927f6747fe93242ef86645ee9092e610244/pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c", size = 29726, upload-time = "2026-03-13T19:27:35.677Z" }, + { url = "https://files.pythonhosted.org/packages/e5/7a/8dd906bd22e79e47397a61742927f6747fe93242ef86645ee9092e610244/pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c", size = 29726 }, ] [package.optional-dependencies] @@ -2745,37 +2757,37 @@ dependencies = [ { name = "setuptools" }, { name = "ujson" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d3/13/899185f025802ba80255faa8e45b3f3bf9cb7bab2d4235e12e3322c8e2a4/pymilvus-2.5.18.tar.gz", hash = "sha256:9e517076068e98dac51c018bc0dfe1f651d936154e2e2d9ad6c7b3dab1164e2d", size = 1285482, upload-time = "2025-12-02T10:58:25.399Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/13/899185f025802ba80255faa8e45b3f3bf9cb7bab2d4235e12e3322c8e2a4/pymilvus-2.5.18.tar.gz", hash = "sha256:9e517076068e98dac51c018bc0dfe1f651d936154e2e2d9ad6c7b3dab1164e2d", size = 1285482 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/9c/a2b50b2b369814003460ca12a3c195fbf11b89bc1a861c2aa737c33ad7f9/pymilvus-2.5.18-py3-none-any.whl", hash = "sha256:1b78badcfa8d62db7d0b29193fc0422e4676873ff1c745a9d75c2c885d7a7e32", size = 244089, upload-time = "2025-12-02T10:58:23.944Z" }, + { url = "https://files.pythonhosted.org/packages/c0/9c/a2b50b2b369814003460ca12a3c195fbf11b89bc1a861c2aa737c33ad7f9/pymilvus-2.5.18-py3-none-any.whl", hash = "sha256:1b78badcfa8d62db7d0b29193fc0422e4676873ff1c745a9d75c2c885d7a7e32", size = 244089 }, ] [[package]] name = "pymssql" version = "2.3.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/35/5a0b79369e42fffd5c04e4e74fa90ef034cc5c3f314e14f6d58cac646ccf/pymssql-2.3.4.tar.gz", hash = "sha256:117c82d7aa9021171aa9be98368475519f33d9c32073cdcf9b0d76231abc6436", size = 184604, upload-time = "2025-04-02T02:08:43.503Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/35/5a0b79369e42fffd5c04e4e74fa90ef034cc5c3f314e14f6d58cac646ccf/pymssql-2.3.4.tar.gz", hash = "sha256:117c82d7aa9021171aa9be98368475519f33d9c32073cdcf9b0d76231abc6436", size = 184604 } wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/89/5a7a4b27ee44b2dc4708de7e897311cb17f15e7c983c299e8bf97ebf98d1/pymssql-2.3.4-cp312-cp312-macosx_13_0_universal2.whl", hash = "sha256:809b75aaeb9bcd061230bace41e275f80f464f70fcbf5dde2ba7ba8f0eea5298", size = 3075736, upload-time = "2025-04-02T02:11:44.347Z" }, - { url = "https://files.pythonhosted.org/packages/43/f9/19bbb0026a47043fb239e821e10a75304b12ba986ce4af71cf8986af411c/pymssql-2.3.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48ab1ee04754fb8ce703b6c154e54fde4f6c7f440766d397b101b748123a12df", size = 4019433, upload-time = "2025-04-02T03:07:58.222Z" }, - { url = "https://files.pythonhosted.org/packages/a6/ac/3aca13f1f527299db4adef594fb9f14d47d68de91b93a220a67391b8ec87/pymssql-2.3.4-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e22bb4d5aed85b084e3b9fb5ae3463301dd69c17703cfef72e0aed746452cc9", size = 3993550, upload-time = "2025-04-02T02:13:16.433Z" }, - { url = "https://files.pythonhosted.org/packages/b9/93/879d92f61afb974f69b9186b16ee6a97adff2abc82777e3b66c9c9efb179/pymssql-2.3.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2c1c8d3632630d52387e5b5b4483027494b5cb8f65401573715b74e7a3f16e5", size = 4381934, upload-time = "2025-04-02T02:12:45.424Z" }, - { url = "https://files.pythonhosted.org/packages/6c/a6/923769b6dbb4e3a4c07a867e0c7fa8e4b230f675095cd7109d4e3eb9ddf0/pymssql-2.3.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f7f245acbdf89b96a41513ef0214b55a3ba2824f1f3119dd1945443b6cac78d3", size = 4849674, upload-time = "2025-04-02T02:13:05.245Z" }, - { url = "https://files.pythonhosted.org/packages/7a/2d/c787f061dcd0603905bf8085dda9cddb8c3c03b18d9239d5d18c953eebba/pymssql-2.3.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9484485fb847eb67f828459b0f4857c9725b20c517c2b7f88a9788fd72b76a6a", size = 4076649, upload-time = "2025-04-02T02:15:13.053Z" }, - { url = "https://files.pythonhosted.org/packages/c1/a2/e55d823e3ab21cf9fc88e4e2424936899392d9d2e6569d5bcce063f84dac/pymssql-2.3.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4a0716482cd5ecce07230925593cefd9137959c18aca4c92fc24c243d3c20e38", size = 4139477, upload-time = "2025-04-02T02:13:42.91Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7c/0fec6587b38081d0d0fca4f9ad31e85ec6c5791879e57f0e559ec6be4d3d/pymssql-2.3.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:ba4f988674b361709821c8173a6471aa6e47ee6e45b5a8e30d4dcbde1f62fb0f", size = 4653837, upload-time = "2025-04-02T02:15:05.102Z" }, - { url = "https://files.pythonhosted.org/packages/5f/7c/77d0251f4b5ad5690226a93547fc8279c1c48bd14e3ccc820f5c580a3b73/pymssql-2.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:51b8ebfbd7d14d5e7c65e76ffaf31584ffabe9fb1bfd2a85f529bd707512e39d", size = 4910914, upload-time = "2025-04-02T02:13:55.446Z" }, - { url = "https://files.pythonhosted.org/packages/4f/22/1b2ef85804872a5940010d3c012722356af1fa24f8ba6f419c0260881032/pymssql-2.3.4-cp312-cp312-win32.whl", hash = "sha256:c8f5718f5e7d2623eaf35e025d5fa288c5789916809a89f00b42346b888673da", size = 1337991, upload-time = "2025-04-02T02:29:43.394Z" }, - { url = "https://files.pythonhosted.org/packages/0f/43/c98f34e7b3cd45653fb233a4bee83bffca0cf5e78c290c291cec34faac21/pymssql-2.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:d72b38b5ba66a4072c680447099bb63ac35d0425e9a29ff91b048e563b999be5", size = 2021760, upload-time = "2025-04-02T02:28:06.757Z" }, + { url = "https://files.pythonhosted.org/packages/bc/89/5a7a4b27ee44b2dc4708de7e897311cb17f15e7c983c299e8bf97ebf98d1/pymssql-2.3.4-cp312-cp312-macosx_13_0_universal2.whl", hash = "sha256:809b75aaeb9bcd061230bace41e275f80f464f70fcbf5dde2ba7ba8f0eea5298", size = 3075736 }, + { url = "https://files.pythonhosted.org/packages/43/f9/19bbb0026a47043fb239e821e10a75304b12ba986ce4af71cf8986af411c/pymssql-2.3.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48ab1ee04754fb8ce703b6c154e54fde4f6c7f440766d397b101b748123a12df", size = 4019433 }, + { url = "https://files.pythonhosted.org/packages/a6/ac/3aca13f1f527299db4adef594fb9f14d47d68de91b93a220a67391b8ec87/pymssql-2.3.4-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e22bb4d5aed85b084e3b9fb5ae3463301dd69c17703cfef72e0aed746452cc9", size = 3993550 }, + { url = "https://files.pythonhosted.org/packages/b9/93/879d92f61afb974f69b9186b16ee6a97adff2abc82777e3b66c9c9efb179/pymssql-2.3.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2c1c8d3632630d52387e5b5b4483027494b5cb8f65401573715b74e7a3f16e5", size = 4381934 }, + { url = "https://files.pythonhosted.org/packages/6c/a6/923769b6dbb4e3a4c07a867e0c7fa8e4b230f675095cd7109d4e3eb9ddf0/pymssql-2.3.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f7f245acbdf89b96a41513ef0214b55a3ba2824f1f3119dd1945443b6cac78d3", size = 4849674 }, + { url = "https://files.pythonhosted.org/packages/7a/2d/c787f061dcd0603905bf8085dda9cddb8c3c03b18d9239d5d18c953eebba/pymssql-2.3.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9484485fb847eb67f828459b0f4857c9725b20c517c2b7f88a9788fd72b76a6a", size = 4076649 }, + { url = "https://files.pythonhosted.org/packages/c1/a2/e55d823e3ab21cf9fc88e4e2424936899392d9d2e6569d5bcce063f84dac/pymssql-2.3.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4a0716482cd5ecce07230925593cefd9137959c18aca4c92fc24c243d3c20e38", size = 4139477 }, + { url = "https://files.pythonhosted.org/packages/c7/7c/0fec6587b38081d0d0fca4f9ad31e85ec6c5791879e57f0e559ec6be4d3d/pymssql-2.3.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:ba4f988674b361709821c8173a6471aa6e47ee6e45b5a8e30d4dcbde1f62fb0f", size = 4653837 }, + { url = "https://files.pythonhosted.org/packages/5f/7c/77d0251f4b5ad5690226a93547fc8279c1c48bd14e3ccc820f5c580a3b73/pymssql-2.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:51b8ebfbd7d14d5e7c65e76ffaf31584ffabe9fb1bfd2a85f529bd707512e39d", size = 4910914 }, + { url = "https://files.pythonhosted.org/packages/4f/22/1b2ef85804872a5940010d3c012722356af1fa24f8ba6f419c0260881032/pymssql-2.3.4-cp312-cp312-win32.whl", hash = "sha256:c8f5718f5e7d2623eaf35e025d5fa288c5789916809a89f00b42346b888673da", size = 1337991 }, + { url = "https://files.pythonhosted.org/packages/0f/43/c98f34e7b3cd45653fb233a4bee83bffca0cf5e78c290c291cec34faac21/pymssql-2.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:d72b38b5ba66a4072c680447099bb63ac35d0425e9a29ff91b048e563b999be5", size = 2021760 }, ] [[package]] name = "pymysql" version = "1.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/9d/ee68dee1c8821c839bb31e6e5f40e61035a5278f7c1307dde758f0c90452/PyMySQL-1.1.0.tar.gz", hash = "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96", size = 47240, upload-time = "2023-06-26T05:34:02.058Z" } +sdist = { url = "https://files.pythonhosted.org/packages/41/9d/ee68dee1c8821c839bb31e6e5f40e61035a5278f7c1307dde758f0c90452/PyMySQL-1.1.0.tar.gz", hash = "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96", size = 47240 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/30/20467e39523d0cfc2b6227902d3687a16364307260c75e6a1cb4422b0c62/PyMySQL-1.1.0-py3-none-any.whl", hash = "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7", size = 44768, upload-time = "2023-06-26T05:33:59.951Z" }, + { url = "https://files.pythonhosted.org/packages/e5/30/20467e39523d0cfc2b6227902d3687a16364307260c75e6a1cb4422b0c62/PyMySQL-1.1.0-py3-none-any.whl", hash = "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7", size = 44768 }, ] [[package]] @@ -2785,22 +2797,22 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/c6/a3124dee667a423f2c637cfd262a54d67d8ccf3e160f3c50f622a85b7723/pynacl-1.6.0.tar.gz", hash = "sha256:cb36deafe6e2bce3b286e5d1f3e1c246e0ccdb8808ddb4550bb2792f2df298f2", size = 3505641, upload-time = "2025-09-10T23:39:22.308Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/c6/a3124dee667a423f2c637cfd262a54d67d8ccf3e160f3c50f622a85b7723/pynacl-1.6.0.tar.gz", hash = "sha256:cb36deafe6e2bce3b286e5d1f3e1c246e0ccdb8808ddb4550bb2792f2df298f2", size = 3505641 } wheels = [ - { url = "https://files.pythonhosted.org/packages/63/37/87c72df19857c5b3b47ace6f211a26eb862ada495cc96daa372d96048fca/pynacl-1.6.0-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:f4b3824920e206b4f52abd7de621ea7a44fd3cb5c8daceb7c3612345dfc54f2e", size = 382610, upload-time = "2025-09-10T23:38:49.459Z" }, - { url = "https://files.pythonhosted.org/packages/0c/64/3ce958a5817fd3cc6df4ec14441c43fd9854405668d73babccf77f9597a3/pynacl-1.6.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:16dd347cdc8ae0b0f6187a2608c0af1c8b7ecbbe6b4a06bff8253c192f696990", size = 798744, upload-time = "2025-09-10T23:38:58.531Z" }, - { url = "https://files.pythonhosted.org/packages/e4/8a/3f0dd297a0a33fa3739c255feebd0206bb1df0b44c52fbe2caf8e8bc4425/pynacl-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:16c60daceee88d04f8d41d0a4004a7ed8d9a5126b997efd2933e08e93a3bd850", size = 1397879, upload-time = "2025-09-10T23:39:00.44Z" }, - { url = "https://files.pythonhosted.org/packages/41/94/028ff0434a69448f61348d50d2c147dda51aabdd4fbc93ec61343332174d/pynacl-1.6.0-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25720bad35dfac34a2bcdd61d9e08d6bfc6041bebc7751d9c9f2446cf1e77d64", size = 833907, upload-time = "2025-09-10T23:38:50.936Z" }, - { url = "https://files.pythonhosted.org/packages/52/bc/a5cff7f8c30d5f4c26a07dfb0bcda1176ab8b2de86dda3106c00a02ad787/pynacl-1.6.0-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8bfaa0a28a1ab718bad6239979a5a57a8d1506d0caf2fba17e524dbb409441cf", size = 1436649, upload-time = "2025-09-10T23:38:52.783Z" }, - { url = "https://files.pythonhosted.org/packages/7a/20/c397be374fd5d84295046e398de4ba5f0722dc14450f65db76a43c121471/pynacl-1.6.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ef214b90556bb46a485b7da8258e59204c244b1b5b576fb71848819b468c44a7", size = 817142, upload-time = "2025-09-10T23:38:54.4Z" }, - { url = "https://files.pythonhosted.org/packages/12/30/5efcef3406940cda75296c6d884090b8a9aad2dcc0c304daebb5ae99fb4a/pynacl-1.6.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:49c336dd80ea54780bcff6a03ee1a476be1612423010472e60af83452aa0f442", size = 1401794, upload-time = "2025-09-10T23:38:56.614Z" }, - { url = "https://files.pythonhosted.org/packages/be/e1/a8fe1248cc17ccb03b676d80fa90763760a6d1247da434844ea388d0816c/pynacl-1.6.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f3482abf0f9815e7246d461fab597aa179b7524628a4bc36f86a7dc418d2608d", size = 772161, upload-time = "2025-09-10T23:39:01.93Z" }, - { url = "https://files.pythonhosted.org/packages/a3/76/8a62702fb657d6d9104ce13449db221a345665d05e6a3fdefb5a7cafd2ad/pynacl-1.6.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:140373378e34a1f6977e573033d1dd1de88d2a5d90ec6958c9485b2fd9f3eb90", size = 1370720, upload-time = "2025-09-10T23:39:03.531Z" }, - { url = "https://files.pythonhosted.org/packages/6d/38/9e9e9b777a1c4c8204053733e1a0269672c0bd40852908c9ad6b6eaba82c/pynacl-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6b393bc5e5a0eb86bb85b533deb2d2c815666665f840a09e0aa3362bb6088736", size = 791252, upload-time = "2025-09-10T23:39:05.058Z" }, - { url = "https://files.pythonhosted.org/packages/63/ef/d972ce3d92ae05c9091363cf185e8646933f91c376e97b8be79ea6e96c22/pynacl-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4a25cfede801f01e54179b8ff9514bd7b5944da560b7040939732d1804d25419", size = 1362910, upload-time = "2025-09-10T23:39:06.924Z" }, - { url = "https://files.pythonhosted.org/packages/35/2c/ee0b373a1861f66a7ca8bdb999331525615061320dd628527a50ba8e8a60/pynacl-1.6.0-cp38-abi3-win32.whl", hash = "sha256:dcdeb41c22ff3c66eef5e63049abf7639e0db4edee57ba70531fc1b6b133185d", size = 226461, upload-time = "2025-09-10T23:39:11.894Z" }, - { url = "https://files.pythonhosted.org/packages/75/f7/41b6c0b9dd9970173b6acc026bab7b4c187e4e5beef2756d419ad65482da/pynacl-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:cf831615cc16ba324240de79d925eacae8265b7691412ac6b24221db157f6bd1", size = 238802, upload-time = "2025-09-10T23:39:08.966Z" }, - { url = "https://files.pythonhosted.org/packages/8e/0f/462326910c6172fa2c6ed07922b22ffc8e77432b3affffd9e18f444dbfbb/pynacl-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:84709cea8f888e618c21ed9a0efdb1a59cc63141c403db8bf56c469b71ad56f2", size = 183846, upload-time = "2025-09-10T23:39:10.552Z" }, + { url = "https://files.pythonhosted.org/packages/63/37/87c72df19857c5b3b47ace6f211a26eb862ada495cc96daa372d96048fca/pynacl-1.6.0-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:f4b3824920e206b4f52abd7de621ea7a44fd3cb5c8daceb7c3612345dfc54f2e", size = 382610 }, + { url = "https://files.pythonhosted.org/packages/0c/64/3ce958a5817fd3cc6df4ec14441c43fd9854405668d73babccf77f9597a3/pynacl-1.6.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:16dd347cdc8ae0b0f6187a2608c0af1c8b7ecbbe6b4a06bff8253c192f696990", size = 798744 }, + { url = "https://files.pythonhosted.org/packages/e4/8a/3f0dd297a0a33fa3739c255feebd0206bb1df0b44c52fbe2caf8e8bc4425/pynacl-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:16c60daceee88d04f8d41d0a4004a7ed8d9a5126b997efd2933e08e93a3bd850", size = 1397879 }, + { url = "https://files.pythonhosted.org/packages/41/94/028ff0434a69448f61348d50d2c147dda51aabdd4fbc93ec61343332174d/pynacl-1.6.0-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25720bad35dfac34a2bcdd61d9e08d6bfc6041bebc7751d9c9f2446cf1e77d64", size = 833907 }, + { url = "https://files.pythonhosted.org/packages/52/bc/a5cff7f8c30d5f4c26a07dfb0bcda1176ab8b2de86dda3106c00a02ad787/pynacl-1.6.0-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8bfaa0a28a1ab718bad6239979a5a57a8d1506d0caf2fba17e524dbb409441cf", size = 1436649 }, + { url = "https://files.pythonhosted.org/packages/7a/20/c397be374fd5d84295046e398de4ba5f0722dc14450f65db76a43c121471/pynacl-1.6.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ef214b90556bb46a485b7da8258e59204c244b1b5b576fb71848819b468c44a7", size = 817142 }, + { url = "https://files.pythonhosted.org/packages/12/30/5efcef3406940cda75296c6d884090b8a9aad2dcc0c304daebb5ae99fb4a/pynacl-1.6.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:49c336dd80ea54780bcff6a03ee1a476be1612423010472e60af83452aa0f442", size = 1401794 }, + { url = "https://files.pythonhosted.org/packages/be/e1/a8fe1248cc17ccb03b676d80fa90763760a6d1247da434844ea388d0816c/pynacl-1.6.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f3482abf0f9815e7246d461fab597aa179b7524628a4bc36f86a7dc418d2608d", size = 772161 }, + { url = "https://files.pythonhosted.org/packages/a3/76/8a62702fb657d6d9104ce13449db221a345665d05e6a3fdefb5a7cafd2ad/pynacl-1.6.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:140373378e34a1f6977e573033d1dd1de88d2a5d90ec6958c9485b2fd9f3eb90", size = 1370720 }, + { url = "https://files.pythonhosted.org/packages/6d/38/9e9e9b777a1c4c8204053733e1a0269672c0bd40852908c9ad6b6eaba82c/pynacl-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6b393bc5e5a0eb86bb85b533deb2d2c815666665f840a09e0aa3362bb6088736", size = 791252 }, + { url = "https://files.pythonhosted.org/packages/63/ef/d972ce3d92ae05c9091363cf185e8646933f91c376e97b8be79ea6e96c22/pynacl-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4a25cfede801f01e54179b8ff9514bd7b5944da560b7040939732d1804d25419", size = 1362910 }, + { url = "https://files.pythonhosted.org/packages/35/2c/ee0b373a1861f66a7ca8bdb999331525615061320dd628527a50ba8e8a60/pynacl-1.6.0-cp38-abi3-win32.whl", hash = "sha256:dcdeb41c22ff3c66eef5e63049abf7639e0db4edee57ba70531fc1b6b133185d", size = 226461 }, + { url = "https://files.pythonhosted.org/packages/75/f7/41b6c0b9dd9970173b6acc026bab7b4c187e4e5beef2756d419ad65482da/pynacl-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:cf831615cc16ba324240de79d925eacae8265b7691412ac6b24221db157f6bd1", size = 238802 }, + { url = "https://files.pythonhosted.org/packages/8e/0f/462326910c6172fa2c6ed07922b22ffc8e77432b3affffd9e18f444dbfbb/pynacl-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:84709cea8f888e618c21ed9a0efdb1a59cc63141c403db8bf56c469b71ad56f2", size = 183846 }, ] [[package]] @@ -2811,47 +2823,47 @@ dependencies = [ { name = "cryptography" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1a/51/27a5ad5f939d08f690a326ef9582cda7140555180db71695f6fb747d6a36/pyopenssl-26.2.0.tar.gz", hash = "sha256:8c6fcecd1183a7fc897548dfe388b0cdb7f37e018200d8409cf33959dbe35387", size = 182195, upload-time = "2026-05-04T23:06:09.72Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/51/27a5ad5f939d08f690a326ef9582cda7140555180db71695f6fb747d6a36/pyopenssl-26.2.0.tar.gz", hash = "sha256:8c6fcecd1183a7fc897548dfe388b0cdb7f37e018200d8409cf33959dbe35387", size = 182195 } wheels = [ - { url = "https://files.pythonhosted.org/packages/73/b8/a0e2790ae249d6f38c9f66de7a211621a7ab2650217bcd04e1262f578a56/pyopenssl-26.2.0-py3-none-any.whl", hash = "sha256:4f9d971bc5298b8bc1fab282803da04bf000c755d4ad9d99b52de2569ca19a70", size = 55823, upload-time = "2026-05-04T23:06:08.395Z" }, + { url = "https://files.pythonhosted.org/packages/73/b8/a0e2790ae249d6f38c9f66de7a211621a7ab2650217bcd04e1262f578a56/pyopenssl-26.2.0-py3-none-any.whl", hash = "sha256:4f9d971bc5298b8bc1fab282803da04bf000c755d4ad9d99b52de2569ca19a70", size = 55823 }, ] [[package]] name = "pyparsing" version = "3.3.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574 } wheels = [ - { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781 }, ] [[package]] name = "pypdfium2" version = "5.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/3d/dc934d3b606c51c3ecc95b6731d84b7dd7ab8e513a50b0e98a4da6c8a719/pypdfium2-5.8.0.tar.gz", hash = "sha256:049397c647e50f83115ee951c49394dab9e9ba52ebdd5a11ab1109390eb3d34e", size = 271934, upload-time = "2026-05-04T17:39:43.794Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/8c/6b75b923cb81368fa3ea7c48a0616b839620a3aeff899885bd930449b89e/pypdfium2-5.8.0-py3-none-android_23_arm64_v8a.whl", hash = "sha256:f67b6c74b716d9ac725ad1af49ae786ad813ac20823d45606d59f1fc06caa8af", size = 3374554, upload-time = "2026-05-04T17:39:05.552Z" }, - { url = "https://files.pythonhosted.org/packages/ef/61/a885c7f36efba89ec98e3d1fe95c83b48c2d6dea321e9194ac6460e7a834/pypdfium2-5.8.0-py3-none-android_23_armeabi_v7a.whl", hash = "sha256:53e82bf3e6a2da170b1bda83f93b7eec57cb6efe3cacd05cba78823879a85203", size = 2831667, upload-time = "2026-05-04T17:39:08.028Z" }, - { url = "https://files.pythonhosted.org/packages/86/1f/04b5627f6dba312d3e707e5b019c9f24d8b03b5aa366866a9e02ec00f8d4/pypdfium2-5.8.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:085e633dcc89b65ff4035a4787e98ce7ae636836eb39c83dd0db26113d9774bc", size = 3450815, upload-time = "2026-05-04T17:39:09.551Z" }, - { url = "https://files.pythonhosted.org/packages/a9/77/8e3a2aba2bc4aef5abe1b1306d05b00588dc0bf7f5c850d1adf6164c786b/pypdfium2-5.8.0-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:bc84b7c6efede88fcfb9467f81daf416f26b973a54fc1cf4d3410d622fda6d7a", size = 3634395, upload-time = "2026-05-04T17:39:11.225Z" }, - { url = "https://files.pythonhosted.org/packages/93/11/6f2b1847d9fa457b3b7251afc2bba2706d104a0c6f01431dfae5d679a839/pypdfium2-5.8.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a63bf09b2e13ba8545c930d243f0650c664a1b51314daa3b5f38df6d1a17b4bc", size = 3617413, upload-time = "2026-05-04T17:39:13.139Z" }, - { url = "https://files.pythonhosted.org/packages/ed/fd/99ce639de5ca06d21743c740dd988cd209dda623bc763ae10b8a162022e1/pypdfium2-5.8.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:937881c1698456749ed203a58db1895baa5eb7178cdb837ef84867790638da28", size = 3347639, upload-time = "2026-05-04T17:39:15.086Z" }, - { url = "https://files.pythonhosted.org/packages/fa/47/82864cc6e26dd8969d5594c168635acb16458d35cf5fed65d6b2e32abb42/pypdfium2-5.8.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6be9dc2b84a8694ad7e626bab133244e8241014d5ed1930d865a9bdf90df1e24", size = 3746404, upload-time = "2026-05-04T17:39:17.094Z" }, - { url = "https://files.pythonhosted.org/packages/82/58/e41e49bba951f61921bac7289e67fe02af5ac57192d0bbfb5f459dc3691d/pypdfium2-5.8.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f27bd82891ae302dd02d736b14809661f6d1220ee1e96dbed9b23e2811922a3", size = 4177893, upload-time = "2026-05-04T17:39:18.729Z" }, - { url = "https://files.pythonhosted.org/packages/b4/15/fa7031010d5cf6853dadb4864680a0bfb7782c5bb6a1a401e0c25c4fca87/pypdfium2-5.8.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26c1089cdbbdc7fe1248f6d17fe3f30214be4f287dd0196b31aaee18a1564240", size = 3665152, upload-time = "2026-05-04T17:39:20.207Z" }, - { url = "https://files.pythonhosted.org/packages/de/6a/5a3520a8b0cfa8d7fdc3f03a07ad9d6146c28ffd519330706f64fd8939a8/pypdfium2-5.8.0-py3-none-manylinux_2_27_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c038a9290864aaa4862dd32e591993d82551ca4d152b4e8ce6d43ba37dc04a8", size = 3095365, upload-time = "2026-05-04T17:39:22.054Z" }, - { url = "https://files.pythonhosted.org/packages/32/d3/845bae4de3cfa36865959046156edb5bf9baea400ccdecdd84fdd911b0f5/pypdfium2-5.8.0-py3-none-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f104bc1a6d8bfc1ff088aa50db13b9729cfdb3722b44975c3c457e9a7b9c7318", size = 2961801, upload-time = "2026-05-04T17:39:23.817Z" }, - { url = "https://files.pythonhosted.org/packages/99/76/cf54eabee4a172241dfcfe63533bd1e11e2162114a983453a5a40bfec114/pypdfium2-5.8.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:04ca7c57a553facf8d46c6ea8ba6fa557e698670cfa4a58e0e01fdae2f6be87d", size = 4133067, upload-time = "2026-05-04T17:39:25.619Z" }, - { url = "https://files.pythonhosted.org/packages/77/66/dcf871d19187ca04ea184a99801a6e7e556d8347aa49540fee33cda6dfc5/pypdfium2-5.8.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ad42b9c22477b32dbedcbc8232833f385d92fd0cf92822547b02383cf9a476d7", size = 3749100, upload-time = "2026-05-04T17:39:27.203Z" }, - { url = "https://files.pythonhosted.org/packages/32/67/0d456c79660959ca45ad307b4d67161d29f9ed4083ee1e8fe8c6925b7c82/pypdfium2-5.8.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:388e3119cf5ca0979b7d5f6d40b7fcd5ab49e17ed4e6de6af89ba116061acfda", size = 4339212, upload-time = "2026-05-04T17:39:29.277Z" }, - { url = "https://files.pythonhosted.org/packages/76/89/e5b0e0f7936be341c91c0f45cd70d693878894ed62aed93a6ee32e9c43c4/pypdfium2-5.8.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:aa05bbfa485ce7916217aa78d856c9f9cd86b08b20846c650392a67975ee72e9", size = 4383943, upload-time = "2026-05-04T17:39:31.287Z" }, - { url = "https://files.pythonhosted.org/packages/82/21/4502ed255f082f579cd3537c2971cf1a57778d43703a08bcd1a92253189f/pypdfium2-5.8.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:f0813a16bb39d5ebd173ea5484430bb67a89b4b181db0a636c73b64ad063c3ea", size = 3925680, upload-time = "2026-05-04T17:39:33.241Z" }, - { url = "https://files.pythonhosted.org/packages/7d/4f/2e59723e7a07779439bd885c1b4960079c9710603308888d29ac926ae69a/pypdfium2-5.8.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:a3c78f7d20dd821bec6c072efdb21a1370b9efe10fdeeb68c969e67608e25385", size = 4269560, upload-time = "2026-05-04T17:39:34.926Z" }, - { url = "https://files.pythonhosted.org/packages/34/4e/7b6b1bde3788c8b880d4b8131d95d9d339cebafb3ad9102d82e234bb65be/pypdfium2-5.8.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:86d302e207c138c827b885a72784f7b306d840646ebeae07e8efdbc39321c629", size = 4182434, upload-time = "2026-05-04T17:39:36.624Z" }, - { url = "https://files.pythonhosted.org/packages/11/7b/6ed4782e0d7a5278330598ce8c4b2df7255f4585a0b3d04520fa580d6507/pypdfium2-5.8.0-py3-none-win32.whl", hash = "sha256:3f25fd436920a907291462b41bdc0ab9f8235c3944b4c9c15398da595ffd1fed", size = 3636680, upload-time = "2026-05-04T17:39:38.49Z" }, - { url = "https://files.pythonhosted.org/packages/19/55/da7223d4202b2461f4f889b0baf10dddec3db7f88e6fd8c52db4a516eecd/pypdfium2-5.8.0-py3-none-win_amd64.whl", hash = "sha256:55592af0bddd2d62bed18e0053c546c9b72041430c5115e54870f7f6163125b0", size = 3754962, upload-time = "2026-05-04T17:39:40.13Z" }, - { url = "https://files.pythonhosted.org/packages/fc/7a/f3dcefe6ee7389aad3ca1488c177e8fbf978206de21c7a99ccf487ea38ab/pypdfium2-5.8.0-py3-none-win_arm64.whl", hash = "sha256:3f17ed97ae8a5a1705301ca93af256a5b02f9009dee4e99c5e175831d46ebd7c", size = 3548362, upload-time = "2026-05-04T17:39:42.304Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/6d/3d/dc934d3b606c51c3ecc95b6731d84b7dd7ab8e513a50b0e98a4da6c8a719/pypdfium2-5.8.0.tar.gz", hash = "sha256:049397c647e50f83115ee951c49394dab9e9ba52ebdd5a11ab1109390eb3d34e", size = 271934 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/8c/6b75b923cb81368fa3ea7c48a0616b839620a3aeff899885bd930449b89e/pypdfium2-5.8.0-py3-none-android_23_arm64_v8a.whl", hash = "sha256:f67b6c74b716d9ac725ad1af49ae786ad813ac20823d45606d59f1fc06caa8af", size = 3374554 }, + { url = "https://files.pythonhosted.org/packages/ef/61/a885c7f36efba89ec98e3d1fe95c83b48c2d6dea321e9194ac6460e7a834/pypdfium2-5.8.0-py3-none-android_23_armeabi_v7a.whl", hash = "sha256:53e82bf3e6a2da170b1bda83f93b7eec57cb6efe3cacd05cba78823879a85203", size = 2831667 }, + { url = "https://files.pythonhosted.org/packages/86/1f/04b5627f6dba312d3e707e5b019c9f24d8b03b5aa366866a9e02ec00f8d4/pypdfium2-5.8.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:085e633dcc89b65ff4035a4787e98ce7ae636836eb39c83dd0db26113d9774bc", size = 3450815 }, + { url = "https://files.pythonhosted.org/packages/a9/77/8e3a2aba2bc4aef5abe1b1306d05b00588dc0bf7f5c850d1adf6164c786b/pypdfium2-5.8.0-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:bc84b7c6efede88fcfb9467f81daf416f26b973a54fc1cf4d3410d622fda6d7a", size = 3634395 }, + { url = "https://files.pythonhosted.org/packages/93/11/6f2b1847d9fa457b3b7251afc2bba2706d104a0c6f01431dfae5d679a839/pypdfium2-5.8.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a63bf09b2e13ba8545c930d243f0650c664a1b51314daa3b5f38df6d1a17b4bc", size = 3617413 }, + { url = "https://files.pythonhosted.org/packages/ed/fd/99ce639de5ca06d21743c740dd988cd209dda623bc763ae10b8a162022e1/pypdfium2-5.8.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:937881c1698456749ed203a58db1895baa5eb7178cdb837ef84867790638da28", size = 3347639 }, + { url = "https://files.pythonhosted.org/packages/fa/47/82864cc6e26dd8969d5594c168635acb16458d35cf5fed65d6b2e32abb42/pypdfium2-5.8.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6be9dc2b84a8694ad7e626bab133244e8241014d5ed1930d865a9bdf90df1e24", size = 3746404 }, + { url = "https://files.pythonhosted.org/packages/82/58/e41e49bba951f61921bac7289e67fe02af5ac57192d0bbfb5f459dc3691d/pypdfium2-5.8.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f27bd82891ae302dd02d736b14809661f6d1220ee1e96dbed9b23e2811922a3", size = 4177893 }, + { url = "https://files.pythonhosted.org/packages/b4/15/fa7031010d5cf6853dadb4864680a0bfb7782c5bb6a1a401e0c25c4fca87/pypdfium2-5.8.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26c1089cdbbdc7fe1248f6d17fe3f30214be4f287dd0196b31aaee18a1564240", size = 3665152 }, + { url = "https://files.pythonhosted.org/packages/de/6a/5a3520a8b0cfa8d7fdc3f03a07ad9d6146c28ffd519330706f64fd8939a8/pypdfium2-5.8.0-py3-none-manylinux_2_27_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c038a9290864aaa4862dd32e591993d82551ca4d152b4e8ce6d43ba37dc04a8", size = 3095365 }, + { url = "https://files.pythonhosted.org/packages/32/d3/845bae4de3cfa36865959046156edb5bf9baea400ccdecdd84fdd911b0f5/pypdfium2-5.8.0-py3-none-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f104bc1a6d8bfc1ff088aa50db13b9729cfdb3722b44975c3c457e9a7b9c7318", size = 2961801 }, + { url = "https://files.pythonhosted.org/packages/99/76/cf54eabee4a172241dfcfe63533bd1e11e2162114a983453a5a40bfec114/pypdfium2-5.8.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:04ca7c57a553facf8d46c6ea8ba6fa557e698670cfa4a58e0e01fdae2f6be87d", size = 4133067 }, + { url = "https://files.pythonhosted.org/packages/77/66/dcf871d19187ca04ea184a99801a6e7e556d8347aa49540fee33cda6dfc5/pypdfium2-5.8.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ad42b9c22477b32dbedcbc8232833f385d92fd0cf92822547b02383cf9a476d7", size = 3749100 }, + { url = "https://files.pythonhosted.org/packages/32/67/0d456c79660959ca45ad307b4d67161d29f9ed4083ee1e8fe8c6925b7c82/pypdfium2-5.8.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:388e3119cf5ca0979b7d5f6d40b7fcd5ab49e17ed4e6de6af89ba116061acfda", size = 4339212 }, + { url = "https://files.pythonhosted.org/packages/76/89/e5b0e0f7936be341c91c0f45cd70d693878894ed62aed93a6ee32e9c43c4/pypdfium2-5.8.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:aa05bbfa485ce7916217aa78d856c9f9cd86b08b20846c650392a67975ee72e9", size = 4383943 }, + { url = "https://files.pythonhosted.org/packages/82/21/4502ed255f082f579cd3537c2971cf1a57778d43703a08bcd1a92253189f/pypdfium2-5.8.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:f0813a16bb39d5ebd173ea5484430bb67a89b4b181db0a636c73b64ad063c3ea", size = 3925680 }, + { url = "https://files.pythonhosted.org/packages/7d/4f/2e59723e7a07779439bd885c1b4960079c9710603308888d29ac926ae69a/pypdfium2-5.8.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:a3c78f7d20dd821bec6c072efdb21a1370b9efe10fdeeb68c969e67608e25385", size = 4269560 }, + { url = "https://files.pythonhosted.org/packages/34/4e/7b6b1bde3788c8b880d4b8131d95d9d339cebafb3ad9102d82e234bb65be/pypdfium2-5.8.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:86d302e207c138c827b885a72784f7b306d840646ebeae07e8efdbc39321c629", size = 4182434 }, + { url = "https://files.pythonhosted.org/packages/11/7b/6ed4782e0d7a5278330598ce8c4b2df7255f4585a0b3d04520fa580d6507/pypdfium2-5.8.0-py3-none-win32.whl", hash = "sha256:3f25fd436920a907291462b41bdc0ab9f8235c3944b4c9c15398da595ffd1fed", size = 3636680 }, + { url = "https://files.pythonhosted.org/packages/19/55/da7223d4202b2461f4f889b0baf10dddec3db7f88e6fd8c52db4a516eecd/pypdfium2-5.8.0-py3-none-win_amd64.whl", hash = "sha256:55592af0bddd2d62bed18e0053c546c9b72041430c5115e54870f7f6163125b0", size = 3754962 }, + { url = "https://files.pythonhosted.org/packages/fc/7a/f3dcefe6ee7389aad3ca1488c177e8fbf978206de21c7a99ccf487ea38ab/pypdfium2-5.8.0-py3-none-win_arm64.whl", hash = "sha256:3f17ed97ae8a5a1705301ca93af256a5b02f9009dee4e99c5e175831d46ebd7c", size = 3548362 }, ] [[package]] @@ -2867,9 +2879,9 @@ dependencies = [ { name = "tcolorpy" }, { name = "typepy", extra = ["datetime"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f6/a1/617730f290f04d347103ab40bf67d317df6691b14746f6e1ea039fb57062/pytablewriter-1.2.1.tar.gz", hash = "sha256:7bd0f4f397e070e3b8a34edcf1b9257ccbb18305493d8350a5dbc9957fced959", size = 619241, upload-time = "2025-01-01T15:37:00.04Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/a1/617730f290f04d347103ab40bf67d317df6691b14746f6e1ea039fb57062/pytablewriter-1.2.1.tar.gz", hash = "sha256:7bd0f4f397e070e3b8a34edcf1b9257ccbb18305493d8350a5dbc9957fced959", size = 619241 } wheels = [ - { url = "https://files.pythonhosted.org/packages/21/4c/c199512f01c845dfe5a7840ab3aae6c60463b5dc2a775be72502dfd9170a/pytablewriter-1.2.1-py3-none-any.whl", hash = "sha256:e906ff7ff5151d70a5f66e0f7b75642a7f2dce8d893c265b79cc9cf6bc04ddb4", size = 91083, upload-time = "2025-01-01T15:36:55.63Z" }, + { url = "https://files.pythonhosted.org/packages/21/4c/c199512f01c845dfe5a7840ab3aae6c60463b5dc2a775be72502dfd9170a/pytablewriter-1.2.1-py3-none-any.whl", hash = "sha256:e906ff7ff5151d70a5f66e0f7b75642a7f2dce8d893c265b79cc9cf6bc04ddb4", size = 91083 }, ] [[package]] @@ -2883,9 +2895,9 @@ dependencies = [ { name = "pluggy" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165 } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249 }, ] [[package]] @@ -2897,9 +2909,9 @@ dependencies = [ { name = "pluggy" }, { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876 }, ] [[package]] @@ -2909,9 +2921,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/2b/db9a193df89e5660137f5428063bcc2ced7ad790003b26974adf5c5ceb3b/pytest_django-4.12.0.tar.gz", hash = "sha256:df94ec819a83c8979c8f6de13d9cdfbe76e8c21d39473cfe2b40c9fc9be3c758", size = 91156, upload-time = "2026-02-14T18:40:49.235Z" } +sdist = { url = "https://files.pythonhosted.org/packages/13/2b/db9a193df89e5660137f5428063bcc2ced7ad790003b26974adf5c5ceb3b/pytest_django-4.12.0.tar.gz", hash = "sha256:df94ec819a83c8979c8f6de13d9cdfbe76e8c21d39473cfe2b40c9fc9be3c758", size = 91156 } wheels = [ - { url = "https://files.pythonhosted.org/packages/83/a5/41d091f697c09609e7ef1d5d61925494e0454ebf51de7de05f0f0a728f1d/pytest_django-4.12.0-py3-none-any.whl", hash = "sha256:3ff300c49f8350ba2953b90297d23bf5f589db69545f56f1ec5f8cff5da83e85", size = 26123, upload-time = "2026-02-14T18:40:47.381Z" }, + { url = "https://files.pythonhosted.org/packages/83/a5/41d091f697c09609e7ef1d5d61925494e0454ebf51de7de05f0f0a728f1d/pytest_django-4.12.0-py3-none-any.whl", hash = "sha256:3ff300c49f8350ba2953b90297d23bf5f589db69545f56f1ec5f8cff5da83e85", size = 26123 }, ] [[package]] @@ -2924,9 +2936,9 @@ dependencies = [ { name = "tcolorpy" }, { name = "typepy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6a/2e/14711f6af84b1f637167b52aef179e307a580dfb54f7da8b0c06c3125453/pytest_md_report-0.8.0.tar.gz", hash = "sha256:c8e3b7f1f91a0e8e7d1b946e1b224f4f39187da0df2f812731361a436a17f472", size = 289649, upload-time = "2026-05-04T04:30:34.384Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/2e/14711f6af84b1f637167b52aef179e307a580dfb54f7da8b0c06c3125453/pytest_md_report-0.8.0.tar.gz", hash = "sha256:c8e3b7f1f91a0e8e7d1b946e1b224f4f39187da0df2f812731361a436a17f472", size = 289649 } wheels = [ - { url = "https://files.pythonhosted.org/packages/31/47/c847f9095e466e5933570d3b603eee78389b3bfc534d017b43f6cc62fa1a/pytest_md_report-0.8.0-py3-none-any.whl", hash = "sha256:d2ba54b4be2071ea91bf3a17b215f70093fd5b8148356cb33f9a7a9ac53f177a", size = 17185, upload-time = "2026-05-04T04:30:32.556Z" }, + { url = "https://files.pythonhosted.org/packages/31/47/c847f9095e466e5933570d3b603eee78389b3bfc534d017b43f6cc62fa1a/pytest_md_report-0.8.0-py3-none-any.whl", hash = "sha256:d2ba54b4be2071ea91bf3a17b215f70093fd5b8148356cb33f9a7a9ac53f177a", size = 17185 }, ] [[package]] @@ -2936,9 +2948,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } +sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095 }, ] [[package]] @@ -2948,9 +2960,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973 } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382 }, ] [[package]] @@ -2961,18 +2973,18 @@ dependencies = [ { name = "execnet" }, { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396 }, ] [[package]] name = "python-crontab" version = "3.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/99/7f/c54fb7e70b59844526aa4ae321e927a167678660ab51dda979955eafb89a/python_crontab-3.3.0.tar.gz", hash = "sha256:007c8aee68dddf3e04ec4dce0fac124b93bd68be7470fc95d2a9617a15de291b", size = 57626, upload-time = "2025-07-13T20:05:35.535Z" } +sdist = { url = "https://files.pythonhosted.org/packages/99/7f/c54fb7e70b59844526aa4ae321e927a167678660ab51dda979955eafb89a/python_crontab-3.3.0.tar.gz", hash = "sha256:007c8aee68dddf3e04ec4dce0fac124b93bd68be7470fc95d2a9617a15de291b", size = 57626 } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/42/bb4afa5b088f64092036221843fc989b7db9d9d302494c1f8b024ee78a46/python_crontab-3.3.0-py3-none-any.whl", hash = "sha256:739a778b1a771379b75654e53fd4df58e5c63a9279a63b5dfe44c0fcc3ee7884", size = 27533, upload-time = "2025-07-13T20:05:34.266Z" }, + { url = "https://files.pythonhosted.org/packages/47/42/bb4afa5b088f64092036221843fc989b7db9d9d302494c1f8b024ee78a46/python_crontab-3.3.0-py3-none-any.whl", hash = "sha256:739a778b1a771379b75654e53fd4df58e5c63a9279a63b5dfe44c0fcc3ee7884", size = 27533 }, ] [[package]] @@ -2982,9 +2994,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "six" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, ] [[package]] @@ -2995,18 +3007,18 @@ dependencies = [ { name = "filelock" }, { name = "platformdirs" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/48/60/e88788207d81e46362cfbef0d4aaf4c0f49efc3c12d4c3fa3f542c34ebec/python_discovery-1.3.1.tar.gz", hash = "sha256:62f6db28064c9613e7ca76cb3f00c38c839a07c31c00dfe7ed0986493d2150a6", size = 68011, upload-time = "2026-05-12T20:53:36.336Z" } +sdist = { url = "https://files.pythonhosted.org/packages/48/60/e88788207d81e46362cfbef0d4aaf4c0f49efc3c12d4c3fa3f542c34ebec/python_discovery-1.3.1.tar.gz", hash = "sha256:62f6db28064c9613e7ca76cb3f00c38c839a07c31c00dfe7ed0986493d2150a6", size = 68011 } wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/6f/a05a317a66fee0aad270011461f1a63a453ed12471249f172f7d2e2bc7b4/python_discovery-1.3.1-py3-none-any.whl", hash = "sha256:ed188687ebb3b82c01a17cd5ac62fc94d9f6487a7f1a0f9dfe89753fec91039c", size = 33185, upload-time = "2026-05-12T20:53:34.969Z" }, + { url = "https://files.pythonhosted.org/packages/b7/6f/a05a317a66fee0aad270011461f1a63a453ed12471249f172f7d2e2bc7b4/python_discovery-1.3.1-py3-none-any.whl", hash = "sha256:ed188687ebb3b82c01a17cd5ac62fc94d9f6487a7f1a0f9dfe89753fec91039c", size = 33185 }, ] [[package]] name = "python-dotenv" version = "1.2.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135 } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101 }, ] [[package]] @@ -3016,18 +3028,18 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "simple-websocket" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/34/12/bdef9dbeedbe2cdeba2a2056ad27b1fb081557d34b69a97f574843462cae/python_engineio-4.13.1.tar.gz", hash = "sha256:0a853fcef52f5b345425d8c2b921ac85023a04dfcf75d7b74696c61e940fd066", size = 92348, upload-time = "2026-02-06T23:38:06.12Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/12/bdef9dbeedbe2cdeba2a2056ad27b1fb081557d34b69a97f574843462cae/python_engineio-4.13.1.tar.gz", hash = "sha256:0a853fcef52f5b345425d8c2b921ac85023a04dfcf75d7b74696c61e940fd066", size = 92348 } wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/54/0cce26da03a981f949bb8449c9778537f75f5917c172e1d2992ff25cb57d/python_engineio-4.13.1-py3-none-any.whl", hash = "sha256:f32ad10589859c11053ad7d9bb3c9695cdf862113bfb0d20bc4d890198287399", size = 59847, upload-time = "2026-02-06T23:38:04.861Z" }, + { url = "https://files.pythonhosted.org/packages/aa/54/0cce26da03a981f949bb8449c9778537f75f5917c172e1d2992ff25cb57d/python_engineio-4.13.1-py3-none-any.whl", hash = "sha256:f32ad10589859c11053ad7d9bb3c9695cdf862113bfb0d20bc4d890198287399", size = 59847 }, ] [[package]] name = "python-magic" version = "0.4.27" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/db/0b3e28ac047452d079d375ec6798bf76a036a08182dbb39ed38116a49130/python-magic-0.4.27.tar.gz", hash = "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b", size = 14677, upload-time = "2022-06-07T20:16:59.508Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/db/0b3e28ac047452d079d375ec6798bf76a036a08182dbb39ed38116a49130/python-magic-0.4.27.tar.gz", hash = "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b", size = 14677 } wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/73/9f872cb81fc5c3bb48f7227872c28975f998f3e7c2b1c16e95e6432bbb90/python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3", size = 13840, upload-time = "2022-06-07T20:16:57.763Z" }, + { url = "https://files.pythonhosted.org/packages/6c/73/9f872cb81fc5c3bb48f7227872c28975f998f3e7c2b1c16e95e6432bbb90/python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3", size = 13840 }, ] [[package]] @@ -3038,9 +3050,9 @@ dependencies = [ { name = "bidict" }, { name = "python-engineio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/59/81/cf8284f45e32efa18d3848ed82cdd4dcc1b657b082458fbe01ad3e1f2f8d/python_socketio-5.16.1.tar.gz", hash = "sha256:f863f98eacce81ceea2e742f6388e10ca3cdd0764be21d30d5196470edf5ea89", size = 128508, upload-time = "2026-02-06T23:42:07Z" } +sdist = { url = "https://files.pythonhosted.org/packages/59/81/cf8284f45e32efa18d3848ed82cdd4dcc1b657b082458fbe01ad3e1f2f8d/python_socketio-5.16.1.tar.gz", hash = "sha256:f863f98eacce81ceea2e742f6388e10ca3cdd0764be21d30d5196470edf5ea89", size = 128508 } wheels = [ - { url = "https://files.pythonhosted.org/packages/07/c7/deb8c5e604404dbf10a3808a858946ca3547692ff6316b698945bb72177e/python_socketio-5.16.1-py3-none-any.whl", hash = "sha256:a3eb1702e92aa2f2b5d3ba00261b61f062cce51f1cfb6900bf3ab4d1934d2d35", size = 82054, upload-time = "2026-02-06T23:42:05.772Z" }, + { url = "https://files.pythonhosted.org/packages/07/c7/deb8c5e604404dbf10a3808a858946ca3547692ff6316b698945bb72177e/python_socketio-5.16.1-py3-none-any.whl", hash = "sha256:a3eb1702e92aa2f2b5d3ba00261b61f062cce51f1cfb6900bf3ab4d1934d2d35", size = 82054 }, ] [[package]] @@ -3050,18 +3062,18 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "defusedxml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5f/4a/29feb8da6c44f77007dcd29518fea73a3d5653ee02a587ae1f17f1f5ddb5/python3-openid-3.2.0.tar.gz", hash = "sha256:33fbf6928f401e0b790151ed2b5290b02545e8775f982485205a066f874aaeaf", size = 305600, upload-time = "2020-06-29T12:15:49.026Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/4a/29feb8da6c44f77007dcd29518fea73a3d5653ee02a587ae1f17f1f5ddb5/python3-openid-3.2.0.tar.gz", hash = "sha256:33fbf6928f401e0b790151ed2b5290b02545e8775f982485205a066f874aaeaf", size = 305600 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a5/c6ba13860bdf5525f1ab01e01cc667578d6f1efc8a1dba355700fb04c29b/python3_openid-3.2.0-py3-none-any.whl", hash = "sha256:6626f771e0417486701e0b4daff762e7212e820ca5b29fcc0d05f6f8736dfa6b", size = 133681, upload-time = "2020-06-29T12:15:47.502Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a5/c6ba13860bdf5525f1ab01e01cc667578d6f1efc8a1dba355700fb04c29b/python3_openid-3.2.0-py3-none-any.whl", hash = "sha256:6626f771e0417486701e0b4daff762e7212e820ca5b29fcc0d05f6f8736dfa6b", size = 133681 }, ] [[package]] name = "pytz" version = "2026.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, + { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141 }, ] [[package]] @@ -3069,27 +3081,27 @@ name = "pywin32" version = "311" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/ab/01ea1943d4eba0f850c3c61e78e8dd59757ff815ff3ccd0a84de5f541f42/pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31", size = 8706543, upload-time = "2025-07-14T20:13:20.765Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a8/a0e8d07d4d051ec7502cd58b291ec98dcc0c3fff027caad0470b72cfcc2f/pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067", size = 9495040, upload-time = "2025-07-14T20:13:22.543Z" }, - { url = "https://files.pythonhosted.org/packages/ba/3a/2ae996277b4b50f17d61f0603efd8253cb2d79cc7ae159468007b586396d/pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852", size = 8710102, upload-time = "2025-07-14T20:13:24.682Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ab/01ea1943d4eba0f850c3c61e78e8dd59757ff815ff3ccd0a84de5f541f42/pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31", size = 8706543 }, + { url = "https://files.pythonhosted.org/packages/d1/a8/a0e8d07d4d051ec7502cd58b291ec98dcc0c3fff027caad0470b72cfcc2f/pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067", size = 9495040 }, + { url = "https://files.pythonhosted.org/packages/ba/3a/2ae996277b4b50f17d61f0603efd8253cb2d79cc7ae159468007b586396d/pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852", size = 8710102 }, ] [[package]] name = "pyyaml" version = "6.0.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960 } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, - { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, - { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, - { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, - { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, - { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, - { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063 }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973 }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116 }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011 }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870 }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089 }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181 }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658 }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003 }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344 }, ] [[package]] @@ -3105,18 +3117,18 @@ dependencies = [ { name = "pydantic" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ca/7d/3cd10e26ae97b35cf856ca1dc67576e42414ae39502c51165bb36bb1dff8/qdrant_client-1.16.2.tar.gz", hash = "sha256:ca4ef5f9be7b5eadeec89a085d96d5c723585a391eb8b2be8192919ab63185f0", size = 331112, upload-time = "2025-12-12T10:58:30.866Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/7d/3cd10e26ae97b35cf856ca1dc67576e42414ae39502c51165bb36bb1dff8/qdrant_client-1.16.2.tar.gz", hash = "sha256:ca4ef5f9be7b5eadeec89a085d96d5c723585a391eb8b2be8192919ab63185f0", size = 331112 } wheels = [ - { url = "https://files.pythonhosted.org/packages/08/13/8ce16f808297e16968269de44a14f4fef19b64d9766be1d6ba5ba78b579d/qdrant_client-1.16.2-py3-none-any.whl", hash = "sha256:442c7ef32ae0f005e88b5d3c0783c63d4912b97ae756eb5e052523be682f17d3", size = 377186, upload-time = "2025-12-12T10:58:29.282Z" }, + { url = "https://files.pythonhosted.org/packages/08/13/8ce16f808297e16968269de44a14f4fef19b64d9766be1d6ba5ba78b579d/qdrant_client-1.16.2-py3-none-any.whl", hash = "sha256:442c7ef32ae0f005e88b5d3c0783c63d4912b97ae756eb5e052523be682f17d3", size = 377186 }, ] [[package]] name = "redis" version = "5.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/da/d283a37303a995cd36f8b92db85135153dc4f7a8e4441aa827721b442cfb/redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f", size = 4608355, upload-time = "2024-12-06T09:50:41.956Z" } +sdist = { url = "https://files.pythonhosted.org/packages/47/da/d283a37303a995cd36f8b92db85135153dc4f7a8e4441aa827721b442cfb/redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f", size = 4608355 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/5f/fa26b9b2672cbe30e07d9a5bdf39cf16e3b80b42916757c5f92bca88e4ba/redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4", size = 261502, upload-time = "2024-12-06T09:50:39.656Z" }, + { url = "https://files.pythonhosted.org/packages/3c/5f/fa26b9b2672cbe30e07d9a5bdf39cf16e3b80b42916757c5f92bca88e4ba/redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4", size = 261502 }, ] [[package]] @@ -3128,33 +3140,33 @@ dependencies = [ { name = "rpds-py" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766 }, ] [[package]] name = "regex" version = "2026.5.9" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/0e/49aee608ad09480e7fd276898c99ec6192985fa331abe4eb3a986094490b/regex-2026.5.9.tar.gz", hash = "sha256:a8234aa23ec39894bfe4a3f1b85616a7032481964a13ac6fc9f10de4f6fca270", size = 416074, upload-time = "2026-05-09T23:15:19.37Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/0e/49aee608ad09480e7fd276898c99ec6192985fa331abe4eb3a986094490b/regex-2026.5.9.tar.gz", hash = "sha256:a8234aa23ec39894bfe4a3f1b85616a7032481964a13ac6fc9f10de4f6fca270", size = 416074 } wheels = [ - { url = "https://files.pythonhosted.org/packages/50/9b/6550044bc44e17c84d312c031c2ec42fbdb6a4ec4e29093be3a172d08772/regex-2026.5.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:57eeeb05db7979413dec5438f2db21d7ecbba787cde7a711df1a6f6df672aa06", size = 490451, upload-time = "2026-05-09T23:12:34.72Z" }, - { url = "https://files.pythonhosted.org/packages/1e/95/fc7ba4303b5a0f92446a12ee6778ef2c6c799233f5060042a31bf390cfe9/regex-2026.5.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:398c521292f4c7fb807001dcd54694d3a1fcafc179a36ad9cc56f98df85930b6", size = 292112, upload-time = "2026-05-09T23:12:36.285Z" }, - { url = "https://files.pythonhosted.org/packages/54/4b/ee27938d1b2c443e89a9a10e00d2d19aa5ee300cd3d61140644e93bb083e/regex-2026.5.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f7a7c26137296beba7784de6eba69c6a93a63ccebc385e4962fe67e267a91225", size = 289599, upload-time = "2026-05-09T23:12:38.089Z" }, - { url = "https://files.pythonhosted.org/packages/d8/dd/ba103dc19614e25f3880800ca67ce093d6e21b325d72b8383c7bf906e9fa/regex-2026.5.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6441cc660d76107934a09c22167200839a0e89604a6297f78a974e66e931d2c0", size = 796732, upload-time = "2026-05-09T23:12:40.062Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e7/f035b4fd858b050b0080bf302968dc0f59ba34e391872d54936758e6844e/regex-2026.5.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:91328f1c23d47595ca3ef0a7557fa129c5a23404b775c770697d2f35b33e0107", size = 865440, upload-time = "2026-05-09T23:12:42.059Z" }, - { url = "https://files.pythonhosted.org/packages/0a/51/8cd301ecc899aea28124357f729f4272f44de7806fc7ca02490bfbe253e8/regex-2026.5.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:93a7860539414dddaefba2b40f8771765ae17949d4c7182b876ce429e11a8309", size = 912329, upload-time = "2026-05-09T23:12:44.373Z" }, - { url = "https://files.pythonhosted.org/packages/cc/1e/3fbe2fa1e8cebd62f3bb7d3321cff1640aca2e240b51d9bd624aad949260/regex-2026.5.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd2810d22146b6d838acc5ec15602cb6b47920aa4e33015df3868eedfd20bab8", size = 801239, upload-time = "2026-05-09T23:12:46.268Z" }, - { url = "https://files.pythonhosted.org/packages/17/2f/6f6008682bf2cf98040a0d3153a8e557b6ab728d7713d045cee4ce544ab8/regex-2026.5.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daff2bdbaf1d23e52fdff7c0b7bc2048b68f978df6a4d107ac981f94caef2e66", size = 777054, upload-time = "2026-05-09T23:12:48.051Z" }, - { url = "https://files.pythonhosted.org/packages/19/2b/eee0d20a6842ba04df4b8847a920b57ef56853f14ef85405473e586b605a/regex-2026.5.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4eeb011098fcb77af513dcef521a3dbecbf8849b1e38940759d293b7a93f5026", size = 785098, upload-time = "2026-05-09T23:12:49.851Z" }, - { url = "https://files.pythonhosted.org/packages/4a/98/6fc1e6410feefb92159edaed5041992bfe390e8d26c721865434acbca558/regex-2026.5.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ea9c8ecfa1b73c73b626534d6626e5340d429630943672b8480724f44e84b962", size = 860095, upload-time = "2026-05-09T23:12:51.666Z" }, - { url = "https://files.pythonhosted.org/packages/18/a3/bd855e0f2cb1a978ecf6fa6bb69632dd9c3f6ea3b81cde62fde14c9daec7/regex-2026.5.9-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cd2846168eb9ee3c513902bc8225409cb1caab31d04728b145171fa1625d9621", size = 765762, upload-time = "2026-05-09T23:12:53.413Z" }, - { url = "https://files.pythonhosted.org/packages/dc/66/0ae8c092e60b14c79d24f8e0b7f0aea5bfbffdcab00b5483d13404d3c3a5/regex-2026.5.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:39617fb0cde9c0e6306dc70e3bfc096f3da793219879f7ae7aa341a69fbdcf6d", size = 852100, upload-time = "2026-05-09T23:12:55.256Z" }, - { url = "https://files.pythonhosted.org/packages/21/de/8dfde60fc1b21c946a893ba273403b72617edb261370cb1087099a83f088/regex-2026.5.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fd03c4f0e33280d15cae17159b899245d6b7c53d21def19b263b39655061f5ce", size = 789479, upload-time = "2026-05-09T23:12:57.573Z" }, - { url = "https://files.pythonhosted.org/packages/c3/1c/bdcc98f9a4af4fdd166c74941174619ccff4726d3ce32faa8e9a2ecd38dd/regex-2026.5.9-cp312-cp312-win32.whl", hash = "sha256:164eba9b755ea6f244b0d881196fbc1fac09714e9782c9e2732b813142033c8e", size = 266699, upload-time = "2026-05-09T23:12:59.14Z" }, - { url = "https://files.pythonhosted.org/packages/78/87/240d36864f9e48ace85f72e79ced97ceb7f27ce87739a947dcb834b4e6bc/regex-2026.5.9-cp312-cp312-win_amd64.whl", hash = "sha256:86f40a5d6444db30a125c9c9177e6b25dad981cbc37451fd838f145e6edac92e", size = 277783, upload-time = "2026-05-09T23:13:00.789Z" }, - { url = "https://files.pythonhosted.org/packages/4f/b5/7b30f312b0669dff5beebe5b0989dc2d1a312b1a44fab852199c387a5b96/regex-2026.5.9-cp312-cp312-win_arm64.whl", hash = "sha256:96f5f58b54a063d7ea9dca08e1cf57bfe10499c4d579ee672da284f57f5f0070", size = 270513, upload-time = "2026-05-09T23:13:02.426Z" }, + { url = "https://files.pythonhosted.org/packages/50/9b/6550044bc44e17c84d312c031c2ec42fbdb6a4ec4e29093be3a172d08772/regex-2026.5.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:57eeeb05db7979413dec5438f2db21d7ecbba787cde7a711df1a6f6df672aa06", size = 490451 }, + { url = "https://files.pythonhosted.org/packages/1e/95/fc7ba4303b5a0f92446a12ee6778ef2c6c799233f5060042a31bf390cfe9/regex-2026.5.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:398c521292f4c7fb807001dcd54694d3a1fcafc179a36ad9cc56f98df85930b6", size = 292112 }, + { url = "https://files.pythonhosted.org/packages/54/4b/ee27938d1b2c443e89a9a10e00d2d19aa5ee300cd3d61140644e93bb083e/regex-2026.5.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f7a7c26137296beba7784de6eba69c6a93a63ccebc385e4962fe67e267a91225", size = 289599 }, + { url = "https://files.pythonhosted.org/packages/d8/dd/ba103dc19614e25f3880800ca67ce093d6e21b325d72b8383c7bf906e9fa/regex-2026.5.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6441cc660d76107934a09c22167200839a0e89604a6297f78a974e66e931d2c0", size = 796732 }, + { url = "https://files.pythonhosted.org/packages/cf/e7/f035b4fd858b050b0080bf302968dc0f59ba34e391872d54936758e6844e/regex-2026.5.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:91328f1c23d47595ca3ef0a7557fa129c5a23404b775c770697d2f35b33e0107", size = 865440 }, + { url = "https://files.pythonhosted.org/packages/0a/51/8cd301ecc899aea28124357f729f4272f44de7806fc7ca02490bfbe253e8/regex-2026.5.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:93a7860539414dddaefba2b40f8771765ae17949d4c7182b876ce429e11a8309", size = 912329 }, + { url = "https://files.pythonhosted.org/packages/cc/1e/3fbe2fa1e8cebd62f3bb7d3321cff1640aca2e240b51d9bd624aad949260/regex-2026.5.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd2810d22146b6d838acc5ec15602cb6b47920aa4e33015df3868eedfd20bab8", size = 801239 }, + { url = "https://files.pythonhosted.org/packages/17/2f/6f6008682bf2cf98040a0d3153a8e557b6ab728d7713d045cee4ce544ab8/regex-2026.5.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daff2bdbaf1d23e52fdff7c0b7bc2048b68f978df6a4d107ac981f94caef2e66", size = 777054 }, + { url = "https://files.pythonhosted.org/packages/19/2b/eee0d20a6842ba04df4b8847a920b57ef56853f14ef85405473e586b605a/regex-2026.5.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4eeb011098fcb77af513dcef521a3dbecbf8849b1e38940759d293b7a93f5026", size = 785098 }, + { url = "https://files.pythonhosted.org/packages/4a/98/6fc1e6410feefb92159edaed5041992bfe390e8d26c721865434acbca558/regex-2026.5.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ea9c8ecfa1b73c73b626534d6626e5340d429630943672b8480724f44e84b962", size = 860095 }, + { url = "https://files.pythonhosted.org/packages/18/a3/bd855e0f2cb1a978ecf6fa6bb69632dd9c3f6ea3b81cde62fde14c9daec7/regex-2026.5.9-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cd2846168eb9ee3c513902bc8225409cb1caab31d04728b145171fa1625d9621", size = 765762 }, + { url = "https://files.pythonhosted.org/packages/dc/66/0ae8c092e60b14c79d24f8e0b7f0aea5bfbffdcab00b5483d13404d3c3a5/regex-2026.5.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:39617fb0cde9c0e6306dc70e3bfc096f3da793219879f7ae7aa341a69fbdcf6d", size = 852100 }, + { url = "https://files.pythonhosted.org/packages/21/de/8dfde60fc1b21c946a893ba273403b72617edb261370cb1087099a83f088/regex-2026.5.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fd03c4f0e33280d15cae17159b899245d6b7c53d21def19b263b39655061f5ce", size = 789479 }, + { url = "https://files.pythonhosted.org/packages/c3/1c/bdcc98f9a4af4fdd166c74941174619ccff4726d3ce32faa8e9a2ecd38dd/regex-2026.5.9-cp312-cp312-win32.whl", hash = "sha256:164eba9b755ea6f244b0d881196fbc1fac09714e9782c9e2732b813142033c8e", size = 266699 }, + { url = "https://files.pythonhosted.org/packages/78/87/240d36864f9e48ace85f72e79ced97ceb7f27ce87739a947dcb834b4e6bc/regex-2026.5.9-cp312-cp312-win_amd64.whl", hash = "sha256:86f40a5d6444db30a125c9c9177e6b25dad981cbc37451fd838f145e6edac92e", size = 277783 }, + { url = "https://files.pythonhosted.org/packages/4f/b5/7b30f312b0669dff5beebe5b0989dc2d1a312b1a44fab852199c387a5b96/regex-2026.5.9-cp312-cp312-win_arm64.whl", hash = "sha256:96f5f58b54a063d7ea9dca08e1cf57bfe10499c4d579ee672da284f57f5f0070", size = 270513 }, ] [[package]] @@ -3167,9 +3179,9 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232 } wheels = [ - { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" }, + { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017 }, ] [[package]] @@ -3180,9 +3192,9 @@ dependencies = [ { name = "oauthlib" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } +sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, + { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179 }, ] [[package]] @@ -3192,9 +3204,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, + { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481 }, ] [[package]] @@ -3205,32 +3217,32 @@ dependencies = [ { name = "markdown-it-py" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680 } wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654 }, ] [[package]] name = "rpds-py" version = "0.30.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } +sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469 } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086, upload-time = "2025-11-30T20:22:17.93Z" }, - { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053, upload-time = "2025-11-30T20:22:19.297Z" }, - { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763, upload-time = "2025-11-30T20:22:21.661Z" }, - { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951, upload-time = "2025-11-30T20:22:23.408Z" }, - { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622, upload-time = "2025-11-30T20:22:25.16Z" }, - { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492, upload-time = "2025-11-30T20:22:26.505Z" }, - { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080, upload-time = "2025-11-30T20:22:27.934Z" }, - { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680, upload-time = "2025-11-30T20:22:29.341Z" }, - { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589, upload-time = "2025-11-30T20:22:31.469Z" }, - { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289, upload-time = "2025-11-30T20:22:32.997Z" }, - { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737, upload-time = "2025-11-30T20:22:34.419Z" }, - { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120, upload-time = "2025-11-30T20:22:35.903Z" }, - { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782, upload-time = "2025-11-30T20:22:37.271Z" }, - { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463, upload-time = "2025-11-30T20:22:39.021Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868, upload-time = "2025-11-30T20:22:40.493Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086 }, + { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053 }, + { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763 }, + { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951 }, + { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622 }, + { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492 }, + { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080 }, + { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680 }, + { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589 }, + { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289 }, + { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737 }, + { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120 }, + { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782 }, + { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463 }, + { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868 }, ] [[package]] @@ -3240,34 +3252,34 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyasn1" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034 } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, + { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696 }, ] [[package]] name = "ruff" version = "0.15.13" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/24/21/a7d5c126d5b557715ef81098f3db2fe20f622a039ff2e626af28d674ab80/ruff-0.15.13.tar.gz", hash = "sha256:f9d89f17f7ba7fb2ed42921f0df75da797a9a5d71bc39049e2c687cf2baf44b7", size = 4678180, upload-time = "2026-05-14T13:44:37.869Z" } +sdist = { url = "https://files.pythonhosted.org/packages/24/21/a7d5c126d5b557715ef81098f3db2fe20f622a039ff2e626af28d674ab80/ruff-0.15.13.tar.gz", hash = "sha256:f9d89f17f7ba7fb2ed42921f0df75da797a9a5d71bc39049e2c687cf2baf44b7", size = 4678180 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/61/11d458dc6ac22504fd8e237b29dfd40504c7fbbcc8930402cfe51a8e63ed/ruff-0.15.13-py3-none-linux_armv6l.whl", hash = "sha256:444b580fc72fd6887e650acd3e575e18cdc79dbcf42fb4030b491057921f61f8", size = 10738279, upload-time = "2026-05-14T13:44:18.7Z" }, - { url = "https://files.pythonhosted.org/packages/86/ca/caa871ee7be718c45256fada4e16a218ee3e33f0c4a46b729a60a24912e6/ruff-0.15.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6590d009e7cb7ebf36f83dbdd44a3fa48a0994ff6f1cdc1b08006abe58f98dc7", size = 11124798, upload-time = "2026-05-14T13:44:06.427Z" }, - { url = "https://files.pythonhosted.org/packages/d3/19/43f5f2e568dddde567fc41f8471f9432c09563e19d3e617a48cfa52f8f0a/ruff-0.15.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1c26d2f66163deeb6e08d8b39fbbe983ce3c71cea06a6d7591cfd1421793c629", size = 10460761, upload-time = "2026-05-14T13:44:04.375Z" }, - { url = "https://files.pythonhosted.org/packages/99/df/cf938cd6de3003178f03ad7c1ea2a6c099468c03a35037985070b37e76be/ruff-0.15.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbd6f94b434f896308e4d57fb7bfde0d02b99f7a64b3bdab0fdfa6a864203a5", size = 10804451, upload-time = "2026-05-14T13:44:25.221Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7d/5d0973129b154ded2225729169d7068f26b467760b146493fde138415f23/ruff-0.15.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3259f3be4d181bda591da5db2571aed6853c6a048157756448020bc6c5cd22", size = 10534285, upload-time = "2026-05-14T13:44:08.888Z" }, - { url = "https://files.pythonhosted.org/packages/1f/e3/6b999bbc66cd51e5f073842bc2a3995e99c5e0e72e16b15e7261f7abf57a/ruff-0.15.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae9c17e5eb4430c154e76abc25d79a318190f5a997f38fb6b114416c5319ffc9", size = 11312063, upload-time = "2026-05-14T13:44:11.274Z" }, - { url = "https://files.pythonhosted.org/packages/af/5a/642639e9f5db04f1e97fbd6e091c6fd20725bdf072fb114d00eefb9e6eb8/ruff-0.15.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e2e39bff6c341f4b577a21b801326fab0b11847f48fcaa83f00a113c9b3cb55", size = 12183079, upload-time = "2026-05-14T13:44:01.634Z" }, - { url = "https://files.pythonhosted.org/packages/19/4c/7585735f6b53b0f12de13618b2f7d250a844f018822efc899df2e7b8295f/ruff-0.15.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e8d9a8e08013542e94d3220bc5b62cc3e5ef87c5f74bff367d3fac14fab013e6", size = 11440833, upload-time = "2026-05-14T13:43:59.043Z" }, - { url = "https://files.pythonhosted.org/packages/e8/31/bf1a0803d077e679cfeee5f2f67290a0fa79c7385b5d9a8c17b9db2c48f0/ruff-0.15.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc411dfebe5eebe55ce041c6ae080eb7668955e866daa2fbb16692a784f1c4ca", size = 11434486, upload-time = "2026-05-14T13:44:27.761Z" }, - { url = "https://files.pythonhosted.org/packages/e1/4e/62c9b999875d4f14db80f277c030578f5e249c9852d65b7ac7ad0b43c041/ruff-0.15.13-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:768494eb08b9cee54e2fd27969966f74db5a57f6eaa7a90fcb3306af34dfc4bd", size = 11385189, upload-time = "2026-05-14T13:44:13.704Z" }, - { url = "https://files.pythonhosted.org/packages/fc/89/7e959047a104df3eb12863447c110140191fc5b6c4f379ea2e803fcdb0e4/ruff-0.15.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:fb75f9a3a7e42ffe117d734494e6c5e5cb3565d66e12612cb63d0e572a41a5b6", size = 10781380, upload-time = "2026-05-14T13:43:56.734Z" }, - { url = "https://files.pythonhosted.org/packages/ff/52/5fd18f3b88cab63e88aa11516b3b4e1e5f720e5c330f8dbe5c26210f41f8/ruff-0.15.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8cb74dd33bb2f6613faf7fc03b660053b5ac4f80e706d5788c6335e2a8048d51", size = 10540605, upload-time = "2026-05-14T13:44:20.748Z" }, - { url = "https://files.pythonhosted.org/packages/e8/e0/9e35f338990d3e41a82875ff7053ffe97541dae81c9d02143177f381d572/ruff-0.15.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:7ef823f817fcd191dc934e984be9cf4094f808effa16f2542ad8e821ba02bbf2", size = 11036554, upload-time = "2026-05-14T13:44:16.256Z" }, - { url = "https://files.pythonhosted.org/packages/c2/13/070fb048c24080fba188f66371e2a92785be257ad02242066dc7255ac6e9/ruff-0.15.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f345a13937bd7f09f6f5d19fa0721b0c103e00e7f62bc67089a8e5e037719e0b", size = 11528133, upload-time = "2026-05-14T13:44:22.808Z" }, - { url = "https://files.pythonhosted.org/packages/6b/8c/b1e1666aef7fc6555094d73ae6cd981701781ae85b97ceefc0eebd0b4668/ruff-0.15.13-py3-none-win32.whl", hash = "sha256:4044f94208b3b05ba0fc4a4abd0558cf4d6459bd18325eead7fd8cc66f909b41", size = 10721455, upload-time = "2026-05-14T13:44:35.697Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a6/870a3e8a50590bb92be184ad928c2922f088b00d9dc5c5ec7b924ee08c22/ruff-0.15.13-py3-none-win_amd64.whl", hash = "sha256:7064884d442b7d477b4e7473d12da7f08851d2b1982763c5d3f388a19468a1a4", size = 11900409, upload-time = "2026-05-14T13:44:30.389Z" }, - { url = "https://files.pythonhosted.org/packages/9b/36/9c015cd052fca743dae8cb2aeb16b551444787467db42ceab0fc968865af/ruff-0.15.13-py3-none-win_arm64.whl", hash = "sha256:2471da9bd1068c8c064b5fd9c0c4b6dddffd6369cb1cd68b29993b1709ff1b21", size = 11179336, upload-time = "2026-05-14T13:44:33.026Z" }, + { url = "https://files.pythonhosted.org/packages/c6/61/11d458dc6ac22504fd8e237b29dfd40504c7fbbcc8930402cfe51a8e63ed/ruff-0.15.13-py3-none-linux_armv6l.whl", hash = "sha256:444b580fc72fd6887e650acd3e575e18cdc79dbcf42fb4030b491057921f61f8", size = 10738279 }, + { url = "https://files.pythonhosted.org/packages/86/ca/caa871ee7be718c45256fada4e16a218ee3e33f0c4a46b729a60a24912e6/ruff-0.15.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6590d009e7cb7ebf36f83dbdd44a3fa48a0994ff6f1cdc1b08006abe58f98dc7", size = 11124798 }, + { url = "https://files.pythonhosted.org/packages/d3/19/43f5f2e568dddde567fc41f8471f9432c09563e19d3e617a48cfa52f8f0a/ruff-0.15.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1c26d2f66163deeb6e08d8b39fbbe983ce3c71cea06a6d7591cfd1421793c629", size = 10460761 }, + { url = "https://files.pythonhosted.org/packages/99/df/cf938cd6de3003178f03ad7c1ea2a6c099468c03a35037985070b37e76be/ruff-0.15.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbd6f94b434f896308e4d57fb7bfde0d02b99f7a64b3bdab0fdfa6a864203a5", size = 10804451 }, + { url = "https://files.pythonhosted.org/packages/c7/7d/5d0973129b154ded2225729169d7068f26b467760b146493fde138415f23/ruff-0.15.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3259f3be4d181bda591da5db2571aed6853c6a048157756448020bc6c5cd22", size = 10534285 }, + { url = "https://files.pythonhosted.org/packages/1f/e3/6b999bbc66cd51e5f073842bc2a3995e99c5e0e72e16b15e7261f7abf57a/ruff-0.15.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae9c17e5eb4430c154e76abc25d79a318190f5a997f38fb6b114416c5319ffc9", size = 11312063 }, + { url = "https://files.pythonhosted.org/packages/af/5a/642639e9f5db04f1e97fbd6e091c6fd20725bdf072fb114d00eefb9e6eb8/ruff-0.15.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e2e39bff6c341f4b577a21b801326fab0b11847f48fcaa83f00a113c9b3cb55", size = 12183079 }, + { url = "https://files.pythonhosted.org/packages/19/4c/7585735f6b53b0f12de13618b2f7d250a844f018822efc899df2e7b8295f/ruff-0.15.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e8d9a8e08013542e94d3220bc5b62cc3e5ef87c5f74bff367d3fac14fab013e6", size = 11440833 }, + { url = "https://files.pythonhosted.org/packages/e8/31/bf1a0803d077e679cfeee5f2f67290a0fa79c7385b5d9a8c17b9db2c48f0/ruff-0.15.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc411dfebe5eebe55ce041c6ae080eb7668955e866daa2fbb16692a784f1c4ca", size = 11434486 }, + { url = "https://files.pythonhosted.org/packages/e1/4e/62c9b999875d4f14db80f277c030578f5e249c9852d65b7ac7ad0b43c041/ruff-0.15.13-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:768494eb08b9cee54e2fd27969966f74db5a57f6eaa7a90fcb3306af34dfc4bd", size = 11385189 }, + { url = "https://files.pythonhosted.org/packages/fc/89/7e959047a104df3eb12863447c110140191fc5b6c4f379ea2e803fcdb0e4/ruff-0.15.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:fb75f9a3a7e42ffe117d734494e6c5e5cb3565d66e12612cb63d0e572a41a5b6", size = 10781380 }, + { url = "https://files.pythonhosted.org/packages/ff/52/5fd18f3b88cab63e88aa11516b3b4e1e5f720e5c330f8dbe5c26210f41f8/ruff-0.15.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8cb74dd33bb2f6613faf7fc03b660053b5ac4f80e706d5788c6335e2a8048d51", size = 10540605 }, + { url = "https://files.pythonhosted.org/packages/e8/e0/9e35f338990d3e41a82875ff7053ffe97541dae81c9d02143177f381d572/ruff-0.15.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:7ef823f817fcd191dc934e984be9cf4094f808effa16f2542ad8e821ba02bbf2", size = 11036554 }, + { url = "https://files.pythonhosted.org/packages/c2/13/070fb048c24080fba188f66371e2a92785be257ad02242066dc7255ac6e9/ruff-0.15.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f345a13937bd7f09f6f5d19fa0721b0c103e00e7f62bc67089a8e5e037719e0b", size = 11528133 }, + { url = "https://files.pythonhosted.org/packages/6b/8c/b1e1666aef7fc6555094d73ae6cd981701781ae85b97ceefc0eebd0b4668/ruff-0.15.13-py3-none-win32.whl", hash = "sha256:4044f94208b3b05ba0fc4a4abd0558cf4d6459bd18325eead7fd8cc66f909b41", size = 10721455 }, + { url = "https://files.pythonhosted.org/packages/ab/a6/870a3e8a50590bb92be184ad928c2922f088b00d9dc5c5ec7b924ee08c22/ruff-0.15.13-py3-none-win_amd64.whl", hash = "sha256:7064884d442b7d477b4e7473d12da7f08851d2b1982763c5d3f388a19468a1a4", size = 11900409 }, + { url = "https://files.pythonhosted.org/packages/9b/36/9c015cd052fca743dae8cb2aeb16b551444787467db42ceab0fc968865af/ruff-0.15.13-py3-none-win_arm64.whl", hash = "sha256:2471da9bd1068c8c064b5fd9c0c4b6dddffd6369cb1cd68b29993b1709ff1b21", size = 11179336 }, ] [[package]] @@ -3279,9 +3291,9 @@ dependencies = [ { name = "aiohttp" }, { name = "fsspec" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/65/4b4c868cff76c036d11dc75dd91e5696dbf16ce626514166f35d5f4a930f/s3fs-2024.10.0.tar.gz", hash = "sha256:58b8c3650f8b99dbedf361543da3533aac8707035a104db5d80b094617ad4a3f", size = 75916, upload-time = "2024-10-21T01:45:49.967Z" } +sdist = { url = "https://files.pythonhosted.org/packages/75/65/4b4c868cff76c036d11dc75dd91e5696dbf16ce626514166f35d5f4a930f/s3fs-2024.10.0.tar.gz", hash = "sha256:58b8c3650f8b99dbedf361543da3533aac8707035a104db5d80b094617ad4a3f", size = 75916 } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/44/bb9ff095ae7b1b6908480f683b6ca6b71c2105d343a5e5cb25334b01f5fa/s3fs-2024.10.0-py3-none-any.whl", hash = "sha256:7a2025d60d5b1a6025726b3a5e292a8e5aa713abc3b16fd1f81735181f7bb282", size = 29855, upload-time = "2024-10-21T01:45:47.905Z" }, + { url = "https://files.pythonhosted.org/packages/99/44/bb9ff095ae7b1b6908480f683b6ca6b71c2105d343a5e5cb25334b01f5fa/s3fs-2024.10.0-py3-none-any.whl", hash = "sha256:7a2025d60d5b1a6025726b3a5e292a8e5aa713abc3b16fd1f81735181f7bb282", size = 29855 }, ] [package.optional-dependencies] @@ -3296,27 +3308,27 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c0/0a/1cdbabf9edd0ea7747efdf6c9ab4e7061b085aa7f9bfc36bb1601563b069/s3transfer-0.10.4.tar.gz", hash = "sha256:29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7", size = 145287, upload-time = "2024-11-20T21:06:05.981Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/0a/1cdbabf9edd0ea7747efdf6c9ab4e7061b085aa7f9bfc36bb1601563b069/s3transfer-0.10.4.tar.gz", hash = "sha256:29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7", size = 145287 } wheels = [ - { url = "https://files.pythonhosted.org/packages/66/05/7957af15543b8c9799209506df4660cba7afc4cf94bfb60513827e96bed6/s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e", size = 83175, upload-time = "2024-11-20T21:06:03.961Z" }, + { url = "https://files.pythonhosted.org/packages/66/05/7957af15543b8c9799209506df4660cba7afc4cf94bfb60513827e96bed6/s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e", size = 83175 }, ] [[package]] name = "setuptools" version = "82.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, + { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223 }, ] [[package]] name = "shellingham" version = "1.5.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755 }, ] [[package]] @@ -3326,33 +3338,33 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "wsproto" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b0/d4/bfa032f961103eba93de583b161f0e6a5b63cebb8f2c7d0c6e6efe1e3d2e/simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4", size = 17300, upload-time = "2024-10-10T22:39:31.412Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/d4/bfa032f961103eba93de583b161f0e6a5b63cebb8f2c7d0c6e6efe1e3d2e/simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4", size = 17300 } wheels = [ - { url = "https://files.pythonhosted.org/packages/52/59/0782e51887ac6b07ffd1570e0364cf901ebc36345fea669969d2084baebb/simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c", size = 13842, upload-time = "2024-10-10T22:39:29.645Z" }, + { url = "https://files.pythonhosted.org/packages/52/59/0782e51887ac6b07ffd1570e0364cf901ebc36345fea669969d2084baebb/simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c", size = 13842 }, ] [[package]] name = "singleton-decorator" version = "1.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/33/98/a8b5c919bee1152a9a1afd82014431f8db5882699754de50d1b3aba4d136/singleton-decorator-1.0.0.tar.gz", hash = "sha256:1a90ad8a8a738be591c9c167fdd677c5d4a43d1bc6b1c128227be1c5e03bee07", size = 2791, upload-time = "2017-08-10T19:52:45.903Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/98/a8b5c919bee1152a9a1afd82014431f8db5882699754de50d1b3aba4d136/singleton-decorator-1.0.0.tar.gz", hash = "sha256:1a90ad8a8a738be591c9c167fdd677c5d4a43d1bc6b1c128227be1c5e03bee07", size = 2791 } [[package]] name = "six" version = "1.17.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031 } wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050 }, ] [[package]] name = "sniffio" version = "1.3.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, ] [[package]] @@ -3378,13 +3390,13 @@ dependencies = [ { name = "tomlkit" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/49/4d/7e6a9088381386b4cfae4c5d1d23ea0c3618ca694bc2290118737af59f36/snowflake_connector_python-4.6.0.tar.gz", hash = "sha256:06e2dba02703da6fd60e07bb0574506f810a85e5831d3461247753ecce4b8335", size = 937999, upload-time = "2026-05-28T13:01:48.582Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/4d/7e6a9088381386b4cfae4c5d1d23ea0c3618ca694bc2290118737af59f36/snowflake_connector_python-4.6.0.tar.gz", hash = "sha256:06e2dba02703da6fd60e07bb0574506f810a85e5831d3461247753ecce4b8335", size = 937999 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/4e/a839eddf87df7fe91fd8086e6a43e10e6afddf7c6b718ef036643f032867/snowflake_connector_python-4.6.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a7701b702dbeb348769c5d1248231e18544c4ff1fb4118ad73d48e8f801cfb6e", size = 1167890, upload-time = "2026-05-28T13:01:56.567Z" }, - { url = "https://files.pythonhosted.org/packages/7d/81/632b4ca9459cd801abfaa5396a60d9e60b9e2f051d015a577af0493782d3/snowflake_connector_python-4.6.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:00abbcfe958f60da18297191f3499b1e61802e64622521a2e8da1c059c14e1c0", size = 1181169, upload-time = "2026-05-28T13:01:58.16Z" }, - { url = "https://files.pythonhosted.org/packages/c9/31/79871d7eea206c60a7891a8d4349fdd8933822101af87204231162a5c3e8/snowflake_connector_python-4.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:72aaee21a70e00fbe4dadcc60b9b1012b6411dddc90f94804d5efe5706fb9621", size = 2878875, upload-time = "2026-05-28T13:01:36.26Z" }, - { url = "https://files.pythonhosted.org/packages/e5/ff/ea43b9f87cf632bd9735f4da18d7982572fb67073fd55c67841091a20f1a/snowflake_connector_python-4.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6d3f6120edeb0d6edd208831d006cc3e769ec51bc346727f22d7aeaecbf20f77", size = 2910491, upload-time = "2026-05-28T13:01:37.957Z" }, - { url = "https://files.pythonhosted.org/packages/52/b1/80bc142ce5afee2e9b0520e4444bcdf1a02627c1066653705e4c36b475ab/snowflake_connector_python-4.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:f15e2493a316ce79ab3d7fb16add10252bb2401723e5cfbc7a2ebc44d89a7b2b", size = 5388193, upload-time = "2026-05-28T13:02:10.267Z" }, + { url = "https://files.pythonhosted.org/packages/ab/4e/a839eddf87df7fe91fd8086e6a43e10e6afddf7c6b718ef036643f032867/snowflake_connector_python-4.6.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a7701b702dbeb348769c5d1248231e18544c4ff1fb4118ad73d48e8f801cfb6e", size = 1167890 }, + { url = "https://files.pythonhosted.org/packages/7d/81/632b4ca9459cd801abfaa5396a60d9e60b9e2f051d015a577af0493782d3/snowflake_connector_python-4.6.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:00abbcfe958f60da18297191f3499b1e61802e64622521a2e8da1c059c14e1c0", size = 1181169 }, + { url = "https://files.pythonhosted.org/packages/c9/31/79871d7eea206c60a7891a8d4349fdd8933822101af87204231162a5c3e8/snowflake_connector_python-4.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:72aaee21a70e00fbe4dadcc60b9b1012b6411dddc90f94804d5efe5706fb9621", size = 2878875 }, + { url = "https://files.pythonhosted.org/packages/e5/ff/ea43b9f87cf632bd9735f4da18d7982572fb67073fd55c67841091a20f1a/snowflake_connector_python-4.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6d3f6120edeb0d6edd208831d006cc3e769ec51bc346727f22d7aeaecbf20f77", size = 2910491 }, + { url = "https://files.pythonhosted.org/packages/52/b1/80bc142ce5afee2e9b0520e4444bcdf1a02627c1066653705e4c36b475ab/snowflake_connector_python-4.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:f15e2493a316ce79ab3d7fb16add10252bb2401723e5cfbc7a2ebc44d89a7b2b", size = 5388193 }, ] [package.optional-dependencies] @@ -3401,9 +3413,9 @@ dependencies = [ { name = "django" }, { name = "social-auth-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ac/f3/be7a7551463a6e7ddf9a4674662ae0fdea54aa4f4c82562d151cf1e41ced/social-auth-app-django-5.3.0.tar.gz", hash = "sha256:8719d57d01d80dcc9629a46e6806889aa9714fe4b658d2ebe3c120450591031d", size = 24519, upload-time = "2023-09-01T11:30:31.772Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/f3/be7a7551463a6e7ddf9a4674662ae0fdea54aa4f4c82562d151cf1e41ced/social-auth-app-django-5.3.0.tar.gz", hash = "sha256:8719d57d01d80dcc9629a46e6806889aa9714fe4b658d2ebe3c120450591031d", size = 24519 } wheels = [ - { url = "https://files.pythonhosted.org/packages/19/65/747ad30653d67c9e65c3028b435a224f0fd9e81cf0bbeca2c889bbdf93ae/social_auth_app_django-5.3.0-py3-none-any.whl", hash = "sha256:2e71234656ddebe0c5b5ad450d42ee49f52a3f2d1708687fccf2a2c92d31a624", size = 26373, upload-time = "2023-09-01T11:30:30.18Z" }, + { url = "https://files.pythonhosted.org/packages/19/65/747ad30653d67c9e65c3028b435a224f0fd9e81cf0bbeca2c889bbdf93ae/social_auth_app_django-5.3.0-py3-none-any.whl", hash = "sha256:2e71234656ddebe0c5b5ad450d42ee49f52a3f2d1708687fccf2a2c92d31a624", size = 26373 }, ] [[package]] @@ -3419,18 +3431,18 @@ dependencies = [ { name = "requests" }, { name = "requests-oauthlib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/87/c0/466383c22767604c573f15aff3ea2c37aacf3c10281f31199c02ac0017ef/social_auth_core-4.7.0.tar.gz", hash = "sha256:2bba127c7b7166a81085ddb0c248d93751b3bc3cdab8569f62d9f70c6bc4ed40", size = 230894, upload-time = "2025-06-27T06:34:27.15Z" } +sdist = { url = "https://files.pythonhosted.org/packages/87/c0/466383c22767604c573f15aff3ea2c37aacf3c10281f31199c02ac0017ef/social_auth_core-4.7.0.tar.gz", hash = "sha256:2bba127c7b7166a81085ddb0c248d93751b3bc3cdab8569f62d9f70c6bc4ed40", size = 230894 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/3e/1b1ed868b840ecf5e7b02fc8ab20718ac24e184b90057815fee2ebbc107d/social_auth_core-4.7.0-py3-none-any.whl", hash = "sha256:9eef9b49c332d1a3265b37dcc698a7ace97c3fc59df2d874b51576d11d31f6a6", size = 427867, upload-time = "2025-06-27T06:34:25.715Z" }, + { url = "https://files.pythonhosted.org/packages/e3/3e/1b1ed868b840ecf5e7b02fc8ab20718ac24e184b90057815fee2ebbc107d/social_auth_core-4.7.0-py3-none-any.whl", hash = "sha256:9eef9b49c332d1a3265b37dcc698a7ace97c3fc59df2d874b51576d11d31f6a6", size = 427867 }, ] [[package]] name = "sortedcontainers" version = "2.4.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594 } wheels = [ - { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575 }, ] [[package]] @@ -3441,16 +3453,16 @@ dependencies = [ { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/09/45/461788f35e0364a8da7bda51a1fe1b09762d0c32f12f63727998d85a873b/sqlalchemy-2.0.49.tar.gz", hash = "sha256:d15950a57a210e36dd4cec1aac22787e2a4d57ba9318233e2ef8b2daf9ff2d5f", size = 9898221, upload-time = "2026-04-03T16:38:11.704Z" } +sdist = { url = "https://files.pythonhosted.org/packages/09/45/461788f35e0364a8da7bda51a1fe1b09762d0c32f12f63727998d85a873b/sqlalchemy-2.0.49.tar.gz", hash = "sha256:d15950a57a210e36dd4cec1aac22787e2a4d57ba9318233e2ef8b2daf9ff2d5f", size = 9898221 } wheels = [ - { url = "https://files.pythonhosted.org/packages/49/b3/2de412451330756aaaa72d27131db6dde23995efe62c941184e15242a5fa/sqlalchemy-2.0.49-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bbccb45260e4ff1b7db0be80a9025bb1e6698bdb808b83fff0000f7a90b2c0b", size = 2157681, upload-time = "2026-04-03T16:53:07.132Z" }, - { url = "https://files.pythonhosted.org/packages/50/84/b2a56e2105bd11ebf9f0b93abddd748e1a78d592819099359aa98134a8bf/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb37f15714ec2652d574f021d479e78cd4eb9d04396dca36568fdfffb3487982", size = 3338976, upload-time = "2026-04-03T17:07:40Z" }, - { url = "https://files.pythonhosted.org/packages/2c/fa/65fcae2ed62f84ab72cf89536c7c3217a156e71a2c111b1305ab6f0690e2/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb9ec6436a820a4c006aad1ac351f12de2f2dbdaad171692ee457a02429b672", size = 3351937, upload-time = "2026-04-03T17:12:23.374Z" }, - { url = "https://files.pythonhosted.org/packages/f8/2f/6fd118563572a7fe475925742eb6b3443b2250e346a0cc27d8d408e73773/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8d6efc136f44a7e8bc8088507eaabbb8c2b55b3dbb63fe102c690da0ddebe55e", size = 3281646, upload-time = "2026-04-03T17:07:41.949Z" }, - { url = "https://files.pythonhosted.org/packages/c5/d7/410f4a007c65275b9cf82354adb4bb8ba587b176d0a6ee99caa16fe638f8/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e06e617e3d4fd9e51d385dfe45b077a41e9d1b033a7702551e3278ac597dc750", size = 3316695, upload-time = "2026-04-03T17:12:25.642Z" }, - { url = "https://files.pythonhosted.org/packages/d9/95/81f594aa60ded13273a844539041ccf1e66c5a7bed0a8e27810a3b52d522/sqlalchemy-2.0.49-cp312-cp312-win32.whl", hash = "sha256:83101a6930332b87653886c01d1ee7e294b1fe46a07dd9a2d2b4f91bcc88eec0", size = 2117483, upload-time = "2026-04-03T17:05:40.896Z" }, - { url = "https://files.pythonhosted.org/packages/47/9e/fd90114059175cac64e4fafa9bf3ac20584384d66de40793ae2e2f26f3bb/sqlalchemy-2.0.49-cp312-cp312-win_amd64.whl", hash = "sha256:618a308215b6cececb6240b9abde545e3acdabac7ae3e1d4e666896bf5ba44b4", size = 2144494, upload-time = "2026-04-03T17:05:42.282Z" }, - { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158, upload-time = "2026-04-03T16:53:44.135Z" }, + { url = "https://files.pythonhosted.org/packages/49/b3/2de412451330756aaaa72d27131db6dde23995efe62c941184e15242a5fa/sqlalchemy-2.0.49-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bbccb45260e4ff1b7db0be80a9025bb1e6698bdb808b83fff0000f7a90b2c0b", size = 2157681 }, + { url = "https://files.pythonhosted.org/packages/50/84/b2a56e2105bd11ebf9f0b93abddd748e1a78d592819099359aa98134a8bf/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb37f15714ec2652d574f021d479e78cd4eb9d04396dca36568fdfffb3487982", size = 3338976 }, + { url = "https://files.pythonhosted.org/packages/2c/fa/65fcae2ed62f84ab72cf89536c7c3217a156e71a2c111b1305ab6f0690e2/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb9ec6436a820a4c006aad1ac351f12de2f2dbdaad171692ee457a02429b672", size = 3351937 }, + { url = "https://files.pythonhosted.org/packages/f8/2f/6fd118563572a7fe475925742eb6b3443b2250e346a0cc27d8d408e73773/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8d6efc136f44a7e8bc8088507eaabbb8c2b55b3dbb63fe102c690da0ddebe55e", size = 3281646 }, + { url = "https://files.pythonhosted.org/packages/c5/d7/410f4a007c65275b9cf82354adb4bb8ba587b176d0a6ee99caa16fe638f8/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e06e617e3d4fd9e51d385dfe45b077a41e9d1b033a7702551e3278ac597dc750", size = 3316695 }, + { url = "https://files.pythonhosted.org/packages/d9/95/81f594aa60ded13273a844539041ccf1e66c5a7bed0a8e27810a3b52d522/sqlalchemy-2.0.49-cp312-cp312-win32.whl", hash = "sha256:83101a6930332b87653886c01d1ee7e294b1fe46a07dd9a2d2b4f91bcc88eec0", size = 2117483 }, + { url = "https://files.pythonhosted.org/packages/47/9e/fd90114059175cac64e4fafa9bf3ac20584384d66de40793ae2e2f26f3bb/sqlalchemy-2.0.49-cp312-cp312-win_amd64.whl", hash = "sha256:618a308215b6cececb6240b9abde545e3acdabac7ae3e1d4e666896bf5ba44b4", size = 2144494 }, + { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158 }, ] [package.optional-dependencies] @@ -3462,9 +3474,9 @@ asyncio = [ name = "sqlparse" version = "0.5.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/90/76/437d71068094df0726366574cf3432a4ed754217b436eb7429415cf2d480/sqlparse-0.5.5.tar.gz", hash = "sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e", size = 120815, upload-time = "2025-12-19T07:17:45.073Z" } +sdist = { url = "https://files.pythonhosted.org/packages/90/76/437d71068094df0726366574cf3432a4ed754217b436eb7429415cf2d480/sqlparse-0.5.5.tar.gz", hash = "sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e", size = 120815 } wheels = [ - { url = "https://files.pythonhosted.org/packages/49/4b/359f28a903c13438ef59ebeee215fb25da53066db67b305c125f1c6d2a25/sqlparse-0.5.5-py3-none-any.whl", hash = "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba", size = 46138, upload-time = "2025-12-19T07:17:46.573Z" }, + { url = "https://files.pythonhosted.org/packages/49/4b/359f28a903c13438ef59ebeee215fb25da53066db67b305c125f1c6d2a25/sqlparse-0.5.5-py3-none-any.whl", hash = "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba", size = 46138 }, ] [[package]] @@ -3475,9 +3487,9 @@ dependencies = [ { name = "ply" }, { name = "six" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/99/6f/ef25bbc1aefeb9c905d527f1d3cd3f41f22f40566d33001b8bb14ae0cdaf/stone-3.3.1.tar.gz", hash = "sha256:4ef0397512f609757975f7ec09b35639d72ba7e3e17ce4ddf399578346b4cb50", size = 190888, upload-time = "2022-01-25T21:32:16.729Z" } +sdist = { url = "https://files.pythonhosted.org/packages/99/6f/ef25bbc1aefeb9c905d527f1d3cd3f41f22f40566d33001b8bb14ae0cdaf/stone-3.3.1.tar.gz", hash = "sha256:4ef0397512f609757975f7ec09b35639d72ba7e3e17ce4ddf399578346b4cb50", size = 190888 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/92/d0c83f63d3518e5f0b8a311937c31347349ec9a47b209ddc17f7566f58fc/stone-3.3.1-py3-none-any.whl", hash = "sha256:e15866fad249c11a963cce3bdbed37758f2e88c8ff4898616bc0caeb1e216047", size = 162257, upload-time = "2022-01-25T21:32:15.155Z" }, + { url = "https://files.pythonhosted.org/packages/5c/92/d0c83f63d3518e5f0b8a311937c31347349ec9a47b209ddc17f7566f58fc/stone-3.3.1-py3-none-any.whl", hash = "sha256:e15866fad249c11a963cce3bdbed37758f2e88c8ff4898616bc0caeb1e216047", size = 162257 }, ] [[package]] @@ -3488,27 +3500,27 @@ dependencies = [ { name = "dataproperty" }, { name = "typepy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/65/2f54f0dedd775dde48e300023d20e13ad329a51e33dcadb6d47b4dc95768/tabledata-1.3.5.tar.gz", hash = "sha256:98c64d0ad6b520846b41000fb3f5b2f42fa7ca2675c2c669e5ccab6b93082a36", size = 25396, upload-time = "2026-05-11T12:03:26.367Z" } +sdist = { url = "https://files.pythonhosted.org/packages/75/65/2f54f0dedd775dde48e300023d20e13ad329a51e33dcadb6d47b4dc95768/tabledata-1.3.5.tar.gz", hash = "sha256:98c64d0ad6b520846b41000fb3f5b2f42fa7ca2675c2c669e5ccab6b93082a36", size = 25396 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/86/37fa0e1437089f08b8b1b8c8ad93f6b57e9427753f002914299323300a9e/tabledata-1.3.5-py3-none-any.whl", hash = "sha256:a1e57afc4767b51bef551114c0df31f205d712dbb75e3caf9be7834a79f23136", size = 11919, upload-time = "2026-05-11T12:03:24.907Z" }, + { url = "https://files.pythonhosted.org/packages/c7/86/37fa0e1437089f08b8b1b8c8ad93f6b57e9427753f002914299323300a9e/tabledata-1.3.5-py3-none-any.whl", hash = "sha256:a1e57afc4767b51bef551114c0df31f205d712dbb75e3caf9be7834a79f23136", size = 11919 }, ] [[package]] name = "tcolorpy" version = "0.1.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/80/cc/44f2d81d8f9093aad81c3467a5bf5718d2b5f786e887b6e4adcfc17ec6b9/tcolorpy-0.1.7.tar.gz", hash = "sha256:0fbf6bf238890bbc2e32662aa25736769a29bf6d880328f310c910a327632614", size = 299437, upload-time = "2024-12-29T15:24:23.847Z" } +sdist = { url = "https://files.pythonhosted.org/packages/80/cc/44f2d81d8f9093aad81c3467a5bf5718d2b5f786e887b6e4adcfc17ec6b9/tcolorpy-0.1.7.tar.gz", hash = "sha256:0fbf6bf238890bbc2e32662aa25736769a29bf6d880328f310c910a327632614", size = 299437 } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/a2/ed023f2edd1e011b4d99b6727bce8253842d66c3fbf9ed0a26fc09a92571/tcolorpy-0.1.7-py3-none-any.whl", hash = "sha256:26a59d52027e175a37e0aba72efc99dda43f074db71f55b316d3de37d3251378", size = 8096, upload-time = "2024-12-29T15:24:21.33Z" }, + { url = "https://files.pythonhosted.org/packages/05/a2/ed023f2edd1e011b4d99b6727bce8253842d66c3fbf9ed0a26fc09a92571/tcolorpy-0.1.7-py3-none-any.whl", hash = "sha256:26a59d52027e175a37e0aba72efc99dda43f074db71f55b316d3de37d3251378", size = 8096 }, ] [[package]] name = "tenacity" version = "9.1.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413 } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926 }, ] [[package]] @@ -3522,9 +3534,9 @@ dependencies = [ { name = "urllib3" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fc/b3/c272537f3ea2f312555efeb86398cc382cd07b740d5f3c730918c36e64e1/testcontainers-4.13.3.tar.gz", hash = "sha256:9d82a7052c9a53c58b69e1dc31da8e7a715e8b3ec1c4df5027561b47e2efe646", size = 79064, upload-time = "2025-11-14T05:08:47.584Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/b3/c272537f3ea2f312555efeb86398cc382cd07b740d5f3c730918c36e64e1/testcontainers-4.13.3.tar.gz", hash = "sha256:9d82a7052c9a53c58b69e1dc31da8e7a715e8b3ec1c4df5027561b47e2efe646", size = 79064 } wheels = [ - { url = "https://files.pythonhosted.org/packages/73/27/c2f24b19dafa197c514abe70eda69bc031c5152c6b1f1e5b20099e2ceedd/testcontainers-4.13.3-py3-none-any.whl", hash = "sha256:063278c4805ffa6dd85e56648a9da3036939e6c0ac1001e851c9276b19b05970", size = 124784, upload-time = "2025-11-14T05:08:46.053Z" }, + { url = "https://files.pythonhosted.org/packages/73/27/c2f24b19dafa197c514abe70eda69bc031c5152c6b1f1e5b20099e2ceedd/testcontainers-4.13.3-py3-none-any.whl", hash = "sha256:063278c4805ffa6dd85e56648a9da3036939e6c0ac1001e851c9276b19b05970", size = 124784 }, ] [package.optional-dependencies] @@ -3546,24 +3558,24 @@ dependencies = [ { name = "regex" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/4d017d0f76ec3171d469d80fc03dfbb4e48a4bcaddaa831b31d526f05edc/tiktoken-0.12.0.tar.gz", hash = "sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931", size = 37806, upload-time = "2025-10-06T20:22:45.419Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/4d017d0f76ec3171d469d80fc03dfbb4e48a4bcaddaa831b31d526f05edc/tiktoken-0.12.0.tar.gz", hash = "sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931", size = 37806 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/85/be65d39d6b647c79800fd9d29241d081d4eeb06271f383bb87200d74cf76/tiktoken-0.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b97f74aca0d78a1ff21b8cd9e9925714c15a9236d6ceacf5c7327c117e6e21e8", size = 1050728, upload-time = "2025-10-06T20:21:52.756Z" }, - { url = "https://files.pythonhosted.org/packages/4a/42/6573e9129bc55c9bf7300b3a35bef2c6b9117018acca0dc760ac2d93dffe/tiktoken-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b90f5ad190a4bb7c3eb30c5fa32e1e182ca1ca79f05e49b448438c3e225a49b", size = 994049, upload-time = "2025-10-06T20:21:53.782Z" }, - { url = "https://files.pythonhosted.org/packages/66/c5/ed88504d2f4a5fd6856990b230b56d85a777feab84e6129af0822f5d0f70/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:65b26c7a780e2139e73acc193e5c63ac754021f160df919add909c1492c0fb37", size = 1129008, upload-time = "2025-10-06T20:21:54.832Z" }, - { url = "https://files.pythonhosted.org/packages/f4/90/3dae6cc5436137ebd38944d396b5849e167896fc2073da643a49f372dc4f/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:edde1ec917dfd21c1f2f8046b86348b0f54a2c0547f68149d8600859598769ad", size = 1152665, upload-time = "2025-10-06T20:21:56.129Z" }, - { url = "https://files.pythonhosted.org/packages/a3/fe/26df24ce53ffde419a42f5f53d755b995c9318908288c17ec3f3448313a3/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:35a2f8ddd3824608b3d650a000c1ef71f730d0c56486845705a8248da00f9fe5", size = 1194230, upload-time = "2025-10-06T20:21:57.546Z" }, - { url = "https://files.pythonhosted.org/packages/20/cc/b064cae1a0e9fac84b0d2c46b89f4e57051a5f41324e385d10225a984c24/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83d16643edb7fa2c99eff2ab7733508aae1eebb03d5dfc46f5565862810f24e3", size = 1254688, upload-time = "2025-10-06T20:21:58.619Z" }, - { url = "https://files.pythonhosted.org/packages/81/10/b8523105c590c5b8349f2587e2fdfe51a69544bd5a76295fc20f2374f470/tiktoken-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc5288f34a8bc02e1ea7047b8d041104791d2ddbf42d1e5fa07822cbffe16bd", size = 878694, upload-time = "2025-10-06T20:21:59.876Z" }, + { url = "https://files.pythonhosted.org/packages/a4/85/be65d39d6b647c79800fd9d29241d081d4eeb06271f383bb87200d74cf76/tiktoken-0.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b97f74aca0d78a1ff21b8cd9e9925714c15a9236d6ceacf5c7327c117e6e21e8", size = 1050728 }, + { url = "https://files.pythonhosted.org/packages/4a/42/6573e9129bc55c9bf7300b3a35bef2c6b9117018acca0dc760ac2d93dffe/tiktoken-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b90f5ad190a4bb7c3eb30c5fa32e1e182ca1ca79f05e49b448438c3e225a49b", size = 994049 }, + { url = "https://files.pythonhosted.org/packages/66/c5/ed88504d2f4a5fd6856990b230b56d85a777feab84e6129af0822f5d0f70/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:65b26c7a780e2139e73acc193e5c63ac754021f160df919add909c1492c0fb37", size = 1129008 }, + { url = "https://files.pythonhosted.org/packages/f4/90/3dae6cc5436137ebd38944d396b5849e167896fc2073da643a49f372dc4f/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:edde1ec917dfd21c1f2f8046b86348b0f54a2c0547f68149d8600859598769ad", size = 1152665 }, + { url = "https://files.pythonhosted.org/packages/a3/fe/26df24ce53ffde419a42f5f53d755b995c9318908288c17ec3f3448313a3/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:35a2f8ddd3824608b3d650a000c1ef71f730d0c56486845705a8248da00f9fe5", size = 1194230 }, + { url = "https://files.pythonhosted.org/packages/20/cc/b064cae1a0e9fac84b0d2c46b89f4e57051a5f41324e385d10225a984c24/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83d16643edb7fa2c99eff2ab7733508aae1eebb03d5dfc46f5565862810f24e3", size = 1254688 }, + { url = "https://files.pythonhosted.org/packages/81/10/b8523105c590c5b8349f2587e2fdfe51a69544bd5a76295fc20f2374f470/tiktoken-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc5288f34a8bc02e1ea7047b8d041104791d2ddbf42d1e5fa07822cbffe16bd", size = 878694 }, ] [[package]] name = "tinytag" version = "2.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/96/59/8a8cb2331e2602b53e4dc06960f57d1387a2b18e7efd24e5f9cb60ea4925/tinytag-2.2.1.tar.gz", hash = "sha256:e6d06610ebe7cd66fd07be2d3b9495914ab32654a5e47657bb8cd44c2484523c", size = 38214, upload-time = "2026-03-15T18:48:01.11Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/59/8a8cb2331e2602b53e4dc06960f57d1387a2b18e7efd24e5f9cb60ea4925/tinytag-2.2.1.tar.gz", hash = "sha256:e6d06610ebe7cd66fd07be2d3b9495914ab32654a5e47657bb8cd44c2484523c", size = 38214 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/34/d50e338631baaf65ec5396e70085e5de0b52b24b28db1ffbc1c6e82190dc/tinytag-2.2.1-py3-none-any.whl", hash = "sha256:ed8b1e6d25367937e3321e054f4974f9abfde1a3e0a538824c87da377130c2b6", size = 32927, upload-time = "2026-03-15T18:47:59.613Z" }, + { url = "https://files.pythonhosted.org/packages/ce/34/d50e338631baaf65ec5396e70085e5de0b52b24b28db1ffbc1c6e82190dc/tinytag-2.2.1-py3-none-any.whl", hash = "sha256:ed8b1e6d25367937e3321e054f4974f9abfde1a3e0a538824c87da377130c2b6", size = 32927 }, ] [[package]] @@ -3573,32 +3585,32 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115 } wheels = [ - { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, - { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, - { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, - { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, - { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, - { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, - { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, - { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, - { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, - { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, - { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, - { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, - { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, - { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, - { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, + { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275 }, + { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472 }, + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736 }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835 }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673 }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818 }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195 }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982 }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245 }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069 }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263 }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429 }, + { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363 }, + { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786 }, + { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133 }, ] [[package]] name = "tomlkit" version = "0.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", size = 161875, upload-time = "2026-05-10T07:38:22.245Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", size = 161875 } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", size = 41328, upload-time = "2026-05-10T07:38:23.517Z" }, + { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", size = 41328 }, ] [[package]] @@ -3608,9 +3620,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" } +sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598 } wheels = [ - { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, + { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374 }, ] [[package]] @@ -3620,9 +3632,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mbstrdecoder" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8b/9f/ae119b0e0fd0fe8dcb0e1eeebfeb62f37fdc0b467267cff15cdb746ba38b/typepy-1.3.5.tar.gz", hash = "sha256:a1c5f54c41860f89bab175f512b11e8c9a57cfe7b8b3d5ae5d52d828b756b6dd", size = 39883, upload-time = "2026-05-04T14:04:32.835Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/9f/ae119b0e0fd0fe8dcb0e1eeebfeb62f37fdc0b467267cff15cdb746ba38b/typepy-1.3.5.tar.gz", hash = "sha256:a1c5f54c41860f89bab175f512b11e8c9a57cfe7b8b3d5ae5d52d828b756b6dd", size = 39883 } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/71/75cf08c49b64a9419f1f2cef9be072ac34f6b784da2851489470b7c7ba15/typepy-1.3.5-py3-none-any.whl", hash = "sha256:de361b59609c7503efc2edbe9d7a4e053ae71307bf90ae1678ec4d6bcd807922", size = 31530, upload-time = "2026-05-04T14:04:31.46Z" }, + { url = "https://files.pythonhosted.org/packages/64/71/75cf08c49b64a9419f1f2cef9be072ac34f6b784da2851489470b7c7ba15/typepy-1.3.5-py3-none-any.whl", hash = "sha256:de361b59609c7503efc2edbe9d7a4e053ae71307bf90ae1678ec4d6bcd807922", size = 31530 }, ] [package.optional-dependencies] @@ -3642,9 +3654,9 @@ dependencies = [ { name = "rich" }, { name = "shellingham" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fd/07/b822e1b307d40e263e8253d2384cf98c51aa2368cc7ba9a07e523a1d964b/typer-0.23.1.tar.gz", hash = "sha256:2070374e4d31c83e7b61362fd859aa683576432fd5b026b060ad6b4cd3b86134", size = 120047, upload-time = "2026-02-13T10:04:30.984Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/07/b822e1b307d40e263e8253d2384cf98c51aa2368cc7ba9a07e523a1d964b/typer-0.23.1.tar.gz", hash = "sha256:2070374e4d31c83e7b61362fd859aa683576432fd5b026b060ad6b4cd3b86134", size = 120047 } wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/91/9b286ab899c008c2cb05e8be99814807e7fbbd33f0c0c960470826e5ac82/typer-0.23.1-py3-none-any.whl", hash = "sha256:3291ad0d3c701cbf522012faccfbb29352ff16ad262db2139e6b01f15781f14e", size = 56813, upload-time = "2026-02-13T10:04:32.008Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/9b286ab899c008c2cb05e8be99814807e7fbbd33f0c0c960470826e5ac82/typer-0.23.1-py3-none-any.whl", hash = "sha256:3291ad0d3c701cbf522012faccfbb29352ff16ad262db2139e6b01f15781f14e", size = 56813 }, ] [[package]] @@ -3654,18 +3666,18 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "types-setuptools" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bd/0b/b352742758a6054d1053783887bf8cfb739deda1102fda8722294bdc01f7/types_cffi-2.0.0.20260518.tar.gz", hash = "sha256:f9707e66c13454789a58f8843d1ded4a66f1e9c8b10bd24d5eb5e0f25c0c5472", size = 17790, upload-time = "2026-05-18T06:06:50.672Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/0b/b352742758a6054d1053783887bf8cfb739deda1102fda8722294bdc01f7/types_cffi-2.0.0.20260518.tar.gz", hash = "sha256:f9707e66c13454789a58f8843d1ded4a66f1e9c8b10bd24d5eb5e0f25c0c5472", size = 17790 } wheels = [ - { url = "https://files.pythonhosted.org/packages/68/44/d3b4aafa20a3f76384ba19a513d39272add13746dcfe0409d8d4974fd464/types_cffi-2.0.0.20260518-py3-none-any.whl", hash = "sha256:5b68a215a95d0eac4203b58e766ff7fe40c2e091b1fa1a9e54111f04cc560084", size = 20198, upload-time = "2026-05-18T06:06:49.83Z" }, + { url = "https://files.pythonhosted.org/packages/68/44/d3b4aafa20a3f76384ba19a513d39272add13746dcfe0409d8d4974fd464/types_cffi-2.0.0.20260518-py3-none-any.whl", hash = "sha256:5b68a215a95d0eac4203b58e766ff7fe40c2e091b1fa1a9e54111f04cc560084", size = 20198 }, ] [[package]] name = "types-pymysql" version = "1.1.0.20260518" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fb/e0/43201060de33285af04263d9bd8e8c6b007bd8e0180bd46df8fe6576842e/types_pymysql-1.1.0.20260518.tar.gz", hash = "sha256:39a2448c4267dc4551e0824d2bfaecf7dfd171e89e6dbba90f4d4d45d55e4342", size = 22427, upload-time = "2026-05-18T06:02:31.239Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/e0/43201060de33285af04263d9bd8e8c6b007bd8e0180bd46df8fe6576842e/types_pymysql-1.1.0.20260518.tar.gz", hash = "sha256:39a2448c4267dc4551e0824d2bfaecf7dfd171e89e6dbba90f4d4d45d55e4342", size = 22427 } wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/5a/db02b5e6633fbe49eaf4e3194bc64ec031e6436a0cfcc610cbda4f1b6a24/types_pymysql-1.1.0.20260518-py3-none-any.whl", hash = "sha256:cf697ce4e44124fc859e8e8a7f047c1dc864745c3c628b85a51b3ee01502ef98", size = 23071, upload-time = "2026-05-18T06:02:30.36Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5a/db02b5e6633fbe49eaf4e3194bc64ec031e6436a0cfcc610cbda4f1b6a24/types_pymysql-1.1.0.20260518-py3-none-any.whl", hash = "sha256:cf697ce4e44124fc859e8e8a7f047c1dc864745c3c628b85a51b3ee01502ef98", size = 23071 }, ] [[package]] @@ -3676,27 +3688,27 @@ dependencies = [ { name = "cryptography" }, { name = "types-cffi" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d8/38/011e2a9916e7afca2cc9c14fff1df42285d697ee0dd9903e4292cd1f5bf6/types-pyOpenSSL-24.0.0.20240417.tar.gz", hash = "sha256:38e75fb828d2717be173770bbae8c22811fdec68e2bc3f5833954113eb84237d", size = 8261, upload-time = "2024-04-17T02:17:34.667Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/38/011e2a9916e7afca2cc9c14fff1df42285d697ee0dd9903e4292cd1f5bf6/types-pyOpenSSL-24.0.0.20240417.tar.gz", hash = "sha256:38e75fb828d2717be173770bbae8c22811fdec68e2bc3f5833954113eb84237d", size = 8261 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/39/4e6dee712d1a93f2f2d39b0f7ebe0ba6168dfe2f6b50efe4b258790b5346/types_pyOpenSSL-24.0.0.20240417-py3-none-any.whl", hash = "sha256:4ce41ddaf383815168b6e21d542fd92135f10a5e82adb3e593a6b79638b0b511", size = 7420, upload-time = "2024-04-17T02:17:33.556Z" }, + { url = "https://files.pythonhosted.org/packages/9d/39/4e6dee712d1a93f2f2d39b0f7ebe0ba6168dfe2f6b50efe4b258790b5346/types_pyOpenSSL-24.0.0.20240417-py3-none-any.whl", hash = "sha256:4ce41ddaf383815168b6e21d542fd92135f10a5e82adb3e593a6b79638b0b511", size = 7420 }, ] [[package]] name = "types-pytz" version = "2026.2.0.20260518" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9f/d9/9fa4019d2235bd374293e1fd4153879b28b6ae1d2bae98addd352c9713f2/types_pytz-2026.2.0.20260518.tar.gz", hash = "sha256:e5d254329e9c4e91f0781b22c43a4bb2d10bb044d97b24c4b05d45567b0eae16", size = 10871, upload-time = "2026-05-18T06:02:45.789Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/d9/9fa4019d2235bd374293e1fd4153879b28b6ae1d2bae98addd352c9713f2/types_pytz-2026.2.0.20260518.tar.gz", hash = "sha256:e5d254329e9c4e91f0781b22c43a4bb2d10bb044d97b24c4b05d45567b0eae16", size = 10871 } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/89/41e80670779a223d8bc8bc83019a619988cfa5c432cedac5cec23884fbc4/types_pytz-2026.2.0.20260518-py3-none-any.whl", hash = "sha256:3a12eaa38f476bd650902a9c9bb442f03f3c7dee2be5c5848bce61bd708d205a", size = 10125, upload-time = "2026-05-18T06:02:44.968Z" }, + { url = "https://files.pythonhosted.org/packages/62/89/41e80670779a223d8bc8bc83019a619988cfa5c432cedac5cec23884fbc4/types_pytz-2026.2.0.20260518-py3-none-any.whl", hash = "sha256:3a12eaa38f476bd650902a9c9bb442f03f3c7dee2be5c5848bce61bd708d205a", size = 10125 }, ] [[package]] name = "types-pyyaml" version = "6.0.12.20260518" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b8/83/4a1afc3fbfcf5b8d46fc390cd95ed6b0dc9010a265f4e9f46314efffa37a/types_pyyaml-6.0.12.20260518.tar.gz", hash = "sha256:d917f83fb38462550338c1297faedd860b3ec83912b96b1e3d73255f7473e466", size = 17850, upload-time = "2026-05-18T06:01:58.675Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b8/83/4a1afc3fbfcf5b8d46fc390cd95ed6b0dc9010a265f4e9f46314efffa37a/types_pyyaml-6.0.12.20260518.tar.gz", hash = "sha256:d917f83fb38462550338c1297faedd860b3ec83912b96b1e3d73255f7473e466", size = 17850 } wheels = [ - { url = "https://files.pythonhosted.org/packages/06/a2/c01db32be2ae7d6a1689972f3c492b149ee4e164b12fdfd9f64b50888215/types_pyyaml-6.0.12.20260518-py3-none-any.whl", hash = "sha256:d2150f75a231c9fe9c7463bd29487d93e60bac90400287351384bc2284eba7cd", size = 20312, upload-time = "2026-05-18T06:01:57.368Z" }, + { url = "https://files.pythonhosted.org/packages/06/a2/c01db32be2ae7d6a1689972f3c492b149ee4e164b12fdfd9f64b50888215/types_pyyaml-6.0.12.20260518-py3-none-any.whl", hash = "sha256:d2150f75a231c9fe9c7463bd29487d93e60bac90400287351384bc2284eba7cd", size = 20312 }, ] [[package]] @@ -3707,9 +3719,9 @@ dependencies = [ { name = "cryptography" }, { name = "types-pyopenssl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3a/95/c054d3ac940e8bac4ca216470c80c26688a0e79e09f520a942bb27da3386/types-redis-4.6.0.20241004.tar.gz", hash = "sha256:5f17d2b3f9091ab75384153bfa276619ffa1cf6a38da60e10d5e6749cc5b902e", size = 49679, upload-time = "2024-10-04T02:43:59.224Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/95/c054d3ac940e8bac4ca216470c80c26688a0e79e09f520a942bb27da3386/types-redis-4.6.0.20241004.tar.gz", hash = "sha256:5f17d2b3f9091ab75384153bfa276619ffa1cf6a38da60e10d5e6749cc5b902e", size = 49679 } wheels = [ - { url = "https://files.pythonhosted.org/packages/55/82/7d25dce10aad92d2226b269bce2f85cfd843b4477cd50245d7d40ecf8f89/types_redis-4.6.0.20241004-py3-none-any.whl", hash = "sha256:ef5da68cb827e5f606c8f9c0b49eeee4c2669d6d97122f301d3a55dc6a63f6ed", size = 58737, upload-time = "2024-10-04T02:43:57.968Z" }, + { url = "https://files.pythonhosted.org/packages/55/82/7d25dce10aad92d2226b269bce2f85cfd843b4477cd50245d7d40ecf8f89/types_redis-4.6.0.20241004-py3-none-any.whl", hash = "sha256:ef5da68cb827e5f606c8f9c0b49eeee4c2669d6d97122f301d3a55dc6a63f6ed", size = 58737 }, ] [[package]] @@ -3719,18 +3731,18 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "types-urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f9/b8/c1e8d39996b4929b918aba10dba5de07a8b3f4c8487bb61bb79882544e69/types-requests-2.31.0.6.tar.gz", hash = "sha256:cd74ce3b53c461f1228a9b783929ac73a666658f223e28ed29753771477b3bd0", size = 15535, upload-time = "2023-09-27T06:19:38.443Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/b8/c1e8d39996b4929b918aba10dba5de07a8b3f4c8487bb61bb79882544e69/types-requests-2.31.0.6.tar.gz", hash = "sha256:cd74ce3b53c461f1228a9b783929ac73a666658f223e28ed29753771477b3bd0", size = 15535 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/a1/6f8dc74d9069e790d604ddae70cb46dcbac668f1bb08136e7b0f2f5cd3bf/types_requests-2.31.0.6-py3-none-any.whl", hash = "sha256:a2db9cb228a81da8348b49ad6db3f5519452dd20a9c1e1a868c83c5fe88fd1a9", size = 14516, upload-time = "2023-09-27T06:19:36.373Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a1/6f8dc74d9069e790d604ddae70cb46dcbac668f1bb08136e7b0f2f5cd3bf/types_requests-2.31.0.6-py3-none-any.whl", hash = "sha256:a2db9cb228a81da8348b49ad6db3f5519452dd20a9c1e1a868c83c5fe88fd1a9", size = 14516 }, ] [[package]] name = "types-setuptools" version = "82.0.0.20260518" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/38/bc/73c2c27e047e42f114ac50fb3bdef986c56cbdb68096f8690eeafb839a93/types_setuptools-82.0.0.20260518.tar.gz", hash = "sha256:3b743cfe63d0981ea4c15b90710fc1ed41e3464a537d51e705be514e891c1d07", size = 44999, upload-time = "2026-05-18T06:02:55.642Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/bc/73c2c27e047e42f114ac50fb3bdef986c56cbdb68096f8690eeafb839a93/types_setuptools-82.0.0.20260518.tar.gz", hash = "sha256:3b743cfe63d0981ea4c15b90710fc1ed41e3464a537d51e705be514e891c1d07", size = 44999 } wheels = [ - { url = "https://files.pythonhosted.org/packages/32/8f/d5e2d493f09a7a98c95619edda1cb37cee377626c0a869d53274c26f2858/types_setuptools-82.0.0.20260518-py3-none-any.whl", hash = "sha256:31c04a62b57a653a5021caf191be0f10f70df890f813b51f02bab3969d300f20", size = 68444, upload-time = "2026-05-18T06:02:54.582Z" }, + { url = "https://files.pythonhosted.org/packages/32/8f/d5e2d493f09a7a98c95619edda1cb37cee377626c0a869d53274c26f2858/types_setuptools-82.0.0.20260518-py3-none-any.whl", hash = "sha256:31c04a62b57a653a5021caf191be0f10f70df890f813b51f02bab3969d300f20", size = 68444 }, ] [[package]] @@ -3740,27 +3752,27 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "types-pytz" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/cf/e4d446e57c0b14ed1da4de180d2a4cac773b667f183e83bdad76ea6e2238/types-tzlocal-5.1.0.1.tar.gz", hash = "sha256:b84a115c0c68f0d0fa9af1c57f0645eeef0e539147806faf1f95ac3ac01ce47b", size = 3549, upload-time = "2023-10-24T02:15:07.127Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/cf/e4d446e57c0b14ed1da4de180d2a4cac773b667f183e83bdad76ea6e2238/types-tzlocal-5.1.0.1.tar.gz", hash = "sha256:b84a115c0c68f0d0fa9af1c57f0645eeef0e539147806faf1f95ac3ac01ce47b", size = 3549 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/13/caeb438290df069ddda6f055d0eb14337ada293c7d43ab89419ba4b1a778/types_tzlocal-5.1.0.1-py3-none-any.whl", hash = "sha256:0302e8067c86936de8f7e0aaedc2cfbf240080802c603df0f80312fbd4efb926", size = 3005, upload-time = "2023-10-24T02:15:05.815Z" }, + { url = "https://files.pythonhosted.org/packages/f8/13/caeb438290df069ddda6f055d0eb14337ada293c7d43ab89419ba4b1a778/types_tzlocal-5.1.0.1-py3-none-any.whl", hash = "sha256:0302e8067c86936de8f7e0aaedc2cfbf240080802c603df0f80312fbd4efb926", size = 3005 }, ] [[package]] name = "types-urllib3" version = "1.26.25.14" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/73/de/b9d7a68ad39092368fb21dd6194b362b98a1daeea5dcfef5e1adb5031c7e/types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f", size = 11239, upload-time = "2023-07-20T15:19:31.307Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/de/b9d7a68ad39092368fb21dd6194b362b98a1daeea5dcfef5e1adb5031c7e/types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f", size = 11239 } wheels = [ - { url = "https://files.pythonhosted.org/packages/11/7b/3fc711b2efea5e85a7a0bbfe269ea944aa767bbba5ec52f9ee45d362ccf3/types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e", size = 15377, upload-time = "2023-07-20T15:19:30.379Z" }, + { url = "https://files.pythonhosted.org/packages/11/7b/3fc711b2efea5e85a7a0bbfe269ea944aa767bbba5ec52f9ee45d362ccf3/types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e", size = 15377 }, ] [[package]] name = "typing-extensions" version = "4.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391 } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614 }, ] [[package]] @@ -3771,9 +3783,9 @@ dependencies = [ { name = "mypy-extensions" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825, upload-time = "2023-05-24T20:25:47.612Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825 } wheels = [ - { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, + { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827 }, ] [[package]] @@ -3783,40 +3795,40 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949 } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611 }, ] [[package]] name = "tzdata" version = "2026.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321 }, ] [[package]] name = "ujson" version = "5.12.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bc/78/937198ea8708182dd1edbf0237bf255a96feab3f511691ad08b84da98e5d/ujson-5.12.1.tar.gz", hash = "sha256:5b7e96406c301a1366534479a7352ec40ec68bb327c0c119091635acd5925e35", size = 7164538, upload-time = "2026-05-05T22:05:01.354Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bc/78/937198ea8708182dd1edbf0237bf255a96feab3f511691ad08b84da98e5d/ujson-5.12.1.tar.gz", hash = "sha256:5b7e96406c301a1366534479a7352ec40ec68bb327c0c119091635acd5925e35", size = 7164538 } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/40/dbb8e2fe6ee33769602fba203dacaa3963b6599f0d0aefdf2b8811af5f70/ujson-5.12.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:10f44bd08ae52ee23ca6e8b472692e5da1768af2d53ff1bad6f40b532e0bc7ee", size = 57951, upload-time = "2026-05-05T22:03:31.606Z" }, - { url = "https://files.pythonhosted.org/packages/8d/db/627472e6b4ac34148ea52e6d3d15f6f366fc21c72fe7d6c7d3729d4b3ac5/ujson-5.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6cc6ea753b7303fa5629fa9ac9257ea4b001c4d72583b2bb36ff1855a07db49f", size = 55562, upload-time = "2026-05-05T22:03:32.853Z" }, - { url = "https://files.pythonhosted.org/packages/be/59/1248c966da197ae7d2673542444a2d9a1ff7c46e3ec2a302c3caf902b922/ujson-5.12.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:289f13095764d03734adfa10107da9b530ceb64dc1b02a5f507588d978d5b7df", size = 59448, upload-time = "2026-05-05T22:03:34.143Z" }, - { url = "https://files.pythonhosted.org/packages/d5/d7/60c1ca71a09c0654c3edca1192a18fc55e6cc06107be86d7d3f2b39fb29b/ujson-5.12.1-cp312-cp312-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:427893168d074e59214b0ee058337c57f5bb80175cdd5b4799a9c931aae22022", size = 61608, upload-time = "2026-05-05T22:03:35.386Z" }, - { url = "https://files.pythonhosted.org/packages/d5/0a/c619525576219bfc50084100117481b1a732a16716a3878355570995de4e/ujson-5.12.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7a81724d5d90a2da7155d15d8b156ce57eaed7cdd622df813f36a8e612fd4c8", size = 59113, upload-time = "2026-05-05T22:03:37.555Z" }, - { url = "https://files.pythonhosted.org/packages/18/4d/79c1674036085e8dfdb77f8d87c1fd2896e97e6affd117c5e8ecc40f0ae4/ujson-5.12.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3a6efff7dc6515416366819de4a1bc449b77107c5b48508b101fd40f7f8bec08", size = 1038914, upload-time = "2026-05-05T22:03:38.954Z" }, - { url = "https://files.pythonhosted.org/packages/94/b1/9409bba17189ee282b6314cdf0ecdcc72e3d38cd565c870c0227d0494569/ujson-5.12.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:77a71fe53427a0cf49d56eafd801d9f7e203b784b7f99cc717783fd6f6f7b732", size = 1198408, upload-time = "2026-05-05T22:03:40.943Z" }, - { url = "https://files.pythonhosted.org/packages/4b/ad/fafbce7ac59f1a10a83892d0a34add23cc06492308e1330493aab707dc20/ujson-5.12.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ea3bed53d2ea8e5642e814a9e41f3e29420a8067874ba03ace8c0462e160490c", size = 1091451, upload-time = "2026-05-05T22:03:42.739Z" }, - { url = "https://files.pythonhosted.org/packages/5a/1f/76fc9d5b1dcb9eb73ed45fd56e5114391bd30808eb1cea7f8bc5c9a64324/ujson-5.12.1-cp312-cp312-win32.whl", hash = "sha256:758e5c8fbe4e6d483041e03b307b01fb5d2f2dd4452d4d4b927ab902e188939e", size = 41049, upload-time = "2026-05-05T22:03:44.341Z" }, - { url = "https://files.pythonhosted.org/packages/35/2a/7ce3b6fda10d05b79a245db03405734b521ba3da6c377f173b018dce6d4e/ujson-5.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:f6074d3d3267ba1914c624b6e1fa3d8152648ff36b0ab77ddf83b92db488c30d", size = 45330, upload-time = "2026-05-05T22:03:45.828Z" }, - { url = "https://files.pythonhosted.org/packages/d7/66/5a37bba7a2e2ab36ae467521c4511e6593ad74c869f62ec4ba6330f3f71e/ujson-5.12.1-cp312-cp312-win_arm64.whl", hash = "sha256:7642a41520ac1b2bc25ea282b66b8da522cc43424442e6fb5e039be4d4f96530", size = 39828, upload-time = "2026-05-05T22:03:47.123Z" }, - { url = "https://files.pythonhosted.org/packages/6d/26/c9d0479236b3f5690d6a8bb45f708aabc2c91ca80d275eba24b1e9e464ab/ujson-5.12.1-graalpy312-graalpy250_312_native-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2c419bf42ae40963fc27f70c59e24e9a97f5cf168dbce2c572f3c0ce3595912", size = 56153, upload-time = "2026-05-05T22:04:40.326Z" }, - { url = "https://files.pythonhosted.org/packages/ee/c8/785f4e132500aff2f1fd2bd4a4b86fe396a5519f830a098358c90ebb92ee/ujson-5.12.1-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0be2b4f2f547b9f0f3d902640e410e5a2fc851576cbe033c88445a23e3e7aef1", size = 57352, upload-time = "2026-05-05T22:04:42.005Z" }, - { url = "https://files.pythonhosted.org/packages/8f/13/b688a905653871b10b4ff0403c2ff562c17a0bd50be0d44324f3c85ca48f/ujson-5.12.1-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:4ea0c490c702c20495e97345acfcf0c2f3153e658ef537ff111929c48b89e10a", size = 45988, upload-time = "2026-05-05T22:04:43.36Z" }, + { url = "https://files.pythonhosted.org/packages/d7/40/dbb8e2fe6ee33769602fba203dacaa3963b6599f0d0aefdf2b8811af5f70/ujson-5.12.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:10f44bd08ae52ee23ca6e8b472692e5da1768af2d53ff1bad6f40b532e0bc7ee", size = 57951 }, + { url = "https://files.pythonhosted.org/packages/8d/db/627472e6b4ac34148ea52e6d3d15f6f366fc21c72fe7d6c7d3729d4b3ac5/ujson-5.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6cc6ea753b7303fa5629fa9ac9257ea4b001c4d72583b2bb36ff1855a07db49f", size = 55562 }, + { url = "https://files.pythonhosted.org/packages/be/59/1248c966da197ae7d2673542444a2d9a1ff7c46e3ec2a302c3caf902b922/ujson-5.12.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:289f13095764d03734adfa10107da9b530ceb64dc1b02a5f507588d978d5b7df", size = 59448 }, + { url = "https://files.pythonhosted.org/packages/d5/d7/60c1ca71a09c0654c3edca1192a18fc55e6cc06107be86d7d3f2b39fb29b/ujson-5.12.1-cp312-cp312-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:427893168d074e59214b0ee058337c57f5bb80175cdd5b4799a9c931aae22022", size = 61608 }, + { url = "https://files.pythonhosted.org/packages/d5/0a/c619525576219bfc50084100117481b1a732a16716a3878355570995de4e/ujson-5.12.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7a81724d5d90a2da7155d15d8b156ce57eaed7cdd622df813f36a8e612fd4c8", size = 59113 }, + { url = "https://files.pythonhosted.org/packages/18/4d/79c1674036085e8dfdb77f8d87c1fd2896e97e6affd117c5e8ecc40f0ae4/ujson-5.12.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3a6efff7dc6515416366819de4a1bc449b77107c5b48508b101fd40f7f8bec08", size = 1038914 }, + { url = "https://files.pythonhosted.org/packages/94/b1/9409bba17189ee282b6314cdf0ecdcc72e3d38cd565c870c0227d0494569/ujson-5.12.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:77a71fe53427a0cf49d56eafd801d9f7e203b784b7f99cc717783fd6f6f7b732", size = 1198408 }, + { url = "https://files.pythonhosted.org/packages/4b/ad/fafbce7ac59f1a10a83892d0a34add23cc06492308e1330493aab707dc20/ujson-5.12.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ea3bed53d2ea8e5642e814a9e41f3e29420a8067874ba03ace8c0462e160490c", size = 1091451 }, + { url = "https://files.pythonhosted.org/packages/5a/1f/76fc9d5b1dcb9eb73ed45fd56e5114391bd30808eb1cea7f8bc5c9a64324/ujson-5.12.1-cp312-cp312-win32.whl", hash = "sha256:758e5c8fbe4e6d483041e03b307b01fb5d2f2dd4452d4d4b927ab902e188939e", size = 41049 }, + { url = "https://files.pythonhosted.org/packages/35/2a/7ce3b6fda10d05b79a245db03405734b521ba3da6c377f173b018dce6d4e/ujson-5.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:f6074d3d3267ba1914c624b6e1fa3d8152648ff36b0ab77ddf83b92db488c30d", size = 45330 }, + { url = "https://files.pythonhosted.org/packages/d7/66/5a37bba7a2e2ab36ae467521c4511e6593ad74c869f62ec4ba6330f3f71e/ujson-5.12.1-cp312-cp312-win_arm64.whl", hash = "sha256:7642a41520ac1b2bc25ea282b66b8da522cc43424442e6fb5e039be4d4f96530", size = 39828 }, + { url = "https://files.pythonhosted.org/packages/6d/26/c9d0479236b3f5690d6a8bb45f708aabc2c91ca80d275eba24b1e9e464ab/ujson-5.12.1-graalpy312-graalpy250_312_native-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2c419bf42ae40963fc27f70c59e24e9a97f5cf168dbce2c572f3c0ce3595912", size = 56153 }, + { url = "https://files.pythonhosted.org/packages/ee/c8/785f4e132500aff2f1fd2bd4a4b86fe396a5519f830a098358c90ebb92ee/ujson-5.12.1-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0be2b4f2f547b9f0f3d902640e410e5a2fc851576cbe033c88445a23e3e7aef1", size = 57352 }, + { url = "https://files.pythonhosted.org/packages/8f/13/b688a905653871b10b4ff0403c2ff562c17a0bd50be0d44324f3c85ca48f/ujson-5.12.1-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:4ea0c490c702c20495e97345acfcf0c2f3153e658ef537ff111929c48b89e10a", size = 45988 }, ] [[package]] @@ -4171,9 +4183,11 @@ source = { editable = "workers" } dependencies = [ { name = "boto3" }, { name = "celery" }, + { name = "croniter" }, { name = "httpx" }, { name = "prometheus-client" }, { name = "psutil" }, + { name = "psycopg2-binary" }, { name = "python-dotenv" }, { name = "python-socketio" }, { name = "redis" }, @@ -4194,9 +4208,11 @@ dependencies = [ requires-dist = [ { name = "boto3", specifier = "~=1.34.0" }, { name = "celery", specifier = ">=5.5.3" }, + { name = "croniter", specifier = ">=3.0.3" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "prometheus-client", specifier = ">=0.17.0,<1.0.0" }, { name = "psutil", specifier = ">=5.9.0,<6.0.0" }, + { name = "psycopg2-binary", specifier = "==2.9.9" }, { name = "python-dotenv", specifier = ">=1.2.2,<2.0.0" }, { name = "python-socketio", specifier = ">=5.9.0" }, { name = "redis", specifier = ">=4.5.0,<6.0.0" }, @@ -4246,6 +4262,7 @@ dependencies = [ { name = "unstract-core" }, { name = "unstract-filesystem" }, { name = "unstract-flags" }, + { name = "unstract-sdk1" }, { name = "unstract-tool-registry" }, { name = "unstract-tool-sandbox" }, ] @@ -4255,6 +4272,7 @@ requires-dist = [ { name = "unstract-core", editable = "unstract/core" }, { name = "unstract-filesystem", editable = "unstract/filesystem" }, { name = "unstract-flags", editable = "unstract/flags" }, + { name = "unstract-sdk1", editable = "unstract/sdk1" }, { name = "unstract-tool-registry", editable = "unstract/tool-registry" }, { name = "unstract-tool-sandbox", editable = "unstract/tool-sandbox" }, ] @@ -4263,36 +4281,36 @@ requires-dist = [ name = "uritemplate" version = "4.2.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/60/f174043244c5306c9988380d2cb10009f91563fc4b31293d27e17201af56/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", size = 33267, upload-time = "2025-06-02T15:12:06.318Z" } +sdist = { url = "https://files.pythonhosted.org/packages/98/60/f174043244c5306c9988380d2cb10009f91563fc4b31293d27e17201af56/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", size = 33267 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/99/3ae339466c9183ea5b8ae87b34c0b897eda475d2aec2307cae60e5cd4f29/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686", size = 11488, upload-time = "2025-06-02T15:12:03.405Z" }, + { url = "https://files.pythonhosted.org/packages/a9/99/3ae339466c9183ea5b8ae87b34c0b897eda475d2aec2307cae60e5cd4f29/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686", size = 11488 }, ] [[package]] name = "urllib3" version = "1.26.20" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/e8/6ff5e6bc22095cfc59b6ea711b687e2b7ed4bdb373f7eeec370a97d7392f/urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32", size = 307380, upload-time = "2024-08-29T15:43:11.37Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/e8/6ff5e6bc22095cfc59b6ea711b687e2b7ed4bdb373f7eeec370a97d7392f/urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32", size = 307380 } wheels = [ - { url = "https://files.pythonhosted.org/packages/33/cf/8435d5a7159e2a9c83a95896ed596f68cf798005fe107cc655b5c5c14704/urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e", size = 144225, upload-time = "2024-08-29T15:43:08.921Z" }, + { url = "https://files.pythonhosted.org/packages/33/cf/8435d5a7159e2a9c83a95896ed596f68cf798005fe107cc655b5c5c14704/urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e", size = 144225 }, ] [[package]] name = "validators" version = "0.35.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/66/a435d9ae49850b2f071f7ebd8119dd4e84872b01630d6736761e6e7fd847/validators-0.35.0.tar.gz", hash = "sha256:992d6c48a4e77c81f1b4daba10d16c3a9bb0dbb79b3a19ea847ff0928e70497a", size = 73399, upload-time = "2025-05-01T05:42:06.7Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/66/a435d9ae49850b2f071f7ebd8119dd4e84872b01630d6736761e6e7fd847/validators-0.35.0.tar.gz", hash = "sha256:992d6c48a4e77c81f1b4daba10d16c3a9bb0dbb79b3a19ea847ff0928e70497a", size = 73399 } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/6e/3e955517e22cbdd565f2f8b2e73d52528b14b8bcfdb04f62466b071de847/validators-0.35.0-py3-none-any.whl", hash = "sha256:e8c947097eae7892cb3d26868d637f79f47b4a0554bc6b80065dfe5aac3705dd", size = 44712, upload-time = "2025-05-01T05:42:04.203Z" }, + { url = "https://files.pythonhosted.org/packages/fa/6e/3e955517e22cbdd565f2f8b2e73d52528b14b8bcfdb04f62466b071de847/validators-0.35.0-py3-none-any.whl", hash = "sha256:e8c947097eae7892cb3d26868d637f79f47b4a0554bc6b80065dfe5aac3705dd", size = 44712 }, ] [[package]] name = "vine" version = "5.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980 } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, + { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636 }, ] [[package]] @@ -4305,18 +4323,18 @@ dependencies = [ { name = "platformdirs" }, { name = "python-discovery" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/15/ba/1f6e8c957e4932be060dcdc482d339c12e0216351478add3645cdaa53c05/virtualenv-21.3.3.tar.gz", hash = "sha256:f5bda277e553b1c2b3c1a8debfc30496e1288cc93ce6b7b71b3280047e317328", size = 7613784, upload-time = "2026-05-13T18:01:30.19Z" } +sdist = { url = "https://files.pythonhosted.org/packages/15/ba/1f6e8c957e4932be060dcdc482d339c12e0216351478add3645cdaa53c05/virtualenv-21.3.3.tar.gz", hash = "sha256:f5bda277e553b1c2b3c1a8debfc30496e1288cc93ce6b7b71b3280047e317328", size = 7613784 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/34/a9dbe051de88a63eb7408ea66630bac38e72f7f6077d4be58737106860d9/virtualenv-21.3.3-py3-none-any.whl", hash = "sha256:7d5987d8369e098e41406efb780a3d4ca79280097293899e351a6407ee153ab3", size = 7594554, upload-time = "2026-05-13T18:01:27.815Z" }, + { url = "https://files.pythonhosted.org/packages/f4/34/a9dbe051de88a63eb7408ea66630bac38e72f7f6077d4be58737106860d9/virtualenv-21.3.3-py3-none-any.whl", hash = "sha256:7d5987d8369e098e41406efb780a3d4ca79280097293899e351a6407ee153ab3", size = 7594554 }, ] [[package]] name = "wcwidth" version = "0.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/ee/afaf0f85a9a18fe47a67f1e4422ed6cf1fe642f0ae0a2f81166231303c52/wcwidth-0.7.0.tar.gz", hash = "sha256:90e3a7ea092341c44b99562e75d09e4d5160fe7a3974c6fb842a101a95e7eed0", size = 182132, upload-time = "2026-05-02T16:04:12.653Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/ee/afaf0f85a9a18fe47a67f1e4422ed6cf1fe642f0ae0a2f81166231303c52/wcwidth-0.7.0.tar.gz", hash = "sha256:90e3a7ea092341c44b99562e75d09e4d5160fe7a3974c6fb842a101a95e7eed0", size = 182132 } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/52/e465037f5375f43533d1a80b6923955201596a99142ed524d77b571a1418/wcwidth-0.7.0-py3-none-any.whl", hash = "sha256:5d69154c429a82910e241c738cd0e2976fac8a2dd47a1a805f4afed1c0f136f2", size = 110825, upload-time = "2026-05-02T16:04:11.033Z" }, + { url = "https://files.pythonhosted.org/packages/41/52/e465037f5375f43533d1a80b6923955201596a99142ed524d77b571a1418/wcwidth-0.7.0-py3-none-any.whl", hash = "sha256:5d69154c429a82910e241c738cd0e2976fac8a2dd47a1a805f4afed1c0f136f2", size = 110825 }, ] [[package]] @@ -4331,37 +4349,37 @@ dependencies = [ { name = "pydantic" }, { name = "validators" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ec/a2/6729149edc0bd5884bd0436186f7823f925ec489d47c327a3e408e514494/weaviate_client-4.21.0.tar.gz", hash = "sha256:050243b07f80349bbbaa3d426ace38466a972a073eff5cff62708ae5e9287dbe", size = 838731, upload-time = "2026-04-23T10:37:11.283Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/a2/6729149edc0bd5884bd0436186f7823f925ec489d47c327a3e408e514494/weaviate_client-4.21.0.tar.gz", hash = "sha256:050243b07f80349bbbaa3d426ace38466a972a073eff5cff62708ae5e9287dbe", size = 838731 } wheels = [ - { url = "https://files.pythonhosted.org/packages/83/6b/5ae0aa935602f58fa2c2f5062c3faed8615a45df4b0406eb2d9deeeae73c/weaviate_client-4.21.0-py3-none-any.whl", hash = "sha256:82904bce3aae8f38a880e860195f4a17e6b55810708780f718132199030c8260", size = 639005, upload-time = "2026-04-23T10:37:09.668Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/5ae0aa935602f58fa2c2f5062c3faed8615a45df4b0406eb2d9deeeae73c/weaviate_client-4.21.0-py3-none-any.whl", hash = "sha256:82904bce3aae8f38a880e860195f4a17e6b55810708780f718132199030c8260", size = 639005 }, ] [[package]] name = "websocket-client" version = "1.9.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576 } wheels = [ - { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616 }, ] [[package]] name = "wrapt" version = "1.17.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" } +sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", size = 53998, upload-time = "2025-08-12T05:51:47.138Z" }, - { url = "https://files.pythonhosted.org/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", size = 39020, upload-time = "2025-08-12T05:51:35.906Z" }, - { url = "https://files.pythonhosted.org/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", size = 39098, upload-time = "2025-08-12T05:51:57.474Z" }, - { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" }, - { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" }, - { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" }, - { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" }, - { url = "https://files.pythonhosted.org/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", size = 36705, upload-time = "2025-08-12T05:53:07.123Z" }, - { url = "https://files.pythonhosted.org/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", size = 38877, upload-time = "2025-08-12T05:53:05.436Z" }, - { url = "https://files.pythonhosted.org/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", size = 36885, upload-time = "2025-08-12T05:52:54.367Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, + { url = "https://files.pythonhosted.org/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", size = 53998 }, + { url = "https://files.pythonhosted.org/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", size = 39020 }, + { url = "https://files.pythonhosted.org/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", size = 39098 }, + { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036 }, + { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156 }, + { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102 }, + { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732 }, + { url = "https://files.pythonhosted.org/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", size = 36705 }, + { url = "https://files.pythonhosted.org/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", size = 38877 }, + { url = "https://files.pythonhosted.org/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", size = 36885 }, + { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591 }, ] [[package]] @@ -4371,9 +4389,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "h11" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116, upload-time = "2025-11-20T18:18:01.871Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454Z" }, + { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405 }, ] [[package]] @@ -4384,9 +4402,9 @@ dependencies = [ { name = "pathspec" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/28/a0/8fc2d68e132cf918f18273fdc8a1b8432b60d75ac12fdae4b0ef5c9d2e8d/yamllint-1.38.0.tar.gz", hash = "sha256:09e5f29531daab93366bb061e76019d5e91691ef0a40328f04c927387d1d364d", size = 142446, upload-time = "2026-01-13T07:47:53.276Z" } +sdist = { url = "https://files.pythonhosted.org/packages/28/a0/8fc2d68e132cf918f18273fdc8a1b8432b60d75ac12fdae4b0ef5c9d2e8d/yamllint-1.38.0.tar.gz", hash = "sha256:09e5f29531daab93366bb061e76019d5e91691ef0a40328f04c927387d1d364d", size = 142446 } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/92/aed08e68de6e6a3d7c2328ce7388072cd6affc26e2917197430b646aed02/yamllint-1.38.0-py3-none-any.whl", hash = "sha256:fc394a5b3be980a4062607b8fdddc0843f4fa394152b6da21722f5d59013c220", size = 68940, upload-time = "2026-01-13T07:47:51.343Z" }, + { url = "https://files.pythonhosted.org/packages/05/92/aed08e68de6e6a3d7c2328ce7388072cd6affc26e2917197430b646aed02/yamllint-1.38.0-py3-none-any.whl", hash = "sha256:fc394a5b3be980a4062607b8fdddc0843f4fa394152b6da21722f5d59013c220", size = 68940 }, ] [[package]] @@ -4398,43 +4416,43 @@ dependencies = [ { name = "multidict" }, { name = "propcache" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/23/6e/beb1beec874a72f23815c1434518bfc4ed2175065173fb138c3705f658d4/yarl-1.23.0.tar.gz", hash = "sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5", size = 194676, upload-time = "2026-03-01T22:07:53.373Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/8a/94615bc31022f711add374097ad4144d569e95ff3c38d39215d07ac153a0/yarl-1.23.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860", size = 124737, upload-time = "2026-03-01T22:05:12.897Z" }, - { url = "https://files.pythonhosted.org/packages/e3/6f/c6554045d59d64052698add01226bc867b52fe4a12373415d7991fdca95d/yarl-1.23.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069", size = 87029, upload-time = "2026-03-01T22:05:14.376Z" }, - { url = "https://files.pythonhosted.org/packages/19/2a/725ecc166d53438bc88f76822ed4b1e3b10756e790bafd7b523fe97c322d/yarl-1.23.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25", size = 86310, upload-time = "2026-03-01T22:05:15.71Z" }, - { url = "https://files.pythonhosted.org/packages/99/30/58260ed98e6ff7f90ba84442c1ddd758c9170d70327394a6227b310cd60f/yarl-1.23.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8", size = 97587, upload-time = "2026-03-01T22:05:17.384Z" }, - { url = "https://files.pythonhosted.org/packages/76/0a/8b08aac08b50682e65759f7f8dde98ae8168f72487e7357a5d684c581ef9/yarl-1.23.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072", size = 92528, upload-time = "2026-03-01T22:05:18.804Z" }, - { url = "https://files.pythonhosted.org/packages/52/07/0b7179101fe5f8385ec6c6bb5d0cb9f76bd9fb4a769591ab6fb5cdbfc69a/yarl-1.23.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8", size = 105339, upload-time = "2026-03-01T22:05:20.235Z" }, - { url = "https://files.pythonhosted.org/packages/d3/8a/36d82869ab5ec829ca8574dfcb92b51286fcfb1e9c7a73659616362dc880/yarl-1.23.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7", size = 105061, upload-time = "2026-03-01T22:05:22.268Z" }, - { url = "https://files.pythonhosted.org/packages/66/3e/868e5c3364b6cee19ff3e1a122194fa4ce51def02c61023970442162859e/yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51", size = 100132, upload-time = "2026-03-01T22:05:23.638Z" }, - { url = "https://files.pythonhosted.org/packages/cf/26/9c89acf82f08a52cb52d6d39454f8d18af15f9d386a23795389d1d423823/yarl-1.23.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67", size = 99289, upload-time = "2026-03-01T22:05:25.749Z" }, - { url = "https://files.pythonhosted.org/packages/6f/54/5b0db00d2cb056922356104468019c0a132e89c8d3ab67d8ede9f4483d2a/yarl-1.23.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7", size = 96950, upload-time = "2026-03-01T22:05:27.318Z" }, - { url = "https://files.pythonhosted.org/packages/f6/40/10fa93811fd439341fad7e0718a86aca0de9548023bbb403668d6555acab/yarl-1.23.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d", size = 93960, upload-time = "2026-03-01T22:05:28.738Z" }, - { url = "https://files.pythonhosted.org/packages/bc/d2/8ae2e6cd77d0805f4526e30ec43b6f9a3dfc542d401ac4990d178e4bf0cf/yarl-1.23.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760", size = 104703, upload-time = "2026-03-01T22:05:30.438Z" }, - { url = "https://files.pythonhosted.org/packages/2f/0c/b3ceacf82c3fe21183ce35fa2acf5320af003d52bc1fcf5915077681142e/yarl-1.23.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2", size = 98325, upload-time = "2026-03-01T22:05:31.835Z" }, - { url = "https://files.pythonhosted.org/packages/9d/e0/12900edd28bdab91a69bd2554b85ad7b151f64e8b521fe16f9ad2f56477a/yarl-1.23.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86", size = 105067, upload-time = "2026-03-01T22:05:33.358Z" }, - { url = "https://files.pythonhosted.org/packages/15/61/74bb1182cf79c9bbe4eb6b1f14a57a22d7a0be5e9cedf8e2d5c2086474c3/yarl-1.23.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34", size = 100285, upload-time = "2026-03-01T22:05:35.4Z" }, - { url = "https://files.pythonhosted.org/packages/69/7f/cd5ef733f2550de6241bd8bd8c3febc78158b9d75f197d9c7baa113436af/yarl-1.23.0-cp312-cp312-win32.whl", hash = "sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d", size = 82359, upload-time = "2026-03-01T22:05:36.811Z" }, - { url = "https://files.pythonhosted.org/packages/f5/be/25216a49daeeb7af2bec0db22d5e7df08ed1d7c9f65d78b14f3b74fd72fc/yarl-1.23.0-cp312-cp312-win_amd64.whl", hash = "sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e", size = 87674, upload-time = "2026-03-01T22:05:38.171Z" }, - { url = "https://files.pythonhosted.org/packages/d2/35/aeab955d6c425b227d5b7247eafb24f2653fedc32f95373a001af5dfeb9e/yarl-1.23.0-cp312-cp312-win_arm64.whl", hash = "sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9", size = 81879, upload-time = "2026-03-01T22:05:40.006Z" }, - { url = "https://files.pythonhosted.org/packages/69/68/c8739671f5699c7dc470580a4f821ef37c32c4cb0b047ce223a7f115757f/yarl-1.23.0-py3-none-any.whl", hash = "sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f", size = 48288, upload-time = "2026-03-01T22:07:51.388Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/23/6e/beb1beec874a72f23815c1434518bfc4ed2175065173fb138c3705f658d4/yarl-1.23.0.tar.gz", hash = "sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5", size = 194676 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/8a/94615bc31022f711add374097ad4144d569e95ff3c38d39215d07ac153a0/yarl-1.23.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860", size = 124737 }, + { url = "https://files.pythonhosted.org/packages/e3/6f/c6554045d59d64052698add01226bc867b52fe4a12373415d7991fdca95d/yarl-1.23.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069", size = 87029 }, + { url = "https://files.pythonhosted.org/packages/19/2a/725ecc166d53438bc88f76822ed4b1e3b10756e790bafd7b523fe97c322d/yarl-1.23.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25", size = 86310 }, + { url = "https://files.pythonhosted.org/packages/99/30/58260ed98e6ff7f90ba84442c1ddd758c9170d70327394a6227b310cd60f/yarl-1.23.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8", size = 97587 }, + { url = "https://files.pythonhosted.org/packages/76/0a/8b08aac08b50682e65759f7f8dde98ae8168f72487e7357a5d684c581ef9/yarl-1.23.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072", size = 92528 }, + { url = "https://files.pythonhosted.org/packages/52/07/0b7179101fe5f8385ec6c6bb5d0cb9f76bd9fb4a769591ab6fb5cdbfc69a/yarl-1.23.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8", size = 105339 }, + { url = "https://files.pythonhosted.org/packages/d3/8a/36d82869ab5ec829ca8574dfcb92b51286fcfb1e9c7a73659616362dc880/yarl-1.23.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7", size = 105061 }, + { url = "https://files.pythonhosted.org/packages/66/3e/868e5c3364b6cee19ff3e1a122194fa4ce51def02c61023970442162859e/yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51", size = 100132 }, + { url = "https://files.pythonhosted.org/packages/cf/26/9c89acf82f08a52cb52d6d39454f8d18af15f9d386a23795389d1d423823/yarl-1.23.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67", size = 99289 }, + { url = "https://files.pythonhosted.org/packages/6f/54/5b0db00d2cb056922356104468019c0a132e89c8d3ab67d8ede9f4483d2a/yarl-1.23.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7", size = 96950 }, + { url = "https://files.pythonhosted.org/packages/f6/40/10fa93811fd439341fad7e0718a86aca0de9548023bbb403668d6555acab/yarl-1.23.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d", size = 93960 }, + { url = "https://files.pythonhosted.org/packages/bc/d2/8ae2e6cd77d0805f4526e30ec43b6f9a3dfc542d401ac4990d178e4bf0cf/yarl-1.23.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760", size = 104703 }, + { url = "https://files.pythonhosted.org/packages/2f/0c/b3ceacf82c3fe21183ce35fa2acf5320af003d52bc1fcf5915077681142e/yarl-1.23.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2", size = 98325 }, + { url = "https://files.pythonhosted.org/packages/9d/e0/12900edd28bdab91a69bd2554b85ad7b151f64e8b521fe16f9ad2f56477a/yarl-1.23.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86", size = 105067 }, + { url = "https://files.pythonhosted.org/packages/15/61/74bb1182cf79c9bbe4eb6b1f14a57a22d7a0be5e9cedf8e2d5c2086474c3/yarl-1.23.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34", size = 100285 }, + { url = "https://files.pythonhosted.org/packages/69/7f/cd5ef733f2550de6241bd8bd8c3febc78158b9d75f197d9c7baa113436af/yarl-1.23.0-cp312-cp312-win32.whl", hash = "sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d", size = 82359 }, + { url = "https://files.pythonhosted.org/packages/f5/be/25216a49daeeb7af2bec0db22d5e7df08ed1d7c9f65d78b14f3b74fd72fc/yarl-1.23.0-cp312-cp312-win_amd64.whl", hash = "sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e", size = 87674 }, + { url = "https://files.pythonhosted.org/packages/d2/35/aeab955d6c425b227d5b7247eafb24f2653fedc32f95373a001af5dfeb9e/yarl-1.23.0-cp312-cp312-win_arm64.whl", hash = "sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9", size = 81879 }, + { url = "https://files.pythonhosted.org/packages/69/68/c8739671f5699c7dc470580a4f821ef37c32c4cb0b047ce223a7f115757f/yarl-1.23.0-py3-none-any.whl", hash = "sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f", size = 48288 }, ] [[package]] name = "zipp" version = "4.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238 }, ] [[package]] name = "zipstream-ng" version = "1.9.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/33/fce793430e56888cfe3d61199b0116fa42b95d54c2e0fe87b85829507d10/zipstream_ng-1.9.2.tar.gz", hash = "sha256:116b7304b00f3251328cb300fa90f0f09d523b7faf2a06b3eaf7277dcb82cc3e", size = 32446, upload-time = "2026-05-17T16:09:26.934Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/33/fce793430e56888cfe3d61199b0116fa42b95d54c2e0fe87b85829507d10/zipstream_ng-1.9.2.tar.gz", hash = "sha256:116b7304b00f3251328cb300fa90f0f09d523b7faf2a06b3eaf7277dcb82cc3e", size = 32446 } wheels = [ - { url = "https://files.pythonhosted.org/packages/63/6e/103ba2f47ae052a9d91e4c3d4d7b1d45128045675325d99a476e9171fa2e/zipstream_ng-1.9.2-py3-none-any.whl", hash = "sha256:7292efc812a437ec688cef2c9523a4e710cd669e4b5abc7d5a15eb4d5e68e4ea", size = 23407, upload-time = "2026-05-17T16:09:25.874Z" }, + { url = "https://files.pythonhosted.org/packages/63/6e/103ba2f47ae052a9d91e4c3d4d7b1d45128045675325d99a476e9171fa2e/zipstream_ng-1.9.2-py3-none-any.whl", hash = "sha256:7292efc812a437ec688cef2c9523a4e710cd669e4b5abc7d5a15eb4d5e68e4ea", size = 23407 }, ] From d5b4a2faa3493c5024c47083d5d8fd9ceb091609 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Wed, 24 Jun 2026 11:51:06 +0530 Subject: [PATCH 40/44] =?UTF-8?q?UN-3618=20[REFACTOR]=20PG=20Queue=20?= =?UTF-8?q?=E2=80=94=20gate=20solely=20on=20the=20Flipt=20flag=20(+=20UN-3?= =?UTF-8?q?619=20transient-connect=20retry)=20(#2109)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3618 [REFACTOR] PG Queue — single-flag gating (drop PG_QUEUE_TRANSPORT_ENABLED) The pg_queue_enabled Flipt flag becomes the sole gate across all four resolvers (execution, scheduler, executor backend+workers). Removes the redundant env master-switch and the master_gate_enabled parameter from the shared resolve_pg_transport. Fully fail-closed (flag off / Flipt blind / Flipt error / no org -> Celery); verified by unit + real-stack dev-test. Co-Authored-By: Claude Opus 4.8 * UN-3619 [FEAT] PG Queue — bounded retry on transient Postgres connect create_pg_connection retries a transient psycopg2.OperationalError with bounded exponential backoff (WORKER_PG_QUEUE_CONNECT_RETRIES / _BACKOFF, defaults 3 / 0.5s). Connecting is side-effect-free so retry is safe; the enqueue INSERT is intentionally NOT retried (double-dispatch risk). Co-Authored-By: Claude Opus 4.8 * UN-3618 UN-3619 [REFACTOR] address PR #2109 review - Flipt observability (UN-3618): check_feature_flag_status / _variant now log a genuine evaluation failure (warning + exc_info) instead of silently returning the disabled default — the now-sole gate's outages are visible at the decision layer. Resolver defense-in-depth guards kept. - Update stale shared executor_rpc docstring (Flipt is the sole gate). - Remove now-dead module logger in workers executor_rpc. - Connect retry (UN-3619): clamp+warn out-of-range RETRIES (cap 10) / negative BACKOFF; drop the unreachable AssertionError sentinel via an inverted loop; document the backoff cap + that permanent misconfigs are retried-then-raised; document both knobs in sample.env. - Tests: add backoff-cap, unset-default, invalid/negative-backoff, attempts-clamp, geometric-growth, and params-forwarded/stable coverage (13 connection tests). Co-Authored-By: Claude Opus 4.8 * UN-3619 [FIX] pg_benchmark connection test — drop generator-throw lambda (SonarCloud) Replace the (_ for _ in ()).throw(...) generator trick with a plain local def connect that raises — matches the other tests' style and clears the SonarCloud comprehension finding. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- backend/backend/settings/base.py | 8 - backend/pg_queue/executor_rpc.py | 16 +- backend/pg_queue/flags.py | 4 +- backend/pg_queue/tests/test_executor_rpc.py | 31 +-- backend/sample.env | 7 - backend/scheduler/ownership.py | 22 +- .../tests/test_pg_schedule_ownership.py | 35 +-- .../workflow_v2/tests/test_transport.py | 43 ++-- .../workflow_manager/workflow_v2/transport.py | 47 ++-- docker/docker-compose.yaml | 5 +- docker/sample.env | 6 +- .../flags/src/unstract/flags/feature_flag.py | 20 ++ .../workflow_execution/executor_rpc.py | 20 +- workers/queue_backend/pg_queue/connection.py | 135 +++++++++-- .../queue_backend/pg_queue/executor_rpc.py | 36 +-- workers/sample.env | 15 +- workers/tests/test_executor_rpc.py | 34 ++- workers/tests/test_pg_connection.py | 210 ++++++++++++++++++ 18 files changed, 451 insertions(+), 243 deletions(-) create mode 100644 workers/tests/test_pg_connection.py diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index 4b3fe0ce90..787a7603e6 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -151,14 +151,6 @@ def get_required_setting(setting_key: str, default: str | None = None) -> str | CELERY_BACKEND_DB_NAME = os.environ.get("CELERY_BACKEND_DB_NAME") or DB_NAME DEFAULT_ORGANIZATION = "default_org" FLIPT_BASE_URL = os.environ.get("FLIPT_BASE_URL", "http://localhost:9005") -# 9e PG-queue transport master-gate (kill-switch). When not "true", -# resolve_transport() never consults Flipt and every workflow execution rides -# Celery — the instant global rollback AND the deploy-ordering safety (stays off -# until PG consumers are running in the fleet). See -# workers/queue_backend/pg_queue/9e-design.md §2. -PG_QUEUE_TRANSPORT_ENABLED = CommonUtils.str_to_bool( - os.environ.get("PG_QUEUE_TRANSPORT_ENABLED", "False") -) PLATFORM_HOST = os.environ.get("PLATFORM_SERVICE_HOST", "http://localhost") PLATFORM_PORT = os.environ.get("PLATFORM_SERVICE_PORT", 3001) PROMPT_HOST = os.environ.get("PROMPT_HOST", "http://localhost") diff --git a/backend/pg_queue/executor_rpc.py b/backend/pg_queue/executor_rpc.py index c50850b2e9..cd850a2440 100644 --- a/backend/pg_queue/executor_rpc.py +++ b/backend/pg_queue/executor_rpc.py @@ -3,19 +3,18 @@ The gate + reply_key/timeout orchestration + routing live ONCE in ``unstract.workflow_execution.executor_rpc`` (shared with the workers). This module is the thin Django half: a :class:`DjangoQueueTransport` that enqueues via the ORM -(``enqueue_task``) and polls ``PgTaskResult``, plus the per-side gate (master switch = -``settings.PG_QUEUE_TRANSPORT_ENABLED``) and the :func:`get_executor_dispatcher` +(``enqueue_task``) and polls ``PgTaskResult``, plus the :func:`get_executor_dispatcher` factory that wires them together. -Zero-regression: gate off ⇒ the routing dispatcher delegates every mode to the -unchanged Celery ``ExecutionDispatcher`` and no ``pg_task_result`` row is created. +Zero-regression: with the ``pg_queue_enabled`` Flipt flag off the routing dispatcher +delegates every mode to the unchanged Celery ``ExecutionDispatcher`` and no +``pg_task_result`` row is created. """ from __future__ import annotations from typing import TYPE_CHECKING -from django.conf import settings from django.db import close_old_connections from pg_queue.models import PgTaskResult @@ -48,12 +47,9 @@ def resolve_executor_transport(context: ExecutionContext) -> bool: """True → route this executor dispatch over PG; False → Celery (default). - The backend gate: master switch ``settings.PG_QUEUE_TRANSPORT_ENABLED``, then the - shared Flipt eval (single ``pg_queue_enabled`` flag, fail-closed). + The single ``pg_queue_enabled`` Flipt flag (fail-closed). """ - return resolve_pg_transport( - context, master_gate_enabled=settings.PG_QUEUE_TRANSPORT_ENABLED - ) + return resolve_pg_transport(context) class DjangoQueueTransport(QueueTransport): diff --git a/backend/pg_queue/flags.py b/backend/pg_queue/flags.py index f2d56d8b73..cf190134c1 100644 --- a/backend/pg_queue/flags.py +++ b/backend/pg_queue/flags.py @@ -5,8 +5,8 @@ executor (``pg_queue/executor_rpc.py``) all read this one key. Kept in a neutral leaf module so the three resolvers import a single constant instead of duplicating the literal (a grep on ``PG_QUEUE_FLAG_KEY`` finds every use), making -"one flag" a structural guarantee. ``PG_QUEUE_TRANSPORT_ENABLED`` (env) remains -the master kill-switch consulted before this flag. +"one flag" a structural guarantee. This flag is the **sole** rollout control — +fail-closed to Celery on a blind/unreachable Flipt or any error. """ PG_QUEUE_FLAG_KEY = "pg_queue_enabled" diff --git a/backend/pg_queue/tests/test_executor_rpc.py b/backend/pg_queue/tests/test_executor_rpc.py index 1361b64f09..610c45ee0b 100644 --- a/backend/pg_queue/tests/test_executor_rpc.py +++ b/backend/pg_queue/tests/test_executor_rpc.py @@ -4,8 +4,8 @@ Flipt gate matrix live ONCE in the shared module and are covered in ``workers/tests/test_executor_rpc.py`` against a fake transport. This suite pins the **backend half**: the :class:`DjangoQueueTransport` (enqueue via the ORM, poll -``PgTaskResult`` → ``ExecResultRow``), the per-side gate (master switch = -``settings.PG_QUEUE_TRANSPORT_ENABLED``), and the factory wiring. +``PgTaskResult`` → ``ExecResultRow``), the resolver's delegation to the shared +single-Flipt-flag gate, and the factory wiring. """ from unittest.mock import MagicMock, patch @@ -95,31 +95,22 @@ def test_multi_iteration_poll_misses_then_hits(self): class TestResolveExecutorTransport: - @staticmethod - def _gate(on: bool): - s = MagicMock() - s.PG_QUEUE_TRANSPORT_ENABLED = on - return patch(f"{_MOD}.settings", s) - - def test_reads_settings_master_gate_on(self): - with self._gate(True), patch( - f"{_MOD}.resolve_pg_transport", return_value=True - ) as r: + def test_delegates_to_shared_flipt_resolver_true(self): + with patch(f"{_MOD}.resolve_pg_transport", return_value=True) as r: assert resolve_executor_transport(_ctx()) is True - assert r.call_args.kwargs["master_gate_enabled"] is True + r.assert_called_once() + # No master-gate is threaded any more — Flipt is the sole gate. + assert "master_gate_enabled" not in r.call_args.kwargs - def test_reads_settings_master_gate_off(self): - with self._gate(False), patch( - f"{_MOD}.resolve_pg_transport", return_value=False - ) as r: - resolve_executor_transport(_ctx()) - assert r.call_args.kwargs["master_gate_enabled"] is False + def test_delegates_to_shared_flipt_resolver_false(self): + with patch(f"{_MOD}.resolve_pg_transport", return_value=False): + assert resolve_executor_transport(_ctx()) is False class TestFactoryWiring: def test_factory_wires_routing_with_django_transport(self): d = get_executor_dispatcher(celery_app="app") assert isinstance(d, RoutingExecutionDispatcher) - # PG dispatcher uses the backend ORM transport; gate = the settings resolver. + # PG dispatcher uses the backend ORM transport; gate = the Flipt resolver. assert isinstance(d._pg._transport, DjangoQueueTransport) assert d._resolve is resolve_executor_transport diff --git a/backend/sample.env b/backend/sample.env index 30450faff3..377016fdec 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -184,13 +184,6 @@ TOOL_REGISTRY_CONFIG_PATH="/data/tool_registry_config" # Flipt Service FLIPT_SERVICE_AVAILABLE=False -# 9e PG-queue transport master-gate (kill-switch). Routing an execution onto the -# PG queue requires ALL THREE: this gate True, FLIPT_SERVICE_AVAILABLE=True (else -# the Flipt client returns False for every flag), and the Flipt flag -# `pg_queue_enabled` on. Keep False until PG queue consumers are running -# in the fleet; set False for instant rollback. -PG_QUEUE_TRANSPORT_ENABLED=False - # File System Configuration for Workflow and API Execution # Directory Prefixes for storing execution files diff --git a/backend/scheduler/ownership.py b/backend/scheduler/ownership.py index f5f3684419..039b583c84 100644 --- a/backend/scheduler/ownership.py +++ b/backend/scheduler/ownership.py @@ -12,9 +12,9 @@ ``PeriodicTask`` disabled, so the two can't both fire. Inert by default: ``resolve_schedule_owner`` fails closed to Beat -(``pg_owned=False``) until ops turns the master gate on AND ramps the single -``pg_queue_enabled`` Flipt flag — so reconciling on every schedule edit is a -no-op (everything stays Beat-owned) until the rollout starts. +(``pg_owned=False``) until ops ramps the single ``pg_queue_enabled`` Flipt flag — so +reconciling on every schedule edit is a no-op (everything stays Beat-owned) until the +rollout starts. """ from __future__ import annotations @@ -22,7 +22,6 @@ import logging import os -from django.conf import settings from django.db import transaction from django.utils import timezone from django_celery_beat.models import PeriodicTask @@ -41,19 +40,14 @@ def resolve_schedule_owner(pipeline_id: str, organization_id: str | None) -> bool: """True → the PG scheduler owns this schedule; False → Celery Beat does. - Mirrors ``resolve_transport``: master-gated by ``PG_QUEUE_TRANSPORT_ENABLED`` - (shared PG kill-switch), then the single ``pg_queue_enabled`` Flipt flag, keyed - on ``pipeline_id`` for a stable percentage bucket. **Fails closed to Beat** - on a closed gate, a blind Flipt, or any error — so a schedule never silently - loses its firer. + Mirrors ``resolve_transport``: gated by the single ``pg_queue_enabled`` Flipt + flag, keyed on ``pipeline_id`` for a stable percentage bucket. **Fails closed to + Beat** on a blind Flipt or any error — so a schedule never silently loses its + firer. """ - # Master gate off → never consult Flipt; every schedule stays on Beat. - if not settings.PG_QUEUE_TRANSPORT_ENABLED: - return False - if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true": logger.warning( - "resolve_schedule_owner: gate ON but FLIPT_SERVICE_AVAILABLE != true " + "resolve_schedule_owner: FLIPT_SERVICE_AVAILABLE != true " "(Flipt blind) for pipeline %s; leaving on Beat", pipeline_id, ) diff --git a/backend/scheduler/tests/test_pg_schedule_ownership.py b/backend/scheduler/tests/test_pg_schedule_ownership.py index ed83030a81..61f16bc4ef 100644 --- a/backend/scheduler/tests/test_pg_schedule_ownership.py +++ b/backend/scheduler/tests/test_pg_schedule_ownership.py @@ -1,9 +1,9 @@ """Unit tests for the schedule-ownership ramp control (Phase 9, ②c). -DB-free: Flipt, settings, and the ORM (``PgPeriodicSchedule`` / ``PeriodicTask``) -are mocked. These pin the fail-closed rollout decision and — the load-bearing -property — that handing a schedule to PG disables its Beat ``PeriodicTask`` in -the same step (no double-fire), with pause state preserved. +DB-free: Flipt and the ORM (``PgPeriodicSchedule`` / ``PeriodicTask``) are mocked. +These pin the fail-closed rollout decision and — the load-bearing property — that +handing a schedule to PG disables its Beat ``PeriodicTask`` in the same step (no +double-fire), with pause state preserved. """ import contextlib @@ -17,32 +17,16 @@ _ORG = "org_abc" -def _gate(on: bool): - s = MagicMock() - s.PG_QUEUE_TRANSPORT_ENABLED = on - return patch("scheduler.ownership.settings", s) - - class TestResolveScheduleOwner: - def test_gate_off_is_beat(self, monkeypatch): - monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with _gate(False), patch( - "scheduler.ownership.check_feature_flag_status" - ) as flag: - assert ownership.resolve_schedule_owner(_PID, _ORG) is False - flag.assert_not_called() # gate off → Flipt never consulted - def test_flipt_unavailable_is_beat(self, monkeypatch): monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false") - with _gate(True), patch( - "scheduler.ownership.check_feature_flag_status" - ) as flag: + with patch("scheduler.ownership.check_feature_flag_status") as flag: assert ownership.resolve_schedule_owner(_PID, _ORG) is False flag.assert_not_called() def test_flag_true_is_pg(self, monkeypatch): monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with _gate(True), patch( + with patch( "scheduler.ownership.check_feature_flag_status", return_value=True ) as flag: assert ownership.resolve_schedule_owner(_PID, _ORG) is True @@ -53,14 +37,14 @@ def test_flag_true_is_pg(self, monkeypatch): def test_flag_false_is_beat(self, monkeypatch): monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with _gate(True), patch( + with patch( "scheduler.ownership.check_feature_flag_status", return_value=False ): assert ownership.resolve_schedule_owner(_PID, _ORG) is False def test_flipt_error_fails_closed_to_beat(self, monkeypatch): monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - with _gate(True), patch( + with patch( "scheduler.ownership.check_feature_flag_status", side_effect=RuntimeError("flipt down"), ): @@ -159,7 +143,8 @@ class TestReconcileAtomicityRealDB: """The load-bearing invariant: the pg_owned write and the PeriodicTask write are ONE transaction — if the PeriodicTask update fails, pg_owned rolls back (so a schedule can't end up pg_owned with Beat still enabled). Needs a real - DB (the mocked atomic() can't prove rollback); skips if unreachable.""" + DB (the mocked atomic() can't prove rollback); skips if unreachable. + """ def test_periodictask_update_failure_rolls_back_pg_owned(self): import uuid diff --git a/backend/workflow_manager/workflow_v2/tests/test_transport.py b/backend/workflow_manager/workflow_v2/tests/test_transport.py index 2b0b100f63..a93c926f4a 100644 --- a/backend/workflow_manager/workflow_v2/tests/test_transport.py +++ b/backend/workflow_manager/workflow_v2/tests/test_transport.py @@ -1,15 +1,14 @@ """Tests for the 9e transport-resolution seam (PR 3 — Flipt canary wiring). ``resolve_transport`` is the single chokepoint that decides whether a new -execution rides the legacy Celery transport or the Postgres queue. It is gated -by an env master-switch and, when that is on, by a Flipt boolean flag; it fails -closed to Celery on any problem. These tests pin that contract. +execution rides the legacy Celery transport or the Postgres queue. It is gated by a +single ``pg_queue_enabled`` Flipt boolean flag; it fails closed to Celery on any +problem. These tests pin that contract. """ from unittest.mock import MagicMock, patch import pytest -from django.test import override_settings from unstract.core.data_models import ( DEFAULT_WORKFLOW_TRANSPORT, WorkflowTransport, @@ -37,58 +36,47 @@ def test_default_is_celery(self): class TestResolveTransport: @pytest.fixture(autouse=True) def _flipt_available(self, monkeypatch): - """The gate-ON path short-circuits to celery when Flipt is marked + """resolve_transport short-circuits to celery when Flipt is marked unavailable; default it available so the flag-evaluation tests exercise - the real path. Individual tests override as needed.""" + the real path. Individual tests override as needed. + """ monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") - @override_settings(PG_QUEUE_TRANSPORT_ENABLED=False) - def test_master_gate_off_never_consults_flipt(self): - """Master-gate off → Celery, and Flipt is not even called.""" - with patch(_FLIPT) as flipt: - result = resolve_transport(execution_id="e1", organization_id="org1") - assert result == WorkflowTransport.CELERY.value - flipt.assert_not_called() - - @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) def test_missing_organization_forces_celery(self): """No org context → can't segment safely → fail closed; Flipt not called - (str(None)/"" must never reach the Flipt org segment).""" + (str(None)/"" must never reach the Flipt org segment). + """ with patch(_FLIPT) as flipt: result = resolve_transport(execution_id="e1", organization_id="") assert result == WorkflowTransport.CELERY.value flipt.assert_not_called() - @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) - def test_gate_on_but_flipt_unavailable_forces_celery(self, monkeypatch): - """Gate ON + Flipt service unavailable → celery, loudly — not a silent - masquerade as a healthy 100%-celery canary.""" + def test_flipt_unavailable_forces_celery(self, monkeypatch): + """Flipt service unavailable → celery, loudly — not a silent masquerade as a + healthy 100%-celery rollout. + """ monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false") with patch(_FLIPT) as flipt: result = resolve_transport(execution_id="e1", organization_id="org1") assert result == WorkflowTransport.CELERY.value flipt.assert_not_called() - @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) - def test_gate_on_flipt_true_resolves_pg_queue(self): + def test_flipt_true_resolves_pg_queue(self): with patch(_FLIPT, return_value=True): result = resolve_transport(execution_id="e1", organization_id="org1") assert result == WorkflowTransport.PG_QUEUE.value - @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) - def test_gate_on_flipt_false_resolves_celery(self): + def test_flipt_false_resolves_celery(self): with patch(_FLIPT, return_value=False): result = resolve_transport(execution_id="e1", organization_id="org1") assert result == WorkflowTransport.CELERY.value - @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) def test_flipt_exception_fails_closed_to_celery(self): """A Flipt outage must never break execution creation.""" with patch(_FLIPT, side_effect=RuntimeError("flipt down")): result = resolve_transport(execution_id="e1", organization_id="org1") assert result == WorkflowTransport.CELERY.value - @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) def test_passes_execution_id_as_entity_and_builds_context(self): """entity_id = execution_id (sticky bucketing); context carries the org/workflow/pipeline for segment rules. @@ -110,7 +98,6 @@ def test_passes_execution_id_as_entity_and_builds_context(self): }, ) - @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) def test_non_string_ids_are_coerced_for_flipt(self): """Callers pass UUID objects for the ids. Flipt's context is a gRPC map and entity_id must hash stably, so every value must @@ -134,14 +121,12 @@ def test_non_string_ids_are_coerced_for_flipt(self): assert kwargs["context"]["workflow_id"] == str(wf) assert kwargs["context"]["pipeline_id"] == str(pl) - @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) def test_context_omits_unset_optional_ids(self): with patch(_FLIPT, return_value=True) as flipt: resolve_transport(execution_id="exec-42", organization_id="org1") _, kwargs = flipt.call_args assert kwargs["context"] == {"organization_id": "org1"} - @override_settings(PG_QUEUE_TRANSPORT_ENABLED=True) def test_result_is_a_valid_transport_value(self): valid = {t.value for t in WorkflowTransport} with patch(_FLIPT, return_value=True): diff --git a/backend/workflow_manager/workflow_v2/transport.py b/backend/workflow_manager/workflow_v2/transport.py index 337e3dfe1d..08b2f31c24 100644 --- a/backend/workflow_manager/workflow_v2/transport.py +++ b/backend/workflow_manager/workflow_v2/transport.py @@ -8,25 +8,23 @@ row: the payload is the single carrier, durable for PG via the queue row's JSONB, and the giant shared table is never migrated for this work. -PR 3 (this change) replaces PR 1's hardwired Celery with a Flipt evaluation: +The transport is resolved from a single Flipt evaluation: - master-gate (env) → Flipt boolean (``pg_queue_enabled``) → transport + Flipt boolean (``pg_queue_enabled``) → transport -Routing onto PG needs **all three** of: the env master-gate on, Flipt reachable -(``FLIPT_SERVICE_AVAILABLE=true``), and the flag enabled for this execution. +Routing onto PG needs **both**: Flipt reachable (``FLIPT_SERVICE_AVAILABLE=true``) +and the single ``pg_queue_enabled`` flag enabled for this execution. The flag is the +sole rollout control — flip it to ramp PG, flip it off (or a Flipt outage) to fall +back to Celery. -- **Master-gate** (``settings.PG_QUEUE_TRANSPORT_ENABLED``, default off): until - ops flips it on, Flipt is never consulted and every execution rides Celery. - This is both the instant global kill-switch *and* the deploy-ordering safety — - the flag stays inert until PG consumers are actually running in the fleet. - **Flipt** decides per-execution: ``entity_id = execution_id`` drives the percentage-rollout hashing (an execution resolves exactly once, so it can never re-bucket mid-flight); ``context`` carries org/workflow/pipeline for segment rules. The flag contract is fixed in 9e-design §2. -- **Fail-closed to Celery**: a Flipt outage must never break execution creation, - mirroring ``normalize_transport`` on the read side. The gate-ON path logs its - decision so a "gate on but still all Celery" situation (e.g. a blind Flipt) - is visible rather than silent. +- **Fail-closed to Celery**: a blind/unreachable Flipt, a missing org, or any error + must never break execution creation — it resolves to Celery, mirroring + ``normalize_transport`` on the read side. The decision is logged so a "flag on but + still all Celery" situation (e.g. a blind Flipt) is visible rather than silent. """ from __future__ import annotations @@ -35,7 +33,6 @@ import os from typing import TYPE_CHECKING -from django.conf import settings from pg_queue.flags import PG_QUEUE_FLAG_KEY from unstract.core.data_models import WorkflowTransport @@ -78,22 +75,12 @@ def resolve_transport( pipeline_id: Optional, carried in ``context`` for future segment rules. Returns: - A :class:`WorkflowTransport` value string — ``"pg_queue"`` only when the - master-gate is on, Flipt is reachable, and Flipt says yes for this - execution; ``"celery"`` otherwise (including any error — fail-closed). + A :class:`WorkflowTransport` value string — ``"pg_queue"`` only when Flipt + is reachable and says yes for this execution; ``"celery"`` otherwise + (including any error — fail-closed). """ celery = WorkflowTransport.CELERY.value - # Master-gate: until ops sets PG_QUEUE_TRANSPORT_ENABLED=true, never consult - # Flipt — every execution rides Celery (kill-switch + deploy-ordering safety). - # Intentionally unlogged: this is the steady state for every execution while - # the gate is off, so a log here would be pure noise. - if not settings.PG_QUEUE_TRANSPORT_ENABLED: - return celery - - # Gate is ON (canary/rollout). From here the decision is logged so a - # "gate on but everything still Celery" situation cannot hide. - # No org context → per-org segment matching can't be trusted (str(None) would # ship a bogus "None" org into the Flipt context and mis-segment). The view # path validates the header non-None, but the helper path reads it from @@ -107,12 +94,12 @@ def resolve_transport( # FliptClient returns False for ALL flags when the service is marked # unavailable — indistinguishable from "rollout says no". Surface it loudly so - # a blind Flipt under an ON gate doesn't masquerade as a healthy 100%-Celery - # canary. Parse exactly as FliptClient does (``.lower()``, no ``.strip()``) - # so the two can never disagree on a value like ``" true"``. + # a blind Flipt doesn't masquerade as a healthy 100%-Celery rollout. Parse + # exactly as FliptClient does (``.lower()``, no ``.strip()``) so the two can + # never disagree on a value like ``" true"``. if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true": logger.warning( - "resolve_transport: gate ON but FLIPT_SERVICE_AVAILABLE != true " + "resolve_transport: FLIPT_SERVICE_AVAILABLE != true " "(Flipt is blind) for execution %s; forcing celery", execution_id, ) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 0dca79ab4c..fffd1d3426 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -553,9 +553,8 @@ services: # still hand off to the Celery executor (tool-execution RPC) / Celery # notifications, so they keep a rabbitmq dependency until the executor and # notifications themselves move to PG. Each maps 1:1 to a future K8s Deployment. - # Executions still route to Celery until the backend gate - # (PG_QUEUE_TRANSPORT_ENABLED) + Flipt flag are flipped — that is the ramp, - # not this change. + # Executions still route to Celery until the single `pg_queue_enabled` Flipt + # flag is ramped — that is the rollout, not this change. # =========================================================================== # Orchestrator consumer for API-deployment executions (async_execute_bin_api). diff --git a/docker/sample.env b/docker/sample.env index 7f114461bd..e07444096e 100644 --- a/docker/sample.env +++ b/docker/sample.env @@ -127,6 +127,6 @@ ENABLE_METRICS=true WORKER_PG_REAPER_INTERVAL_SECONDS=5 # Reaper sweep interval (seconds) # # NOTE: routing executions to PG is a SEPARATE, later step. Running these -# services does NOT move any traffic — the backend gate PG_QUEUE_TRANSPORT_ENABLED -# (in backend/.env, default off) plus the Flipt flag still decide per-execution -# transport, and both stay off until the rollout ramp. +# services does NOT move any traffic — the single `pg_queue_enabled` Flipt flag +# (default off, fail-closed) decides per-execution transport, and stays off until +# the rollout ramp. diff --git a/unstract/flags/src/unstract/flags/feature_flag.py b/unstract/flags/src/unstract/flags/feature_flag.py index f9833c3aa2..05bc5de899 100644 --- a/unstract/flags/src/unstract/flags/feature_flag.py +++ b/unstract/flags/src/unstract/flags/feature_flag.py @@ -44,6 +44,18 @@ def check_feature_flag_status( return bool(result) except Exception: + # A genuine evaluation failure (Flipt unreachable, renamed/deleted flag, + # wrong namespace) otherwise collapses into the same False as a healthy + # "flag off" — log it so a real outage is visible at the decision layer + # rather than silent. Callers fail closed on False either way. warning + + # exc_info (not logger.exception) so a persistently-down Flipt doesn't + # bury every call as a Sentry error. + logger.warning( + "check_feature_flag_status: evaluation failed for flag %r; " + "treating as disabled", + flag_key, + exc_info=True, + ) return False @@ -136,4 +148,12 @@ def check_feature_flag_variant( return result except Exception: + # Same rationale as check_feature_flag_status — surface a genuine + # evaluation failure instead of silently returning the disabled default. + logger.warning( + "check_feature_flag_variant: evaluation failed for flag %r; " + "returning disabled default", + flag_key, + exc_info=True, + ) return default_result diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py index 5d256a9132..14dcc6e041 100644 --- a/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py +++ b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py @@ -16,10 +16,9 @@ ``dispatch_with_callback`` + the reply_key/timeout orchestration and the never-raises contract (timeout/failure → ``ExecutionResult.failure``). It calls ``transport.enqueue(...)`` and ``transport.wait_for_result(...)``. -- :func:`resolve_pg_transport` — the gate: a master kill-switch (its boolean value - supplied by the caller — a Django setting on the backend, an env var on the - workers) then the single ``pg_queue_enabled`` Flipt flag, bucketed per org. Fails - closed to Celery. +- :func:`resolve_pg_transport` — the gate: the single ``pg_queue_enabled`` Flipt + flag (bucketed per org) is the sole control. Fails closed to Celery on a + blind/unreachable Flipt or any error. - :class:`RoutingExecutionDispatcher` — picks PG-vs-Celery per call (instant rollout/rollback) for every mode; the Celery dispatcher, the PG dispatcher and the per-side ``resolve`` are all injected. @@ -135,22 +134,17 @@ def wait_for_result(self, reply_key: str, timeout: float) -> ExecResultRow | Non def resolve_pg_transport( context: ExecutionContext, *, - master_gate_enabled: bool, flag_key: str = PG_QUEUE_FLAG_KEY, ) -> bool: """True → route this executor dispatch over PG; False → Celery (default). - Master-gated by ``master_gate_enabled`` (the caller supplies its value — a Django - setting on the backend, an env var on the workers), then the single - ``pg_queue_enabled`` Flipt flag, bucketed per org. **Fails closed to Celery** on a - closed gate, a blind Flipt, or any error — so the executor never silently loses - its transport. + Gated by the single ``pg_queue_enabled`` Flipt flag, bucketed per org. + **Fails closed to Celery** on a blind Flipt or any error — so the executor + never silently loses its transport. """ - if not master_gate_enabled: - return False if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true": logger.warning( - "resolve_pg_transport: gate ON but FLIPT_SERVICE_AVAILABLE != true " + "resolve_pg_transport: FLIPT_SERVICE_AVAILABLE != true " "(Flipt blind); using Celery" ) return False diff --git a/workers/queue_backend/pg_queue/connection.py b/workers/queue_backend/pg_queue/connection.py index 53c1f414f9..89e5999c5f 100644 --- a/workers/queue_backend/pg_queue/connection.py +++ b/workers/queue_backend/pg_queue/connection.py @@ -16,6 +16,8 @@ import logging import os +import time +from collections.abc import Callable from typing import TYPE_CHECKING import psycopg2 @@ -25,6 +27,68 @@ logger = logging.getLogger(__name__) +# Bounded retry for *transient* connect failures (DB restart, PgBouncer pool +# wait, brief network partition, "too many clients" spikes) — the common cloud +# blips. Opening a connection is side-effect-free, so retrying it is safe and +# purely additive. Note we deliberately do NOT auto-retry the enqueue INSERT +# (queue_backend.pg_queue.client.send): an ambiguous commit-time failure could +# double-enqueue → double-dispatch, so that path stays fail-and-surface. +_DEFAULT_CONNECT_RETRIES = 3 # total attempts (1 = no retry) +_MAX_CONNECT_RETRIES = 10 # sane upper bound — a fat-fingered value can't wedge startup +_DEFAULT_CONNECT_BACKOFF = 0.5 # base seconds between attempts; doubles each retry +_CONNECT_BACKOFF_CAP = ( + 5.0 # fixed ceiling on a single inter-attempt sleep (not env-tunable) +) + + +def _connect_env[T](suffix: str, default: T, cast: Callable[[str], T]) -> T: + """Read ``WORKER_PG_QUEUE_CONNECT_{suffix}``; fall back to ``default``. + + Empty/unset → default; an unparseable value warns and uses the default + (a bad knob must not wedge the only direct-DB worker path). + """ + raw = os.getenv(f"WORKER_PG_QUEUE_CONNECT_{suffix}") + if raw is None or raw == "": + return default + try: + return cast(raw) + except (TypeError, ValueError): + logger.warning( + "PG-queue: invalid WORKER_PG_QUEUE_CONNECT_%s=%r; using default %r", + suffix, + raw, + default, + ) + return default + + +def _connect_retry_policy() -> tuple[int, float]: + """Resolve ``(attempts, base_backoff)`` from env, clamping out-of-range values. + + Out-of-range knobs are clamped *and warned* (not silently coerced), mirroring + the unparseable-value path: attempts to ``[1, _MAX_CONNECT_RETRIES]``, backoff + to ``>= 0``. The per-sleep ``_CONNECT_BACKOFF_CAP`` is a separate fixed ceiling. + """ + raw_retries = _connect_env("RETRIES", _DEFAULT_CONNECT_RETRIES, int) + attempts = min(_MAX_CONNECT_RETRIES, max(1, raw_retries)) + if attempts != raw_retries: + logger.warning( + "PG-queue: WORKER_PG_QUEUE_CONNECT_RETRIES=%d out of range [1, %d]; " + "clamped to %d", + raw_retries, + _MAX_CONNECT_RETRIES, + attempts, + ) + raw_backoff = _connect_env("BACKOFF", _DEFAULT_CONNECT_BACKOFF, float) + backoff = max(0.0, raw_backoff) + if backoff != raw_backoff: + logger.warning( + "PG-queue: WORKER_PG_QUEUE_CONNECT_BACKOFF=%s is negative; clamped to " + "0.0 (retry without sleep)", + raw_backoff, + ) + return attempts, backoff + def create_pg_connection(env_prefix: str = "DB_") -> PgConnection: """Open a direct Postgres connection from ``{env_prefix}*`` env. @@ -37,31 +101,58 @@ def create_pg_connection(env_prefix: str = "DB_") -> PgConnection: is the compose service name); real deployments always set the env explicitly, so the fallback host intentionally differs from the backend's own ``base.py`` default. + + A transient ``OperationalError`` is retried with exponential backoff + (``WORKER_PG_QUEUE_CONNECT_RETRIES`` total attempts, base + ``WORKER_PG_QUEUE_CONNECT_BACKOFF`` seconds, each sleep capped at + ``_CONNECT_BACKOFF_CAP``). Non-operational errors (e.g. a bad ``options`` + string) are not transient and raise immediately. **Permanent** misconfigs + (auth failure, unknown database, bad host) also surface as ``OperationalError``, + so they are retried-then-raised — the final ``logger.error`` makes them obvious. """ host = os.getenv(f"{env_prefix}HOST", "unstract-db") port = os.getenv(f"{env_prefix}PORT", "5432") dbname = os.getenv(f"{env_prefix}NAME", "unstract_db") user = os.getenv(f"{env_prefix}USER", "unstract_dev") schema = os.getenv(f"{env_prefix}SCHEMA", "unstract") - try: - return psycopg2.connect( - host=host, - port=port, - dbname=dbname, - user=user, - password=os.getenv(f"{env_prefix}PASSWORD", "unstract_pass"), - options=f"-c search_path={schema}", - ) - except psycopg2.OperationalError: - # First direct-DB worker component — make the failure self-identifying - # so a misconfigured DB_* var is obvious. Secrets (password) omitted. - logger.error( - "PG-queue: failed to connect to Postgres " - "(host=%s port=%s dbname=%s schema=%s, env_prefix=%r)", - host, - port, - dbname, - schema, - env_prefix, - ) - raise + + attempts, backoff = _connect_retry_policy() + + for attempt in range(1, attempts + 1): + try: + return psycopg2.connect( + host=host, + port=port, + dbname=dbname, + user=user, + password=os.getenv(f"{env_prefix}PASSWORD", "unstract_pass"), + options=f"-c search_path={schema}", + ) + except psycopg2.OperationalError: + if attempt >= attempts: + # Every attempt exhausted — keep the failure self-identifying so a + # misconfigured DB_* var is obvious. Secrets (password) omitted. + logger.error( + "PG-queue: failed to connect to Postgres after %d attempt(s) " + "(host=%s port=%s dbname=%s schema=%s, env_prefix=%r)", + attempts, + host, + port, + dbname, + schema, + env_prefix, + ) + raise + sleep_for = min(backoff * (2 ** (attempt - 1)), _CONNECT_BACKOFF_CAP) + logger.warning( + "PG-queue: connect attempt %d/%d failed " + "(host=%s port=%s dbname=%s schema=%s); retrying in %.2fs", + attempt, + attempts, + host, + port, + dbname, + schema, + sleep_for, + ) + time.sleep(sleep_for) diff --git a/workers/queue_backend/pg_queue/executor_rpc.py b/workers/queue_backend/pg_queue/executor_rpc.py index 1b3367bb03..892866a18b 100644 --- a/workers/queue_backend/pg_queue/executor_rpc.py +++ b/workers/queue_backend/pg_queue/executor_rpc.py @@ -4,18 +4,16 @@ ``unstract.workflow_execution.executor_rpc`` (shared with the backend). This module is the thin psycopg2 half: a :class:`PgClientQueueTransport` that enqueues via :class:`~queue_backend.pg_queue.client.PgQueueClient` and polls via -:class:`~queue_backend.pg_queue.result_backend.PgResultBackend`, plus the per-side -gate (master switch = the ``PG_QUEUE_TRANSPORT_ENABLED`` env, the workers analogue of -the backend's Django setting) and the :func:`get_executor_dispatcher` factory. +:class:`~queue_backend.pg_queue.result_backend.PgResultBackend`, plus the +:func:`get_executor_dispatcher` factory. -Zero-regression: gate off ⇒ the routing dispatcher delegates every mode to the -unchanged Celery ``ExecutionDispatcher`` and no ``pg_task_result`` row is created. +Zero-regression: with the ``pg_queue_enabled`` Flipt flag off the routing dispatcher +delegates every mode to the unchanged Celery ``ExecutionDispatcher`` and no +``pg_task_result`` row is created. """ from __future__ import annotations -import logging -import os from typing import TYPE_CHECKING from unstract.sdk1.execution.dispatcher import ExecutionDispatcher @@ -46,33 +44,13 @@ "resolve_executor_transport", ] -logger = logging.getLogger(__name__) - -# Master kill-switch + deploy-ordering gate — the workers analogue of the backend's -# ``settings.PG_QUEUE_TRANSPORT_ENABLED``, read straight from the env here. -_MASTER_GATE_ENV = "PG_QUEUE_TRANSPORT_ENABLED" -_TRUE = "true" -_FALSE = "false" - def resolve_executor_transport(context: ExecutionContext) -> bool: """True → route this executor dispatch over PG; False → Celery (default). - The workers gate: master switch = the ``PG_QUEUE_TRANSPORT_ENABLED`` env, then the - shared Flipt eval (single ``pg_queue_enabled`` flag, fail-closed). + The single ``pg_queue_enabled`` Flipt flag (fail-closed). """ - raw = os.environ.get(_MASTER_GATE_ENV, _FALSE) - master = raw.strip().lower() == _TRUE - if not master and raw.strip().lower() != _FALSE: - # A fat-fingered value ("1"/"yes"/"on"/" True ") parses to OFF — warn so it - # isn't a silent no-op for an operator who expected it to enable PG. - logger.warning( - "resolve_executor_transport: %s=%r is not 'true'/'false' — treating as " - "OFF (PG transport disabled); use exactly 'true' to enable", - _MASTER_GATE_ENV, - raw, - ) - return resolve_pg_transport(context, master_gate_enabled=master) + return resolve_pg_transport(context) class PgClientQueueTransport(QueueTransport): diff --git a/workers/sample.env b/workers/sample.env index d5564536ce..2df037e51a 100644 --- a/workers/sample.env +++ b/workers/sample.env @@ -474,14 +474,6 @@ EVALUATION_SERVER_IP=unstract-flipt EVALUATION_SERVER_PORT=9005 PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python -# PG-queue transport master-gate (kill-switch), worker side. Mirrors the backend's -# PG_QUEUE_TRANSPORT_ENABLED: the file_processing worker reads it to route the -# structure_tool executor RPC onto the PG queue. Routing onto PG requires ALL -# THREE: this gate True, FLIPT_SERVICE_AVAILABLE=True, and the Flipt flag -# `pg_queue_enabled` on. Keep False until the worker-pg-executor consumer is -# running in the fleet; set False for instant rollback. -PG_QUEUE_TRANSPORT_ENABLED=False - # PG-queue consumer prefork concurrency (UN-3606). >1 makes the consumer launcher # fork N isolated consumer children — the PG analogue of Celery's prefork # --concurrency, so file batches run in parallel. SKIP LOCKED distributes work @@ -490,3 +482,10 @@ PG_QUEUE_TRANSPORT_ENABLED=False # (k8s HPA scales replicas). 1 = single process (default). Set per consumer; for # the file-processing consumer it mirrors WORKER_FILE_PROCESSING_CONCURRENCY. # WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=1 + +# PG-queue direct-connect retry (UN-3619). create_pg_connection retries a transient +# OperationalError (DB restart, PgBouncer pool wait, brief network blip) with +# exponential backoff. RETRIES = total attempts, clamped to [1, 10] (1 = no retry); +# BACKOFF = base seconds, doubling each attempt, with each sleep capped at 5.0s. +# WORKER_PG_QUEUE_CONNECT_RETRIES=3 +# WORKER_PG_QUEUE_CONNECT_BACKOFF=0.5 diff --git a/workers/tests/test_executor_rpc.py b/workers/tests/test_executor_rpc.py index 75ed262a58..e2bec43a62 100644 --- a/workers/tests/test_executor_rpc.py +++ b/workers/tests/test_executor_rpc.py @@ -180,42 +180,37 @@ def test_dispatch_with_callback_defaults_task_id(self): assert t.enqueue_calls[0]["task_id"] == handle.id -# --- Shared gate: resolve_pg_transport (master-gated, then Flipt, fail-closed) --- +# --- Shared gate: resolve_pg_transport (single Flipt flag, fail-closed) --- class TestResolvePgTransport: - def test_master_gate_off_is_celery(self): - with patch(f"{_SMOD}.check_feature_flag_status") as flag: - assert resolve_pg_transport(_ctx(), master_gate_enabled=False) is False - flag.assert_not_called() - def test_flipt_unavailable_is_celery(self, monkeypatch): monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false") with patch(f"{_SMOD}.check_feature_flag_status") as flag: - assert resolve_pg_transport(_ctx(), master_gate_enabled=True) is False + assert resolve_pg_transport(_ctx()) is False flag.assert_not_called() def test_flag_true_is_pg_keyed_on_org(self, monkeypatch): monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") with patch(f"{_SMOD}.check_feature_flag_status", return_value=True) as flag: - assert resolve_pg_transport(_ctx("orgX"), master_gate_enabled=True) is True + assert resolve_pg_transport(_ctx("orgX")) is True assert flag.call_args.kwargs["entity_id"] == "orgX" assert flag.call_args.kwargs["flag_key"] == "pg_queue_enabled" def test_flag_false_is_celery(self, monkeypatch): monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") with patch(f"{_SMOD}.check_feature_flag_status", return_value=False): - assert resolve_pg_transport(_ctx(), master_gate_enabled=True) is False + assert resolve_pg_transport(_ctx()) is False def test_flipt_error_fails_closed(self, monkeypatch): monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") with patch(f"{_SMOD}.check_feature_flag_status", side_effect=RuntimeError("x")): - assert resolve_pg_transport(_ctx(), master_gate_enabled=True) is False + assert resolve_pg_transport(_ctx()) is False def test_org_less_context_buckets_on_run_id(self, monkeypatch): monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true") with patch(f"{_SMOD}.check_feature_flag_status", return_value=True) as flag: - assert resolve_pg_transport(_ctx(org=None), master_gate_enabled=True) is True + assert resolve_pg_transport(_ctx(org=None)) is True assert flag.call_args.kwargs["entity_id"] == "run-1" assert "organization_id" not in flag.call_args.kwargs["context"] @@ -328,23 +323,22 @@ def test_wait_for_result_none_passes_through(self): with patch(f"{_WMOD}.PgResultBackend", return_value=rb): assert PgClientQueueTransport().wait_for_result("rk", 5) is None - def test_resolve_reads_env_master_gate(self, monkeypatch): - monkeypatch.setenv("PG_QUEUE_TRANSPORT_ENABLED", "true") + def test_resolve_delegates_to_shared_flipt_resolver_true(self): with patch(f"{_WMOD}.resolve_pg_transport", return_value=True) as r: assert resolve_executor_transport(self._ctx()) is True - assert r.call_args.kwargs["master_gate_enabled"] is True + r.assert_called_once() + # No env master-gate is threaded any more — Flipt is the sole gate. + assert "master_gate_enabled" not in r.call_args.kwargs - def test_resolve_env_off_is_false(self, monkeypatch): - monkeypatch.delenv("PG_QUEUE_TRANSPORT_ENABLED", raising=False) - with patch(f"{_WMOD}.resolve_pg_transport", return_value=False) as r: - resolve_executor_transport(self._ctx()) - assert r.call_args.kwargs["master_gate_enabled"] is False + def test_resolve_delegates_to_shared_flipt_resolver_false(self): + with patch(f"{_WMOD}.resolve_pg_transport", return_value=False): + assert resolve_executor_transport(self._ctx()) is False def test_factory_wires_routing_with_workers_transport(self): d = get_executor_dispatcher(celery_app="app") assert isinstance(d, RoutingExecutionDispatcher) # The PG dispatcher is wired with the workers psycopg2 transport, and the gate - # is the workers env-master-gate resolver. + # is the workers' Flipt resolver. assert isinstance(d._pg._transport, PgClientQueueTransport) assert d._resolve is resolve_executor_transport diff --git a/workers/tests/test_pg_connection.py b/workers/tests/test_pg_connection.py new file mode 100644 index 0000000000..b9b619141c --- /dev/null +++ b/workers/tests/test_pg_connection.py @@ -0,0 +1,210 @@ +"""Tests for ``create_pg_connection``'s bounded connect-retry. + +Connecting is side-effect-free, so a *transient* ``OperationalError`` is +retried with exponential backoff (the common cloud blip: DB restart, +PgBouncer pool wait, brief network partition). Non-transient errors raise +immediately, and the enqueue INSERT is intentionally NOT retried elsewhere +(double-dispatch risk) — these tests pin the connect-layer contract only. + +``psycopg2.connect`` and ``time.sleep`` are patched on the module, so no DB +and no real waiting. +""" + +from __future__ import annotations + +import psycopg2 +import pytest +from queue_backend.pg_queue import connection as conn_mod + + +def _patch(monkeypatch, *, connect, retries="3", backoff="0.5"): + """Patch connect + sleep; return the list that records sleep durations.""" + monkeypatch.setattr(conn_mod.psycopg2, "connect", connect) + sleeps: list[float] = [] + monkeypatch.setattr(conn_mod.time, "sleep", lambda s: sleeps.append(s)) + monkeypatch.setenv("WORKER_PG_QUEUE_CONNECT_RETRIES", retries) + monkeypatch.setenv("WORKER_PG_QUEUE_CONNECT_BACKOFF", backoff) + return sleeps + + +class TestCreatePgConnectionRetry: + def test_succeeds_first_try_without_sleeping(self, monkeypatch): + sentinel = object() + calls: list[int] = [] + + def connect(**_kwargs): + calls.append(1) + return sentinel + + sleeps = _patch(monkeypatch, connect=connect) + assert conn_mod.create_pg_connection() is sentinel + assert len(calls) == 1 + assert sleeps == [] + + def test_retries_transient_then_succeeds(self, monkeypatch): + sentinel = object() + seq: list = [ + psycopg2.OperationalError("nope"), + psycopg2.OperationalError("nope"), + sentinel, + ] + + def connect(**_kwargs): + item = seq.pop(0) + if isinstance(item, Exception): + raise item + return item + + sleeps = _patch(monkeypatch, connect=connect) + assert conn_mod.create_pg_connection() is sentinel + # Slept before attempts 2 and 3 with exponential backoff (0.5, 1.0). + assert sleeps == [0.5, 1.0] + + def test_exhausts_attempts_and_raises(self, monkeypatch): + calls: list[int] = [] + + def connect(**_kwargs): + calls.append(1) + raise psycopg2.OperationalError("down") + + sleeps = _patch(monkeypatch, connect=connect, retries="3", backoff="0.1") + with pytest.raises(psycopg2.OperationalError): + conn_mod.create_pg_connection() + assert len(calls) == 3 # all attempts used + assert len(sleeps) == 2 # no sleep after the final attempt + + def test_non_operational_error_is_not_retried(self, monkeypatch): + calls: list[int] = [] + + def connect(**_kwargs): + calls.append(1) + raise psycopg2.ProgrammingError("bad options string") + + sleeps = _patch(monkeypatch, connect=connect) + with pytest.raises(psycopg2.ProgrammingError): + conn_mod.create_pg_connection() + assert len(calls) == 1 # raised on the first attempt + assert sleeps == [] + + def test_single_attempt_when_retries_is_one(self, monkeypatch): + calls: list[int] = [] + + def connect(**_kwargs): + calls.append(1) + raise psycopg2.OperationalError("down") + + sleeps = _patch(monkeypatch, connect=connect, retries="1") + with pytest.raises(psycopg2.OperationalError): + conn_mod.create_pg_connection() + assert len(calls) == 1 + assert sleeps == [] + + def test_invalid_retries_env_falls_back_to_default(self, monkeypatch): + calls: list[int] = [] + + def connect(**_kwargs): + calls.append(1) + raise psycopg2.OperationalError("down") + + # Garbage RETRIES → default of 3 attempts (a bad knob must not wedge it). + sleeps = _patch(monkeypatch, connect=connect, retries="abc", backoff="0.1") + with pytest.raises(psycopg2.OperationalError): + conn_mod.create_pg_connection() + assert len(calls) == 3 + assert len(sleeps) == 2 + + def test_unset_env_uses_production_defaults(self, monkeypatch): + # The _patch helper always sets both knobs; this drives the real default + # path (env absent → 3 attempts / 0.5s base) that production actually uses. + monkeypatch.delenv("WORKER_PG_QUEUE_CONNECT_RETRIES", raising=False) + monkeypatch.delenv("WORKER_PG_QUEUE_CONNECT_BACKOFF", raising=False) + + def connect(**_kwargs): + raise psycopg2.OperationalError("down") + + monkeypatch.setattr(conn_mod.psycopg2, "connect", connect) + sleeps: list[float] = [] + monkeypatch.setattr(conn_mod.time, "sleep", lambda s: sleeps.append(s)) + with pytest.raises(psycopg2.OperationalError): + conn_mod.create_pg_connection() + assert sleeps == [0.5, 1.0] # default base 0.5, 3 attempts → 2 sleeps + + def test_backoff_grows_geometrically(self, monkeypatch): + def connect(**_kwargs): + raise psycopg2.OperationalError("down") + + # 4 attempts, base 1 → [1, 2, 4] — pins geometric (not linear) growth. + sleeps = _patch(monkeypatch, connect=connect, retries="4", backoff="1") + with pytest.raises(psycopg2.OperationalError): + conn_mod.create_pg_connection() + assert sleeps == [1.0, 2.0, 4.0] + + def test_sleep_is_capped_at_backoff_cap(self, monkeypatch): + def connect(**_kwargs): + raise psycopg2.OperationalError("down") + + # base 4 doubles past the 5.0s ceiling: 4, 8→5, 16→5, 32→5. + sleeps = _patch(monkeypatch, connect=connect, retries="5", backoff="4") + with pytest.raises(psycopg2.OperationalError): + conn_mod.create_pg_connection() + assert sleeps == [4.0, 5.0, 5.0, 5.0] + + def test_negative_backoff_clamped_to_zero(self, monkeypatch): + def connect(**_kwargs): + raise psycopg2.OperationalError("down") + + # Negative base → clamped to 0.0 (retry without sleep), not silently kept. + sleeps = _patch(monkeypatch, connect=connect, retries="3", backoff="-1") + with pytest.raises(psycopg2.OperationalError): + conn_mod.create_pg_connection() + assert sleeps == [0.0, 0.0] + + def test_invalid_backoff_env_falls_back_to_default(self, monkeypatch): + def connect(**_kwargs): + raise psycopg2.OperationalError("down") + + sleeps = _patch(monkeypatch, connect=connect, retries="3", backoff="xyz") + with pytest.raises(psycopg2.OperationalError): + conn_mod.create_pg_connection() + assert sleeps == [0.5, 1.0] # default base 0.5 + + def test_retries_above_max_is_clamped(self, monkeypatch): + calls: list[int] = [] + + def connect(**_kwargs): + calls.append(1) + raise psycopg2.OperationalError("down") + + # 100 attempts requested → clamped to the 10-attempt ceiling (9 sleeps). + sleeps = _patch(monkeypatch, connect=connect, retries="100", backoff="0.01") + with pytest.raises(psycopg2.OperationalError): + conn_mod.create_pg_connection() + assert len(calls) == 10 + assert len(sleeps) == 9 + + def test_connection_params_forwarded_and_stable_across_retries(self, monkeypatch): + monkeypatch.setenv("DB_HOST", "pg.test") + monkeypatch.setenv("DB_NAME", "mydb") + monkeypatch.setenv("DB_SCHEMA", "myschema") + sentinel = object() + seq: list = [ + psycopg2.OperationalError("x"), + psycopg2.OperationalError("x"), + sentinel, + ] + calls: list[dict] = [] + + def connect(**kwargs): + calls.append(kwargs) + item = seq.pop(0) + if isinstance(item, Exception): + raise item + return item + + _patch(monkeypatch, connect=connect) + assert conn_mod.create_pg_connection() is sentinel + assert len(calls) == 3 + # The DB_* params (incl. the search_path options) are forwarded and + # identical on every retry. + assert all(c["options"] == "-c search_path=myschema" for c in calls) + assert all(c["host"] == "pg.test" and c["dbname"] == "mydb" for c in calls) From d7b27ceb74c6d27a9289a947267399cac13cd474 Mon Sep 17 00:00:00 2001 From: ali Date: Fri, 26 Jun 2026 17:02:13 +0530 Subject: [PATCH 41/44] =?UTF-8?q?UN-3445=20[FIX]=20workflow=5Fv2=20?= =?UTF-8?q?=E2=80=94=20merge=20migration=20after=20main=20backmerge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backmerge brought workflow_v2.0021_alter_workflow_organization (main) alongside our 0020_workflowexecution_queue_message_id; both fork from 0019, leaving two leaf nodes → 'Conflicting migrations detected' on migrate. Add the standard no-op merge migration 0022 depending on both leaves. detect_conflicts now clean. Co-Authored-By: Claude Opus 4.8 --- .../migrations/0022_merge_20260626_1131.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 backend/workflow_manager/workflow_v2/migrations/0022_merge_20260626_1131.py diff --git a/backend/workflow_manager/workflow_v2/migrations/0022_merge_20260626_1131.py b/backend/workflow_manager/workflow_v2/migrations/0022_merge_20260626_1131.py new file mode 100644 index 0000000000..9e463e8356 --- /dev/null +++ b/backend/workflow_manager/workflow_v2/migrations/0022_merge_20260626_1131.py @@ -0,0 +1,12 @@ +# Generated by Django 4.2.1 on 2026-06-26 11:31 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("workflow_v2", "0020_workflowexecution_queue_message_id"), + ("workflow_v2", "0021_alter_workflow_organization"), + ] + + operations = [] From 163355221648b17d511d1daec59f98148dd799ef Mon Sep 17 00:00:00 2001 From: ali Date: Sat, 27 Jun 2026 10:17:56 +0530 Subject: [PATCH 42/44] =?UTF-8?q?UN-3624=20[FIX]=20PG=20queue=20=E2=80=94?= =?UTF-8?q?=20schema-qualify=20worker=20SQL=20for=20PgBouncer=20txn=20pool?= =?UTF-8?q?ing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PG-queue workers connect through PgBouncer's transaction-pool alias (unstract_txn), where the `options=-c search_path=` startup parameter is stripped (IGNORE_STARTUP_PARAMETERS=options). Bare table names then resolve to the default (public) path -> `relation "pg_queue_message" does not exist`. This works in OSS/local (direct Postgres, search_path honoured) but breaks in cloud. Fix: name every queue table explicitly as `"".` via a new `qualified()` helper (schema read lazily from DB_SCHEMA, validated), so the SQL is self-contained and resolves through transaction pooling without search_path — the same way the Celery result backend reaches its public tables. Keeps the unstract_txn transaction pool (no session-pooling / scale regression). - new schema.py: qualified() + queue_schema() + QUEUE_TABLES registry (rejects unknown / typo'd table names — single source of truth) - qualify the SQL in client, result_backend, reaper, leader_election, pg_scheduler, pg_barrier (24 sites) - guard test fails the build on any bare unqualified FROM/INTO/UPDATE pg_
, so a future query cannot silently regress the cloud path Co-Authored-By: Claude Opus 4.8 --- workers/queue_backend/pg_barrier.py | 20 ++-- workers/queue_backend/pg_queue/client.py | 45 +++++--- .../queue_backend/pg_queue/leader_election.py | 8 +- .../queue_backend/pg_queue/pg_scheduler.py | 22 ++-- workers/queue_backend/pg_queue/reaper.py | 16 ++- .../queue_backend/pg_queue/result_backend.py | 32 ++++-- workers/queue_backend/pg_queue/schema.py | 86 +++++++++++++++ workers/tests/test_pg_queue_client.py | 15 +-- workers/tests/test_pg_reaper.py | 28 +++-- workers/tests/test_pg_schema.py | 103 ++++++++++++++++++ 10 files changed, 310 insertions(+), 65 deletions(-) create mode 100644 workers/queue_backend/pg_queue/schema.py create mode 100644 workers/tests/test_pg_schema.py diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.py index 19e80f29a1..f8951f779d 100644 --- a/workers/queue_backend/pg_barrier.py +++ b/workers/queue_backend/pg_barrier.py @@ -83,6 +83,7 @@ ) from .handle import BarrierHandle from .pg_queue.connection import create_pg_connection +from .pg_queue.schema import qualified if TYPE_CHECKING: from celery.canvas import Signature @@ -130,7 +131,8 @@ def _cursor() -> Iterator[Any]: def _delete_barrier(execution_id: str) -> None: with _cursor() as cur: cur.execute( - "DELETE FROM pg_barrier_state WHERE execution_id = %s", (execution_id,) + f"DELETE FROM {qualified('pg_barrier_state')} WHERE execution_id = %s", + (execution_id,), ) @@ -154,7 +156,8 @@ def claim_batch(execution_id: str, batch_index: int) -> bool: """ with _cursor() as cur: cur.execute( - "INSERT INTO pg_batch_dedup (execution_id, batch_index, created_at) " + f"INSERT INTO {qualified('pg_batch_dedup')} " + "(execution_id, batch_index, created_at) " "VALUES (%s, %s, now()) " "ON CONFLICT (execution_id, batch_index) DO NOTHING " "RETURNING execution_id", @@ -178,7 +181,10 @@ def clear_execution_batches(execution_id: str) -> int: Called from the barrier-finalise + abort paths. """ with _cursor() as cur: - cur.execute("DELETE FROM pg_batch_dedup WHERE execution_id = %s", (execution_id,)) + cur.execute( + f"DELETE FROM {qualified('pg_batch_dedup')} WHERE execution_id = %s", + (execution_id,), + ) return cur.rowcount @@ -312,7 +318,7 @@ def enqueue( # concurrent enqueues; orphan reclaim is a separate (future) # periodic sweep keyed on pg_barrier_expires_idx. cur.execute( - "INSERT INTO pg_barrier_state " + f"INSERT INTO {qualified('pg_barrier_state')} " "(execution_id, organization_id, remaining, results, " " created_at, expires_at) " "VALUES (%s, %s, %s, '[]'::jsonb, now(), " @@ -329,7 +335,7 @@ def enqueue( # returns False → all batches skip → barrier hangs to expiry. # (Markers are written by claim_batch() on the in-body PG path.) cur.execute( - "DELETE FROM pg_batch_dedup WHERE execution_id = %s", + f"DELETE FROM {qualified('pg_batch_dedup')} WHERE execution_id = %s", (execution_id,), ) @@ -578,7 +584,7 @@ def _barrier_pg_decrement( try: with _cursor() as cur: cur.execute( - "UPDATE pg_barrier_state " + f"UPDATE {qualified('pg_barrier_state')} " " SET remaining = remaining - 1, " " results = results || jsonb_build_array(%s::jsonb) " " WHERE execution_id = %s " @@ -718,7 +724,7 @@ def barrier_pg_abort( with _cursor() as cur: cur.execute( "WITH claimed AS (" - " DELETE FROM pg_barrier_state WHERE execution_id = %s " + f" DELETE FROM {qualified('pg_barrier_state')} WHERE execution_id = %s " " RETURNING execution_id" ") SELECT execution_id FROM claimed", (execution_id,), diff --git a/workers/queue_backend/pg_queue/client.py b/workers/queue_backend/pg_queue/client.py index 769ae0b666..05832a1487 100644 --- a/workers/queue_backend/pg_queue/client.py +++ b/workers/queue_backend/pg_queue/client.py @@ -37,12 +37,14 @@ from ..fairness import DEFAULT_PRIORITY, MAX_PRIORITY, MIN_PRIORITY from .connection import create_pg_connection +from .schema import qualified if TYPE_CHECKING: from psycopg2.extensions import connection as PgConnection logger = logging.getLogger(__name__) + # Atomic claim. Takes up to %(qty)s ready messages no other transaction # holds, makes them invisible for %(vt)s seconds, returns them. SKIP LOCKED # => concurrent readers never claim the same visible row (no concurrent @@ -78,17 +80,27 @@ # ``vt <= now()`` applied as a per-row filter — not a guaranteed top-N: vt is # not in the index, so claimed-but-unacked rows (future vt) at the front of a # priority band are scanned past on each claim. Cheap at low in-flight depth. -_DEQUEUE_SQL = """ +def _dequeue_sql() -> str: + """Build the atomic-claim SQL, schema-qualifying ``pg_queue_message``. + + Built per call (not a module constant) so the schema is resolved from the + live ``DB_SCHEMA`` — the table is named ``"".pg_queue_message`` so + it resolves through PgBouncer transaction pooling without ``search_path`` + (see :mod:`queue_backend.pg_queue.schema`). Cost is a trivial f-string vs. + the DB round trip that follows. + """ + msg = qualified("pg_queue_message") + return f""" WITH locked AS ( SELECT msg_id - FROM pg_queue_message + FROM {msg} WHERE queue_name = %s AND vt <= now() ORDER BY priority DESC, msg_id FOR UPDATE SKIP LOCKED LIMIT %s ), claimed AS ( - UPDATE pg_queue_message q + UPDATE {msg} q SET vt = now() + make_interval(secs => %s), read_ct = read_ct + 1 FROM locked @@ -123,12 +135,16 @@ class QueueMessage: # appends ``RETURNING msg_id``; the PG scheduler (pg_scheduler.py) executes this # verbatim inside its own transaction so the enqueue + next_run advance commit # atomically (it can't call send(), which commits internally). Keep callers in -# sync by sharing this constant rather than copying the SQL. -INSERT_MESSAGE_SQL: str = ( - "INSERT INTO pg_queue_message " - "(queue_name, message, org_id, priority, enqueued_at, vt, read_ct) " - "VALUES (%s, %s::jsonb, %s, %s, now(), now(), 0)" -) +# sync by calling this helper rather than copying the SQL. Built per call so +# ``pg_queue_message`` is schema-qualified from the live ``DB_SCHEMA`` (resolves +# through PgBouncer txn pooling without ``search_path`` — see +# :mod:`queue_backend.pg_queue.schema`). +def insert_message_sql() -> str: + return ( + f"INSERT INTO {qualified('pg_queue_message')} " + "(queue_name, message, org_id, priority, enqueued_at, vt, read_ct) " + "VALUES (%s, %s::jsonb, %s, %s, now(), now(), 0)" + ) class PgQueueClient: @@ -211,7 +227,7 @@ def send( ) with self._cursor() as cur: cur.execute( - INSERT_MESSAGE_SQL + " RETURNING msg_id", + insert_message_sql() + " RETURNING msg_id", # "" rather than NULL for "no org" — the column is non-null # (string fields shouldn't have two empty values; Django S6553). ( @@ -242,9 +258,9 @@ def read( if qty <= 0: raise ValueError(f"qty must be positive, got {qty}") with self._cursor() as cur: - # Param order matches the %s positions in _DEQUEUE_SQL: + # Param order matches the %s positions in _dequeue_sql(): # queue_name (locked CTE), qty (LIMIT), vt_seconds (UPDATE SET). - cur.execute(_DEQUEUE_SQL, (queue_name, qty, vt_seconds)) + cur.execute(_dequeue_sql(), (queue_name, qty, vt_seconds)) rows = cur.fetchall() return [ QueueMessage(msg_id=int(r[0]), message=r[1], read_ct=int(r[2])) for r in rows @@ -260,7 +276,10 @@ def delete(self, msg_id: int) -> bool: avoids a duplicate warning per double-run. """ with self._cursor() as cur: - cur.execute("DELETE FROM pg_queue_message WHERE msg_id = %s", (msg_id,)) + cur.execute( + f"DELETE FROM {qualified('pg_queue_message')} WHERE msg_id = %s", + (msg_id,), + ) deleted = cur.rowcount if deleted == 0: logger.debug( diff --git a/workers/queue_backend/pg_queue/leader_election.py b/workers/queue_backend/pg_queue/leader_election.py index 0a1d8415d8..403e84e388 100644 --- a/workers/queue_backend/pg_queue/leader_election.py +++ b/workers/queue_backend/pg_queue/leader_election.py @@ -55,6 +55,7 @@ import psycopg2 from .connection import create_pg_connection +from .schema import qualified if TYPE_CHECKING: from psycopg2.extensions import connection as PgConnection @@ -203,7 +204,7 @@ def try_acquire(self) -> bool: """ with self._cursor() as cur: cur.execute( - "UPDATE pg_orchestrator_lock " + f"UPDATE {qualified('pg_orchestrator_lock')} " " SET leader = %s, acquired_at = now() " " WHERE id = 1 " " AND (leader = '' " @@ -227,7 +228,7 @@ def renew(self) -> bool: """ with self._cursor() as cur: cur.execute( - "UPDATE pg_orchestrator_lock SET acquired_at = now() " + f"UPDATE {qualified('pg_orchestrator_lock')} SET acquired_at = now() " "WHERE id = 1 AND leader = %s RETURNING id", (self._worker_id,), ) @@ -250,7 +251,8 @@ def release(self) -> None: """ with self._cursor() as cur: cur.execute( - "UPDATE pg_orchestrator_lock SET leader = '', acquired_at = now() " + f"UPDATE {qualified('pg_orchestrator_lock')} " + "SET leader = '', acquired_at = now() " "WHERE id = 1 AND leader = %s", (self._worker_id,), ) diff --git a/workers/queue_backend/pg_queue/pg_scheduler.py b/workers/queue_backend/pg_queue/pg_scheduler.py index 42755e7b05..6ae64a5a93 100644 --- a/workers/queue_backend/pg_queue/pg_scheduler.py +++ b/workers/queue_backend/pg_queue/pg_scheduler.py @@ -41,7 +41,8 @@ from unstract.core.data_models import TaskPayload from ..fairness import DEFAULT_PRIORITY -from .client import INSERT_MESSAGE_SQL +from .client import insert_message_sql +from .schema import qualified from .task_payload import to_payload if TYPE_CHECKING: @@ -114,7 +115,8 @@ def _quiesce_invalid_cron(conn: PgConnection, schedule: _DueSchedule) -> None: try: with conn.cursor() as cur: cur.execute( - "UPDATE pg_periodic_schedule SET enabled = FALSE WHERE pipeline_id = %s", + f"UPDATE {qualified('pg_periodic_schedule')} " + "SET enabled = FALSE WHERE pipeline_id = %s", (schedule.pipeline_id,), ) conn.commit() @@ -140,10 +142,10 @@ def dispatch_due_schedules(conn: PgConnection) -> int: cur.execute("SELECT now()") base = cur.fetchone()[0] cur.execute( - """ + f""" SELECT pipeline_id, organization_id, workflow_id, pipeline_name, cron_string, next_run_at - FROM pg_periodic_schedule + FROM {qualified('pg_periodic_schedule')} WHERE pg_owned AND enabled AND (next_run_at IS NULL OR next_run_at <= %s) """, @@ -170,8 +172,8 @@ def dispatch_due_schedules(conn: PgConnection) -> int: # next time and do NOT fire (no burst when handed over). with conn.cursor() as cur: cur.execute( - "UPDATE pg_periodic_schedule SET next_run_at = %s " - "WHERE pipeline_id = %s", + f"UPDATE {qualified('pg_periodic_schedule')} " + "SET next_run_at = %s WHERE pipeline_id = %s", (nxt, schedule.pipeline_id), ) conn.commit() @@ -189,11 +191,11 @@ def dispatch_due_schedules(conn: PgConnection) -> int: pipeline_name=schedule.pipeline_name, ) # Enqueue + advance in ONE transaction so a crash between them can't - # re-fire next cycle. INSERT_MESSAGE_SQL is the shared enqueue - # contract from client.py (send() uses the same constant). + # re-fire next cycle. insert_message_sql() is the shared enqueue + # contract from client.py (send() uses the same helper). with conn.cursor() as cur: cur.execute( - INSERT_MESSAGE_SQL, + insert_message_sql(), ( SCHEDULER_QUEUE_NAME, json.dumps(payload), @@ -202,7 +204,7 @@ def dispatch_due_schedules(conn: PgConnection) -> int: ), ) cur.execute( - "UPDATE pg_periodic_schedule " + f"UPDATE {qualified('pg_periodic_schedule')} " "SET last_run_at = %s, next_run_at = %s WHERE pipeline_id = %s", (base, nxt, schedule.pipeline_id), ) diff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.py index 4a0c79f5f7..5d6f54b794 100644 --- a/workers/queue_backend/pg_queue/reaper.py +++ b/workers/queue_backend/pg_queue/reaper.py @@ -60,6 +60,7 @@ from .leader_election import LeaderLease, default_worker_id from .liveness import LivenessServer as _BaseLivenessServer from .pg_scheduler import dispatch_due_schedules +from .schema import qualified if TYPE_CHECKING: from psycopg2.extensions import connection as PgConnection @@ -212,7 +213,9 @@ def sweep_expired_results(conn: PgConnection) -> int: """ try: with conn.cursor() as cur: - cur.execute("DELETE FROM pg_task_result WHERE expires_at <= now()") + cur.execute( + f"DELETE FROM {qualified('pg_task_result')} WHERE expires_at <= now()" + ) deleted = cur.rowcount conn.commit() return deleted @@ -236,7 +239,7 @@ def sweep_orphan_dedup(conn: PgConnection, retention_seconds: int) -> int: try: with conn.cursor() as cur: cur.execute( - "DELETE FROM pg_batch_dedup " + f"DELETE FROM {qualified('pg_batch_dedup')} " "WHERE created_at <= now() - make_interval(secs => %s)", (retention_seconds,), ) @@ -280,7 +283,7 @@ def _still_expired(conn: PgConnection, execution_id: str) -> bool: """ with conn.cursor() as cur: cur.execute( - "SELECT 1 FROM pg_barrier_state " + f"SELECT 1 FROM {qualified('pg_barrier_state')} " "WHERE execution_id = %s AND expires_at < now()", (execution_id,), ) @@ -401,14 +404,15 @@ def _recover_one_barrier( # Queue-infra cleanup (direct PG), re-guarded against a concurrent re-arm. with conn.cursor() as cur: cur.execute( - "DELETE FROM pg_barrier_state WHERE execution_id = %s " + f"DELETE FROM {qualified('pg_barrier_state')} WHERE execution_id = %s " "AND expires_at < now()", (execution_id,), ) deleted = cur.rowcount > 0 if deleted: cur.execute( - "DELETE FROM pg_batch_dedup WHERE execution_id = %s", (execution_id,) + f"DELETE FROM {qualified('pg_batch_dedup')} WHERE execution_id = %s", + (execution_id,), ) else: logger.warning( @@ -441,7 +445,7 @@ def recover_expired_barriers( with conn.cursor() as cur: cur.execute( "SELECT execution_id, organization_id, remaining " - "FROM pg_barrier_state WHERE expires_at < now()" + f"FROM {qualified('pg_barrier_state')} WHERE expires_at < now()" ) rows = cur.fetchall() conn.commit() diff --git a/workers/queue_backend/pg_queue/result_backend.py b/workers/queue_backend/pg_queue/result_backend.py index e6544afc49..0968c1521e 100644 --- a/workers/queue_backend/pg_queue/result_backend.py +++ b/workers/queue_backend/pg_queue/result_backend.py @@ -41,6 +41,7 @@ from unstract.core.polling import poll_for_row from .connection import create_pg_connection +from .schema import qualified if TYPE_CHECKING: from psycopg2.extensions import connection as PgConnection @@ -52,15 +53,26 @@ # outlives any caller still waiting on it. DEFAULT_RETENTION_SECONDS = 3600 + # First write wins — an at-least-once redelivery of the executor message must -# not overwrite a recorded result. -_STORE_SQL = ( - "INSERT INTO pg_task_result " - "(task_id, status, result, error, created_at, expires_at) " - "VALUES (%s, %s, %s::jsonb, %s, now(), now() + make_interval(secs => %s)) " - "ON CONFLICT (task_id) DO NOTHING" -) -_GET_SQL = "SELECT status, result, error FROM pg_task_result WHERE task_id = %s" +# not overwrite a recorded result. Built per call so ``pg_task_result`` is +# schema-qualified from the live ``DB_SCHEMA`` (resolves through PgBouncer txn +# pooling without ``search_path`` — see :mod:`queue_backend.pg_queue.schema`). +def _store_sql() -> str: + return ( + f"INSERT INTO {qualified('pg_task_result')} " + "(task_id, status, result, error, created_at, expires_at) " + "VALUES (%s, %s, %s::jsonb, %s, now(), now() + make_interval(secs => %s)) " + "ON CONFLICT (task_id) DO NOTHING" + ) + + +def _get_sql() -> str: + return ( + f"SELECT status, result, error FROM {qualified('pg_task_result')} " + "WHERE task_id = %s" + ) + # Poll cadence for wait_for_result: start tight (low latency for fast tasks), # back off to a ceiling so a long-running task doesn't hammer the DB. @@ -137,7 +149,7 @@ def store_result( status, result_json, error_text = STATUS_FAILED, None, error or "" with self._cursor() as cur: cur.execute( - _STORE_SQL, + _store_sql(), (str(task_id), status, result_json, error_text, retention_seconds), ) @@ -148,7 +160,7 @@ def get_result(self, task_id: str) -> dict[str, Any] | None: Python ``dict``); ``None`` means the task has not finished yet. """ with self._cursor() as cur: - cur.execute(_GET_SQL, (str(task_id),)) + cur.execute(_get_sql(), (str(task_id),)) row = cur.fetchone() if row is None: return None diff --git a/workers/queue_backend/pg_queue/schema.py b/workers/queue_backend/pg_queue/schema.py new file mode 100644 index 0000000000..ddb4b69b9c --- /dev/null +++ b/workers/queue_backend/pg_queue/schema.py @@ -0,0 +1,86 @@ +"""Schema-qualified table names for the bespoke PG queue. + +The queue's tables (``pg_queue_message``, ``pg_task_result``, +``pg_barrier_state``, ``pg_batch_dedup``, ``pg_orchestrator_lock``, +``pg_periodic_schedule``) live in the schema the backend manages +(``DB_SCHEMA`` — ``unstract`` on-prem, a per-developer schema such as ``ali`` +in cloud dev). + +**Why the SQL must name the schema explicitly** instead of relying on +``search_path``: in cloud the worker connects through PgBouncer, which strips +the ``options=-c search_path=…`` startup parameter +(``IGNORE_STARTUP_PARAMETERS=options``), so the connection never gets the +queue's schema on its path and bare table names resolve to the default +(``public``) → ``UndefinedTable``. Transaction pooling also wouldn't preserve a +``SET search_path`` between statements. Qualifying every table as +``"".
`` makes the SQL self-contained, so it works through the +PgBouncer **transaction** pool unchanged — the same way the Celery result +backend reaches its ``public`` tables. + +The schema is read at call time (not import) from the same ``{prefix}SCHEMA`` +env the connection uses (:func:`create_pg_connection`), so the qualified name +always matches where the connection looks. +""" + +from __future__ import annotations + +import os +import re + +# DB_SCHEMA is operator-supplied (helm values / env), never user input — but we +# still validate it as a bare SQL identifier before interpolating it into SQL, +# as defence-in-depth so a typo'd/hostile value can't become injection. +_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +# Mirror create_pg_connection's default so an unset DB_SCHEMA qualifies to the +# same schema the connection targets (search_path default ``unstract``). +_DEFAULT_SCHEMA = "unstract" + +# Single source of truth for the bespoke queue tables. Every raw query qualifies +# its table THROUGH :func:`qualified`, which rejects any name not listed here — so +# a typo fails loud, and a genuinely new table forces an explicit addition. The +# no-bare-table guard test (tests/test_pg_schema.py) reads this same set, so the +# registry and the lint stay in lockstep. +QUEUE_TABLES = frozenset( + { + "pg_queue_message", + "pg_task_result", + "pg_barrier_state", + "pg_batch_dedup", + "pg_orchestrator_lock", + "pg_periodic_schedule", + } +) + + +def queue_schema(env_prefix: str = "DB_") -> str: + """Return the validated queue schema from ``{env_prefix}SCHEMA``. + + Raises ``ValueError`` if the configured schema is not a plain SQL + identifier — failing loud at the first query beats silently building + injectable or malformed SQL. + """ + schema = os.getenv(f"{env_prefix}SCHEMA", _DEFAULT_SCHEMA) + if not _IDENT_RE.match(schema): + raise ValueError( + f"PG queue: {env_prefix}SCHEMA={schema!r} is not a valid SQL " + "identifier (expected [A-Za-z_][A-Za-z0-9_]*)" + ) + return schema + + +def qualified(table: str, env_prefix: str = "DB_") -> str: + """Return ``"".
`` for use in the queue's raw SQL. + + ``table`` must be one of :data:`QUEUE_TABLES` — an unknown name (typo, or a + new table not yet registered) raises, so it can never silently produce a + query against the wrong/non-existent relation. Only the schema is quoted (it + can be a reserved word or mixed case); the table names are fixed lowercase + identifiers owned by this module. + """ + if table not in QUEUE_TABLES: + raise ValueError( + f"PG queue: {table!r} is not a known queue table. Add it to " + "QUEUE_TABLES (the single registry) if it is genuinely new." + ) + return f'"{queue_schema(env_prefix)}".{table}' diff --git a/workers/tests/test_pg_queue_client.py b/workers/tests/test_pg_queue_client.py index 4a473af01d..ad348ad58d 100644 --- a/workers/tests/test_pg_queue_client.py +++ b/workers/tests/test_pg_queue_client.py @@ -23,6 +23,7 @@ from queue_backend.fairness import DEFAULT_PRIORITY, MAX_PRIORITY, MIN_PRIORITY from queue_backend.pg_queue import PgQueueClient, QueueMessage from queue_backend.pg_queue.connection import create_pg_connection +from queue_backend.pg_queue.schema import qualified # --- Unit: SQL shape against a mocked connection --- @@ -56,7 +57,7 @@ def test_send_inserts_and_returns_msg_id(self): msg_id = PgQueueClient(conn=conn).send("q1", {"a": 1}, org_id="org-9") assert msg_id == 42 sql, params = cur.execute.call_args.args - assert "INSERT INTO pg_queue_message" in sql + assert f"INSERT INTO {qualified('pg_queue_message')}" in sql assert params[0] == "q1" assert params[2] == "org-9" assert '"a": 1' in params[1] # message JSON-serialised @@ -95,7 +96,7 @@ def test_read_runs_skip_locked_dequeue(self): msgs = PgQueueClient(conn=conn).read("q1", vt_seconds=15, qty=3) sql, params = cur.execute.call_args.args assert "FOR UPDATE SKIP LOCKED" in sql - assert "UPDATE pg_queue_message" in sql + assert f"UPDATE {qualified('pg_queue_message')}" in sql assert "ORDER BY priority DESC" in sql # fairness L3 claim order # Param order follows the %s positions: queue_name, qty, vt_seconds. assert params == ("q1", 3, 15) @@ -106,7 +107,7 @@ def test_delete_returns_true_when_row_removed(self): conn, cur = _mock_conn(rowcount=1) assert PgQueueClient(conn=conn).delete(7) is True sql, params = cur.execute.call_args.args - assert "DELETE FROM pg_queue_message" in sql + assert f"DELETE FROM {qualified('pg_queue_message')}" in sql assert params == (7,) conn.commit.assert_called_once() @@ -145,9 +146,7 @@ def _owned_client(monkeypatch, *, execute_raises=None): conn.closed = 0 conn.cursor.return_value = _CursorCtx(cur) factory = MagicMock(return_value=conn) - monkeypatch.setattr( - "queue_backend.pg_queue.client.create_pg_connection", factory - ) + monkeypatch.setattr("queue_backend.pg_queue.client.create_pg_connection", factory) return PgQueueClient(), conn, factory def test_owned_conn_recovered_on_operational_error(self, monkeypatch): @@ -243,9 +242,7 @@ def test_connect_failure_is_logged_and_reraised(self, monkeypatch, caplog): def boom(**_): raise psycopg2.OperationalError("nope") - monkeypatch.setattr( - "queue_backend.pg_queue.connection.psycopg2.connect", boom - ) + monkeypatch.setattr("queue_backend.pg_queue.connection.psycopg2.connect", boom) with ( caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.connection"), pytest.raises(psycopg2.OperationalError), diff --git a/workers/tests/test_pg_reaper.py b/workers/tests/test_pg_reaper.py index 01db9fda9a..d36bd42989 100644 --- a/workers/tests/test_pg_reaper.py +++ b/workers/tests/test_pg_reaper.py @@ -35,6 +35,7 @@ sweep_expired_results, sweep_orphan_dedup, ) +from queue_backend.pg_queue.schema import qualified # The reaper's leader tick also runs the PG scheduler tick (②b). Its behaviour @@ -163,14 +164,18 @@ def test_non_positive_sweep_interval_rejected(self): # An injected knob bypasses the env parser's guard → re-validated in __init__. with pytest.raises(ValueError, match="sweep_interval"): PgReaper( - _FakeLease(), interval_seconds=1, sweep_interval_seconds=0, + _FakeLease(), + interval_seconds=1, + sweep_interval_seconds=0, sweep_conn=object(), ) def test_non_positive_dedup_retention_rejected(self): with pytest.raises(ValueError, match="dedup_retention"): PgReaper( - _FakeLease(), interval_seconds=1, dedup_retention_seconds=0, + _FakeLease(), + interval_seconds=1, + dedup_retention_seconds=0, sweep_conn=object(), ) @@ -264,7 +269,8 @@ def boom(): class TestSchedulerTick: """The orchestrator's second job: the leader (and only the leader) runs the PG scheduler tick each cycle. Scheduling behaviour itself is in - test_pg_scheduler.py; here we only assert the wiring + leader gating.""" + test_pg_scheduler.py; here we only assert the wiring + leader gating. + """ def _reaper(self, lease): return PgReaper( @@ -335,14 +341,17 @@ def test_sweep_expired_results_sql(self): conn, cur = self._conn_cur(3) assert sweep_expired_results(conn) == 3 sql = cur.execute.call_args[0][0] - assert "DELETE FROM pg_task_result" in sql and "expires_at <= now()" in sql + assert ( + f"DELETE FROM {qualified('pg_task_result')}" in sql + and "expires_at <= now()" in sql + ) conn.commit.assert_called_once() def test_sweep_orphan_dedup_sql(self): conn, cur = self._conn_cur(2) assert sweep_orphan_dedup(conn, 999) == 2 args = cur.execute.call_args[0] - assert "DELETE FROM pg_batch_dedup" in args[0] + assert f"DELETE FROM {qualified('pg_batch_dedup')}" in args[0] assert "created_at <= now() - make_interval" in args[0] assert args[1] == (999,) # the retention param is bound, not interpolated conn.commit.assert_called_once() @@ -502,8 +511,13 @@ class _FakeApiClient: """ def __init__( - self, status="EXECUTING", *, fail_read=False, fail_update=False, - fail_update_for=None, on_get=None, + self, + status="EXECUTING", + *, + fail_read=False, + fail_update=False, + fail_update_for=None, + on_get=None, ): self._status = status self._fail_read = fail_read diff --git a/workers/tests/test_pg_schema.py b/workers/tests/test_pg_schema.py new file mode 100644 index 0000000000..22cf0df05d --- /dev/null +++ b/workers/tests/test_pg_schema.py @@ -0,0 +1,103 @@ +"""Unit tests for the PG-queue schema qualifier (queue_backend.pg_queue.schema). + +The qualifier is what lets the worker's raw SQL resolve its tables through +PgBouncer transaction pooling (which strips the ``search_path`` startup param) — +so these guard that the schema is read from ``DB_SCHEMA``, validated, rendered as +a quoted ``"".
`` prefix, and that no production query forgets to +go through it. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +from queue_backend.pg_queue.schema import QUEUE_TABLES, qualified, queue_schema + + +class TestQueueSchema: + def test_defaults_to_unstract_when_unset(self, monkeypatch): + monkeypatch.delenv("DB_SCHEMA", raising=False) + assert queue_schema() == "unstract" + + def test_reads_db_schema(self, monkeypatch): + monkeypatch.setenv("DB_SCHEMA", "acme") + assert queue_schema() == "acme" + + def test_honours_env_prefix(self, monkeypatch): + # The integration suite connects via TEST_DB_* — the qualifier must be + # able to follow the same prefix the connection used. + monkeypatch.setenv("TEST_DB_SCHEMA", "test_tenant") + assert queue_schema(env_prefix="TEST_DB_") == "test_tenant" + + @pytest.mark.parametrize( + "bad", ["pg_queue; DROP TABLE x", "a.b", "1schema", "has space", '"quoted"', ""] + ) + def test_rejects_non_identifier_schema(self, monkeypatch, bad): + # Defence-in-depth: an operator typo / hostile value must fail loud, not + # build injectable or malformed SQL. + monkeypatch.setenv("DB_SCHEMA", bad) + with pytest.raises(ValueError, match="not a valid SQL identifier"): + queue_schema() + + +class TestQualified: + def test_quotes_schema_and_appends_table(self, monkeypatch): + monkeypatch.setenv("DB_SCHEMA", "acme") + assert qualified("pg_queue_message") == '"acme".pg_queue_message' + + def test_default_schema(self, monkeypatch): + monkeypatch.delenv("DB_SCHEMA", raising=False) + assert qualified("pg_task_result") == '"unstract".pg_task_result' + + def test_propagates_schema_validation(self, monkeypatch): + monkeypatch.setenv("DB_SCHEMA", "bad;schema") + with pytest.raises(ValueError, match="not a valid SQL identifier"): + qualified("pg_queue_message") + + def test_rejects_unknown_table(self, monkeypatch): + # A typo'd / unregistered table must fail loud rather than build a query + # against a non-existent relation. + monkeypatch.setenv("DB_SCHEMA", "acme") + with pytest.raises(ValueError, match="not a known queue table"): + qualified("pg_queue_mesage") # typo + + def test_every_registered_table_qualifies(self, monkeypatch): + monkeypatch.setenv("DB_SCHEMA", "acme") + for table in QUEUE_TABLES: + assert qualified(table) == f'"acme".{table}' + + +# Production modules that run raw queue SQL. Globbed (not hardcoded) so a NEW +# module under queue_backend/ is automatically covered by the guard below. +_WORKERS_ROOT = Path(__file__).resolve().parent.parent +_PROD_SQL_DIR = _WORKERS_ROOT / "queue_backend" + + +class TestNoBareTableGuard: + """Fail the build if any production query names a queue table WITHOUT going + through :func:`qualified`. + + A bare ``FROM/INTO/UPDATE pg_
`` resolves via ``search_path`` — which + works in OSS/local (direct Postgres) but is stripped by PgBouncer in cloud → + ``UndefinedTable``. So such a slip passes local tests and only breaks in + cloud; this static guard is what makes "forgetting to qualify" un-mergeable. + """ + + def test_no_unqualified_queue_table_reference(self): + # SQL keywords are uppercase by convention here, so the case-sensitive + # match ignores lowercase prose / docstrings. A qualified use reads + # ``FROM {qualified('pg_x')}`` — the table name never follows the keyword + # directly, so only BARE references match. + tables = "|".join(sorted(QUEUE_TABLES)) + bare = re.compile(rf"\b(?:FROM|INTO|UPDATE)\s+(?:{tables})\b") + offenders: list[str] = [] + for path in sorted(_PROD_SQL_DIR.rglob("*.py")): + for m in bare.finditer(path.read_text()): + rel = path.relative_to(_WORKERS_ROOT) + offenders.append(f"{rel}: {m.group(0)!r}") + assert not offenders, ( + "Unqualified queue-table reference(s) — wrap the table with " + "queue_backend.pg_queue.schema.qualified():\n" + "\n".join(offenders) + ) From 2159ffd9f7f5d7c378dc7efbcc6e5b0511d8b5f8 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:22:01 +0530 Subject: [PATCH 43/44] =?UTF-8?q?UN-3651=20[FIX]=20PG=20barrier=20enqueue?= =?UTF-8?q?=20=E2=80=94=20reconnect-retry=20on=20stale=20cached=20connecti?= =?UTF-8?q?on=20(#2121)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3651 [FIX] PG barrier enqueue — reconnect-retry on stale cached connection A transient `psycopg2.OperationalError: server closed the connection unexpectedly` during the PgBarrier enqueue (the barrier UPSERT) aborted the whole ETL execution -> status ERROR with 0 files. PgBarrier keeps a thread-local cached connection reused across barrier ops; while idle between ops it can be reaped server-side (PgBouncer server_idle_timeout / DB failover) and `_get_conn` can't tell (conn.closed is client-side only), so the first statement after the idle gap fails. `_cursor` recovered the dead conn but did not retry the op. Fix: a one-shot reconnect-retry (`_run_idempotent_write`, attempts=2) scoped to the IDEMPOTENT, pre-dispatch barrier write only — the UPSERT (ON CONFLICT DO UPDATE -> same row/state) + the per-execution dedup reset. Re-running them after an ambiguous commit is a no-op and no header has been dispatched yet, so a retry can neither duplicate a row nor double-dispatch work. Idempotency boundary preserved: the non-idempotent decrement (remaining - 1) and claim_batch stay on the plain `_cursor` (recover-but-don't-retry) — a re-applied decrement could fire the callback early. Scope: workers only, under the pg_queue_enabled flag (PgBarrier runs only on the PG transport; flag off -> CeleryChordBarrier, this code never runs -> zero Celery-path regression). Tests: +3 unit tests (retry-then-succeed / no-retry-on-non-connection-error / reraise-after-exhaust); full PgBarrier suite 54 passed; ruff clean. Dev-tested live (A/B against real Postgres via pg_terminate_backend): old path raises + loses the write; new path reconnects, retries, and the write lands. Co-Authored-By: Claude Opus 4.8 * UN-3651 [FIX] address PR #2121 review — observability, backoff, naming, tests Review feedback (greptile + PR-review-toolkit) — all non-blocking; fix was confirmed correct. Changes: - Retry log now names the real error (type + message) and keeps the traceback (exc_info=True); reworded so it no longer asserts a connection drop the broad psycopg2 catch can't be sure of. (HIGH — observability) - Add a small fixed backoff (0.5s) before the retry — widens the self-heal window on a brief failover and avoids immediately re-hammering a struggling DB on the server-side errors the broad OperationalError catch also covers. (MED) - Rename _run_idempotent_write -> _run_idempotent_pre_dispatch_write and type the op as Callable[[PgCursor], None] so the idempotent/pre-dispatch contract is loud at the call site (it can't be type-enforced). (MED — type design) - _BARRIER_WRITE_ATTEMPTS: Final = 2 # total attempts; doc/comment precision: decrement hazard reworded (premature/incomplete callback or strand, not "fire twice"), "same state" softened (timestamps refresh harmlessly), trimmed the redundant call-site comment to a one-line pointer. (LOW) - Tests: add an end-to-end self-heal test through enqueue() against the real DB (asserts the row lands once + every header dispatched exactly once — pins the wiring a no-DB test can't); parametrize the retry test over OperationalError AND InterfaceError; assert no partial commit (commits==0), exactly one extra attempt (executes counters), and the on-retry warning via caplog. (HIGH + MED) Full PgBarrier suite: 56 passed; ruff clean. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- workers/queue_backend/pg_barrier.py | 78 ++++++++++++- workers/tests/test_pg_barrier.py | 172 +++++++++++++++++++++++++++- 2 files changed, 244 insertions(+), 6 deletions(-) diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.py index f8951f779d..235229f528 100644 --- a/workers/queue_backend/pg_barrier.py +++ b/workers/queue_backend/pg_barrier.py @@ -60,9 +60,10 @@ import json import logging import threading +import time from collections.abc import Callable, Iterator from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Final import psycopg2 import psycopg2.extensions @@ -88,6 +89,7 @@ if TYPE_CHECKING: from celery.canvas import Signature from psycopg2.extensions import connection as PgConnection + from psycopg2.extensions import cursor as PgCursor logger = logging.getLogger(__name__) @@ -128,6 +130,72 @@ def _cursor() -> Iterator[Any]: raise +# One retry for an IDEMPOTENT, pre-dispatch barrier write. The thread-local +# connection is cached across barrier ops, so it can be reaped server-side +# (PgBouncer server_idle_timeout / DB failover) while sitting idle BETWEEN ops — +# and ``_get_conn`` can't tell, since ``conn.closed`` is a client-side flag only. +# The first statement after the idle gap then fails; ``_cursor`` discards the +# dead conn, so a single retry runs against a freshly reconnected one. This turns +# a transient blip (which previously aborted the whole execution at barrier +# enqueue) into a self-heal. Kept a literal (not env-driven) so the idempotency +# bound can't be weakened operationally. +_BARRIER_WRITE_ATTEMPTS: Final = 2 # total attempts: 1 initial + 1 retry +# Small fixed pause before the retry. The idle-reap case reconnects instantly +# regardless; this only widens the self-heal window for a brief DB failover, and +# avoids immediately re-hammering a struggling server on the rarer server-side +# errors the broad psycopg2.OperationalError catch also covers. +_BARRIER_RETRY_BACKOFF_SECONDS: Final = 0.5 + + +def _run_idempotent_pre_dispatch_write( + operation: Callable[[PgCursor], None], *, what: str +) -> None: + """Run ``operation(cur)`` in a committed cursor, retrying ONCE if the cached + connection was dead. + + The name spells out the contract because it can't be type-enforced: only call + with an **idempotent** statement run **before** any task dispatch — the + barrier UPSERT (``ON CONFLICT … DO UPDATE`` → same row, ``remaining``/ + ``results`` reset identically; timestamps refresh, harmlessly) + the + per-execution dedup reset (``DELETE WHERE execution_id``). Re-running them + after an ambiguous commit is a no-op, so a retry can neither duplicate a row + nor double-dispatch work (no header has been enqueued yet). The ``-> None`` + op signature also blocks passing a ``RETURNING``-reading op by construction. + + Deliberately NOT used for the barrier **decrement** (``remaining = + remaining - 1``): it is not idempotent — a re-applied decrement can fire the + callback **prematurely with incomplete results**, or skip past 0 and + **strand the barrier** to expiry — so it stays on the plain :func:`_cursor` + (recover-but-don't-retry). Same for ``claim_batch``, whose ``RETURNING`` + answer flips on a retry. + """ + for attempt in range(1, _BARRIER_WRITE_ATTEMPTS + 1): + try: + with _cursor() as cur: + operation(cur) + return + except (psycopg2.OperationalError, psycopg2.InterfaceError) as exc: + # _cursor already dropped the dead thread-local conn → the next + # _get_conn() reconnects. Retry once; re-raise if it still fails + # (a genuinely-down DB surfaces as ERROR, as before). Name the real + # error + keep the traceback: the broad catch also covers server-side + # conditions (statement timeout, deadlock, admin shutdown), so the + # message must not assert a connection drop it can't be sure of. + if attempt >= _BARRIER_WRITE_ATTEMPTS: + raise + logger.warning( + "PgBarrier: %s — DB write failed (%s: %s); cached connection " + "likely stale, reconnecting and retrying (attempt %d/%d)", + what, + type(exc).__name__, + exc, + attempt, + _BARRIER_WRITE_ATTEMPTS, + exc_info=True, + ) + time.sleep(_BARRIER_RETRY_BACKOFF_SECONDS) + + def _delete_barrier(execution_id: str) -> None: with _cursor() as cur: cur.execute( @@ -311,7 +379,7 @@ def enqueue( f"could not be reaper-recovered. This is a bug in the caller." ) - with _cursor() as cur: + def _reset_barrier(cur: PgCursor) -> None: # UPSERT clears any leftover state from a prior run with this id. # No inline expiry sweep here — an unbounded global DELETE on the # enqueue hot path risks lock contention / deadlocks between @@ -339,6 +407,12 @@ def enqueue( (execution_id,), ) + # Idempotent + pre-dispatch → safe to retry; see + # _run_idempotent_pre_dispatch_write. + _run_idempotent_pre_dispatch_write( + _reset_barrier, what=f"enqueue exec={execution_id}" + ) + self._dispatch_headers( header_tasks, is_pg=is_pg, diff --git a/workers/tests/test_pg_barrier.py b/workers/tests/test_pg_barrier.py index a4fca8b146..7d032a3b19 100644 --- a/workers/tests/test_pg_barrier.py +++ b/workers/tests/test_pg_barrier.py @@ -43,6 +43,7 @@ def _pg_header(task_name="process_file_batch", args=None, queue="file_processing sig.options = {"queue": queue} return sig + _CALLBACK = { "task_name": "process_batch_callback_api", "kwargs": {"execution_id": "exec-1", "pipeline_id": "pipe-1"}, @@ -115,6 +116,140 @@ def test_missing_execution_id_raises(self): ) +# --- Idempotent-write reconnect-retry (no DB) --- + + +class _FakeCursorCtx: + def __init__(self, on_execute): + self._on_execute = on_execute + + def __enter__(self): + cur = MagicMock(name="cursor") + cur.execute.side_effect = self._on_execute + return cur + + def __exit__(self, *exc): + return False + + +class _FakeConn: + """Minimal psycopg2-connection stub. ``execute_error`` (if set) is raised by + every cursor.execute on this connection — simulating a stale/dead socket. + """ + + def __init__(self, *, execute_error=None): + self.closed = False + self._execute_error = execute_error + self.commits = 0 + self.rollbacks = 0 + self.executes = 0 + + def cursor(self): + def _on_execute(*_a, **_k): + self.executes += 1 + if self._execute_error is not None: + raise self._execute_error + + return _FakeCursorCtx(_on_execute) + + def commit(self): + self.commits += 1 + + def rollback(self): + self.rollbacks += 1 + + def close(self): + self.closed = True + + +@pytest.fixture +def _clean_local(): + """Ensure the module thread-local connection is reset around each test.""" + pg_barrier._local.conn = None + yield + pg_barrier._local.conn = None + + +class TestIdempotentWriteRetry: + """`_run_idempotent_pre_dispatch_write` self-heals a stale cached connection + with ONE retry — the fix for the `server closed the connection unexpectedly` + abort at barrier enqueue. It must retry ONLY on connection errors, stay + bounded, and never silently swallow a real (non-connection) failure. + """ + + @pytest.fixture(autouse=True) + def _no_sleep(self, monkeypatch): + # The retry backoff is real (0.5s); skip the wall-clock wait in tests. + monkeypatch.setattr(pg_barrier.time, "sleep", lambda *_a, **_k: None) + + @pytest.mark.parametrize( + "exc", + [ + psycopg2.OperationalError("server closed the connection unexpectedly"), + psycopg2.InterfaceError("connection already closed"), + ], + ids=["OperationalError", "InterfaceError"], + ) + def test_retries_once_on_dead_connection_then_succeeds( + self, _clean_local, monkeypatch, caplog, exc + ): + # InterfaceError is the more common stale-socket symptom, so both arms of + # the (OperationalError, InterfaceError) catch must self-heal. + dead = _FakeConn(execute_error=exc) + healthy = _FakeConn() + pg_barrier._local.conn = dead + # On reconnect (_cursor discarded the dead conn), hand back the healthy one. + monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: healthy) + + attempts = [] + + def op(cur): + attempts.append(1) + cur.execute("UPSERT ...", ("x",)) + + with caplog.at_level("WARNING"): + pg_barrier._run_idempotent_pre_dispatch_write(op, what="test") + + assert len(attempts) == 2 # first failed mid-execute, retry succeeded + assert dead.executes == 1 and healthy.executes == 1 # exactly one extra try + assert dead.commits == 0 # no partial commit on the failed attempt + assert dead.closed is True # stale conn discarded + assert healthy.commits == 1 # committed exactly once, on the retry + assert pg_barrier._local.conn is healthy + # logs on retry (not on success) and names the real error, not a guess. + assert "reconnecting and retrying" in caplog.text + assert type(exc).__name__ in caplog.text + + def test_does_not_retry_non_connection_error(self, _clean_local, monkeypatch): + healthy = _FakeConn() + pg_barrier._local.conn = healthy + reconnects = [] + monkeypatch.setattr( + pg_barrier, + "create_pg_connection", + lambda **_k: reconnects.append(1) or _FakeConn(), + ) + + def op(cur): + raise ValueError("not a connection problem") + + with pytest.raises(ValueError, match="not a connection problem"): + pg_barrier._run_idempotent_pre_dispatch_write(op, what="test") + assert reconnects == [] # a real error must surface immediately, no retry + + def test_reraises_after_exhausting_attempts(self, _clean_local, monkeypatch): + err = psycopg2.OperationalError("still down") + # Every connection (initial + reconnect) is dead → both attempts fail. + monkeypatch.setattr( + pg_barrier, "create_pg_connection", lambda **_k: _FakeConn(execute_error=err) + ) + + with pytest.raises(psycopg2.OperationalError, match="still down"): + pg_barrier._run_idempotent_pre_dispatch_write( + lambda cur: cur.execute("X"), what="test" + ) + + # --- Layer 2: enqueue + link/abort with a real injected connection --- @@ -182,6 +317,38 @@ def test_upsert_creates_row_and_attaches_links(self, barrier_db): cloned.link_error.assert_called_once() cloned.apply_async.assert_called_once() + def test_enqueue_self_heals_on_stale_connection(self, barrier_db, monkeypatch): + # End-to-end through enqueue(): the first barrier write hits a dead cached + # conn; the fix must reconnect to the REAL db, land the row exactly once, + # and still dispatch every header exactly once (no double-dispatch). This + # pins the wiring — reverting enqueue() to the inline `with _cursor()` + # would make this fail (the no-DB retry tests alone would not catch that). + monkeypatch.setattr(pg_barrier.time, "sleep", lambda *_a, **_k: None) + dead = _FakeConn( + execute_error=psycopg2.OperationalError( + "server closed the connection unexpectedly" + ) + ) + pg_barrier._local.conn = ( + dead # the barrier_db fixture's real conn is the reconnect target + ) + monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: barrier_db) + + tasks = [_mock_header_task() for _ in range(3)] + handle = PgBarrier().enqueue( + [t for t, _ in tasks], + callback_task_name="cb", + callback_kwargs={"execution_id": "exec-HEAL"}, + callback_queue="general", + app_instance=None, + ) + + assert handle.id == "exec-HEAL" + assert dead.closed is True # stale conn discarded by _cursor + assert _row(barrier_db, "exec-HEAL") == (3, []) # row landed once, remaining=N + for _, cloned in tasks: # every header dispatched exactly once + cloned.apply_async.assert_called_once() + def test_fairness_header_stamped(self, barrier_db): task, cloned = _mock_header_task() PgBarrier().enqueue( @@ -484,7 +651,6 @@ def test_concurrent_aborts_deduplicate(self, barrier_db): def test_registered_under_canonical_name(self): assert barrier_pg_abort.name == "barrier_pg_abort" - def test_max_retries_zero(self): # A Celery retry would replay the decrement and corrupt the count. assert barrier_pg_decr_and_check.max_retries == 0 @@ -795,9 +961,7 @@ def dispatch_side_effect(*args, **kwargs): raise RuntimeError("broker down") dispatch_side_effect.claimed = False - with patch( - "queue_backend.dispatch.dispatch", side_effect=dispatch_side_effect - ): + with patch("queue_backend.dispatch.dispatch", side_effect=dispatch_side_effect): with pytest.raises(RuntimeError, match="broker down"): PgBarrier().enqueue( [_pg_header(), _pg_header()], From 5d07155678e37e4de9cf314ba4ea061b12206dc6 Mon Sep 17 00:00:00 2001 From: ali <117142933+muhammad-ali-e@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:49:57 +0530 Subject: [PATCH 44/44] =?UTF-8?q?UN-3652=20[FIX]=20PG=20orchestration=20fa?= =?UTF-8?q?ilure=20=E2=80=94=20reconcile=20file=20counters=20+=20surface?= =?UTF-8?q?=20error=20to=20UI=20(#2122)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UN-3652 [FIX] PG orchestration failure — reconcile file counters + surface error to UI When a PG-routed orchestration fails (e.g. the barrier-enqueue blip, or any failure after total_files is set), two UX gaps remained: the execution went ERROR but its files showed as perpetually "in progress" (UI = total - successful - failed, with successful/failed left NULL), and the failure reason never reached the UI (only workflow_execution.error_message + worker logs). Fix, gated behind is_pg_transport (Celery branch byte-identical): - New pure helper WorkflowOrchestrationUtils.pg_failure_file_counts(transport, total_files): PG -> {total_files, successful_files: 0, failed_files: total} so a failed run reads "N failed" (in-progress = 0); non-PG -> {} (the gate). total_files is included because the backend update_status serializer rejects file aggregates without it ("total_files is required when file aggregates are provided"; also successful + failed <= total). - general/tasks.py _execute_general_workflow + api-deployment/tasks.py _run_workflow_api failure handlers: on PG, pass the counts to update_workflow_execution_status (B) and emit the error to the UI/WS via the workflow logger (C). api resolves transport defensively (assigned inside try). Tests: +14 helper/gating unit tests (incl. serializer-constraint guards); related suites green; ruff clean. Dev-tested live E2E on a rebuilt image (flag on, real benchmark API deployment, barrier table renamed to force a non-retryable orchestration failure): - B: failed PG run -> total=1, successful=0, failed=1 (UI in-progress = 0). - C: "❌ Workflow orchestration failed: ..." published to the execution's WS log. The E2E caught a real bug a unit test missed: the first cut omitted total_files and the backend rejected the update with 400 — fixed and re-verified. Co-Authored-By: Claude Opus 4.8 * UN-3652 [FIX] address PR #2122 review — guard status update, hoist transport, test handler Review (greptile P1 + PR-review-toolkit). Changes: - Extract the PG failure recording into one tested seam, WorkflowOrchestrationUtils.record_pg_orchestration_failure (replaces pg_failure_file_counts). It (C) surfaces the error to the UI logger (guarded) and (B) reconciles counters via update_status, with the update WRAPPED in try/except so a status-update failure (e.g. a 400 serializer error) is logged but never re-raised — it can no longer mask the original orchestration error or skip the caller's re-raise (greptile P1). Never raises. - api-deployment: resolve `transport` ONCE up front from kwargs (above the try) and reference it directly, instead of `locals().get("transport", ...)` which fell back to celery for a genuine PG run that failed BEFORE the in-try assignment — re-introducing the exact "stuck in progress" symptom. Dropped the duplicate normalize_transport. - Both handlers now branch: PG -> record_pg_orchestration_failure; else -> the original bare status update, so the Celery path stays byte-identical. - general: the UI-logger call is now None-guarded inside the helper (was an unguarded workflow_logger.log_error). - Tests: replace the helper-only dict tests with behaviour tests of the recorder (mocked api_client/logger): counter reconciliation + total_files, error surfacing, logger-hiccup-doesn't-block-update, update-failure-swallowed-not- raised, and a kwarg-seam guard binding the counts against InternalAPIClient.update_workflow_execution_status's signature. Re-verified live E2E on the refactored handler (renamed pg_barrier_state to force a non-retryable failure): PG run -> ERROR total=1/ok=0/fail=1 (in-progress 0) + "❌ Workflow orchestration failed" published to the WS log channel. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- workers/api-deployment/tasks.py | 56 ++++++++--- workers/general/tasks.py | 25 ++++- .../workflow/execution/orchestration_utils.py | 61 +++++++++++- .../tests/test_pg_orchestration_failure.py | 97 +++++++++++++++++++ 4 files changed, 219 insertions(+), 20 deletions(-) create mode 100644 workers/tests/test_pg_orchestration_failure.py diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index 1aafe70bad..a07ef7b7bd 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -31,6 +31,7 @@ ExecutionStatus, FileHashData, WorkerFileData, + is_pg_transport, normalize_transport, ) from unstract.core.worker_models import ApiDeploymentResultStatus @@ -421,6 +422,15 @@ def _run_workflow_api( WorkflowHelper.process_input_files() methods. """ total_files = len(hash_values_of_files) + # Resolve transport ONCE, up front from the authoritative kwargs source, so + # the failure handler can rely on it even if orchestration fails before the + # fan-out (the PR-1 seam; the fan-out honours it → PG path = PgBarrier). + # Fail-closed to celery on any unrecognised value. + transport = normalize_transport( + kwargs.get("transport", DEFAULT_WORKFLOW_TRANSPORT), + logger=logger, + context=f" [exec:{execution_id}]", + ) # TOOL VALIDATION: Validate tool instances before API workflow orchestration # Get workflow execution context to retrieve tool instances @@ -689,14 +699,7 @@ def _run_workflow_api( org_id=str(schema_name), workload_type=WorkloadType.API, ) - # Transport rides in via the dispatched task's kwargs (PR 1 seam); the - # fan-out honours it (PG path → fire-and-forget PgBarrier). Fail-closed - # to celery on any unrecognised value. - transport = normalize_transport( - kwargs.get("transport", DEFAULT_WORKFLOW_TRANSPORT), - logger=logger, - context=f" [exec:{execution_id}]", - ) + # transport was resolved up front (see top of _run_workflow_api). result = WorkflowOrchestrationUtils.create_chord_execution( batch_tasks=batch_tasks, callback_task_name="process_batch_callback_api", @@ -806,12 +809,37 @@ def _run_workflow_api( } except Exception as e: - # Update execution to ERROR status matching Django pattern - api_client.update_workflow_execution_status( - execution_id=execution_id, - status=ExecutionStatus.ERROR.value, - error_message=f"Error while processing files: {str(e)}", - ) + error_message = f"Error while processing files: {str(e)}" + # On PG, surface the error to the UI + reconcile file counters so a failed + # run reads "N failed" not "N in progress"; the wrapped update inside the + # helper guarantees a status-update failure can't mask `e` or skip the + # re-raise below (see record_pg_orchestration_failure). The Celery branch + # is the original bare update, untouched. + if is_pg_transport(transport): + error_logger = None + try: + error_logger = WorkerWorkflowLogger.create_for_api_workflow( + execution_id=str(execution_id), + organization_id=str(schema_name), + pipeline_id=str(pipeline_id) if pipeline_id else None, + ) + except Exception as log_error: + logger.warning(f"Failed to build UI logger: {log_error}") + WorkflowOrchestrationUtils.record_pg_orchestration_failure( + api_client=api_client, + execution_id=execution_id, + total_files=total_files, + error_message=error_message, + logger=logger, + workflow_logger=error_logger, + ) + else: + # Update execution to ERROR status matching Django pattern + api_client.update_workflow_execution_status( + execution_id=execution_id, + status=ExecutionStatus.ERROR.value, + error_message=error_message, + ) logger.error(f"Execution {execution_id} failed: {str(e)}", exc_info=True) raise diff --git a/workers/general/tasks.py b/workers/general/tasks.py index 82669be43a..877f867056 100644 --- a/workers/general/tasks.py +++ b/workers/general/tasks.py @@ -57,6 +57,7 @@ FileBatchData, FileHashData, WorkerFileData, + is_pg_transport, normalize_transport, ) @@ -753,14 +754,28 @@ def _execute_general_workflow( except Exception as e: logger.error(f"Workflow execution failed: {e}") - try: - api_client.update_workflow_execution_status( + # On PG, surface the error to the UI + reconcile file counters so a + # failed run reads "N failed" not "N in progress" (see + # WorkflowOrchestrationUtils.record_pg_orchestration_failure). The + # Celery branch below is the original status update, untouched. + if is_pg_transport(transport): + WorkflowOrchestrationUtils.record_pg_orchestration_failure( + api_client=api_client, execution_id=execution_id, - status=ExecutionStatus.ERROR.value, + total_files=total_files, error_message=str(e), + logger=logger, + workflow_logger=workflow_logger, ) - except Exception as status_error: - logger.warning(f"Failed to update error status: {status_error}") + else: + try: + api_client.update_workflow_execution_status( + execution_id=execution_id, + status=ExecutionStatus.ERROR.value, + error_message=str(e), + ) + except Exception as status_error: + logger.warning(f"Failed to update error status: {status_error}") orchestration_result = WorkerTaskResponse.error_response( execution_id=execution_id, diff --git a/workers/shared/workflow/execution/orchestration_utils.py b/workers/shared/workflow/execution/orchestration_utils.py index 560af7b91d..80bfa0bc23 100644 --- a/workers/shared/workflow/execution/orchestration_utils.py +++ b/workers/shared/workflow/execution/orchestration_utils.py @@ -12,7 +12,11 @@ from queue_backend import BarrierHandle, FairnessKey, get_barrier from queue_backend.pg_barrier import PgBarrier -from unstract.core.data_models import DEFAULT_WORKFLOW_TRANSPORT, is_pg_transport +from unstract.core.data_models import ( + DEFAULT_WORKFLOW_TRANSPORT, + ExecutionStatus, + is_pg_transport, +) from ...enums import FileDestinationType, PipelineType from ...enums.worker_enums import QueueName @@ -48,6 +52,61 @@ def _barrier_for_transport(transport: str) -> Barrier: class WorkflowOrchestrationUtils: """Centralized workflow orchestration patterns and utilities.""" + @staticmethod + def record_pg_orchestration_failure( + *, + api_client: Any, + execution_id: str, + total_files: int, + error_message: str, + logger: Any, + workflow_logger: Any | None = None, + ) -> None: + """Record a **PG** orchestration failure: surface the error to the UI and + reconcile the file counters, both best-effort. + + Call this ONLY on the PG transport (the caller gates on + ``is_pg_transport``); the Celery failure path keeps its own original + status update untouched, so it stays byte-identical. + + Two things happen, neither of which may disturb the caller's control flow + (it still has the original orchestration exception to log / re-raise): + + * **(C)** if a ``workflow_logger`` is given, publish the error to the UI / + WebSocket logs (guarded — a logging hiccup must not abort the rest). + * **(B)** mark the attempted files failed via ``update_status`` so the run + reads "N failed" not "N in progress" (UI derives in-progress as + ``total - successful - failed``). ``total_files`` is sent alongside the + aggregates because the backend serializer requires it ("total_files is + required when file aggregates are provided") and enforces + ``successful + failed <= total`` — both hold here (0 + N <= N). The call + is wrapped: a status-update failure (e.g. a serializer/validation error) + is logged but **not** re-raised, so it can never mask the real + orchestration error or skip the caller's re-raise (greptile P1). + """ + if workflow_logger: + try: + workflow_logger.log_error( + logger, f"❌ Workflow orchestration failed: {error_message}" + ) + except Exception as log_error: + logger.warning(f"Failed to publish error to UI logs: {log_error}") + try: + api_client.update_workflow_execution_status( + execution_id=execution_id, + status=ExecutionStatus.ERROR.value, + error_message=error_message, + total_files=total_files, + successful_files=0, + failed_files=total_files, + ) + except Exception as status_error: + logger.error( + f"Failed to write ERROR status + file counts for execution " + f"{execution_id}: {status_error}", + exc_info=True, + ) + @staticmethod def create_chord_execution( batch_tasks: list[Signature], diff --git a/workers/tests/test_pg_orchestration_failure.py b/workers/tests/test_pg_orchestration_failure.py new file mode 100644 index 0000000000..75adc03a18 --- /dev/null +++ b/workers/tests/test_pg_orchestration_failure.py @@ -0,0 +1,97 @@ +"""Tests for ``WorkflowOrchestrationUtils.record_pg_orchestration_failure`` (UN-3652). + +This is the shared PG orchestration-failure recorder used by both the general and +api workers. The tests pin the actual behaviour the PR adds — error surfacing, +counter reconciliation, and the guarantee that **neither a UI-logging hiccup nor +a status-update failure may disturb the caller's original exception/flow** (it +must never raise, so the handler's own re-raise / error-response always runs). + +The Celery path is byte-identical by construction: this recorder is only invoked +on the ``is_pg_transport`` branch, so a Celery failure never reaches it (and so +never sends file aggregates). +""" + +from __future__ import annotations + +import inspect +from unittest.mock import MagicMock + +from shared.workflow.execution.orchestration_utils import WorkflowOrchestrationUtils + +from unstract.core.data_models import ExecutionStatus + + +def _record(**overrides): + api_client = overrides.pop("api_client", MagicMock()) + logger = overrides.pop("logger", MagicMock()) + workflow_logger = overrides.pop("workflow_logger", MagicMock()) + kwargs = { + "api_client": api_client, + "execution_id": "exec-1", + "total_files": 2, + "error_message": "boom", + "logger": logger, + "workflow_logger": workflow_logger, + } + kwargs.update(overrides) + WorkflowOrchestrationUtils.record_pg_orchestration_failure(**kwargs) + return api_client, logger, workflow_logger + + +class TestRecordPgOrchestrationFailure: + def test_reconciles_counters_via_update_status(self): + # B: marks the attempted files failed (in-progress = total - 0 - total = 0) + # AND includes total_files (the serializer requires it with aggregates). + api_client, _, _ = _record(total_files=2) + api_client.update_workflow_execution_status.assert_called_once_with( + execution_id="exec-1", + status=ExecutionStatus.ERROR.value, + error_message="boom", + total_files=2, + successful_files=0, + failed_files=2, + ) + + def test_surfaces_error_to_ui_logger(self): + # C: the failure reason is published to the UI/WS logger. + _, _, wl = _record() + assert wl.log_error.called + assert "boom" in wl.log_error.call_args.args[1] + + def test_no_logger_still_reconciles(self): + api_client, _, _ = _record(workflow_logger=None) + api_client.update_workflow_execution_status.assert_called_once() + + def test_ui_logger_failure_does_not_block_update(self): + # A WS hiccup must not abort the counter reconciliation. + wl = MagicMock() + wl.log_error.side_effect = RuntimeError("ws down") + api_client, logger, _ = _record(workflow_logger=wl) + api_client.update_workflow_execution_status.assert_called_once() + assert logger.warning.called + + def test_status_update_failure_is_swallowed_not_raised(self): + # The greptile P1 guarantee: a rejected update (e.g. a 400 serializer + # error) is logged but NOT re-raised, so it can't mask the caller's + # original orchestration exception or skip its re-raise. + api_client = MagicMock() + api_client.update_workflow_execution_status.side_effect = ValueError("400") + # Must not raise: + _, logger, _ = _record(api_client=api_client) + assert logger.error.called # logged at error level (a real defect signal) + + def test_count_keys_are_valid_update_status_kwargs(self): + # Guard the ``**counts`` seam against a kwarg rename on either side — a + # mismatch would otherwise only surface as a runtime TypeError on the rare + # PG-failure path. + from shared.api.internal_client import InternalAPIClient + + sig = inspect.signature(InternalAPIClient.update_workflow_execution_status) + sig.bind_partial( + execution_id="x", + status=ExecutionStatus.ERROR.value, + error_message="e", + total_files=2, + successful_files=0, + failed_files=2, + )