chore: import upstream snapshot with attribution

tests/api/user/agents/test_webhooks_idempotency.py

@@ -0,0 +1,407 @@
+"""Idempotency-Key behavior on the agent webhook listener route."""
+
+from contextlib import contextmanager
+from unittest.mock import MagicMock, patch
+
+import pytest
+from flask import Flask
+
+
+@pytest.fixture
+def app():
+    return Flask(__name__)
+
+
+@contextmanager
+def _patch_db(conn):
+    @contextmanager
+    def _yield():
+        yield conn
+
+    with patch(
+        "application.api.user.agents.webhooks.db_session", _yield
+    ), patch(
+        "application.api.user.agents.webhooks.db_readonly", _yield
+    ):
+        yield
+
+
+def _seed_agent(pg_conn, user="u", token="tk", **kw):
+    from application.storage.db.repositories.agents import AgentsRepository
+    return AgentsRepository(pg_conn).create(
+        user, "a", "published", incoming_webhook_token=token, **kw,
+    )
+
+
+def _apply_async_mock():
+    """Mock for ``process_agent_webhook.apply_async``; ``task.id`` mirrors the predetermined id."""
+    def _side_effect(*args, **kwargs):
+        return MagicMock(id=kwargs.get("task_id") or "auto-task-id")
+    return MagicMock(side_effect=_side_effect)
+
+
+class TestWebhookIdempotency:
+    def test_no_header_enqueues_normally(self, app, pg_conn):
+        from application.api.user.agents.webhooks import AgentWebhookListener
+
+        agent = _seed_agent(pg_conn, user="u-noh", token="tk-noh")
+        apply_mock = _apply_async_mock()
+
+        with _patch_db(pg_conn), patch(
+            "application.api.user.agents.webhooks.process_agent_webhook.apply_async",
+            apply_mock,
+        ), app.test_request_context(
+            "/api/webhooks/agents/tk-noh", method="POST",
+            json={"event": "x"},
+        ):
+            listener = AgentWebhookListener()
+            response = listener.post(
+                webhook_token="tk-noh",
+                agent=agent,
+                agent_id_str=str(agent["id"]),
+            )
+        assert response.status_code == 200
+        assert apply_mock.call_count == 1
+
+    def test_header_first_post_records_row(self, app, pg_conn):
+        from sqlalchemy import text
+
+        from application.api.user.agents.webhooks import AgentWebhookListener
+
+        agent = _seed_agent(pg_conn, user="u-first", token="tk-first")
+        apply_mock = _apply_async_mock()
+
+        with _patch_db(pg_conn), patch(
+            "application.api.user.agents.webhooks.process_agent_webhook.apply_async",
+            apply_mock,
+        ), app.test_request_context(
+            "/api/webhooks/agents/tk-first", method="POST",
+            json={"event": "x"},
+            headers={"Idempotency-Key": "key-abc"},
+        ):
+            listener = AgentWebhookListener()
+            response = listener.post(
+                webhook_token="tk-first",
+                agent=agent,
+                agent_id_str=str(agent["id"]),
+            )
+        assert response.status_code == 200
+        assert apply_mock.call_count == 1
+        predetermined_id = apply_mock.call_args.kwargs["task_id"]
+        assert response.json["task_id"] == predetermined_id
+
+        # Stored under the *scoped* form ``"{agent_id}:{key}"`` so two
+        # agents sharing the same raw header don't collapse on PK.
+        scoped_key = f"{agent['id']}:key-abc"
+        row = pg_conn.execute(
+            text("SELECT task_id, agent_id FROM webhook_dedup WHERE idempotency_key = :k"),
+            {"k": scoped_key},
+        ).fetchone()
+        assert row is not None
+        assert row[0] == predetermined_id
+        assert str(row[1]) == str(agent["id"])
+
+    def test_header_forwards_idempotency_key_to_delay(self, app, pg_conn):
+        """The Celery task body needs the key so ``with_idempotency`` can
+        record terminal status and ``_derive_source_id`` can pick it up.
+        """
+        from application.api.user.agents.webhooks import AgentWebhookListener
+
+        agent = _seed_agent(pg_conn, user="u-fwd", token="tk-fwd")
+        apply_mock = _apply_async_mock()
+
+        with _patch_db(pg_conn), patch(
+            "application.api.user.agents.webhooks.process_agent_webhook.apply_async",
+            apply_mock,
+        ), app.test_request_context(
+            "/api/webhooks/agents/tk-fwd", method="POST",
+            json={"event": "x"},
+            headers={"Idempotency-Key": "key-fwd"},
+        ):
+            listener = AgentWebhookListener()
+            listener.post(
+                webhook_token="tk-fwd",
+                agent=agent,
+                agent_id_str=str(agent["id"]),
+            )
+        # Worker sees the agent-scoped form so its dedup row is also
+        # agent-distinct.
+        scoped_key = f"{agent['id']}:key-fwd"
+        assert (
+            apply_mock.call_args.kwargs["kwargs"]["idempotency_key"]
+            == scoped_key
+        )
+
+    def test_same_header_second_post_returns_cached(self, app, pg_conn):
+        from application.api.user.agents.webhooks import AgentWebhookListener
+
+        agent = _seed_agent(pg_conn, user="u-rep", token="tk-rep")
+        apply_mock = _apply_async_mock()
+
+        with _patch_db(pg_conn), patch(
+            "application.api.user.agents.webhooks.process_agent_webhook.apply_async",
+            apply_mock,
+        ):
+            with app.test_request_context(
+                "/api/webhooks/agents/tk-rep", method="POST",
+                json={"event": "x"},
+                headers={"Idempotency-Key": "key-rep"},
+            ):
+                listener = AgentWebhookListener()
+                first = listener.post(
+                    webhook_token="tk-rep",
+                    agent=agent,
+                    agent_id_str=str(agent["id"]),
+                )
+            with app.test_request_context(
+                "/api/webhooks/agents/tk-rep", method="POST",
+                json={"event": "x"},
+                headers={"Idempotency-Key": "key-rep"},
+            ):
+                listener = AgentWebhookListener()
+                second = listener.post(
+                    webhook_token="tk-rep",
+                    agent=agent,
+                    agent_id_str=str(agent["id"]),
+                )
+
+        assert first.status_code == 200
+        assert second.status_code == 200
+        assert first.json == second.json
+        assert apply_mock.call_count == 1
+
+    def test_concurrent_same_key_only_one_apply_async(self, app, pg_engine):
+        """Race test (M3): N parallel webhook POSTs with same key → only ONE apply_async.
+
+        Uses ``pg_engine`` so each thread checks out its own DB connection
+        (sharing a single Connection serializes at the driver level).
+        """
+        from concurrent.futures import ThreadPoolExecutor
+        from contextlib import contextmanager
+
+        from application.api.user.agents.webhooks import AgentWebhookListener
+        from application.storage.db.repositories.agents import AgentsRepository
+
+        with pg_engine.begin() as conn:
+            agent = AgentsRepository(conn).create(
+                "u-race", "a", "published", incoming_webhook_token="tk-race",
+            )
+
+        apply_mock = _apply_async_mock()
+
+        @contextmanager
+        def _engine_session():
+            with pg_engine.begin() as conn:
+                yield conn
+
+        @contextmanager
+        def _engine_readonly():
+            with pg_engine.connect() as conn:
+                yield conn
+
+        def fire(idx):
+            # Patches sit outside the thread pool (see below); only the
+            # per-thread Flask request context is set up inside.
+            with app.test_request_context(
+                "/api/webhooks/agents/tk-race", method="POST",
+                json={"event": idx},
+                headers={"Idempotency-Key": "wh-race"},
+            ):
+                listener = AgentWebhookListener()
+                return listener.post(
+                    webhook_token="tk-race",
+                    agent=agent,
+                    agent_id_str=str(agent["id"]),
+                )
+
+        # ``unittest.mock.patch`` is not thread-safe; set up
+        # module-attribute patches once before fanning out so every
+        # thread sees the mock instead of racing on save/restore.
+        with patch(
+            "application.api.user.agents.webhooks.db_session",
+            _engine_session,
+        ), patch(
+            "application.api.user.agents.webhooks.db_readonly",
+            _engine_readonly,
+        ), patch(
+            "application.api.user.agents.webhooks.process_agent_webhook.apply_async",
+            apply_mock,
+        ), ThreadPoolExecutor(max_workers=8) as ex:
+            responses = list(ex.map(fire, range(8)))
+        assert all(r.status_code == 200 for r in responses)
+        assert apply_mock.call_count == 1
+        ids = {r.json["task_id"] for r in responses}
+        assert len(ids) == 1
+        assert "deduplicated" not in ids
+
+    def test_same_key_different_agent_does_not_collide(self, app, pg_conn):
+        """Idempotency keys are now scoped by ``agent_id`` — two agents
+        sending the same raw header each get their own dedup row, both
+        requests enqueue work, and the responses carry distinct
+        ``task_id``s. (Pre-fix, the second agent's request was silently
+        deduplicated against the first agent's row.)
+        """
+        from sqlalchemy import text as sql_text
+
+        from application.api.user.agents.webhooks import AgentWebhookListener
+
+        agent_a = _seed_agent(pg_conn, user="u-a", token="tk-a")
+        agent_b = _seed_agent(pg_conn, user="u-b", token="tk-b")
+        apply_mock = _apply_async_mock()
+
+        with _patch_db(pg_conn), patch(
+            "application.api.user.agents.webhooks.process_agent_webhook.apply_async",
+            apply_mock,
+        ):
+            with app.test_request_context(
+                "/api/webhooks/agents/tk-a", method="POST",
+                json={"event": "x"},
+                headers={"Idempotency-Key": "global-key"},
+            ):
+                listener = AgentWebhookListener()
+                first = listener.post(
+                    webhook_token="tk-a",
+                    agent=agent_a,
+                    agent_id_str=str(agent_a["id"]),
+                )
+            with app.test_request_context(
+                "/api/webhooks/agents/tk-b", method="POST",
+                json={"event": "x"},
+                headers={"Idempotency-Key": "global-key"},
+            ):
+                listener = AgentWebhookListener()
+                second = listener.post(
+                    webhook_token="tk-b",
+                    agent=agent_b,
+                    agent_id_str=str(agent_b["id"]),
+                )
+
+        assert first.status_code == 200
+        assert second.status_code == 200
+        assert first.json["task_id"] != second.json["task_id"]
+        assert apply_mock.call_count == 2
+
+        # And there are two ``webhook_dedup`` rows: one per agent scope.
+        rows = pg_conn.execute(
+            sql_text(
+                "SELECT idempotency_key, agent_id FROM webhook_dedup "
+                "WHERE idempotency_key LIKE :pat ORDER BY idempotency_key"
+            ),
+            {"pat": "%:global-key"},
+        ).fetchall()
+        assert len(rows) == 2
+        scopes = {str(r[1]) for r in rows}
+        assert scopes == {str(agent_a["id"]), str(agent_b["id"])}
+
+    def test_empty_header_treated_as_absent(self, app, pg_conn):
+        from sqlalchemy import text
+
+        from application.api.user.agents.webhooks import AgentWebhookListener
+
+        agent = _seed_agent(pg_conn, user="u-empty", token="tk-empty")
+        apply_mock = _apply_async_mock()
+
+        with _patch_db(pg_conn), patch(
+            "application.api.user.agents.webhooks.process_agent_webhook.apply_async",
+            apply_mock,
+        ), app.test_request_context(
+            "/api/webhooks/agents/tk-empty", method="POST",
+            json={"event": "x"},
+            headers={"Idempotency-Key": ""},
+        ):
+            listener = AgentWebhookListener()
+            response = listener.post(
+                webhook_token="tk-empty",
+                agent=agent,
+                agent_id_str=str(agent["id"]),
+            )
+        assert response.status_code == 200
+        assert apply_mock.call_count == 1
+        count = pg_conn.execute(
+            text("SELECT count(*) FROM webhook_dedup")
+        ).scalar()
+        assert count == 0
+
+    def test_oversized_header_rejected_with_400(self, app, pg_conn):
+        from application.api.user.agents.webhooks import AgentWebhookListener
+
+        agent = _seed_agent(pg_conn, user="u-big", token="tk-big")
+        oversized = "x" * 257
+
+        with _patch_db(pg_conn), patch(
+            "application.api.user.agents.webhooks.process_agent_webhook.apply_async",
+        ) as mock_apply, app.test_request_context(
+            "/api/webhooks/agents/tk-big", method="POST",
+            json={"event": "x"},
+            headers={"Idempotency-Key": oversized},
+        ):
+            listener = AgentWebhookListener()
+            response = listener.post(
+                webhook_token="tk-big",
+                agent=agent,
+                agent_id_str=str(agent["id"]),
+            )
+        assert response.status_code == 400
+        assert mock_apply.call_count == 0
+
+    def test_stale_dedup_row_does_not_block_new_work(self, app, pg_conn):
+        """Regression for the TTL fail-shut bug: a >24h-old dedup row
+        must not silently drop a new request. Pre-fix, the second POST
+        returned ``task_id="deduplicated"`` and never enqueued.
+        """
+        from sqlalchemy import text
+
+        from application.api.user.agents.webhooks import AgentWebhookListener
+
+        agent = _seed_agent(pg_conn, user="u-stale", token="tk-stale")
+        apply_mock = _apply_async_mock()
+
+        # First POST creates a dedup row.
+        with _patch_db(pg_conn), patch(
+            "application.api.user.agents.webhooks.process_agent_webhook.apply_async",
+            apply_mock,
+        ), app.test_request_context(
+            "/api/webhooks/agents/tk-stale", method="POST",
+            json={"event": "x"},
+            headers={"Idempotency-Key": "stale-key"},
+        ):
+            listener = AgentWebhookListener()
+            first = listener.post(
+                webhook_token="tk-stale",
+                agent=agent,
+                agent_id_str=str(agent["id"]),
+            )
+        assert first.status_code == 200
+        first_task_id = first.json["task_id"]
+        assert first_task_id != "deduplicated"
+
+        # Backdate the row so it looks 25h old.
+        scoped_key = f"{agent['id']}:stale-key"
+        pg_conn.execute(
+            text(
+                "UPDATE webhook_dedup SET created_at = "
+                "clock_timestamp() - make_interval(hours => 25) "
+                "WHERE idempotency_key = :k"
+            ),
+            {"k": scoped_key},
+        )
+
+        # Second POST with the same key must enqueue again, not silently dedup.
+        with _patch_db(pg_conn), patch(
+            "application.api.user.agents.webhooks.process_agent_webhook.apply_async",
+            apply_mock,
+        ), app.test_request_context(
+            "/api/webhooks/agents/tk-stale", method="POST",
+            json={"event": "x2"},
+            headers={"Idempotency-Key": "stale-key"},
+        ):
+            listener = AgentWebhookListener()
+            second = listener.post(
+                webhook_token="tk-stale",
+                agent=agent,
+                agent_id_str=str(agent["id"]),
+            )
+        assert second.status_code == 200
+        assert second.json["task_id"] != "deduplicated"
+        assert second.json["task_id"] != first_task_id
+        assert apply_mock.call_count == 2