Hermes-agent

tests/stress/README.md

@@ -0,0 +1,41 @@
+# Stress / battle-test suite
+
+Long-running tests that exercise the Kanban kernel under adversarial
+conditions. **Not run by `scripts/run_tests.sh`** because they can
+take 30+ seconds each and spawn real subprocesses.
+
+Run manually:
+
+```bash
+./venv/bin/python -m pytest tests/stress/ -v -s
+# or individual files:
+./venv/bin/python tests/stress/test_concurrency.py
+./venv/bin/python tests/stress/test_subprocess_e2e.py
+./venv/bin/python tests/stress/test_property_fuzzing.py
+./venv/bin/python tests/stress/test_benchmarks.py
+```
+
+## What's covered
+
+- **test_concurrency.py** — 5 workers, 100 tasks, race-for-claim. Asserts
+  no double-claims, no orphan runs, no SQLite errors escape retry.
+- **test_concurrency_mixed.py** — 10 workers + 1 reclaimer, 500 tasks,
+  random ops (claim/complete/block/unblock/archive). Same invariants
+  under adversarial scheduling.
+- **test_concurrency_reclaim_race.py** — TTL < work duration so the
+  reclaimer intentionally yanks tasks mid-work; verifies the worker's
+  late-complete is refused cleanly (CAS guard works).
+- **test_subprocess_e2e.py** — dispatcher spawns real Python subprocess
+  workers that heartbeat + complete via the CLI; crash detection
+  against a real dead PID.
+- **test_property_fuzzing.py** — 500 random operation sequences,
+  ~40k operations total, 9 invariant checks after each step.
+- **test_atypical_scenarios.py** — 28 scenarios covering atypical
+  user inputs: unicode/emoji/RTL, 1 MB strings, SQL injection
+  attempts, cycles, self-parents, wide fan-in/out, clock skew,
+  HERMES_HOME with spaces/unicode/symlinks, 1000 runs on one
+  task, idempotency-key race across processes, terminal-state
+  resurrection attempts, dashboard REST with weird JSON.
+- **test_benchmarks.py** — latency at 100/1k/10k tasks for dispatch,
+  recompute_ready, list_tasks, build_worker_context, etc. Results saved
+  to JSON for regression diffing.

tests/stress/_fake_worker.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""Fake worker process that exercises the real subprocess contract.
+
+Reads HERMES_KANBAN_TASK from env, heartbeats periodically, does short
+work, completes via the CLI. Designed to be spawned by the dispatcher
+exactly the way `hermes chat -q` would be, minus the LLM cost.
+"""
+
+import json
+import os
+import subprocess
+import time
+
+
+def main():
+    tid = os.environ["HERMES_KANBAN_TASK"]
+    workspace = os.environ.get("HERMES_KANBAN_WORKSPACE", "")
+
+    # Announce via CLI (goes through real argparse + init_db + etc)
+    subprocess.run(
+        ["hermes", "kanban", "heartbeat", tid, "--note", "started"],
+        check=True, capture_output=True,
+    )
+
+    # Simulate work with periodic heartbeats
+    for i in range(3):
+        time.sleep(0.3)
+        subprocess.run(
+            ["hermes", "kanban", "heartbeat", tid, "--note", f"progress {i+1}/3"],
+            check=True, capture_output=True,
+        )
+
+    # Complete with structured handoff
+    subprocess.run(
+        [
+            "hermes", "kanban", "complete", tid,
+            "--summary", f"real-subprocess worker finished {tid}",
+            "--metadata", json.dumps({
+                "workspace": workspace,
+                "worker_pid": os.getpid(),
+                "iterations": 3,
+            }),
+        ],
+        check=True, capture_output=True,
+    )
+
+
+if __name__ == "__main__":
+    main()

tests/stress/conftest.py

@@ -0,0 +1,37 @@
+"""pytest config for the stress/ subdirectory.
+
+These tests are slow (30s+), spawn subprocesses, and are not run by
+default. Enable via `pytest --run-stress` or by running the scripts
+directly.
+
+The scripts are primarily __main__-executable entry points; pytest
+isn't expected to collect individual test functions from them.
+"""
+import pytest
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--run-stress", default=False):
+        return
+    skip_stress = pytest.mark.skip(
+        reason="stress test (opt-in via --run-stress or run script directly)"
+    )
+    for item in items:
+        if "tests/stress" in str(item.fspath):
+            item.add_marker(skip_stress)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--run-stress",
+        action="store_true",
+        default=False,
+        help="Run the stress/battle-test suite (slow, spawns subprocesses).",
+    )
+
+
+collect_ignore_glob = [
+    # The stress scripts have top-level code and hard-coded paths; they're
+    # meant to run as `python tests/stress/<name>.py`, not as pytest modules.
+    "*.py",
+]

tests/stress/test_benchmarks.py

@@ -0,0 +1,221 @@
+"""Scale benchmarks for the Kanban kernel.
+
+Measures:
+  - dispatch_once latency at 100, 1000, 10000 tasks
+  - recompute_ready latency at 100, 1000, 10000 todo tasks with wide parent graphs
+  - build_worker_context latency with 1, 10, 50 parent dependencies
+  - board list/stats query latency
+  - task_runs query latency at scale
+
+Results printed as a table. Saved to JSON for regression-diffing in CI
+or future reviews. Not a pass/fail test — records numbers so we know
+when a change regresses latency by 10x and can decide whether to care.
+"""
+
+import json
+import os
+import random
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+WT = str(Path(__file__).resolve().parents[2])
+
+
+def bench(label, fn, iterations=5):
+    """Time fn over `iterations` runs, return (min, median, max) in ms."""
+    times = []
+    for _ in range(iterations):
+        t0 = time.perf_counter()
+        fn()
+        times.append((time.perf_counter() - t0) * 1000)
+    times.sort()
+    mn = times[0]
+    md = times[len(times) // 2]
+    mx = times[-1]
+    return {"label": label, "iter": iterations, "min_ms": mn, "median_ms": md, "max_ms": mx}
+
+
+def seed_tasks(conn, kb, n, assignee="bench-worker", with_parents=False):
+    """Seed n tasks. Optionally give each task 5 parents."""
+    ids = []
+    for i in range(n):
+        if with_parents and i >= 5:
+            parents = random.sample(ids[:i], 5)
+        else:
+            parents = ()
+        tid = kb.create_task(
+            conn, title=f"bench {i}", assignee=assignee,
+            tenant="bench", parents=parents,
+        )
+        ids.append(tid)
+    return ids
+
+
+def main():
+    home = tempfile.mkdtemp(prefix="hermes_bench_")
+    os.environ["HERMES_HOME"] = home
+    os.environ["HOME"] = home
+    sys.path.insert(0, WT)
+    from hermes_cli import kanban_db as kb
+
+    kb.init_db()
+
+    results = []
+
+    # ============ dispatch_once latency ============
+    for n in [100, 1000, 10000]:
+        print(f"\n== dispatch_once @ {n} tasks ==")
+        # Fresh DB each time so we're not measuring cumulative effects
+        import shutil
+        shutil.rmtree(home, ignore_errors=True)
+        os.makedirs(home)
+        kb._INITIALIZED_PATHS.clear()
+        kb.init_db()
+        conn = kb.connect()
+        seed_tasks(conn, kb, n, assignee=None)  # no assignee → won't spawn
+        r = bench(
+            f"dispatch_once (n={n}, no spawn)",
+            lambda: kb.dispatch_once(conn, spawn_fn=lambda *_: None),
+            iterations=5,
+        )
+        print(f"  min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms")
+        r["n"] = n
+        results.append(r)
+        conn.close()
+
+    # ============ recompute_ready at scale with parent graphs ============
+    for n in [100, 1000, 10000]:
+        print(f"\n== recompute_ready @ {n} tasks (5 parents each) ==")
+        shutil.rmtree(home, ignore_errors=True)
+        os.makedirs(home)
+        kb._INITIALIZED_PATHS.clear()
+        kb.init_db()
+        conn = kb.connect()
+        ids = seed_tasks(conn, kb, n, assignee=None, with_parents=True)
+        # Complete the first 100 so some todo tasks might get promoted
+        for tid in ids[:min(100, n // 10)]:
+            kb.complete_task(conn, tid, result="bench")
+        r = bench(
+            f"recompute_ready (n={n}, with parents)",
+            lambda: kb.recompute_ready(conn),
+            iterations=5,
+        )
+        print(f"  min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms")
+        r["n"] = n
+        results.append(r)
+        conn.close()
+
+    # ============ build_worker_context with N parents ============
+    for parent_count in [1, 10, 50]:
+        print(f"\n== build_worker_context with {parent_count} parents ==")
+        shutil.rmtree(home, ignore_errors=True)
+        os.makedirs(home)
+        kb._INITIALIZED_PATHS.clear()
+        kb.init_db()
+        conn = kb.connect()
+        # Create parents, complete them with summaries+metadata
+        parent_ids = []
+        for i in range(parent_count):
+            pid = kb.create_task(conn, title=f"parent {i}", assignee="p")
+            kb.claim_task(conn, pid)
+            kb.complete_task(
+                conn, pid,
+                summary=f"parent {i} result that is longer than a single token "
+                        f"so we actually measure the IO",
+                metadata={"files": [f"file_{j}.py" for j in range(5)], "i": i},
+            )
+            parent_ids.append(pid)
+        child_id = kb.create_task(
+            conn, title="child", assignee="c", parents=parent_ids,
+        )
+        r = bench(
+            f"build_worker_context (parents={parent_count})",
+            lambda: kb.build_worker_context(conn, child_id),
+            iterations=10,
+        )
+        print(f"  min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms")
+        r["parent_count"] = parent_count
+        results.append(r)
+        conn.close()
+
+    # ============ list_tasks at scale ============
+    for n in [100, 1000, 10000]:
+        print(f"\n== list_tasks @ {n} ==")
+        shutil.rmtree(home, ignore_errors=True)
+        os.makedirs(home)
+        kb._INITIALIZED_PATHS.clear()
+        kb.init_db()
+        conn = kb.connect()
+        seed_tasks(conn, kb, n)
+        r = bench(
+            f"list_tasks (n={n})",
+            lambda: kb.list_tasks(conn),
+            iterations=5,
+        )
+        print(f"  min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms")
+        r["n"] = n
+        results.append(r)
+        conn.close()
+
+    # ============ board_stats at scale ============
+    for n in [100, 1000, 10000]:
+        print(f"\n== board_stats @ {n} ==")
+        shutil.rmtree(home, ignore_errors=True)
+        os.makedirs(home)
+        kb._INITIALIZED_PATHS.clear()
+        kb.init_db()
+        conn = kb.connect()
+        seed_tasks(conn, kb, n)
+        r = bench(
+            f"board_stats (n={n})",
+            lambda: kb.board_stats(conn),
+            iterations=5,
+        )
+        print(f"  min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms")
+        r["n"] = n
+        results.append(r)
+        conn.close()
+
+    # ============ list_runs at scale ============
+    for n in [100, 1000]:
+        print(f"\n== list_runs for task with {n} attempts ==")
+        shutil.rmtree(home, ignore_errors=True)
+        os.makedirs(home)
+        kb._INITIALIZED_PATHS.clear()
+        kb.init_db()
+        conn = kb.connect()
+        tid = kb.create_task(conn, title="x", assignee="w")
+        # Create N attempts via claim/release
+        for i in range(n):
+            kb.claim_task(conn, tid, ttl_seconds=0)
+            kb.release_stale_claims(conn)
+        r = bench(
+            f"list_runs (runs={n})",
+            lambda: kb.list_runs(conn, tid),
+            iterations=10,
+        )
+        print(f"  min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms")
+        r["run_count"] = n
+        results.append(r)
+        conn.close()
+
+    # ============ SUMMARY TABLE ============
+    print()
+    print("=" * 60)
+    print("SUMMARY")
+    print("=" * 60)
+    print(f"{'Benchmark':<50} {'min':>8} {'median':>8} {'max':>8}")
+    for r in results:
+        print(f"{r['label']:<50} {r['min_ms']:>7.1f}ms {r['median_ms']:>7.1f}ms {r['max_ms']:>7.1f}ms")
+
+    # Save for future diffing.
+    out_path = "/tmp/kanban_bench_results.json"
+    with open(out_path, "w") as f:
+        json.dump(results, f, indent=2)
+    print(f"\nResults saved to {out_path}")
+
+
+if __name__ == "__main__":
+    main()

tests/stress/test_concurrency.py

@@ -0,0 +1,301 @@
+"""Multi-process concurrency stress test for the Kanban kernel.
+
+5 worker processes race for claims on a shared DB with 100 tasks. Each
+worker loops: claim -> simulate work -> complete. Asserts the invariants
+that make the system worth building:
+
+  - No task claimed by two workers simultaneously
+  - No task completed twice
+  - Every claim produces exactly one run row
+  - Every completion closes exactly one run row
+  - Zero SQLite locking errors that escape the retry layer
+  - Total run count == total claim events == total completed events
+
+This test is the primary justification for WAL + CAS-based claim. If it
+passes, the architecture holds. If it fails, we have a real bug to fix
+before anyone runs this in anger.
+"""
+
+import json
+import multiprocessing as mp
+import os
+import random
+import sqlite3
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+
+NUM_WORKERS = 5
+NUM_TASKS = 100
+WORKER_TIMEOUT_S = 60
+WT = str(Path(__file__).resolve().parents[2])
+
+
+def worker_loop(worker_id: int, hermes_home: str, result_file: str) -> None:
+    """One worker's inner loop. Runs in a fresh Python process.
+
+    Tries to claim a ready task, marks it done with a per-worker summary,
+    repeats until the ready pool is empty. Records every claim + complete
+    into its own JSON result file for later aggregation.
+    """
+    os.environ["HERMES_HOME"] = hermes_home
+    os.environ["HOME"] = hermes_home
+    sys.path.insert(0, WT)
+
+    from hermes_cli import kanban_db as kb
+
+    events = []
+    empty_polls = 0
+    start = time.monotonic()
+
+    while time.monotonic() - start < WORKER_TIMEOUT_S:
+        conn = kb.connect()
+        try:
+            # Find any ready task (non-deterministic order intentional — we
+            # want workers to race on popular assignees).
+            row = conn.execute(
+                "SELECT id FROM tasks WHERE status = 'ready' "
+                "AND claim_lock IS NULL LIMIT 1"
+            ).fetchone()
+            if row is None:
+                empty_polls += 1
+                if empty_polls > 20:
+                    break  # queue empty long enough, stop
+                time.sleep(0.01)
+                continue
+            empty_polls = 0
+
+            tid = row["id"]
+            try:
+                claimed = kb.claim_task(
+                    conn, tid, claimer=f"worker-{worker_id}",
+                )
+            except sqlite3.OperationalError as e:
+                events.append({"kind": "sqlite_err_on_claim", "task": tid, "err": str(e)})
+                continue
+            if claimed is None:
+                # Someone else beat us — expected contention, not an error.
+                events.append({"kind": "lost_claim_race", "task": tid})
+                continue
+
+            run = kb.latest_run(conn, tid)
+            events.append({
+                "kind": "claimed",
+                "task": tid,
+                "worker": worker_id,
+                "run_id": run.id,
+                "t": time.monotonic() - start,
+            })
+
+            # Simulate short, variable work
+            time.sleep(random.uniform(0.001, 0.05))
+
+            try:
+                kb.complete_task(
+                    conn, tid,
+                    result=f"done by worker-{worker_id}",
+                    summary=f"worker-{worker_id} finished task {tid}",
+                    metadata={"worker_id": worker_id, "run_id": run.id},
+                )
+            except sqlite3.OperationalError as e:
+                events.append({"kind": "sqlite_err_on_complete", "task": tid, "err": str(e)})
+                continue
+            events.append({
+                "kind": "completed",
+                "task": tid,
+                "worker": worker_id,
+                "run_id": run.id,
+                "t": time.monotonic() - start,
+            })
+        finally:
+            conn.close()
+
+    with open(result_file, "w") as f:
+        json.dump(events, f)
+
+
+def main():
+    home = tempfile.mkdtemp(prefix="hermes_concurrency_")
+    print(f"HERMES_HOME = {home}")
+
+    # Seed.
+    os.environ["HERMES_HOME"] = home
+    os.environ["HOME"] = home
+    sys.path.insert(0, WT)
+    from hermes_cli import kanban_db as kb
+
+    kb.init_db()
+    conn = kb.connect()
+    tids = []
+    for i in range(NUM_TASKS):
+        tid = kb.create_task(
+            conn, title=f"task #{i}", assignee="shared",
+            tenant="concurrency-test",
+        )
+        tids.append(tid)
+    conn.close()
+    print(f"Seeded {NUM_TASKS} tasks.")
+
+    # Spawn workers.
+    ctx = mp.get_context("spawn")
+    result_files = [f"/tmp/concurrency_worker_{i}.json" for i in range(NUM_WORKERS)]
+    procs = []
+    start = time.monotonic()
+    for i in range(NUM_WORKERS):
+        p = ctx.Process(target=worker_loop, args=(i, home, result_files[i]))
+        p.start()
+        procs.append(p)
+
+    for p in procs:
+        p.join(timeout=WORKER_TIMEOUT_S + 30)
+        if p.is_alive():
+            p.terminate()
+            p.join()
+
+    elapsed = time.monotonic() - start
+    print(f"All workers done in {elapsed:.1f}s")
+
+    # Aggregate worker events.
+    all_events = []
+    for i, f in enumerate(result_files):
+        if not os.path.isfile(f):
+            print(f"  WORKER {i} produced no result file — died?")
+            continue
+        with open(f) as fh:
+            events = json.load(fh)
+        all_events.extend(events)
+
+    # ============ INVARIANT CHECKS ============
+    print()
+    print("=" * 60)
+    print("INVARIANT CHECKS")
+    print("=" * 60)
+
+    failures = []
+
+    # Check 1: no task claimed by two different workers
+    claims_by_task = {}
+    for e in all_events:
+        if e["kind"] == "claimed":
+            if e["task"] in claims_by_task:
+                prev = claims_by_task[e["task"]]
+                if prev["worker"] != e["worker"]:
+                    failures.append(
+                        f"DOUBLE CLAIM: task {e['task']} claimed by "
+                        f"worker {prev['worker']} AND worker {e['worker']}"
+                    )
+            claims_by_task[e["task"]] = e
+
+    # Check 2: every completion has a matching claim from the same worker
+    for e in all_events:
+        if e["kind"] == "completed":
+            prev_claim = claims_by_task.get(e["task"])
+            if prev_claim is None:
+                failures.append(f"COMPLETION WITHOUT CLAIM: task {e['task']}")
+            elif prev_claim["worker"] != e["worker"]:
+                failures.append(
+                    f"WORKER MISMATCH: task {e['task']} claimed by "
+                    f"{prev_claim['worker']} but completed by {e['worker']}"
+                )
+
+    # Check 3: DB state — every task should be in 'done', no dangling claims
+    conn = kb.connect()
+    try:
+        bad_status = conn.execute(
+            "SELECT id, status, claim_lock, current_run_id FROM tasks "
+            "WHERE status != 'done' OR claim_lock IS NOT NULL "
+            "OR current_run_id IS NOT NULL"
+        ).fetchall()
+        if bad_status:
+            for row in bad_status:
+                failures.append(
+                    f"BAD FINAL STATE: task {row['id']} status={row['status']} "
+                    f"claim_lock={row['claim_lock']} current_run_id={row['current_run_id']}"
+                )
+
+        # Check 4: exactly one run per task, all closed as completed
+        bad_runs = conn.execute(
+            "SELECT task_id, COUNT(*) as n FROM task_runs "
+            "GROUP BY task_id HAVING n != 1"
+        ).fetchall()
+        if bad_runs:
+            for row in bad_runs:
+                failures.append(
+                    f"WRONG RUN COUNT: task {row['task_id']} has {row['n']} runs (expected 1)"
+                )
+
+        open_runs = conn.execute(
+            "SELECT id, task_id FROM task_runs WHERE ended_at IS NULL"
+        ).fetchall()
+        for row in open_runs:
+            failures.append(f"OPEN RUN: run {row['id']} on task {row['task_id']}")
+
+        wrong_outcomes = conn.execute(
+            "SELECT task_id, outcome FROM task_runs "
+            "WHERE outcome IS NULL OR outcome != 'completed'"
+        ).fetchall()
+        for row in wrong_outcomes:
+            failures.append(
+                f"WRONG OUTCOME: task {row['task_id']} run outcome={row['outcome']}"
+            )
+
+        # Check 5: event counts — exactly NUM_TASKS completed events
+        completed_events = conn.execute(
+            "SELECT COUNT(*) as n FROM task_events WHERE kind='completed'"
+        ).fetchone()["n"]
+        if completed_events != NUM_TASKS:
+            failures.append(
+                f"EVENT COUNT MISMATCH: {completed_events} completed events "
+                f"expected {NUM_TASKS}"
+            )
+
+        # Check 6: count SQLite errors that escaped retry
+        sqlite_errs = sum(
+            1 for e in all_events if e["kind"].startswith("sqlite_err")
+        )
+        if sqlite_errs > 0:
+            failures.append(f"UNRETRIED SQLITE ERRORS: {sqlite_errs}")
+
+    finally:
+        conn.close()
+
+    # ============ STATS ============
+    print()
+    total_claims = sum(1 for e in all_events if e["kind"] == "claimed")
+    total_completes = sum(1 for e in all_events if e["kind"] == "completed")
+    total_lost_races = sum(1 for e in all_events if e["kind"] == "lost_claim_race")
+
+    per_worker = {}
+    for e in all_events:
+        if e["kind"] == "completed":
+            per_worker.setdefault(e["worker"], 0)
+            per_worker[e["worker"]] += 1
+
+    print(f"Total claims:      {total_claims}")
+    print(f"Total completes:   {total_completes}")
+    print(f"Lost claim races:  {total_lost_races}  (expected contention; not a bug)")
+    print(f"Elapsed:           {elapsed:.2f}s")
+    print(f"Throughput:        {NUM_TASKS/elapsed:.1f} tasks/sec")
+    print(f"Per-worker completions:")
+    for w in sorted(per_worker.keys()):
+        print(f"  worker-{w}: {per_worker[w]}")
+
+    if failures:
+        print()
+        print("=" * 60)
+        print(f"FAILURES ({len(failures)}):")
+        print("=" * 60)
+        for f in failures[:20]:
+            print(f"  {f}")
+        if len(failures) > 20:
+            print(f"  ... and {len(failures) - 20} more")
+        sys.exit(1)
+    else:
+        print()
+        print("✔ ALL INVARIANTS HELD")
+
+
+if __name__ == "__main__":
+    main()

tests/stress/test_concurrency_mixed.py

@@ -0,0 +1,350 @@
+"""Harder concurrency stress: mixed operations + larger scale.
+
+Scales to 500 tasks, 10 workers, 60s runtime. Each worker randomly:
+  - claims + completes (70%)
+  - claims + blocks with a reason (15%)
+  - unblocks a random blocked task (10%)
+  - archives a random done task (5%)
+
+Adds a background "dispatcher" process that calls release_stale_claims
+and detect_crashed_workers every 200ms, racing against the workers to
+surface TTL + crash detection races.
+
+Pass criteria: runs invariant holds, no double-completions, no orphan
+runs, no SQLite errors escape the retry layer.
+"""
+
+import json
+import multiprocessing as mp
+import os
+import random
+import sqlite3
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+NUM_WORKERS = 10
+NUM_TASKS = 500
+RUN_DURATION_S = 30
+WT = str(Path(__file__).resolve().parents[2])
+
+
+def worker_loop(worker_id: int, hermes_home: str, result_file: str) -> None:
+    os.environ["HERMES_HOME"] = hermes_home
+    os.environ["HOME"] = hermes_home
+    sys.path.insert(0, WT)
+    from hermes_cli import kanban_db as kb
+
+    events = []
+    start = time.monotonic()
+    idle_rounds = 0
+
+    while time.monotonic() - start < RUN_DURATION_S:
+        conn = kb.connect()
+        try:
+            op = random.random()
+
+            if op < 0.10:
+                # Try to unblock a blocked task.
+                row = conn.execute(
+                    "SELECT id FROM tasks WHERE status='blocked' "
+                    "ORDER BY RANDOM() LIMIT 1"
+                ).fetchone()
+                if row:
+                    try:
+                        ok = kb.unblock_task(conn, row["id"])
+                        events.append({"kind": "unblocked" if ok else "unblock_noop",
+                                       "task": row["id"], "worker": worker_id})
+                    except sqlite3.OperationalError as e:
+                        events.append({"kind": "sqlite_err", "op": "unblock",
+                                       "task": row["id"], "err": str(e)[:100]})
+                continue
+
+            if op < 0.15:
+                # Try to archive a done task.
+                row = conn.execute(
+                    "SELECT id FROM tasks WHERE status='done' "
+                    "ORDER BY RANDOM() LIMIT 1"
+                ).fetchone()
+                if row:
+                    try:
+                        kb.archive_task(conn, row["id"])
+                        events.append({"kind": "archived", "task": row["id"],
+                                       "worker": worker_id})
+                    except sqlite3.OperationalError as e:
+                        events.append({"kind": "sqlite_err", "op": "archive",
+                                       "task": row["id"], "err": str(e)[:100]})
+                continue
+
+            # Default: claim + complete-or-block.
+            row = conn.execute(
+                "SELECT id FROM tasks WHERE status='ready' "
+                "AND claim_lock IS NULL LIMIT 1"
+            ).fetchone()
+            if row is None:
+                idle_rounds += 1
+                if idle_rounds > 50:
+                    break
+                time.sleep(0.02)
+                continue
+            idle_rounds = 0
+
+            tid = row["id"]
+            try:
+                claimed = kb.claim_task(
+                    conn, tid, claimer=f"worker-{worker_id}",
+                    ttl_seconds=5,  # short TTL so reclaim races in
+                )
+            except sqlite3.OperationalError as e:
+                events.append({"kind": "sqlite_err", "op": "claim",
+                               "task": tid, "err": str(e)[:100]})
+                continue
+            if claimed is None:
+                events.append({"kind": "lost_claim_race", "task": tid})
+                continue
+
+            run = kb.latest_run(conn, tid)
+            events.append({"kind": "claimed", "task": tid, "worker": worker_id,
+                           "run_id": run.id, "t": time.monotonic() - start})
+
+            time.sleep(random.uniform(0.005, 0.05))
+
+            # 20% of the time, block instead of complete
+            if random.random() < 0.20:
+                try:
+                    kb.block_task(conn, tid,
+                                  reason=f"blocked by worker-{worker_id}")
+                    events.append({"kind": "blocked", "task": tid,
+                                   "worker": worker_id, "run_id": run.id})
+                except sqlite3.OperationalError as e:
+                    events.append({"kind": "sqlite_err", "op": "block",
+                                   "task": tid, "err": str(e)[:100]})
+            else:
+                try:
+                    kb.complete_task(
+                        conn, tid,
+                        result=f"done by worker-{worker_id}",
+                        summary=f"worker-{worker_id} ok",
+                        metadata={"worker_id": worker_id},
+                    )
+                    events.append({"kind": "completed", "task": tid,
+                                   "worker": worker_id, "run_id": run.id,
+                                   "t": time.monotonic() - start})
+                except sqlite3.OperationalError as e:
+                    events.append({"kind": "sqlite_err", "op": "complete",
+                                   "task": tid, "err": str(e)[:100]})
+        finally:
+            conn.close()
+
+    with open(result_file, "w") as f:
+        json.dump(events, f)
+
+
+def reclaimer_loop(hermes_home: str, result_file: str) -> None:
+    """Background dispatcher-like loop that reclaims stale tasks."""
+    os.environ["HERMES_HOME"] = hermes_home
+    os.environ["HOME"] = hermes_home
+    sys.path.insert(0, WT)
+    from hermes_cli import kanban_db as kb
+
+    events = []
+    start = time.monotonic()
+    while time.monotonic() - start < RUN_DURATION_S + 2:
+        conn = kb.connect()
+        try:
+            try:
+                reclaimed = kb.release_stale_claims(conn)
+                if reclaimed:
+                    events.append({"kind": "reclaimed", "count": reclaimed,
+                                   "t": time.monotonic() - start})
+            except sqlite3.OperationalError as e:
+                events.append({"kind": "sqlite_err", "op": "reclaim",
+                               "err": str(e)[:100]})
+        finally:
+            conn.close()
+        time.sleep(0.2)
+
+    with open(result_file, "w") as f:
+        json.dump(events, f)
+
+
+def main():
+    home = tempfile.mkdtemp(prefix="hermes_mixed_stress_")
+    print(f"HERMES_HOME = {home}")
+
+    os.environ["HERMES_HOME"] = home
+    os.environ["HOME"] = home
+    sys.path.insert(0, WT)
+    from hermes_cli import kanban_db as kb
+
+    kb.init_db()
+    conn = kb.connect()
+    for i in range(NUM_TASKS):
+        kb.create_task(
+            conn, title=f"t#{i}", assignee="shared", tenant="mixed-stress",
+        )
+    conn.close()
+    print(f"Seeded {NUM_TASKS} tasks, launching {NUM_WORKERS} workers + 1 reclaimer")
+
+    ctx = mp.get_context("spawn")
+    worker_results = [f"/tmp/mixed_worker_{i}.json" for i in range(NUM_WORKERS)]
+    reclaim_result = "/tmp/mixed_reclaim.json"
+
+    procs = []
+    start = time.monotonic()
+    for i in range(NUM_WORKERS):
+        p = ctx.Process(target=worker_loop, args=(i, home, worker_results[i]))
+        p.start()
+        procs.append(p)
+    r = ctx.Process(target=reclaimer_loop, args=(home, reclaim_result))
+    r.start()
+    procs.append(r)
+
+    for p in procs:
+        p.join(timeout=RUN_DURATION_S + 30)
+        if p.is_alive():
+            p.terminate()
+            p.join()
+
+    elapsed = time.monotonic() - start
+    print(f"Done in {elapsed:.1f}s")
+
+    # Aggregate.
+    all_events = []
+    for i, f in enumerate(worker_results):
+        if os.path.isfile(f):
+            with open(f) as fh:
+                all_events.extend(json.load(fh))
+        else:
+            print(f"  WORKER {i} died with no result file!")
+    reclaim_events = []
+    if os.path.isfile(reclaim_result):
+        with open(reclaim_result) as fh:
+            reclaim_events = json.load(fh)
+
+    # ============ INVARIANT CHECKS ============
+    print()
+    print("=" * 60)
+    print("INVARIANT CHECKS")
+    print("=" * 60)
+
+    failures = []
+
+    # Per-run attribution tracking
+    claims = [e for e in all_events if e["kind"] == "claimed"]
+    completions = [e for e in all_events if e["kind"] == "completed"]
+    blocks = [e for e in all_events if e["kind"] == "blocked"]
+
+    # Every completion must have a matching claim on the same run_id AND
+    # the same worker (workers don't steal each other's runs).
+    claims_by_run = {c["run_id"]: c for c in claims}
+    for comp in completions:
+        claim = claims_by_run.get(comp["run_id"])
+        if claim is None:
+            # It's possible this worker saw a reclaimed run from another worker
+            # — that's still a bug: the worker shouldn't be able to complete
+            # a run it didn't claim. But let me check if reclaim happened first.
+            failures.append(
+                f"COMPLETION WITHOUT CLAIM: task {comp['task']} run {comp['run_id']} "
+                f"by worker {comp['worker']}"
+            )
+        elif claim["worker"] != comp["worker"]:
+            failures.append(
+                f"CROSS-WORKER COMPLETION: run {comp['run_id']} claimed by "
+                f"worker {claim['worker']} but completed by worker {comp['worker']}"
+            )
+
+    # SQLite errors that escaped the retry layer
+    sqlite_errs = [e for e in all_events if e["kind"] == "sqlite_err"]
+    if sqlite_errs:
+        for e in sqlite_errs[:5]:
+            failures.append(f"SQLITE ERROR: op={e.get('op')} err={e.get('err')}")
+        if len(sqlite_errs) > 5:
+            failures.append(f"  ... and {len(sqlite_errs) - 5} more sqlite errs")
+
+    # DB final state — every task should be in a clean terminal state.
+    conn = kb.connect()
+    try:
+        # Invariant: current_run_id NULL iff latest run is terminal
+        inconsistent = conn.execute("""
+            SELECT t.id, t.status, t.current_run_id
+            FROM tasks t
+            WHERE t.current_run_id IS NOT NULL
+              AND EXISTS (SELECT 1 FROM task_runs r
+                          WHERE r.id = t.current_run_id AND r.ended_at IS NOT NULL)
+        """).fetchall()
+        for row in inconsistent:
+            failures.append(
+                f"INVARIANT VIOLATION: task {row['id']} status={row['status']} "
+                f"has current_run_id={row['current_run_id']} but run is ended"
+            )
+
+        # Invariant: no orphan open runs
+        orphans = conn.execute("""
+            SELECT r.id, r.task_id, r.status
+            FROM task_runs r
+            LEFT JOIN tasks t ON t.current_run_id = r.id
+            WHERE r.ended_at IS NULL AND t.id IS NULL
+        """).fetchall()
+        for row in orphans:
+            failures.append(
+                f"ORPHAN OPEN RUN: run {row['id']} on task {row['task_id']}"
+            )
+
+        # Counts — should roughly balance.
+        status_counts = dict(
+            conn.execute("SELECT status, COUNT(*) FROM tasks GROUP BY status").fetchall()
+        )
+        run_outcome_counts = dict(
+            conn.execute(
+                "SELECT outcome, COUNT(*) FROM task_runs "
+                "WHERE ended_at IS NOT NULL GROUP BY outcome"
+            ).fetchall()
+        )
+        active_runs = conn.execute(
+            "SELECT COUNT(*) FROM task_runs WHERE ended_at IS NULL"
+        ).fetchone()[0]
+
+    finally:
+        conn.close()
+
+    # ============ STATS ============
+    print()
+    print(f"Workers: {NUM_WORKERS}, Tasks: {NUM_TASKS}")
+    print(f"Elapsed: {elapsed:.1f}s")
+    print(f"Events collected: {len(all_events)} (+{len(reclaim_events)} reclaim)")
+    print()
+    print("Operations:")
+    op_counts = {}
+    for e in all_events:
+        op_counts[e["kind"]] = op_counts.get(e["kind"], 0) + 1
+    for k in sorted(op_counts.keys()):
+        print(f"  {k:<25} {op_counts[k]}")
+
+    print()
+    print("Final task status:")
+    for s, n in sorted(status_counts.items()):
+        print(f"  {s:<10} {n}")
+    print("Final run outcomes:")
+    for o, n in sorted(run_outcome_counts.items(), key=lambda x: (x[0] or '',)):
+        print(f"  {o:<12} {n}")
+    print(f"  active       {active_runs}")
+
+    if failures:
+        print()
+        print("=" * 60)
+        print(f"FAILURES ({len(failures)}):")
+        print("=" * 60)
+        for f in failures[:30]:
+            print(f"  {f}")
+        if len(failures) > 30:
+            print(f"  ... and {len(failures) - 30} more")
+        sys.exit(1)
+    else:
+        print()
+        print("✔ ALL INVARIANTS HELD UNDER MIXED STRESS")
+
+
+if __name__ == "__main__":
+    main()

tests/stress/test_concurrency_parent_gate.py

@@ -0,0 +1,183 @@
+"""Stress test for parent-completion invariant at the claim gate.
+
+Simulates the create-then-link race described in RCA t_a6acd07d:
+
+  Thread A: repeatedly inserts a child row with status='ready' (racy
+            writer) and a split-second-later inserts the parent link,
+            emulating the pre-fix _kanban_create path.
+  Thread B: repeatedly runs claim_task against every ready task.
+
+Pass criteria: no task is ever 'claimed' while any of its parents is
+not 'done'. The claim_task gate added in hermes_cli/kanban_db.py must
+demote such tasks back to 'todo' and emit a 'claim_rejected' event
+instead of spawning.
+
+Run as a script (`python tests/stress/test_concurrency_parent_gate.py`)
+or via `pytest --run-stress`. The default pytest collection in
+tests/stress/conftest.py ignores *.py globs, so this is a script.
+"""
+from __future__ import annotations
+
+import os
+import random
+import sys
+import tempfile
+import threading
+import time
+from pathlib import Path
+
+WT = str(Path(__file__).resolve().parents[2])
+sys.path.insert(0, WT)
+
+NUM_CREATE_ROUNDS = 200
+WORKERS_RUN_DURATION_S = 8
+
+
+def run() -> int:
+    home = tempfile.mkdtemp(prefix="hermes_parent_gate_stress_")
+    os.environ["HERMES_HOME"] = home
+    os.environ["HOME"] = home
+
+    from hermes_cli import kanban_db as kb
+
+    kb.init_db()
+
+    # Seed N parents in 'ready' state. They stay ready for the whole run
+    # (never 'done'), so every child linked to one of them must remain
+    # unclaimable.
+    parent_ids: list[str] = []
+    conn = kb.connect()
+    try:
+        for i in range(10):
+            parent_ids.append(
+                kb.create_task(conn, title=f"parent-{i}", assignee="a")
+            )
+    finally:
+        conn.close()
+
+    created_children: list[str] = []
+    created_lock = threading.Lock()
+    stop = threading.Event()
+    violations: list[str] = []
+
+    def racy_creator() -> None:
+        """Inserts child rows with status='ready' and links them after.
+
+        This is the pre-fix _kanban_create behavior — the very race
+        the gate in claim_task must catch.
+        """
+        conn = kb.connect()
+        try:
+            for _ in range(NUM_CREATE_ROUNDS):
+                if stop.is_set():
+                    return
+                parents = random.sample(parent_ids, k=2)
+                # Step 1: insert child WITHOUT parents (ends up ready).
+                child = kb.create_task(
+                    conn, title="child", assignee="a", parents=[],
+                )
+                # Tiny delay so worker threads get a chance to see the
+                # ready row before the links are inserted.
+                time.sleep(random.uniform(0.0001, 0.002))
+                # Step 2: add the parent links after the fact.
+                for p in parents:
+                    try:
+                        kb.link_tasks(conn, parent_id=p, child_id=child)
+                    except Exception:
+                        pass
+                with created_lock:
+                    created_children.append(child)
+        finally:
+            conn.close()
+
+    def worker_loop() -> None:
+        conn = kb.connect()
+        try:
+            end = time.monotonic() + WORKERS_RUN_DURATION_S
+            while time.monotonic() < end and not stop.is_set():
+                row = conn.execute(
+                    "SELECT id FROM tasks WHERE status='ready' "
+                    "AND claim_lock IS NULL ORDER BY RANDOM() LIMIT 1"
+                ).fetchone()
+                if row is None:
+                    time.sleep(0.002)
+                    continue
+                tid = row["id"]
+                try:
+                    claimed = kb.claim_task(conn, tid, claimer="w")
+                except Exception:
+                    continue
+                if claimed is None:
+                    continue
+                # Invariant: a successful claim on `tid` must mean all
+                # parents are 'done'. Check in the same connection txn
+                # so we see the post-claim state.
+                undone = conn.execute(
+                    "SELECT l.parent_id, p.status FROM task_links l "
+                    "JOIN tasks p ON p.id = l.parent_id "
+                    "WHERE l.child_id = ? AND p.status != 'done'",
+                    (tid,),
+                ).fetchall()
+                if undone:
+                    violations.append(
+                        f"claimed {tid} while parents not done: "
+                        + ",".join(f"{r['parent_id']}={r['status']}" for r in undone)
+                    )
+                # Release so the run doesn't leak and the next round sees ready.
+                kb.complete_task(conn, tid, result="stress-ok")
+        finally:
+            conn.close()
+
+    creator = threading.Thread(target=racy_creator, daemon=True)
+    workers = [threading.Thread(target=worker_loop, daemon=True)
+               for _ in range(4)]
+    creator.start()
+    for w in workers:
+        w.start()
+    creator.join()
+    # Give the workers a chance to fully drain ready rows before we stop.
+    time.sleep(0.5)
+    stop.set()
+    for w in workers:
+        w.join(timeout=WORKERS_RUN_DURATION_S + 2)
+
+    # Post-run audit: the DB event log must show no 'claimed' event on any
+    # task whose parents were not 'done' at the time of the claim.
+    conn = kb.connect()
+    try:
+        bad = conn.execute(
+            """
+            WITH claims AS (
+              SELECT task_id, created_at AS t
+              FROM task_events WHERE kind='claimed'
+            )
+            SELECT c.task_id, l.parent_id, p.status, p.completed_at
+            FROM claims c
+            JOIN task_links l ON l.child_id = c.task_id
+            JOIN tasks p ON p.id = l.parent_id
+            WHERE p.completed_at IS NULL OR p.completed_at > c.t
+            """
+        ).fetchall()
+        rejections = conn.execute(
+            "SELECT COUNT(*) FROM task_events WHERE kind='claim_rejected'"
+        ).fetchone()[0]
+    finally:
+        conn.close()
+
+    print(f"children created:  {len(created_children)}")
+    print(f"violations:        {len(violations)}")
+    print(f"event-log bad:     {len(bad)}")
+    print(f"claim_rejected:    {rejections}")
+
+    if violations or bad:
+        for v in violations[:10]:
+            print("  VIOLATION:", v)
+        for row in list(bad)[:10]:
+            print("  EVENT-LOG BAD:", dict(row))
+        return 1
+    print("PARENT-GATE INVARIANT HELD UNDER RACE")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(run())

tests/stress/test_concurrency_reclaim_race.py

@@ -0,0 +1,241 @@
+"""Target the reclaim race specifically.
+
+Workers claim tasks with a 1s TTL but sleep 2s before completing. The
+reclaimer runs every 200ms. Scenario: worker claims, reclaimer expires
+the claim mid-work, worker tries to complete AFTER its run has been
+reclaimed.
+
+Expected behavior (per design): the worker's complete_task should
+either succeed on the reclaimed-and-re-claimed-by-another-worker case
+(no, it should refuse — the claim was invalidated), OR succeed by
+grace (we "forgive" a late complete from the original worker if no
+one else picked it up).
+
+Actually looking at complete_task: it doesn't check claim_lock. It just
+transitions from 'running' -> 'done'. So if the reclaimer moved it back
+to 'ready', the late worker's complete_task will fail (CAS on
+status='running' fails). This is the CORRECT behavior.
+
+Invariant being tested: race between worker.complete and
+dispatcher.reclaim must not produce a double-run-close or other
+inconsistency.
+"""
+
+import json
+import multiprocessing as mp
+import os
+import random
+import sqlite3
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+NUM_WORKERS = 5
+NUM_TASKS = 50
+TTL = 1
+WORK_DURATION_S = 2.0  # longer than TTL => reclaimer wins
+WT = str(Path(__file__).resolve().parents[2])
+
+
+def worker_loop(worker_id: int, hermes_home: str, result_file: str) -> None:
+    os.environ["HERMES_HOME"] = hermes_home
+    os.environ["HOME"] = hermes_home
+    sys.path.insert(0, WT)
+    from hermes_cli import kanban_db as kb
+
+    events = []
+    start = time.monotonic()
+    idle = 0
+
+    while time.monotonic() - start < 40:
+        conn = kb.connect()
+        try:
+            row = conn.execute(
+                "SELECT id FROM tasks WHERE status='ready' AND claim_lock IS NULL LIMIT 1"
+            ).fetchone()
+            if row is None:
+                idle += 1
+                if idle > 30:
+                    break
+                time.sleep(0.05)
+                continue
+            idle = 0
+            tid = row["id"]
+            try:
+                claimed = kb.claim_task(conn, tid, claimer=f"worker-{worker_id}",
+                                        ttl_seconds=TTL)
+            except sqlite3.OperationalError as e:
+                events.append({"kind": "sqlite_err", "op": "claim", "err": str(e)[:100]})
+                continue
+            if claimed is None:
+                events.append({"kind": "lost_claim", "task": tid})
+                continue
+            run = kb.latest_run(conn, tid)
+            events.append({"kind": "claimed", "task": tid, "worker": worker_id,
+                           "run_id": run.id})
+
+            # Sleep longer than TTL so reclaimer has a chance to intervene
+            time.sleep(WORK_DURATION_S + random.uniform(-0.3, 0.3))
+
+            try:
+                ok = kb.complete_task(
+                    conn, tid,
+                    result=f"by worker-{worker_id}",
+                    summary=f"worker-{worker_id} finished",
+                )
+                events.append({"kind": "complete_ok" if ok else "complete_refused",
+                               "task": tid, "worker": worker_id, "run_id": run.id})
+            except sqlite3.OperationalError as e:
+                events.append({"kind": "sqlite_err", "op": "complete", "err": str(e)[:100]})
+        finally:
+            conn.close()
+
+    with open(result_file, "w") as f:
+        json.dump(events, f)
+
+
+def reclaimer_loop(hermes_home: str, result_file: str) -> None:
+    os.environ["HERMES_HOME"] = hermes_home
+    os.environ["HOME"] = hermes_home
+    sys.path.insert(0, WT)
+    from hermes_cli import kanban_db as kb
+
+    events = []
+    start = time.monotonic()
+    while time.monotonic() - start < 42:
+        conn = kb.connect()
+        try:
+            try:
+                n = kb.release_stale_claims(conn)
+                if n:
+                    events.append({"kind": "reclaimed", "count": n,
+                                   "t": time.monotonic() - start})
+            except sqlite3.OperationalError as e:
+                events.append({"kind": "sqlite_err", "err": str(e)[:100]})
+        finally:
+            conn.close()
+        time.sleep(0.2)
+    with open(result_file, "w") as f:
+        json.dump(events, f)
+
+
+def main():
+    home = tempfile.mkdtemp(prefix="hermes_reclaim_race_")
+    os.environ["HERMES_HOME"] = home
+    os.environ["HOME"] = home
+    sys.path.insert(0, WT)
+    from hermes_cli import kanban_db as kb
+
+    kb.init_db()
+    conn = kb.connect()
+    for i in range(NUM_TASKS):
+        kb.create_task(conn, title=f"t{i}", assignee="shared",
+                       tenant="reclaim-race")
+    conn.close()
+    print(f"Seeded {NUM_TASKS} tasks. TTL={TTL}s, work_duration={WORK_DURATION_S}s")
+    print(f"(worker work > TTL guarantees reclaims)")
+
+    ctx = mp.get_context("spawn")
+    worker_results = [f"/tmp/rc_worker_{i}.json" for i in range(NUM_WORKERS)]
+    reclaim_result = "/tmp/rc_reclaim.json"
+    procs = []
+    for i in range(NUM_WORKERS):
+        p = ctx.Process(target=worker_loop, args=(i, home, worker_results[i]))
+        p.start()
+        procs.append(p)
+    r = ctx.Process(target=reclaimer_loop, args=(home, reclaim_result))
+    r.start()
+    procs.append(r)
+
+    for p in procs:
+        p.join(timeout=60)
+        if p.is_alive():
+            p.terminate()
+            p.join()
+
+    # Aggregate.
+    all_events = []
+    for f in worker_results:
+        if os.path.isfile(f):
+            with open(f) as fh:
+                all_events.extend(json.load(fh))
+    reclaim_events = []
+    if os.path.isfile(reclaim_result):
+        with open(reclaim_result) as fh:
+            reclaim_events = json.load(fh)
+
+    op_counts = {}
+    for e in all_events:
+        op_counts[e["kind"]] = op_counts.get(e["kind"], 0) + 1
+    total_reclaims = sum(e.get("count", 0) for e in reclaim_events)
+    print(f"\nReclaimer fired {len(reclaim_events)} times, total tasks reclaimed: {total_reclaims}")
+    print("Worker events:")
+    for k in sorted(op_counts):
+        print(f"  {k:<25} {op_counts[k]}")
+
+    # Invariant checks
+    failures = []
+    conn = kb.connect()
+    try:
+        # Any task stuck with current_run_id pointing at a closed run?
+        bad = conn.execute("""
+            SELECT t.id, t.status, t.current_run_id, r.ended_at, r.outcome
+            FROM tasks t
+            JOIN task_runs r ON r.id = t.current_run_id
+            WHERE r.ended_at IS NOT NULL
+        """).fetchall()
+        for row in bad:
+            failures.append(
+                f"INVARIANT VIOLATION: task {row['id']} status={row['status']} "
+                f"current_run_id={row['current_run_id']} but run ended "
+                f"outcome={row['outcome']}"
+            )
+        # Every run with NULL ended_at should still have the task pointing at it
+        orphans = conn.execute("""
+            SELECT r.id, r.task_id
+            FROM task_runs r
+            LEFT JOIN tasks t ON t.current_run_id = r.id
+            WHERE r.ended_at IS NULL AND t.id IS NULL
+        """).fetchall()
+        for row in orphans:
+            failures.append(f"ORPHAN OPEN RUN: run {row['id']} on task {row['task_id']}")
+        # Event counts
+        claim_evts = conn.execute(
+            "SELECT COUNT(*) FROM task_events WHERE kind='claimed'").fetchone()[0]
+        reclaim_evts = conn.execute(
+            "SELECT COUNT(*) FROM task_events WHERE kind='reclaimed'").fetchone()[0]
+        comp_evts = conn.execute(
+            "SELECT COUNT(*) FROM task_events WHERE kind='completed'").fetchone()[0]
+        print(f"\nDB event counts: claimed={claim_evts} reclaimed={reclaim_evts} completed={comp_evts}")
+        # Every reclaimed run must have ended_at set
+        unended_reclaims = conn.execute(
+            "SELECT COUNT(*) FROM task_runs WHERE outcome='reclaimed' AND ended_at IS NULL"
+        ).fetchone()[0]
+        if unended_reclaims:
+            failures.append(f"UNENDED RECLAIMED RUNS: {unended_reclaims}")
+        # Count of completed runs
+        comp_runs = conn.execute(
+            "SELECT COUNT(*) FROM task_runs WHERE outcome='completed'"
+        ).fetchone()[0]
+        reclaim_runs = conn.execute(
+            "SELECT COUNT(*) FROM task_runs WHERE outcome='reclaimed'"
+        ).fetchone()[0]
+        print(f"DB run outcomes: completed={comp_runs} reclaimed={reclaim_runs}")
+    finally:
+        conn.close()
+
+    if reclaim_runs == 0:
+        failures.append("NO RECLAIMS HAPPENED — test didn't stress what it was supposed to")
+
+    if failures:
+        print(f"\nFAILURES ({len(failures)}):")
+        for f in failures[:20]:
+            print(f"  {f}")
+        sys.exit(1)
+    else:
+        print("\n✔ RECLAIM RACE INVARIANTS HELD")
+
+
+if __name__ == "__main__":
+    main()

tests/stress/test_property_fuzzing.py

@@ -0,0 +1,282 @@
+"""Randomized property testing for the Kanban kernel.
+
+Generates 1000 random operation sequences, each 20-50 ops, on small
+task graphs. After each step, checks the full invariant set:
+
+  I1. If tasks.current_run_id IS NOT NULL, the run MUST exist AND
+      ended_at MUST be NULL (we never point at a closed run).
+  I2. If a run has ended_at NULL, SOME task MUST have current_run_id
+      pointing at it (no orphan open runs).
+  I3. task.status in the valid set {triage, todo, ready, running,
+      blocked, done, archived}.
+  I4. task.claim_lock NULL iff status not in (running,).
+  I5. Every run has started_at <= ended_at (or ended_at is NULL).
+  I6. If outcome is set, ended_at must also be set.
+  I7. Events are strictly monotonic in (created_at, id).
+  I8. task_events.run_id references a task_runs.id that exists
+      (or is NULL).
+  I9. Parent completion invariant: if all parents are 'done', the
+      child cannot be in 'todo' status (recompute_ready should have
+      promoted it). This is called out in the comment on
+      recompute_ready; verify it holds after every random seq.
+
+Not using hypothesis the lib; just Python random for simplicity.
+"""
+
+import os
+import random
+import sys
+import tempfile
+from pathlib import Path
+
+WT = str(Path(__file__).resolve().parents[2])
+NUM_SEQUENCES = 500
+OPS_PER_SEQUENCE = 100
+TASK_POOL = 10
+
+OPS = [
+    "create", "create_child", "claim", "complete", "block", "unblock",
+    "archive", "heartbeat", "release_stale", "detect_crashed",
+    "recompute_ready", "reassign",
+]
+
+
+def assert_invariants(conn, kb, ops_log):
+    """Run all invariant checks; raise AssertionError with context on any."""
+    failures = []
+
+    # I1: current_run_id → run exists and not ended
+    bad_ptr = conn.execute("""
+        SELECT t.id, t.current_run_id, r.ended_at, r.outcome
+        FROM tasks t
+        LEFT JOIN task_runs r ON r.id = t.current_run_id
+        WHERE t.current_run_id IS NOT NULL
+          AND (r.id IS NULL OR r.ended_at IS NOT NULL)
+    """).fetchall()
+    for row in bad_ptr:
+        if row["ended_at"] is None and row["outcome"] is None:
+            detail = "missing"
+        else:
+            detail = f"closed ({row['outcome']})"
+        failures.append(
+            f"I1: task {row['id']} points at run {row['current_run_id']} "
+            f"which is {detail}"
+        )
+
+    # I2: open run → some task points at it
+    orphans = conn.execute("""
+        SELECT r.id, r.task_id
+        FROM task_runs r
+        WHERE r.ended_at IS NULL
+          AND NOT EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id)
+    """).fetchall()
+    for row in orphans:
+        failures.append(f"I2: open run {row['id']} on task {row['task_id']} has no pointer")
+
+    # I3: valid statuses
+    valid = {"triage", "todo", "ready", "running", "blocked", "done", "archived"}
+    bad_status = conn.execute("SELECT id, status FROM tasks").fetchall()
+    for row in bad_status:
+        if row["status"] not in valid:
+            failures.append(f"I3: task {row['id']} has invalid status {row['status']!r}")
+
+    # I4: claim_lock set only when running
+    bad_lock = conn.execute("""
+        SELECT id, status, claim_lock FROM tasks
+        WHERE (status != 'running' AND claim_lock IS NOT NULL)
+    """).fetchall()
+    for row in bad_lock:
+        failures.append(
+            f"I4: task {row['id']} status={row['status']} but claim_lock={row['claim_lock']!r}"
+        )
+
+    # I5: run started_at <= ended_at
+    bad_times = conn.execute("""
+        SELECT id, started_at, ended_at FROM task_runs
+        WHERE ended_at IS NOT NULL AND started_at > ended_at
+    """).fetchall()
+    for row in bad_times:
+        failures.append(
+            f"I5: run {row['id']} started_at={row['started_at']} > ended_at={row['ended_at']}"
+        )
+
+    # I6: outcome set → ended_at set
+    bad_outcome = conn.execute("""
+        SELECT id, outcome, ended_at FROM task_runs
+        WHERE outcome IS NOT NULL AND ended_at IS NULL
+    """).fetchall()
+    for row in bad_outcome:
+        failures.append(f"I6: run {row['id']} outcome={row['outcome']} but ended_at NULL")
+
+    # I7: events monotonic in id (always true for autoincrement)
+    # Skip — autoincrement guarantees it.
+
+    # I8: event.run_id references existing run
+    bad_ev_fk = conn.execute("""
+        SELECT e.id, e.run_id FROM task_events e
+        LEFT JOIN task_runs r ON r.id = e.run_id
+        WHERE e.run_id IS NOT NULL AND r.id IS NULL
+    """).fetchall()
+    for row in bad_ev_fk:
+        failures.append(f"I8: event {row['id']} references missing run {row['run_id']}")
+
+    # I9: if all parents done → child not in todo
+    # (Only applies to children with at least one parent)
+    orphaned_todo = conn.execute("""
+        SELECT c.id AS child_id,
+               COUNT(*) AS n_parents,
+               SUM(CASE WHEN p.status = 'done' THEN 1 ELSE 0 END) AS done_parents
+        FROM tasks c
+        JOIN task_links l ON l.child_id = c.id
+        JOIN tasks p ON p.id = l.parent_id
+        WHERE c.status = 'todo'
+        GROUP BY c.id
+        HAVING n_parents > 0 AND n_parents = done_parents
+    """).fetchall()
+    for row in orphaned_todo:
+        failures.append(
+            f"I9: task {row['child_id']} is todo but all {row['n_parents']} parents are done"
+        )
+
+    if failures:
+        print(f"\n!!! INVARIANT VIOLATION after {len(ops_log)} ops:")
+        for f in failures[:10]:
+            print(f"  {f}")
+        if len(failures) > 10:
+            print(f"  ... and {len(failures) - 10} more")
+        print("\nLast 10 ops:")
+        for op in ops_log[-10:]:
+            print(f"  {op}")
+        return False
+    return True
+
+
+def random_op(rng, conn, kb, task_pool):
+    op = rng.choice(OPS)
+
+    if op == "create":
+        tid = kb.create_task(
+            conn,
+            title=f"rand {rng.randint(0, 1000)}",
+            assignee=rng.choice(["w1", "w2", "w3", None]),
+        )
+        task_pool.append(tid)
+        return {"op": "create", "tid": tid}
+
+    if op == "create_child" and task_pool:
+        parent = rng.choice(task_pool)
+        tid = kb.create_task(
+            conn, title=f"child of {parent}",
+            assignee=rng.choice(["w1", "w2", "w3", None]),
+            parents=[parent],
+        )
+        task_pool.append(tid)
+        return {"op": "create_child", "tid": tid, "parent": parent}
+
+    if not task_pool:
+        return None
+
+    tid = rng.choice(task_pool)
+    task = kb.get_task(conn, tid)
+    if task is None:
+        task_pool.remove(tid)
+        return None
+
+    if op == "claim":
+        claimed = kb.claim_task(conn, tid, ttl_seconds=rng.choice([1, 3, 10]))
+        return {"op": "claim", "tid": tid, "ok": claimed is not None}
+    if op == "complete":
+        summary = rng.choice([None, f"done via op {rng.randint(0, 1000)}"])
+        ok = kb.complete_task(conn, tid, summary=summary)
+        return {"op": "complete", "tid": tid, "ok": ok}
+    if op == "block":
+        reason = rng.choice([None, "rand block"])
+        ok = kb.block_task(conn, tid, reason=reason)
+        return {"op": "block", "tid": tid, "ok": ok}
+    if op == "unblock":
+        ok = kb.unblock_task(conn, tid)
+        return {"op": "unblock", "tid": tid, "ok": ok}
+    if op == "archive":
+        ok = kb.archive_task(conn, tid)
+        if ok:
+            task_pool.remove(tid)
+        return {"op": "archive", "tid": tid, "ok": ok}
+    if op == "heartbeat":
+        ok = kb.heartbeat_worker(conn, tid)
+        return {"op": "heartbeat", "tid": tid, "ok": ok}
+    if op == "release_stale":
+        n = kb.release_stale_claims(conn)
+        return {"op": "release_stale", "n": n}
+    if op == "detect_crashed":
+        # Force-kill a fake PID first so there's something to detect
+        crashed = kb.detect_crashed_workers(conn)
+        return {"op": "detect_crashed", "n": len(crashed)}
+    if op == "recompute_ready":
+        n = kb.recompute_ready(conn)
+        return {"op": "recompute_ready", "promoted": n}
+    if op == "reassign":
+        # Reassignment isn't a direct API; simulate via assign_task
+        new_a = rng.choice(["w1", "w2", "w3", None])
+        try:
+            kb.assign_task(conn, tid, new_a)
+            return {"op": "reassign", "tid": tid, "to": new_a}
+        except Exception as e:
+            return {"op": "reassign", "tid": tid, "err": str(e)[:50]}
+
+    return None
+
+
+def main():
+    total_ops = 0
+    total_violations = 0
+
+    for seq_idx in range(NUM_SEQUENCES):
+        seed = random.randint(0, 10**9)
+        rng = random.Random(seed)
+        home = tempfile.mkdtemp(prefix=f"hermes_fuzz_{seq_idx}_")
+        os.environ["HERMES_HOME"] = home
+        os.environ["HOME"] = home
+        sys.path.insert(0, WT)
+
+        # Fresh module state per sequence to avoid cached init paths.
+        for m in list(sys.modules.keys()):
+            if m.startswith("hermes_cli"):
+                del sys.modules[m]
+        from hermes_cli import kanban_db as kb
+
+        kb.init_db()
+        conn = kb.connect()
+        task_pool = []
+        ops_log = []
+
+        try:
+            for i in range(OPS_PER_SEQUENCE):
+                result = random_op(rng, conn, kb, task_pool)
+                if result is None:
+                    continue
+                ops_log.append(result)
+                total_ops += 1
+                if not assert_invariants(conn, kb, ops_log):
+                    total_violations += 1
+                    print(f"  sequence {seq_idx} (seed={seed}) failed at op {i}")
+                    break
+        finally:
+            conn.close()
+
+        if seq_idx % 10 == 0:
+            print(f"  seq {seq_idx:3d}: {total_ops} ops so far, {total_violations} violations")
+
+    print()
+    print("=" * 60)
+    print(f"Total sequences: {NUM_SEQUENCES}")
+    print(f"Total operations: {total_ops}")
+    print(f"Invariant violations: {total_violations}")
+    if total_violations == 0:
+        print("\n✔ ALL INVARIANTS HELD ACROSS RANDOMIZED SEQUENCES")
+    else:
+        print("\n✗ INVARIANT VIOLATIONS FOUND")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tests/stress/test_subprocess_e2e.py

@@ -0,0 +1,228 @@
+"""E2E: dispatcher spawns real Python subprocess workers.
+
+This validates the IPC + lifecycle story that mocks can't:
+  - spawn_fn returns a real PID
+  - the child process resolves hermes_cli.kanban_db on its own
+  - the child writes heartbeats via the CLI (real argparse, real init_db)
+  - the child completes via the CLI with --summary + --metadata
+  - the dispatcher observes all of this through the DB only
+  - worker logs are captured to HERMES_HOME/kanban/logs/<task>.log
+  - crash detection works against a real dead PID
+"""
+
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+
+WT = str(Path(__file__).resolve().parents[2])
+FAKE_WORKER = str(Path(__file__).parent / "_fake_worker.py")
+PY = sys.executable
+
+
+def make_spawn_fn(home: str):
+    """Return a spawn_fn the dispatcher can call. Launches the fake
+    worker as a detached subprocess."""
+
+    def _spawn(task, workspace):
+        log_path = os.path.join(home, f"worker_{task.id}.log")
+        env = {
+            **os.environ,
+            "HERMES_HOME": home,
+            "HOME": home,
+            "PYTHONPATH": WT,
+            "HERMES_KANBAN_TASK": task.id,
+            "HERMES_KANBAN_WORKSPACE": workspace,
+            "PATH": f"{os.path.dirname(PY)}:{os.environ.get('PATH','')}",
+        }
+        log_f = open(log_path, "ab")
+        proc = subprocess.Popen(
+            [PY, FAKE_WORKER],
+            stdin=subprocess.DEVNULL,
+            stdout=log_f,
+            stderr=subprocess.STDOUT,
+            env=env,
+            start_new_session=True,
+        )
+        return proc.pid
+
+    return _spawn
+
+
+def main():
+    home = tempfile.mkdtemp(prefix="hermes_e2e_")
+    os.environ["HERMES_HOME"] = home
+    os.environ["HOME"] = home
+    sys.path.insert(0, WT)
+    from hermes_cli import kanban_db as kb
+
+    # Point the `hermes` CLI child processes will run at the worktree
+    # hermes_cli.main. We do this by putting a shim on PATH.
+    shim_dir = os.path.join(home, "bin")
+    os.makedirs(shim_dir, exist_ok=True)
+    shim_path = os.path.join(shim_dir, "hermes")
+    with open(shim_path, "w") as f:
+        f.write(f"""#!/bin/sh
+exec {PY} -m hermes_cli.main "$@"
+""")
+    os.chmod(shim_path, 0o755)
+    os.environ["PATH"] = f"{shim_dir}:{os.environ.get('PATH','')}"
+
+    kb.init_db()
+    conn = kb.connect()
+
+    # ============ SCENARIO A: happy path, 3 tasks ============
+    print("=" * 60)
+    print("A. Real-subprocess happy path (3 tasks)")
+    print("=" * 60)
+
+    tids = []
+    for i in range(3):
+        tid = kb.create_task(
+            conn, title=f"real-e2e-{i}", assignee="default",
+        )
+        tids.append(tid)
+
+    spawn_fn = make_spawn_fn(home)
+    result = kb.dispatch_once(conn, spawn_fn=spawn_fn)
+    print(f"  dispatched: {len(result.spawned)} spawned")
+    spawned_pids = []
+    # The dispatcher sets worker_pid on each claimed task via _set_worker_pid.
+    for tid in tids:
+        task = kb.get_task(conn, tid)
+        spawned_pids.append(task.worker_pid)
+        print(f"  task {tid}: pid={task.worker_pid} status={task.status}")
+
+    # Wait for all workers to complete (up to 10s).
+    deadline = time.monotonic() + 10
+    while time.monotonic() < deadline:
+        statuses = [kb.get_task(conn, tid).status for tid in tids]
+        if all(s == "done" for s in statuses):
+            break
+        time.sleep(0.2)
+
+    print()
+    failures = []
+    for tid in tids:
+        task = kb.get_task(conn, tid)
+        runs = kb.list_runs(conn, tid)
+        print(f"  task {tid}: status={task.status}, current_run_id={task.current_run_id}, "
+              f"runs={[(r.id, r.outcome) for r in runs]}")
+        if task.status != "done":
+            failures.append(f"task {tid} not done: status={task.status}")
+        if task.current_run_id is not None:
+            failures.append(f"task {tid} has dangling current_run_id={task.current_run_id}")
+        if len(runs) != 1:
+            failures.append(f"task {tid} has {len(runs)} runs, expected 1")
+        else:
+            r = runs[0]
+            if r.outcome != "completed":
+                failures.append(f"task {tid} run outcome={r.outcome}, expected completed")
+            if not r.summary or "real-subprocess worker finished" not in r.summary:
+                failures.append(f"task {tid} summary missing: {r.summary!r}")
+            if not r.metadata or r.metadata.get("iterations") != 3:
+                failures.append(f"task {tid} metadata missing iterations: {r.metadata}")
+            # Heartbeat events should be present
+            events = kb.list_events(conn, tid)
+            heartbeats = [e for e in events if e.kind == "heartbeat"]
+            if len(heartbeats) < 3:  # start + 3 progress
+                failures.append(f"task {tid} heartbeats={len(heartbeats)} expected >=3")
+
+    if failures:
+        print("\nFAILURES:")
+        for f in failures:
+            print(f"  {f}")
+        sys.exit(1)
+
+    print("\n  ✔ Scenario A: all 3 real-subprocess workers completed cleanly")
+
+    # ============ SCENARIO B: crashed worker ============
+    print()
+    print("=" * 60)
+    print("B. Crashed worker (kill -9 mid-heartbeat)")
+    print("=" * 60)
+
+    crash_tid = kb.create_task(
+        conn, title="crash-e2e", assignee="default",
+    )
+
+    # Spawn a worker that sleeps long enough for us to kill it.
+    # CRITICAL: spawn through a double-fork so when we kill the child it
+    # doesn't zombify under our pid (which would fool kill -0 liveness
+    # checks into thinking it's still alive). In production the
+    # dispatcher daemon is long-lived but its workers are reaped by init
+    # after exit; the test needs to match that orphaning behavior.
+    def spawn_sleeper(task, workspace):
+        r, w = os.pipe()
+        middleman = subprocess.Popen(
+            [
+                PY, "-c",
+                "import os,sys,subprocess;"
+                "p=subprocess.Popen(['sleep','30'],"
+                "stdin=subprocess.DEVNULL,"
+                "stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,"
+                "start_new_session=True);"
+                "os.write(int(sys.argv[1]), str(p.pid).encode());"
+                "sys.exit(0)",
+                str(w),
+            ],
+            pass_fds=(w,),
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        os.close(w)
+        middleman.wait()  # middleman exits immediately, orphaning the sleep
+        grandchild_pid = int(os.read(r, 16))
+        os.close(r)
+        return grandchild_pid
+
+    result = kb.dispatch_once(conn, spawn_fn=spawn_sleeper)
+    task = kb.get_task(conn, crash_tid)
+    print(f"  spawned sleeper pid={task.worker_pid} for {crash_tid}")
+    # Kill the sleeper forcibly
+    os.kill(task.worker_pid, 9)
+    # Give the OS a moment to reap
+    time.sleep(0.5)
+
+    # Simulate next dispatcher tick — should detect the crashed PID
+    crashed = kb.detect_crashed_workers(conn)
+    print(f"  detect_crashed_workers returned {len(crashed)} crashed (expected 1)")
+
+    task = kb.get_task(conn, crash_tid)
+    runs = kb.list_runs(conn, crash_tid)
+    print(f"  task status={task.status}, runs={[(r.id, r.outcome) for r in runs]}")
+
+    if len(crashed) < 1:
+        print("  ✗ crash NOT detected")
+        sys.exit(1)
+    if task.status != "ready":
+        print(f"  ✗ task should be back to ready, got {task.status}")
+        sys.exit(1)
+    if runs[0].outcome != "crashed":
+        print(f"  ✗ run outcome should be 'crashed', got {runs[0].outcome!r}")
+        sys.exit(1)
+    print("\n  ✔ Scenario B: crash detected, task re-queued, run outcome=crashed")
+
+    # ============ SCENARIO C: worker log was captured ============
+    print()
+    print("=" * 60)
+    print("C. Worker log captured to disk")
+    print("=" * 60)
+    # Scenario A workers wrote to /tmp/hermes_e2e_*/worker_*.log
+    import glob
+    logs = glob.glob(os.path.join(home, "worker_*.log"))
+    print(f"  {len(logs)} worker log files")
+    for lp in logs[:3]:
+        size = os.path.getsize(lp)
+        print(f"    {os.path.basename(lp)}: {size} bytes")
+        # Our fake worker is quiet (no prints); size=0 is fine
+
+    conn.close()
+    print("\n✔ ALL E2E SCENARIOS PASS")
+
+
+if __name__ == "__main__":
+    main()