forked from Zakaria/hermes-agent
Hermes-agent
This commit is contained in:
Zakaria
@@ -0,0 +1,282 @@
|
||||
"""Randomized property testing for the Kanban kernel.
|
||||
|
||||
Generates 1000 random operation sequences, each 20-50 ops, on small
|
||||
task graphs. After each step, checks the full invariant set:
|
||||
|
||||
I1. If tasks.current_run_id IS NOT NULL, the run MUST exist AND
|
||||
ended_at MUST be NULL (we never point at a closed run).
|
||||
I2. If a run has ended_at NULL, SOME task MUST have current_run_id
|
||||
pointing at it (no orphan open runs).
|
||||
I3. task.status in the valid set {triage, todo, ready, running,
|
||||
blocked, done, archived}.
|
||||
I4. task.claim_lock NULL iff status not in (running,).
|
||||
I5. Every run has started_at <= ended_at (or ended_at is NULL).
|
||||
I6. If outcome is set, ended_at must also be set.
|
||||
I7. Events are strictly monotonic in (created_at, id).
|
||||
I8. task_events.run_id references a task_runs.id that exists
|
||||
(or is NULL).
|
||||
I9. Parent completion invariant: if all parents are 'done', the
|
||||
child cannot be in 'todo' status (recompute_ready should have
|
||||
promoted it). This is called out in the comment on
|
||||
recompute_ready; verify it holds after every random seq.
|
||||
|
||||
Not using hypothesis the lib; just Python random for simplicity.
|
||||
"""
|
||||
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
WT = str(Path(__file__).resolve().parents[2])
|
||||
NUM_SEQUENCES = 500
|
||||
OPS_PER_SEQUENCE = 100
|
||||
TASK_POOL = 10
|
||||
|
||||
OPS = [
|
||||
"create", "create_child", "claim", "complete", "block", "unblock",
|
||||
"archive", "heartbeat", "release_stale", "detect_crashed",
|
||||
"recompute_ready", "reassign",
|
||||
]
|
||||
|
||||
|
||||
def assert_invariants(conn, kb, ops_log):
|
||||
"""Run all invariant checks; raise AssertionError with context on any."""
|
||||
failures = []
|
||||
|
||||
# I1: current_run_id → run exists and not ended
|
||||
bad_ptr = conn.execute("""
|
||||
SELECT t.id, t.current_run_id, r.ended_at, r.outcome
|
||||
FROM tasks t
|
||||
LEFT JOIN task_runs r ON r.id = t.current_run_id
|
||||
WHERE t.current_run_id IS NOT NULL
|
||||
AND (r.id IS NULL OR r.ended_at IS NOT NULL)
|
||||
""").fetchall()
|
||||
for row in bad_ptr:
|
||||
if row["ended_at"] is None and row["outcome"] is None:
|
||||
detail = "missing"
|
||||
else:
|
||||
detail = f"closed ({row['outcome']})"
|
||||
failures.append(
|
||||
f"I1: task {row['id']} points at run {row['current_run_id']} "
|
||||
f"which is {detail}"
|
||||
)
|
||||
|
||||
# I2: open run → some task points at it
|
||||
orphans = conn.execute("""
|
||||
SELECT r.id, r.task_id
|
||||
FROM task_runs r
|
||||
WHERE r.ended_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id)
|
||||
""").fetchall()
|
||||
for row in orphans:
|
||||
failures.append(f"I2: open run {row['id']} on task {row['task_id']} has no pointer")
|
||||
|
||||
# I3: valid statuses
|
||||
valid = {"triage", "todo", "ready", "running", "blocked", "done", "archived"}
|
||||
bad_status = conn.execute("SELECT id, status FROM tasks").fetchall()
|
||||
for row in bad_status:
|
||||
if row["status"] not in valid:
|
||||
failures.append(f"I3: task {row['id']} has invalid status {row['status']!r}")
|
||||
|
||||
# I4: claim_lock set only when running
|
||||
bad_lock = conn.execute("""
|
||||
SELECT id, status, claim_lock FROM tasks
|
||||
WHERE (status != 'running' AND claim_lock IS NOT NULL)
|
||||
""").fetchall()
|
||||
for row in bad_lock:
|
||||
failures.append(
|
||||
f"I4: task {row['id']} status={row['status']} but claim_lock={row['claim_lock']!r}"
|
||||
)
|
||||
|
||||
# I5: run started_at <= ended_at
|
||||
bad_times = conn.execute("""
|
||||
SELECT id, started_at, ended_at FROM task_runs
|
||||
WHERE ended_at IS NOT NULL AND started_at > ended_at
|
||||
""").fetchall()
|
||||
for row in bad_times:
|
||||
failures.append(
|
||||
f"I5: run {row['id']} started_at={row['started_at']} > ended_at={row['ended_at']}"
|
||||
)
|
||||
|
||||
# I6: outcome set → ended_at set
|
||||
bad_outcome = conn.execute("""
|
||||
SELECT id, outcome, ended_at FROM task_runs
|
||||
WHERE outcome IS NOT NULL AND ended_at IS NULL
|
||||
""").fetchall()
|
||||
for row in bad_outcome:
|
||||
failures.append(f"I6: run {row['id']} outcome={row['outcome']} but ended_at NULL")
|
||||
|
||||
# I7: events monotonic in id (always true for autoincrement)
|
||||
# Skip — autoincrement guarantees it.
|
||||
|
||||
# I8: event.run_id references existing run
|
||||
bad_ev_fk = conn.execute("""
|
||||
SELECT e.id, e.run_id FROM task_events e
|
||||
LEFT JOIN task_runs r ON r.id = e.run_id
|
||||
WHERE e.run_id IS NOT NULL AND r.id IS NULL
|
||||
""").fetchall()
|
||||
for row in bad_ev_fk:
|
||||
failures.append(f"I8: event {row['id']} references missing run {row['run_id']}")
|
||||
|
||||
# I9: if all parents done → child not in todo
|
||||
# (Only applies to children with at least one parent)
|
||||
orphaned_todo = conn.execute("""
|
||||
SELECT c.id AS child_id,
|
||||
COUNT(*) AS n_parents,
|
||||
SUM(CASE WHEN p.status = 'done' THEN 1 ELSE 0 END) AS done_parents
|
||||
FROM tasks c
|
||||
JOIN task_links l ON l.child_id = c.id
|
||||
JOIN tasks p ON p.id = l.parent_id
|
||||
WHERE c.status = 'todo'
|
||||
GROUP BY c.id
|
||||
HAVING n_parents > 0 AND n_parents = done_parents
|
||||
""").fetchall()
|
||||
for row in orphaned_todo:
|
||||
failures.append(
|
||||
f"I9: task {row['child_id']} is todo but all {row['n_parents']} parents are done"
|
||||
)
|
||||
|
||||
if failures:
|
||||
print(f"\n!!! INVARIANT VIOLATION after {len(ops_log)} ops:")
|
||||
for f in failures[:10]:
|
||||
print(f" {f}")
|
||||
if len(failures) > 10:
|
||||
print(f" ... and {len(failures) - 10} more")
|
||||
print("\nLast 10 ops:")
|
||||
for op in ops_log[-10:]:
|
||||
print(f" {op}")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def random_op(rng, conn, kb, task_pool):
|
||||
op = rng.choice(OPS)
|
||||
|
||||
if op == "create":
|
||||
tid = kb.create_task(
|
||||
conn,
|
||||
title=f"rand {rng.randint(0, 1000)}",
|
||||
assignee=rng.choice(["w1", "w2", "w3", None]),
|
||||
)
|
||||
task_pool.append(tid)
|
||||
return {"op": "create", "tid": tid}
|
||||
|
||||
if op == "create_child" and task_pool:
|
||||
parent = rng.choice(task_pool)
|
||||
tid = kb.create_task(
|
||||
conn, title=f"child of {parent}",
|
||||
assignee=rng.choice(["w1", "w2", "w3", None]),
|
||||
parents=[parent],
|
||||
)
|
||||
task_pool.append(tid)
|
||||
return {"op": "create_child", "tid": tid, "parent": parent}
|
||||
|
||||
if not task_pool:
|
||||
return None
|
||||
|
||||
tid = rng.choice(task_pool)
|
||||
task = kb.get_task(conn, tid)
|
||||
if task is None:
|
||||
task_pool.remove(tid)
|
||||
return None
|
||||
|
||||
if op == "claim":
|
||||
claimed = kb.claim_task(conn, tid, ttl_seconds=rng.choice([1, 3, 10]))
|
||||
return {"op": "claim", "tid": tid, "ok": claimed is not None}
|
||||
if op == "complete":
|
||||
summary = rng.choice([None, f"done via op {rng.randint(0, 1000)}"])
|
||||
ok = kb.complete_task(conn, tid, summary=summary)
|
||||
return {"op": "complete", "tid": tid, "ok": ok}
|
||||
if op == "block":
|
||||
reason = rng.choice([None, "rand block"])
|
||||
ok = kb.block_task(conn, tid, reason=reason)
|
||||
return {"op": "block", "tid": tid, "ok": ok}
|
||||
if op == "unblock":
|
||||
ok = kb.unblock_task(conn, tid)
|
||||
return {"op": "unblock", "tid": tid, "ok": ok}
|
||||
if op == "archive":
|
||||
ok = kb.archive_task(conn, tid)
|
||||
if ok:
|
||||
task_pool.remove(tid)
|
||||
return {"op": "archive", "tid": tid, "ok": ok}
|
||||
if op == "heartbeat":
|
||||
ok = kb.heartbeat_worker(conn, tid)
|
||||
return {"op": "heartbeat", "tid": tid, "ok": ok}
|
||||
if op == "release_stale":
|
||||
n = kb.release_stale_claims(conn)
|
||||
return {"op": "release_stale", "n": n}
|
||||
if op == "detect_crashed":
|
||||
# Force-kill a fake PID first so there's something to detect
|
||||
crashed = kb.detect_crashed_workers(conn)
|
||||
return {"op": "detect_crashed", "n": len(crashed)}
|
||||
if op == "recompute_ready":
|
||||
n = kb.recompute_ready(conn)
|
||||
return {"op": "recompute_ready", "promoted": n}
|
||||
if op == "reassign":
|
||||
# Reassignment isn't a direct API; simulate via assign_task
|
||||
new_a = rng.choice(["w1", "w2", "w3", None])
|
||||
try:
|
||||
kb.assign_task(conn, tid, new_a)
|
||||
return {"op": "reassign", "tid": tid, "to": new_a}
|
||||
except Exception as e:
|
||||
return {"op": "reassign", "tid": tid, "err": str(e)[:50]}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
total_ops = 0
|
||||
total_violations = 0
|
||||
|
||||
for seq_idx in range(NUM_SEQUENCES):
|
||||
seed = random.randint(0, 10**9)
|
||||
rng = random.Random(seed)
|
||||
home = tempfile.mkdtemp(prefix=f"hermes_fuzz_{seq_idx}_")
|
||||
os.environ["HERMES_HOME"] = home
|
||||
os.environ["HOME"] = home
|
||||
sys.path.insert(0, WT)
|
||||
|
||||
# Fresh module state per sequence to avoid cached init paths.
|
||||
for m in list(sys.modules.keys()):
|
||||
if m.startswith("hermes_cli"):
|
||||
del sys.modules[m]
|
||||
from hermes_cli import kanban_db as kb
|
||||
|
||||
kb.init_db()
|
||||
conn = kb.connect()
|
||||
task_pool = []
|
||||
ops_log = []
|
||||
|
||||
try:
|
||||
for i in range(OPS_PER_SEQUENCE):
|
||||
result = random_op(rng, conn, kb, task_pool)
|
||||
if result is None:
|
||||
continue
|
||||
ops_log.append(result)
|
||||
total_ops += 1
|
||||
if not assert_invariants(conn, kb, ops_log):
|
||||
total_violations += 1
|
||||
print(f" sequence {seq_idx} (seed={seed}) failed at op {i}")
|
||||
break
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if seq_idx % 10 == 0:
|
||||
print(f" seq {seq_idx:3d}: {total_ops} ops so far, {total_violations} violations")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print(f"Total sequences: {NUM_SEQUENCES}")
|
||||
print(f"Total operations: {total_ops}")
|
||||
print(f"Invariant violations: {total_violations}")
|
||||
if total_violations == 0:
|
||||
print("\n✔ ALL INVARIANTS HELD ACROSS RANDOMIZED SEQUENCES")
|
||||
else:
|
||||
print("\n✗ INVARIANT VIOLATIONS FOUND")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user