Hermes-agent

2026-06-14 14:30:48 -04:00
commit dac4b88b94
5058 changed files with 1884848 additions and 0 deletions
@@ -0,0 +1,328 @@
+"""Container-restart survives per-profile gateway registrations.
+
+The s6 dynamic scandir at /run/service/ lives on tmpfs and is wiped
+on every container restart. Phase 4 Task 4.0's container_boot module
+ cont-init.d/02-reconcile-profiles regenerate the service slots from
+$HERMES_HOME/profiles/<name>/gateway_state.json on every boot and
+auto-start only those whose last state was `running`.
+
+These tests stand up a container with a named volume, create profiles
+inside it in various gateway states, restart the container, and
+assert the reconciler did the right thing.
+
+Every ``docker exec`` here runs as the unprivileged ``hermes`` user
+(via :func:`docker_exec` / :func:`docker_exec_sh` in conftest); see
+the conftest module docstring.
+"""
+from __future__ import annotations
+
+import subprocess
+import time
+
+import pytest
+
+from tests.docker.conftest import docker_exec, docker_exec_sh
+
+
+def _docker(*args: str, **kw) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["docker", *args],
+        capture_output=True, text=True, timeout=kw.pop("timeout", 60),
+        **kw,
+    )
+
+
+def _exec(container: str, *args: str, timeout: int = 30) -> subprocess.CompletedProcess[str]:
+    return docker_exec(container, *args, timeout=timeout)
+
+
+def _sh(container: str, cmd: str, timeout: int = 30) -> subprocess.CompletedProcess[str]:
+    return docker_exec_sh(container, cmd, timeout=timeout)
+
+
+def _wait_for_path(
+    container: str,
+    path: str,
+    *,
+    kind: str = "f",
+    deadline_s: float = 30.0,
+    interval_s: float = 0.25,
+) -> bool:
+    """Poll `test -<kind> <path>` inside container until success or timeout.
+
+    `kind` is the `test` flag: 'f' for file, 'd' for directory, 'e' for
+    existence. Returns True on success, False on timeout. Strictly
+    better than a fixed `time.sleep()` because:
+
+      * we don't wait the full budget when the path appears early, and
+      * the test fails with a precise "waited N seconds" assertion
+        instead of a confusing one-line failure mid-test when the
+        sleep was too short.
+    """
+    end = time.monotonic() + deadline_s
+    while time.monotonic() < end:
+        r = _sh(container, f"test -{kind} {path}", timeout=5)
+        if r.returncode == 0:
+            return True
+        time.sleep(interval_s)
+    return False
+
+
+def _wait_for_reconcile_log_mention(
+    container: str,
+    profile: str,
+    *,
+    deadline_s: float = 30.0,
+    interval_s: float = 0.25,
+) -> str:
+    """Poll until /opt/data/logs/container-boot.log mentions `profile`.
+
+    Returns the matching log content on success. On timeout, returns
+    the last observed contents so the assertion can render a
+    meaningful diagnostic. The container-boot.log is the explicit
+    signal that the reconciler has finished — much more reliable
+    than a fixed sleep that hopes 8 seconds is enough.
+    """
+    end = time.monotonic() + deadline_s
+    last = ""
+    while time.monotonic() < end:
+        r = _sh(container, "cat /opt/data/logs/container-boot.log", timeout=5)
+        if r.returncode == 0:
+            last = r.stdout
+            if f"profile={profile}" in last:
+                return last
+        time.sleep(interval_s)
+    return last
+
+
+@pytest.fixture
+def restart_container(request, built_image: str):
+    """A long-running container with a named volume so docker restart
+    preserves $HERMES_HOME/profiles/."""
+    safe = request.node.name.replace("[", "_").replace("]", "_")
+    name = f"hermes-restart-{safe}"
+    volume = f"hermes-restart-vol-{safe}"
+    _docker("rm", "-f", name)
+    _docker("volume", "rm", "-f", volume)
+    _docker("volume", "create", volume, timeout=10).check_returncode()
+    r = _docker(
+        "run", "-d", "--name", name,
+        "-v", f"{volume}:/opt/data",
+        built_image, "sleep", "infinity",
+        timeout=30,
+    )
+    r.check_returncode()
+    # Wait for s6 + stage2 + 02-reconcile to publish the boot log so
+    # the test can rely on the default slot being registered before
+    # it starts issuing commands. The reconciler always writes one
+    # 'default' line on every boot (PR #30136 item I1) — that's our
+    # readiness signal.
+    deadline = time.monotonic() + 30.0
+    while time.monotonic() < deadline:
+        r = _docker(
+            "exec", "-u", "hermes", name, "sh", "-c",
+            "cat /opt/data/logs/container-boot.log 2>/dev/null",
+            timeout=5,
+        )
+        if r.returncode == 0 and "profile=default" in r.stdout:
+            break
+        time.sleep(0.25)
+    else:
+        # Defensive: surface a timeout from the fixture itself so the
+        # test failure points at "container never finished cont-init"
+        # rather than mid-test where the symptom would be obscure.
+        raise RuntimeError(
+            f"container {name} did not finish cont-init within 30s"
+        )
+    yield name
+    _docker("rm", "-f", name)
+    _docker("volume", "rm", "-f", volume)
+
+
+def test_running_gateway_survives_container_restart(restart_container: str) -> None:
+    container = restart_container
+
+    # Create the profile + start its gateway. The Phase 4 hooks
+    # register the s6 service slot during create and the dispatch
+    # path brings it up via s6-svc -u.
+    r = _exec(container, "hermes", "profile", "create", "coder")
+    assert r.returncode == 0, f"profile create failed: {r.stderr}"
+
+    r = _exec(container, "hermes", "-p", "coder", "gateway", "start", timeout=60)
+    assert r.returncode == 0, f"gateway start failed: {r.stderr}"
+
+    # Give the service time to actually come up under supervision.
+    deadline = time.monotonic() + 15.0
+    while time.monotonic() < deadline:
+        r = _sh(container, "/command/s6-svstat /run/service/gateway-coder")
+        if r.returncode == 0 and "up " in r.stdout:
+            break
+        time.sleep(0.5)
+    assert "up " in r.stdout, f"gateway never came up pre-restart: {r.stdout!r}"
+
+    # Persist state so the reconciler will treat the slot as 'running'
+    # post-restart. The gateway process itself writes gateway_state.json
+    # via gateway/status.py — but we don't want to wait for or assert
+    # against the live process here; just stamp the file directly to
+    # exercise the reconciler's contract.
+    write_state = (
+        "import json, pathlib; "
+        "p = pathlib.Path('/opt/data/profiles/coder/gateway_state.json'); "
+        "p.write_text(json.dumps({'gateway_state': 'running', 'timestamp': 1}))"
+    )
+    _exec(container, "python3", "-c", write_state, timeout=10).check_returncode()
+
+    # Restart. After this, /run/service/ is empty until cont-init.d
+    # runs the reconciler. We need to wait long enough for the
+    # reconciler to write coder's entry to the boot log AND for
+    # s6-svscan to spin up the service supervise tree from the
+    # restored slot. Polling the boot log gives us the first signal.
+    _docker("restart", container, timeout=60).check_returncode()
+    log = _wait_for_reconcile_log_mention(container, "coder", deadline_s=30.0)
+    assert "profile=coder" in log, (
+        f"reconciler never logged coder after restart: {log!r}"
+    )
+    assert "action=started" in log
+
+    # Service slot exists.
+    assert _wait_for_path(
+        container, "/run/service/gateway-coder", kind="d", deadline_s=10.0,
+    ), "slot not recreated after restart"
+
+    # No `down` marker — we asked for auto-start.
+    r = _sh(container, "test -f /run/service/gateway-coder/down")
+    assert r.returncode != 0, "down marker present despite prior_state=running"
+
+
+def test_stopped_gateway_stays_stopped_after_restart(restart_container: str) -> None:
+    container = restart_container
+
+    _exec(container, "hermes", "profile", "create", "writer").check_returncode()
+
+    # Write 'stopped' directly so we don't have to race against the
+    # gateway's own state writes.
+    write_state = (
+        "import json, pathlib; "
+        "p = pathlib.Path('/opt/data/profiles/writer/gateway_state.json'); "
+        "p.write_text(json.dumps({'gateway_state': 'stopped', 'timestamp': 1}))"
+    )
+    _exec(container, "python3", "-c", write_state, timeout=10).check_returncode()
+
+    _docker("restart", container, timeout=60).check_returncode()
+    log = _wait_for_reconcile_log_mention(container, "writer", deadline_s=30.0)
+    assert "profile=writer" in log
+
+    # Slot exists.
+    assert _wait_for_path(
+        container, "/run/service/gateway-writer", kind="d", deadline_s=10.0,
+    )
+
+    # Down marker present.
+    r = _sh(container, "test -f /run/service/gateway-writer/down")
+    assert r.returncode == 0, "down marker missing despite prior_state=stopped"
+
+
+def test_stale_gateway_pid_cleaned_up_on_restart(restart_container: str) -> None:
+    """A dead container's gateway.pid + processes.json must NOT
+    survive the restart — a numerically-equal live PID in the new
+    container is a different process and would confuse the gateway
+    process-mismatch checks."""
+    container = restart_container
+
+    _exec(container, "hermes", "profile", "create", "ghost").check_returncode()
+
+    # Stamp stale runtime files alongside a 'running' state so the
+    # reconciler walks this profile.
+    stamp = (
+        "import json, pathlib; "
+        "p = pathlib.Path('/opt/data/profiles/ghost'); "
+        "(p / 'gateway_state.json').write_text(json.dumps({'gateway_state': 'stopped', 'timestamp': 1})); "
+        "(p / 'gateway.pid').write_text(json.dumps({'pid': 99999, 'host': 'old'})); "
+        "(p / 'processes.json').write_text('[]')"
+    )
+    _exec(container, "python3", "-c", stamp, timeout=10).check_returncode()
+
+    _docker("restart", container, timeout=60).check_returncode()
+    _wait_for_reconcile_log_mention(container, "ghost", deadline_s=30.0)
+
+    # Stale runtime files swept.
+    r = _sh(container, "test -f /opt/data/profiles/ghost/gateway.pid")
+    assert r.returncode != 0, "stale gateway.pid survived restart"
+    r = _sh(container, "test -f /opt/data/profiles/ghost/processes.json")
+    assert r.returncode != 0, "stale processes.json survived restart"
+
+
+def test_live_gateway_autostarts_after_real_restart_without_manual_state_stamp(
+    restart_container: str,
+) -> None:
+    """End-to-end guard for issue #42675.
+
+    The other tests in this module stamp gateway_state.json directly to
+    exercise the reconciler's READ side. This one exercises the WRITE
+    side: a real, live gateway is killed by the container/s6 SIGTERM that
+    `docker restart` sends — no manual state stamp — and must come back up
+    on the next boot.
+
+    Before the fix, the shutdown handler unconditionally persisted
+    gateway_state=stopped on that SIGTERM, so the reconciler saw 'stopped'
+    and registered the slot DOWN — the gateway silently stayed dark after
+    every container restart. The fix classifies an unmarked SIGTERM as
+    signal-initiated and persists 'running' instead, so auto-start works.
+    """
+    container = restart_container
+
+    _exec(container, "hermes", "profile", "create", "live").check_returncode()
+    r = _exec(container, "hermes", "-p", "live", "gateway", "start", timeout=60)
+    assert r.returncode == 0, f"gateway start failed: {r.stderr}"
+
+    # Wait for the gateway to actually come up under supervision AND write
+    # its own gateway_state=running (we do NOT stamp it ourselves).
+    deadline = time.monotonic() + 20.0
+    while time.monotonic() < deadline:
+        r = _sh(container, "/command/s6-svstat /run/service/gateway-live")
+        if r.returncode == 0 and "up " in r.stdout:
+            break
+        time.sleep(0.5)
+    assert "up " in r.stdout, f"gateway never came up pre-restart: {r.stdout!r}"
+
+    # Confirm the gateway persisted its own 'running' state (sanity: we're
+    # testing the real write path, not a stamped fixture).
+    deadline = time.monotonic() + 15.0
+    state = ""
+    while time.monotonic() < deadline:
+        r = _sh(
+            container,
+            "cat /opt/data/profiles/live/gateway_state.json 2>/dev/null",
+        )
+        if r.returncode == 0 and '"gateway_state"' in r.stdout:
+            state = r.stdout
+            break
+        time.sleep(0.5)
+    assert '"running"' in state, (
+        f"gateway never persisted running state pre-restart: {state!r}"
+    )
+
+    # Real restart — Docker sends SIGTERM to PID 1; s6 propagates it to the
+    # supervised gateway. No planned-stop marker is written (this is not an
+    # operator `hermes gateway stop`), so the shutdown is signal-initiated.
+    _docker("restart", container, timeout=60).check_returncode()
+
+    log = _wait_for_reconcile_log_mention(container, "live", deadline_s=30.0)
+    assert "profile=live" in log, (
+        f"reconciler never logged live after restart: {log!r}"
+    )
+    # The crux: the reconciler must AUTO-START it, not register it down.
+    assert "action=started" in log, (
+        f"gateway did NOT auto-start after a real restart (issue #42675 "
+        f"regression): {log!r}"
+    )
+
+    # Slot recreated, and NO down marker (we expect auto-start).
+    assert _wait_for_path(
+        container, "/run/service/gateway-live", kind="d", deadline_s=10.0,
+    ), "slot not recreated after restart"
+    r = _sh(container, "test -f /run/service/gateway-live/down")
+    assert r.returncode != 0, (
+        "down marker present despite a live gateway being restarted — "
+        "the signal-initiated shutdown wrongly persisted 'stopped' (#42675)"
+    )