first commit

strix/telemetry/README.md

@@ -0,0 +1,36 @@
+### Overview
+
+To help make Strix better for everyone, we collect anonymized data that helps us understand how to better improve our AI security agent for our users, guide the addition of new features, and fix common errors and bugs. This feedback loop is crucial for improving Strix's capabilities and user experience.
+
+We use [PostHog](https://posthog.com), an open-source analytics platform, for data collection and analysis. Our telemetry implementation is fully transparent - you can review the [source code](https://github.com/usestrix/strix/blob/main/strix/telemetry/posthog.py) to see exactly what we track.
+
+### Telemetry Policy
+
+Privacy is our priority. All collected data is anonymized by default. Each session gets a random UUID that is not persisted or tied to you. Your code, scan targets, vulnerability details, and findings always remain private and are never collected.
+
+### What We Track
+
+We collect only very **basic** usage data including:
+
+**Session Errors:** Duration and error types (not messages or stack traces)\
+**System Context:** OS type, architecture, Strix version\
+**Scan Context:** Scan mode (quick/standard/deep), scan type (whitebox/blackbox)\
+**Model Usage:** Which LLM model is being used (not prompts or responses)\
+**Aggregate Metrics:** Vulnerability counts by severity
+
+### What We **Never** Collect
+
+- Usernames, or any identifying information
+- Scan targets, file paths, target URLs, or domains
+- Vulnerability details, descriptions, or code
+- LLM requests and responses
+
+### How to Opt Out
+
+Telemetry in Strix is entirely **optional**:
+
+```bash
+export STRIX_TELEMETRY=0
+```
+
+You can set this environment variable before running Strix to disable **all** telemetry.

strix/telemetry/__init__.py

@@ -0,0 +1,7 @@
+from . import posthog, scarf
+
+
+__all__ = [
+    "posthog",
+    "scarf",
+]

strix/telemetry/_common.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import logging
+import platform
+import sys
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+
+logger = logging.getLogger(__name__)
+
+SESSION_ID: str = uuid4().hex[:16]
+
+_FIRST_RUN_CACHED: bool | None = None
+
+
+def get_version() -> str:
+    try:
+        return version("strix-agent")
+    except PackageNotFoundError:
+        logger.debug("strix-agent version lookup failed", exc_info=True)
+        return "unknown"
+
+
+def is_first_run() -> bool:
+    global _FIRST_RUN_CACHED  # noqa: PLW0603
+    if _FIRST_RUN_CACHED is not None:
+        return _FIRST_RUN_CACHED
+    marker = Path.home() / ".strix" / ".seen"
+    if marker.exists():
+        _FIRST_RUN_CACHED = False
+        return False
+    try:
+        marker.parent.mkdir(parents=True, exist_ok=True)
+        marker.touch()
+    except Exception:  # noqa: BLE001, S110
+        pass  # nosec B110
+    _FIRST_RUN_CACHED = True
+    return True
+
+
+def base_props() -> dict[str, Any]:
+    return {
+        "os": platform.system().lower(),
+        "arch": platform.machine(),
+        "python": f"{sys.version_info.major}.{sys.version_info.minor}",
+        "strix_version": get_version(),
+    }

strix/telemetry/logging.py

@@ -0,0 +1,143 @@
+"""Per-scan logging setup."""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import os
+import warnings
+from contextvars import ContextVar
+from pathlib import Path  # noqa: TC003  used at runtime by ``setup_scan_logging``
+from typing import TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+
+_SCAN_ID: ContextVar[str | None] = ContextVar("strix_scan_id", default=None)
+_AGENT_ID: ContextVar[str | None] = ContextVar("strix_agent_id", default=None)
+
+
+def set_scan_id(scan_id: str) -> None:
+    """Set the scan_id seen on every log record from this point in the task tree."""
+    _SCAN_ID.set(scan_id)
+
+
+def set_agent_id(agent_id: str | None) -> None:
+    """Set or clear the agent_id seen on every log record from this point.
+
+    ``None`` clears (renders as ``-`` in the log line). Mutations are
+    isolated to the current asyncio task and tasks created from it after
+    the call.
+    """
+    _AGENT_ID.set(agent_id)
+
+
+class _StrixContextFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.scan_id = _SCAN_ID.get() or "-"
+        record.agent_id = _AGENT_ID.get() or "-"
+        return True
+
+
+_FORMAT = "%(asctime)s.%(msecs)03d %(levelname)-7s %(scan_id)s %(agent_id)s %(name)s: %(message)s"
+_DATEFMT = "%Y-%m-%d %H:%M:%S"
+
+
+# Third-party loggers that get noisy at DEBUG. Capped so the file isn't
+# drowned in their internals when STRIX_DEBUG=1.
+_NOISY_LIBS: tuple[str, ...] = (
+    "httpx",
+    "httpcore",
+    "urllib3",
+    "litellm",
+    "openai",
+    "anthropic",
+)
+
+
+_HANDLER_TAG = "_strix_scan_handler"
+
+
+# ``openai.agents`` is the openai-agents SDK's canonical logger root.
+_TRACKED_ROOTS: tuple[str, ...] = ("strix", "openai.agents")
+
+
+def configure_dependency_logging() -> None:
+    """Quiet dependency logging/warnings that obscure Strix scan logs."""
+    with contextlib.suppress(Exception):
+        import litellm
+
+        litellm_logging = litellm._logging
+        litellm_logging._disable_debugging()  # type: ignore[no-untyped-call]
+
+    logging.getLogger("asyncio").setLevel(logging.CRITICAL)
+    logging.getLogger("asyncio").propagate = False
+    warnings.filterwarnings("ignore", category=RuntimeWarning, module="asyncio")
+
+
+def setup_scan_logging(run_dir: Path, *, debug: bool | None = None) -> Callable[[], None]:
+    """Attach scan-scoped handlers; return a teardown callable.
+
+    Args:
+        run_dir: Per-scan output directory. ``{run_dir}/strix.log`` is
+            created if missing and opened append-mode (so re-runs of the
+            same scan_id concatenate cleanly).
+        debug: When ``True``, stderr handler runs at DEBUG instead of
+            ERROR. ``None`` (default) reads ``STRIX_DEBUG`` env: ``1`` /
+            ``true`` / ``yes`` / ``on`` enables debug.
+
+    Returns:
+        A no-arg callable that flushes/closes/removes the handlers this
+        call attached. Idempotent — calling twice is a no-op the second
+        time. Safe to call from a ``finally`` block.
+    """
+    configure_dependency_logging()
+
+    if debug is None:
+        debug = (os.environ.get("STRIX_DEBUG") or "").strip().lower() in {
+            "1",
+            "true",
+            "yes",
+            "on",
+        }
+
+    run_dir.mkdir(parents=True, exist_ok=True)
+    log_path = run_dir / "strix.log"
+
+    formatter = logging.Formatter(_FORMAT, datefmt=_DATEFMT)
+    context_filter = _StrixContextFilter()
+
+    file_handler = logging.FileHandler(log_path, mode="a", encoding="utf-8")
+    file_handler.setLevel(logging.DEBUG)
+    file_handler.setFormatter(formatter)
+    file_handler.addFilter(context_filter)
+    setattr(file_handler, _HANDLER_TAG, True)
+
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(logging.DEBUG if debug else logging.ERROR)
+    stream_handler.setFormatter(formatter)
+    stream_handler.addFilter(context_filter)
+    setattr(stream_handler, _HANDLER_TAG, True)
+
+    tracked_loggers = [logging.getLogger(name) for name in _TRACKED_ROOTS]
+    for tracked in tracked_loggers:
+        tracked.setLevel(logging.DEBUG)
+        tracked.addHandler(file_handler)
+        tracked.addHandler(stream_handler)
+        tracked.propagate = False
+
+    for name in _NOISY_LIBS:
+        logging.getLogger(name).setLevel(logging.WARNING)
+
+    def _teardown() -> None:
+        for tracked in tracked_loggers:
+            for handler in list(tracked.handlers):
+                if getattr(handler, _HANDLER_TAG, False):
+                    tracked.removeHandler(handler)
+                    with contextlib.suppress(Exception):
+                        handler.flush()
+                        handler.close()
+
+    return _teardown

strix/telemetry/posthog.py

@@ -0,0 +1,128 @@
+import json
+import logging
+import urllib.request
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+from strix.config import load_settings
+from strix.telemetry._common import (
+    SESSION_ID,
+    base_props,
+    is_first_run,
+)
+
+
+if TYPE_CHECKING:
+    from strix.report.state import ReportState
+
+
+logger = logging.getLogger(__name__)
+
+_POSTHOG_PUBLIC_API_KEY = "phc_7rO3XRuNT5sgSKAl6HDIrWdSGh1COzxw0vxVIAR6vVZ"
+_POSTHOG_HOST = "https://us.i.posthog.com"
+
+
+def _is_enabled() -> bool:
+    return load_settings().telemetry.enabled
+
+
+def _send(event: str, properties: dict[str, Any]) -> None:
+    if not _is_enabled():
+        logger.debug("posthog disabled; skipping event %s", event)
+        return
+    try:
+        payload = {
+            "api_key": _POSTHOG_PUBLIC_API_KEY,
+            "event": event,
+            "distinct_id": SESSION_ID,
+            "properties": properties,
+        }
+        req = urllib.request.Request(  # noqa: S310
+            f"{_POSTHOG_HOST}/capture/",
+            data=json.dumps(payload).encode(),
+            headers={"Content-Type": "application/json"},
+        )
+        with urllib.request.urlopen(req, timeout=10):  # noqa: S310  # nosec B310
+            pass
+    except Exception:  # noqa: BLE001
+        logger.debug("posthog send failed for event %s", event, exc_info=True)
+    else:
+        logger.debug("posthog event sent: %s", event)
+
+
+def start(
+    model: str | None,
+    scan_mode: str | None,
+    is_whitebox: bool,
+    interactive: bool,
+    has_instructions: bool,
+) -> None:
+    _send(
+        "scan_started",
+        {
+            **base_props(),
+            "model": model or "unknown",
+            "scan_mode": scan_mode or "unknown",
+            "scan_type": "whitebox" if is_whitebox else "blackbox",
+            "interactive": interactive,
+            "has_instructions": has_instructions,
+            "first_run": is_first_run(),
+        },
+    )
+
+
+def finding(severity: str) -> None:
+    _send(
+        "finding_reported",
+        {
+            **base_props(),
+            "severity": severity.lower(),
+        },
+    )
+
+
+def end(report_state: "ReportState", exit_reason: str = "completed") -> None:
+    vulnerabilities_counts = {"critical": 0, "high": 0, "medium": 0, "low": 0, "info": 0}
+    for v in report_state.vulnerability_reports:
+        sev = v.get("severity", "info").lower()
+        if sev in vulnerabilities_counts:
+            vulnerabilities_counts[sev] += 1
+
+    duration = 0.0
+    try:
+        start = datetime.fromisoformat(report_state.start_time.replace("Z", "+00:00"))
+        end_iso = report_state.end_time or datetime.now(start.tzinfo).isoformat()
+        duration = (datetime.fromisoformat(end_iso.replace("Z", "+00:00")) - start).total_seconds()
+    except (ValueError, TypeError, AttributeError):
+        pass
+
+    llm_props: dict[str, int | float] = {}
+    try:
+        usage = report_state.get_total_llm_usage()
+        if isinstance(usage, dict):
+            llm_props = {
+                "llm_requests": int(usage.get("requests") or 0),
+                "llm_input_tokens": int(usage.get("input_tokens") or 0),
+                "llm_output_tokens": int(usage.get("output_tokens") or 0),
+                "llm_tokens": int(usage.get("total_tokens") or 0),
+                "llm_cost": float(usage.get("cost") or 0.0),
+            }
+    except (TypeError, ValueError, AttributeError):
+        pass
+
+    _send(
+        "scan_ended",
+        {
+            **base_props(),
+            "exit_reason": exit_reason,
+            "duration_seconds": round(duration),
+            "vulnerabilities_total": len(report_state.vulnerability_reports),
+            **{f"vulnerabilities_{k}": v for k, v in vulnerabilities_counts.items()},
+            **llm_props,
+        },
+    )
+
+
+def error(error_type: str) -> None:
+    props = {**base_props(), "error_type": error_type}
+    _send("error", props)

strix/telemetry/scarf.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import logging
+import urllib.parse
+import urllib.request
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+from strix.config import load_settings
+from strix.telemetry._common import (
+    SESSION_ID,
+    base_props,
+    get_version,
+    is_first_run,
+)
+
+
+if TYPE_CHECKING:
+    from strix.report.state import ReportState
+
+
+logger = logging.getLogger(__name__)
+
+_SCARF_ENDPOINT = "https://strix.gateway.scarf.sh"
+
+
+def _is_enabled() -> bool:
+    return load_settings().telemetry.enabled
+
+
+def _send(event: str, properties: dict[str, Any]) -> None:
+    if not _is_enabled():
+        logger.debug("scarf disabled; skipping event %s", event)
+        return
+    try:
+        props = dict(properties)
+        version = str(props.pop("strix_version", get_version()) or "unknown")
+        path = f"/{urllib.parse.quote(event, safe='')}/{urllib.parse.quote(version, safe='')}"
+        query = urllib.parse.urlencode(
+            {k: ("" if v is None else str(v)) for k, v in props.items()},
+        )
+        url = f"{_SCARF_ENDPOINT}{path}"
+        if query:
+            url = f"{url}?{query}"
+        req = urllib.request.Request(url, method="POST")  # noqa: S310
+        with urllib.request.urlopen(req, timeout=10):  # noqa: S310  # nosec B310
+            pass
+    except Exception:  # noqa: BLE001
+        logger.debug("scarf send failed for event %s", event, exc_info=True)
+    else:
+        logger.debug("scarf event sent: %s", event)
+
+
+def start(
+    model: str | None,
+    scan_mode: str | None,
+    is_whitebox: bool,
+    interactive: bool,
+    has_instructions: bool,
+) -> None:
+    _send(
+        "scan_started",
+        {
+            **base_props(),
+            "session": SESSION_ID,
+            "model": model or "unknown",
+            "scan_mode": scan_mode or "unknown",
+            "scan_type": "whitebox" if is_whitebox else "blackbox",
+            "interactive": interactive,
+            "has_instructions": has_instructions,
+            "first_run": is_first_run(),
+        },
+    )
+
+
+def finding(severity: str) -> None:
+    _send(
+        "finding_reported",
+        {
+            **base_props(),
+            "session": SESSION_ID,
+            "severity": severity.lower(),
+        },
+    )
+
+
+def end(report_state: ReportState, exit_reason: str = "completed") -> None:
+    vulnerabilities_counts = {"critical": 0, "high": 0, "medium": 0, "low": 0, "info": 0}
+    for v in report_state.vulnerability_reports:
+        sev = v.get("severity", "info").lower()
+        if sev in vulnerabilities_counts:
+            vulnerabilities_counts[sev] += 1
+
+    duration = 0.0
+    try:
+        scan_start = datetime.fromisoformat(report_state.start_time.replace("Z", "+00:00"))
+        end_iso = report_state.end_time or datetime.now(scan_start.tzinfo).isoformat()
+        duration = (
+            datetime.fromisoformat(end_iso.replace("Z", "+00:00")) - scan_start
+        ).total_seconds()
+    except (ValueError, TypeError, AttributeError):
+        pass
+
+    llm_props: dict[str, int | float] = {}
+    try:
+        usage = report_state.get_total_llm_usage()
+        if isinstance(usage, dict):
+            llm_props = {
+                "llm_requests": int(usage.get("requests") or 0),
+                "llm_input_tokens": int(usage.get("input_tokens") or 0),
+                "llm_output_tokens": int(usage.get("output_tokens") or 0),
+                "llm_tokens": int(usage.get("total_tokens") or 0),
+                "llm_cost": float(usage.get("cost") or 0.0),
+            }
+    except (TypeError, ValueError, AttributeError):
+        pass
+
+    _send(
+        "scan_ended",
+        {
+            **base_props(),
+            "session": SESSION_ID,
+            "exit_reason": exit_reason,
+            "duration_seconds": round(duration),
+            "vulnerabilities_total": len(report_state.vulnerability_reports),
+            **{f"vulnerabilities_{k}": v for k, v in vulnerabilities_counts.items()},
+            **llm_props,
+        },
+    )
+
+
+def error(error_type: str) -> None:
+    props: dict[str, Any] = {
+        **base_props(),
+        "session": SESSION_ID,
+        "error_type": error_type,
+    }
+    _send("error", props)