Hermes-agent
This commit is contained in:
Zakaria
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the Automation Blueprints catalog JSON for the docs site.
|
||||
|
||||
Mirrors ``extract-skills.py``: imports the single-source-of-truth blueprint
|
||||
definitions from ``cron/blueprint_catalog.py`` and emits a flat JSON array the
|
||||
docs page renders into cards (description, schedule, copy-paste slash command,
|
||||
and a ``hermes://`` "Send to App" deep-link).
|
||||
|
||||
Output: ``website/static/api/automation-blueprints-index.json`` (served at
|
||||
``/docs/api/automation-blueprints-index.json``). Run automatically by
|
||||
``website/scripts/prebuild.mjs`` before ``npm start`` / ``npm run build``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Repo root = two levels up from website/scripts/.
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
OUTPUT = REPO_ROOT / "website" / "static" / "api" / "automation-blueprints-index.json"
|
||||
|
||||
|
||||
def build_index() -> list:
|
||||
from cron.blueprint_catalog import CATALOG, blueprint_catalog_entry
|
||||
|
||||
return [blueprint_catalog_entry(r) for r in CATALOG]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
index = build_index()
|
||||
except Exception as e: # pragma: no cover - import/build failure
|
||||
# Match extract-skills.py's resilience: write an empty array so the
|
||||
# docs build never hard-fails on a generator hiccup.
|
||||
sys.stderr.write(f"extract-automation-blueprints: {e}; writing empty index\n")
|
||||
index = []
|
||||
|
||||
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(OUTPUT, "w", encoding="utf-8") as f:
|
||||
json.dump(index, f, separators=(",", ":"))
|
||||
sys.stderr.write(f"extract-automation-blueprints: wrote {len(index)} blueprints -> {OUTPUT}\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,683 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract skill metadata into website/static/api/skills.json for the Skills Hub page.
|
||||
|
||||
Two data sources:
|
||||
|
||||
1. Local SKILL.md files under ``skills/`` (built-in) and ``optional-skills/``
|
||||
(official optional). These give us full metadata — overview prose, version,
|
||||
license, env vars, commands — that the unified index doesn't carry.
|
||||
|
||||
2. The unified Hermes Skills Index at ``website/static/api/skills-index.json``,
|
||||
built twice daily by ``scripts/build_skills_index.py`` (workflow
|
||||
``.github/workflows/skills-index.yml``). Covers skills.sh, ClawHub, browse.sh,
|
||||
LobeHub, Claude Marketplace, well-known endpoints, and the GitHub taps
|
||||
(openai/skills, anthropics/skills, huggingface/skills, VoltAgent, etc.).
|
||||
|
||||
Legacy fallback: if the unified index is missing AND ``skills/index-cache/``
|
||||
contains pre-baked JSON dumps, we read those (preserves behaviour from before
|
||||
the unified index existed).
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
LOCAL_SKILL_DIRS = [
|
||||
("skills", "built-in"),
|
||||
("optional-skills", "optional"),
|
||||
]
|
||||
UNIFIED_INDEX_PATH = os.path.join(REPO_ROOT, "website", "static", "api", "skills-index.json")
|
||||
LEGACY_INDEX_CACHE_DIR = os.path.join(REPO_ROOT, "skills", "index-cache")
|
||||
# Output to static/api/ so the file is CDN-served at /api/skills.json
|
||||
# rather than bundled into the page's JS chunk. At 50k+ skills the
|
||||
# bundled payload was ~26 MB; lazy-fetch keeps the initial page load
|
||||
# fast and shrinks the JS chunk back to a few hundred KB.
|
||||
OUTPUT = os.path.join(REPO_ROOT, "website", "static", "api", "skills.json")
|
||||
META_OUTPUT = os.path.join(REPO_ROOT, "website", "static", "api", "skills-meta.json")
|
||||
|
||||
CATEGORY_LABELS = {
|
||||
"apple": "Apple",
|
||||
"autonomous-ai-agents": "AI Agents",
|
||||
"blockchain": "Blockchain",
|
||||
"communication": "Communication",
|
||||
"creative": "Creative",
|
||||
"data-science": "Data Science",
|
||||
"devops": "DevOps",
|
||||
"dogfood": "Dogfood",
|
||||
"domain": "Business & Finance",
|
||||
"email": "Email",
|
||||
"gaming": "Gaming",
|
||||
"gifs": "GIFs",
|
||||
"github": "GitHub",
|
||||
"health": "Health",
|
||||
"inference-sh": "Inference",
|
||||
"leisure": "Leisure",
|
||||
"mcp": "MCP",
|
||||
"media": "Media",
|
||||
"migration": "Migration",
|
||||
"mlops": "MLOps",
|
||||
"note-taking": "Note-Taking",
|
||||
"productivity": "Productivity",
|
||||
"red-teaming": "Red Teaming",
|
||||
"research": "Research",
|
||||
"security": "Security",
|
||||
"smart-home": "Smart Home",
|
||||
"social-media": "Social Media",
|
||||
"software-development": "Software Dev",
|
||||
"translation": "Translation",
|
||||
"other": "Other",
|
||||
}
|
||||
|
||||
# Map the source ids the unified index emits to the friendly labels the
|
||||
# Skills Hub UI uses. Keep these in sync with the SOURCE_CONFIG dict in
|
||||
# website/src/pages/skills/index.tsx.
|
||||
UNIFIED_SOURCE_LABELS = {
|
||||
"official": "official", # treated as our "optional" tier in the UI
|
||||
"skills.sh": "skills.sh",
|
||||
"skills-sh": "skills.sh",
|
||||
"clawhub": "ClawHub",
|
||||
"browse-sh": "browse.sh",
|
||||
"lobehub": "LobeHub",
|
||||
"claude-marketplace": "Claude Marketplace",
|
||||
"well-known": "Well-Known",
|
||||
"github": "GitHub", # default for non-named GitHub taps
|
||||
}
|
||||
|
||||
# Repo-specific labels for the unified index's "github" source. Lets us
|
||||
# call out the well-known taps with their vendor name instead of a generic
|
||||
# "GitHub" pill. Match is checked against the leading "owner/repo/" prefix
|
||||
# of the identifier.
|
||||
GITHUB_TAP_LABELS = {
|
||||
"openai/skills": "OpenAI",
|
||||
"anthropics/skills": "Anthropic",
|
||||
"huggingface/skills": "HuggingFace",
|
||||
"NVIDIA/skills": "NVIDIA",
|
||||
"VoltAgent/awesome-agent-skills": "VoltAgent",
|
||||
"garrytan/gstack": "gstack",
|
||||
"MiniMax-AI/cli": "MiniMax",
|
||||
}
|
||||
|
||||
# Legacy filename -> label mapping for the deprecated skills/index-cache/
|
||||
# fallback. Used only when website/static/api/skills-index.json is absent.
|
||||
LEGACY_SOURCE_LABELS = {
|
||||
"anthropics_skills": "Anthropic",
|
||||
"openai_skills": "OpenAI",
|
||||
"claude_marketplace": "Claude Marketplace",
|
||||
"lobehub": "LobeHub",
|
||||
}
|
||||
|
||||
|
||||
def _extract_overview(body: str) -> str:
|
||||
"""Pull the first non-heading paragraph from a SKILL.md body."""
|
||||
if not body:
|
||||
return ""
|
||||
paragraphs = [p.strip() for p in body.split("\n\n") if p.strip()]
|
||||
for p in paragraphs[:6]:
|
||||
if p.startswith("#"):
|
||||
lines = [ln for ln in p.split("\n") if ln.strip() and not ln.lstrip().startswith("#")]
|
||||
if lines:
|
||||
p = "\n".join(lines).strip()
|
||||
else:
|
||||
continue
|
||||
if p.startswith(":::"):
|
||||
continue
|
||||
if p.startswith("```") or p.startswith("~~~"):
|
||||
continue
|
||||
if len(p) > 500:
|
||||
cut = p[:500]
|
||||
last_period = cut.rfind(". ")
|
||||
if last_period > 200:
|
||||
p = cut[: last_period + 1]
|
||||
else:
|
||||
p = cut.rstrip() + "…"
|
||||
return p
|
||||
return ""
|
||||
|
||||
|
||||
def _docs_page_path(rel_dir: str, source_label: str) -> str:
|
||||
"""Compute the per-skill docs-site URL slug for a given SKILL.md location.
|
||||
|
||||
Mirrors the slug logic in website/scripts/generate-skill-docs.py:
|
||||
bundled + skills/<cat>/<slug>/SKILL.md -> bundled/<cat>/<cat>-<slug>
|
||||
bundled + skills/<cat>/<sub>/<slug>/SKILL.md -> bundled/<cat>/<cat>-<sub>-<slug>
|
||||
optional + optional-skills/<cat>/<slug>/SKILL.md -> optional/<cat>/<cat>-<slug>
|
||||
"""
|
||||
parts = [p for p in rel_dir.split(os.sep) if p]
|
||||
if not parts:
|
||||
return ""
|
||||
source_dir = "bundled" if source_label == "built-in" else "optional"
|
||||
if len(parts) == 1:
|
||||
category, slug = parts[0], parts[0]
|
||||
return f"{source_dir}/{category}/{category}-{slug}"
|
||||
if len(parts) == 2:
|
||||
category, slug = parts
|
||||
return f"{source_dir}/{category}/{category}-{slug}"
|
||||
if len(parts) == 3:
|
||||
category, sub, slug = parts
|
||||
return f"{source_dir}/{category}/{category}-{sub}-{slug}"
|
||||
return ""
|
||||
|
||||
|
||||
def _install_command(source: str, identifier: str, name: str) -> str:
|
||||
"""Build the ``hermes skills install …`` command for a unified-index entry.
|
||||
|
||||
These show up in the SkillCard panel so users can copy-paste them. We try
|
||||
to use the most idiomatic identifier per source.
|
||||
"""
|
||||
if not identifier:
|
||||
return f"hermes skills install {name}"
|
||||
src = source.lower()
|
||||
if src in {"official", "built-in", "optional"}:
|
||||
# OptionalSkillSource emits identifiers like "official/security/1password"
|
||||
return f"hermes skills install {identifier}"
|
||||
if src in {"skills.sh", "skills-sh"}:
|
||||
# Already wrapped as "skills-sh/owner/repo/skill" by the source
|
||||
return f"hermes skills install {identifier}"
|
||||
if src == "clawhub":
|
||||
return f"hermes skills install clawhub/{identifier}"
|
||||
if src == "browse-sh":
|
||||
# Identifier already includes the "browse-sh/" prefix from BrowseShSource
|
||||
return f"hermes skills install {identifier}"
|
||||
if src == "lobehub":
|
||||
return f"hermes skills install {identifier}"
|
||||
if src == "claude-marketplace":
|
||||
return f"hermes skills install {identifier}"
|
||||
if src == "github":
|
||||
return f"hermes skills install {identifier}"
|
||||
if src == "well-known":
|
||||
return f"hermes skills install {identifier}"
|
||||
return f"hermes skills install {identifier}"
|
||||
|
||||
|
||||
def _source_url(source: str, identifier: str, extra: dict) -> str:
|
||||
"""Best-effort clickable URL to the skill's origin (repo / detail page).
|
||||
|
||||
Community skills have no generated docs page, so without this the
|
||||
expanded card on the Skills Hub gives users nowhere to go to read the
|
||||
actual SKILL.md before installing. We prefer an explicit URL the source
|
||||
adapter already collected (``extra.detail_url`` / ``extra.repo_url``),
|
||||
then fall back to synthesizing one from the identifier shape.
|
||||
"""
|
||||
extra = extra or {}
|
||||
for key in ("detail_url", "source_url", "repo_url", "url", "index_url"):
|
||||
val = extra.get(key)
|
||||
if isinstance(val, str) and val.startswith("http"):
|
||||
return val
|
||||
|
||||
if not identifier:
|
||||
return ""
|
||||
src = (source or "").lower()
|
||||
|
||||
# GitHub-backed taps (openai/anthropic/nvidia/hf/gstack/VoltAgent/...):
|
||||
# identifier is "owner/repo/<path...>" — link to the directory on GitHub.
|
||||
if src in {"github", "openai", "anthropic", "huggingface", "nvidia",
|
||||
"gstack", "voltagent", "minimax", "claude marketplace",
|
||||
"claude-marketplace"}:
|
||||
parts = [p for p in identifier.split("/") if p]
|
||||
if len(parts) >= 2:
|
||||
owner, repo = parts[0], parts[1]
|
||||
sub = "/".join(parts[2:])
|
||||
base = f"https://github.com/{owner}/{repo}"
|
||||
return f"{base}/tree/main/{sub}" if sub else base
|
||||
return ""
|
||||
|
||||
if src == "clawhub":
|
||||
# identifier is a bare slug (the "clawhub/" prefix is added at install time)
|
||||
slug = identifier[len("clawhub/"):] if identifier.startswith("clawhub/") else identifier
|
||||
return f"https://clawhub.ai/skills/{slug}"
|
||||
|
||||
if src in {"skills.sh", "skills-sh"}:
|
||||
# "skills-sh/owner/repo/skill" -> the skills.sh detail page
|
||||
rest = identifier[len("skills-sh/"):] if identifier.startswith("skills-sh/") else identifier
|
||||
return f"https://skills.sh/skills/{rest}"
|
||||
|
||||
if src == "lobehub":
|
||||
slug = identifier[len("lobehub/"):] if identifier.startswith("lobehub/") else identifier
|
||||
return f"https://lobehub.com/agent/{slug}"
|
||||
|
||||
if src in {"browse.sh", "browse-sh"}:
|
||||
# "browse-sh/<hostname>/<task-id>" -> browse.sh task page
|
||||
rest = identifier[len("browse-sh/"):] if identifier.startswith("browse-sh/") else identifier
|
||||
return f"https://browse.sh/skills/{rest}"
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def extract_local_skills():
|
||||
skills = []
|
||||
|
||||
for base_dir, source_label in LOCAL_SKILL_DIRS:
|
||||
base_path = os.path.join(REPO_ROOT, base_dir)
|
||||
if not os.path.isdir(base_path):
|
||||
continue
|
||||
|
||||
for root, _dirs, files in os.walk(base_path):
|
||||
if "SKILL.md" not in files:
|
||||
continue
|
||||
|
||||
skill_path = os.path.join(root, "SKILL.md")
|
||||
with open(skill_path, encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
if not content.startswith("---"):
|
||||
continue
|
||||
|
||||
parts = content.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
continue
|
||||
|
||||
try:
|
||||
fm = yaml.safe_load(parts[1])
|
||||
except yaml.YAMLError:
|
||||
continue
|
||||
|
||||
if not fm or not isinstance(fm, dict):
|
||||
continue
|
||||
|
||||
body = parts[2].strip()
|
||||
overview = _extract_overview(body)
|
||||
|
||||
rel = os.path.relpath(root, base_path)
|
||||
category = rel.split(os.sep)[0]
|
||||
|
||||
tags = []
|
||||
metadata = fm.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
hermes_meta = metadata.get("hermes", {})
|
||||
if isinstance(hermes_meta, dict):
|
||||
tags = hermes_meta.get("tags", [])
|
||||
if not tags:
|
||||
tags = fm.get("tags", [])
|
||||
if isinstance(tags, str):
|
||||
tags = [tags]
|
||||
|
||||
prereq = fm.get("prerequisites") or {}
|
||||
env_vars = []
|
||||
commands = []
|
||||
if isinstance(prereq, dict):
|
||||
ev = prereq.get("env_vars")
|
||||
if isinstance(ev, list):
|
||||
env_vars = [str(x) for x in ev if x]
|
||||
elif isinstance(ev, str) and ev.strip():
|
||||
env_vars = [ev.strip()]
|
||||
cmds = prereq.get("commands")
|
||||
if isinstance(cmds, list):
|
||||
commands = [str(x) for x in cmds if x]
|
||||
elif isinstance(cmds, str) and cmds.strip():
|
||||
commands = [cmds.strip()]
|
||||
|
||||
skills.append({
|
||||
"name": fm.get("name", os.path.basename(root)),
|
||||
"description": fm.get("description", ""),
|
||||
"overview": overview,
|
||||
"category": category,
|
||||
"categoryLabel": CATEGORY_LABELS.get(category, category.replace("-", " ").title()),
|
||||
"source": source_label,
|
||||
"tags": tags or [],
|
||||
"platforms": fm.get("platforms", []),
|
||||
"author": fm.get("author", ""),
|
||||
"version": fm.get("version", ""),
|
||||
"license": fm.get("license", ""),
|
||||
"envVars": env_vars,
|
||||
"commands": commands,
|
||||
"docsPath": _docs_page_path(rel, source_label),
|
||||
})
|
||||
|
||||
return skills
|
||||
|
||||
|
||||
def _label_for_github_identifier(identifier: str) -> str:
|
||||
"""Return a friendly source label for a unified-index 'github' entry."""
|
||||
if not identifier:
|
||||
return "GitHub"
|
||||
for prefix, label in GITHUB_TAP_LABELS.items():
|
||||
if identifier.startswith(prefix + "/") or identifier == prefix:
|
||||
return label
|
||||
return "GitHub"
|
||||
|
||||
|
||||
def extract_unified_index_skills():
|
||||
"""Read website/static/api/skills-index.json — the canonical multi-source index.
|
||||
|
||||
Returns ``(skills, meta)`` where ``meta`` carries the index's
|
||||
``generated_at`` timestamp and total count so the Skills Hub page can
|
||||
show a "Last refreshed …" badge. Returns ``(None, None)`` when the
|
||||
index file is absent or malformed (caller falls back to the legacy
|
||||
cache).
|
||||
"""
|
||||
if not os.path.isfile(UNIFIED_INDEX_PATH):
|
||||
return None, None
|
||||
|
||||
try:
|
||||
with open(UNIFIED_INDEX_PATH, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
print(f"[extract-skills] Failed to read unified index: {e}")
|
||||
return None, None
|
||||
|
||||
if not isinstance(data, dict) or "skills" not in data:
|
||||
return None, None
|
||||
|
||||
meta = {
|
||||
"indexGeneratedAt": data.get("generated_at", ""),
|
||||
"indexSkillCount": data.get("skill_count", 0),
|
||||
"indexVersion": data.get("version", 0),
|
||||
}
|
||||
|
||||
out = []
|
||||
for entry in data.get("skills", []):
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
source_id = (entry.get("source") or "").lower()
|
||||
identifier = entry.get("identifier", "") or ""
|
||||
name = entry.get("name") or identifier.split("/")[-1] or "unknown"
|
||||
description = (entry.get("description") or "").split("\n")[0]
|
||||
if len(description) > 280:
|
||||
description = description[:277] + "…"
|
||||
tags = entry.get("tags", []) or []
|
||||
if not isinstance(tags, list):
|
||||
tags = []
|
||||
|
||||
# Skip official entries here — extract_local_skills() already covered
|
||||
# those from optional-skills/ with full metadata (overview, version, etc.).
|
||||
if source_id == "official":
|
||||
continue
|
||||
|
||||
# Map source id -> display label
|
||||
if source_id == "github":
|
||||
source_label = _label_for_github_identifier(identifier)
|
||||
else:
|
||||
source_label = UNIFIED_SOURCE_LABELS.get(source_id, source_id or "community")
|
||||
|
||||
# Guess a category from tags so the UI's category filter has a chance.
|
||||
category = _guess_category(tags)
|
||||
extra = entry.get("extra", {}) or {}
|
||||
|
||||
# A skills.sh.json grouping sidecar (if the tap ships one) gives us a
|
||||
# real, human-readable category — prefer it over the tag heuristic.
|
||||
# extra["category"] holds the grouping title, e.g. "Inference AI".
|
||||
sidecar_category = extra.get("category") if isinstance(extra, dict) else None
|
||||
category_label_override = ""
|
||||
if isinstance(sidecar_category, str) and sidecar_category.strip():
|
||||
category_label_override = sidecar_category.strip()
|
||||
category = category_label_override.lower().replace(" ", "-")
|
||||
|
||||
# Author hint from extras when available (skills.sh has installs;
|
||||
# clawhub doesn't expose author).
|
||||
author = ""
|
||||
if source_id in {"skills.sh", "skills-sh"}:
|
||||
repo = entry.get("repo", "")
|
||||
if repo:
|
||||
author = repo.split("/")[0]
|
||||
|
||||
install_cmd = _install_command(source_id, identifier, name)
|
||||
source_url = _source_url(source_id, identifier, extra)
|
||||
|
||||
out.append({
|
||||
"name": name,
|
||||
"description": description,
|
||||
"overview": "",
|
||||
"category": category,
|
||||
"categoryLabel": category_label_override, # set from sidecar, else filled in _consolidate_small_categories
|
||||
"fixedCategory": bool(category_label_override), # sidecar categories are exempt from small-cat collapse
|
||||
"source": source_label,
|
||||
"tags": tags,
|
||||
"platforms": [],
|
||||
"author": author,
|
||||
"version": "",
|
||||
"license": "",
|
||||
"envVars": [],
|
||||
"commands": [],
|
||||
"docsPath": "",
|
||||
"identifier": identifier,
|
||||
"installCmd": install_cmd,
|
||||
"sourceUrl": source_url,
|
||||
})
|
||||
|
||||
return out, meta
|
||||
|
||||
|
||||
def extract_legacy_cache_skills():
|
||||
"""Read the deprecated skills/index-cache/ snapshots — fallback only."""
|
||||
skills = []
|
||||
|
||||
if not os.path.isdir(LEGACY_INDEX_CACHE_DIR):
|
||||
return skills
|
||||
|
||||
for filename in os.listdir(LEGACY_INDEX_CACHE_DIR):
|
||||
if not filename.endswith(".json"):
|
||||
continue
|
||||
|
||||
filepath = os.path.join(LEGACY_INDEX_CACHE_DIR, filename)
|
||||
try:
|
||||
with open(filepath, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
continue
|
||||
|
||||
stem = filename.replace(".json", "")
|
||||
source_label = "community"
|
||||
for key, label in LEGACY_SOURCE_LABELS.items():
|
||||
if key in stem:
|
||||
source_label = label
|
||||
break
|
||||
|
||||
if isinstance(data, dict) and "agents" in data:
|
||||
for agent in data["agents"]:
|
||||
if not isinstance(agent, dict):
|
||||
continue
|
||||
skills.append({
|
||||
"name": agent.get("identifier", agent.get("meta", {}).get("title", "unknown")),
|
||||
"description": (agent.get("meta", {}).get("description", "") or "").split("\n")[0][:200],
|
||||
"category": _guess_category(agent.get("meta", {}).get("tags", [])),
|
||||
"categoryLabel": "",
|
||||
"source": source_label,
|
||||
"tags": agent.get("meta", {}).get("tags", []),
|
||||
"platforms": [],
|
||||
"author": agent.get("author", ""),
|
||||
"version": "",
|
||||
})
|
||||
continue
|
||||
|
||||
if isinstance(data, list):
|
||||
for entry in data:
|
||||
if not isinstance(entry, dict) or not entry.get("name"):
|
||||
continue
|
||||
if "skills" in entry and isinstance(entry["skills"], list):
|
||||
continue
|
||||
skills.append({
|
||||
"name": entry.get("name", ""),
|
||||
"description": entry.get("description", ""),
|
||||
"category": "uncategorized",
|
||||
"categoryLabel": "",
|
||||
"source": source_label,
|
||||
"tags": entry.get("tags", []),
|
||||
"platforms": [],
|
||||
"author": "",
|
||||
"version": "",
|
||||
})
|
||||
|
||||
for s in skills:
|
||||
if not s["categoryLabel"]:
|
||||
s["categoryLabel"] = CATEGORY_LABELS.get(
|
||||
s["category"],
|
||||
s["category"].replace("-", " ").title() if s["category"] else "Uncategorized",
|
||||
)
|
||||
|
||||
return skills
|
||||
|
||||
|
||||
TAG_TO_CATEGORY = {}
|
||||
for _cat, _tags in {
|
||||
"software-development": [
|
||||
"programming", "code", "coding", "software-development",
|
||||
"frontend-development", "backend-development", "web-development",
|
||||
"react", "python", "typescript", "java", "rust", "cli",
|
||||
"developer-tools", "development", "api", "database", "debugging",
|
||||
"documentation", "testing", "test", "architecture",
|
||||
],
|
||||
"autonomous-ai-agents": [
|
||||
"ai", "agent", "agents", "ai-agent", "ai-agents", "agentic",
|
||||
"agentic-ai", "ai-assistant", "assistant", "multi-agent",
|
||||
"autonomous", "llm", "rag", "prompt", "prompts", "a2a", "acp",
|
||||
],
|
||||
"creative": [
|
||||
"writing", "design", "creative", "art", "image-generation",
|
||||
"image", "content", "video-editing", "content-creation",
|
||||
],
|
||||
"research": ["education", "academic", "academic-writing", "research", "knowledge"],
|
||||
"social-media": ["marketing", "seo", "social-media", "advertising", "creator"],
|
||||
"productivity": [
|
||||
"productivity", "business", "automation", "calendar", "email",
|
||||
"document", "documents", "office", "notes", "note-taking",
|
||||
"collaboration", "workflow", "crm",
|
||||
],
|
||||
"data-science": ["data", "data-science", "analytics", "analysis", "visualization"],
|
||||
"mlops": ["machine-learning", "deep-learning", "mlops", "training", "fine-tuning"],
|
||||
"devops": ["devops", "docker", "kubernetes", "infrastructure", "deployment", "monitoring", "ci-cd"],
|
||||
"gaming": ["gaming", "game", "game-development"],
|
||||
"media": ["music", "media", "video", "audio", "podcast", "youtube"],
|
||||
"health": ["health", "fitness", "medical", "wellness"],
|
||||
"translation": ["translation", "language-learning", "i18n", "localization"],
|
||||
"security": ["security", "cybersecurity", "auth", "compliance", "audit", "privacy"],
|
||||
"blockchain": [
|
||||
"blockchain", "crypto", "cryptocurrency", "defi", "web3",
|
||||
"bitcoin", "ethereum", "nft", "trading", "arbitrage",
|
||||
],
|
||||
"communication": ["communication", "chat", "messaging", "slack", "discord"],
|
||||
"domain": [
|
||||
"finance", "accounting", "banking", "ecommerce", "e-commerce",
|
||||
"shopping", "travel", "booking", "real-estate", "legal",
|
||||
"government", "b2b", "b2b-sales", "entrepreneur", "budget",
|
||||
],
|
||||
}.items():
|
||||
for _t in _tags:
|
||||
TAG_TO_CATEGORY[_t] = _cat
|
||||
|
||||
|
||||
def _guess_category(tags: list) -> str:
|
||||
"""Map a skill's tags to a curated category, or 'uncategorized'.
|
||||
|
||||
Previously this fell back to ``tags[0]`` verbatim, which produced
|
||||
hundreds of junk one-off "categories" in the sidebar (e.g.
|
||||
"Doramagic Crystal", "0.10.7 Dev", "Ap2") — version strings, brand
|
||||
names, and tag noise. We now ONLY accept categories that map to a
|
||||
known curated bucket; everything else becomes "uncategorized", which
|
||||
_consolidate_small_categories folds into "Other". Sidecar-declared
|
||||
categories (skills.sh groupings) bypass this entirely via fixedCategory.
|
||||
"""
|
||||
if not tags:
|
||||
return "uncategorized"
|
||||
for tag in tags:
|
||||
if not isinstance(tag, str):
|
||||
continue
|
||||
cat = TAG_TO_CATEGORY.get(tag.lower())
|
||||
if cat:
|
||||
return cat
|
||||
# Also accept a tag that's already a known curated category key
|
||||
# (e.g. a skill tagged literally "security" or "devops").
|
||||
normalized = tag.lower().replace(" ", "-")
|
||||
if normalized in CATEGORY_LABELS and normalized != "other":
|
||||
return normalized
|
||||
return "uncategorized"
|
||||
|
||||
|
||||
MIN_CATEGORY_SIZE = 4
|
||||
|
||||
|
||||
def _consolidate_small_categories(skills: list) -> list:
|
||||
for s in skills:
|
||||
if s["category"] in {"uncategorized", ""}:
|
||||
s["category"] = "other"
|
||||
s["categoryLabel"] = "Other"
|
||||
|
||||
# Skills with a sidecar-declared category (skills.sh.json grouping) keep
|
||||
# their category even if it's the only skill in it — the tap explicitly
|
||||
# chose that label, so it's not a heuristic guess to collapse away.
|
||||
counts = Counter(
|
||||
s["category"] for s in skills if not s.get("fixedCategory")
|
||||
)
|
||||
small_cats = {cat for cat, n in counts.items() if n < MIN_CATEGORY_SIZE}
|
||||
|
||||
for s in skills:
|
||||
if s.get("fixedCategory"):
|
||||
continue
|
||||
if s["category"] in small_cats:
|
||||
s["category"] = "other"
|
||||
s["categoryLabel"] = "Other"
|
||||
elif not s["categoryLabel"]:
|
||||
s["categoryLabel"] = CATEGORY_LABELS.get(
|
||||
s["category"],
|
||||
s["category"].replace("-", " ").title() if s["category"] else "Uncategorized",
|
||||
)
|
||||
|
||||
return skills
|
||||
|
||||
|
||||
def main():
|
||||
local = extract_local_skills()
|
||||
|
||||
unified, index_meta = extract_unified_index_skills()
|
||||
if unified is not None:
|
||||
external = unified
|
||||
external_source = "unified index"
|
||||
else:
|
||||
external = extract_legacy_cache_skills()
|
||||
external_source = "legacy index-cache"
|
||||
index_meta = None
|
||||
print(
|
||||
f"[extract-skills] WARNING: unified index not found at "
|
||||
f"{UNIFIED_INDEX_PATH}; falling back to {external_source}. "
|
||||
f"Run `python3 scripts/build_skills_index.py` to refresh."
|
||||
)
|
||||
|
||||
all_skills = _consolidate_small_categories(local + external)
|
||||
|
||||
source_order = {"built-in": 0, "optional": 1}
|
||||
all_skills.sort(key=lambda s: (
|
||||
source_order.get(s["source"], 2),
|
||||
1 if s["category"] == "other" else 0,
|
||||
s["category"],
|
||||
s["name"],
|
||||
))
|
||||
|
||||
os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
|
||||
with open(OUTPUT, "w", encoding="utf-8") as f:
|
||||
# Minified — file is served over the wire, not read by humans.
|
||||
# At 50k+ skills the indented version was ~30% larger.
|
||||
json.dump(all_skills, f, separators=(",", ":"), ensure_ascii=False)
|
||||
|
||||
# Sidecar meta file so the page can render a "Last refreshed" badge
|
||||
# without changing the shape of skills.json.
|
||||
by_source = Counter(s["source"] for s in all_skills)
|
||||
meta = {
|
||||
"extractedAt": datetime.now(timezone.utc).isoformat(),
|
||||
"totalSkills": len(all_skills),
|
||||
"localSkills": len(local),
|
||||
"externalSkills": len(external),
|
||||
"externalSource": external_source,
|
||||
"bySource": dict(by_source.most_common()),
|
||||
}
|
||||
if index_meta:
|
||||
meta.update(index_meta)
|
||||
with open(META_OUTPUT, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f, separators=(",", ":"), ensure_ascii=False)
|
||||
|
||||
print(f"Extracted {len(all_skills)} skills to {OUTPUT}")
|
||||
print(f" {len(local)} local ({sum(1 for s in local if s['source'] == 'built-in')} built-in, "
|
||||
f"{sum(1 for s in local if s['source'] == 'optional')} optional)")
|
||||
print(f" {len(external)} from {external_source}")
|
||||
|
||||
print("By source:")
|
||||
for src, count in by_source.most_common():
|
||||
print(f" {src}: {count}")
|
||||
if index_meta and index_meta.get("indexGeneratedAt"):
|
||||
print(f"Unified index built at: {index_meta['indexGeneratedAt']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,306 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate llms.txt and llms-full.txt for the Hermes docs site.
|
||||
|
||||
Outputs:
|
||||
website/static/llms.txt — short curated index of the docs, one link per page,
|
||||
grouped by section. Conforms to https://llmstxt.org.
|
||||
website/static/llms-full.txt — every `.md` file under `website/docs/` concatenated,
|
||||
with `# <title>` headings and `<!-- source: … -->`
|
||||
comments separating files.
|
||||
|
||||
Both publish at:
|
||||
https://hermes-agent.nousresearch.com/docs/llms.txt
|
||||
https://hermes-agent.nousresearch.com/docs/llms-full.txt
|
||||
|
||||
The `/docs/` prefix is not a mistake — Docusaurus serves `website/static/`
|
||||
at the `docs/` base path. Clients and IDE plugins that probe the classic
|
||||
`/llms.txt` root will miss these. Document the canonical URLs in the docs
|
||||
index and in the repo README.
|
||||
|
||||
Called from `website/scripts/prebuild.mjs` on every `npm run start` /
|
||||
`npm run build` so the output stays in sync with the docs tree.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
WEBSITE = SCRIPT_DIR.parent
|
||||
DOCS = WEBSITE / "docs"
|
||||
STATIC = WEBSITE / "static"
|
||||
|
||||
SITE_BASE = "https://hermes-agent.nousresearch.com/docs"
|
||||
|
||||
# Curated sections for llms.txt — mirrors the product story, not the filesystem.
|
||||
# Each entry: (docs-relative path without .md, display title, optional short desc).
|
||||
# `None` desc → pulled from frontmatter `description:` field.
|
||||
SECTIONS: list[tuple[str, list[tuple[str, str, str | None]]]] = [
|
||||
("Getting Started", [
|
||||
("getting-started/installation", "Installation", None),
|
||||
("getting-started/quickstart", "Quickstart", None),
|
||||
("getting-started/learning-path", "Learning Path", None),
|
||||
("getting-started/updating", "Updating", None),
|
||||
("getting-started/termux", "Termux (Android)", None),
|
||||
("getting-started/nix-setup", "Nix Setup", None),
|
||||
]),
|
||||
("Using Hermes", [
|
||||
("user-guide/cli", "CLI", None),
|
||||
("user-guide/tui", "TUI (Ink terminal UI)", None),
|
||||
("user-guide/configuration", "Configuration", None),
|
||||
("user-guide/configuring-models", "Configuring Models", None),
|
||||
("user-guide/sessions", "Sessions", None),
|
||||
("user-guide/profiles", "Profiles", None),
|
||||
("user-guide/git-worktrees", "Git Worktrees", None),
|
||||
("user-guide/docker", "Docker Backend", None),
|
||||
("user-guide/security", "Security", None),
|
||||
("user-guide/checkpoints-and-rollback", "Checkpoints & Rollback", None),
|
||||
]),
|
||||
("Core Features", [
|
||||
("user-guide/features/overview", "Features Overview", None),
|
||||
("user-guide/features/tools", "Tools", None),
|
||||
("user-guide/features/skills", "Skills System", None),
|
||||
("user-guide/features/curator", "Curator", None),
|
||||
("user-guide/features/memory", "Memory", None),
|
||||
("user-guide/features/memory-providers", "Memory Providers", None),
|
||||
("user-guide/features/context-files", "Context Files", None),
|
||||
("user-guide/features/context-references", "Context References", None),
|
||||
("user-guide/features/personality", "Personality & SOUL.md", None),
|
||||
("user-guide/features/plugins", "Plugins", None),
|
||||
("user-guide/features/built-in-plugins", "Built-in Plugins", None),
|
||||
]),
|
||||
("Automation", [
|
||||
("user-guide/features/cron", "Cron Jobs", None),
|
||||
("user-guide/features/delegation", "Delegation", None),
|
||||
("user-guide/features/kanban", "Kanban Multi-Agent", None),
|
||||
("user-guide/features/kanban-tutorial", "Kanban Tutorial", None),
|
||||
("user-guide/features/goals", "Persistent Goals", None),
|
||||
("user-guide/features/code-execution", "Code Execution", None),
|
||||
("user-guide/features/hooks", "Hooks", None),
|
||||
("user-guide/features/batch-processing", "Batch Processing", None),
|
||||
]),
|
||||
("Media & Web", [
|
||||
("user-guide/features/voice-mode", "Voice Mode", None),
|
||||
("user-guide/features/browser", "Browser", None),
|
||||
("user-guide/features/vision", "Vision", None),
|
||||
("user-guide/features/image-generation", "Image Generation", None),
|
||||
("user-guide/features/tts", "Text-to-Speech", None),
|
||||
]),
|
||||
("Messaging Platforms", [
|
||||
("user-guide/messaging/index", "Overview", None),
|
||||
("user-guide/messaging/telegram", "Telegram", None),
|
||||
("user-guide/messaging/discord", "Discord", None),
|
||||
("user-guide/messaging/slack", "Slack", None),
|
||||
("user-guide/messaging/whatsapp", "WhatsApp", None),
|
||||
("user-guide/messaging/signal", "Signal", None),
|
||||
("user-guide/messaging/email", "Email", None),
|
||||
("user-guide/messaging/sms", "SMS", None),
|
||||
("user-guide/messaging/matrix", "Matrix", None),
|
||||
("user-guide/messaging/mattermost", "Mattermost", None),
|
||||
("user-guide/messaging/homeassistant", "Home Assistant", None),
|
||||
("user-guide/messaging/webhooks", "Webhooks", None),
|
||||
]),
|
||||
("Integrations", [
|
||||
("integrations/index", "Integrations Overview", None),
|
||||
("integrations/providers", "Providers", None),
|
||||
("user-guide/features/mcp", "MCP (Model Context Protocol)", None),
|
||||
("user-guide/features/acp", "ACP (Agent Context Protocol)", None),
|
||||
("user-guide/features/api-server", "API Server", None),
|
||||
("user-guide/features/honcho", "Honcho Memory", None),
|
||||
("user-guide/features/provider-routing", "Provider Routing", None),
|
||||
("user-guide/features/fallback-providers", "Fallback Providers", None),
|
||||
("user-guide/features/credential-pools", "Credential Pools", None),
|
||||
]),
|
||||
("Guides & Tutorials", [
|
||||
("guides/tips", "Tips & Best Practices", None),
|
||||
("guides/local-llm-on-mac", "Local LLMs on Mac", None),
|
||||
("guides/daily-briefing-bot", "Daily Briefing Bot", None),
|
||||
("guides/team-telegram-assistant", "Team Telegram Assistant", None),
|
||||
("guides/python-library", "Use Hermes as a Python Library", None),
|
||||
("guides/use-mcp-with-hermes", "Use MCP with Hermes", None),
|
||||
("guides/use-voice-mode-with-hermes", "Use Voice Mode with Hermes", None),
|
||||
("guides/use-soul-with-hermes", "Use SOUL.md with Hermes", None),
|
||||
("guides/build-a-hermes-plugin", "Build a Hermes Plugin", None),
|
||||
("guides/automate-with-cron", "Automate with Cron", None),
|
||||
("guides/work-with-skills", "Work with Skills", None),
|
||||
("guides/delegation-patterns", "Delegation Patterns", None),
|
||||
("guides/github-pr-review-agent", "GitHub PR Review Agent", None),
|
||||
]),
|
||||
("Developer Guide", [
|
||||
("developer-guide/contributing", "Contributing", None),
|
||||
("developer-guide/architecture", "Architecture", None),
|
||||
("developer-guide/agent-loop", "Agent Loop", None),
|
||||
("developer-guide/prompt-assembly", "Prompt Assembly", None),
|
||||
("developer-guide/context-compression-and-caching", "Context Compression & Caching", None),
|
||||
("developer-guide/gateway-internals", "Gateway Internals", None),
|
||||
("developer-guide/session-storage", "Session Storage", None),
|
||||
("developer-guide/provider-runtime", "Provider Runtime", None),
|
||||
("developer-guide/adding-tools", "Adding Tools", None),
|
||||
("developer-guide/adding-providers", "Adding Providers", None),
|
||||
("developer-guide/adding-platform-adapters", "Adding Platform Adapters", None),
|
||||
("developer-guide/creating-skills", "Creating Skills", None),
|
||||
("developer-guide/extending-the-cli", "Extending the CLI", None),
|
||||
]),
|
||||
("Reference", [
|
||||
("reference/cli-commands", "CLI Commands", None),
|
||||
("reference/slash-commands", "Slash Commands", None),
|
||||
("reference/profile-commands", "Profile Commands", None),
|
||||
("reference/environment-variables", "Environment Variables", None),
|
||||
("reference/tools-reference", "Tools Reference", None),
|
||||
("reference/toolsets-reference", "Toolsets Reference", None),
|
||||
("reference/mcp-config-reference", "MCP Config Reference", None),
|
||||
("reference/model-catalog", "Model Catalog", None),
|
||||
("reference/skills-catalog", "Bundled Skills Catalog", "Table of all ~90 skills bundled with Hermes"),
|
||||
("reference/optional-skills-catalog", "Optional Skills Catalog", "Table of ~60 additional installable skills"),
|
||||
("reference/faq", "FAQ & Troubleshooting", None),
|
||||
]),
|
||||
]
|
||||
|
||||
|
||||
FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL)
|
||||
DESC_RE = re.compile(r"^description:\s*[\"'](.+?)[\"']\s*$", re.MULTILINE)
|
||||
TITLE_RE = re.compile(r"^title:\s*[\"'](.+?)[\"']\s*$", re.MULTILINE)
|
||||
|
||||
|
||||
def read_frontmatter(path: Path) -> tuple[dict[str, str], str]:
|
||||
"""Return ({title, description}, body-markdown) for a doc file."""
|
||||
text = path.read_text(encoding="utf-8")
|
||||
m = FRONTMATTER_RE.match(text)
|
||||
meta: dict[str, str] = {}
|
||||
body = text
|
||||
if m:
|
||||
fm = m.group(1)
|
||||
body = text[m.end():]
|
||||
dm = DESC_RE.search(fm)
|
||||
if dm:
|
||||
meta["description"] = dm.group(1)
|
||||
tm = TITLE_RE.search(fm)
|
||||
if tm:
|
||||
meta["title"] = tm.group(1)
|
||||
return meta, body
|
||||
|
||||
|
||||
def resolve_desc(slug: str, provided: str | None) -> str:
|
||||
"""Resolve short description for llms.txt entry."""
|
||||
if provided:
|
||||
return provided
|
||||
path = DOCS / f"{slug}.md"
|
||||
if not path.exists():
|
||||
path = DOCS / slug / "index.md"
|
||||
if not path.exists():
|
||||
return ""
|
||||
meta, _ = read_frontmatter(path)
|
||||
return meta.get("description", "")
|
||||
|
||||
|
||||
def emit_llms_index() -> str:
|
||||
"""Build the short llms.txt index."""
|
||||
lines: list[str] = []
|
||||
lines.append("# Hermes Agent")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"> The self-improving AI agent built by Nous Research. A terminal-native "
|
||||
"autonomous coding and task agent with persistent memory, agent-created skills, "
|
||||
"and a messaging gateway that lives on 21+ messaging platforms — 19 native to "
|
||||
"the gateway plus IRC and Microsoft Teams via plugins (Telegram, Discord, Slack, "
|
||||
"SMS, Matrix, ...). Runs on local, Docker, SSH, Daytona, Modal, or Singularity "
|
||||
"backends. Works with Nous Portal, OpenRouter, OpenAI, Anthropic, Google, or any "
|
||||
"OpenAI-compatible endpoint."
|
||||
)
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"Install: `curl -fsSL https://raw.githubusercontent.com/NousResearch/"
|
||||
"hermes-agent/main/scripts/install.sh | bash` "
|
||||
"(Linux, macOS, WSL2, Termux)"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("Repo: https://github.com/NousResearch/hermes-agent")
|
||||
lines.append("")
|
||||
|
||||
for section, items in SECTIONS:
|
||||
lines.append(f"## {section}")
|
||||
lines.append("")
|
||||
for slug, title, desc_override in items:
|
||||
desc = resolve_desc(slug, desc_override)
|
||||
url = f"{SITE_BASE}/{slug}"
|
||||
if desc:
|
||||
lines.append(f"- [{title}]({url}): {desc}")
|
||||
else:
|
||||
lines.append(f"- [{title}]({url})")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def emit_llms_full() -> str:
|
||||
"""Concatenate every doc under website/docs/ into a single markdown file.
|
||||
|
||||
Order: mirrors the curated SECTIONS list first (so the most important
|
||||
pages are front-loaded for agents that truncate on token budget), then
|
||||
appends any remaining .md files sorted by path.
|
||||
"""
|
||||
seen: set[Path] = set()
|
||||
chunks: list[str] = [
|
||||
"# Hermes Agent — Full Documentation\n",
|
||||
(
|
||||
"This file is the entire Hermes Agent documentation concatenated for LLM "
|
||||
"context ingestion. Section order reflects docs-site navigation: Getting "
|
||||
"Started, Using Hermes, Features, Messaging, Integrations, Guides, "
|
||||
"Developer Guide, Reference, then everything else.\n"
|
||||
),
|
||||
"Canonical site: https://hermes-agent.nousresearch.com/docs\n",
|
||||
"Short index: https://hermes-agent.nousresearch.com/docs/llms.txt\n",
|
||||
"\n---\n\n",
|
||||
]
|
||||
|
||||
def emit_file(rel: str) -> None:
|
||||
path = DOCS / f"{rel}.md"
|
||||
if not path.exists():
|
||||
path = DOCS / rel / "index.md"
|
||||
if not path.exists() or path in seen:
|
||||
return
|
||||
seen.add(path)
|
||||
meta, body = read_frontmatter(path)
|
||||
title = meta.get("title") or rel
|
||||
chunks.append(f"<!-- source: website/docs/{path.relative_to(DOCS)} -->\n")
|
||||
chunks.append(f"# {title}\n\n")
|
||||
chunks.append(body.rstrip() + "\n\n---\n\n")
|
||||
|
||||
# Curated order first
|
||||
for _, items in SECTIONS:
|
||||
for slug, _t, _d in items:
|
||||
emit_file(slug)
|
||||
|
||||
# Everything else (sorted, skipping already emitted and auto-gen skill pages
|
||||
# — those are covered by the two catalog reference pages, emitting every
|
||||
# individual skill would add ~1.4 MB of largely duplicative material).
|
||||
for path in sorted(DOCS.rglob("*.md")):
|
||||
if path in seen:
|
||||
continue
|
||||
rel = path.relative_to(DOCS)
|
||||
parts = rel.parts
|
||||
if len(parts) >= 3 and parts[0] == "user-guide" and parts[1] == "skills" \
|
||||
and parts[2] in {"bundled", "optional"}:
|
||||
continue
|
||||
seen.add(path)
|
||||
meta, body = read_frontmatter(path)
|
||||
title = meta.get("title") or str(rel)
|
||||
chunks.append(f"<!-- source: website/docs/{rel} -->\n")
|
||||
chunks.append(f"# {title}\n\n")
|
||||
chunks.append(body.rstrip() + "\n\n---\n\n")
|
||||
|
||||
return "".join(chunks).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
STATIC.mkdir(exist_ok=True)
|
||||
index = emit_llms_index()
|
||||
full = emit_llms_full()
|
||||
(STATIC / "llms.txt").write_text(index, encoding="utf-8")
|
||||
(STATIC / "llms-full.txt").write_text(full, encoding="utf-8")
|
||||
print(f"Wrote {STATIC / 'llms.txt'} ({len(index):,} bytes)")
|
||||
print(f"Wrote {STATIC / 'llms-full.txt'} ({len(full):,} bytes)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+773
@@ -0,0 +1,773 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate per-skill Docusaurus pages from skills/ and optional-skills/ SKILL.md files.
|
||||
|
||||
Each skill gets website/docs/user-guide/skills/<source>/<category>/<skill-name>.md
|
||||
where <source> is "bundled" or "optional".
|
||||
|
||||
Also regenerates:
|
||||
- website/docs/reference/skills-catalog.md
|
||||
- website/docs/reference/optional-skills-catalog.md
|
||||
(so their table rows link to the new dedicated pages)
|
||||
|
||||
Sidebar is updated to nest all per-skill pages under Skills → Bundled / Optional.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
DOCS = REPO / "website" / "docs"
|
||||
SKILLS_PAGES = DOCS / "user-guide" / "skills"
|
||||
|
||||
SKILL_SOURCES = [
|
||||
("bundled", REPO / "skills"),
|
||||
("optional", REPO / "optional-skills"),
|
||||
]
|
||||
|
||||
# Pages the user had previously hand-written in user-guide/skills/.
|
||||
# We leave these alone (they get first-class sidebar treatment separately).
|
||||
HAND_WRITTEN = {"google-workspace.md"}
|
||||
|
||||
|
||||
_FENCE_RE = re.compile(r"^(?P<indent>\s*)(?P<fence>```+|~~~+)", re.MULTILINE)
|
||||
|
||||
# Unicode box-drawing characters. If a generated fenced code block contains any
|
||||
# of these, wrap it in `<!-- ascii-guard-ignore -->` so the docs-site-checks
|
||||
# lint (which scans inside code fences) can't reject the page for a skill's
|
||||
# own ASCII diagram. Skill authors shouldn't need to remember to add the
|
||||
# ignore markers in every SKILL.md — the generator handles it defensively.
|
||||
_BOX_DRAWING_CHARS = frozenset("┌┐└┘─│═║╔╗╚╝╠╣╦╩╬├┤┬┴┼╭╮╯╰▶◀▲▼")
|
||||
|
||||
|
||||
def _wrap_ascii_art_code_blocks(code_segment: str) -> str:
|
||||
"""Wrap a fenced code segment in ascii-guard-ignore markers if it contains
|
||||
box-drawing characters. No-op otherwise, so plain bash/python code blocks
|
||||
stay uncluttered.
|
||||
|
||||
Already-wrapped segments (the SKILL.md source added its own markers) are
|
||||
left alone — double-wrapping is harmless but we'd rather keep the output
|
||||
clean.
|
||||
"""
|
||||
if not any(ch in _BOX_DRAWING_CHARS for ch in code_segment):
|
||||
return code_segment
|
||||
return (
|
||||
"<!-- ascii-guard-ignore -->\n"
|
||||
f"{code_segment}\n"
|
||||
"<!-- ascii-guard-ignore-end -->"
|
||||
)
|
||||
|
||||
|
||||
def mdx_escape_body(body: str) -> str:
|
||||
"""Escape MDX-dangerous characters in markdown body, leaving fenced code blocks alone.
|
||||
|
||||
Outside fenced code blocks:
|
||||
* `{` -> `{` (prevents MDX from parsing JSX expressions)
|
||||
* `}` -> `}`
|
||||
* `<tag>` for bare tags that aren't whitelisted HTML get HTML-entity-escaped
|
||||
* inline `` `code` `` content is preserved (backticks handled naturally)
|
||||
Inside fenced code blocks: untouched.
|
||||
|
||||
We also preserve `<br>`, `<br/>`, `<img ...>`, `<a ...>`, and a handful of
|
||||
other markup-safe tags because Docusaurus/MDX accepts them as HTML.
|
||||
"""
|
||||
# Split the body into segments by fenced code blocks, alternating
|
||||
# (text, code, text, code, ...). A line like ``` or ~~~ opens a fence;
|
||||
# a matching marker closes it.
|
||||
lines = body.split("\n")
|
||||
segments: list[tuple[str, str]] = [] # ("text"|"code", content)
|
||||
buf: list[str] = []
|
||||
mode = "text"
|
||||
fence_char: str | None = None
|
||||
fence_len = 0
|
||||
for line in lines:
|
||||
stripped = line.lstrip()
|
||||
if mode == "text":
|
||||
if stripped.startswith("```") or stripped.startswith("~~~"):
|
||||
# Opening fence
|
||||
if buf:
|
||||
segments.append(("text", "\n".join(buf)))
|
||||
buf = []
|
||||
buf.append(line)
|
||||
# Detect fence char + length
|
||||
m = re.match(r"(`{3,}|~{3,})", stripped)
|
||||
if m:
|
||||
fence_char = m.group(1)[0]
|
||||
fence_len = len(m.group(1))
|
||||
mode = "code"
|
||||
else:
|
||||
buf.append(line)
|
||||
else: # code mode
|
||||
buf.append(line)
|
||||
if fence_char is not None and stripped.startswith(fence_char * fence_len):
|
||||
# Closing fence
|
||||
segments.append(("code", "\n".join(buf)))
|
||||
buf = []
|
||||
mode = "text"
|
||||
fence_char = None
|
||||
fence_len = 0
|
||||
if buf:
|
||||
segments.append((mode, "\n".join(buf)))
|
||||
|
||||
def escape_text(text: str) -> str:
|
||||
# Walk inline-code runs (backticks) and leave them alone.
|
||||
# Pattern matches runs of backticks, then the matched content, then the
|
||||
# same number of backticks.
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
while i < len(text):
|
||||
ch = text[i]
|
||||
if ch == "`":
|
||||
# Find the run of backticks
|
||||
j = i
|
||||
while j < len(text) and text[j] == "`":
|
||||
j += 1
|
||||
run = text[i:j]
|
||||
# Find matching run
|
||||
end = text.find(run, j)
|
||||
if end == -1:
|
||||
# No closing -- just keep as-is
|
||||
out.append(text[i:])
|
||||
i = len(text)
|
||||
continue
|
||||
out.append(text[i : end + len(run)])
|
||||
i = end + len(run)
|
||||
else:
|
||||
# Escape MDX metacharacters
|
||||
if ch == "{":
|
||||
out.append("{")
|
||||
elif ch == "}":
|
||||
out.append("}")
|
||||
elif ch == "<":
|
||||
# Preserve full HTML comments (e.g. ascii-guard ignore markers) — they
|
||||
# are not HTML tags, so the tag regex below would escape the leading <.
|
||||
if text[i:].startswith("<!--"):
|
||||
end = text.find("-->", i)
|
||||
if end != -1:
|
||||
out.append(text[i : end + 3])
|
||||
i = end + 3
|
||||
continue
|
||||
# Look ahead to see if this is a valid HTML-ish tag.
|
||||
# If it looks like a tag name then alnum/-/_ chars, leave it.
|
||||
# Otherwise escape.
|
||||
m = re.match(
|
||||
r"<(/?)([A-Za-z][A-Za-z0-9]*)([^<>]*)>",
|
||||
text[i:],
|
||||
)
|
||||
if m:
|
||||
tag = m.group(2).lower()
|
||||
# Whitelist known-safe HTML tags
|
||||
safe_tags = {
|
||||
"br",
|
||||
"hr",
|
||||
"img",
|
||||
"a",
|
||||
"b",
|
||||
"i",
|
||||
"em",
|
||||
"strong",
|
||||
"code",
|
||||
"kbd",
|
||||
"sup",
|
||||
"sub",
|
||||
"span",
|
||||
"div",
|
||||
"p",
|
||||
"ul",
|
||||
"ol",
|
||||
"li",
|
||||
"table",
|
||||
"thead",
|
||||
"tbody",
|
||||
"tr",
|
||||
"td",
|
||||
"th",
|
||||
"details",
|
||||
"summary",
|
||||
"blockquote",
|
||||
"pre",
|
||||
"mark",
|
||||
"small",
|
||||
"u",
|
||||
"s",
|
||||
"del",
|
||||
"ins",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
}
|
||||
if tag in safe_tags:
|
||||
out.append(m.group(0))
|
||||
i += len(m.group(0))
|
||||
continue
|
||||
# Escape the `<`
|
||||
out.append("<")
|
||||
else:
|
||||
out.append(ch)
|
||||
i += 1
|
||||
return "".join(out)
|
||||
|
||||
processed: list[str] = []
|
||||
for kind, content in segments:
|
||||
if kind == "code":
|
||||
processed.append(_wrap_ascii_art_code_blocks(content))
|
||||
else:
|
||||
processed.append(escape_text(content))
|
||||
return "\n".join(processed)
|
||||
|
||||
|
||||
def rewrite_relative_links(body: str, meta: dict[str, Any]) -> str:
|
||||
"""Rewrite references/foo.md style links in the SKILL.md body.
|
||||
|
||||
The source SKILL.md lives in `skills/<...>` and references sibling files
|
||||
with paths like `references/foo.md` or `./templates/bar.md`. Those files
|
||||
are NOT copied into docs/, so we rewrite these to absolute GitHub URLs
|
||||
pointing to the file in the repo.
|
||||
"""
|
||||
source_dir = "skills" if meta["source_kind"] == "bundled" else "optional-skills"
|
||||
base = f"https://github.com/NousResearch/hermes-agent/blob/main/{source_dir}/{meta['rel_path']}"
|
||||
|
||||
def sub_link(m: re.Match) -> str:
|
||||
text = m.group(1)
|
||||
url = m.group(2).strip()
|
||||
# Skip URLs that already start with a scheme or //
|
||||
if re.match(r"^[a-z]+://", url) or url.startswith("#") or url.startswith("/"):
|
||||
return m.group(0)
|
||||
# Skip mailto
|
||||
if url.startswith("mailto:"):
|
||||
return m.group(0)
|
||||
# Strip leading ./
|
||||
url_clean = url[2:] if url.startswith("./") else url
|
||||
full = f"{base}/{url_clean}"
|
||||
return f"[{text}]({full})"
|
||||
|
||||
return re.sub(r"\[([^\]]+)\]\(([^)]+)\)", sub_link, body)
|
||||
|
||||
|
||||
def parse_skill_md(path: Path) -> dict[str, Any]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if not text.startswith("---"):
|
||||
raise ValueError(f"{path}: no frontmatter")
|
||||
parts = text.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
raise ValueError(f"{path}: malformed frontmatter")
|
||||
fm_text, body = parts[1], parts[2]
|
||||
try:
|
||||
fm = yaml.safe_load(fm_text) or {}
|
||||
except yaml.YAMLError as exc:
|
||||
raise ValueError(f"{path}: YAML error: {exc}") from exc
|
||||
return {"frontmatter": fm, "body": body.lstrip("\n")}
|
||||
|
||||
|
||||
def sanitize_yaml_string(s: str) -> str:
|
||||
"""Make a string safe to embed in a YAML double-quoted scalar."""
|
||||
s = s.replace("\\", "\\\\").replace('"', '\\"')
|
||||
# Collapse newlines to spaces.
|
||||
s = re.sub(r"\s+", " ", s).strip()
|
||||
return s
|
||||
|
||||
|
||||
def derive_skill_meta(skill_path: Path, source_dir: Path, source_kind: str) -> dict[str, Any]:
|
||||
"""Extract category + skill slug from filesystem layout.
|
||||
|
||||
skills/<cat>/<skill>/SKILL.md -> cat=<cat>, slug=<skill>
|
||||
skills/<cat>/<sub>/<skill>/SKILL.md -> cat=<cat>, sub=<sub>, slug=<skill>
|
||||
optional-skills/<cat>/<skill>/SKILL.md -> cat=<cat>, slug=<skill>
|
||||
"""
|
||||
rel = skill_path.parent.relative_to(source_dir)
|
||||
parts = rel.parts
|
||||
if len(parts) == 1:
|
||||
# Top-level skill (e.g. skills/dogfood/SKILL.md) -- rare
|
||||
category = parts[0]
|
||||
sub = None
|
||||
slug = parts[0]
|
||||
elif len(parts) == 2:
|
||||
category, slug = parts
|
||||
sub = None
|
||||
elif len(parts) == 3:
|
||||
category, sub, slug = parts
|
||||
else:
|
||||
raise ValueError(f"Unexpected skill layout: {skill_path}")
|
||||
return {
|
||||
"source_kind": source_kind, # bundled | optional
|
||||
"category": category,
|
||||
"sub": sub,
|
||||
"slug": slug,
|
||||
"rel_path": str(rel),
|
||||
}
|
||||
|
||||
|
||||
def page_id(meta: dict[str, Any]) -> str:
|
||||
"""Stable slug used for filename + sidebar id."""
|
||||
if meta["sub"]:
|
||||
return f"{meta['category']}-{meta['sub']}-{meta['slug']}"
|
||||
return f"{meta['category']}-{meta['slug']}"
|
||||
|
||||
|
||||
def page_output_path(meta: dict[str, Any]) -> Path:
|
||||
return (
|
||||
SKILLS_PAGES
|
||||
/ meta["source_kind"]
|
||||
/ meta["category"]
|
||||
/ f"{page_id(meta)}.md"
|
||||
)
|
||||
|
||||
|
||||
def sidebar_doc_id(meta: dict[str, Any]) -> str:
|
||||
"""Docusaurus sidebar id, relative to docs/."""
|
||||
return f"user-guide/skills/{meta['source_kind']}/{meta['category']}/{page_id(meta)}"
|
||||
|
||||
|
||||
def render_skill_page(
|
||||
meta: dict[str, Any],
|
||||
fm: dict[str, Any],
|
||||
body: str,
|
||||
skill_index: dict[str, dict[str, Any]] | None = None,
|
||||
) -> str:
|
||||
name = fm.get("name", meta["slug"])
|
||||
description = fm.get("description", "").strip()
|
||||
short_desc = description.split(".")[0].strip() if description else name
|
||||
if len(short_desc) > 160:
|
||||
short_desc = short_desc[:157] + "..."
|
||||
|
||||
title = f"{name}"
|
||||
# Heuristic nicer title from name
|
||||
display_name = name.replace("-", " ").replace("_", " ").title()
|
||||
|
||||
hermes_meta = (fm.get("metadata") or {}).get("hermes") or {}
|
||||
tags = hermes_meta.get("tags") or []
|
||||
related = hermes_meta.get("related_skills") or []
|
||||
platforms = fm.get("platforms")
|
||||
version = fm.get("version")
|
||||
author = fm.get("author")
|
||||
license_ = fm.get("license")
|
||||
deps = fm.get("dependencies")
|
||||
|
||||
# Build metadata info block
|
||||
info_rows: list[tuple[str, str]] = []
|
||||
if meta["source_kind"] == "bundled":
|
||||
info_rows.append(("Source", "Bundled (installed by default)"))
|
||||
else:
|
||||
info_rows.append(
|
||||
(
|
||||
"Source",
|
||||
"Optional — install with `hermes skills install official/"
|
||||
+ meta["category"]
|
||||
+ "/"
|
||||
+ meta["slug"]
|
||||
+ "`",
|
||||
)
|
||||
)
|
||||
source_dir = "skills" if meta["source_kind"] == "bundled" else "optional-skills"
|
||||
info_rows.append(("Path", f"`{source_dir}/{meta['rel_path']}`"))
|
||||
if version:
|
||||
info_rows.append(("Version", f"`{version}`"))
|
||||
if author:
|
||||
info_rows.append(("Author", str(author)))
|
||||
if license_:
|
||||
info_rows.append(("License", str(license_)))
|
||||
if deps:
|
||||
if isinstance(deps, list):
|
||||
deps_str = ", ".join(f"`{d}`" for d in deps) if deps else "None"
|
||||
else:
|
||||
deps_str = f"`{deps}`"
|
||||
info_rows.append(("Dependencies", deps_str))
|
||||
if platforms:
|
||||
if isinstance(platforms, list):
|
||||
plat_str = ", ".join(platforms)
|
||||
else:
|
||||
plat_str = str(platforms)
|
||||
info_rows.append(("Platforms", plat_str))
|
||||
if tags:
|
||||
info_rows.append(("Tags", ", ".join(f"`{t}`" for t in tags)))
|
||||
if related:
|
||||
# link to sibling pages when possible -- fall back to plain code
|
||||
link_parts = []
|
||||
for r in related:
|
||||
target_meta = None
|
||||
if skill_index is not None:
|
||||
target_meta = skill_index.get(r)
|
||||
if target_meta is not None:
|
||||
href = (
|
||||
f"/docs/user-guide/skills/{target_meta['source_kind']}"
|
||||
f"/{target_meta['category']}/{page_id(target_meta)}"
|
||||
)
|
||||
link_parts.append(f"[`{r}`]({href})")
|
||||
else:
|
||||
link_parts.append(f"`{r}`")
|
||||
info_rows.append(("Related skills", ", ".join(link_parts)))
|
||||
|
||||
info_block = "\n".join(f"| {k} | {v} |" for k, v in info_rows)
|
||||
info_table = (
|
||||
"| | |\n|---|---|\n" + info_block
|
||||
)
|
||||
|
||||
# Frontmatter for Docusaurus
|
||||
fm_title = sanitize_yaml_string(display_name + " — " + (short_desc or name))
|
||||
if len(fm_title) > 120:
|
||||
fm_title = sanitize_yaml_string(display_name)
|
||||
fm_desc = sanitize_yaml_string(short_desc or description or name)
|
||||
sidebar_label = sanitize_yaml_string(display_name)
|
||||
|
||||
body_clean = mdx_escape_body(rewrite_relative_links(body.strip(), meta))
|
||||
|
||||
# Guard against the first heading in body being `# Xxx Skill` which would
|
||||
# duplicate the page title -- Docusaurus handles this fine because the
|
||||
# frontmatter `title` drives the page header and TOC.
|
||||
|
||||
return (
|
||||
"---\n"
|
||||
f'title: "{fm_title}"\n'
|
||||
f'sidebar_label: "{sidebar_label}"\n'
|
||||
f'description: "{fm_desc}"\n'
|
||||
"---\n"
|
||||
"\n"
|
||||
"{/* This page is auto-generated from the skill's SKILL.md by website/scripts/generate-skill-docs.py. Edit the source SKILL.md, not this page. */}\n"
|
||||
"\n"
|
||||
f"# {display_name}\n"
|
||||
"\n"
|
||||
f"{mdx_escape_body(description)}\n"
|
||||
"\n"
|
||||
"## Skill metadata\n"
|
||||
"\n"
|
||||
f"{info_table}\n"
|
||||
"\n"
|
||||
"## Reference: full SKILL.md\n"
|
||||
"\n"
|
||||
":::info\n"
|
||||
"The following is the complete skill definition that Hermes loads when this skill is triggered. This is what the agent sees as instructions when the skill is active.\n"
|
||||
":::\n"
|
||||
"\n"
|
||||
f"{body_clean}\n"
|
||||
)
|
||||
|
||||
|
||||
def discover_skills() -> list[tuple[dict[str, Any], dict[str, Any]]]:
|
||||
results: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
||||
for kind, source_dir in SKILL_SOURCES:
|
||||
for skill_md in sorted(source_dir.rglob("SKILL.md")):
|
||||
meta = derive_skill_meta(skill_md, source_dir, kind)
|
||||
parsed = parse_skill_md(skill_md)
|
||||
results.append((meta, parsed))
|
||||
return results
|
||||
|
||||
|
||||
def build_catalog_md_bundled(entries: list[tuple[dict[str, Any], dict[str, Any]]]) -> str:
|
||||
by_cat: dict[str, list[tuple[dict[str, Any], dict[str, Any]]]] = defaultdict(list)
|
||||
for meta, parsed in entries:
|
||||
if meta["source_kind"] != "bundled":
|
||||
continue
|
||||
by_cat[meta["category"]].append((meta, parsed))
|
||||
for k in by_cat:
|
||||
by_cat[k].sort(key=lambda e: e[0]["slug"])
|
||||
|
||||
lines = [
|
||||
"---",
|
||||
"sidebar_position: 5",
|
||||
'title: "Bundled Skills Catalog"',
|
||||
'description: "Catalog of bundled skills that ship with Hermes Agent"',
|
||||
"---",
|
||||
"",
|
||||
"# Bundled Skills Catalog",
|
||||
"",
|
||||
"Hermes ships with a large built-in skill library copied into `~/.hermes/skills/` on install. Each skill below links to a dedicated page with its full definition, setup, and usage.",
|
||||
"",
|
||||
"Hermes also syncs bundled skills on `hermes update`, but the sync manifest respects local deletions and user edits. If a skill listed here is missing from your profile's `~/.hermes/skills/` tree, it is still shipped with Hermes; restore it with `hermes skills reset <name> --restore`.",
|
||||
"",
|
||||
"If a skill is missing from this list but present in the repo, the catalog is regenerated by `website/scripts/generate-skill-docs.py`.",
|
||||
"",
|
||||
]
|
||||
for category in sorted(by_cat):
|
||||
lines.append(f"## {category}")
|
||||
lines.append("")
|
||||
lines.append("| Skill | Description | Path |")
|
||||
lines.append("|-------|-------------|------|")
|
||||
for meta, parsed in by_cat[category]:
|
||||
fm = parsed["frontmatter"]
|
||||
name = fm.get("name", meta["slug"])
|
||||
desc = (fm.get("description") or "").strip()
|
||||
if len(desc) > 240:
|
||||
desc = desc[:237].rstrip() + "..."
|
||||
link_target = f"/docs/user-guide/skills/bundled/{meta['category']}/{page_id(meta)}"
|
||||
path = f"`{meta['rel_path']}`"
|
||||
desc_esc = mdx_escape_body(desc).replace("|", "\\|").replace("\n", " ")
|
||||
lines.append(
|
||||
f"| [`{name}`]({link_target}) | {desc_esc} | {path} |"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def build_catalog_md_optional(entries: list[tuple[dict[str, Any], dict[str, Any]]]) -> str:
|
||||
by_cat: dict[str, list[tuple[dict[str, Any], dict[str, Any]]]] = defaultdict(list)
|
||||
for meta, parsed in entries:
|
||||
if meta["source_kind"] != "optional":
|
||||
continue
|
||||
by_cat[meta["category"]].append((meta, parsed))
|
||||
for k in by_cat:
|
||||
by_cat[k].sort(key=lambda e: e[0]["slug"])
|
||||
|
||||
lines = [
|
||||
"---",
|
||||
"sidebar_position: 9",
|
||||
'title: "Optional Skills Catalog"',
|
||||
'description: "Official optional skills shipped with hermes-agent — install via hermes skills install official/<category>/<skill>"',
|
||||
"---",
|
||||
"",
|
||||
"# Optional Skills Catalog",
|
||||
"",
|
||||
"Optional skills ship with hermes-agent under `optional-skills/` but are **not active by default**. Install them explicitly:",
|
||||
"",
|
||||
"```bash",
|
||||
"hermes skills install official/<category>/<skill>",
|
||||
"```",
|
||||
"",
|
||||
"For example:",
|
||||
"",
|
||||
"```bash",
|
||||
"hermes skills install official/blockchain/solana",
|
||||
"hermes skills install official/mlops/flash-attention",
|
||||
"```",
|
||||
"",
|
||||
"Each skill below links to a dedicated page with its full definition, setup, and usage.",
|
||||
"",
|
||||
"To uninstall:",
|
||||
"",
|
||||
"```bash",
|
||||
"hermes skills uninstall <skill-name>",
|
||||
"```",
|
||||
"",
|
||||
]
|
||||
for category in sorted(by_cat):
|
||||
lines.append(f"## {category}")
|
||||
lines.append("")
|
||||
lines.append("| Skill | Description |")
|
||||
lines.append("|-------|-------------|")
|
||||
for meta, parsed in by_cat[category]:
|
||||
fm = parsed["frontmatter"]
|
||||
name = fm.get("name", meta["slug"])
|
||||
desc = (fm.get("description") or "").strip()
|
||||
if len(desc) > 240:
|
||||
desc = desc[:237].rstrip() + "..."
|
||||
link_target = f"/docs/user-guide/skills/optional/{meta['category']}/{page_id(meta)}"
|
||||
desc_esc = mdx_escape_body(desc).replace("|", "\\|").replace("\n", " ")
|
||||
lines.append(f"| [**{name}**]({link_target}) | {desc_esc} |")
|
||||
lines.append("")
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"---",
|
||||
"",
|
||||
"## Contributing Optional Skills",
|
||||
"",
|
||||
"To add a new optional skill to the repository:",
|
||||
"",
|
||||
"1. Create a directory under `optional-skills/<category>/<skill-name>/`",
|
||||
"2. Add a `SKILL.md` with standard frontmatter (name, description, version, author)",
|
||||
"3. Include any supporting files in `references/`, `templates/`, or `scripts/` subdirectories",
|
||||
"4. Submit a pull request — the skill will appear in this catalog and get its own docs page once merged",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def build_sidebar_items(entries: list[tuple[dict[str, Any], dict[str, Any]]]) -> dict:
|
||||
"""Build a dict representing the Skills sidebar tree.
|
||||
|
||||
Structure:
|
||||
Skills
|
||||
├── (hand-written pages first: google-workspace)
|
||||
├── Bundled
|
||||
│ ├── apple
|
||||
│ │ ├── apple-apple-notes
|
||||
│ │ └── ...
|
||||
│ └── ...
|
||||
└── Optional
|
||||
└── ...
|
||||
"""
|
||||
bundled = defaultdict(list)
|
||||
optional = defaultdict(list)
|
||||
for meta, _ in entries:
|
||||
if meta["source_kind"] == "bundled":
|
||||
bundled[meta["category"]].append(meta)
|
||||
else:
|
||||
optional[meta["category"]].append(meta)
|
||||
|
||||
def cat_section(bucket: dict[str, list[dict[str, Any]]], source: str) -> list[dict]:
|
||||
result = []
|
||||
for category in sorted(bucket):
|
||||
items = sorted(bucket[category], key=lambda m: m["slug"])
|
||||
result.append(
|
||||
{
|
||||
"type": "category",
|
||||
"label": category,
|
||||
# Docusaurus generates a translation key from the label by
|
||||
# default (e.g. sidebar.docs.category.productivity). When
|
||||
# the same category name appears under both Bundled and
|
||||
# Optional, the duplicate keys break i18n extraction and
|
||||
# fail the build. Scope each category by source to keep
|
||||
# the keys unique.
|
||||
"key": f"skills-{source}-{category}",
|
||||
"collapsed": True,
|
||||
"items": [sidebar_doc_id(m) for m in items],
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
return {
|
||||
"bundled_categories": cat_section(bundled, "bundled"),
|
||||
"optional_categories": cat_section(optional, "optional"),
|
||||
}
|
||||
|
||||
|
||||
def _render_sidebar_item(item: Any, indent: int) -> list[str]:
|
||||
"""Render one sidebar item (string doc id, or category dict) as ts lines."""
|
||||
pad = " " * indent
|
||||
lines: list[str] = []
|
||||
if isinstance(item, str):
|
||||
lines.append(f"{pad}'{item}',")
|
||||
return lines
|
||||
# category dict
|
||||
lines.append(f"{pad}{{")
|
||||
lines.append(f"{pad} type: 'category',")
|
||||
lines.append(f"{pad} label: '{item['label']}',")
|
||||
if item.get("key"):
|
||||
lines.append(f"{pad} key: '{item['key']}',")
|
||||
if item.get("collapsed", True):
|
||||
lines.append(f"{pad} collapsed: true,")
|
||||
lines.append(f"{pad} items: [")
|
||||
for child in item.get("items", []):
|
||||
lines.extend(_render_sidebar_item(child, indent + 4))
|
||||
lines.append(f"{pad} ],")
|
||||
lines.append(f"{pad}}},")
|
||||
return lines
|
||||
|
||||
|
||||
def write_sidebar(entries):
|
||||
# Sidebar layout:
|
||||
# Skills
|
||||
# ├── reference/skills-catalog
|
||||
# ├── reference/optional-skills-catalog
|
||||
# ├── Bundled
|
||||
# │ ├── apple/
|
||||
# │ │ ├── apple-apple-notes
|
||||
# │ │ └── ...
|
||||
# │ └── ...
|
||||
# └── Optional
|
||||
# └── ...
|
||||
#
|
||||
# The two catalog index pages stay at the top of the Skills section so
|
||||
# the at-a-glance table view is one click away, and the per-category
|
||||
# subtrees give individual skill pages real sidebar navigation when
|
||||
# users land on them directly.
|
||||
tree = build_sidebar_items(entries)
|
||||
|
||||
skills_block: list[dict[str, Any]] = [
|
||||
{
|
||||
"label": "Bundled",
|
||||
"collapsed": True,
|
||||
"items": tree["bundled_categories"],
|
||||
},
|
||||
{
|
||||
"label": "Optional",
|
||||
"collapsed": True,
|
||||
"items": tree["optional_categories"],
|
||||
},
|
||||
]
|
||||
skills_items: list[Any] = [
|
||||
"reference/skills-catalog",
|
||||
"reference/optional-skills-catalog",
|
||||
*skills_block,
|
||||
]
|
||||
|
||||
skills_top = {
|
||||
"label": "Skills",
|
||||
"collapsed": True,
|
||||
"items": skills_items,
|
||||
}
|
||||
skills_subtree = "\n".join(_render_sidebar_item(skills_top, 8)) + "\n"
|
||||
|
||||
sidebar_path = REPO / "website" / "sidebars.ts"
|
||||
text = sidebar_path.read_text(encoding="utf-8")
|
||||
# Replace the existing Skills block.
|
||||
pattern = re.compile(
|
||||
r" \{\n"
|
||||
r" type: 'category',\n"
|
||||
r" label: 'Skills',\n"
|
||||
r"(?:.*?\n)*?"
|
||||
r" \},\n",
|
||||
re.DOTALL,
|
||||
)
|
||||
# Safer: match the exact current block shape.
|
||||
old_block_start = " {\n type: 'category',\n label: 'Skills',\n"
|
||||
i = text.find(old_block_start)
|
||||
if i == -1:
|
||||
raise RuntimeError("Could not find Skills sidebar block to replace")
|
||||
# Find matching closing of this block -- walk brace depth
|
||||
depth = 0
|
||||
j = i
|
||||
while j < len(text):
|
||||
ch = text[j]
|
||||
if ch == "{":
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
# Include the trailing ,\n after the closing brace
|
||||
end = text.find("\n", j) + 1
|
||||
break
|
||||
j += 1
|
||||
else:
|
||||
raise RuntimeError("Could not find end of Skills sidebar block")
|
||||
|
||||
new_text = text[:i] + skills_subtree + text[end:]
|
||||
sidebar_path.write_text(new_text, encoding="utf-8")
|
||||
print(f"Updated sidebar: {sidebar_path}")
|
||||
|
||||
|
||||
def main():
|
||||
entries = discover_skills()
|
||||
print(f"Discovered {len(entries)} skills")
|
||||
|
||||
# Build name -> meta index for related-skill cross-linking
|
||||
skill_index: dict[str, dict[str, Any]] = {}
|
||||
for meta, parsed in entries:
|
||||
name = parsed["frontmatter"].get("name", meta["slug"])
|
||||
# Prefer bundled over optional if a name collision exists
|
||||
if name not in skill_index or meta["source_kind"] == "bundled":
|
||||
skill_index[name] = meta
|
||||
|
||||
# Write per-skill pages
|
||||
written = 0
|
||||
for meta, parsed in entries:
|
||||
out_path = page_output_path(meta)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
content = render_skill_page(
|
||||
meta, parsed["frontmatter"], parsed["body"], skill_index=skill_index
|
||||
)
|
||||
out_path.write_text(content, encoding="utf-8")
|
||||
written += 1
|
||||
print(f"Wrote {written} per-skill pages under {SKILLS_PAGES}")
|
||||
|
||||
# Regenerate catalogs
|
||||
bundled_catalog = build_catalog_md_bundled(entries)
|
||||
(DOCS / "reference" / "skills-catalog.md").write_text(bundled_catalog, encoding="utf-8")
|
||||
print("Updated reference/skills-catalog.md")
|
||||
|
||||
optional_catalog = build_catalog_md_optional(entries)
|
||||
(DOCS / "reference" / "optional-skills-catalog.md").write_text(optional_catalog, encoding="utf-8")
|
||||
print("Updated reference/optional-skills-catalog.md")
|
||||
|
||||
# Update sidebar
|
||||
write_sidebar(entries)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env node
|
||||
// Runs website/scripts/extract-skills.py and generate-llms-txt.py before
|
||||
// docusaurus build/start so that:
|
||||
// - website/static/api/skills.json (lazy-fetched by src/pages/skills/index.tsx)
|
||||
// - website/static/api/skills-meta.json (sidecar metadata for the Skills Hub)
|
||||
// - website/static/llms.txt (agent-friendly short docs index)
|
||||
// - website/static/llms-full.txt (full docs concat for LLM context)
|
||||
// all exist without contributors remembering to run Python scripts manually.
|
||||
// CI workflows still run the extraction explicitly, which is a no-op duplicate
|
||||
// but matches their historical behaviour.
|
||||
//
|
||||
// We also try to pull a fresh copy of skills-index.json (the unified
|
||||
// multi-source catalog) from the live docs site if it's not already on disk.
|
||||
// That way local `npm run build` doesn't have to wait on
|
||||
// scripts/build_skills_index.py crawling every skill source — which takes
|
||||
// several minutes and burns GitHub API quota — but still gets the same
|
||||
// 2000+ external skills the deployed site has.
|
||||
//
|
||||
// If python3 or its deps (pyyaml) aren't available on the local machine, we
|
||||
// fall back to writing an empty skills.json so `npm run build` still
|
||||
// succeeds — the Skills Hub page just shows an empty state, and llms.txt
|
||||
// generation is skipped. CI always has the deps installed, so production
|
||||
// deploys get real data.
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, writeFileSync, existsSync, statSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const websiteDir = resolve(scriptDir, "..");
|
||||
const extractScript = join(scriptDir, "extract-skills.py");
|
||||
const llmsScript = join(scriptDir, "generate-llms-txt.py");
|
||||
const cronBlueprintsScript = join(scriptDir, "extract-automation-blueprints.py");
|
||||
const outputFile = join(websiteDir, "static", "api", "skills.json");
|
||||
const unifiedIndexFile = join(websiteDir, "static", "api", "skills-index.json");
|
||||
const UNIFIED_INDEX_URL =
|
||||
"https://hermes-agent.nousresearch.com/docs/api/skills-index.json";
|
||||
const UNIFIED_INDEX_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24h
|
||||
|
||||
function writeEmptyFallback(reason) {
|
||||
mkdirSync(dirname(outputFile), { recursive: true });
|
||||
writeFileSync(outputFile, "[]\n");
|
||||
console.warn(
|
||||
`[prebuild] extract-skills.py skipped (${reason}); wrote empty skills.json. ` +
|
||||
`Install python3 + pyyaml locally for a populated Skills Hub page.`,
|
||||
);
|
||||
}
|
||||
|
||||
function runPython(script, label) {
|
||||
if (!existsSync(script)) {
|
||||
console.warn(`[prebuild] ${label} skipped (script missing)`);
|
||||
return false;
|
||||
}
|
||||
const r = spawnSync("python3", [script], { stdio: "inherit", cwd: websiteDir });
|
||||
if (r.error && r.error.code === "ENOENT") {
|
||||
console.warn(`[prebuild] ${label} skipped (python3 not found)`);
|
||||
return false;
|
||||
}
|
||||
if (r.status !== 0) {
|
||||
console.warn(`[prebuild] ${label} exited with status ${r.status}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function ensureUnifiedIndex() {
|
||||
// If we have a recent copy on disk, trust it.
|
||||
if (existsSync(unifiedIndexFile)) {
|
||||
try {
|
||||
const age = Date.now() - statSync(unifiedIndexFile).mtimeMs;
|
||||
if (age < UNIFIED_INDEX_MAX_AGE_MS) {
|
||||
return true;
|
||||
}
|
||||
console.log(
|
||||
`[prebuild] skills-index.json is ${(age / 3600000).toFixed(1)}h old; ` +
|
||||
`refreshing from ${UNIFIED_INDEX_URL}`,
|
||||
);
|
||||
} catch {
|
||||
// fall through to re-fetch
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const resp = await fetch(UNIFIED_INDEX_URL, {
|
||||
headers: { accept: "application/json" },
|
||||
});
|
||||
if (!resp.ok) {
|
||||
console.warn(
|
||||
`[prebuild] skills-index.json fetch returned HTTP ${resp.status}; ` +
|
||||
`using local copy if any`,
|
||||
);
|
||||
return existsSync(unifiedIndexFile);
|
||||
}
|
||||
const text = await resp.text();
|
||||
// Sanity check: must be valid JSON with a skills array
|
||||
try {
|
||||
const parsed = JSON.parse(text);
|
||||
if (!parsed || !Array.isArray(parsed.skills)) {
|
||||
console.warn(
|
||||
"[prebuild] skills-index.json from live site has no skills array; ignoring",
|
||||
);
|
||||
return existsSync(unifiedIndexFile);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(`[prebuild] skills-index.json from live site is not valid JSON: ${e}`);
|
||||
return existsSync(unifiedIndexFile);
|
||||
}
|
||||
mkdirSync(dirname(unifiedIndexFile), { recursive: true });
|
||||
writeFileSync(unifiedIndexFile, text);
|
||||
console.log(
|
||||
`[prebuild] downloaded skills-index.json from ${UNIFIED_INDEX_URL} ` +
|
||||
`(${(text.length / 1024).toFixed(0)} KB)`,
|
||||
);
|
||||
return true;
|
||||
} catch (e) {
|
||||
console.warn(`[prebuild] skills-index.json fetch failed: ${e}`);
|
||||
return existsSync(unifiedIndexFile);
|
||||
}
|
||||
}
|
||||
|
||||
// 0) Pull unified index if we don't have a fresh one.
|
||||
await ensureUnifiedIndex();
|
||||
|
||||
// 1) skills.json — required for the Skills Hub page.
|
||||
if (!existsSync(extractScript)) {
|
||||
writeEmptyFallback("extract script missing");
|
||||
} else {
|
||||
const r = spawnSync("python3", [extractScript], {
|
||||
stdio: "inherit",
|
||||
cwd: websiteDir,
|
||||
});
|
||||
if (r.error && r.error.code === "ENOENT") {
|
||||
writeEmptyFallback("python3 not found");
|
||||
} else if (r.status !== 0) {
|
||||
writeEmptyFallback(`extract-skills.py exited with status ${r.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 2) llms.txt + llms-full.txt — agent-friendly docs entrypoints. Non-fatal.
|
||||
runPython(llmsScript, "generate-llms-txt.py");
|
||||
|
||||
// 3) automation-blueprints-index.json — Automation Blueprints catalog page. Non-fatal; the page
|
||||
// renders an empty state if the generator can't run.
|
||||
runPython(cronBlueprintsScript, "extract-automation-blueprints.py");
|
||||
Reference in New Issue
Block a user