Zakaria/FlociClone
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

second first commit

Browse Source
This commit is contained in:
Zakaria
2026-09-22 08:57:00 -04:00
parent e88d7e2ff3
commit 0aef7fb2d8
4766 changed files with 1236986 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/docs/service_inventory.py
+577
View File
@@ -0,0 +1,577 @@
#!/usr/bin/env python3
"""Reconcile every source that answers "how many AWS services does Floci support?".
There is no single number, and that is not a bug: twelve sources in this repo each
count something slightly different, and each is right on its own terms. A descriptor
is a signing endpoint, a matrix row is an API surface, a doc page is a service a user
browses to, a README category entry is a marketing name. Asked "how many services",
people reach for whichever is nearest and get a different answer every time.
This script prints all twelve, groups them into the five keyspaces that can actually
be compared, and for each comparison prints the delta *as a number and as the named
entries that explain it*. An unexplained delta is a finding; an explained one is a
design decision that should read as deliberate.
It is a human-facing report, not a CI gate: it is read-only, and it exits non-zero
only when a source cannot be parsed. It deliberately does not duplicate the two real
gates that already exist (check_service_matrix.py for catalog-to-matrix,
ServiceConfigYamlCoverageTest for config-to-YAML); both are named in the output so an
advisory comparison can never be mistaken for an enforced one.
python3 tools/docs/service_inventory.py # the reconciliation report
python3 tools/docs/service_inventory.py --json # the same data, machine-readable
**Never copy a number from this output into README.md or docs/.** The repo's rule is
that no service count is published anywhere, derived or not; user-facing pages link to
the Service Matrix instead. This tool exists so the question can be answered in
conversation, not so the answer can be pasted into a page that then goes stale.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from datetime import date, timedelta
from pathlib import Path
import yaml
import check_service_matrix as csm
CONFIG_SOURCE = "src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java"
APPLICATION_YML = "src/main/resources/application.yml"
README = "README.md"
SERVICES_YAML = "tools/docs/services.yaml"
PACKAGE_DIR = "src/main/java/io/github/hectorvent/floci/services"
# Warn this far ahead of a deferred entry's expiry, so a red main is a decision rather
# than a surprise on the morning the date passes.
DEFERRAL_WARN_DAYS = 30
# Entries under floci.services.* that emulate nothing from AWS. The config and YAML
# collectors count them correctly (they are services Floci runs), so they have to be
# subtracted before those counts can be compared with anything AWS-shaped.
NON_AWS_CONFIG_KEYS = {"duck", "ui"}
# Packages under services/ that back no descriptor of their own, each with the reason.
# The point of listing them is that the package-count delta reads as explained; an
# entry appearing in the UNREGISTERED list instead is the signal that a package was
# added without being registered in the catalog.
PACKAGE_NOTES = {
"cloudwatch": "one package, two descriptors: cloudwatchlogs and cloudwatchmetrics",
"floci": "Floci's own control surface, not an AWS service",
"lambdamicrovms": "controllers attached to the lambda descriptor",
"resourcegroupstagging": "registered under the AWS signing name, tagging",
"securityadmin": "shared admin-accounts endpoint over GuardDuty and Macie",
"signin": "controller attached to the signin descriptor, which is keyed iam",
"ssoportal": "controller attached to the sso descriptor, which is keyed ssoadmin",
}
# README category names whose wording differs from the matrix row label they refer to.
# Matching is by normalised name, so only genuine wording differences need an entry,
# and one README name may cover several matrix rows. This assists a human-facing
# report and gates nothing, which is why it lives here rather than in
# service_matrix.yaml next to the aliases the CI gate reads.
README_NAME_ALIASES: dict[str, str | list[str]] = {
"API Gateway REST": "API Gateway v1",
"Bedrock": "Amazon Bedrock",
"ELB Classic": "ELB Classic (v1)",
"Firehose": "Data Firehose",
"Resource Explorer 2": "Resource Explorer",
"Service Catalog": "AWS Service Catalog",
"IAM Identity Center (SSO Admin, OIDC, Access Portal, SCIM)": [
"IAM Identity Center (SSO Admin)",
"IAM Identity Center OIDC",
"IAM Identity Center Access Portal",
],
}
CONFIG_ACCESSOR_RE = re.compile(r"^\s{8}[A-Za-z0-9]+ +([a-zA-Z0-9]+)\(\);", re.M)
ENABLED_CALL_RE = re.compile(r"config\.services\(\)\.([a-zA-Z0-9]+)\(\)\.enabled\(\)")
MATRIX_ROW_RE = re.compile(r"^\| \[([^\]]+)\]\(([a-z0-9][a-z0-9\-]*)\.md([^)]*)\)", re.M)
def _repo_root() -> Path:
return Path(__file__).resolve().parent.parent.parent
def key(name: str) -> str:
"""Canonical form of a service key, for comparing across naming conventions.
The same service is `bcmDataExports` as a config accessor, `bcm-data-exports` in
application.yml and `bcmdataexports` as a package directory. Comparing the raw
spellings reports eight bogus mismatches in a repo where nothing is actually wrong,
so every cross-source comparison goes through here.
"""
return re.sub(r"[^a-z0-9]", "", name.lower())
# --------------------------------------------------------------------------- #
# Java catalog
# --------------------------------------------------------------------------- #
def split_top_level_args(text: str, start: int) -> list[str]:
"""Split the argument list of a call whose opening paren is at or after `start`.
Splits on commas at paren depth 1 only, so a nested `config.services().x().enabled()`
or `Set.of("a", "b")` argument stays in one piece. Raises ValueError if the parens
never balance.
"""
depth = 0
args: list[str] = []
current: list[str] = []
for i in range(start, len(text)):
ch = text[i]
if ch == "(":
depth += 1
if depth == 1:
continue
elif ch == ")":
depth -= 1
if depth == 0:
args.append("".join(current).strip())
return args
elif ch == "," and depth == 1:
args.append("".join(current).strip())
current = []
continue
current.append(ch)
raise ValueError("unbalanced parentheses in descriptor(...) call")
def parse_descriptors(java_source: str) -> list[dict]:
"""externalKey, configKey, enable toggles and includeInStatus of every call.
check_service_matrix.py needs only the externalKey, so one regex serves it. This
needs the first four arguments, and the fourth can sit on a continuation line:
rds-data and redshift-data wrap because their enabled expression is a conjunction
of two toggles. Matching off the opening line would read `true` from the wrong
argument for exactly those two, so the arguments are split properly instead.
"""
out: list[dict] = []
for m in csm.ALL_DESCRIPTOR_CALLS_RE.finditer(java_source):
open_paren = java_source.index("(", m.start())
args = split_top_level_args(java_source, open_paren)
if len(args) < 4:
raise ValueError(f"descriptor(...) call with only {len(args)} arguments")
external, config_key, enabled, include = args[0], args[1], args[2], args[3]
if not (external.startswith('"') and external.endswith('"')):
raise ValueError(
f"descriptor(...) externalKey is not a plain string literal: {external!r}"
)
out.append(
{
"external_key": external.strip('"'),
"config_key": config_key.strip('"') if config_key.startswith('"') else None,
"toggles": sorted(set(ENABLED_CALL_RE.findall(enabled))),
"in_status": include.strip() == "true",
}
)
return out
# --------------------------------------------------------------------------- #
# Config plane
# --------------------------------------------------------------------------- #
def parse_config_accessors(java_source: str) -> list[str]:
"""Accessors on EmulatorConfig.ServicesConfig that return a per-service config.
Scoped to that interface body, because the same accessor shape appears on other
nested interfaces. `dockerNetwork()` is excluded by the return-type pattern: it
returns `Optional<String>`, and the `<` keeps it out.
"""
try:
start = java_source.index(" interface ServicesConfig {")
except ValueError:
raise ValueError(f"{CONFIG_SOURCE}: interface ServicesConfig not found") from None
end = java_source.index("\n }", start)
return sorted(CONFIG_ACCESSOR_RE.findall(java_source[start:end]))
def parse_yaml_service_keys(path: Path) -> tuple[list[str], list[str]]:
"""(every floci.services.* key, the subset declaring `enabled:`)."""
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
services = (data.get("floci") or {}).get("services") or {}
keys = [k for k, v in services.items() if isinstance(v, dict)]
return sorted(keys), sorted(k for k in keys if "enabled" in services[k])
# --------------------------------------------------------------------------- #
# Docs and README
# --------------------------------------------------------------------------- #
def parse_matrix_rows(md_source: str) -> list[dict]:
"""One entry per Service Matrix row: label, page slug, anchor.
A row is a distinct API surface; a page is a documented service. They differ
because eight pages carry two rows each (DynamoDB and DynamoDB Streams, IAM and
AWS Sign-In, and so on). The facet test is **two rows sharing a slug**, not "the
link carries an anchor": IoT Core and IoT Data both link plain `iot.md` with no
anchor at all, so an anchor-based rule misses that pair and over-counts by one.
"""
start = md_source.index(csm.MATRIX_HEADING)
end = md_source.index(csm.MATRIX_END_HEADING, start)
return [
{"label": label, "slug": slug, "anchor": anchor.lstrip("#") or None}
for label, slug, anchor in MATRIX_ROW_RE.findall(md_source[start:end])
]
def parse_readme_category_names(md_source: str) -> list[str]:
"""Service names from the README's Category/Services table.
Split on commas at paren depth 0 only. One entry reads `IAM Identity Center (SSO
Admin, OIDC, Access Portal, SCIM)`; splitting on every comma turns that single
name into four, three of which ("OIDC", "SCIM)") are not service names at all and
inflate the count by three.
"""
start = md_source.index("| Category | Services |")
end = md_source.index("For operation-level compatibility", start)
names: list[str] = []
for line in md_source[start:end].splitlines():
if not line.startswith("| ") or line.startswith("|---") or line.startswith("| Category"):
continue
cells = [c.strip() for c in line.strip("|").split("|")]
if len(cells) < 2:
continue
depth = 0
current: list[str] = []
for ch in cells[1]:
if ch == "(":
depth += 1
elif ch == ")":
depth -= 1
elif ch == "," and depth == 0:
names.append("".join(current).strip())
current = []
continue
current.append(ch)
if "".join(current).strip():
names.append("".join(current).strip())
return names
def parse_readme_detail_rows(md_source: str) -> list[str]:
"""Service names from the README's collapsed "Detailed service notes" table."""
header = "| Service | How it works | Notable features |"
start = md_source.index(header)
end = md_source.index("</details>", start)
rows = []
for line in md_source[start:end].splitlines():
if not line.startswith("| ") or line.startswith("|---") or line == header:
continue
rows.append(line.strip("|").split("|")[0].strip())
return rows
def parse_action_table_services(path: Path) -> list[str]:
"""Service keys in services.yaml, the opt-in list for generated action tables."""
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
return sorted(entry["service"] for entry in (data.get("services") or []))
# --------------------------------------------------------------------------- #
# Reconciliation
# --------------------------------------------------------------------------- #
def collect(repo_root: Path) -> dict:
"""Every source, parsed. Raises ValueError if any source cannot be read."""
descriptors = parse_descriptors((repo_root / csm.CATALOG_SOURCE).read_text(encoding="utf-8"))
if not descriptors:
raise ValueError(f"no descriptor(...) calls found in {csm.CATALOG_SOURCE}")
yaml_keys, yaml_enabled = parse_yaml_service_keys(repo_root / APPLICATION_YML)
readme_src = (repo_root / README).read_text(encoding="utf-8")
aliases, deferred, facet_pages = csm._load_registry(repo_root)
return {
"descriptors": descriptors,
"config_accessors": parse_config_accessors(
(repo_root / CONFIG_SOURCE).read_text(encoding="utf-8")
),
"yaml_keys": yaml_keys,
"yaml_enabled": yaml_enabled,
"packages": sorted(p.name for p in (repo_root / PACKAGE_DIR).iterdir() if p.is_dir()),
"matrix_rows": parse_matrix_rows(
(repo_root / csm.MATRIX_DOC).read_text(encoding="utf-8")
),
"doc_pages": sorted(
p.stem for p in (repo_root / csm.SERVICES_DIR).glob("*.md") if p.name != "index.md"
),
"readme_categories": parse_readme_category_names(readme_src),
"readme_details": parse_readme_detail_rows(readme_src),
"action_tables": parse_action_table_services(repo_root / SERVICES_YAML),
"aliases": aliases,
"facet_pages": sorted(facet_pages),
"deferred": [{"key": d.key, "reason": d.reason, "by": d.by.isoformat()} for d in deferred],
}
def reconcile(data: dict, today: date | None = None) -> dict:
"""Counts, deltas, and the named entries that explain each delta."""
today = today or date.today()
descriptors = data["descriptors"]
aliases = data["aliases"]
# --- catalog ---------------------------------------------------------- #
keys = [d["external_key"] for d in descriptors]
page_of = {k: aliases.get(k, k) for k in keys}
owners: dict[str, list[str]] = {}
for k in keys:
owners.setdefault(page_of[k], []).append(k)
facet_descriptors = sorted(k for k in keys if owners[page_of[k]][0] != k)
status_hidden = sorted(d["external_key"] for d in descriptors if not d["in_status"])
toggle_users: dict[str, list[str]] = {}
for d in descriptors:
for t in d["toggles"]:
toggle_users.setdefault(key(t), []).append(d["external_key"])
shared = {t: v for t, v in sorted(toggle_users.items()) if len(v) > 1}
# --- config plane ----------------------------------------------------- #
accessors = {key(a): a for a in data["config_accessors"]}
yaml_keys = {key(k): k for k in data["yaml_keys"]}
aws_keys = {k: v for k, v in yaml_keys.items() if v not in NON_AWS_CONFIG_KEYS}
untoggled = sorted(v for k, v in aws_keys.items() if k not in toggle_users)
# --- packages --------------------------------------------------------- #
descriptor_keys = {key(d["config_key"]) for d in descriptors if d["config_key"]}
packages = data["packages"]
noted = sorted(p for p in packages if p in PACKAGE_NOTES)
unregistered = sorted(
p for p in packages if key(p) not in descriptor_keys and p not in PACKAGE_NOTES
)
# --- docs ------------------------------------------------------------- #
rows = data["matrix_rows"]
slug_rows: dict[str, list[str]] = {}
for r in rows:
slug_rows.setdefault(r["slug"], []).append(r["label"])
facet_rows = sorted(r["label"] for r in rows if slug_rows[r["slug"]][0] != r["label"])
doc_pages = set(data["doc_pages"])
expiring = [
d for d in data["deferred"]
if date.fromisoformat(d["by"]) <= today + timedelta(days=DEFERRAL_WARN_DAYS)
]
# --- README ----------------------------------------------------------- #
row_labels = {key(r["label"]) for r in rows}
facet_norms = {key(label) for label in facet_rows}
def cover(names: list[str]) -> tuple[set[str], list[str]]:
"""(matrix labels these README names account for, names matching no row).
A name resolves through README_NAME_ALIASES when its wording differs from the
matrix label, and one alias may cover several rows: the README names IAM
Identity Center once where the matrix carries three rows for it.
"""
covered: set[str] = set()
unmatched: list[str] = []
for name in names:
target = README_NAME_ALIASES.get(name, name)
norms = [key(t) for t in (target if isinstance(target, list) else [target])]
if all(n in row_labels for n in norms):
covered |= set(norms)
else:
unmatched.append(name)
return covered, unmatched
category_covered, unmatched_names = cover(data["readme_categories"])
detail_covered, _ = cover(data["readme_details"])
def uncovered(covered: set[str], facets: bool) -> list[str]:
return [
r["label"]
for r in rows
if key(r["label"]) not in covered and (key(r["label"]) in facet_norms) is facets
]
return {
"counts": {
"catalog_descriptors": len(keys),
"catalog_distinct_products": len(keys) - len(facet_descriptors),
"catalog_status_visible": len(keys) - len(status_hidden),
"catalog_toggles": len(toggle_users),
"service_packages": len(packages),
"config_accessors": len(accessors),
"yaml_keys": len(yaml_keys),
"yaml_enabled": len(data["yaml_enabled"]),
"yaml_aws_keys": len(aws_keys),
"matrix_rows": len(rows),
"matrix_pages": len(slug_rows),
"doc_pages": len(doc_pages),
"readme_category_names": len(data["readme_categories"]),
"readme_detail_rows": len(data["readme_details"]),
"action_tables": len(data["action_tables"]),
},
"catalog": {
"facet_descriptors": facet_descriptors,
"status_hidden": status_hidden,
"shared_toggles": {t: v for t, v in shared.items()},
},
"config": {
"non_aws": sorted(v for k, v in yaml_keys.items() if v in NON_AWS_CONFIG_KEYS),
"accessor_only": sorted(accessors[k] for k in accessors.keys() - yaml_keys.keys()),
"yaml_only": sorted(yaml_keys[k] for k in yaml_keys.keys() - accessors.keys()),
"no_enabled_key": sorted(set(data["yaml_keys"]) - set(data["yaml_enabled"])),
"untoggled": untoggled,
"toggle_without_key": sorted(
t for t in toggle_users if t not in yaml_keys
),
},
"packages": {"noted": noted, "unregistered": unregistered},
"docs": {
"facet_rows": facet_rows,
"pages_without_row": sorted(doc_pages - set(slug_rows) - set(data["facet_pages"])),
"rows_without_page": sorted(set(slug_rows) - doc_pages),
"facet_pages": data["facet_pages"],
"deferred": data["deferred"],
"deferred_expiring": expiring,
},
"readme": {
"unmatched_category_names": unmatched_names,
"missing_from_categories": uncovered(category_covered, facets=False),
"missing_facet_rows": uncovered(category_covered, facets=True),
"missing_from_details": uncovered(detail_covered, facets=False),
},
}
# --------------------------------------------------------------------------- #
# Report
# --------------------------------------------------------------------------- #
def _names(items: list[str], limit: int = 10) -> str:
if not items:
return ""
shown = ", ".join(items[:limit])
return shown if len(items) <= limit else f"{shown}, +{len(items) - limit} more"
def render(result: dict) -> str:
c = result["counts"]
cat, cfg, pkg, docs, rd = (
result["catalog"], result["config"], result["packages"],
result["docs"], result["readme"],
)
out: list[str] = []
w = out.append
def line(label: str, count: int | str, note: str = "") -> None:
w(f" {label:<32}{str(count):>5} {note}".rstrip())
def delta(label: str, items: list[str], limit: int = 10) -> None:
w(f" {'- ' + label:<32}{len(items):>5} {_names(items, limit)}".rstrip())
def note(text: str) -> None:
w(f" {text}")
w("")
w("THE ANSWER")
w(f" Floci documents {c['matrix_pages']} AWS services, exposing {c['matrix_rows']} distinct")
w(f" API surfaces across {c['catalog_descriptors']} signing endpoints.")
w(" Quote the first for \"how many services\", the second when the question is really")
w(" about API coverage. Never write any of them into README.md or docs/: link to the")
w(" Service Matrix instead.")
w("")
w("1. CATALOG (keyspace: descriptor externalKey)")
line("catalog descriptors", c["catalog_descriptors"], csm.CATALOG_SOURCE)
delta("facet endpoints", cat["facet_descriptors"])
line("= distinct products", c["catalog_distinct_products"])
line("status-visible at runtime", c["catalog_status_visible"],
f"GET /health; hidden: {_names(cat['status_hidden'])}")
line("distinct enable toggles", c["catalog_toggles"])
for toggle, users in cat["shared_toggles"].items():
note(f"{toggle} gates {len(users)}: {_names(users)}")
w("")
w("2. CONFIG PLANE (keyspace: floci.services.* key, naming normalised)")
line("EmulatorConfig accessors", c["config_accessors"], CONFIG_SOURCE)
line("application.yml keys", c["yaml_keys"], APPLICATION_YML)
if cfg["accessor_only"] or cfg["yaml_only"]:
delta("accessor with no YAML key", cfg["accessor_only"])
delta("YAML key with no accessor", cfg["yaml_only"])
else:
line("= identical sets", "", "GATED by ServiceConfigYamlCoverageTest")
delta("no `enabled:` in YAML", cfg["no_enabled_key"])
delta("non-AWS entries", cfg["non_aws"])
line("= AWS service keys", c["yaml_aws_keys"])
delta("no descriptor reads its toggle", cfg["untoggled"])
delta("toggle with no YAML key", cfg["toggle_without_key"])
w("")
w("3. SOURCE (keyspace: Java package, naming normalised)")
line("services/ packages", c["service_packages"], PACKAGE_DIR)
delta("no descriptor, explained", pkg["noted"])
for p in pkg["noted"]:
note(f"{p}: {PACKAGE_NOTES[p]}")
if pkg["unregistered"]:
delta("UNREGISTERED", pkg["unregistered"])
note(f"A package under {PACKAGE_DIR} that no descriptor(...) names.")
note("Register it, or add it to PACKAGE_NOTES with the reason it has none.")
else:
line("= every other package registered", "", "ADVISORY, no gate")
w("")
w("4. DOCS (keyspace: docs/services slug)")
line("matrix rows", c["matrix_rows"], csm.MATRIX_DOC)
delta("facet rows (share a page)", docs["facet_rows"])
line("= matrix pages", c["matrix_pages"])
line("docs/services/*.md", c["doc_pages"])
if docs["pages_without_row"] or docs["rows_without_page"]:
delta("page with no row", docs["pages_without_row"])
delta("row with no page", docs["rows_without_page"])
else:
line("= identical sets", "", "GATED by check_service_matrix.py --strict")
if docs["facet_pages"]:
note(f"facet_pages, row-exempt by registry: {_names(docs['facet_pages'])}")
for d in docs["deferred"]:
flag = " <-- EXPIRING" if d in docs["deferred_expiring"] else ""
note(f"deferred: {d['key']} until {d['by']}{flag}")
note(f" {d['reason']}")
w("")
w("5. README (keyspace: service name, matched to matrix row labels)")
line("category-table names", c["readme_category_names"], f"{README}, Supported Services")
line("matrix rows they cover",
c["matrix_rows"] - len(rd["missing_from_categories"]) - len(rd["missing_facet_rows"]))
delta("row missing from README", rd["missing_from_categories"])
delta("facet row, absent by design", rd["missing_facet_rows"])
if rd["unmatched_category_names"]:
delta("README name matching no row", rd["unmatched_category_names"], limit=4)
note("Wording drift, a service not in the matrix, or a missing")
note("README_NAME_ALIASES entry. Read it, do not assume.")
note("ADVISORY, no gate: docs/contributing.md asks contributors for a README")
note("row as well as a matrix row, but only the matrix row is machine-checked.")
w("")
line("detail-table rows", c["readme_detail_rows"], "editorial \"notable features\"")
delta("documented service with no note", rd["missing_from_details"], limit=6)
note("Deliberately ungated: the detail table is editorial, and a row per")
note(f"service would buy {len(rd['missing_from_details'])} lines of filler prose.")
w("")
w("6. ACTION TABLES (keyspace: services.yaml entry)")
line("generated action tables", c["action_tables"], SERVICES_YAML)
note(f"Opt-in by design, not a gap: {c['doc_pages'] - c['action_tables']} pages carry no")
note("generated operation table. regen_action_docs.py owns this list.")
w("")
return "\n".join(out)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--json", action="store_true", help="emit the raw data instead of the report"
)
args = parser.parse_args(argv)
try:
result = reconcile(collect(_repo_root()))
except (ValueError, KeyError, yaml.YAMLError, OSError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
print(json.dumps(result, indent=2, sort_keys=True) if args.json else render(result))
return 0
if __name__ == "__main__":
sys.exit(main())