second first commit
This commit is contained in:
Zakaria
@@ -0,0 +1,241 @@
|
||||
"""Opt-in AWS oracle capture. Only synthetic, read-only TestMetricFilter requests."""
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
def cases():
|
||||
result = []
|
||||
|
||||
def add(name, pattern, messages):
|
||||
result.append({"id": name, "request": {
|
||||
"filterPattern": pattern, "logEventMessages": messages}})
|
||||
|
||||
terms = ["ERROR", "ERRORS", "xERROR", "error", "[ERROR]", "ERROR_code",
|
||||
"ERROR timeout", "INFO timeout", "WARN", "two words", "two words"]
|
||||
for name, pattern in [
|
||||
("substring", "ERROR"), ("case", "error"), ("and", "ERROR timeout"),
|
||||
("negative", "ERROR -timeout"), ("negative-only", "-ERROR"),
|
||||
("optional", "?ERROR ?WARN"), ("optional-ignored", "?ERROR timeout"),
|
||||
("optional-negative", "?ERROR -timeout"), ("phrase", '"two words"'),
|
||||
("negative-phrase", '-"two words"'), ("optional-phrase", '?"two words" ?WARN'),
|
||||
("wildcard", "ERR*"), ("quoted-wildcard", '"ERR*"'),
|
||||
("bare-symbol", "ERROR!"), ("bare-colon", "ERROR:"),
|
||||
("empty", ""), ("blank", " "), ("quoted-empty", '""'),
|
||||
("missing-quote", '"ERROR'), ("adjacent-quote", '"ERROR"WARN'),
|
||||
]:
|
||||
add("terms-" + name, pattern, terms)
|
||||
|
||||
regex_messages = ["ERROR", "WARN", "aaaa", "aa", "cat", "a cat", "hat",
|
||||
"color", "colour", "a-b", "a b", "a\nb", "123", "_", "café",
|
||||
"key:value", "ab", "back\\slash", "!"]
|
||||
for name, pattern in [
|
||||
("alternation", "%ERROR|WARN%"), ("anchors", "%^[hc]at$%"),
|
||||
("quantifier", "%^a{3,5}$%"), ("optional", "%colou?r%"),
|
||||
("digits", "%\\d+%"), ("word", "%\\w+%"), ("space", "%a\\sb%"),
|
||||
("hex", "%\\x3A%"), ("dot-newline", "%a.b%"),
|
||||
("escaped-symbol", "%a\\-b%"), ("group-rejected", "%(ab)+%"),
|
||||
("unicode-rejected", "%café%"), ("symbol-rejected", "%!%"),
|
||||
("escape-rejected", "%\\p%"), ("escaped-group", "%\\(ab\\)%"),
|
||||
("empty", "%%"), ("unclosed", "%abc"), ("bad-range", "%[z-a]%"),
|
||||
("lazy", "%a+?%"), ("possessive", "%a++%"),
|
||||
("class-intersection", "%[a&&b]%"), ("two", "%ERROR% %WARN%"),
|
||||
("three", "%ERROR% %WARN% %INFO%"),
|
||||
]:
|
||||
add("regex-" + name, pattern, regex_messages)
|
||||
|
||||
json_messages = [
|
||||
'{"a":1}', '{"a":"1"}', '{"a":1.0}', '{"a":"01"}', '{"a":true}',
|
||||
'{"a":false}', '{"a":"true"}', '{"a":null}', '{}', '{"a":{}}',
|
||||
'{"a":[]}', '{"a":"foo"}', '{"a":"foobar"}', '{"a":"FOO"}',
|
||||
'{"a":1e3}', '{"a":"1e3"}', '{"a":1e309}', '{"a":"1e309"}',
|
||||
'{"a":1e-999}', '{"a":-0.0}', '{"a":9007199254740993}',
|
||||
]
|
||||
for name, pattern in [
|
||||
("numeric", "{ $.a = 1 }"), ("quoted-number", '{ $.a = "1" }'),
|
||||
("numeric-ne", "{ $.a != 1 }"), ("string-ne", '{ $.a != "foo" }'),
|
||||
("ordering", "{ $.a > 0 }"), ("quoted-order", '{ $.a > "0" }'),
|
||||
("bare-word", "{ $.a = foo }"), ("bare-boolean", "{ $.a = true }"),
|
||||
("quoted-boolean", '{ $.a = "true" }'), ("is-true", "{ $.a IS TRUE }"),
|
||||
("is-false", "{ $.a IS FALSE }"), ("null", "{ $.a IS NULL }"),
|
||||
("missing", "{ $.a NOT EXISTS }"), ("star", "{ $.a = * }"),
|
||||
("quoted-star", '{ $.a = "*" }'), ("glob", "{ $.a = foo* }"),
|
||||
("exponent", "{ $.a = 1e3 }"), ("plus", "{ $.a = +1 }"),
|
||||
("overflow", "{ $.a = 1e309 }"), ("underflow", "{ $.a = 1e-999 }"),
|
||||
("precision", "{ $.a = 9007199254740992 }"),
|
||||
("regex-number", "{ $.a = %1% }"), ("keyword-and", "{ $.a = 1 AND $.b = 2 }"),
|
||||
("double-equal", "{ $.a == 1 }"), ("trailing", "{ $.a = 1 } x"),
|
||||
]:
|
||||
add("json-" + name, pattern, json_messages)
|
||||
|
||||
selectors = [
|
||||
'{"a":[1,2]}', '{"a":[2,1]}', '{"a":[null,2]}',
|
||||
'{"a":{"x":1,"y":2}}', '{"a":{"x":2,"y":1}}',
|
||||
'{"a":[{"b":1},{"b":2}]}', '{"a.b":2}', '{"a-b":2}',
|
||||
'{"a":null}', '{}', '[1,2]', '{"a":2} trailing',
|
||||
]
|
||||
for name, pattern in [
|
||||
("index", "{ $.a[1] = 2 }"), ("wildcard", "{ $.a[*] = 2 }"),
|
||||
("object-wildcard", "{ $.a.* = 2 }"), ("wildcard-child", "{ $.a[*].b = 2 }"),
|
||||
("quoted-property", "{ $.['a.b'] = 2 }"), ("bracket-property", "{ $['a.b'] = 2 }"),
|
||||
("double-property", '{ $.["a.b"] = 2 }'), ("hyphen", "{ $.a-b = 2 }"),
|
||||
("root-index", "{ $[1] = 2 }"), ("negative-index", "{ $.a[-1] = 2 }"),
|
||||
("null-extraction", "{ $.a IS NULL }"), ("or-extraction", "{ $.a = 9 || $.missing NOT EXISTS }"),
|
||||
("array-ne", "{ $.a[*] != 2 }"), ("nested-wildcard", "{ $.a[*].* = 2 }"),
|
||||
]:
|
||||
add("selector-" + name, pattern, selectors)
|
||||
|
||||
spaced = ["ERROR", "ERROR one", "ERROR one two", "WARN one", "INFO one",
|
||||
'a "two words" c', "a [two words] c", 'a "two words"',
|
||||
"a [two words]", "a b c ", "a\tb\tc", 'a "unclosed', "a [unclosed",
|
||||
'a "b"c d', "one", "1 2", "1.0 2", "a b c d"]
|
||||
for name, pattern in [
|
||||
("fields", "[a, b, c]"), ("last-rest", "[a, rest]"), ("single", "[a]"),
|
||||
("empty", "[]"), ("leading-ellipsis", "[..., last]"),
|
||||
("middle-ellipsis", "[first, ..., last]"), ("trailing-ellipsis", "[first, ...]"),
|
||||
("unnamed", "[, b,]"), ("trailing-comma", "[a,]"),
|
||||
("constraint", "[level=ERROR, rest]"),
|
||||
("or", "[level=ERROR || level=WARN, rest]"),
|
||||
("numeric", "[a=1, rest]"), ("quoted-numeric", '[a="1", rest]'),
|
||||
("cross-field", "[a=ERROR || b=one, b]"),
|
||||
("duplicate", "[a, a]"), ("invalid-name", "[1a]"),
|
||||
("double-ellipsis", "[a, ..., b, ...]"),
|
||||
("bad-operator", "[a == 1]"), ("regex", "[a=%ERR%, rest]"),
|
||||
]:
|
||||
add("space-" + name, pattern, spaced)
|
||||
return result
|
||||
|
||||
|
||||
def edge_cases():
|
||||
numbers = ['{"a":' + value + '}' for value in [
|
||||
'1', '1.0', '1e0', '"1"', '"1.0"', '"01"', '0.0', '-0.0', '-1',
|
||||
'9007199254740992', '9007199254740993', '1e308', '1e309', '"1e309"',
|
||||
'1e-999', '"1e-999"', '0', 'true', '"true"', 'null', '[1,2]', '[null,2]',
|
||||
]]
|
||||
structures = ['{}', '{"a":null}', '{"a":2}', '{"a":1}', '{"a":"2"}',
|
||||
'{"a":[]}', '{"a":[1,2]}', '{"a":[2,1]}', '{"a":[1]}',
|
||||
'{"a":[null,2]}', '{"a":{}}', '{"a":{"b":2}}',
|
||||
'{"a":[{"b":1},{"b":2}]}', '{"a":[{"b":null}]}',
|
||||
'{"a":[[1,2]]}', '{"a":[true,false]}']
|
||||
result = []
|
||||
for name, pattern, messages in [
|
||||
("decimal", "{ $.a = 1.0 }", numbers),
|
||||
("zero", "{ $.a = 0 }", numbers),
|
||||
("negative-zero", "{ $.a = -0.0 }", numbers),
|
||||
("precise-long", "{ $.a = 9007199254740993 }", numbers),
|
||||
("large-finite", "{ $.a = 1e308 }", numbers),
|
||||
("large-ordered", "{ $.a > 1e307 }", numbers),
|
||||
("underflow-ordered", "{ $.a > 1e-999 }", numbers),
|
||||
("string-exponent", '{ $.a = "1e0" }', numbers),
|
||||
("is-null", "{ $.a IS NULL }", structures),
|
||||
("equals", "{ $.a = 2 }", structures),
|
||||
("not-equals", "{ $.a != 2 }", structures),
|
||||
("wildcard-not-equals", "{ $.a[*] != 2 }", structures),
|
||||
("wildcard-null", "{ $.a[*] IS NULL }", structures),
|
||||
("wildcard-missing", "{ $.a[*] NOT EXISTS }", structures),
|
||||
("missing", "{ $.a NOT EXISTS }", structures),
|
||||
("child", "{ $.a.b = 2 }", structures),
|
||||
("child-missing", "{ $.a.b NOT EXISTS }", structures),
|
||||
("star", "{ $.a = * }", structures),
|
||||
("quoted-number", '{ $.a = "2" }', structures),
|
||||
("bare-null", "{ $.a = null }", structures),
|
||||
("is-not-null", "{ $.a IS NOT NULL }", structures),
|
||||
("selector-root", "{ $ = 2 }", structures),
|
||||
("json-negative-regex", "{ $.a != %2% }", structures),
|
||||
("regex-text", "ERROR %WARN%", ["ERROR WARN", "ERROR", "WARN"]),
|
||||
("regex-optional", "?%WARN%", ["WARN", "WARN INFO"]),
|
||||
("term-punctuation", "a-b", ["a-b", "a b"]),
|
||||
("term-underscore", "a_b", ["a_b", "a b"]),
|
||||
("term-dot", "a.b", ["a.b", "a b"]),
|
||||
("term-question", "a?b", ["a?b", "a b"]),
|
||||
("regex-structured-two", "{ $.a = %1% || $.a = %2% }", structures),
|
||||
("regex-structured-three", "{ $.a = %1% || $.a = %2% || $.a = %3% }", structures),
|
||||
("quoted-order-space", '[a>"1", b]', ["2 x", "0 x"]),
|
||||
("json-keyword-case", "{ $.a is null }", structures),
|
||||
("json-unquoted-symbol", "{ $.a = a:b }", ['{"a":"a:b"}']),
|
||||
("space-quote-trailing", "[a, b]", ['x "y" ', "x [y] ", "x y ", "x ", 'x ""', "x []"]),
|
||||
("space-incomplete", "[a, b, c]", ['x "y z', "x [y z", 'x "y z q', "x [y z q"]),
|
||||
]:
|
||||
result.append({"id": "edge-" + name, "request": {
|
||||
"filterPattern": pattern, "logEventMessages": messages}})
|
||||
return result
|
||||
|
||||
|
||||
def boundary_cases():
|
||||
messages = [
|
||||
'{"a":true}', '{"a":false}', '{"a":[true,false]}', '{"a":[null]}',
|
||||
'{"a":[[null]]}', '{"a":[{"b":2},{"b":1}]}', '{"a":[{}]}', '{}',
|
||||
]
|
||||
result = []
|
||||
for name, pattern, inputs in [
|
||||
("array-true", "{ $.a IS TRUE }", messages),
|
||||
("array-false", "{ $.a IS FALSE }", messages),
|
||||
("array-null", "{ $.a IS NULL }", messages),
|
||||
("nested-negative", "{ $.a[*].b != 2 }", messages),
|
||||
("wildcard-null", "{ $.a[*] IS NULL }", messages),
|
||||
("term-slash", "a/b", ["a/b", "a b"]),
|
||||
("term-at", "a@b", ["a@b", "a b"]),
|
||||
("term-unicode", "café", ["café", "cafe"]),
|
||||
("quoted-space", '" "', ["x", "two words", " "]),
|
||||
("huge-exponent", "{ $.a = 1e2147483648 }", ['{"a":1}', '{"a":"1e2147483648"}']),
|
||||
("bare-negative", "a-b", ["a", "b", "a-b", "ab", "a b"]),
|
||||
]:
|
||||
result.append({"id": "boundary-" + name, "request": {
|
||||
"filterPattern": pattern, "logEventMessages": inputs}})
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--profile", required=True)
|
||||
parser.add_argument("--region", required=True)
|
||||
parser.add_argument("--endpoint-url", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--suite", choices=("base", "edges", "boundaries"), default="base")
|
||||
args = parser.parse_args()
|
||||
if args.endpoint_url != f"https://logs.{args.region}.amazonaws.com":
|
||||
parser.error("capture requires the explicit regional AWS Logs endpoint")
|
||||
corpus = {"base": cases, "edges": edge_cases, "boundaries": boundary_cases}[args.suite]()
|
||||
assert len(cases()) + len(edge_cases()) + len(boundary_cases()) <= 149
|
||||
for case in corpus:
|
||||
for attempt in range(4):
|
||||
command = ["aws", "--profile", args.profile, "--region", args.region,
|
||||
"--endpoint-url", args.endpoint_url, "--no-cli-pager",
|
||||
"logs", "test-metric-filter", "--output", "json",
|
||||
"--cli-input-json", json.dumps(case["request"])]
|
||||
completed = subprocess.run(command, capture_output=True, text=True)
|
||||
if "Throttling" not in completed.stderr:
|
||||
break
|
||||
time.sleep(2 ** attempt)
|
||||
if completed.returncode == 0:
|
||||
case["response"] = json.loads(completed.stdout)
|
||||
else:
|
||||
error = re.search(r"An error occurred \((\w+)\).*?: (.*)", completed.stderr)
|
||||
if not error or error.group(1) != "InvalidParameterException":
|
||||
raise RuntimeError("AWS capture blocked: " + completed.stderr)
|
||||
case["error"] = {"code": error.group(1), "message": error.group(2)}
|
||||
print(case["id"], "error" if "error" in case else
|
||||
str(len(case["response"]["matches"])) + " matches", flush=True)
|
||||
time.sleep(0.15)
|
||||
fixture = {
|
||||
"provenance": {
|
||||
"source": "AWS CloudWatch Logs TestMetricFilter",
|
||||
"capturedAt": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"region": args.region,
|
||||
"endpoint": args.endpoint_url,
|
||||
"inputs": "Synthetic only. No resource writes or user logs.",
|
||||
"eventNumberBase": 1,
|
||||
"reference": "https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_TestMetricFilter.html",
|
||||
},
|
||||
"cases": corpus,
|
||||
}
|
||||
args.output.write_text(json.dumps(fixture, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,188 @@
|
||||
# Presentation for the generated "Supported Resource Types" table in
|
||||
# docs/services/cloudformation.md. The set of types is NOT configured here: it comes from
|
||||
# src/test/resources/cloudformation/supported-resource-types.tsv, which CfnResourceInventoryTest
|
||||
# pins to the CDI-resolved provisioner registry and the legacy switch. This file only decides how
|
||||
# those types are labelled, ordered and annotated.
|
||||
#
|
||||
# Adding a resource type: add the provisioner, update the TSV, run `make docs-sync`. Touch this
|
||||
# file only for a new namespace (which needs a label) or to add a note.
|
||||
|
||||
# Row order. Namespaces omitted here are appended alphabetically after these, so a forgotten
|
||||
# entry degrades to a sensible position rather than disappearing.
|
||||
order:
|
||||
- AWS::S3
|
||||
- AWS::SQS
|
||||
- AWS::SNS
|
||||
- AWS::DynamoDB
|
||||
- AWS::Lambda
|
||||
- AWS::IAM
|
||||
- AWS::Organizations
|
||||
- AWS::SSM
|
||||
- AWS::KMS
|
||||
- AWS::SecretsManager
|
||||
- AWS::ECR
|
||||
- AWS::ECS
|
||||
- AWS::EKS
|
||||
- AWS::RDS
|
||||
- AWS::Redshift
|
||||
- AWS::EC2
|
||||
- AWS::ElasticLoadBalancingV2
|
||||
- AWS::AutoScaling
|
||||
- AWS::Route53
|
||||
- AWS::ApiGateway
|
||||
- AWS::ApiGatewayV2
|
||||
- AWS::AppSync
|
||||
- AWS::StepFunctions
|
||||
- AWS::CodePipeline
|
||||
- AWS::CodeBuild
|
||||
- AWS::Batch
|
||||
- AWS::Cognito
|
||||
- AWS::CertificateManager
|
||||
- AWS::Events
|
||||
- AWS::Scheduler
|
||||
- AWS::Backup
|
||||
- AWS::Pipes
|
||||
- AWS::Kinesis
|
||||
- AWS::KinesisFirehose
|
||||
- AWS::IoT
|
||||
- AWS::CloudFront
|
||||
- AWS::CloudWatch
|
||||
- AWS::Logs
|
||||
- AWS::WAFv2
|
||||
- AWS::Config
|
||||
- AWS::CloudFormation
|
||||
- AWS::CloudTrail
|
||||
- Custom
|
||||
- AWS::CDK
|
||||
|
||||
# Namespaces folded into another row. Custom::* types are CloudFormation's own extension
|
||||
# mechanism, not a service of their own, so they belong in the CloudFormation row.
|
||||
merge_namespaces:
|
||||
Custom: AWS::CloudFormation
|
||||
|
||||
# Overrides the name shown for a type. Only needed where the bare leaf would be ambiguous.
|
||||
display_names:
|
||||
AWS::CDK::Metadata: CDK::Metadata
|
||||
|
||||
# Order of type names within a row. Listed names come first in this order; anything else is
|
||||
# appended alphabetically, so a newly provisioned type appears without needing an entry here.
|
||||
# Curated where the reading order carries meaning (primary resources before their accessories).
|
||||
type_order:
|
||||
AWS::S3: [Bucket, BucketPolicy]
|
||||
AWS::SQS: [Queue, QueuePolicy]
|
||||
AWS::SNS: [Topic, Subscription]
|
||||
AWS::DynamoDB: [Table, GlobalTable]
|
||||
AWS::Lambda: [Function, LayerVersion, EventSourceMapping, Version, Alias, Permission]
|
||||
AWS::IAM: [Role, User, AccessKey, Policy, ManagedPolicy, InstanceProfile]
|
||||
AWS::Organizations: [Organization, OrganizationalUnit, Account, Policy, ResourcePolicy]
|
||||
AWS::KMS: [Key, Alias]
|
||||
AWS::SecretsManager: [Secret, SecretTargetAttachment]
|
||||
AWS::ECS: [Cluster, TaskDefinition, Service]
|
||||
AWS::EKS: [Cluster, Nodegroup]
|
||||
AWS::RDS:
|
||||
[DBInstance, DBCluster, DBSubnetGroup, DBParameterGroup, DBClusterParameterGroup]
|
||||
AWS::Redshift:
|
||||
[Cluster, ClusterParameterGroup, ClusterSubnetGroup, ClusterSecurityGroup]
|
||||
AWS::EC2:
|
||||
[VPC, Subnet, SecurityGroup, SecurityGroupIngress, SecurityGroupEgress, InternetGateway,
|
||||
RouteTable, SubnetRouteTableAssociation, Route, NatGateway, EIP, Instance, LaunchTemplate,
|
||||
VPCGatewayAttachment, VPCEndpoint, NetworkAcl, NetworkAclEntry, SubnetNetworkAclAssociation,
|
||||
FlowLog]
|
||||
AWS::ElasticLoadBalancingV2: [LoadBalancer, TargetGroup, Listener, ListenerRule]
|
||||
AWS::AutoScaling: [LaunchConfiguration, AutoScalingGroup, LifecycleHook]
|
||||
AWS::Route53: [HostedZone, RecordSet]
|
||||
AWS::ApiGateway: [RestApi, Resource, Authorizer, Method, Deployment, Stage, Account, DomainName, BasePathMapping, GatewayResponse]
|
||||
AWS::ApiGatewayV2: [Api, Authorizer, Route, Integration, Stage, Deployment]
|
||||
AWS::AppSync: [GraphQLApi, GraphQLSchema, DataSource, FunctionConfiguration, Resolver, ApiKey]
|
||||
AWS::CodePipeline: [Pipeline, CustomActionType, Webhook]
|
||||
AWS::Batch: [ComputeEnvironment, JobQueue, JobDefinition]
|
||||
AWS::Cognito: [UserPool, UserPoolClient]
|
||||
AWS::Events: [Rule, EventBus, EventBusPolicy]
|
||||
AWS::CloudFormation: [CustomResource]
|
||||
|
||||
# Namespace -> the service name shown in the Service column.
|
||||
service_labels:
|
||||
AWS::ApiGateway: API Gateway (v1)
|
||||
AWS::ApiGatewayV2: API Gateway v2
|
||||
AWS::AppSync: AppSync
|
||||
AWS::AutoScaling: Auto Scaling
|
||||
AWS::Batch: Batch
|
||||
AWS::Backup: Backup
|
||||
AWS::CDK: CDK
|
||||
AWS::CertificateManager: ACM
|
||||
AWS::CloudFormation: CloudFormation
|
||||
AWS::CloudFront: CloudFront
|
||||
AWS::CloudTrail: CloudTrail
|
||||
AWS::CloudWatch: CloudWatch
|
||||
AWS::CodeBuild: CodeBuild
|
||||
AWS::CodePipeline: CodePipeline
|
||||
AWS::Cognito: Cognito
|
||||
AWS::Config: Config
|
||||
AWS::DynamoDB: DynamoDB
|
||||
AWS::EC2: EC2
|
||||
AWS::ECR: ECR
|
||||
AWS::ECS: ECS
|
||||
AWS::EKS: EKS
|
||||
AWS::ElasticLoadBalancingV2: Elastic Load Balancing v2
|
||||
AWS::Events: EventBridge
|
||||
AWS::IAM: IAM
|
||||
AWS::IoT: IoT Core
|
||||
AWS::KMS: KMS
|
||||
AWS::Kinesis: Kinesis
|
||||
AWS::KinesisFirehose: Kinesis Data Firehose
|
||||
AWS::Lambda: Lambda
|
||||
AWS::Logs: CloudWatch Logs
|
||||
AWS::Organizations: Organizations
|
||||
AWS::Pipes: Pipes
|
||||
AWS::RDS: RDS
|
||||
AWS::Redshift: Redshift
|
||||
AWS::Route53: Route 53
|
||||
AWS::S3: S3
|
||||
AWS::SNS: SNS
|
||||
AWS::SQS: SQS
|
||||
AWS::SSM: SSM
|
||||
AWS::Scheduler: EventBridge Scheduler
|
||||
AWS::SecretsManager: Secrets Manager
|
||||
AWS::StepFunctions: Step Functions
|
||||
AWS::WAFv2: WAFv2
|
||||
Custom: CloudFormation
|
||||
|
||||
# Parenthetical shown after a type name. Keyed by the full type.
|
||||
notes:
|
||||
AWS::S3::BucketPolicy: document stored on the bucket; S3 does not evaluate it
|
||||
AWS::SQS::QueuePolicy: accepted; policy not enforced
|
||||
AWS::Lambda::Function: Zip via S3/inline `ZipFile`, and Image
|
||||
AWS::Lambda::EventSourceMapping: SQS, Kinesis, DynamoDB Streams
|
||||
AWS::Lambda::Alias: also what SAM's `AutoPublishAlias` expands into
|
||||
AWS::EC2::SecurityGroup: inline `SecurityGroupIngress`/`SecurityGroupEgress` supported
|
||||
AWS::RDS::DBInstance: starts a real container
|
||||
AWS::RDS::DBCluster: starts a real container
|
||||
AWS::Redshift::ClusterSecurityGroup: accepted; no EC2-Classic security group model
|
||||
AWS::Redshift::Cluster: single-node container; Port and non-dev DBName ignored; ManageMasterPassword unsupported
|
||||
AWS::CDK::Metadata: accepted; no-op
|
||||
AWS::IAM::User: 'template `LoginProfile` and `PermissionsBoundary` are ignored; an API-created login profile is removed on delete'
|
||||
AWS::IoT::DomainConfiguration: '`ServerCertificates` resolves to a JSON string'
|
||||
AWS::IoT::Policy: 'deleted after detaching it from its principals; on AWS the delete fails with `DeleteConflictException` while the policy is attached'
|
||||
Custom::DynamoDBReplica: applied natively against DynamoDB, not via a provider Lambda
|
||||
AWS::Cognito::UserPool: '`ProviderURL` is the local issuer of the tokens Floci mints, `<base-url>/<pool id>`'
|
||||
|
||||
# Trailing prose appended to a row after its type list, for behaviour that belongs to the service
|
||||
# rather than to one type.
|
||||
row_notes:
|
||||
AWS::Lambda: >-
|
||||
Inline `ZipFile` packages include the `cfn-response` (Node.js) / `cfnresponse` (Python) module
|
||||
AWS injects for that code path, so Solutions-style custom-resource handlers work.
|
||||
|
||||
# Capabilities that belong in this table but are not provisioner types, so they cannot come from
|
||||
# the inventory. Each needs a reason, because an entry here is exempt from the drift check.
|
||||
extra_types:
|
||||
AWS::CloudFormation:
|
||||
- name: Stack
|
||||
note: nested stacks
|
||||
reason: >-
|
||||
Handled by CloudFormationService as a child-stack lifecycle, not by a resource provisioner.
|
||||
- name: Custom::*
|
||||
note: Lambda-backed
|
||||
reason: >-
|
||||
A wildcard prefix, not an exact type. The registry is keyed by exact type, so this is
|
||||
dispatched by a prefix branch and can never appear in the inventory.
|
||||
@@ -0,0 +1,255 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail CI when a registered service has no row in the docs/services/index.md matrix.
|
||||
|
||||
`ResolvedServiceCatalog.java` is the source of truth for which AWS services Floci
|
||||
emulates: every `descriptor("externalKey", "configKey", ...)` call registers one.
|
||||
`docs/services/index.md` is the human-facing matrix users browse to find out what's
|
||||
supported. Nothing previously connected the two, so a service could be registered,
|
||||
enabled and routable while remaining absent from the matrix with every other check
|
||||
green (see floci-io/floci#2454 - four services, and Bedrock AgentCore for five days,
|
||||
shipped without a row).
|
||||
|
||||
This script closes that gap the same way regen_action_docs.py's unregistered-handler
|
||||
sentinel closes the equivalent gap for action tables: extract descriptor keys from
|
||||
Java source, extract documented filenames from the matrix, and fail on anything that
|
||||
resolves to neither - unless tools/docs/service_matrix.yaml says otherwise (an alias
|
||||
for a key whose AWS signing name differs from its doc filename, or a time-boxed
|
||||
deferred entry for a service that is genuinely not documented yet).
|
||||
|
||||
The reverse direction is checked too: every docs/services/*.md page must have a
|
||||
matrix row, unless listed under `facet_pages:` in the registry, so a page can no
|
||||
longer exist in the nav with no row (elb-classic.md did, hidden by the `elb` key
|
||||
exact-matching the v2 page).
|
||||
|
||||
Run from anywhere in the repo:
|
||||
python3 tools/docs/check_service_matrix.py # print warnings only
|
||||
python3 tools/docs/check_service_matrix.py --strict # exit non-zero on warnings
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
CATALOG_SOURCE = "src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java"
|
||||
MATRIX_DOC = "docs/services/index.md"
|
||||
REGISTRY = "tools/docs/service_matrix.yaml"
|
||||
|
||||
MATRIX_HEADING = "## Service Matrix"
|
||||
MATRIX_END_HEADING = "## Common Setup"
|
||||
|
||||
# The externalKey is the first string literal argument to descriptor(...). It's always
|
||||
# a plain double-quoted literal (no concatenation, no constants) in current usage.
|
||||
DESCRIPTOR_KEY_RE = re.compile(r'\bdescriptor\(\s*"([^"]+)"')
|
||||
|
||||
# Every descriptor(...) call site, literal-keyed or not - used only to catch a call this
|
||||
# parser's literal-only DESCRIPTOR_KEY_RE would otherwise miss in total silence (a future
|
||||
# externalKey built from a variable or constant instead of a plain string literal). The
|
||||
# negative lookbehind excludes the `private static ServiceDescriptor descriptor(...)`
|
||||
# definition itself, the one place "descriptor(" appears that isn't a call.
|
||||
ALL_DESCRIPTOR_CALLS_RE = re.compile(r"(?<!ServiceDescriptor )\bdescriptor\(")
|
||||
|
||||
# A matrix row's first cell is a markdown link to its doc page: `[Label](slug.md...)`.
|
||||
# The slug is everything up to `.md`; a trailing `#anchor` (e.g. `cloudwatch.md#metrics`)
|
||||
# is not part of the filename and is dropped by stopping the match at `.md` itself.
|
||||
MATRIX_LINK_RE = re.compile(r"\]\(([a-z0-9][a-z0-9\-]*)\.md")
|
||||
|
||||
SERVICES_DIR = "docs/services"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeferredEntry:
|
||||
key: str
|
||||
reason: str
|
||||
by: date
|
||||
|
||||
|
||||
def extract_descriptor_keys(java_source: str) -> list[str]:
|
||||
"""externalKey of every descriptor(...) call, in source order.
|
||||
|
||||
May contain dupes only if the source itself registers the same externalKey twice
|
||||
(a copy-paste bug); the caller dedups via set() before comparing against the matrix.
|
||||
"""
|
||||
return DESCRIPTOR_KEY_RE.findall(java_source)
|
||||
|
||||
|
||||
def count_descriptor_calls(java_source: str) -> int:
|
||||
"""Total descriptor(...) call sites, regardless of how the first argument is built."""
|
||||
return len(ALL_DESCRIPTOR_CALLS_RE.findall(java_source))
|
||||
|
||||
|
||||
def extract_matrix_slugs(md_source: str) -> set[str]:
|
||||
"""Doc filenames (no extension, no anchor) linked from the Service Matrix table.
|
||||
|
||||
Scoped to the table between MATRIX_HEADING and MATRIX_END_HEADING so an unrelated
|
||||
markdown link elsewhere on the page (prose, footer) can never be mistaken for a
|
||||
documented row. Raises ValueError if either heading is missing, rather than letting
|
||||
a renamed heading silently empty the table (and every service look undocumented).
|
||||
"""
|
||||
try:
|
||||
start = md_source.index(MATRIX_HEADING)
|
||||
end = md_source.index(MATRIX_END_HEADING, start)
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
f"{MATRIX_DOC}: could not find '{MATRIX_HEADING}' ... '{MATRIX_END_HEADING}' "
|
||||
"(has a heading been renamed or reordered?)"
|
||||
) from None
|
||||
table = md_source[start:end]
|
||||
return set(MATRIX_LINK_RE.findall(table))
|
||||
|
||||
|
||||
def find_unlinked_pages(services_dir: Path, slugs: set[str], facet_pages: set[str]) -> list[str]:
|
||||
"""docs/services pages (by slug) that no matrix row links and no facet entry excuses.
|
||||
|
||||
index.md is the matrix page itself and is never expected to link to itself.
|
||||
"""
|
||||
pages = sorted(p.stem for p in services_dir.glob("*.md") if p.name != "index.md")
|
||||
return [page for page in pages if page not in slugs and page not in facet_pages]
|
||||
|
||||
|
||||
def _load_registry(repo_root: Path) -> tuple[dict[str, str], list[DeferredEntry], set[str]]:
|
||||
"""Load service_matrix.yaml. Returns (aliases, deferred entries, facet pages).
|
||||
|
||||
Schema: `aliases:` maps an externalKey to the doc slug that documents it;
|
||||
`deferred:` lists externalKeys with no row yet, each requiring `reason` and a
|
||||
`by` expiry date (see service_matrix.yaml's header comment for why expiry is
|
||||
mandatory rather than a permanent bypass); `facet_pages:` lists docs/services
|
||||
slugs that deliberately have no matrix row of their own.
|
||||
"""
|
||||
raw = yaml.safe_load((repo_root / REGISTRY).read_text(encoding="utf-8")) or {}
|
||||
aliases = dict(raw.get("aliases") or {})
|
||||
facet_pages = set(raw.get("facet_pages") or [])
|
||||
deferred: list[DeferredEntry] = []
|
||||
for item in raw.get("deferred") or []:
|
||||
missing = [f for f in ("key", "reason", "by") if f not in item]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"{REGISTRY}: deferred entry {item!r} is missing required field(s): {missing}"
|
||||
)
|
||||
deferred.append(
|
||||
DeferredEntry(
|
||||
key=item["key"],
|
||||
reason=item["reason"],
|
||||
by=date.fromisoformat(str(item["by"])),
|
||||
)
|
||||
)
|
||||
return aliases, deferred, facet_pages
|
||||
|
||||
|
||||
def find_undocumented(
|
||||
keys: list[str],
|
||||
slugs: set[str],
|
||||
aliases: dict[str, str],
|
||||
deferred: list[DeferredEntry],
|
||||
today: date,
|
||||
) -> tuple[list[str], list[DeferredEntry]]:
|
||||
"""Classify descriptor keys against the matrix.
|
||||
|
||||
Returns (undocumented, expired_deferred):
|
||||
- undocumented: keys that resolve to no matrix row by exact match, alias, or an
|
||||
unexpired deferred entry. Includes keys whose only deferred entry has expired.
|
||||
- expired_deferred: the deferred entries (still present in the registry) whose
|
||||
`by` date has passed - reported separately so the warning names the date and
|
||||
reason instead of reading like a brand-new, unexplained gap.
|
||||
"""
|
||||
active_deferred = {d.key: d for d in deferred if d.by >= today}
|
||||
expired = [d for d in deferred if d.by < today]
|
||||
|
||||
undocumented: list[str] = []
|
||||
for key in sorted(set(keys)):
|
||||
target = aliases.get(key, key)
|
||||
if target in slugs:
|
||||
continue
|
||||
if key in active_deferred:
|
||||
continue
|
||||
undocumented.append(key)
|
||||
|
||||
expired_deferred = [d for d in expired if d.key in set(keys) and aliases.get(d.key, d.key) not in slugs]
|
||||
return undocumented, expired_deferred
|
||||
|
||||
|
||||
def _repo_root() -> Path:
|
||||
return Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
help="treat warnings (undocumented services, expired deferrals) as errors",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
repo_root = _repo_root()
|
||||
|
||||
try:
|
||||
aliases, deferred, facet_pages = _load_registry(repo_root)
|
||||
except (ValueError, yaml.YAMLError) as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
java_source = (repo_root / CATALOG_SOURCE).read_text(encoding="utf-8")
|
||||
keys = extract_descriptor_keys(java_source)
|
||||
if not keys:
|
||||
print(f"error: no descriptor(...) calls found in {CATALOG_SOURCE}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
total_calls = count_descriptor_calls(java_source)
|
||||
if total_calls != len(keys):
|
||||
# DESCRIPTOR_KEY_RE only matches a plain double-quoted externalKey. A call built
|
||||
# from a variable or constant would be silently invisible to every check below -
|
||||
# this is the one place that gap gets caught instead of just under-counting.
|
||||
print(
|
||||
f"error: {CATALOG_SOURCE} has {total_calls} descriptor(...) call(s) but only "
|
||||
f"{len(keys)} have a plain string-literal externalKey as the first argument; "
|
||||
"the rest are invisible to this script and need a parser update",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
md_source = (repo_root / MATRIX_DOC).read_text(encoding="utf-8")
|
||||
try:
|
||||
slugs = extract_matrix_slugs(md_source)
|
||||
except ValueError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
undocumented, expired_deferred = find_undocumented(
|
||||
keys, slugs, aliases, deferred, date.today()
|
||||
)
|
||||
|
||||
warnings: list[str] = []
|
||||
for d in expired_deferred:
|
||||
warnings.append(
|
||||
f"deferred entry '{d.key}' expired on {d.by.isoformat()} ({d.reason}); "
|
||||
f"add its docs/services/index.md row or renew the expiry in {REGISTRY}"
|
||||
)
|
||||
still_undocumented = set(undocumented) - {d.key for d in expired_deferred}
|
||||
for key in sorted(still_undocumented):
|
||||
warnings.append(
|
||||
f"service '{key}' is registered in {CATALOG_SOURCE} but has no row in "
|
||||
f"{MATRIX_DOC} (add a row, an alias, or a time-boxed entry in {REGISTRY})"
|
||||
)
|
||||
|
||||
for page in find_unlinked_pages(repo_root / SERVICES_DIR, slugs, facet_pages):
|
||||
warnings.append(
|
||||
f"{SERVICES_DIR}/{page}.md has no row in {MATRIX_DOC} (add a row, or list it "
|
||||
f"under facet_pages in {REGISTRY} if it is a facet of another service's page)"
|
||||
)
|
||||
|
||||
for w in warnings:
|
||||
print(f"warning: {w}", file=sys.stderr)
|
||||
|
||||
if args.strict and warnings:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,428 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regenerate the action tables in docs/services/*.md from handler source.
|
||||
|
||||
The action LIST is derived from handler source (never hand-edited, never drifts).
|
||||
The Description column is hand-written prose: regen PRESERVES existing descriptions
|
||||
keyed by action name and leaves a "-" placeholder for new/unknown actions. The
|
||||
generator only ever rewrites the bytes between the marker pair; everything else in
|
||||
the file is byte-identical pre and post.
|
||||
|
||||
Assumption: a switch `case` label counts as supported purely by existing, so an arm
|
||||
that is present but throws `UnsupportedOperation` in its body would still be listed.
|
||||
Handlers here reject unsupported actions from the `default` arm, so this holds; a
|
||||
per-action stub that throws would be a false positive.
|
||||
|
||||
Run from anywhere in the repo:
|
||||
python3 tools/docs/regen_action_docs.py # rewrite docs in place
|
||||
python3 tools/docs/regen_action_docs.py --strict # exit non-zero on warnings
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
MARKER_START = "<!-- floci:actions:start -->"
|
||||
MARKER_END = "<!-- floci:actions:end -->"
|
||||
|
||||
PLACEHOLDER = "-"
|
||||
|
||||
# `case "ActionName" ->` switch arms (Query / JSON 1.1 protocols). A single arm may
|
||||
# carry several labels: `case "Foo", "Bar", "Baz" ->`; all are extracted, in order.
|
||||
# The label list may also wrap across lines (valid Java 14+ switch syntax), so the
|
||||
# capture group excludes only the `{ } ;` statement/body boundaries rather than
|
||||
# newlines: it spans line breaks between labels but stops at the arm's `->` and can't
|
||||
# run into an arm body or a colon-style `case` that has no `->`. Without this a
|
||||
# wrapped arm would be silently dropped, and `docs-check` would stay green because the
|
||||
# doc and the tool's output would agree on the omission.
|
||||
# The label head requires `[A-Z][a-z]` (PascalCase), which deliberately excludes
|
||||
# SCREAMING_CASE enum labels in non-action switches a handler may also contain
|
||||
# (e.g. `switch (state) { case "DISABLED" -> ... }`). AWS action names are always
|
||||
# PascalCase, so no real action is lost.
|
||||
SWITCH_CASE_RE = re.compile(r"^\s*case\s+([^{};]+?)\s*->", re.MULTILINE)
|
||||
SWITCH_LABEL_RE = re.compile(r'"([A-Z][a-z][A-Za-z0-9]*)"')
|
||||
|
||||
# REST controllers: a JAX-RS verb annotation followed by the public method it decorates,
|
||||
# inside a class that carries a class-level @Path. The action is ucfirst(methodName),
|
||||
# corrected per-service via rename_actions / exclude_actions (method names are not
|
||||
# always the canonical AWS action, e.g. updateV2Integration -> UpdateIntegration).
|
||||
CLASS_PATH_RE = re.compile(r"^\s*@Path\b", re.MULTILINE)
|
||||
HTTP_ANNO_RE = re.compile(r"^\s*@(?:GET|POST|PUT|DELETE|PATCH)\b")
|
||||
ANNOTATION_LINE_RE = re.compile(r"^\s*@\w")
|
||||
PUBLIC_METHOD_RE = re.compile(r"^\s*public\s+\S+(?:<[^>]*>)?\s+([a-z][A-Za-z0-9]*)\s*\(")
|
||||
# Line/block-comment lines (Javadoc) that may sit between the verb annotation and the
|
||||
# method signature; skipped so they don't reset the `pending` flag mid-lookahead.
|
||||
COMMENT_LINE_RE = re.compile(r"^\s*(?://|/\*|\*)")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Source:
|
||||
path: Path
|
||||
mode: str # 'switch' | 'rest'
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServiceEntry:
|
||||
service: str
|
||||
doc: Path
|
||||
sources: list[Source]
|
||||
rename_actions: dict[str, str] = field(default_factory=dict)
|
||||
exclude_actions: frozenset[str] = field(default_factory=frozenset)
|
||||
handler: Path | None = None # optional explicit handler path for excluded services
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Extraction (pure functions over Java source text)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def extract_switch_actions(java_source: str) -> list[str]:
|
||||
"""Action names from `case "X" ->` arms, in source order.
|
||||
|
||||
Handles multi-label arms (`case "A", "B" ->`) by extracting every quoted
|
||||
PascalCase label up to the arm's `->`.
|
||||
"""
|
||||
out: list[str] = []
|
||||
for arm in SWITCH_CASE_RE.finditer(java_source):
|
||||
out.extend(m.group(1) for m in SWITCH_LABEL_RE.finditer(arm.group(1)))
|
||||
return out
|
||||
|
||||
|
||||
CLASS_EXTENDS_RE = re.compile(r"public\s+(?:abstract\s+)?class\s+\w+\s+extends\s+([A-Za-z0-9_]+)")
|
||||
|
||||
|
||||
def _extract_raw_rest_methods(java_source: str) -> list[str]:
|
||||
actions: list[str] = []
|
||||
pending = False
|
||||
for line in java_source.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
if HTTP_ANNO_RE.match(line):
|
||||
pending = True
|
||||
continue
|
||||
if not pending:
|
||||
continue
|
||||
if ANNOTATION_LINE_RE.match(line):
|
||||
continue
|
||||
if COMMENT_LINE_RE.match(line):
|
||||
continue
|
||||
m = PUBLIC_METHOD_RE.match(line)
|
||||
if m:
|
||||
name = m.group(1)
|
||||
actions.append(name[0].upper() + name[1:])
|
||||
pending = False
|
||||
return actions
|
||||
|
||||
|
||||
def extract_rest_actions(java_source: str, source_path: Path | None = None) -> list[str]:
|
||||
"""ucfirst(method name) of @GET/@POST/@PUT/@DELETE/@PATCH methods, source order.
|
||||
|
||||
Only applies to classes containing a class-level @Path. Returns [] otherwise.
|
||||
Inherited methods from a superclass in the same directory are included first.
|
||||
"""
|
||||
if not CLASS_PATH_RE.search(java_source):
|
||||
return []
|
||||
actions: list[str] = []
|
||||
if source_path is not None:
|
||||
extends_match = CLASS_EXTENDS_RE.search(java_source)
|
||||
if extends_match:
|
||||
super_name = extends_match.group(1)
|
||||
super_path = source_path.parent / f"{super_name}.java"
|
||||
if super_path.exists():
|
||||
actions.extend(_extract_raw_rest_methods(super_path.read_text(encoding="utf-8")))
|
||||
actions.extend(_extract_raw_rest_methods(java_source))
|
||||
return actions
|
||||
|
||||
|
||||
def extract_actions(entry: ServiceEntry) -> list[str]:
|
||||
"""Merged, deduped, override-corrected action list for a service.
|
||||
|
||||
Order: first appearance across the source list in registry order. rename_actions
|
||||
is applied before dedup (so a renamed collision dedupes correctly); excluded
|
||||
names are dropped after rename.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for source in entry.sources:
|
||||
text = source.path.read_text(encoding="utf-8")
|
||||
if source.mode == "switch":
|
||||
raw = extract_switch_actions(text)
|
||||
elif source.mode == "rest":
|
||||
raw = extract_rest_actions(text, source.path)
|
||||
else:
|
||||
raise ValueError(f"unknown mode: {source.mode!r}")
|
||||
for name in raw:
|
||||
name = entry.rename_actions.get(name, name)
|
||||
if name in entry.exclude_actions:
|
||||
continue
|
||||
if name not in seen:
|
||||
seen.add(name)
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Markdown marker block: parse existing descriptions, render new table
|
||||
# --------------------------------------------------------------------------- #
|
||||
_SEPARATOR_ROW_RE = re.compile(r"^\|[\s:|-]+\|$")
|
||||
|
||||
|
||||
def _split_row(row: str) -> list[str]:
|
||||
"""Split a markdown table row into trimmed cells, honoring escaped pipes (\\|).
|
||||
|
||||
A literal pipe inside a cell must be written as \\| in markdown; an unescaped |
|
||||
is always a column separator (including pipes inside inline code, which is why
|
||||
descriptions should escape them). The split is on unescaped pipes only.
|
||||
"""
|
||||
cells: list[str] = []
|
||||
buf: list[str] = []
|
||||
i = 0
|
||||
n = len(row)
|
||||
while i < n:
|
||||
ch = row[i]
|
||||
if ch == "\\" and i + 1 < n and row[i + 1] == "|":
|
||||
buf.append("|")
|
||||
i += 2
|
||||
continue
|
||||
if ch == "|":
|
||||
cells.append("".join(buf).strip())
|
||||
buf = []
|
||||
i += 1
|
||||
continue
|
||||
buf.append(ch)
|
||||
i += 1
|
||||
cells.append("".join(buf).strip())
|
||||
# A well-formed row starts and ends with |, producing empty edge cells; drop them.
|
||||
if cells and cells[0] == "":
|
||||
cells = cells[1:]
|
||||
if cells and cells[-1] == "":
|
||||
cells = cells[:-1]
|
||||
return cells
|
||||
|
||||
|
||||
def _strip_action_cell(cell: str) -> str:
|
||||
"""Normalize an action cell to the bare action name (handles `Backticked` form)."""
|
||||
return cell.strip().strip("`").strip()
|
||||
|
||||
|
||||
def parse_marker_block(md: str) -> tuple[str, dict[str, str], str]:
|
||||
"""Split a doc around the marker block.
|
||||
|
||||
Returns (prefix_through_start_marker_line, {action: description}, suffix_from_end_marker_line).
|
||||
Raises ValueError if markers are missing, unbalanced, or the enclosed table is malformed.
|
||||
"""
|
||||
if md.count(MARKER_START) != 1 or md.count(MARKER_END) != 1:
|
||||
raise ValueError(
|
||||
f"document must contain exactly one '{MARKER_START}' and one '{MARKER_END}'"
|
||||
)
|
||||
start_idx = md.index(MARKER_START)
|
||||
end_idx = md.index(MARKER_END)
|
||||
if end_idx < start_idx:
|
||||
raise ValueError("end marker appears before start marker")
|
||||
|
||||
try:
|
||||
after_start = md.index("\n", start_idx) + 1
|
||||
before_end = md.rindex("\n", 0, end_idx) + 1
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
f"'{MARKER_START}' and '{MARKER_END}' must each sit on their own line"
|
||||
) from None
|
||||
|
||||
prefix = md[:after_start]
|
||||
suffix = md[before_end:]
|
||||
body = md[after_start:before_end]
|
||||
return prefix, _parse_table(body), suffix
|
||||
|
||||
|
||||
def _parse_table(body: str) -> dict[str, str]:
|
||||
"""Parse {action: description} from the table inside the marker block.
|
||||
|
||||
Tolerant of a missing/extra Description column, padded alignment rows, and an
|
||||
empty block (first migration). Rejects rows that don't look like a table row
|
||||
once a table has started, loudly, rather than silently dropping content.
|
||||
"""
|
||||
descriptions: dict[str, str] = {}
|
||||
seen_header = False
|
||||
for line in body.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
if not stripped.startswith("|"):
|
||||
raise ValueError(f"unexpected non-table line inside marker block: {stripped!r}")
|
||||
if _SEPARATOR_ROW_RE.match(stripped):
|
||||
continue
|
||||
cells = _split_row(stripped)
|
||||
if not cells:
|
||||
continue
|
||||
if len(cells) > 2:
|
||||
raise ValueError(
|
||||
f"malformed table row, expected at most 2 columns "
|
||||
f"(escape literal pipes as '\\|'): {stripped!r}"
|
||||
)
|
||||
action = _strip_action_cell(cells[0])
|
||||
# Skip the header row (its first cell is the literal "Action").
|
||||
if not seen_header and action.lower() == "action":
|
||||
seen_header = True
|
||||
continue
|
||||
seen_header = True
|
||||
if not action:
|
||||
continue
|
||||
desc = cells[1].strip() if len(cells) > 1 else ""
|
||||
if desc == PLACEHOLDER:
|
||||
desc = ""
|
||||
descriptions[action] = desc
|
||||
return descriptions
|
||||
|
||||
|
||||
def render_marker_block(actions: list[str], descriptions: dict[str, str]) -> str:
|
||||
"""Render the table that goes between the marker lines (two columns).
|
||||
|
||||
Preserves a known description; emits the "-" placeholder for an unknown action.
|
||||
"""
|
||||
lines = ["| Action | Description |", "| --- | --- |"]
|
||||
for action in actions:
|
||||
desc = descriptions.get(action) or PLACEHOLDER
|
||||
# Descriptions are held unescaped in memory; re-escape literal pipes so the
|
||||
# cell round-trips through _split_row on the next regen instead of splitting.
|
||||
desc = desc.replace("|", r"\|")
|
||||
lines.append(f"| `{action}` | {desc} |")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def regenerate_doc_content(
|
||||
actions: list[str], doc_content: str
|
||||
) -> tuple[str, list[str]]:
|
||||
"""Pure: given the new action list and the current doc, return (new_doc, orphans).
|
||||
|
||||
Orphans are actions present in the doc's old block but no longer in source; their
|
||||
descriptions are dropped from the new block (the row goes away).
|
||||
"""
|
||||
prefix, prior, suffix = parse_marker_block(doc_content)
|
||||
keep = set(actions)
|
||||
orphans = [a for a in prior if a not in keep]
|
||||
body = render_marker_block(actions, prior)
|
||||
return prefix + body + suffix, orphans
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Registry + driver
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _repo_root() -> Path:
|
||||
return Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
def _load_registry(repo_root: Path) -> tuple[list[ServiceEntry], set[Path]]:
|
||||
"""Load services.yaml. Returns (service entries, acknowledged deferred handler paths).
|
||||
|
||||
Schema: a top-level mapping with `services:` (the registered entries) and
|
||||
`deferred_handlers:` (switch handlers known to exist but intentionally not yet
|
||||
covered, so the unregistered-handler sentinel doesn't flag them every run).
|
||||
"""
|
||||
registry_path = repo_root / "tools" / "docs" / "services.yaml"
|
||||
raw = yaml.safe_load(registry_path.read_text(encoding="utf-8")) or {}
|
||||
entries: list[ServiceEntry] = []
|
||||
for item in raw.get("services") or []:
|
||||
sources = [
|
||||
Source(path=repo_root / s["path"], mode=s["mode"])
|
||||
for s in item.get("sources") or []
|
||||
]
|
||||
handler = item.get("handler")
|
||||
entries.append(
|
||||
ServiceEntry(
|
||||
service=item["service"],
|
||||
doc=repo_root / item["doc"],
|
||||
sources=sources,
|
||||
rename_actions=dict(item.get("rename_actions") or {}),
|
||||
exclude_actions=frozenset(item.get("exclude_actions") or []),
|
||||
handler=repo_root / handler if handler else None,
|
||||
)
|
||||
)
|
||||
deferred = {
|
||||
(repo_root / d["path"]).resolve()
|
||||
for d in raw.get("deferred_handlers") or []
|
||||
}
|
||||
return entries, deferred
|
||||
|
||||
|
||||
def _process_entry(entry: ServiceEntry) -> tuple[bool, list[str]]:
|
||||
"""Regenerate one service's doc. Returns (changed, orphans). Skip-mode is a no-op."""
|
||||
if not entry.sources:
|
||||
return False, []
|
||||
actions = extract_actions(entry)
|
||||
doc_content = entry.doc.read_text(encoding="utf-8")
|
||||
new_content, orphans = regenerate_doc_content(actions, doc_content)
|
||||
if new_content != doc_content:
|
||||
entry.doc.write_text(new_content, encoding="utf-8")
|
||||
return True, orphans
|
||||
return False, orphans
|
||||
|
||||
|
||||
def _find_unregistered_handlers(
|
||||
repo_root: Path, entries: list[ServiceEntry], deferred: set[Path]
|
||||
) -> list[str]:
|
||||
"""Top-level *Handler.java files that produce switch actions but aren't registered
|
||||
and aren't on the deferred allowlist.
|
||||
|
||||
New-service drift sentinel: a brand-new switch handler must be either registered
|
||||
or added to deferred_handlers (with a reason), or this fails --strict. REST
|
||||
controllers are opt-in (per-judgment) and not flagged here.
|
||||
"""
|
||||
services_root = repo_root / "src/main/java/io/github/hectorvent/floci/services"
|
||||
registered = {s.path.resolve() for entry in entries for s in entry.sources}
|
||||
registered |= {entry.handler.resolve() for entry in entries if entry.handler}
|
||||
registered |= deferred
|
||||
unregistered: list[str] = []
|
||||
# Recursive: handlers live at varying depths (e.g. cloudwatch/logs/, cloudwatch/metrics/).
|
||||
for path in sorted(services_root.glob("**/[A-Z]*Handler.java")):
|
||||
if path.resolve() in registered:
|
||||
continue
|
||||
if not extract_switch_actions(path.read_text(encoding="utf-8")):
|
||||
continue
|
||||
unregistered.append(str(path.relative_to(repo_root)))
|
||||
return unregistered
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
help="treat warnings (orphan actions, unregistered handlers) as errors",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
repo_root = _repo_root()
|
||||
entries, deferred = _load_registry(repo_root)
|
||||
|
||||
warnings: list[str] = []
|
||||
for entry in entries:
|
||||
try:
|
||||
changed, orphans = _process_entry(entry)
|
||||
except (ValueError, FileNotFoundError) as exc:
|
||||
print(f"error: {entry.service}: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if changed:
|
||||
print(f"updated {entry.doc.relative_to(repo_root)}")
|
||||
for orphan in orphans:
|
||||
warnings.append(
|
||||
f"{entry.service}: action '{orphan}' was in the doc but not in source (row dropped)"
|
||||
)
|
||||
|
||||
for handler in _find_unregistered_handlers(repo_root, entries, deferred):
|
||||
warnings.append(
|
||||
f"unregistered handler '{handler}' produces actions but is not in "
|
||||
f"tools/docs/services.yaml (register it or add to deferred_handlers)"
|
||||
)
|
||||
|
||||
for w in warnings:
|
||||
print(f"warning: {w}", file=sys.stderr)
|
||||
|
||||
if args.strict and warnings:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regenerate the "Supported Resource Types" table in docs/services/cloudformation.md.
|
||||
|
||||
The set of types comes from src/test/resources/cloudformation/supported-resource-types.tsv,
|
||||
which CfnResourceInventoryTest pins to the CDI-resolved provisioner registry plus the legacy
|
||||
switch. Presentation (row order, service labels, notes) comes from cfn_resource_types.yaml.
|
||||
|
||||
Deliberately reads the TSV rather than parsing the Java: several provisioners return
|
||||
`Set.of(CONSTANT, CONSTANT)` from resourceTypes(), so a source regex sees only a subset. The
|
||||
TSV is the one representation that is both machine-checked against the running registry and
|
||||
readable without a JVM.
|
||||
|
||||
python3 tools/docs/regen_cfn_resource_types.py # rewrite in place
|
||||
python3 tools/docs/regen_cfn_resource_types.py --check # exit 1 if stale
|
||||
python3 tools/docs/regen_cfn_resource_types.py --strict # also fail on warnings
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
INVENTORY = REPO_ROOT / "src/test/resources/cloudformation/supported-resource-types.tsv"
|
||||
CONFIG = Path(__file__).resolve().parent / "cfn_resource_types.yaml"
|
||||
DOC = REPO_ROOT / "docs/services/cloudformation.md"
|
||||
PROVISIONER_DIR = REPO_ROOT / "src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners"
|
||||
|
||||
MARKER_START = "<!-- floci:cfn-types:start -->"
|
||||
MARKER_END = "<!-- floci:cfn-types:end -->"
|
||||
|
||||
|
||||
def load_inventory(path: Path = INVENTORY) -> list[tuple[str, str]]:
|
||||
"""[(resource_type, owner)] in file order."""
|
||||
rows = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
parts = line.split("\t")
|
||||
if len(parts) != 2:
|
||||
raise ValueError(f"{path.name}: expected 'type<TAB>owner', got: {line!r}")
|
||||
rows.append((parts[0], parts[1]))
|
||||
return rows
|
||||
|
||||
|
||||
def namespace_of(resource_type: str) -> str:
|
||||
"""AWS::SQS::Queue -> AWS::SQS; Custom::DynamoDBReplica -> Custom."""
|
||||
parts = resource_type.split("::")
|
||||
return "::".join(parts[:2]) if len(parts) == 3 else parts[0]
|
||||
|
||||
|
||||
def leaf_of(resource_type: str) -> str:
|
||||
"""AWS::SQS::Queue -> Queue; Custom::DynamoDBReplica -> Custom::DynamoDBReplica.
|
||||
|
||||
A two-segment type has no namespace to strip, so it is shown whole; abbreviating
|
||||
Custom::DynamoDBReplica to DynamoDBReplica would read as an AWS type that does not exist.
|
||||
"""
|
||||
parts = resource_type.split("::")
|
||||
return parts[2] if len(parts) == 3 else resource_type
|
||||
|
||||
|
||||
def render_rows(inventory, config) -> tuple[list[str], list[str]]:
|
||||
"""Returns (table_lines, warnings)."""
|
||||
warnings: list[str] = []
|
||||
labels = config.get("service_labels", {}) or {}
|
||||
notes = config.get("notes", {}) or {}
|
||||
row_notes = config.get("row_notes", {}) or {}
|
||||
extras = config.get("extra_types", {}) or {}
|
||||
order = config.get("order", []) or []
|
||||
|
||||
merges = config.get("merge_namespaces", {}) or {}
|
||||
displays = config.get("display_names", {}) or {}
|
||||
type_order = config.get("type_order", {}) or {}
|
||||
|
||||
by_namespace: dict[str, list[str]] = {}
|
||||
for resource_type, _owner in inventory:
|
||||
ns = namespace_of(resource_type)
|
||||
by_namespace.setdefault(merges.get(ns, ns), []).append(resource_type)
|
||||
|
||||
# Curated order first (it reads by importance, not alphabet), then anything new
|
||||
# alphabetically so a type added without touching this config still lands somewhere sane.
|
||||
for ns, types in by_namespace.items():
|
||||
wanted = type_order.get(ns, [])
|
||||
rank = {name: i for i, name in enumerate(wanted)}
|
||||
types.sort(key=lambda t: (rank.get(leaf_of(t), len(rank)), leaf_of(t)))
|
||||
for leaf in wanted:
|
||||
if leaf not in {leaf_of(t) for t in types}:
|
||||
warnings.append(
|
||||
f"type_order for {ns} lists {leaf}, which nothing provisions; "
|
||||
f"remove it or fix the spelling"
|
||||
)
|
||||
|
||||
namespaces = list(by_namespace) + [ns for ns in extras if ns not in by_namespace]
|
||||
ranked = [ns for ns in order if ns in namespaces]
|
||||
ranked += sorted(ns for ns in namespaces if ns not in order)
|
||||
|
||||
for ns in namespaces:
|
||||
if ns not in labels:
|
||||
warnings.append(
|
||||
f"no service_labels entry for {ns}; add one to cfn_resource_types.yaml "
|
||||
f"so the row gets a readable service name"
|
||||
)
|
||||
if ns not in order:
|
||||
warnings.append(f"{ns} is not in the order list; appended alphabetically")
|
||||
|
||||
lines = ["| Service | Resource types |", "|---|---|"]
|
||||
for ns in ranked:
|
||||
cells = []
|
||||
for resource_type in by_namespace.get(ns, []):
|
||||
note = notes.get(resource_type)
|
||||
shown = displays.get(resource_type, leaf_of(resource_type))
|
||||
cells.append(f"`{shown}`" + (f" ({note})" if note else ""))
|
||||
for extra in extras.get(ns, []):
|
||||
if not extra.get("reason"):
|
||||
warnings.append(
|
||||
f"extra_types entry {ns} / {extra.get('name')} has no reason; "
|
||||
f"an entry exempt from the inventory check must say why"
|
||||
)
|
||||
note = extra.get("note")
|
||||
cells.append(f"`{extra['name']}`" + (f" ({note})" if note else ""))
|
||||
if not cells:
|
||||
continue
|
||||
types = ", ".join(cells)
|
||||
trailing = (row_notes.get(ns) or "").strip()
|
||||
if trailing:
|
||||
types = f"{types}. {trailing}"
|
||||
lines.append(f"| {labels.get(ns, ns)} | {types} |")
|
||||
return lines, warnings
|
||||
|
||||
|
||||
def check_provisioner_annotations() -> list[str]:
|
||||
"""A CfnResourceProvisioner without @ApplicationScoped is never discovered by CDI.
|
||||
|
||||
CfnResourceInventoryTest catches this at runtime; flagging it here too means the cheap
|
||||
check runs in docs-check, before anyone waits on a Quarkus test boot.
|
||||
"""
|
||||
warnings = []
|
||||
if not PROVISIONER_DIR.is_dir():
|
||||
return warnings
|
||||
for path in sorted(PROVISIONER_DIR.glob("*CfnProvisioner.java")):
|
||||
src = path.read_text(encoding="utf-8")
|
||||
if "implements CfnResourceProvisioner" not in src:
|
||||
continue
|
||||
if "@ApplicationScoped" not in src:
|
||||
warnings.append(
|
||||
f"{path.name} implements CfnResourceProvisioner without @ApplicationScoped, "
|
||||
f"so CDI never registers it and its types are silently stubbed"
|
||||
)
|
||||
return warnings
|
||||
|
||||
|
||||
def check_no_double_ownership(inventory) -> list[str]:
|
||||
"""A type served by a provisioner must not also sit in the legacy switch."""
|
||||
owners: dict[str, set[str]] = {}
|
||||
for resource_type, owner in inventory:
|
||||
owners.setdefault(resource_type, set()).add(owner)
|
||||
return [
|
||||
f"{resource_type} is claimed by both the legacy switch and {sorted(o - {'LEGACY_SWITCH'})[0]}; "
|
||||
f"the registry wins, leaving the switch arm as dead code"
|
||||
for resource_type, o in sorted(owners.items())
|
||||
if len(o) > 1 and "LEGACY_SWITCH" in o
|
||||
]
|
||||
|
||||
|
||||
def splice(md: str, table: list[str]) -> str:
|
||||
if md.count(MARKER_START) != 1 or md.count(MARKER_END) != 1:
|
||||
raise ValueError(
|
||||
f"{DOC.name} must contain exactly one {MARKER_START} and one {MARKER_END}"
|
||||
)
|
||||
start, end = md.index(MARKER_START), md.index(MARKER_END)
|
||||
if end < start:
|
||||
raise ValueError("end marker appears before start marker")
|
||||
after_start = md.index("\n", start) + 1
|
||||
before_end = md.rindex("\n", 0, end) + 1
|
||||
return md[:after_start] + "\n".join(table) + "\n" + md[before_end:]
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--check", action="store_true", help="exit 1 if the doc is stale")
|
||||
parser.add_argument("--strict", action="store_true", help="also exit 1 on warnings")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
inventory = load_inventory()
|
||||
config = yaml.safe_load(CONFIG.read_text(encoding="utf-8")) or {}
|
||||
table, warnings = render_rows(inventory, config)
|
||||
warnings += check_no_double_ownership(inventory)
|
||||
warnings += check_provisioner_annotations()
|
||||
|
||||
for warning in warnings:
|
||||
print(f"warning: {warning}", file=sys.stderr)
|
||||
|
||||
current = DOC.read_text(encoding="utf-8")
|
||||
updated = splice(current, table)
|
||||
stale = updated != current
|
||||
|
||||
if args.check or args.strict:
|
||||
if stale:
|
||||
print(
|
||||
f"error: {DOC.relative_to(REPO_ROOT)} resource-type table is stale. "
|
||||
f"Run 'make docs-sync' and commit the result.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1 if (stale or (args.strict and warnings)) else 0
|
||||
|
||||
if stale:
|
||||
DOC.write_text(updated, encoding="utf-8")
|
||||
print(f"updated {DOC.relative_to(REPO_ROOT)} ({len(table) - 2} rows)")
|
||||
else:
|
||||
print(f"{DOC.relative_to(REPO_ROOT)} already up to date ({len(table) - 2} rows)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,2 @@
|
||||
PyYAML>=6.0
|
||||
pytest>=7.0
|
||||
@@ -0,0 +1,577 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reconcile every source that answers "how many AWS services does Floci support?".
|
||||
|
||||
There is no single number, and that is not a bug: twelve sources in this repo each
|
||||
count something slightly different, and each is right on its own terms. A descriptor
|
||||
is a signing endpoint, a matrix row is an API surface, a doc page is a service a user
|
||||
browses to, a README category entry is a marketing name. Asked "how many services",
|
||||
people reach for whichever is nearest and get a different answer every time.
|
||||
|
||||
This script prints all twelve, groups them into the five keyspaces that can actually
|
||||
be compared, and for each comparison prints the delta *as a number and as the named
|
||||
entries that explain it*. An unexplained delta is a finding; an explained one is a
|
||||
design decision that should read as deliberate.
|
||||
|
||||
It is a human-facing report, not a CI gate: it is read-only, and it exits non-zero
|
||||
only when a source cannot be parsed. It deliberately does not duplicate the two real
|
||||
gates that already exist (check_service_matrix.py for catalog-to-matrix,
|
||||
ServiceConfigYamlCoverageTest for config-to-YAML); both are named in the output so an
|
||||
advisory comparison can never be mistaken for an enforced one.
|
||||
|
||||
python3 tools/docs/service_inventory.py # the reconciliation report
|
||||
python3 tools/docs/service_inventory.py --json # the same data, machine-readable
|
||||
|
||||
**Never copy a number from this output into README.md or docs/.** The repo's rule is
|
||||
that no service count is published anywhere, derived or not; user-facing pages link to
|
||||
the Service Matrix instead. This tool exists so the question can be answered in
|
||||
conversation, not so the answer can be pasted into a page that then goes stale.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
import check_service_matrix as csm
|
||||
|
||||
CONFIG_SOURCE = "src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java"
|
||||
APPLICATION_YML = "src/main/resources/application.yml"
|
||||
README = "README.md"
|
||||
SERVICES_YAML = "tools/docs/services.yaml"
|
||||
PACKAGE_DIR = "src/main/java/io/github/hectorvent/floci/services"
|
||||
|
||||
# Warn this far ahead of a deferred entry's expiry, so a red main is a decision rather
|
||||
# than a surprise on the morning the date passes.
|
||||
DEFERRAL_WARN_DAYS = 30
|
||||
|
||||
# Entries under floci.services.* that emulate nothing from AWS. The config and YAML
|
||||
# collectors count them correctly (they are services Floci runs), so they have to be
|
||||
# subtracted before those counts can be compared with anything AWS-shaped.
|
||||
NON_AWS_CONFIG_KEYS = {"duck", "ui"}
|
||||
|
||||
# Packages under services/ that back no descriptor of their own, each with the reason.
|
||||
# The point of listing them is that the package-count delta reads as explained; an
|
||||
# entry appearing in the UNREGISTERED list instead is the signal that a package was
|
||||
# added without being registered in the catalog.
|
||||
PACKAGE_NOTES = {
|
||||
"cloudwatch": "one package, two descriptors: cloudwatchlogs and cloudwatchmetrics",
|
||||
"floci": "Floci's own control surface, not an AWS service",
|
||||
"lambdamicrovms": "controllers attached to the lambda descriptor",
|
||||
"resourcegroupstagging": "registered under the AWS signing name, tagging",
|
||||
"securityadmin": "shared admin-accounts endpoint over GuardDuty and Macie",
|
||||
"signin": "controller attached to the signin descriptor, which is keyed iam",
|
||||
"ssoportal": "controller attached to the sso descriptor, which is keyed ssoadmin",
|
||||
}
|
||||
|
||||
# README category names whose wording differs from the matrix row label they refer to.
|
||||
# Matching is by normalised name, so only genuine wording differences need an entry,
|
||||
# and one README name may cover several matrix rows. This assists a human-facing
|
||||
# report and gates nothing, which is why it lives here rather than in
|
||||
# service_matrix.yaml next to the aliases the CI gate reads.
|
||||
README_NAME_ALIASES: dict[str, str | list[str]] = {
|
||||
"API Gateway REST": "API Gateway v1",
|
||||
"Bedrock": "Amazon Bedrock",
|
||||
"ELB Classic": "ELB Classic (v1)",
|
||||
"Firehose": "Data Firehose",
|
||||
"Resource Explorer 2": "Resource Explorer",
|
||||
"Service Catalog": "AWS Service Catalog",
|
||||
"IAM Identity Center (SSO Admin, OIDC, Access Portal, SCIM)": [
|
||||
"IAM Identity Center (SSO Admin)",
|
||||
"IAM Identity Center OIDC",
|
||||
"IAM Identity Center Access Portal",
|
||||
],
|
||||
}
|
||||
|
||||
CONFIG_ACCESSOR_RE = re.compile(r"^\s{8}[A-Za-z0-9]+ +([a-zA-Z0-9]+)\(\);", re.M)
|
||||
ENABLED_CALL_RE = re.compile(r"config\.services\(\)\.([a-zA-Z0-9]+)\(\)\.enabled\(\)")
|
||||
MATRIX_ROW_RE = re.compile(r"^\| \[([^\]]+)\]\(([a-z0-9][a-z0-9\-]*)\.md([^)]*)\)", re.M)
|
||||
|
||||
|
||||
def _repo_root() -> Path:
|
||||
return Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
def key(name: str) -> str:
|
||||
"""Canonical form of a service key, for comparing across naming conventions.
|
||||
|
||||
The same service is `bcmDataExports` as a config accessor, `bcm-data-exports` in
|
||||
application.yml and `bcmdataexports` as a package directory. Comparing the raw
|
||||
spellings reports eight bogus mismatches in a repo where nothing is actually wrong,
|
||||
so every cross-source comparison goes through here.
|
||||
"""
|
||||
return re.sub(r"[^a-z0-9]", "", name.lower())
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Java catalog
|
||||
# --------------------------------------------------------------------------- #
|
||||
def split_top_level_args(text: str, start: int) -> list[str]:
|
||||
"""Split the argument list of a call whose opening paren is at or after `start`.
|
||||
|
||||
Splits on commas at paren depth 1 only, so a nested `config.services().x().enabled()`
|
||||
or `Set.of("a", "b")` argument stays in one piece. Raises ValueError if the parens
|
||||
never balance.
|
||||
"""
|
||||
depth = 0
|
||||
args: list[str] = []
|
||||
current: list[str] = []
|
||||
for i in range(start, len(text)):
|
||||
ch = text[i]
|
||||
if ch == "(":
|
||||
depth += 1
|
||||
if depth == 1:
|
||||
continue
|
||||
elif ch == ")":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
args.append("".join(current).strip())
|
||||
return args
|
||||
elif ch == "," and depth == 1:
|
||||
args.append("".join(current).strip())
|
||||
current = []
|
||||
continue
|
||||
current.append(ch)
|
||||
raise ValueError("unbalanced parentheses in descriptor(...) call")
|
||||
|
||||
|
||||
def parse_descriptors(java_source: str) -> list[dict]:
|
||||
"""externalKey, configKey, enable toggles and includeInStatus of every call.
|
||||
|
||||
check_service_matrix.py needs only the externalKey, so one regex serves it. This
|
||||
needs the first four arguments, and the fourth can sit on a continuation line:
|
||||
rds-data and redshift-data wrap because their enabled expression is a conjunction
|
||||
of two toggles. Matching off the opening line would read `true` from the wrong
|
||||
argument for exactly those two, so the arguments are split properly instead.
|
||||
"""
|
||||
out: list[dict] = []
|
||||
for m in csm.ALL_DESCRIPTOR_CALLS_RE.finditer(java_source):
|
||||
open_paren = java_source.index("(", m.start())
|
||||
args = split_top_level_args(java_source, open_paren)
|
||||
if len(args) < 4:
|
||||
raise ValueError(f"descriptor(...) call with only {len(args)} arguments")
|
||||
external, config_key, enabled, include = args[0], args[1], args[2], args[3]
|
||||
if not (external.startswith('"') and external.endswith('"')):
|
||||
raise ValueError(
|
||||
f"descriptor(...) externalKey is not a plain string literal: {external!r}"
|
||||
)
|
||||
out.append(
|
||||
{
|
||||
"external_key": external.strip('"'),
|
||||
"config_key": config_key.strip('"') if config_key.startswith('"') else None,
|
||||
"toggles": sorted(set(ENABLED_CALL_RE.findall(enabled))),
|
||||
"in_status": include.strip() == "true",
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Config plane
|
||||
# --------------------------------------------------------------------------- #
|
||||
def parse_config_accessors(java_source: str) -> list[str]:
|
||||
"""Accessors on EmulatorConfig.ServicesConfig that return a per-service config.
|
||||
|
||||
Scoped to that interface body, because the same accessor shape appears on other
|
||||
nested interfaces. `dockerNetwork()` is excluded by the return-type pattern: it
|
||||
returns `Optional<String>`, and the `<` keeps it out.
|
||||
"""
|
||||
try:
|
||||
start = java_source.index(" interface ServicesConfig {")
|
||||
except ValueError:
|
||||
raise ValueError(f"{CONFIG_SOURCE}: interface ServicesConfig not found") from None
|
||||
end = java_source.index("\n }", start)
|
||||
return sorted(CONFIG_ACCESSOR_RE.findall(java_source[start:end]))
|
||||
|
||||
|
||||
def parse_yaml_service_keys(path: Path) -> tuple[list[str], list[str]]:
|
||||
"""(every floci.services.* key, the subset declaring `enabled:`)."""
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
services = (data.get("floci") or {}).get("services") or {}
|
||||
keys = [k for k, v in services.items() if isinstance(v, dict)]
|
||||
return sorted(keys), sorted(k for k in keys if "enabled" in services[k])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Docs and README
|
||||
# --------------------------------------------------------------------------- #
|
||||
def parse_matrix_rows(md_source: str) -> list[dict]:
|
||||
"""One entry per Service Matrix row: label, page slug, anchor.
|
||||
|
||||
A row is a distinct API surface; a page is a documented service. They differ
|
||||
because eight pages carry two rows each (DynamoDB and DynamoDB Streams, IAM and
|
||||
AWS Sign-In, and so on). The facet test is **two rows sharing a slug**, not "the
|
||||
link carries an anchor": IoT Core and IoT Data both link plain `iot.md` with no
|
||||
anchor at all, so an anchor-based rule misses that pair and over-counts by one.
|
||||
"""
|
||||
start = md_source.index(csm.MATRIX_HEADING)
|
||||
end = md_source.index(csm.MATRIX_END_HEADING, start)
|
||||
return [
|
||||
{"label": label, "slug": slug, "anchor": anchor.lstrip("#") or None}
|
||||
for label, slug, anchor in MATRIX_ROW_RE.findall(md_source[start:end])
|
||||
]
|
||||
|
||||
|
||||
def parse_readme_category_names(md_source: str) -> list[str]:
|
||||
"""Service names from the README's Category/Services table.
|
||||
|
||||
Split on commas at paren depth 0 only. One entry reads `IAM Identity Center (SSO
|
||||
Admin, OIDC, Access Portal, SCIM)`; splitting on every comma turns that single
|
||||
name into four, three of which ("OIDC", "SCIM)") are not service names at all and
|
||||
inflate the count by three.
|
||||
"""
|
||||
start = md_source.index("| Category | Services |")
|
||||
end = md_source.index("For operation-level compatibility", start)
|
||||
names: list[str] = []
|
||||
for line in md_source[start:end].splitlines():
|
||||
if not line.startswith("| ") or line.startswith("|---") or line.startswith("| Category"):
|
||||
continue
|
||||
cells = [c.strip() for c in line.strip("|").split("|")]
|
||||
if len(cells) < 2:
|
||||
continue
|
||||
depth = 0
|
||||
current: list[str] = []
|
||||
for ch in cells[1]:
|
||||
if ch == "(":
|
||||
depth += 1
|
||||
elif ch == ")":
|
||||
depth -= 1
|
||||
elif ch == "," and depth == 0:
|
||||
names.append("".join(current).strip())
|
||||
current = []
|
||||
continue
|
||||
current.append(ch)
|
||||
if "".join(current).strip():
|
||||
names.append("".join(current).strip())
|
||||
return names
|
||||
|
||||
|
||||
def parse_readme_detail_rows(md_source: str) -> list[str]:
|
||||
"""Service names from the README's collapsed "Detailed service notes" table."""
|
||||
header = "| Service | How it works | Notable features |"
|
||||
start = md_source.index(header)
|
||||
end = md_source.index("</details>", start)
|
||||
rows = []
|
||||
for line in md_source[start:end].splitlines():
|
||||
if not line.startswith("| ") or line.startswith("|---") or line == header:
|
||||
continue
|
||||
rows.append(line.strip("|").split("|")[0].strip())
|
||||
return rows
|
||||
|
||||
|
||||
def parse_action_table_services(path: Path) -> list[str]:
|
||||
"""Service keys in services.yaml, the opt-in list for generated action tables."""
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
return sorted(entry["service"] for entry in (data.get("services") or []))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Reconciliation
|
||||
# --------------------------------------------------------------------------- #
|
||||
def collect(repo_root: Path) -> dict:
|
||||
"""Every source, parsed. Raises ValueError if any source cannot be read."""
|
||||
descriptors = parse_descriptors((repo_root / csm.CATALOG_SOURCE).read_text(encoding="utf-8"))
|
||||
if not descriptors:
|
||||
raise ValueError(f"no descriptor(...) calls found in {csm.CATALOG_SOURCE}")
|
||||
|
||||
yaml_keys, yaml_enabled = parse_yaml_service_keys(repo_root / APPLICATION_YML)
|
||||
readme_src = (repo_root / README).read_text(encoding="utf-8")
|
||||
aliases, deferred, facet_pages = csm._load_registry(repo_root)
|
||||
|
||||
return {
|
||||
"descriptors": descriptors,
|
||||
"config_accessors": parse_config_accessors(
|
||||
(repo_root / CONFIG_SOURCE).read_text(encoding="utf-8")
|
||||
),
|
||||
"yaml_keys": yaml_keys,
|
||||
"yaml_enabled": yaml_enabled,
|
||||
"packages": sorted(p.name for p in (repo_root / PACKAGE_DIR).iterdir() if p.is_dir()),
|
||||
"matrix_rows": parse_matrix_rows(
|
||||
(repo_root / csm.MATRIX_DOC).read_text(encoding="utf-8")
|
||||
),
|
||||
"doc_pages": sorted(
|
||||
p.stem for p in (repo_root / csm.SERVICES_DIR).glob("*.md") if p.name != "index.md"
|
||||
),
|
||||
"readme_categories": parse_readme_category_names(readme_src),
|
||||
"readme_details": parse_readme_detail_rows(readme_src),
|
||||
"action_tables": parse_action_table_services(repo_root / SERVICES_YAML),
|
||||
"aliases": aliases,
|
||||
"facet_pages": sorted(facet_pages),
|
||||
"deferred": [{"key": d.key, "reason": d.reason, "by": d.by.isoformat()} for d in deferred],
|
||||
}
|
||||
|
||||
|
||||
def reconcile(data: dict, today: date | None = None) -> dict:
|
||||
"""Counts, deltas, and the named entries that explain each delta."""
|
||||
today = today or date.today()
|
||||
descriptors = data["descriptors"]
|
||||
aliases = data["aliases"]
|
||||
|
||||
# --- catalog ---------------------------------------------------------- #
|
||||
keys = [d["external_key"] for d in descriptors]
|
||||
page_of = {k: aliases.get(k, k) for k in keys}
|
||||
owners: dict[str, list[str]] = {}
|
||||
for k in keys:
|
||||
owners.setdefault(page_of[k], []).append(k)
|
||||
facet_descriptors = sorted(k for k in keys if owners[page_of[k]][0] != k)
|
||||
status_hidden = sorted(d["external_key"] for d in descriptors if not d["in_status"])
|
||||
|
||||
toggle_users: dict[str, list[str]] = {}
|
||||
for d in descriptors:
|
||||
for t in d["toggles"]:
|
||||
toggle_users.setdefault(key(t), []).append(d["external_key"])
|
||||
shared = {t: v for t, v in sorted(toggle_users.items()) if len(v) > 1}
|
||||
|
||||
# --- config plane ----------------------------------------------------- #
|
||||
accessors = {key(a): a for a in data["config_accessors"]}
|
||||
yaml_keys = {key(k): k for k in data["yaml_keys"]}
|
||||
aws_keys = {k: v for k, v in yaml_keys.items() if v not in NON_AWS_CONFIG_KEYS}
|
||||
untoggled = sorted(v for k, v in aws_keys.items() if k not in toggle_users)
|
||||
|
||||
# --- packages --------------------------------------------------------- #
|
||||
descriptor_keys = {key(d["config_key"]) for d in descriptors if d["config_key"]}
|
||||
packages = data["packages"]
|
||||
noted = sorted(p for p in packages if p in PACKAGE_NOTES)
|
||||
unregistered = sorted(
|
||||
p for p in packages if key(p) not in descriptor_keys and p not in PACKAGE_NOTES
|
||||
)
|
||||
|
||||
# --- docs ------------------------------------------------------------- #
|
||||
rows = data["matrix_rows"]
|
||||
slug_rows: dict[str, list[str]] = {}
|
||||
for r in rows:
|
||||
slug_rows.setdefault(r["slug"], []).append(r["label"])
|
||||
facet_rows = sorted(r["label"] for r in rows if slug_rows[r["slug"]][0] != r["label"])
|
||||
doc_pages = set(data["doc_pages"])
|
||||
expiring = [
|
||||
d for d in data["deferred"]
|
||||
if date.fromisoformat(d["by"]) <= today + timedelta(days=DEFERRAL_WARN_DAYS)
|
||||
]
|
||||
|
||||
# --- README ----------------------------------------------------------- #
|
||||
row_labels = {key(r["label"]) for r in rows}
|
||||
facet_norms = {key(label) for label in facet_rows}
|
||||
|
||||
def cover(names: list[str]) -> tuple[set[str], list[str]]:
|
||||
"""(matrix labels these README names account for, names matching no row).
|
||||
|
||||
A name resolves through README_NAME_ALIASES when its wording differs from the
|
||||
matrix label, and one alias may cover several rows: the README names IAM
|
||||
Identity Center once where the matrix carries three rows for it.
|
||||
"""
|
||||
covered: set[str] = set()
|
||||
unmatched: list[str] = []
|
||||
for name in names:
|
||||
target = README_NAME_ALIASES.get(name, name)
|
||||
norms = [key(t) for t in (target if isinstance(target, list) else [target])]
|
||||
if all(n in row_labels for n in norms):
|
||||
covered |= set(norms)
|
||||
else:
|
||||
unmatched.append(name)
|
||||
return covered, unmatched
|
||||
|
||||
category_covered, unmatched_names = cover(data["readme_categories"])
|
||||
detail_covered, _ = cover(data["readme_details"])
|
||||
|
||||
def uncovered(covered: set[str], facets: bool) -> list[str]:
|
||||
return [
|
||||
r["label"]
|
||||
for r in rows
|
||||
if key(r["label"]) not in covered and (key(r["label"]) in facet_norms) is facets
|
||||
]
|
||||
|
||||
return {
|
||||
"counts": {
|
||||
"catalog_descriptors": len(keys),
|
||||
"catalog_distinct_products": len(keys) - len(facet_descriptors),
|
||||
"catalog_status_visible": len(keys) - len(status_hidden),
|
||||
"catalog_toggles": len(toggle_users),
|
||||
"service_packages": len(packages),
|
||||
"config_accessors": len(accessors),
|
||||
"yaml_keys": len(yaml_keys),
|
||||
"yaml_enabled": len(data["yaml_enabled"]),
|
||||
"yaml_aws_keys": len(aws_keys),
|
||||
"matrix_rows": len(rows),
|
||||
"matrix_pages": len(slug_rows),
|
||||
"doc_pages": len(doc_pages),
|
||||
"readme_category_names": len(data["readme_categories"]),
|
||||
"readme_detail_rows": len(data["readme_details"]),
|
||||
"action_tables": len(data["action_tables"]),
|
||||
},
|
||||
"catalog": {
|
||||
"facet_descriptors": facet_descriptors,
|
||||
"status_hidden": status_hidden,
|
||||
"shared_toggles": {t: v for t, v in shared.items()},
|
||||
},
|
||||
"config": {
|
||||
"non_aws": sorted(v for k, v in yaml_keys.items() if v in NON_AWS_CONFIG_KEYS),
|
||||
"accessor_only": sorted(accessors[k] for k in accessors.keys() - yaml_keys.keys()),
|
||||
"yaml_only": sorted(yaml_keys[k] for k in yaml_keys.keys() - accessors.keys()),
|
||||
"no_enabled_key": sorted(set(data["yaml_keys"]) - set(data["yaml_enabled"])),
|
||||
"untoggled": untoggled,
|
||||
"toggle_without_key": sorted(
|
||||
t for t in toggle_users if t not in yaml_keys
|
||||
),
|
||||
},
|
||||
"packages": {"noted": noted, "unregistered": unregistered},
|
||||
"docs": {
|
||||
"facet_rows": facet_rows,
|
||||
"pages_without_row": sorted(doc_pages - set(slug_rows) - set(data["facet_pages"])),
|
||||
"rows_without_page": sorted(set(slug_rows) - doc_pages),
|
||||
"facet_pages": data["facet_pages"],
|
||||
"deferred": data["deferred"],
|
||||
"deferred_expiring": expiring,
|
||||
},
|
||||
"readme": {
|
||||
"unmatched_category_names": unmatched_names,
|
||||
"missing_from_categories": uncovered(category_covered, facets=False),
|
||||
"missing_facet_rows": uncovered(category_covered, facets=True),
|
||||
"missing_from_details": uncovered(detail_covered, facets=False),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Report
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _names(items: list[str], limit: int = 10) -> str:
|
||||
if not items:
|
||||
return ""
|
||||
shown = ", ".join(items[:limit])
|
||||
return shown if len(items) <= limit else f"{shown}, +{len(items) - limit} more"
|
||||
|
||||
|
||||
def render(result: dict) -> str:
|
||||
c = result["counts"]
|
||||
cat, cfg, pkg, docs, rd = (
|
||||
result["catalog"], result["config"], result["packages"],
|
||||
result["docs"], result["readme"],
|
||||
)
|
||||
out: list[str] = []
|
||||
w = out.append
|
||||
|
||||
def line(label: str, count: int | str, note: str = "") -> None:
|
||||
w(f" {label:<32}{str(count):>5} {note}".rstrip())
|
||||
|
||||
def delta(label: str, items: list[str], limit: int = 10) -> None:
|
||||
w(f" {'- ' + label:<32}{len(items):>5} {_names(items, limit)}".rstrip())
|
||||
|
||||
def note(text: str) -> None:
|
||||
w(f" {text}")
|
||||
|
||||
w("")
|
||||
w("THE ANSWER")
|
||||
w(f" Floci documents {c['matrix_pages']} AWS services, exposing {c['matrix_rows']} distinct")
|
||||
w(f" API surfaces across {c['catalog_descriptors']} signing endpoints.")
|
||||
w(" Quote the first for \"how many services\", the second when the question is really")
|
||||
w(" about API coverage. Never write any of them into README.md or docs/: link to the")
|
||||
w(" Service Matrix instead.")
|
||||
w("")
|
||||
|
||||
w("1. CATALOG (keyspace: descriptor externalKey)")
|
||||
line("catalog descriptors", c["catalog_descriptors"], csm.CATALOG_SOURCE)
|
||||
delta("facet endpoints", cat["facet_descriptors"])
|
||||
line("= distinct products", c["catalog_distinct_products"])
|
||||
line("status-visible at runtime", c["catalog_status_visible"],
|
||||
f"GET /health; hidden: {_names(cat['status_hidden'])}")
|
||||
line("distinct enable toggles", c["catalog_toggles"])
|
||||
for toggle, users in cat["shared_toggles"].items():
|
||||
note(f"{toggle} gates {len(users)}: {_names(users)}")
|
||||
w("")
|
||||
|
||||
w("2. CONFIG PLANE (keyspace: floci.services.* key, naming normalised)")
|
||||
line("EmulatorConfig accessors", c["config_accessors"], CONFIG_SOURCE)
|
||||
line("application.yml keys", c["yaml_keys"], APPLICATION_YML)
|
||||
if cfg["accessor_only"] or cfg["yaml_only"]:
|
||||
delta("accessor with no YAML key", cfg["accessor_only"])
|
||||
delta("YAML key with no accessor", cfg["yaml_only"])
|
||||
else:
|
||||
line("= identical sets", "", "GATED by ServiceConfigYamlCoverageTest")
|
||||
delta("no `enabled:` in YAML", cfg["no_enabled_key"])
|
||||
delta("non-AWS entries", cfg["non_aws"])
|
||||
line("= AWS service keys", c["yaml_aws_keys"])
|
||||
delta("no descriptor reads its toggle", cfg["untoggled"])
|
||||
delta("toggle with no YAML key", cfg["toggle_without_key"])
|
||||
w("")
|
||||
|
||||
w("3. SOURCE (keyspace: Java package, naming normalised)")
|
||||
line("services/ packages", c["service_packages"], PACKAGE_DIR)
|
||||
delta("no descriptor, explained", pkg["noted"])
|
||||
for p in pkg["noted"]:
|
||||
note(f"{p}: {PACKAGE_NOTES[p]}")
|
||||
if pkg["unregistered"]:
|
||||
delta("UNREGISTERED", pkg["unregistered"])
|
||||
note(f"A package under {PACKAGE_DIR} that no descriptor(...) names.")
|
||||
note("Register it, or add it to PACKAGE_NOTES with the reason it has none.")
|
||||
else:
|
||||
line("= every other package registered", "", "ADVISORY, no gate")
|
||||
w("")
|
||||
|
||||
w("4. DOCS (keyspace: docs/services slug)")
|
||||
line("matrix rows", c["matrix_rows"], csm.MATRIX_DOC)
|
||||
delta("facet rows (share a page)", docs["facet_rows"])
|
||||
line("= matrix pages", c["matrix_pages"])
|
||||
line("docs/services/*.md", c["doc_pages"])
|
||||
if docs["pages_without_row"] or docs["rows_without_page"]:
|
||||
delta("page with no row", docs["pages_without_row"])
|
||||
delta("row with no page", docs["rows_without_page"])
|
||||
else:
|
||||
line("= identical sets", "", "GATED by check_service_matrix.py --strict")
|
||||
if docs["facet_pages"]:
|
||||
note(f"facet_pages, row-exempt by registry: {_names(docs['facet_pages'])}")
|
||||
for d in docs["deferred"]:
|
||||
flag = " <-- EXPIRING" if d in docs["deferred_expiring"] else ""
|
||||
note(f"deferred: {d['key']} until {d['by']}{flag}")
|
||||
note(f" {d['reason']}")
|
||||
w("")
|
||||
|
||||
w("5. README (keyspace: service name, matched to matrix row labels)")
|
||||
line("category-table names", c["readme_category_names"], f"{README}, Supported Services")
|
||||
line("matrix rows they cover",
|
||||
c["matrix_rows"] - len(rd["missing_from_categories"]) - len(rd["missing_facet_rows"]))
|
||||
delta("row missing from README", rd["missing_from_categories"])
|
||||
delta("facet row, absent by design", rd["missing_facet_rows"])
|
||||
if rd["unmatched_category_names"]:
|
||||
delta("README name matching no row", rd["unmatched_category_names"], limit=4)
|
||||
note("Wording drift, a service not in the matrix, or a missing")
|
||||
note("README_NAME_ALIASES entry. Read it, do not assume.")
|
||||
note("ADVISORY, no gate: docs/contributing.md asks contributors for a README")
|
||||
note("row as well as a matrix row, but only the matrix row is machine-checked.")
|
||||
w("")
|
||||
line("detail-table rows", c["readme_detail_rows"], "editorial \"notable features\"")
|
||||
delta("documented service with no note", rd["missing_from_details"], limit=6)
|
||||
note("Deliberately ungated: the detail table is editorial, and a row per")
|
||||
note(f"service would buy {len(rd['missing_from_details'])} lines of filler prose.")
|
||||
w("")
|
||||
|
||||
w("6. ACTION TABLES (keyspace: services.yaml entry)")
|
||||
line("generated action tables", c["action_tables"], SERVICES_YAML)
|
||||
note(f"Opt-in by design, not a gap: {c['doc_pages'] - c['action_tables']} pages carry no")
|
||||
note("generated operation table. regen_action_docs.py owns this list.")
|
||||
w("")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--json", action="store_true", help="emit the raw data instead of the report"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
result = reconcile(collect(_repo_root()))
|
||||
except (ValueError, KeyError, yaml.YAMLError, OSError) as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(json.dumps(result, indent=2, sort_keys=True) if args.json else render(result))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,69 @@
|
||||
# Registry for the service-matrix presence check (tools/docs/check_service_matrix.py).
|
||||
#
|
||||
# Every descriptor("externalKey", "configKey", ...) registered in
|
||||
# core/common/ResolvedServiceCatalog.java must resolve to a row in the Service Matrix
|
||||
# table in docs/services/index.md, keyed by the .md filename its row links to (no
|
||||
# extension, no #anchor). A key resolves one of three ways:
|
||||
#
|
||||
# 1. exact match - externalKey literally equals a documented filename. No entry
|
||||
# needed in this file; most services work this way.
|
||||
# 2. aliases - externalKey differs from the filename that documents it, because
|
||||
# it's an AWS SigV4 signing-name quirk (`kafka` -> msk.md) or a
|
||||
# facet folded into another service's page (`ec2messages` -> ssm.md).
|
||||
# 3. deferred - genuinely undocumented today, tracked with a reason. A NEW
|
||||
# descriptor key that is neither an exact match, an alias target,
|
||||
# nor listed here fails `check_service_matrix.py --strict`.
|
||||
#
|
||||
# The reverse direction is checked too: every docs/services/*.md page must be linked
|
||||
# from a matrix row unless its slug is listed under `facet_pages:` (a page that
|
||||
# documents a facet of another service and deliberately has no row of its own).
|
||||
|
||||
aliases:
|
||||
apigateway: api-gateway # "API Gateway v1" row title
|
||||
apigatewayv2: api-gateway # v2 facet, documented at api-gateway.md#v2
|
||||
aps: managed-prometheus # AMP's SigV4 signing name is "aps"
|
||||
kafka: msk # MSK's SigV4 signing name is "kafka"
|
||||
mq: amazonmq # Amazon MQ's SigV4 signing name is "mq"
|
||||
signin: iam # AWS Sign-In facet, at iam.md#aws-sign-in-login-credentials
|
||||
sso: ssoadmin # descriptor("sso") is the Identity Center admin API; ssoadmin.md is its page
|
||||
sso-oidc: ssooidc # IAM Identity Center OIDC REST API
|
||||
events: eventbridge
|
||||
servicediscovery: cloudmap # Cloud Map's SigV4 signing name
|
||||
elasticmapreduce: emr # EMR's SigV4 signing name
|
||||
logs: cloudwatch # CloudWatch Logs facet, at cloudwatch.md
|
||||
monitoring: cloudwatch # CloudWatch Metrics facet, at cloudwatch.md#metrics
|
||||
secretsmanager: secrets-manager
|
||||
codeguru-reviewer: codegurureviewer # SigV4 signing name is hyphenated; the doc slug is not
|
||||
cognito-identity: cognitoidentity # SigV4 signing name is hyphenated; the doc slug is not
|
||||
app-integrations: appintegrations # SigV4 signing name is hyphenated; the doc slug is not
|
||||
servicecatalog: service-catalog
|
||||
cognito-idp: cognito # Cognito's SigV4 signing name
|
||||
states: step-functions # Step Functions' SigV4 signing name
|
||||
email: ses # SES's SigV4 signing name
|
||||
es: opensearch # OpenSearch's SigV4 signing name (legacy Elasticsearch)
|
||||
appconfigdata: appconfig # data-plane facet, at appconfig.md#data-plane
|
||||
tagging: resource-groups-tagging # Resource Groups Tagging API's SigV4 signing name
|
||||
elasticloadbalancing: elb # ELB v2's SigV4 signing name
|
||||
elb: elb-classic # Classic (2012-06-01) key; the v2 page is elb.md
|
||||
application-autoscaling: applicationautoscaling
|
||||
elasticbeanstalk: elastic-beanstalk
|
||||
ec2messages: ssm # SSM Messages facet, documented alongside ssm.md
|
||||
iotdata: iot # IoT Data row shares iot.md with IoT Core
|
||||
resource-explorer-2: resource-explorer # API version suffix; the page is resource-explorer.md
|
||||
bedrock-agentcore-control: bedrock-agentcore # control plane owns the page; the data-plane row is the facet
|
||||
|
||||
# A NEW switch/REST descriptor not listed here and not resolvable via an alias fails
|
||||
# `--strict`. Every entry MUST carry a `by:` expiry date (ISO, YYYY-MM-DD): once that
|
||||
# date passes, `--strict` fails for it exactly as if it were never listed, even though
|
||||
# the entry is still physically present in this file. This is deliberate - an
|
||||
# unbounded deferred list is a silent, permanent bypass of the whole check, which is
|
||||
# how Bedrock AgentCore (#2316) shipped a working service that stayed off the matrix
|
||||
# for five days with nothing red anywhere. An expiring entry forces a human to either
|
||||
# add the row or come back and renew the date with a fresh justification; it can never
|
||||
# just be forgotten. Remove an entry outright once the referenced PR merges and the
|
||||
# row lands - don't wait for it to expire.
|
||||
deferred: []
|
||||
|
||||
# docs/services pages that intentionally have no Service Matrix row of their own.
|
||||
facet_pages:
|
||||
- cloudwatch-metric-filters-verification
|
||||
@@ -0,0 +1,437 @@
|
||||
# Registry of services whose Supported Actions tables are auto-generated.
|
||||
#
|
||||
# The action LIST comes from handler source (mode: switch|rest); the Description
|
||||
# column is hand-written and PRESERVED across regen, keyed by action name.
|
||||
# Per-service overrides: rename_actions (method->canonical), exclude_actions.
|
||||
#
|
||||
# Phase 1 covers single-table switch-mode services. REST controllers, multi-table
|
||||
# docs, mixed-mode (ses), S3 query-string dispatch, and non-tabular docs are tracked
|
||||
# as follow-ups; their switch handlers are acknowledged in deferred_handlers below so
|
||||
# the new-service sentinel only fires on genuinely unregistered handlers.
|
||||
|
||||
services:
|
||||
- service: account
|
||||
doc: docs/services/account.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/account/AccountController.java, mode: rest }
|
||||
- service: accessanalyzer
|
||||
doc: docs/services/access-analyzer.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/accessanalyzer/AccessAnalyzerController.java, mode: rest }
|
||||
- service: acm
|
||||
doc: docs/services/acm.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/acm/AcmJsonHandler.java, mode: switch }
|
||||
- service: applicationautoscaling
|
||||
doc: docs/services/applicationautoscaling.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/applicationautoscaling/ApplicationAutoScalingJsonHandler.java, mode: switch }
|
||||
- service: athena
|
||||
doc: docs/services/athena.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/athena/AthenaJsonHandler.java, mode: switch }
|
||||
- service: bedrock
|
||||
doc: docs/services/bedrock.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/bedrock/BedrockController.java, mode: rest }
|
||||
- service: bedrock-agentcore-control
|
||||
doc: docs/services/bedrock-agentcore.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreIdentityController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreGatewayController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreToolsController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreGatewayRuleController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreResourcePolicyController.java, mode: rest }
|
||||
rename_actions:
|
||||
CreateEndpoint: CreateAgentRuntimeEndpoint
|
||||
GetEndpoint: GetAgentRuntimeEndpoint
|
||||
UpdateEndpoint: UpdateAgentRuntimeEndpoint
|
||||
DeleteEndpoint: DeleteAgentRuntimeEndpoint
|
||||
ListEndpoints: ListAgentRuntimeEndpoints
|
||||
Create: CreateWorkloadIdentity
|
||||
Get: GetWorkloadIdentity
|
||||
Update: UpdateWorkloadIdentity
|
||||
Delete: DeleteWorkloadIdentity
|
||||
List: ListWorkloadIdentities
|
||||
CreateTarget: CreateGatewayTarget
|
||||
GetTarget: GetGatewayTarget
|
||||
UpdateTarget: UpdateGatewayTarget
|
||||
DeleteTarget: DeleteGatewayTarget
|
||||
ListTargets: ListGatewayTargets
|
||||
- service: cloudformation
|
||||
doc: docs/services/cloudformation.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationQueryHandler.java, mode: switch }
|
||||
- service: cloudhsmv2
|
||||
doc: docs/services/cloudhsmv2.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cloudhsmv2/CloudHsmV2JsonHandler.java, mode: switch }
|
||||
- service: cloudtrail
|
||||
doc: docs/services/cloudtrail.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailJsonHandler.java, mode: switch }
|
||||
- service: codeartifact
|
||||
doc: docs/services/codeartifact.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/core/common/V1TagsController.java, mode: rest }
|
||||
rename_actions:
|
||||
ListTagsForResourceByQuery: ListTagsForResource
|
||||
exclude_actions:
|
||||
- ListTags
|
||||
- TagResourcePost
|
||||
- TagResourcePut
|
||||
- service: codegurureviewer
|
||||
doc: docs/services/codegurureviewer.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/codegurureviewer/CodeGuruReviewerController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/core/common/SharedTagsController.java, mode: rest }
|
||||
rename_actions:
|
||||
ListTags: ListTagsForResource
|
||||
TagResourcePost: TagResource
|
||||
exclude_actions:
|
||||
- ListTagsByQuery
|
||||
- TagResourceByBody
|
||||
- TagResourcePut
|
||||
- UntagResourceByQuery
|
||||
- service: marketplace
|
||||
doc: docs/services/marketplace.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/marketplace/MarketplaceCatalogController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/marketplace/MarketplaceAgreementService.java, mode: switch }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/marketplace/MarketplaceEntitlementController.java, mode: switch }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/marketplace/MarketplaceDeploymentController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/marketplace/MarketplaceReportingController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/marketplace/MarketplaceMeteringService.java, mode: switch }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/marketplace/MarketplaceDiscoveryController.java, mode: rest }
|
||||
exclude_actions:
|
||||
- AgreementType
|
||||
- Status
|
||||
- AcceptorAccountId
|
||||
- OfferId
|
||||
- OfferSetId
|
||||
- ResourceIdentifier
|
||||
- ResourceType
|
||||
- PartyType
|
||||
- BeforeStartTime
|
||||
- AfterStartTime
|
||||
- BeforeEndTime
|
||||
- AfterEndTime
|
||||
- BeforeLastUpdateTime
|
||||
- AfterLastUpdateTime
|
||||
- EndTimeBehaviorType
|
||||
- EndTimeBehaviorReasonCode
|
||||
- InitialAgreementId
|
||||
- LicenseArn
|
||||
- StartTime
|
||||
- LastUpdateTime
|
||||
- EndTime
|
||||
- ASCENDING
|
||||
- DESCENDING
|
||||
- service: comprehend
|
||||
doc: docs/services/comprehend.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/comprehend/ComprehendJsonHandler.java, mode: switch }
|
||||
- service: controlcatalog
|
||||
doc: docs/services/controlcatalog.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/controlcatalog/ControlCatalogController.java, mode: rest }
|
||||
rename_actions:
|
||||
GetControl: GetControl
|
||||
ListControls: ListControls
|
||||
- service: appintegrations
|
||||
doc: docs/services/appintegrations.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/appintegrations/AppIntegrationsController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/core/common/SharedTagsController.java, mode: rest }
|
||||
rename_actions:
|
||||
ListTags: ListTagsForResource
|
||||
TagResourcePost: TagResource
|
||||
exclude_actions:
|
||||
- ListTagsByQuery
|
||||
- TagResourceByBody
|
||||
- TagResourcePut
|
||||
- UntagResourceByQuery
|
||||
- service: connect
|
||||
doc: docs/services/connect.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/connect/ConnectController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/core/common/SharedTagsController.java, mode: rest }
|
||||
rename_actions:
|
||||
ListTags: ListTagsForResource
|
||||
TagResourcePost: TagResource
|
||||
exclude_actions:
|
||||
- ListTagsByQuery
|
||||
- TagResourceByBody
|
||||
- TagResourcePut
|
||||
- UntagResourceByQuery
|
||||
- service: rekognition
|
||||
doc: docs/services/rekognition.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/rekognition/RekognitionJsonHandler.java, mode: switch }
|
||||
- service: translate
|
||||
doc: docs/services/translate.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/translate/TranslateJsonHandler.java, mode: switch }
|
||||
- service: detective
|
||||
doc: docs/services/detective.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/detective/DetectiveController.java, mode: rest }
|
||||
- service: dms
|
||||
doc: docs/services/dms.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/dms/DmsJsonHandler.java, mode: switch }
|
||||
- service: docdb
|
||||
doc: docs/services/docdb.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/docdb/DocDbQueryHandler.java, mode: switch }
|
||||
- service: efs
|
||||
doc: docs/services/efs.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/efs/EfsController.java, mode: rest }
|
||||
- service: organizations
|
||||
doc: docs/services/organizations.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/organizations/OrganizationsJsonHandler.java, mode: switch }
|
||||
- service: elasticache
|
||||
doc: docs/services/elasticache.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java, mode: switch }
|
||||
- service: emr
|
||||
doc: docs/services/emr.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java, mode: switch }
|
||||
- service: eventbridge
|
||||
doc: docs/services/eventbridge.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/eventbridge/EventBridgeHandler.java, mode: switch }
|
||||
- service: firehose
|
||||
doc: docs/services/firehose.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java, mode: switch }
|
||||
- service: resource-explorer-2
|
||||
doc: docs/services/resource-explorer.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2Controller.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/core/common/SharedTagsController.java, mode: rest }
|
||||
rename_actions:
|
||||
ListTags: ListTagsForResource
|
||||
TagResourcePost: TagResource
|
||||
exclude_actions:
|
||||
- ListTagsByQuery
|
||||
- TagResourceByBody
|
||||
- TagResourcePut
|
||||
- UntagResourceByQuery
|
||||
- service: fis
|
||||
doc: docs/services/fis.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/fis/FisController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/core/common/SharedTagsController.java, mode: rest }
|
||||
rename_actions:
|
||||
ListTags: ListTagsForResource
|
||||
TagResourcePost: TagResource
|
||||
exclude_actions:
|
||||
- ListTagsByQuery
|
||||
- TagResourceByBody
|
||||
- TagResourcePut
|
||||
- UntagResourceByQuery
|
||||
- service: kinesis
|
||||
doc: docs/services/kinesis.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java, mode: switch }
|
||||
- service: kinesisanalytics
|
||||
doc: docs/services/kinesisanalytics.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/kinesisanalytics/KinesisAnalyticsV2JsonHandler.java, mode: switch }
|
||||
- service: kms
|
||||
doc: docs/services/kms.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/kms/KmsJsonHandler.java, mode: switch }
|
||||
- service: memorydb
|
||||
doc: docs/services/memorydb.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/memorydb/MemoryDbHandler.java, mode: switch }
|
||||
- service: macie2
|
||||
doc: docs/services/macie2.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/macie2/MacieController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/securityadmin/SecurityAdminController.java, mode: rest }
|
||||
rename_actions:
|
||||
ListAdmin: ListOrganizationAdminAccounts
|
||||
- service: inspector2
|
||||
doc: docs/services/inspector2.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/inspector2/Inspector2Controller.java, mode: rest }
|
||||
- service: neptune
|
||||
doc: docs/services/neptune.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/neptune/NeptuneQueryHandler.java, mode: switch }
|
||||
- service: networkfirewall
|
||||
doc: docs/services/network-firewall.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/networkfirewall/NetworkFirewallJsonHandler.java, mode: switch }
|
||||
- service: ram
|
||||
doc: docs/services/ram.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/ram/RamController.java, mode: rest }
|
||||
- service: rds
|
||||
doc: docs/services/rds.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/rds/RdsQueryHandler.java, mode: switch }
|
||||
exclude_actions: [SubnetIds, VpcSecurityGroupIds, OptionsToRemove, ValuesToAdd, ValuesToRemove]
|
||||
- service: redshift
|
||||
doc: docs/services/redshift.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java, mode: switch }
|
||||
# SubnetIds / VpcSecurityGroupIds / IamRoles / AddIamRoles / RemoveIamRoles / TagKeys / DbGroups
|
||||
# are labels of an inner member-name switch (memberKeyRegex), not actions, same shape as the
|
||||
# rds exclusions above.
|
||||
exclude_actions: [SubnetIds, VpcSecurityGroupIds, IamRoles, AddIamRoles, RemoveIamRoles, TagKeys, DbGroups]
|
||||
- service: redshiftdata
|
||||
doc: docs/services/redshift-data.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataJsonHandler.java, mode: switch }
|
||||
# ExecuteSql / BatchExecuteSql are the deprecated pre-2020 operations: the handler
|
||||
# recognizes them only to return a ValidationException, they are not supported actions.
|
||||
exclude_actions: [ExecuteSql, BatchExecuteSql]
|
||||
- service: cognitoidentity
|
||||
doc: docs/services/cognitoidentity.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cognitoidentity/CognitoIdentityJsonHandler.java, mode: switch }
|
||||
- service: globalaccelerator
|
||||
doc: docs/services/globalaccelerator.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/globalaccelerator/GlobalAcceleratorJsonHandler.java, mode: switch }
|
||||
- service: datasync
|
||||
doc: docs/services/datasync.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/datasync/DataSyncJsonHandler.java, mode: switch }
|
||||
- service: route53
|
||||
doc: docs/services/route53.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/route53/Route53Controller.java, mode: rest }
|
||||
# JAX-RS method names use Java casing; map them onto the canonical Route 53 action names.
|
||||
rename_actions:
|
||||
AssociateVpcWithHostedZone: AssociateVPCWithHostedZone
|
||||
DisassociateVpcFromHostedZone: DisassociateVPCFromHostedZone
|
||||
CreateVpcAssociationAuthorization: CreateVPCAssociationAuthorization
|
||||
DeleteVpcAssociationAuthorization: DeleteVPCAssociationAuthorization
|
||||
ListVpcAssociationAuthorizations: ListVPCAssociationAuthorizations
|
||||
ListHostedZonesByVpc: ListHostedZonesByVPC
|
||||
GetDnssec: GetDNSSEC
|
||||
- service: route53resolver
|
||||
doc: docs/services/route53resolver.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/route53resolver/Route53ResolverJsonHandler.java, mode: switch }
|
||||
- service: sagemaker
|
||||
doc: docs/services/sagemaker.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerJsonHandler.java, mode: switch }
|
||||
- service: securityhub
|
||||
doc: docs/services/securityhub.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/securityhub/SecurityHubController.java, mode: rest }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/core/common/SharedTagsController.java, mode: rest }
|
||||
rename_actions:
|
||||
ListTags: ListTagsForResource
|
||||
TagResourcePost: TagResource
|
||||
exclude_actions:
|
||||
- ListTagsByQuery
|
||||
- TagResourceByBody
|
||||
- TagResourcePut
|
||||
- UntagResourceByQuery
|
||||
- service: secretsmanager
|
||||
doc: docs/services/secrets-manager.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerJsonHandler.java, mode: switch }
|
||||
- service: servicecatalog
|
||||
doc: docs/services/service-catalog.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/servicecatalog/ServiceCatalogJsonHandler.java, mode: switch }
|
||||
- service: servicequotas
|
||||
doc: docs/services/servicequotas.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/servicequotas/ServiceQuotasJsonHandler.java, mode: switch }
|
||||
- service: ssoadmin
|
||||
doc: docs/services/ssoadmin.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/ssoadmin/SsoAdminJsonHandler.java, mode: switch }
|
||||
- service: identitystore
|
||||
doc: docs/services/identitystore.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/identitystore/IdentityStoreJsonHandler.java, mode: switch }
|
||||
- service: budgets
|
||||
doc: docs/services/budgets.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/budgets/BudgetsJsonHandler.java, mode: switch }
|
||||
- service: verifiedpermissions
|
||||
doc: docs/services/verifiedpermissions.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/verifiedpermissions/VerifiedPermissionsJsonHandler.java, mode: switch }
|
||||
- service: timestream-influxdb
|
||||
doc: docs/services/timestream-influxdb.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/timestreaminfluxdb/TimestreamInfluxDbJsonHandler.java, mode: switch }
|
||||
- service: sqs
|
||||
doc: docs/services/sqs.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/sqs/SqsJsonHandler.java, mode: switch }
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/sqs/SqsQueryHandler.java, mode: switch }
|
||||
- service: stepfunctions
|
||||
doc: docs/services/step-functions.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonHandler.java, mode: switch }
|
||||
- service: swf
|
||||
doc: docs/services/swf.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/swf/SwfJsonHandler.java, mode: switch }
|
||||
- service: wafv2
|
||||
doc: docs/services/wafv2.md
|
||||
sources:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/wafv2/WafV2Handler.java, mode: switch }
|
||||
|
||||
# Switch handlers that exist but are intentionally not yet covered (see reasons).
|
||||
# A NEW switch handler not listed here and not registered above fails `--strict`.
|
||||
deferred_handlers:
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/apigatewayv2/ApiGatewayV2JsonHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/applicationautoscaling/ScalingPolicyAlarmActionHandler.java } # not a JSON/Query action dispatcher — its `case "X" ->` arms are the StepScaling AdjustmentType switch, not AWS actions
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/autoscaling/AutoScalingQueryHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/bcmpricingcalculator/BcmPricingCalculatorJsonHandler.java } # focused workload-estimate surface
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/bcmdataexports/BcmDataExportsJsonHandler.java } # no standalone action table yet
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/ce/CostExplorerJsonHandler.java } # no standalone action table yet
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlJsonHandler.java } # no standalone action table yet
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cloudmap/CloudMapHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsHandler.java } # multi-table doc (nested handler)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsMetricFilterHandler.java } # multi-table doc (nested handler, reached through CloudWatchLogsHandler)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/CloudWatchMetricsJsonHandler.java } # multi-table doc (nested handler)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cloudwatch/metrics/CloudWatchMetricsQueryHandler.java } # multi-table doc (nested handler)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildJsonHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployJsonHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cognito/CognitoJsonHandler.java } # non-tabular doc (grouped/inline); separate docs effort
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/configservice/ConfigServiceJsonHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/cur/CurJsonHandler.java } # no standalone action table yet
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java } # multi-table doc
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamsJsonHandler.java } # multi-table doc
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/ec2/Ec2QueryHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/ecr/EcrJsonHandler.java } # doc packs multiple actions per row (`A` / `B`) + REST endpoints; needs de-grouping
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/ecs/EcsJsonHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/elasticbeanstalk/ElasticBeanstalkQueryHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/elb/ElbClassicQueryHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2QueryHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/iam/IamQueryHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/iam/StsQueryHandler.java } # sts (lives in iam package); single-table, registerable next
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/lakeformation/LakeFormationJsonHandler.java } # no standalone action table yet
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/lightsail/LightsailJsonHandler.java } # non-tabular doc (grouped/inline); standalone action table to be added
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/pricing/PricingJsonHandler.java } # no standalone action table yet
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/resourcegroupstagging/ResourceGroupsTaggingJsonHandler.java } # non-tabular doc (grouped/inline)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/ses/SesQueryHandler.java } # mixed-mode (Query + REST surface)
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/sns/SnsJsonHandler.java } # multi-table doc
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/sns/SnsQueryHandler.java } # multi-table doc
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/ssm/Ec2MessagesJsonHandler.java } # multi-table doc
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/ssm/SsmJsonHandler.java } # multi-table doc
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/textract/TextractJsonHandler.java } # no standalone action table yet
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeJsonHandler.java } # no standalone action table yet
|
||||
- { path: src/main/java/io/github/hectorvent/floci/services/transfer/TransferHandler.java } # multi-table doc
|
||||
@@ -0,0 +1,201 @@
|
||||
"""Tests for check_service_matrix.
|
||||
|
||||
Run with: pytest tools/docs -q (or: make docs-test)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
import check_service_matrix as c
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Extraction
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_extract_descriptor_keys_basic():
|
||||
src = '''
|
||||
descriptor("ssm", "ssm", config.services().ssm().enabled(), true,
|
||||
"ssm", storageMode(...), 5000L, null, ServiceProtocol.JSON,
|
||||
protocols(ServiceProtocol.JSON),
|
||||
Set.of("AmazonSSM."), Set.of("ssm"), Set.of(), Set.of()),
|
||||
descriptor("sqs", "sqs", config.services().sqs().enabled(), true, ...),
|
||||
'''
|
||||
assert c.extract_descriptor_keys(src) == ["ssm", "sqs"]
|
||||
|
||||
|
||||
def test_extract_descriptor_keys_ignores_other_string_literals():
|
||||
# Only the first argument of descriptor(...) counts; targetPrefixes, credential
|
||||
# scopes and the like must not be picked up even though they're also quoted.
|
||||
src = 'descriptor("kafka", "msk", true, true, "msk", null, 5000L, null, X, Y, Set.of("kafka-signing-prefix."), Set.of("kafka"), Set.of(), Set.of())'
|
||||
assert c.extract_descriptor_keys(src) == ["kafka"]
|
||||
|
||||
|
||||
def test_count_descriptor_calls_matches_literal_keys_when_all_literal():
|
||||
src = '''
|
||||
descriptor("ssm", "ssm", true, true, ...),
|
||||
descriptor("sqs", "sqs", true, true, ...),
|
||||
'''
|
||||
assert c.count_descriptor_calls(src) == len(c.extract_descriptor_keys(src)) == 2
|
||||
|
||||
|
||||
def test_count_descriptor_calls_excludes_the_definition():
|
||||
src = '''
|
||||
private static ServiceDescriptor descriptor(
|
||||
String externalKey,
|
||||
String configKey) {
|
||||
return new ServiceDescriptor(externalKey, configKey);
|
||||
}
|
||||
'''
|
||||
assert c.count_descriptor_calls(src) == 0
|
||||
assert c.extract_descriptor_keys(src) == []
|
||||
|
||||
|
||||
def test_count_descriptor_calls_catches_a_non_literal_key():
|
||||
# A future externalKey built from a variable/constant instead of a plain string
|
||||
# literal must not be silently invisible: the call-site count and the literal-key
|
||||
# count have to disagree so main() can fail loudly instead of under-counting.
|
||||
src = '''
|
||||
descriptor("ssm", "ssm", true, true, ...),
|
||||
descriptor(SOME_CONSTANT, "other", true, true, ...),
|
||||
'''
|
||||
assert c.count_descriptor_calls(src) == 2
|
||||
assert len(c.extract_descriptor_keys(src)) == 1
|
||||
|
||||
|
||||
def test_extract_matrix_slugs_scoped_to_table():
|
||||
md = """
|
||||
# Services Overview
|
||||
|
||||
Some prose with an unrelated [link](not-a-service.md) that must not be picked up.
|
||||
|
||||
## Service Matrix
|
||||
|
||||
| Service | Endpoint | Protocol | Supported operations |
|
||||
|---|---|---|---|
|
||||
| [SSM](ssm.md) | ... | JSON 1.1 | 22 |
|
||||
| [CloudWatch Metrics](cloudwatch.md#metrics) | ... | Query | 11 |
|
||||
|
||||
## Common Setup
|
||||
|
||||
Another [link](also-not-a-service.md) below the table.
|
||||
"""
|
||||
assert c.extract_matrix_slugs(md) == {"ssm", "cloudwatch"}
|
||||
|
||||
|
||||
def test_extract_matrix_slugs_strips_anchor():
|
||||
md = "## Service Matrix\n\n| [API Gateway v2](api-gateway.md#v2) | | | 48 |\n\n## Common Setup\n"
|
||||
assert c.extract_matrix_slugs(md) == {"api-gateway"}
|
||||
|
||||
|
||||
def test_extract_matrix_slugs_raises_on_missing_heading():
|
||||
# A renamed/reordered heading must fail loudly rather than silently returning an
|
||||
# empty slug set, which would make every registered service look undocumented.
|
||||
md = "# Services Overview\n\nNo matrix heading here.\n"
|
||||
with pytest.raises(ValueError, match="Service Matrix"):
|
||||
c.extract_matrix_slugs(md)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Classification
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_find_undocumented_exact_match_needs_no_alias():
|
||||
undocumented, expired = c.find_undocumented(
|
||||
keys=["ssm", "sqs"],
|
||||
slugs={"ssm", "sqs"},
|
||||
aliases={},
|
||||
deferred=[],
|
||||
today=date(2026, 1, 1),
|
||||
)
|
||||
assert undocumented == []
|
||||
assert expired == []
|
||||
|
||||
|
||||
def test_find_undocumented_alias_resolves():
|
||||
undocumented, expired = c.find_undocumented(
|
||||
keys=["kafka"],
|
||||
slugs={"msk"},
|
||||
aliases={"kafka": "msk"},
|
||||
deferred=[],
|
||||
today=date(2026, 1, 1),
|
||||
)
|
||||
assert undocumented == []
|
||||
|
||||
|
||||
def test_find_undocumented_flags_genuine_gap():
|
||||
undocumented, expired = c.find_undocumented(
|
||||
keys=["swf"],
|
||||
slugs={"ssm"},
|
||||
aliases={},
|
||||
deferred=[],
|
||||
today=date(2026, 1, 1),
|
||||
)
|
||||
assert undocumented == ["swf"]
|
||||
assert expired == []
|
||||
|
||||
|
||||
def test_find_undocumented_unexpired_deferred_is_silenced():
|
||||
d = c.DeferredEntry(key="bedrock-agentcore", reason="tracked in #2436", by=date(2026, 9, 15))
|
||||
undocumented, expired = c.find_undocumented(
|
||||
keys=["bedrock-agentcore"],
|
||||
slugs=set(),
|
||||
aliases={},
|
||||
deferred=[d],
|
||||
today=date(2026, 8, 22),
|
||||
)
|
||||
assert undocumented == []
|
||||
assert expired == []
|
||||
|
||||
|
||||
def test_find_undocumented_expired_deferred_reports_and_still_flags():
|
||||
# This is the case the mandatory `by:` date exists to catch: a deferred entry that
|
||||
# has run past its expiry is reported distinctly (so the message names the date
|
||||
# and reason) AND still counts as undocumented - it cannot silently keep passing.
|
||||
d = c.DeferredEntry(key="bedrock-agentcore", reason="tracked in #2436", by=date(2026, 9, 15))
|
||||
undocumented, expired = c.find_undocumented(
|
||||
keys=["bedrock-agentcore"],
|
||||
slugs=set(),
|
||||
aliases={},
|
||||
deferred=[d],
|
||||
today=date(2026, 10, 1),
|
||||
)
|
||||
assert undocumented == ["bedrock-agentcore"]
|
||||
assert expired == [d]
|
||||
|
||||
|
||||
def test_find_undocumented_deferred_entry_already_resolved_is_not_flagged():
|
||||
# If the row was added but the registry entry wasn't cleaned up, the key resolves
|
||||
# via the matrix itself regardless of expiry - no warning either way.
|
||||
d = c.DeferredEntry(key="bedrock-agentcore", reason="tracked in #2436", by=date(2020, 1, 1))
|
||||
undocumented, expired = c.find_undocumented(
|
||||
keys=["bedrock-agentcore"],
|
||||
slugs={"bedrock-agentcore"},
|
||||
aliases={},
|
||||
deferred=[d],
|
||||
today=date(2026, 8, 22),
|
||||
)
|
||||
assert undocumented == []
|
||||
assert expired == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Unlinked pages
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_find_unlinked_pages_flags_a_page_with_no_row(tmp_path):
|
||||
for name in ("index.md", "ssm.md", "elb.md", "elb-classic.md"):
|
||||
(tmp_path / name).write_text("# page\n")
|
||||
# index.md is the matrix page itself; elb.md is linked; elb-classic.md is not.
|
||||
assert c.find_unlinked_pages(tmp_path, slugs={"ssm", "elb"}, facet_pages=set()) == ["elb-classic"]
|
||||
|
||||
|
||||
def test_find_unlinked_pages_honours_facet_exemption(tmp_path):
|
||||
for name in ("index.md", "lambda.md", "lambda-microvms.md"):
|
||||
(tmp_path / name).write_text("# page\n")
|
||||
assert c.find_unlinked_pages(tmp_path, slugs={"lambda"}, facet_pages={"lambda-microvms"}) == []
|
||||
|
||||
|
||||
def test_find_unlinked_pages_ignores_non_markdown(tmp_path):
|
||||
(tmp_path / "ssm.md").write_text("# page\n")
|
||||
(tmp_path / "diagram.svg").write_text("<svg/>")
|
||||
assert c.find_unlinked_pages(tmp_path, slugs={"ssm"}, facet_pages=set()) == []
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Tests for regen_action_docs.
|
||||
|
||||
Run with: pytest tools/docs -q (or: make docs-test)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import regen_action_docs as r
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Extraction: switch
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_extract_switch_single_label():
|
||||
src = """
|
||||
return switch (action) {
|
||||
case "CreateSecret" -> handleCreate(req);
|
||||
case "GetSecretValue" -> handleGet(req);
|
||||
default -> error();
|
||||
};
|
||||
"""
|
||||
assert r.extract_switch_actions(src) == ["CreateSecret", "GetSecretValue"]
|
||||
|
||||
|
||||
def test_extract_switch_multi_label_arm():
|
||||
# A single arm carrying several labels (real case: CloudFormation StackSets).
|
||||
src = 'case "ListStackSets", "DescribeStackSet", "CreateStackSet" -> handleStackSets(req);'
|
||||
assert r.extract_switch_actions(src) == [
|
||||
"ListStackSets",
|
||||
"DescribeStackSet",
|
||||
"CreateStackSet",
|
||||
]
|
||||
|
||||
|
||||
def test_extract_switch_multiline_arm():
|
||||
# Labels wrapped across lines (valid Java 14+ switch syntax) must all be captured,
|
||||
# and the capture must stop at the arm's `->` rather than running into the next
|
||||
# arm's `{ ... }` body. A miss here would be invisible: the doc and the regenerated
|
||||
# output would agree on the omission and docs-check would stay green.
|
||||
src = '''
|
||||
return switch (action) {
|
||||
case "CreateStackSet",
|
||||
"DescribeStackSet",
|
||||
"ListStackSets" -> handleStackSets(req);
|
||||
case "GetTemplate" -> { return handleGet(req); }
|
||||
};
|
||||
'''
|
||||
assert r.extract_switch_actions(src) == [
|
||||
"CreateStackSet",
|
||||
"DescribeStackSet",
|
||||
"ListStackSets",
|
||||
"GetTemplate",
|
||||
]
|
||||
|
||||
|
||||
def test_extract_switch_ignores_non_pascal_and_default():
|
||||
src = '''
|
||||
case "lowercase" -> a();
|
||||
case null, default -> b();
|
||||
case "Good" -> c();
|
||||
'''
|
||||
assert r.extract_switch_actions(src) == ["Good"]
|
||||
|
||||
|
||||
def test_extract_switch_excludes_screaming_case_enum_labels():
|
||||
# A handler may contain non-action switches (e.g. state enums). SCREAMING_CASE
|
||||
# labels must not be picked up as actions; PascalCase actions still are.
|
||||
src = '''
|
||||
return switch (action) {
|
||||
case "EnableRule" -> enable();
|
||||
case "DisableRule" -> disable();
|
||||
};
|
||||
return switch (state.toUpperCase()) {
|
||||
case "DISABLED" -> RuleState.DISABLED;
|
||||
case "ENABLED" -> RuleState.ENABLED;
|
||||
};
|
||||
'''
|
||||
assert r.extract_switch_actions(src) == ["EnableRule", "DisableRule"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Extraction: rest + overrides
|
||||
# --------------------------------------------------------------------------- #
|
||||
REST_CONTROLLER = """
|
||||
@Path("/")
|
||||
public class ThingController {
|
||||
@POST
|
||||
public Response createThing(Req r) { return ok(); }
|
||||
|
||||
@GET
|
||||
public Response listThingsPost(Req r) { return ok(); }
|
||||
|
||||
@PUT
|
||||
public Response updateV2Thing(Req r) { return ok(); }
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def test_extract_rest_ucfirsts_method_names():
|
||||
assert r.extract_rest_actions(REST_CONTROLLER) == [
|
||||
"CreateThing",
|
||||
"ListThingsPost",
|
||||
"UpdateV2Thing",
|
||||
]
|
||||
|
||||
|
||||
def test_extract_rest_requires_class_path():
|
||||
no_path = REST_CONTROLLER.replace('@Path("/")\n', "")
|
||||
assert r.extract_rest_actions(no_path) == []
|
||||
|
||||
|
||||
def test_extract_rest_skips_comments_between_annotation_and_method():
|
||||
# A Javadoc or inline comment sitting between the verb annotation and the method
|
||||
# signature must not reset the lookahead and drop the action.
|
||||
src = '''
|
||||
@Path("/")
|
||||
public class C {
|
||||
@POST
|
||||
/** Creates a thing. */
|
||||
public Response createThing(Req r) { return ok(); }
|
||||
|
||||
@GET
|
||||
// list them
|
||||
public Response listThings(Req r) { return ok(); }
|
||||
}
|
||||
'''
|
||||
assert r.extract_rest_actions(src) == ["CreateThing", "ListThings"]
|
||||
|
||||
|
||||
def test_extract_rest_inherits_methods_from_superclass(tmp_path: Path):
|
||||
base_src = """
|
||||
public abstract class BaseController {
|
||||
@GET
|
||||
public Response getBaseItem() { return ok(); }
|
||||
|
||||
@DELETE
|
||||
public Response deleteBaseItem() { return ok(); }
|
||||
}
|
||||
"""
|
||||
sub_src = """
|
||||
@Path("/items")
|
||||
public class ChildController extends BaseController {
|
||||
@POST
|
||||
public Response createChildItem() { return ok(); }
|
||||
}
|
||||
"""
|
||||
base_file = tmp_path / "BaseController.java"
|
||||
sub_file = tmp_path / "ChildController.java"
|
||||
base_file.write_text(base_src, encoding="utf-8")
|
||||
sub_file.write_text(sub_src, encoding="utf-8")
|
||||
|
||||
actions = r.extract_rest_actions(sub_src, sub_file)
|
||||
assert actions == ["GetBaseItem", "DeleteBaseItem", "CreateChildItem"]
|
||||
|
||||
|
||||
def test_extract_actions_applies_rename_and_exclude(tmp_path: Path):
|
||||
src = tmp_path / "ThingController.java"
|
||||
src.write_text(REST_CONTROLLER)
|
||||
entry = r.ServiceEntry(
|
||||
service="thing",
|
||||
doc=tmp_path / "thing.md",
|
||||
sources=[r.Source(path=src, mode="rest")],
|
||||
rename_actions={"UpdateV2Thing": "UpdateThing"},
|
||||
exclude_actions=frozenset({"ListThingsPost"}),
|
||||
)
|
||||
assert r.extract_actions(entry) == ["CreateThing", "UpdateThing"]
|
||||
|
||||
|
||||
def test_extract_actions_dedupes_across_sources(tmp_path: Path):
|
||||
a = tmp_path / "A.java"
|
||||
b = tmp_path / "B.java"
|
||||
a.write_text('case "Foo" -> x();\ncase "Bar" -> y();')
|
||||
b.write_text('case "Bar" -> y();\ncase "Baz" -> z();')
|
||||
entry = r.ServiceEntry(
|
||||
service="s",
|
||||
doc=tmp_path / "s.md",
|
||||
sources=[r.Source(path=a, mode="switch"), r.Source(path=b, mode="switch")],
|
||||
)
|
||||
assert r.extract_actions(entry) == ["Foo", "Bar", "Baz"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Table parsing: robustness
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_split_row_honors_escaped_pipe():
|
||||
assert r._split_row(r"| `Foo` | a \| b |") == ["`Foo`", "a | b"]
|
||||
|
||||
|
||||
def test_parse_marker_block_preserves_descriptions():
|
||||
md = (
|
||||
"pre\n"
|
||||
f"{r.MARKER_START}\n"
|
||||
"| Action | Description |\n"
|
||||
"| --- | --- |\n"
|
||||
"| `CreateSecret` | Create a new secret |\n"
|
||||
"| `GetSecretValue` | - |\n"
|
||||
f"{r.MARKER_END}\n"
|
||||
"post\n"
|
||||
)
|
||||
_, descs, _ = r.parse_marker_block(md)
|
||||
assert descs == {"CreateSecret": "Create a new secret", "GetSecretValue": ""}
|
||||
|
||||
|
||||
def test_parse_marker_block_rejects_missing_markers():
|
||||
with pytest.raises(ValueError):
|
||||
r.parse_marker_block("no markers here")
|
||||
|
||||
|
||||
def test_parse_table_rejects_non_table_line():
|
||||
body = "| Action | Description |\n| --- | --- |\nstray prose line\n"
|
||||
with pytest.raises(ValueError):
|
||||
r._parse_table(body)
|
||||
|
||||
|
||||
def test_parse_table_rejects_extra_columns():
|
||||
# An unescaped pipe in a hand-edited description yields a 3-cell row; reject it
|
||||
# loudly rather than silently dropping the trailing text.
|
||||
body = "| Action | Description |\n| --- | --- |\n| `Foo` | a | b |\n"
|
||||
with pytest.raises(ValueError):
|
||||
r._parse_table(body)
|
||||
|
||||
|
||||
def test_pipe_in_description_round_trips():
|
||||
# A description containing a literal pipe survives render -> parse -> render
|
||||
# unchanged (render escapes, the parser unescapes), and stays idempotent.
|
||||
block = r.render_marker_block(["Foo"], {"Foo": "does a | b"})
|
||||
assert r"\|" in block # escaped on render
|
||||
md = f"x\n{r.MARKER_START}\n{block}{r.MARKER_END}\n"
|
||||
_, descs, _ = r.parse_marker_block(md)
|
||||
assert descs["Foo"] == "does a | b" # unescaped on parse
|
||||
assert r.render_marker_block(["Foo"], descs) == block # idempotent
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Regeneration: preservation, placeholders, orphans, idempotency
|
||||
# --------------------------------------------------------------------------- #
|
||||
DOC = (
|
||||
"# Svc\n\n## Supported Actions\n\n"
|
||||
f"{r.MARKER_START}\n"
|
||||
"| Action | Description |\n"
|
||||
"| --- | --- |\n"
|
||||
"| `CreateSecret` | Create a new secret |\n"
|
||||
"| `OldAction` | gone from code |\n"
|
||||
f"{r.MARKER_END}\n\n## Configuration\nprose\n"
|
||||
)
|
||||
|
||||
|
||||
def test_regenerate_preserves_known_and_placeholders_new():
|
||||
actions = ["CreateSecret", "GetSecretValue"]
|
||||
new, orphans = r.regenerate_doc_content(actions, DOC)
|
||||
_, descs, _ = r.parse_marker_block(new)
|
||||
assert descs["CreateSecret"] == "Create a new secret" # preserved
|
||||
assert descs["GetSecretValue"] == "" # new -> placeholder
|
||||
assert orphans == ["OldAction"] # dropped + reported
|
||||
assert "OldAction" not in new
|
||||
|
||||
|
||||
def test_regenerate_is_idempotent():
|
||||
actions = ["CreateSecret", "GetSecretValue"]
|
||||
once, _ = r.regenerate_doc_content(actions, DOC)
|
||||
twice, _ = r.regenerate_doc_content(actions, once)
|
||||
assert once == twice
|
||||
|
||||
|
||||
def test_regenerate_touches_only_marker_block():
|
||||
actions = ["CreateSecret", "GetSecretValue"]
|
||||
new, _ = r.regenerate_doc_content(actions, DOC)
|
||||
assert new.split(r.MARKER_START)[0] == DOC.split(r.MARKER_START)[0]
|
||||
assert new.split(r.MARKER_END)[1] == DOC.split(r.MARKER_END)[1]
|
||||
|
||||
|
||||
def test_render_marker_block_shape():
|
||||
out = r.render_marker_block(["Foo"], {"Foo": "does foo"})
|
||||
assert out == "| Action | Description |\n| --- | --- |\n| `Foo` | does foo |\n"
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Tests for regen_cfn_resource_types.
|
||||
|
||||
Run with: pytest tools/docs -q (or: make docs-test)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
import regen_cfn_resource_types as r
|
||||
|
||||
|
||||
BASE_CONFIG = {
|
||||
"order": ["AWS::S3", "AWS::CloudFormation"],
|
||||
"service_labels": {
|
||||
"AWS::S3": "S3",
|
||||
"AWS::CloudFormation": "CloudFormation",
|
||||
"AWS::SQS": "SQS",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render(inventory, **overrides):
|
||||
config = {**BASE_CONFIG, **overrides}
|
||||
return r.render_rows(inventory, config)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Type naming
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_namespace_and_leaf_of_a_three_segment_type():
|
||||
assert r.namespace_of("AWS::SQS::Queue") == "AWS::SQS"
|
||||
assert r.leaf_of("AWS::SQS::Queue") == "Queue"
|
||||
|
||||
|
||||
def test_two_segment_type_keeps_its_prefix():
|
||||
# Abbreviating Custom::DynamoDBReplica to DynamoDBReplica would read as an AWS
|
||||
# resource type that does not exist.
|
||||
assert r.namespace_of("Custom::DynamoDBReplica") == "Custom"
|
||||
assert r.leaf_of("Custom::DynamoDBReplica") == "Custom::DynamoDBReplica"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Rendering
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_renders_one_row_per_namespace_in_configured_order():
|
||||
lines, warnings = render(
|
||||
[("AWS::SQS::Queue", "SqsCfnProvisioner"), ("AWS::S3::Bucket", "LEGACY_SWITCH")]
|
||||
)
|
||||
assert lines[0] == "| Service | Resource types |"
|
||||
# S3 is in the order list, SQS is not, so SQS is appended after it.
|
||||
assert lines[2] == "| S3 | `Bucket` |"
|
||||
assert lines[3] == "| SQS | `Queue` |"
|
||||
assert any("not in the order list" in w for w in warnings)
|
||||
|
||||
|
||||
def test_unlisted_types_sort_alphabetically_after_curated_ones():
|
||||
inventory = [
|
||||
("AWS::S3::AccessPoint", "LEGACY_SWITCH"),
|
||||
("AWS::S3::Bucket", "LEGACY_SWITCH"),
|
||||
("AWS::S3::BucketPolicy", "LEGACY_SWITCH"),
|
||||
]
|
||||
lines, _ = render(inventory, type_order={"AWS::S3": ["Bucket", "BucketPolicy"]})
|
||||
assert lines[2] == "| S3 | `Bucket`, `BucketPolicy`, `AccessPoint` |"
|
||||
|
||||
|
||||
def test_notes_and_row_notes_are_appended():
|
||||
lines, _ = render(
|
||||
[("AWS::S3::Bucket", "LEGACY_SWITCH")],
|
||||
notes={"AWS::S3::Bucket": "accepted; policy not enforced"},
|
||||
row_notes={"AWS::S3": "Trailing prose."},
|
||||
)
|
||||
assert lines[2] == "| S3 | `Bucket` (accepted; policy not enforced). Trailing prose. |"
|
||||
|
||||
|
||||
def test_display_names_override_the_leaf():
|
||||
lines, _ = render(
|
||||
[("AWS::CDK::Metadata", "LEGACY_SWITCH")],
|
||||
service_labels={**BASE_CONFIG["service_labels"], "AWS::CDK": "CDK"},
|
||||
display_names={"AWS::CDK::Metadata": "CDK::Metadata"},
|
||||
)
|
||||
assert "`CDK::Metadata`" in lines[2]
|
||||
|
||||
|
||||
def test_merged_namespace_folds_into_one_row():
|
||||
lines, _ = render(
|
||||
[
|
||||
("AWS::CloudFormation::CustomResource", "LEGACY_SWITCH"),
|
||||
("Custom::DynamoDBReplica", "LEGACY_SWITCH"),
|
||||
],
|
||||
merge_namespaces={"Custom": "AWS::CloudFormation"},
|
||||
)
|
||||
rows = [line for line in lines if line.startswith("| CloudFormation ")]
|
||||
assert len(rows) == 1, "Custom::* belongs in the CloudFormation row, not a second one"
|
||||
assert "`Custom::DynamoDBReplica`" in rows[0]
|
||||
|
||||
|
||||
def test_extra_types_appear_after_inventory_types():
|
||||
lines, _ = render(
|
||||
[("AWS::CloudFormation::CustomResource", "LEGACY_SWITCH")],
|
||||
extra_types={
|
||||
"AWS::CloudFormation": [
|
||||
{"name": "Stack", "note": "nested stacks", "reason": "not a provisioner type"}
|
||||
]
|
||||
},
|
||||
)
|
||||
assert lines[2] == "| CloudFormation | `CustomResource`, `Stack` (nested stacks) |"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Warnings: these are the drift the gate exists to catch
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_warns_when_a_namespace_has_no_label():
|
||||
_, warnings = render([("AWS::Athena::WorkGroup", "AthenaCfnProvisioner")])
|
||||
assert any("no service_labels entry for AWS::Athena" in w for w in warnings)
|
||||
|
||||
|
||||
def test_warns_when_type_order_names_a_type_nothing_provisions():
|
||||
_, warnings = render(
|
||||
[("AWS::S3::Bucket", "LEGACY_SWITCH")],
|
||||
type_order={"AWS::S3": ["Bucket", "Buckett"]},
|
||||
)
|
||||
assert any("lists Buckett" in w for w in warnings)
|
||||
|
||||
|
||||
def test_warns_when_an_extra_type_has_no_reason():
|
||||
_, warnings = render(
|
||||
[("AWS::CloudFormation::CustomResource", "LEGACY_SWITCH")],
|
||||
extra_types={"AWS::CloudFormation": [{"name": "Stack"}]},
|
||||
)
|
||||
assert any("has no reason" in w for w in warnings)
|
||||
|
||||
|
||||
def test_warns_when_a_type_is_claimed_by_both_the_switch_and_a_provisioner():
|
||||
# The registry wins, so the switch arm is dead code that still looks live.
|
||||
warnings = r.check_no_double_ownership(
|
||||
[("AWS::SQS::Queue", "LEGACY_SWITCH"), ("AWS::SQS::Queue", "SqsCfnProvisioner")]
|
||||
)
|
||||
assert len(warnings) == 1
|
||||
assert "SqsCfnProvisioner" in warnings[0]
|
||||
|
||||
|
||||
def test_no_double_ownership_warning_when_each_type_has_one_owner():
|
||||
assert r.check_no_double_ownership(
|
||||
[("AWS::SQS::Queue", "SqsCfnProvisioner"), ("AWS::S3::Bucket", "LEGACY_SWITCH")]
|
||||
) == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Inventory parsing and splicing
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_load_inventory_rejects_a_malformed_row(tmp_path):
|
||||
path = tmp_path / "inv.tsv"
|
||||
path.write_text("AWS::SQS::Queue\tSqsCfnProvisioner\nAWS::S3::Bucket\n", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="expected 'type<TAB>owner'"):
|
||||
r.load_inventory(path)
|
||||
|
||||
|
||||
def test_splice_replaces_only_the_marked_block():
|
||||
md = f"before\n{r.MARKER_START}\nold table\n{r.MARKER_END}\nafter\n"
|
||||
assert r.splice(md, ["new table"]) == (
|
||||
f"before\n{r.MARKER_START}\nnew table\n{r.MARKER_END}\nafter\n"
|
||||
)
|
||||
|
||||
|
||||
def test_splice_requires_exactly_one_marker_pair():
|
||||
with pytest.raises(ValueError, match="exactly one"):
|
||||
r.splice("no markers here\n", ["x"])
|
||||
@@ -0,0 +1,320 @@
|
||||
"""Tests for service_inventory.
|
||||
|
||||
Every test here pins a parser trap that was found by hand-checking the real sources,
|
||||
and that would otherwise have made this tool produce a *tenth* wrong number in a repo
|
||||
that already had nine. They are written against synthetic sources so they keep failing
|
||||
for the right reason when the real files change.
|
||||
|
||||
Run with: pytest tools/docs -q (or: make docs-test)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
import service_inventory as si
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Key normalisation
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_key_normalises_across_naming_conventions():
|
||||
# The same service, spelled three ways by three sources.
|
||||
assert si.key("bcmDataExports") == si.key("bcm-data-exports") == si.key("bcmdataexports")
|
||||
|
||||
|
||||
def test_key_does_not_collide_across_distinct_services():
|
||||
assert si.key("rds") != si.key("rds-data")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Descriptor parsing
|
||||
# --------------------------------------------------------------------------- #
|
||||
SIMPLE = '''
|
||||
descriptor("ssm", "ssm", config.services().ssm().enabled(), true,
|
||||
"ssm", storageMode(...), 5000L, null, ServiceProtocol.JSON,
|
||||
protocols(ServiceProtocol.JSON),
|
||||
Set.of("AmazonSSM."), Set.of("ssm"), Set.of(), Set.of()),
|
||||
'''
|
||||
|
||||
|
||||
def test_parses_key_config_key_and_status_flag():
|
||||
[d] = si.parse_descriptors(SIMPLE)
|
||||
assert d["external_key"] == "ssm"
|
||||
assert d["config_key"] == "ssm"
|
||||
assert d["toggles"] == ["ssm"]
|
||||
assert d["in_status"] is True
|
||||
|
||||
|
||||
def test_conjunction_toggle_wrapping_onto_a_continuation_line():
|
||||
# rds-data and redshift-data wrap because their enabled expression is a conjunction.
|
||||
# A line-oriented parser reads `true` off the *first* line, which for these two is
|
||||
# the configKey's line, and gets both the toggles and includeInStatus wrong.
|
||||
src = '''
|
||||
descriptor("rds-data", "rds-data",
|
||||
config.services().rds().enabled() && config.services().rdsData().enabled(), false,
|
||||
null, null, 5000L, null, ServiceProtocol.REST_JSON,
|
||||
protocols(ServiceProtocol.REST_JSON),
|
||||
Set.of(), Set.of("rds-data"), Set.of(), Set.of(RdsDataController.class)),
|
||||
'''
|
||||
[d] = si.parse_descriptors(src)
|
||||
assert d["external_key"] == "rds-data"
|
||||
assert d["toggles"] == ["rds", "rdsData"]
|
||||
assert d["in_status"] is False
|
||||
|
||||
|
||||
def test_status_flag_is_read_from_the_fourth_argument_not_the_first_true():
|
||||
# `includeInStatus=false` on a descriptor whose second line contains other literals.
|
||||
src = '''
|
||||
descriptor("sts", "iam", config.services().iam().enabled(), false,
|
||||
null, null, 5000L, AwsNamespaces.STS, ServiceProtocol.QUERY,
|
||||
protocols(ServiceProtocol.QUERY),
|
||||
Set.of(), Set.of("sts"), Set.of(), Set.of()),
|
||||
'''
|
||||
assert si.parse_descriptors(src)[0]["in_status"] is False
|
||||
|
||||
|
||||
def test_nested_commas_do_not_split_an_argument():
|
||||
args = si.split_top_level_args('f("a", Set.of("b", "c"), true)', 1)
|
||||
assert args == ['"a"', 'Set.of("b", "c")', "true"]
|
||||
|
||||
|
||||
def test_unbalanced_parens_raise_rather_than_returning_a_short_list():
|
||||
with pytest.raises(ValueError, match="unbalanced"):
|
||||
si.split_top_level_args('f("a", "b"', 1)
|
||||
|
||||
|
||||
def test_non_literal_external_key_raises():
|
||||
# check_service_matrix.py catches this case by counting call sites; here it must
|
||||
# fail loudly rather than silently dropping a service from every count.
|
||||
with pytest.raises(ValueError, match="not a plain string literal"):
|
||||
si.parse_descriptors("descriptor(KEY, \"ssm\", true, true, ...)")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Config accessors
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_config_accessors_exclude_docker_network_and_include_non_service_configs():
|
||||
# DuckConfig does not end in ServiceConfig, so a `*ServiceConfig` pattern misses it
|
||||
# and under-counts by one. dockerNetwork() returns Optional<String> and must not
|
||||
# be counted at all.
|
||||
src = """
|
||||
interface ServicesConfig {
|
||||
Optional<String> dockerNetwork();
|
||||
|
||||
SsmServiceConfig ssm();
|
||||
SqsServiceConfig sqs();
|
||||
DuckConfig duck();
|
||||
}
|
||||
"""
|
||||
assert si.parse_config_accessors(src) == ["duck", "sqs", "ssm"]
|
||||
|
||||
|
||||
def test_missing_services_config_interface_raises():
|
||||
with pytest.raises(ValueError, match="ServicesConfig not found"):
|
||||
si.parse_config_accessors("interface Unrelated { }")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Matrix rows
|
||||
# --------------------------------------------------------------------------- #
|
||||
MATRIX = f"""{si.csm.MATRIX_HEADING}
|
||||
|
||||
| Service | Endpoint | Protocol | Supported operations |
|
||||
|---|---|---|---|
|
||||
| [DynamoDB](dynamodb.md) | `POST /` | JSON 1.1 | 28 |
|
||||
| [DynamoDB Streams](dynamodb.md#streams) | `POST /` | JSON 1.1 | 4 |
|
||||
| [IoT Core](iot.md) | `/things` | REST JSON | 40 |
|
||||
| [IoT Data](iot.md) | `/topics` | REST JSON | 5 |
|
||||
| [S3](s3.md) | `/{{bucket}}` | REST XML | 58 |
|
||||
|
||||
{si.csm.MATRIX_END_HEADING}
|
||||
"""
|
||||
|
||||
|
||||
def test_facet_rows_are_detected_by_duplicate_slug_not_by_anchor():
|
||||
# IoT Core and IoT Data both link plain `iot.md` with no anchor at all. An
|
||||
# anchor-based rule finds only the DynamoDB pair and over-counts pages by one.
|
||||
rows = si.parse_matrix_rows(MATRIX)
|
||||
assert len(rows) == 5
|
||||
assert {r["slug"] for r in rows} == {"dynamodb", "iot", "s3"}
|
||||
assert rows[3]["anchor"] is None
|
||||
|
||||
|
||||
def test_rows_outside_the_matrix_headings_are_ignored():
|
||||
src = f"| [Elsewhere](elsewhere.md) | x |\n\n{MATRIX}"
|
||||
assert all(r["slug"] != "elsewhere" for r in si.parse_matrix_rows(src))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# README
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_category_names_do_not_split_inside_parentheses():
|
||||
# "IAM Identity Center (SSO Admin, OIDC, Access Portal, SCIM)" is one service name.
|
||||
# Splitting on every comma turns it into four, three of which ("OIDC", "SCIM)")
|
||||
# are not service names and inflate the count.
|
||||
src = (
|
||||
"| Category | Services |\n|---|---|\n"
|
||||
"| Identity | IAM, IAM Identity Center (SSO Admin, OIDC, Access Portal, SCIM), KMS |\n"
|
||||
"\nFor operation-level compatibility, see the Services Overview.\n"
|
||||
)
|
||||
assert si.parse_readme_category_names(src) == [
|
||||
"IAM",
|
||||
"IAM Identity Center (SSO Admin, OIDC, Access Portal, SCIM)",
|
||||
"KMS",
|
||||
]
|
||||
|
||||
|
||||
def test_detail_rows_take_the_first_cell_only():
|
||||
src = (
|
||||
"| Service | How it works | Notable features |\n|---|---|---|\n"
|
||||
"| SQS | In-process | Standard and FIFO queues, DLQ |\n"
|
||||
"</details>\n"
|
||||
)
|
||||
assert si.parse_readme_detail_rows(src) == ["SQS"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Reconciliation
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _data(**overrides) -> dict:
|
||||
base = {
|
||||
"descriptors": [
|
||||
{"external_key": "iam", "config_key": "iam", "toggles": ["iam"], "in_status": True},
|
||||
{"external_key": "sts", "config_key": "iam", "toggles": ["iam"], "in_status": False},
|
||||
{"external_key": "s3", "config_key": "s3", "toggles": ["s3"], "in_status": True},
|
||||
],
|
||||
"config_accessors": ["iam", "s3", "duck"],
|
||||
"yaml_keys": ["iam", "s3", "duck"],
|
||||
"yaml_enabled": ["iam", "s3"],
|
||||
"packages": ["iam", "s3"],
|
||||
"matrix_rows": [
|
||||
{"label": "IAM", "slug": "iam", "anchor": None},
|
||||
{"label": "AWS Sign-In", "slug": "iam", "anchor": "signin"},
|
||||
{"label": "S3", "slug": "s3", "anchor": None},
|
||||
],
|
||||
"doc_pages": ["iam", "s3"],
|
||||
"readme_categories": ["IAM", "S3"],
|
||||
"readme_details": ["S3"],
|
||||
"action_tables": ["s3"],
|
||||
"aliases": {"sts": "iam"},
|
||||
"facet_pages": [],
|
||||
"deferred": [],
|
||||
}
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
|
||||
def test_shared_toggle_is_reported_with_every_descriptor_it_gates():
|
||||
r = si.reconcile(_data())
|
||||
assert r["catalog"]["shared_toggles"] == {"iam": ["iam", "sts"]}
|
||||
assert r["counts"]["catalog_toggles"] == 2
|
||||
|
||||
|
||||
def test_facet_descriptor_is_the_non_owner_of_a_shared_page():
|
||||
r = si.reconcile(_data())
|
||||
assert r["catalog"]["facet_descriptors"] == ["sts"]
|
||||
assert r["counts"]["catalog_distinct_products"] == 2
|
||||
|
||||
|
||||
def test_non_aws_entries_are_subtracted_from_the_aws_key_count():
|
||||
r = si.reconcile(_data())
|
||||
assert r["config"]["non_aws"] == ["duck"]
|
||||
assert r["counts"]["yaml_aws_keys"] == 2
|
||||
|
||||
|
||||
def test_config_and_yaml_agree_after_normalisation():
|
||||
r = si.reconcile(_data(config_accessors=["rdsData"], yaml_keys=["rds-data"]))
|
||||
assert r["config"]["accessor_only"] == []
|
||||
assert r["config"]["yaml_only"] == []
|
||||
|
||||
|
||||
def test_unregistered_package_is_reported_when_not_noted():
|
||||
r = si.reconcile(_data(packages=["iam", "s3", "newthing"]))
|
||||
assert r["packages"]["unregistered"] == ["newthing"]
|
||||
|
||||
|
||||
def test_noted_package_is_not_reported_as_unregistered():
|
||||
si.PACKAGE_NOTES["temporary"] = "test fixture"
|
||||
try:
|
||||
r = si.reconcile(_data(packages=["iam", "s3", "temporary"]))
|
||||
assert r["packages"]["unregistered"] == []
|
||||
assert "temporary" in r["packages"]["noted"]
|
||||
finally:
|
||||
del si.PACKAGE_NOTES["temporary"]
|
||||
|
||||
|
||||
def test_facet_row_missing_from_readme_is_separated_from_a_real_gap():
|
||||
# "AWS Sign-In" shares iam.md, so its absence from the README is by design.
|
||||
# A non-facet row missing from the README is a genuine finding.
|
||||
r = si.reconcile(_data(readme_categories=["IAM"]))
|
||||
assert r["readme"]["missing_from_categories"] == ["S3"]
|
||||
assert r["readme"]["missing_facet_rows"] == ["AWS Sign-In"]
|
||||
|
||||
|
||||
def test_readme_alias_covers_several_matrix_rows_at_once():
|
||||
si.README_NAME_ALIASES["Identity"] = ["IAM", "AWS Sign-In"]
|
||||
try:
|
||||
r = si.reconcile(_data(readme_categories=["Identity", "S3"]))
|
||||
assert r["readme"]["unmatched_category_names"] == []
|
||||
assert r["readme"]["missing_facet_rows"] == []
|
||||
finally:
|
||||
del si.README_NAME_ALIASES["Identity"]
|
||||
|
||||
|
||||
def test_readme_name_matching_no_row_is_reported_rather_than_dropped():
|
||||
r = si.reconcile(_data(readme_categories=["IAM", "S3", "Imaginary Service"]))
|
||||
assert r["readme"]["unmatched_category_names"] == ["Imaginary Service"]
|
||||
|
||||
|
||||
def test_deferral_within_the_warning_window_is_flagged():
|
||||
deferred = [{"key": "x", "reason": "r", "by": "2026-09-15"}]
|
||||
r = si.reconcile(_data(deferred=deferred), today=date(2026, 9, 14))
|
||||
assert r["docs"]["deferred_expiring"] == deferred
|
||||
|
||||
|
||||
def test_deferral_beyond_the_warning_window_is_not_flagged():
|
||||
deferred = [{"key": "x", "reason": "r", "by": "2027-01-01"}]
|
||||
r = si.reconcile(_data(deferred=deferred), today=date(2026, 9, 14))
|
||||
assert r["docs"]["deferred_expiring"] == []
|
||||
|
||||
|
||||
def test_row_exempt_facet_pages_are_counted_without_requiring_a_matrix_row():
|
||||
r = si.reconcile(_data(doc_pages=["iam", "s3", "iam-verification"],
|
||||
facet_pages=["iam-verification"]))
|
||||
assert r["counts"]["matrix_pages"] == 2
|
||||
assert r["counts"]["doc_pages"] == 3
|
||||
assert r["docs"]["pages_without_row"] == []
|
||||
assert r["docs"]["rows_without_page"] == []
|
||||
|
||||
|
||||
def test_facet_exemption_does_not_hide_an_unregistered_page():
|
||||
r = si.reconcile(_data(doc_pages=["iam", "s3", "iam-verification", "unregistered"],
|
||||
facet_pages=["iam-verification"]))
|
||||
assert r["docs"]["pages_without_row"] == ["unregistered"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# End to end, against the real repo
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_real_sources_parse_and_render():
|
||||
data = si.collect(si._repo_root())
|
||||
result = si.reconcile(data)
|
||||
counts = result["counts"]
|
||||
# Not pinned to exact values: they move every time a service lands. What must hold
|
||||
# is that every source parsed to something plausible and the two gated equalities
|
||||
# this tool reports as green really are green.
|
||||
assert all(v > 50 for v in counts.values())
|
||||
assert counts["matrix_pages"] == len(set(data["doc_pages"]) - set(data["facet_pages"]))
|
||||
assert counts["config_accessors"] == counts["yaml_keys"]
|
||||
assert result["docs"]["pages_without_row"] == []
|
||||
assert result["docs"]["rows_without_page"] == []
|
||||
assert result["packages"]["unregistered"] == []
|
||||
assert "THE ANSWER" in si.render(result)
|
||||
|
||||
|
||||
def test_report_never_suggests_publishing_a_count():
|
||||
# The hard rule the tool exists to protect, asserted rather than trusted to prose.
|
||||
text = si.render(si.reconcile(si.collect(si._repo_root())))
|
||||
assert "Never write any of them into README.md or docs/" in text
|
||||
Reference in New Issue
Block a user