feat(cognito): show the client's identity providers on the hosted login page

The hosted login page now offers a "Continue with <provider>" button for each
of the app client's other SupportedIdentityProviders, carrying the same
authorize parameters (state, nonce, PKCE) into the federated flow, as Cognito's
managed login does. A client that lists only external providers gets the
buttons without the password form instead of an error.

README.md

@@ -203,7 +203,7 @@ Run AWS-compatible services locally without an AWS account, auth token, or paid
 <details>
 <summary><strong>Real Docker where fidelity matters</strong></summary>
 
-Lambda, RDS, Neptune, ElastiCache, MSK, ECS, EC2, EKS, OpenSearch, CodeBuild, and Managed Service for Apache Flink use real Docker-backed execution instead of shallow mocks.
+Lambda, RDS, Neptune, ElastiCache, MSK, MSK Connect, ECS, EC2, EKS, OpenSearch, CodeBuild, and Managed Service for Apache Flink use real Docker-backed execution instead of shallow mocks.
 
 </details>
 
@@ -244,7 +244,7 @@ LocalStack's community edition [sunset in March 2026](https://blog.localstack.cl
 | License | MIT | Restricted |
 | API Gateway v2 / HTTP API | Yes | No |
 | Cognito | Yes | No |
-| RDS, ElastiCache, MSK | Real Docker | No |
+| RDS, ElastiCache, MSK, MSK Connect | Real Docker | No |
 | Neptune (graph DB + Gremlin WebSocket) | Real Docker | No |
 | DocumentDB (MongoDB-compatible) | Real Docker | No |
 | ECS, EC2, EKS | Real Docker | No |
@@ -298,7 +298,7 @@ Floci supports local emulation for application services, data services, eventing
 | Containers and compute | ECS, EC2, Lightsail, EKS, MWAA, ECR, EFS, CodeBuild, CodeDeploy, CodePipeline, CodeGuru Reviewer, CodeArtifact, AWS Batch, Auto Scaling, Application Auto Scaling, Elastic Beanstalk, ELB v2, ELB Classic |
 | Data, analytics, and AI | Athena, Glue, Lake Formation, EMR, EMR Serverless, Redshift, Redshift Data API, Firehose, Managed Service for Apache Flink, OpenSearch, S3 Tables, S3 Vectors, Textract, Transcribe, Comprehend, Rekognition, Translate, Bedrock, Bedrock Runtime, Bedrock AgentCore, Bedrock AgentCore Control, SageMaker |
 | Databases and caching | RDS, RDS Data API, Neptune, DocumentDB, DMS, MemoryDB, ElastiCache, Timestream for InfluxDB |
-| Messaging and transfer | SES, Kinesis, MSK, Amazon MQ, Transfer Family, DataSync, IoT Core, Amazon Connect, Amazon AppIntegrations |
+| Messaging and transfer | SES, Kinesis, MSK, MSK Connect, Amazon MQ, Transfer Family, DataSync, IoT Core, Amazon Connect, Amazon AppIntegrations |
 | Security and governance | AWS Network Firewall, AWS RAM, Service Quotas, WAF v2, GuardDuty, Amazon Inspector, CloudTrail, CloudFront, Resource Groups Tagging API, Resource Explorer 2, CloudHSM v2, Organizations, AWS Account Management, IAM Access Analyzer, IAM Identity Center (SSO Admin, OIDC, Access Portal, SCIM), Identity Store, Amazon Macie, Amazon Detective, Security Hub, Amazon Verified Permissions, Control Catalog, Control Tower, Service Catalog, AWS Marketplace |
 | Cost and billing | AWS Budgets, Pricing, Cost Explorer, Cost and Usage Reports, BCM Pricing Calculator, BCM Data Exports |
 | Resilience, backup, and config | AWS FIS, AWS Backup, AWS Config, AppConfig, AppConfigData, CloudFormation, Cloud Control API |
@@ -342,6 +342,7 @@ For operation-level compatibility, see the [Services Overview](https://floci.io/
 | Neptune | Real Docker | Graph DB via TinkerPop Gremlin Server (default) or Neo4j for openCypher/Bolt (`NEPTUNE_DB_TYPE`); RDS-shaped control plane; SigV4 proxy on port 8182 |
 | DocumentDB | Real Docker, mock mode available | MongoDB-compatible cluster via real MongoDB containers; RDS-shaped control plane; MongoDB wire protocol on port 27017 |
 | MSK | Real Docker | Kafka-compatible broker via Redpanda |
+| MSK Connect | Real Docker | Stock Kafka Connect workers with S3 custom plugin ZIPs |
 | Amazon MQ | Real Docker | RabbitMQ broker via rabbitmq:3-management; AMQP + management console |
 | Athena | In-process with DuckDB sidecar | Real SQL execution over S3 and Glue-backed views |
 | Glue | In-process | Data Catalog, Schema Registry, tables consumed by Athena |

docker/ec2/ami-images/debian-13/Dockerfile

@@ -0,0 +1,22 @@
+FROM docker.io/library/debian:trixie-slim
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        systemd \
+        systemd-sysv \
+        openssh-client \
+        openssh-server \
+        iproute2 \
+        iptables \
+        socat \
+        curl \
+        ca-certificates \
+    && systemctl enable ssh \
+    && printf '#!/bin/sh\nmkdir -p /run/sshd\nexec "$@"\n' > /usr/local/sbin/floci-init \
+    && chmod +x /usr/local/sbin/floci-init \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV container=docker
+STOPSIGNAL SIGRTMIN+3
+ENTRYPOINT ["/usr/local/sbin/floci-init"]
+CMD ["/sbin/init"]

docker/ec2/ami-images/debian-13/README.md

@@ -0,0 +1,10 @@
+# Debian 13 EC2 guest
+
+Build the local guest image before launching `ami-debian13`:
+
+```text
+docker build --platform linux/amd64 -t floci/ami-debian:13 docker/ec2/ami-images/debian-13
+```
+
+The image runs systemd and starts SSH on every EC2 boot. Enable application services with
+`systemctl enable --now <service>` so they restart after `StopInstances` and `StartInstances`.

docker/ec2/ami-images/fedora-42/Dockerfile

@@ -0,0 +1,17 @@
+FROM registry.fedoraproject.org/fedora:42
+
+RUN dnf install -y \
+        systemd \
+        openssh-clients \
+        openssh-server \
+        iproute \
+        iptables \
+        socat \
+        curl \
+        ca-certificates \
+    && dnf clean all \
+    && systemctl enable sshd
+
+ENV container=docker
+STOPSIGNAL SIGRTMIN+3
+CMD ["/sbin/init"]

docker/ec2/ami-images/fedora-42/README.md

@@ -0,0 +1,10 @@
+# Fedora 42 systemd EC2 guest
+
+Build the local guest image before launching `ami-fedora42`:
+
+```text
+docker build --platform linux/amd64 -t floci/ami-fedora:42 docker/ec2/ami-images/fedora-42
+```
+
+The image runs systemd and starts SSH on every EC2 boot. Enable application services with
+`systemctl enable --now <service>` so they restart after `StopInstances` and `StartInstances`.

docker/ec2/ami-images/opensuse-leap-15.6/Dockerfile

@@ -0,0 +1,4 @@
+FROM registry.opensuse.org/opensuse/leap:15.6
+
+RUN zypper --non-interactive install -y openssh iproute2 iptables socat curl ca-certificates \
+    && zypper clean --all

docker/ec2/ami-images/opensuse-leap-15.6/README.md

@@ -0,0 +1,9 @@
+# openSUSE Leap 15.6 EC2 guest
+
+Build the local guest image before launching `ami-opensuse-leap156`:
+
+```text
+docker build --platform linux/amd64 -t floci/ami-opensuse:15.6 docker/ec2/ami-images/opensuse-leap-15.6
+```
+
+The image includes OpenSSH and the EC2 metadata proxy dependencies.

docker/ec2/ami-images/ubuntu-20.04/Dockerfile

@@ -0,0 +1,19 @@
+FROM public.ecr.aws/docker/library/ubuntu:20.04
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        systemd \
+        systemd-sysv \
+        openssh-client \
+        openssh-server \
+        iproute2 \
+        iptables \
+        socat \
+        curl \
+        ca-certificates \
+    && systemctl enable ssh \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV container=docker
+STOPSIGNAL SIGRTMIN+3
+CMD ["/sbin/init"]

docker/ec2/ami-images/ubuntu-20.04/README.md

@@ -0,0 +1,10 @@
+# Ubuntu 20.04 systemd EC2 guest
+
+Build the local guest image before launching `ami-0abcdef1234567892`:
+
+```text
+docker build --platform linux/amd64 -t floci/ami-ubuntu:20.04 docker/ec2/ami-images/ubuntu-20.04
+```
+
+The image runs systemd and starts SSH on every EC2 boot. Enable application services with
+`systemctl enable --now <service>` so they restart after `StopInstances` and `StartInstances`.

docs/services/codedeploy.md

@@ -109,8 +109,8 @@ For `computePlatform: ECS`, `CreateDeployment` performs a full blue/green traffi
 3. Runs lifecycle hook Lambdas in order: `BeforeInstall` → (install) → `AfterInstall` → `BeforeAllowTraffic` → (traffic shift) → `AfterAllowTraffic`
 4. **Traffic shift** — atomically updates the ELB v2 listener's default forward rule:
    - `ECSAllAtOnce`: immediately shifts 100% of traffic to the green target group
-   - `ECSCanary*`: shifts the canary percentage first, waits a short interval (capped at 5 s in emulator), then shifts to 100%
-   - `ECSLinear*`: shifts traffic in equal increments (capped at 2 s per step in emulator)
+    - `ECSCanary*`: shifts the canary percentage first, waits the configured interval, then shifts to 100%
+    - `ECSLinear*`: shifts traffic in equal increments at the configured interval
 5. Promotes the green task set to **PRIMARY** on the ECS service and deletes the original blue task set
 6. Marks the deployment `Succeeded`; if any lifecycle hook reports `Failed`, the deployment is marked `Failed`
 

docs/services/cognito.md

@@ -75,9 +75,9 @@ AWS-shaped `/oauth2/authorize` and `/oauth2/idpresponse` endpoints to the config
 OIDC provider, exchanges the returned code, provisions or reconciles the federated user,
 and issues a Cognito authorization code for the registered callback.
 
-Only `ProviderType=OIDC` is supported by this flow. Floci does not provide a built-in hosted
-UI and does not implement social providers such as Google, Facebook, Login with Amazon or
-SignInWithApple.
+Only `ProviderType=OIDC` is supported by this flow. Floci does not implement social providers
+such as Google, Facebook, Login with Amazon or SignInWithApple. The pool's own users sign in on
+the hosted login page instead (see Well-Known And OAuth Endpoints below).
 
 Two deliberate divergences from AWS, both consequences of not calling out to a third
 party:
@@ -280,8 +280,8 @@ up, so an oversized request naming something that does not exist reports the req
 problem rather than `ResourceNotFoundException`, and an update violating both reports them
 in one message with the asset list first.
 
-Branding is presentation for the hosted UI, which Floci does not serve, so it is stored
-and returned rather than rendered. Two divergences follow from that:
+Branding is presentation for the hosted UI. Floci's hosted login page does not apply it, so it
+is stored and returned rather than rendered. Two divergences follow from that:
 
 - **`Settings` is stored opaquely.** AWS validates it against a deep schema, rejecting
   unknown properties with `Invalid settings provided. Validation errors: [{property:
@@ -304,17 +304,40 @@ and returned rather than rendered. Two divergences follow from that:
 | `GET /{userPoolId}/.well-known/jwks.json`            | JSON Web Key Set for JWT validation                              |
 | `GET /cognito-idp/oauth2/authorize`                  | OIDC authorization-code start endpoint                         |
 | `GET /cognito-idp/oauth2/idpresponse`                | OIDC provider callback endpoint                                |
-| `POST /cognito-idp/oauth2/token`                     | OAuth authorization-code and client-credentials token endpoint |
+| `GET, POST /cognito-idp/login`                       | Hosted login page for the pool's own users                     |
+| `GET /cognito-idp/logout`                            | Ends the hosted login session                                  |
+| `POST /cognito-idp/oauth2/token`                     | OAuth authorization-code, refresh-token and client-credentials token endpoint |
 
 The OAuth endpoints support browser-style OIDC authorization-code sign-in as well as the
 emulator-friendly client-credentials flow:
 
-- `GET /cognito-idp/oauth2/authorize` validates the app client and callback, then redirects
-  to the configured provider with an opaque state and nonce.
+- `GET /cognito-idp/oauth2/authorize` validates the app client and callback. With
+  `identity_provider` naming an OIDC provider it redirects there with an opaque state and nonce.
+  Without `identity_provider`, or with `identity_provider=COGNITO`, the pool's own users sign in
+  on the hosted login page; the client must list `COGNITO` in `SupportedIdentityProviders` (the
+  default when none are given).
+- `GET /cognito-idp/login` shows a user name (or email) and password form when the client lists
+  `COGNITO`, and a "Continue with <provider>" button for each of its other
+  `SupportedIdentityProviders`, which starts the federated sign-in above with the same authorize
+  parameters. A correct password
+  redirects to the registered callback with a one-time authorization code and `state`, and starts
+  a one-hour browser session (the `cognito` cookie), so the next authorize request returns a code
+  without the form unless it sends `prompt=login`. Wrong passwords, unknown users and
+  unconfirmed users get the same answer until the password is right. Users with a temporary
+  password, MFA or a pending reset are told so; those steps are not served on the page yet.
+- `GET /cognito-idp/logout?client_id=...&logout_uri=...` ends the session and redirects to
+  `logout_uri`, which must be one of the client's `LogoutURLs`. With `redirect_uri` and
+  `response_type=code` instead, it ends the session and shows the login page again.
+- PKCE: send `code_challenge` with `code_challenge_method=S256` to authorize, and the matching
+  `code_verifier` when redeeming the code. `nonce` is copied into the ID token.
+- Codes from the hosted login and from federated sign-in fire the Pre Token Generation trigger
+  with `triggerSource=TokenGeneration_HostedAuth`, and the hosted login fires Pre and Post
+  Authentication.
 - `GET /cognito-idp/oauth2/idpresponse` consumes the provider state, exchanges the provider
   code and redirects to the registered callback with a one-time Cognito authorization code.
-- `POST /cognito-idp/oauth2/token` redeems that authorization code once, or issues a machine
-  token for `grant_type=client_credentials`.
+- `POST /cognito-idp/oauth2/token` redeems that authorization code once, exchanges a
+  `refresh_token` for new access and ID tokens, or issues a machine token for
+  `grant_type=client_credentials`.
 
 `POST /cognito-idp/oauth2/token` is intentionally emulator-friendly rather than full Cognito parity:
 

docs/services/ec2.md

@@ -26,14 +26,16 @@ metadata.
 | AMI ID | Aliases | Docker image |
 |---|---|---|
 | `ami-0abcdef1234567890` | `ami-amazonlinux2` | `public.ecr.aws/amazonlinux/amazonlinux:2` |
-| `ami-0abcdef1234567891` | `ami-amazonlinux2023` | `public.ecr.aws/amazonlinux/amazonlinux:2023` |
-| `ami-0abcdef1234567892` | `ami-ubuntu2004` | `public.ecr.aws/docker/library/ubuntu:20.04` |
+| `ami-0abcdef1234567891` | `ami-amazonlinux2023` | `floci/ami-amazonlinux:2023` |
+| `ami-0abcdef1234567892` | `ami-ubuntu2004` | `floci/ami-ubuntu:20.04` |
 | `ami-ubuntu2204` | | `public.ecr.aws/docker/library/ubuntu:22.04` |
 | `ami-ubuntu2404-arm64` | `ami-ubuntu2404` | `public.ecr.aws/docker/library/ubuntu:24.04` |
 | `ami-ubuntu2404-amd64` | | `public.ecr.aws/docker/library/ubuntu:24.04` |
 | `ami-ubuntu2404-cloud-arm64` | `ami-ubuntu2404-cloud` | `floci/ami-ubuntu:24.04-arm64` |
 | `ami-debian12` | | `public.ecr.aws/docker/library/debian:12` |
+| `ami-debian13` | | `floci/ami-debian:13` |
 | `ami-alpine` | | `public.ecr.aws/docker/library/alpine:latest` |
+| `ami-fedora42` | | `floci/ami-fedora:42` |
 | `ami-0abcdef1234567893` | | `public.ecr.aws/amazonlinux/amazonlinux:2023` |
 
 Any unrecognized AMI ID (including real AWS AMI IDs like `ami-0abc12345678`) falls back to the catalog `defaultDockerImage` (`public.ecr.aws/amazonlinux/amazonlinux:2023` by default).

docs/services/index.md

@@ -65,6 +65,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac
 | [RDS Data API](rds-data.md) | `/Execute`, `/BeginTransaction`, `/CommitTransaction`, `/RollbackTransaction` | REST JSON | 4 |
 | [Timestream for InfluxDB](timestream-influxdb.md) | `POST /` + `X-Amz-Target: AmazonTimestreamInfluxDB.*` + InfluxDB container | JSON 1.0 + InfluxDB HTTP | 24 |
 | [MSK](msk.md) | `/v1/clusters/...`, `/api/v2/clusters/...` + Redpanda broker | REST JSON + Kafka | 8 |
+| [MSK Connect](kafka-connect.md) | `/v1/custom-plugins`, `/v1/connectors`, `/v1/worker-configurations` + Kafka Connect worker | REST JSON + Kafka Connect | 13 |
 | [Amazon MQ](amazonmq.md) | `/v1/brokers/...` + RabbitMQ broker | REST JSON + AMQP | 5 |
 | [MWAA](mwaa.md) | `/` REST paths for environments + CLI/web proxy | REST JSON | 10 |
 | [Athena](athena.md) | `POST /` + `X-Amz-Target: AmazonAthena.*` | JSON 1.1 | 4 |

docs/services/kafka-connect.md

@@ -0,0 +1,45 @@
+# MSK Connect
+
+**Protocol:** REST JSON
+**Signing name:** `kafkaconnect`
+
+Floci implements the MSK Connect control plane separately from MSK. Custom plugins are ZIP
+archives stored in Floci S3. A connector starts a pinned stock Apache Kafka Connect worker, extracts
+the referenced ZIP, and mounts only that extracted plugin directory into the worker.
+
+## Supported Actions
+
+| Action | Description |
+|---|---|
+| `CreateCustomPlugin` | Register a ZIP custom plugin from an S3 bucket ARN and key |
+| `ListCustomPlugins` | List custom plugins |
+| `DescribeCustomPlugin` | Describe a custom plugin |
+| `DeleteCustomPlugin` | Delete a custom plugin |
+| `CreateWorkerConfiguration` | Create a worker properties configuration |
+| `ListWorkerConfigurations` | List worker configurations |
+| `DescribeWorkerConfiguration` | Describe a worker configuration |
+| `DeleteWorkerConfiguration` | Delete a worker configuration |
+| `CreateConnector` | Start a connector using a custom plugin |
+| `ListConnectors` | List connectors |
+| `DescribeConnector` | Describe a connector |
+| `UpdateConnector` | Update connector configuration |
+| `DeleteConnector` | Stop and delete a connector |
+
+## Worker behavior
+
+Workers use the internal Redpanda address of a Floci-managed MSK cluster. The public bootstrap
+address supplied to `CreateConnector` is resolved to that cluster but is never passed into the
+container. This keeps the worker on Floci's Docker network while host clients continue using the
+public MSK bootstrap endpoint.
+
+No connector plugin is bundled. To run an S3 sink, upload a compatible S3 sink connector ZIP to
+Floci S3, create a custom plugin for it, and use that plugin in `CreateConnector`. The worker's
+container logs are streamed to CloudWatch Logs under `/aws/msk-connect/{connectorName}`.
+
+## Configuration
+
+| Variable | Default | Description |
+|---|---|---|
+| `FLOCI_SERVICES_KAFKACONNECT_ENABLED` | `true` | Enable or disable MSK Connect |
+| `FLOCI_SERVICES_KAFKACONNECT_MOCK` | `false` | Persist control-plane metadata without starting workers |
+| `FLOCI_SERVICES_KAFKACONNECT_WORKER_IMAGE` | `apache/kafka:3.9.0` | Pinned stock Kafka Connect worker image |

docs/services/kinesis.md

@@ -91,6 +91,15 @@ TPS over 10 will trigger the LimitExceededException"), independent of the ten-ca
 limit above. Floci does not simulate real-time request throttling for this or any other
 action, so that limit is intentionally unenforced here.
 
+## Current Limitations
+
+- `ListStreams`, `ListShards`, `ListTagsForStream`, and `ListStreamConsumers` do not yet
+  support AWS pagination tokens. `ListShards` honors `MaxResults`, but does not return a
+  continuation token.
+- Resource policy actions are not implemented.
+- KMS encryption settings are recorded in stream metadata, but record payloads are not
+  cryptographically encrypted.
+
 ## Configuration
 
 | Variable | Default | Description |

docs/services/lightsail.md

@@ -16,6 +16,7 @@ The local implementation keeps Lightsail state in Floci storage and is designed
 - Key pairs: create, import, get, list, delete, and download default key pair
 - Discovery: regions, blueprints, bundles, active names, operations, and operations for resource
 - Tags: tag and untag local Lightsail resources
+- Node.js instances: the `nodejs` blueprint uses the multi-architecture `node:24-bookworm` current LTS image, supports the selected SSH key, and persists application state in a named volume at `/opt/lightsail/app`. The volume starts with `releases/<timestamp>` and `current`; the in-container supervisor runs `npm start` from `current`, captures stdout and stderr under `logs/`, restarts crashes with capped exponential backoff, and retains the three newest releases. Application deployment remains SSH/manual in this phase: no deployment API or console workflow is provided.
 
 Read-only list operations for cloud-side Lightsail resource families that are not locally implemented yet return empty AWS-shaped lists. Cloud-only provisioning workflows such as container services, distributions, managed databases, load balancers, buckets, and snapshots are recognized from the AWS Lightsail API model and return an explicit `UnsupportedOperation` error instead of a custom endpoint or non-AWS response shape.
 

mkdocs.yml

@@ -142,6 +142,7 @@ nav:
     - RDS Data API: services/rds-data.md
     - Timestream for InfluxDB: services/timestream-influxdb.md
     - MSK (Kafka): services/msk.md
+    - MSK Connect: services/kafka-connect.md
     - Amazon MQ: services/amazonmq.md
     - Glue: services/glue.md
     - Lake Formation: services/lakeformation.md

src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java

@@ -302,6 +302,7 @@ public interface EmulatorConfig {
 
     interface ServiceStorageOverrides {
         SsmStorageConfig ssm();
+        KafkaConnectStorageConfig kafkaconnect();
         SqsStorageConfig sqs();
         S3StorageConfig s3();
         DynamoDbStorageConfig dynamodb();
@@ -365,6 +366,13 @@ public interface EmulatorConfig {
         long flushIntervalMs();
     }
 
+    interface KafkaConnectStorageConfig {
+        Optional<String> mode();
+
+        @WithDefault("5000")
+        long flushIntervalMs();
+    }
+
     interface SqsStorageConfig {
         Optional<String> mode();
     }
@@ -673,6 +681,7 @@ public interface EmulatorConfig {
         ApiGatewayServiceConfig apigateway();
         IamServiceConfig iam();
         MskServiceConfig msk();
+        KafkaConnectServiceConfig kafkaconnect();
         AmazonMqServiceConfig amazonmq();
         KinesisAnalyticsServiceConfig kinesisAnalytics();
         ElastiCacheServiceConfig elasticache();
@@ -1345,6 +1354,19 @@ public interface EmulatorConfig {
         int kafkaHostPortMax();
     }
 
+    interface KafkaConnectServiceConfig {
+        @WithDefault("true")
+        boolean enabled();
+
+        /** Metadata-only mode is useful where Docker is intentionally unavailable. */
+        @WithDefault("false")
+        boolean mock();
+
+        /** Stock Apache Kafka distribution, never a connector-plugin image. */
+        @WithDefault("apache/kafka:3.9.0")
+        String workerImage();
+    }
+
     interface AmazonMqServiceConfig {
         @WithDefault("true")
         boolean enabled();

src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java

@@ -146,6 +146,12 @@ public class ResolvedServiceCatalog {
                         "msk", config.storage().mode(), 5000L, null, ServiceProtocol.REST_JSON,
                         protocols(ServiceProtocol.REST_JSON),
                         Set.of(), Set.of("kafka"), Set.of(), Set.of(io.github.hectorvent.floci.services.msk.MskController.class)),
+                descriptor("kafkaconnect", "kafkaconnect", config.services().kafkaconnect().enabled(), true,
+                        "kafkaconnect", storageMode(config.storage().services().kafkaconnect().mode(), config.storage().mode()),
+                        config.storage().services().kafkaconnect().flushIntervalMs(), null, ServiceProtocol.REST_JSON,
+                        protocols(ServiceProtocol.REST_JSON),
+                        Set.of(), Set.of("kafkaconnect"), Set.of(),
+                        Set.of(io.github.hectorvent.floci.services.kafkaconnect.KafkaConnectController.class)),
                 descriptor("mq", "amazonmq", config.services().amazonmq().enabled(), true,
                         "amazonmq", config.storage().mode(), 5000L, null, ServiceProtocol.REST_JSON,
                         protocols(ServiceProtocol.REST_JSON),

src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java

@@ -110,6 +110,10 @@ public class ContainerLifecycleManager {
         }
     }
 
+    public String refreshImage(String image) {
+        return imageCacheService.refreshImage(image);
+    }
+
     /**
      * Creates a container without starting it. Use {@link #startCreated} to
      * start it after any pre-start setup (e.g. copying files into the
@@ -120,7 +124,7 @@ public class ContainerLifecycleManager {
      */
     public String create(ContainerSpec spec) {
         String resolvedImage = imageCacheService.ensureImageExists(spec.image());
-        return create(spec, resolvedImage, null);
+        return create(spec, resolvedImage, (String) null);
     }
 
     /**
@@ -135,6 +139,35 @@ public class ContainerLifecycleManager {
         return create(spec, resolvedImage, platform);
     }
 
+    public String create(ContainerSpec spec, String platform, ContainerCreationObserver observer) {
+        String resolvedImage = imageCacheService.ensureImageExists(spec.image(), platform,
+                new ImageCacheService.ImagePullObserver() {
+                    @Override
+                    public void started(String imageUri) {
+                        observer.imagePullStarted(imageUri);
+                    }
+
+                    @Override
+                    public void completed(String imageUri) {
+                        observer.imagePullCompleted(imageUri);
+                    }
+                });
+        observer.containerCreationStarted();
+        String containerId = create(spec, resolvedImage, platform);
+        observer.containerCreated(containerId);
+        return containerId;
+    }
+
+    public interface ContainerCreationObserver {
+        default void imagePullStarted(String imageUri) {}
+
+        default void imagePullCompleted(String imageUri) {}
+
+        default void containerCreationStarted() {}
+
+        default void containerCreated(String containerId) {}
+    }
+
     private String create(ContainerSpec spec, String resolvedImage, String platform) {
         String containerId = createWithCaBundle(spec, resolvedImage, platform);
         if (spec.hasNetworkConfiguration()) {

src/main/java/io/github/hectorvent/floci/core/common/docker/CurrentContainerNetworkResolver.java

@@ -5,6 +5,7 @@ import com.github.dockerjava.api.command.InspectContainerResponse;
 import com.github.dockerjava.api.model.ContainerNetwork;
 import com.github.dockerjava.api.model.ExposedPort;
 import com.github.dockerjava.api.model.Ports;
+import io.github.hectorvent.floci.config.EmulatorConfig;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 import org.jboss.logging.Logger;
@@ -32,14 +33,21 @@ public class CurrentContainerNetworkResolver {
 
     private final DockerClient dockerClient;
     private final ContainerDetector containerDetector;
+    private final EmulatorConfig config;
     private final Map<Integer, Integer> cachedPublishedPorts = new ConcurrentHashMap<>();
 
     private volatile Optional<CurrentContainerNetwork> cachedNetwork;
 
     @Inject
-    public CurrentContainerNetworkResolver(DockerClient dockerClient, ContainerDetector containerDetector) {
+    public CurrentContainerNetworkResolver(DockerClient dockerClient, ContainerDetector containerDetector,
+                                           EmulatorConfig config) {
         this.dockerClient = dockerClient;
         this.containerDetector = containerDetector;
+        this.config = config;
+    }
+
+    public CurrentContainerNetworkResolver(DockerClient dockerClient, ContainerDetector containerDetector) {
+        this(dockerClient, containerDetector, null);
     }
 
     public Optional<String> resolveNetworkName() {
@@ -148,11 +156,21 @@ public class CurrentContainerNetworkResolver {
         }
     }
 
+    /**
+     * Spawned containers join services.docker-network when it is set, so Floci's address on that
+     * network is the one they can reach. Floci can be attached to other networks too (for example
+     * the emulated VPC networks), whose addresses spawned containers cannot route to.
+     */
     private Optional<CurrentContainerNetwork> selectNetwork(Map<String, ContainerNetwork> networks) {
-        return networks.entrySet().stream()
+        Optional<String> configured = config == null ? Optional.empty()
+                : config.services().dockerNetwork().filter(name -> !name.isBlank());
+        return configured
+                .filter(name -> isUsable(networks.get(name)))
+                .map(name -> Map.entry(name, networks.get(name)))
+                .or(() -> networks.entrySet().stream()
                         .filter(entry -> isUsable(entry.getValue()))
                         .filter(entry -> isUserDefinedNetwork(entry.getKey()))
-                .findFirst()
+                        .findFirst())
                 .or(() -> networks.entrySet().stream()
                         .filter(entry -> isUsable(entry.getValue()))
                         .findFirst())

src/main/java/io/github/hectorvent/floci/core/common/docker/PortAllocator.java

@@ -13,6 +13,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.function.IntConsumer;
 
 /**
  * Utility for allocating free TCP ports for Docker container port bindings.
@@ -55,6 +56,10 @@ public class PortAllocator {
      * @throws RuntimeException if no free port is available in the range
      */
     public synchronized int allocate(int basePort, int maxPort) {
+        return allocate(basePort, maxPort, ignored -> {});
+    }
+
+    public synchronized int allocate(int basePort, int maxPort, IntConsumer occupiedPortObserver) {
         // One Docker API round-trip per allocation, not one per candidate: the
         // snapshot is only as fresh as the moment allocate() started either way,
         // and this loop runs inside a lock every consumer of the shared
@@ -66,6 +71,7 @@ public class PortAllocator {
                 LOG.debugv("Allocated port {0} from range {1}-{2}", String.valueOf(port), String.valueOf(basePort), String.valueOf(maxPort));
                 return port;
             }
+            occupiedPortObserver.accept(port);
         }
         throw new RuntimeException("No free port available in range " + basePort + "-" + maxPort);
     }

src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoController.java

@@ -6,13 +6,20 @@ import io.github.hectorvent.floci.config.FlociCertificateAuthority;
 import io.github.hectorvent.floci.core.common.ContainerTeardown;
 import io.github.hectorvent.floci.core.common.ContainerTeardowns;
 import io.github.hectorvent.floci.core.common.ServiceRegistry;
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.services.lightsail.LightsailService;
+import io.github.hectorvent.floci.services.lightsail.LightsailContainerManager;
 import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook;
 import jakarta.inject.Inject;
 import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Consumes;
 import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
 import jakarta.ws.rs.Produces;
 import jakarta.ws.rs.core.MediaType;
 import jakarta.ws.rs.core.Response;
+import org.jboss.resteasy.reactive.RestForm;
+import org.jboss.resteasy.reactive.multipart.FileUpload;
 import org.jboss.logging.Logger;
 
 import io.github.hectorvent.floci.core.storage.StorageFactory;
@@ -44,6 +51,8 @@ public class EmulatorInfoController {
     private final Instance<ContainerTeardown> containerTeardowns;
     private final FlociCertificateAuthority certificateAuthority;
     private final EmulatorConfig config;
+    private final Ec2Service ec2Service;
+    private final LightsailService lightsailService;
 
     @Inject
     public EmulatorInfoController(ServiceRegistry serviceRegistry,
@@ -52,7 +61,8 @@ public class EmulatorInfoController {
                                   Instance<Resettable> resettables,
                                    Instance<ContainerTeardown> containerTeardowns,
                                    FlociCertificateAuthority certificateAuthority,
-                                  EmulatorConfig config) {
+                                   EmulatorConfig config,
+                                    Ec2Service ec2Service, LightsailService lightsailService) {
         this.serviceRegistry = serviceRegistry;
         this.initLifecycleState = initLifecycleState;
         this.storageFactory = storageFactory;
@@ -60,6 +70,8 @@ public class EmulatorInfoController {
         this.containerTeardowns = containerTeardowns;
         this.certificateAuthority = certificateAuthority;
         this.config = config;
+        this.ec2Service = ec2Service;
+        this.lightsailService = lightsailService;
         this.version = resolveVersion();
     }
 
@@ -104,7 +116,41 @@ public class EmulatorInfoController {
     @GET
     @Path("/diagnose")
     public Response diagnose() {
-        return Response.ok(Map.of()).build();
+        return Response.ok(Map.of("ec2", Map.of("instances", ec2Service.diagnosticInstances()))).build();
+    }
+
+    @GET
+    @Path("/diagnostics/lightsail/instances/{instanceName}/host-ports")
+    public Response lightsailHostPorts(@PathParam("instanceName") String instanceName) {
+        return Response.ok(Map.of("mappings", lightsailService.hostPortMappings(config.defaultRegion(), instanceName))).build();
+    }
+
+    @POST
+    @Path("/diagnostics/lightsail/instances/{instanceName}/static-files")
+    @Consumes(MediaType.MULTIPART_FORM_DATA)
+    public Response deployLightsailStaticFiles(@PathParam("instanceName") String instanceName,
+                                                @RestForm("files") List<FileUpload> files) {
+        List<LightsailContainerManager.StaticFile> staticFiles = files == null ? List.of() : files.stream()
+                .map(file -> new LightsailContainerManager.StaticFile(file.fileName(), file.uploadedFile()))
+                .toList();
+        int deployed = lightsailService.deployStaticFiles(config.defaultRegion(), instanceName, staticFiles);
+        return Response.ok(Map.of("status", "deployed", "files", deployed)).build();
+    }
+
+    @POST
+    @Path("/diagnostics/lightsail/instances/{instanceName}/node-application")
+    @Consumes(MediaType.MULTIPART_FORM_DATA)
+    public Response deployLightsailNodeApplication(@PathParam("instanceName") String instanceName,
+                                                    @RestForm("files") List<FileUpload> files,
+                                                    @RestForm("startCommand") String startCommand,
+                                                    @RestForm("port") Integer port) {
+        List<LightsailContainerManager.StaticFile> applicationFiles = files == null ? List.of() : files.stream()
+                .map(file -> new LightsailContainerManager.StaticFile(file.fileName(), file.uploadedFile()))
+                .toList();
+        LightsailContainerManager.NodeDeployment deployment = lightsailService.deployNodeApplication(
+                config.defaultRegion(), instanceName, applicationFiles, startCommand, port == null ? 3000 : port);
+        return Response.ok(Map.of("status", "deployed", "files", deployment.files(), "port", deployment.port(),
+                "releaseId", deployment.releaseId())).build();
     }
 
     @GET

src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LambdaCfnProvisioner.java

@@ -111,7 +111,7 @@ public class LambdaCfnProvisioner implements CfnResourceProvisioner {
                         "Cannot replace Lambda function " + r.getPhysicalId()
                                 + " without a new FunctionName", 400);
             }
-            func = createLambdaFunction(region, desired, !replacement);
+            func = createLambdaFunction(region, desired, !replacement, r);
             if (replacement && r.getPhysicalId() != null) {
                 deleteReplacedLambda(region, r.getPhysicalId());
             }
@@ -343,13 +343,15 @@ public class LambdaCfnProvisioner implements CfnResourceProvisioner {
         return !Objects.equals(existingPackageType, desired.packageType());
     }
 
-    private LambdaFunction createLambdaFunction(String region, LambdaDesiredState desired, boolean allowAdopt) {
+    private LambdaFunction createLambdaFunction(String region, LambdaDesiredState desired, boolean allowAdopt,
+                                                StackResource resource) {
         try {
             return lambdaService.createFunction(region, desired.createRequest());
         } catch (AwsException e) {
             if (allowAdopt && ("ResourceConflictException".equals(e.getErrorCode())
                     || (e.getMessage() != null && e.getMessage().contains("Function already exist")))) {
-                return lambdaService.getFunction(region, desired.functionName());
+                LambdaFunction existing = lambdaService.getFunction(region, desired.functionName());
+                return updateLambdaFunction(region, existing, desired, resource);
             }
             throw e;
         }

src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LambdaVersionAliasCfnProvisioner.java

@@ -85,16 +85,9 @@ public class LambdaVersionAliasCfnProvisioner implements CfnResourceProvisioner
         }
         String description = ctx.resolveOptional(props, "Description");
 
-        // Version resources are immutable in CloudFormation. An update re-invokes provision with
-        // the prior physical id, and publishing again would mint a new version and repoint every
-        // Ref to it, so the version this resource already published is kept while it still exists.
-        String previous = r.getPhysicalId();
-        if (previous != null && existingVersionIsCurrent(ctx.region(), functionName, previous)) {
-            r.getAttributes().put("Version", previous.substring(previous.lastIndexOf(':') + 1));
-            r.getAttributes().put("FunctionArn", previous);
-            return;
-        }
-
+        // Lambda deduplicates an unchanged publish by $LATEST revision. Calling it here is
+        // essential on UpdateStack: after a dependent function updates its code, the Version
+        // must snapshot that new revision rather than retaining its prior physical id.
         LambdaFunction version = lambdaService.publishVersion(ctx.region(), functionName, description);
 
         // Matches real CloudFormation: Ref returns the qualified ARN, GetAtt Version the version
@@ -104,26 +97,6 @@ public class LambdaVersionAliasCfnProvisioner implements CfnResourceProvisioner
         r.getAttributes().put("FunctionArn", version.getFunctionArn());
     }
 
-    /** True when {@code physicalId} names a still-published version of the function in the template. */
-    private boolean existingVersionIsCurrent(String region, String functionName, String physicalId) {
-        int sep = physicalId.lastIndexOf(':');
-        if (sep <= 0) {
-            return false;
-        }
-        String version = physicalId.substring(sep + 1);
-        String baseArn = physicalId.substring(0, sep);
-        try {
-            LambdaFunction fn = lambdaService.getFunction(region, functionName);
-            if (!baseArn.equals(fn.getFunctionArn().replace(":$LATEST", ""))) {
-                return false; // the FunctionName property now points at a different function
-            }
-            return lambdaService.listVersionsByFunction(region, functionName).stream()
-                    .anyMatch(v -> version.equals(v.getVersion()));
-        } catch (AwsException e) {
-            return false;
-        }
-    }
-
     private void provisionAlias(StackResource r, JsonNode props, ProvisionContext ctx) {
         String functionName = ctx.resolveOptional(props, "FunctionName");
         String aliasName = ctx.resolveOptional(props, "Name");

src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java

@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
 import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
 import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
 import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.core.common.docker.DockerClientProducer;
 import io.github.hectorvent.floci.services.codebuild.BuildspecParser.ParsedArtifacts;
 import io.github.hectorvent.floci.services.codebuild.BuildspecParser.ParsedBuildspec;
 import io.github.hectorvent.floci.services.codebuild.model.Build;
@@ -38,6 +39,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -195,9 +197,10 @@ public class CodeBuildRunner implements ContainerTeardown {
             String logGroup = "/aws/codebuild/" + project.getName();
             String logStream = logStreamer.generateLogStreamName(buildId.replace(":", "/"));
 
-            String image = build.getEnvironment() != null && build.getEnvironment().getImage() != null
+            String requestedImage = build.getEnvironment() != null && build.getEnvironment().getImage() != null
                     ? build.getEnvironment().getImage()
                     : project.getEnvironment().getImage();
+            String image = resolveBuildImage(requestedImage);
 
             boolean privileged = (project.getEnvironment() != null
                     && Boolean.TRUE.equals(project.getEnvironment().getPrivilegedMode()))
@@ -214,8 +217,11 @@ public class CodeBuildRunner implements ContainerTeardown {
 
             // Keep the container alive so each phase can be run with docker exec.
             // No bind mount needed — source and artifacts are transferred with docker cp.
-            ContainerSpec spec = containerBuilder.newContainer(image)
-                    .withCmd(List.of("sh", "-c", "tail -f /dev/null"))
+            String startupCommand = "aws/codebuild/standard:7.0".equals(requestedImage)
+                    ? "apt-get update && apt-get install -y --no-install-recommends zip && tail -f /dev/null"
+                    : "tail -f /dev/null";
+            ContainerBuilder.Builder container = containerBuilder.newContainer(image)
+                    .withCmd(List.of("sh", "-c", startupCommand))
                     .withEnv(envList)
                     .withDockerNetwork(config.services().codebuild().dockerNetwork())
                     .withEmbeddedDns()
@@ -223,8 +229,9 @@ public class CodeBuildRunner implements ContainerTeardown {
                     .withPrivileged(privileged)
                     .withLogRotation()
                     .withLabels(ContainerStorageHelper.resourceIdentityLabels(
-                            "codebuild", buildId, regionResolver.getAccountId(), region))
-                    .build();
+                            "codebuild", buildId, regionResolver.getAccountId(), region));
+            exposeDockerSocketIfPrivileged(container, privileged, config.docker());
+            ContainerSpec spec = container.build();
 
             ContainerLifecycleManager.ContainerInfo info = lifecycleManager.createAndStart(spec);
             containerId = info.containerId();
@@ -261,7 +268,8 @@ public class CodeBuildRunner implements ContainerTeardown {
             if (installResult.stopped()) { finishStopped(build); return; }
             if (installResult.failed()) {
                 completePhaseWithError(build, "INSTALL", "FAILED", installResult.errorMessage());
-                buildFailed = true;
+                finishFailed(build);
+                return;
             } else {
                 completePhase(build, "INSTALL", "SUCCEEDED");
             }
@@ -276,7 +284,8 @@ public class CodeBuildRunner implements ContainerTeardown {
                 if (preBuildResult.stopped()) { finishStopped(build); return; }
                 if (preBuildResult.failed()) {
                     completePhaseWithError(build, "PRE_BUILD", "FAILED", preBuildResult.errorMessage());
-                    buildFailed = true;
+                    finishFailed(build);
+                    return;
                 } else {
                     completePhase(build, "PRE_BUILD", "SUCCEEDED");
                 }
@@ -294,7 +303,8 @@ public class CodeBuildRunner implements ContainerTeardown {
                 if (buildResult.stopped()) { finishStopped(build); return; }
                 if (buildResult.failed()) {
                     completePhaseWithError(build, "BUILD", "FAILED", buildResult.errorMessage());
-                    buildFailed = true;
+                    finishFailed(build);
+                    return;
                 } else {
                     completePhase(build, "BUILD", "SUCCEEDED");
                 }
@@ -311,7 +321,8 @@ public class CodeBuildRunner implements ContainerTeardown {
             if (postBuildResult.stopped()) { finishStopped(build); return; }
             if (postBuildResult.failed()) {
                 completePhaseWithError(build, "POST_BUILD", "FAILED", postBuildResult.errorMessage());
-                buildFailed = true;
+                finishFailed(build);
+                return;
             } else {
                 completePhase(build, "POST_BUILD", "SUCCEEDED");
             }
@@ -384,6 +395,27 @@ public class CodeBuildRunner implements ContainerTeardown {
         }
     }
 
+    private String resolveBuildImage(String image) {
+        if ("aws/codebuild/standard:7.0".equals(image)) {
+            return "golang:1.23-bookworm";
+        }
+        return image;
+    }
+
+    static void exposeDockerSocketIfPrivileged(ContainerBuilder.Builder container, boolean privileged,
+                                               EmulatorConfig.DockerConfig dockerConfig) {
+        if (!privileged) {
+            return;
+        }
+        String dockerHost = DockerClientProducer.resolveDockerConnection(dockerConfig, System::getenv).host();
+        URI dockerUri = URI.create(dockerHost);
+        if (!"unix".equals(dockerUri.getScheme()) || dockerUri.getPath() == null || dockerUri.getPath().isBlank()) {
+            LOG.debugv("CodeBuild privileged mode cannot mount non-Unix Docker endpoint {0}", dockerHost);
+            return;
+        }
+        container.withBind(dockerUri.getPath(), "/var/run/docker.sock");
+    }
+
     private String resolveAndAcquireSource(String region, Build build, Project project,
                                            String buildspecOverride, Path workspace) throws IOException {
         String sourceType = project.getSource() != null ? project.getSource().getType() : "NO_SOURCE";
@@ -660,25 +692,20 @@ public class CodeBuildRunner implements ContainerTeardown {
     private void uploadArtifacts(String region, Build build, Project project,
                                  ParsedArtifacts artifacts, Path workspace) throws IOException {
         String type = artifacts.type();
-        if ("NO_ARTIFACTS".equals(type) && project.getArtifacts() != null) {
-            type = project.getArtifacts().getType();
+        if ("NO_ARTIFACTS".equals(type) && build.getArtifacts() != null) {
+            type = build.getArtifacts().getType();
         }
-        if (type == null || "NO_ARTIFACTS".equals(type) || "CODEPIPELINE".equals(type)) {
+        if (type == null || "NO_ARTIFACTS".equals(type)) {
             return;
         }
-        if (!"S3".equals(type)) {
+        if (!"S3".equals(type) && !"CODEPIPELINE".equals(type)) {
             return;
         }
 
         String packaging = artifacts.packaging();
-        if ("ZIP".equals(packaging) && project.getArtifacts() != null
-                && project.getArtifacts().getPackaging() != null) {
-            packaging = project.getArtifacts().getPackaging();
-        }
-
-        String location = project.getArtifacts() != null ? project.getArtifacts().getLocation() : null;
-        if (location == null || location.isBlank()) {
-            return;
+        if ("ZIP".equals(packaging) && build.getArtifacts() != null
+                && build.getArtifacts().getPackaging() != null) {
+            packaging = build.getArtifacts().getPackaging();
         }
 
         List<String> filePatterns = artifacts.files();
@@ -698,6 +725,16 @@ public class CodeBuildRunner implements ContainerTeardown {
             return;
         }
 
+        if ("CODEPIPELINE".equals(type)) {
+            build.setOutputArtifact(zipFiles(baseDir, matchedFiles, artifacts.discardPaths()));
+            return;
+        }
+
+        String location = build.getArtifacts() != null ? build.getArtifacts().getLocation() : null;
+        if (location == null || location.isBlank()) {
+            return;
+        }
+
         int slash = location.indexOf('/');
         String bucket = slash > 0 ? location.substring(0, slash) : location;
         String prefix = slash > 0 ? location.substring(slash + 1) : "";
@@ -841,6 +878,13 @@ public class CodeBuildRunner implements ContainerTeardown {
         build.setBuildComplete(true);
         build.setBuildStatus("FAILED");
         build.setCurrentPhase("COMPLETED");
+        BuildPhase completed = new BuildPhase();
+        completed.setPhaseType("COMPLETED");
+        completed.setPhaseStatus("FAILED");
+        completed.setStartTime(System.currentTimeMillis() / 1000.0);
+        completed.setEndTime(System.currentTimeMillis() / 1000.0);
+        completed.setDurationInSeconds(0L);
+        build.getPhases().add(completed);
     }
 
     private Optional<BuildPhase> findPhase(Build build, String phaseType) {

src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java

@@ -489,6 +489,14 @@ public class CodeBuildService {
         return build;
     }
 
+    /**
+     * Returns the CODEPIPELINE output produced by one completed build. This is runtime state,
+     * like the build itself, and must not be inferred from a project-wide artifact location.
+     */
+    public byte[] getBuildOutputArtifact(String region, String account, String buildId) {
+        return getBuild(region, account, buildId).getOutputArtifact();
+    }
+
     public List<Build> batchGetBuilds(String region, String account, List<String> buildIds) {
         Map<String, Build> store = buildsFor(account, region);
         return buildIds.stream()

src/main/java/io/github/hectorvent/floci/services/codebuild/model/Build.java

@@ -1,5 +1,6 @@
 package io.github.hectorvent.floci.services.codebuild.model;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 
@@ -29,6 +30,7 @@ public class Build {
     private Integer timeoutInMinutes;
     private Integer queuedTimeoutInMinutes;
     private String encryptionKey;
+    private byte[] outputArtifact;
 
     public String getId() { return id; }
     public void setId(String id) { this.id = id; }
@@ -83,4 +85,11 @@ public class Build {
 
     public String getEncryptionKey() { return encryptionKey; }
     public void setEncryptionKey(String encryptionKey) { this.encryptionKey = encryptionKey; }
+
+    @JsonIgnore
+    public byte[] getOutputArtifact() { return outputArtifact != null ? outputArtifact.clone() : null; }
+
+    public void setOutputArtifact(byte[] outputArtifact) {
+        this.outputArtifact = outputArtifact != null ? outputArtifact.clone() : null;
+    }
 }

src/main/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployService.java

@@ -17,10 +17,19 @@ import io.github.hectorvent.floci.services.codedeploy.model.OnPremisesInstance;
 import io.github.hectorvent.floci.services.ec2.Ec2Service;
 import io.github.hectorvent.floci.services.ecs.EcsService;
 import io.github.hectorvent.floci.services.ecs.model.CreateTaskSetRequest;
+import io.github.hectorvent.floci.services.ecs.model.EcsLoadBalancer;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
 import io.github.hectorvent.floci.services.ecs.model.TaskSet;
 import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
+import io.github.hectorvent.floci.services.elbv2.model.Action;
+import io.github.hectorvent.floci.services.elbv2.model.Rule;
 import io.github.hectorvent.floci.services.elbv2.model.TargetGroup;
+import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;
 import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.model.LambdaAlias;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import io.github.hectorvent.floci.services.s3.S3Service;
 import io.github.hectorvent.floci.services.ssm.SsmCommandService;
 import io.github.hectorvent.floci.services.lambda.model.InvocationType;
 import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
@@ -29,6 +38,8 @@ import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 import org.jboss.logging.Logger;
 
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
@@ -36,6 +47,8 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -53,8 +66,16 @@ public class CodeDeployService {
     // Lambda/ECS validation hooks must call PutLifecycleEventHookExecutionStatus within one hour.
     private static final Duration DEFAULT_HOOK_CALLBACK_TIMEOUT = Duration.ofHours(1);
     private static final Duration STOP_POLL_INTERVAL = Duration.ofMillis(200);
+    private static final int ECS_READINESS_MAX_POLLS = 150;
+    private static final int ECS_TARGET_HEALTH_MAX_POLLS = 900;
+
+    @FunctionalInterface
+    interface Sleeper {
+        void sleep(long millis) throws InterruptedException;
+    }
 
     private final LambdaService lambdaService;
+    private final S3Service s3Service;
     private final EcsService ecsService;
     private final ElbV2Service elbV2Service;
     private final SsmCommandService ssmCommandService;
@@ -65,21 +86,48 @@ public class CodeDeployService {
     private final StorageFactory storageFactory;
     private final Duration hookCallbackTimeout;
     private final Clock clock;
+    private final Sleeper sleeper;
 
     @Inject
     public CodeDeployService(LambdaService lambdaService, EcsService ecsService,
+                             ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
+                             Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
+                              StorageFactory storageFactory, S3Service s3Service) {
+        this(lambdaService, ecsService, elbV2Service, ssmCommandService, ec2Service, mapper,
+                regionResolver, storageFactory, DEFAULT_HOOK_CALLBACK_TIMEOUT, Clock.systemUTC(), s3Service, Thread::sleep);
+    }
+
+    CodeDeployService(LambdaService lambdaService, EcsService ecsService,
                       ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
                       Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
                       StorageFactory storageFactory) {
         this(lambdaService, ecsService, elbV2Service, ssmCommandService, ec2Service, mapper,
-                regionResolver, storageFactory, DEFAULT_HOOK_CALLBACK_TIMEOUT, Clock.systemUTC());
+                regionResolver, storageFactory, DEFAULT_HOOK_CALLBACK_TIMEOUT, Clock.systemUTC(), null, Thread::sleep);
     }
 
     CodeDeployService(LambdaService lambdaService, EcsService ecsService,
                       ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
                        Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
                        StorageFactory storageFactory, Duration hookCallbackTimeout, Clock clock) {
+        this(lambdaService, ecsService, elbV2Service, ssmCommandService, ec2Service, mapper,
+                regionResolver, storageFactory, hookCallbackTimeout, clock, null, Thread::sleep);
+    }
+
+    CodeDeployService(LambdaService lambdaService, EcsService ecsService,
+                      ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
+                      Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
+                      StorageFactory storageFactory, Duration hookCallbackTimeout, Clock clock, S3Service s3Service) {
+        this(lambdaService, ecsService, elbV2Service, ssmCommandService, ec2Service, mapper,
+                regionResolver, storageFactory, hookCallbackTimeout, clock, s3Service, Thread::sleep);
+    }
+
+    CodeDeployService(LambdaService lambdaService, EcsService ecsService,
+                      ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
+                      Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
+                      StorageFactory storageFactory, Duration hookCallbackTimeout, Clock clock, S3Service s3Service,
+                      Sleeper sleeper) {
         this.lambdaService = lambdaService;
+        this.s3Service = s3Service;
         this.ecsService = ecsService;
         this.elbV2Service = elbV2Service;
         this.ssmCommandService = ssmCommandService;
@@ -90,6 +138,7 @@ public class CodeDeployService {
         this.storageFactory = storageFactory;
         this.hookCallbackTimeout = hookCallbackTimeout;
         this.clock = clock;
+        this.sleeper = sleeper;
     }
 
     // ---- Durable (persisted) ----
@@ -109,8 +158,8 @@ public class CodeDeployService {
     private final ConcurrentHashMap<String, ConcurrentHashMap<String, Deployment>> deployments = new ConcurrentHashMap<>();
     // key: region -> deploymentId -> targetId -> DeploymentTarget wrapper
     private final ConcurrentHashMap<String, ConcurrentHashMap<String, ConcurrentHashMap<String, Map<String, Object>>>> deploymentTargets = new ConcurrentHashMap<>();
-    // key: lifecycleEventHookExecutionId -> CompletableFuture<status>
-    private final ConcurrentHashMap<String, CompletableFuture<String>> hookFutures = new ConcurrentHashMap<>();
+    // key: lifecycleEventHookExecutionId -> the deployment event that owns the callback
+    private final ConcurrentHashMap<String, HookExecution> hookExecutions = new ConcurrentHashMap<>();
     // key: deploymentId -> stop flag
     private final ConcurrentHashMap<String, AtomicBoolean> stopFlags = new ConcurrentHashMap<>();
 
@@ -203,6 +252,28 @@ public class CodeDeployService {
         String afterAllowTraffic;
     }
 
+    private static final class HookExecution {
+        final String deploymentId;
+        final String lifecycleEventName;
+        final CompletableFuture<String> future;
+
+        HookExecution(String deploymentId, String lifecycleEventName, CompletableFuture<String> future) {
+            this.deploymentId = deploymentId;
+            this.lifecycleEventName = lifecycleEventName;
+            this.future = future;
+        }
+    }
+
+    private static final class AliasRoutingSnapshot {
+        final String functionVersion;
+        final Map<String, Double> routingConfig;
+
+        AliasRoutingSnapshot(LambdaAlias alias) {
+            this.functionVersion = alias.getFunctionVersion();
+            this.routingConfig = alias.getRoutingConfig() == null ? Map.of() : Map.copyOf(alias.getRoutingConfig());
+        }
+    }
+
     private static final class ServerAppSpecInfo {
         String os;
         Map<String, List<Map<String, Object>>> hooks; // hookName → [{location, timeout, runas}]
@@ -578,8 +649,10 @@ public class CodeDeployService {
             return createServerDeployment(region, appName, groupName, group, configName, revision, description);
         }
 
-        AppSpecInfo appSpec = parseAppSpec(revision);
         String effectiveConfig = configName != null ? configName : group.getDeploymentConfigName();
+        validateLambdaDeployment(region, app, group, effectiveConfig);
+        AppSpecInfo appSpec = parseAppSpec(revision);
+        validateLambdaTarget(region, appSpec);
 
         String deploymentId = generateDeploymentId();
         double now = Instant.now().toEpochMilli() / 1000.0;
@@ -695,10 +768,16 @@ public class CodeDeployService {
     }
 
     public String putLifecycleEventHookExecutionStatus(String deploymentId, String executionId, String status) {
-        CompletableFuture<String> future = hookFutures.get(executionId);
-        if (future != null && !future.isDone()) {
-            future.complete(status);
+        if (!"Succeeded".equals(status) && !"Failed".equals(status)) {
+            throw new AwsException("InvalidLifecycleEventHookExecutionStatusException",
+                    "Lifecycle event hook execution status must be Succeeded or Failed", 400);
         }
+        HookExecution execution = hookExecutions.get(executionId);
+        if (execution == null || !execution.deploymentId.equals(deploymentId) || execution.future.isDone()) {
+            throw new AwsException("InvalidLifecycleEventHookExecutionIdException",
+                    "Lifecycle event hook execution ID is not valid for this deployment", 400);
+        }
+        execution.future.complete(status);
         return executionId;
     }
 
@@ -1063,12 +1142,17 @@ public class CodeDeployService {
             for (Map<String, String> filter : ec2TagFilters) {
                 String key = filter.get("Key");
                 String value = filter.get("Value");
-                if (key != null && value != null) {
+                String type = filter.get("Type");
+                if ("KEY_ONLY".equals(type) && key != null) {
+                    filters.computeIfAbsent("tag-key", k -> new ArrayList<>()).add(key);
+                } else if ("VALUE_ONLY".equals(type) && value != null) {
+                    filters.computeIfAbsent("tag-value", k -> new ArrayList<>()).add(value);
+                } else if (key != null && value != null) {
                     filters.computeIfAbsent("tag:" + key, k -> new ArrayList<>()).add(value);
                 }
             }
             try {
-                ec2Service.describeInstances(region, null, filters).stream()
+                ec2Service.describeInstances(region, List.of(), filters).stream()
                         .flatMap(r -> r.getInstances().stream())
                         .map(inst -> inst.getInstanceId())
                         .filter(id -> id != null)
@@ -1118,38 +1202,53 @@ public class CodeDeployService {
         if (revision == null) {
             throw new AwsException("InvalidRevisionException", "Revision is required", 400);
         }
-        Object appSpecContent = revision.get("appSpecContent");
-        if (!(appSpecContent instanceof Map)) {
-            throw new AwsException("InvalidRevisionException", "Missing appSpecContent in revision", 400);
-        }
-        String content = (String) ((Map<String, Object>) appSpecContent).get("content");
+        String content = appSpecContent(revision);
         if (content == null || content.isBlank()) {
             throw new AwsException("InvalidRevisionException", "Missing content in appSpecContent", 400);
         }
 
         AppSpecInfo info = new AppSpecInfo();
         try {
-            JsonNode root = mapper.readTree(content);
-            JsonNode resources = root.get("Resources");
-            if (resources != null && resources.isArray() && !resources.isEmpty()) {
-                JsonNode firstResource = resources.get(0);
-                if (firstResource.isObject()) {
-                    JsonNode resourceNode = firstResource.fields().next().getValue();
-                    JsonNode props = resourceNode.path("Properties");
-                    info.functionName = props.path("Name").asText(null);
-                    info.aliasName = props.path("Alias").asText(null);
-                    info.currentVersion = props.path("CurrentVersion").asText(null);
-                    info.targetVersion = props.path("TargetVersion").asText(null);
+            JsonNode root = yamlMapper.readTree(content);
+            if (!root.isObject() || !"0.0".equals(root.path("version").asText())) {
+                throw new IllegalArgumentException("AppSpec version must be 0.0");
             }
+            JsonNode resources = root.path("Resources");
+            if (!resources.isArray() || resources.size() != 1 || !resources.get(0).isObject()
+                    || resources.get(0).size() != 1) {
+                throw new IllegalArgumentException("Lambda AppSpec must contain exactly one resource");
+            }
+            JsonNode resourceNode = resources.get(0).elements().next();
+            if (!"AWS::Lambda::Function".equals(resourceNode.path("Type").asText())) {
+                throw new IllegalArgumentException("Lambda AppSpec resource Type must be AWS::Lambda::Function");
+            }
+            JsonNode props = resourceNode.path("Properties");
+            info.functionName = requiredAppSpecText(props, "Name");
+            info.aliasName = requiredAppSpecText(props, "Alias");
+            info.currentVersion = requiredAppSpecText(props, "CurrentVersion");
+            info.targetVersion = requiredAppSpecText(props, "TargetVersion");
+            if (info.currentVersion.equals(info.targetVersion)) {
+                throw new IllegalArgumentException("CurrentVersion and TargetVersion must differ");
             }
             JsonNode hooks = root.get("Hooks");
-            if (hooks != null && hooks.isArray()) {
-                for (JsonNode hook : hooks) {
-                    if (hook.has("BeforeAllowTraffic")) {
-                        info.beforeAllowTraffic = hook.get("BeforeAllowTraffic").asText(null);
+            if (hooks != null) {
+                if (!hooks.isArray()) {
+                    throw new IllegalArgumentException("Hooks must be an array");
                 }
-                    if (hook.has("AfterAllowTraffic")) {
-                        info.afterAllowTraffic = hook.get("AfterAllowTraffic").asText(null);
+                for (JsonNode hook : hooks) {
+                    if (!hook.isObject() || hook.size() != 1) {
+                        throw new IllegalArgumentException("Each Hook must contain exactly one lifecycle event");
+                    }
+                    Map.Entry<String, JsonNode> entry = hook.fields().next();
+                    String hookFunction = requiredAppSpecText(hook, entry.getKey());
+                    if ("BeforeAllowTraffic".equals(entry.getKey())) {
+                        if (info.beforeAllowTraffic != null) { throw new IllegalArgumentException("Duplicate BeforeAllowTraffic hook"); }
+                        info.beforeAllowTraffic = hookFunction;
+                    } else if ("AfterAllowTraffic".equals(entry.getKey())) {
+                        if (info.afterAllowTraffic != null) { throw new IllegalArgumentException("Duplicate AfterAllowTraffic hook"); }
+                        info.afterAllowTraffic = hookFunction;
+                    } else {
+                        throw new IllegalArgumentException("Unsupported Lambda lifecycle event: " + entry.getKey());
                     }
                 }
             }
@@ -1157,13 +1256,73 @@ public class CodeDeployService {
             throw new AwsException("InvalidRevisionException", "Failed to parse AppSpec content: " + e.getMessage(), 400);
         }
 
-        if (info.functionName == null || info.aliasName == null || info.targetVersion == null) {
-            throw new AwsException("InvalidRevisionException",
-                    "AppSpec must specify Name, Alias, and TargetVersion", 400);
-        }
         return info;
     }
 
+    @SuppressWarnings("unchecked")
+    private String appSpecContent(Map<String, Object> revision) {
+        Object appSpecContent = revision.get("appSpecContent");
+        if (appSpecContent instanceof Map<?, ?> content) {
+            Object value = ((Map<String, Object>) content).get("content");
+            if (value instanceof String text && !text.isBlank()) {
+                return text;
+            }
+        }
+        Object s3Location = revision.get("s3Location");
+        if (!(s3Location instanceof Map<?, ?> location) || s3Service == null) {
+            throw new AwsException("InvalidRevisionException", "Missing AppSpec content in revision", 400);
+        }
+        String bucket = String.valueOf(location.get("bucket"));
+        String key = String.valueOf(location.get("key"));
+        byte[] artifact = s3Service.getObject(bucket, key).getData();
+        InputStream input = artifact == null ? InputStream.nullInputStream() : new ByteArrayInputStream(artifact);
+        try (ZipInputStream zip = new ZipInputStream(input)) {
+            ZipEntry entry;
+            while ((entry = zip.getNextEntry()) != null) {
+                String name = entry.getName();
+                if (!entry.isDirectory() && ("appspec.yml".equalsIgnoreCase(name) || "appspec.yaml".equalsIgnoreCase(name))) {
+                    return new String(zip.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8);
+                }
+            }
+        } catch (Exception e) {
+            throw new AwsException("InvalidRevisionException", "Could not read AppSpec from S3 revision: " + e.getMessage(), 400);
+        }
+        throw new AwsException("InvalidRevisionException", "S3 revision does not contain appspec.yml or appspec.yaml", 400);
+    }
+
+    private String requiredAppSpecText(JsonNode node, String field) {
+        JsonNode value = node.get(field);
+        if (value == null || !value.isTextual() || value.asText().isBlank()) {
+            throw new IllegalArgumentException("AppSpec must specify a non-empty " + field);
+        }
+        return value.asText();
+    }
+
+    private void validateLambdaDeployment(String region, Application app, DeploymentGroup group, String configName) {
+        if (!"Lambda".equals(app.getComputePlatform())) {
+            throw new AwsException("InvalidDeploymentConfigException", "Application compute platform must be Lambda", 400);
+        }
+        if (group.getComputePlatform() != null && !"Lambda".equals(group.getComputePlatform())) {
+            throw new AwsException("InvalidDeploymentConfigException", "Deployment group compute platform must be Lambda", 400);
+        }
+        DeploymentConfig config = getDeploymentConfig(region, configName);
+        if (!"Lambda".equals(config.getComputePlatform())) {
+            throw new AwsException("InvalidDeploymentConfigException", "Deployment configuration compute platform must be Lambda", 400);
+        }
+    }
+
+    private void validateLambdaTarget(String region, AppSpecInfo appSpec) {
+        LambdaAlias alias = lambdaService.getAlias(region, appSpec.functionName, appSpec.aliasName);
+        if (!appSpec.currentVersion.equals(alias.getFunctionVersion())) {
+            throw new AwsException("InvalidRevisionException", "AppSpec CurrentVersion does not match the alias version", 400);
+        }
+        List<String> versions = lambdaService.listVersionsByFunction(region, appSpec.functionName).stream()
+                .map(LambdaFunction::getVersion).toList();
+        if (!versions.contains(appSpec.currentVersion) || !versions.contains(appSpec.targetVersion)) {
+            throw new AwsException("InvalidRevisionException", "AppSpec version does not exist", 400);
+        }
+    }
+
     @SuppressWarnings("unchecked")
     private EcsAppSpecInfo parseEcsAppSpec(Map<String, Object> revision) {
         if (revision == null) {
@@ -1180,7 +1339,13 @@ public class CodeDeployService {
 
         EcsAppSpecInfo info = new EcsAppSpecInfo();
         try {
-            JsonNode root = mapper.readTree(content);
+            JsonNode root;
+            try {
+                root = mapper.readTree(content);
+            } catch (Exception ignored) {
+                // CodePipeline's AppSpecTemplatePath conventionally points to YAML.
+                root = yamlMapper.readTree(content);
+            }
             JsonNode resources = root.get("Resources");
             if (resources != null && resources.isArray() && !resources.isEmpty()) {
                 JsonNode firstResource = resources.get(0);
@@ -1367,7 +1532,28 @@ public class CodeDeployService {
                 // its startedBy, which is how a client tells a CodeDeploy set from an external one.
                 createTaskSet.setExternalId(deploymentId);
                 createTaskSet.setStartedBy("CODE_DEPLOY");
+                EcsServiceModel targetService = ecsService.describeServices(clusterName, List.of(serviceName), region)
+                        .getFirst();
+                if (targetService.getCapacityProviderStrategy() != null
+                        && !targetService.getCapacityProviderStrategy().isEmpty()) {
+                    createTaskSet.setCapacityProviderStrategy(targetService.getCapacityProviderStrategy());
+                } else {
+                    createTaskSet.setLaunchType(targetService.getLaunchType());
+                }
+                if (greenTgArn != null) {
+                    EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+                    loadBalancer.setTargetGroupArn(greenTgArn);
+                    loadBalancer.setContainerName(appSpec.containerName);
+                    loadBalancer.setContainerPort(appSpec.containerPort);
+                    createTaskSet.setLoadBalancers(List.of(loadBalancer));
+                }
                 greenTaskSet = ecsService.createTaskSet(createTaskSet, region);
+                verifyGreenTaskSet(region, clusterName, serviceName, deploymentId, targetService, greenTaskSet,
+                        greenTgArn);
+                if (greenTaskSet.getLaunchType() == LaunchType.FARGATE) {
+                    greenTaskSet = waitForGreenTaskSetRunning(region, clusterName, serviceName, greenTaskSet,
+                            stopFlag);
+                }
                 appendTaskSetInfo(ecsTargetMap, greenTaskSet, greenTgArn, 0.0);
                 finishLifecycleEvent(installEvent, "Succeeded");
             } catch (Exception e) {
@@ -1406,11 +1592,13 @@ public class CodeDeployService {
 
             // AllowTraffic: shift ELB traffic blue → green
             if (!listenerArns.isEmpty() && blueTgArn != null && greenTgArn != null) {
+                waitForGreenTargets(region, greenTgArn, stopFlag);
+                if (stopFlag.get()) { finishEcsStopped(deployment, ecsTargetMap); return; }
                 executeEcsAllowTraffic(region, deployment, configName, ecsTargetMap,
                         listenerArns, blueTgArn, greenTgArn, stopFlag);
+                verifyListenersForwardToGreen(region, listenerArns, greenTgArn);
             } else {
-                Map<String, Object> allowEvent = addLifecycleEvent(ecsTargetMap, "AllowTraffic");
-                finishLifecycleEvent(allowEvent, "Succeeded");
+                throw new IllegalStateException("ECS deployment group must have production listeners and blue/green target groups");
             }
 
             if (stopFlag.get()) { finishEcsStopped(deployment, ecsTargetMap); return; }
@@ -1438,10 +1626,11 @@ public class CodeDeployService {
                 try {
                     ecsService.deleteTaskSet(clusterName, serviceName,
                             blueTaskSet.getTaskSetArn(), true, region);
+                    verifyBlueTaskSetDrained(region, clusterName, serviceName, blueTaskSet.getTaskSetArn());
                     finishLifecycleEvent(terminateEvent, "Succeeded");
                 } catch (Exception e) {
-                    LOG.debugv("Could not delete blue task set: {0}", e.getMessage());
-                    finishLifecycleEvent(terminateEvent, "Succeeded");
+                    finishLifecycleEvent(terminateEvent, "Failed");
+                    throw e;
                 }
             }
 
@@ -1460,7 +1649,123 @@ public class CodeDeployService {
         }
     }
 
-    private void executeEcsAllowTraffic(String region, Deployment deployment, String configName,
+    void verifyGreenTaskSet(String region, String clusterName, String serviceName, String deploymentId,
+                            EcsServiceModel targetService, TaskSet createdTaskSet, String greenTgArn) {
+        if (createdTaskSet == null || createdTaskSet.getTaskSetArn() == null) {
+            throw new IllegalStateException("ECS did not create a green task set");
+        }
+        List<TaskSet> taskSets = ecsService.describeTaskSets(clusterName, serviceName,
+                List.of(createdTaskSet.getTaskSetArn()), region);
+        if (taskSets.size() != 1) {
+            throw new IllegalStateException("ECS did not retain the green task set");
+        }
+        TaskSet greenTaskSet = taskSets.getFirst();
+        if (!deploymentId.equals(greenTaskSet.getExternalId())
+                || !"CODE_DEPLOY".equals(greenTaskSet.getStartedBy())
+                || !"ACTIVE".equals(greenTaskSet.getStatus())) {
+            throw new IllegalStateException("ECS green task set is not a CodeDeploy active task set");
+        }
+        if (greenTaskSet.getLaunchType() != LaunchType.FARGATE && greenTaskSet.getLaunchType() != LaunchType.EC2) {
+            throw new IllegalStateException("ECS green task set does not use FARGATE or EC2");
+        }
+        if (targetService.getLaunchType() != null && greenTaskSet.getLaunchType() != targetService.getLaunchType()) {
+            throw new IllegalStateException("ECS green task set launch type does not match the service");
+        }
+        if (greenTaskSet.getLaunchType() == LaunchType.EC2
+                && !hasRunningTasks(greenTaskSet)) {
+            throw new IllegalStateException("ECS green task set has no running tasks");
+        }
+        if (greenTgArn == null || greenTaskSet.getLoadBalancers() == null || greenTaskSet.getLoadBalancers().stream()
+                .noneMatch(loadBalancer -> greenTgArn.equals(loadBalancer.getTargetGroupArn()))) {
+            throw new IllegalStateException("ECS green task set is not registered with the green target group");
+        }
+    }
+
+    /** Fargate task launch is asynchronous, so the initial CreateTaskSet result is not yet stable. */
+    TaskSet waitForGreenTaskSetRunning(String region, String clusterName, String serviceName,
+                                       TaskSet createdTaskSet, AtomicBoolean stopFlag) throws InterruptedException {
+        for (int poll = 0; poll < ECS_READINESS_MAX_POLLS && !stopFlag.get(); poll++) {
+            List<TaskSet> taskSets = ecsService.describeTaskSets(clusterName, serviceName,
+                    List.of(createdTaskSet.getTaskSetArn()), region);
+            if (taskSets.size() == 1 && hasRunningTasks(taskSets.getFirst())) {
+                return taskSets.getFirst();
+            }
+            sleeper.sleep(STOP_POLL_INTERVAL.toMillis());
+        }
+        if (!stopFlag.get()) {
+            throw new IllegalStateException("ECS green task set has no running tasks");
+        }
+        return createdTaskSet;
+    }
+
+    private boolean hasRunningTasks(TaskSet taskSet) {
+        return taskSet.getComputedDesiredCount() > 0
+                && taskSet.getRunningCount() >= taskSet.getComputedDesiredCount();
+    }
+
+    /** A production listener is switched only once green tasks are registered and routable. */
+    void waitForGreenTargets(String region, String greenTgArn, AtomicBoolean stopFlag)
+            throws InterruptedException {
+        boolean registeredTargets = false;
+        for (int poll = 0; poll < ECS_TARGET_HEALTH_MAX_POLLS && !stopFlag.get(); poll++) {
+            List<TargetHealth> targets = elbV2Service.describeTargetHealth(region, greenTgArn, null);
+            registeredTargets = !targets.isEmpty();
+            if (registeredTargets && targets.stream().allMatch(target -> "healthy".equals(target.getState()))) {
+                return;
+            }
+            sleeper.sleep(STOP_POLL_INTERVAL.toMillis());
+        }
+        if (!stopFlag.get()) {
+            throw new IllegalStateException(registeredTargets
+                    ? "ECS green target group has unhealthy targets"
+                    : "ECS green target group has no registered targets");
+        }
+    }
+
+    void verifyListenersForwardToGreen(String region, List<String> listenerArns, String greenTgArn) {
+        for (String listenerArn : listenerArns) {
+            List<Rule> defaultRules = elbV2Service.describeRules(region, listenerArn, null).stream()
+                    .filter(Rule::isDefault)
+                    .toList();
+            if (defaultRules.size() != 1 || !forwardsOnlyTo(defaultRules.getFirst().getActions(), greenTgArn)) {
+                throw new IllegalStateException("Production listener did not switch to the green target group");
+            }
+        }
+    }
+
+    private boolean forwardsOnlyTo(List<Action> actions, String greenTgArn) {
+        if (actions == null) {
+            return false;
+        }
+        List<Action> forwardActions = actions.stream()
+                .filter(action -> "forward".equals(action.getType()))
+                .toList();
+        return !forwardActions.isEmpty()
+                && forwardActions.stream().allMatch(action -> forwardsOnlyTo(action, greenTgArn));
+    }
+
+    private boolean forwardsOnlyTo(Action action, String greenTgArn) {
+        if (action.getTargetGroups() == null || action.getTargetGroups().isEmpty()) {
+            return greenTgArn.equals(action.getTargetGroupArn());
+        }
+        boolean greenReceivesTraffic = action.getTargetGroups().stream()
+                .anyMatch(target -> greenTgArn.equals(target.getTargetGroupArn()) && effectiveWeight(target) > 0);
+        boolean otherTargetReceivesTraffic = action.getTargetGroups().stream()
+                .anyMatch(target -> !greenTgArn.equals(target.getTargetGroupArn()) && effectiveWeight(target) > 0);
+        return greenReceivesTraffic && !otherTargetReceivesTraffic;
+    }
+
+    private int effectiveWeight(Action.TargetGroupTuple target) {
+        return target.getWeight() != null ? target.getWeight() : 1;
+    }
+
+    void verifyBlueTaskSetDrained(String region, String clusterName, String serviceName, String blueTaskSetArn) {
+        if (!ecsService.describeTaskSets(clusterName, serviceName, List.of(blueTaskSetArn), region).isEmpty()) {
+            throw new IllegalStateException("ECS blue task set was not drained");
+        }
+    }
+
+    void executeEcsAllowTraffic(String region, Deployment deployment, String configName,
                                 Map<String, Object> ecsTargetMap, List<String> listenerArns,
                                 String blueTgArn, String greenTgArn,
                                 AtomicBoolean stopFlag) throws InterruptedException {
@@ -1473,9 +1778,9 @@ public class CodeDeployService {
                         elbV2Service.shiftListenerForward(region, listenerArn,
                                 blueTgArn, greenTgArn, trc.percentage);
                     }
-                    long waitMs = Math.min(trc.intervalSeconds * 1000L, 5000L);
+                    long waitMs = trc.intervalSeconds * 1000L;
                     if (waitMs > 0 && !stopFlag.get()) {
-                        Thread.sleep(waitMs);
+                        sleeper.sleep(waitMs);
                     }
                     if (!stopFlag.get()) {
                         for (String listenerArn : listenerArns) {
@@ -1493,9 +1798,9 @@ public class CodeDeployService {
                                     blueTgArn, greenTgArn, pct);
                         }
                         if (pct < 100) {
-                            long waitMs = Math.min(trc.intervalSeconds * 1000L, 2000L);
+                            long waitMs = trc.intervalSeconds * 1000L;
                             if (waitMs > 0) {
-                                Thread.sleep(waitMs);
+                                sleeper.sleep(waitMs);
                             }
                         }
                     }
@@ -1588,6 +1893,9 @@ public class CodeDeployService {
     private void runStateMachine(String region, Deployment deployment, AppSpecInfo appSpec,
                                   Map<String, Object> lambdaTargetMap, AtomicBoolean stopFlag, String configName) {
         String deploymentId = deployment.getDeploymentId();
+        AliasRoutingSnapshot aliasSnapshot = new AliasRoutingSnapshot(
+                lambdaService.getAlias(region, appSpec.functionName, appSpec.aliasName));
+        boolean trafficShifted = false;
         try {
             deployment.setStatus("InProgress");
             deployment.setStartTime(Instant.now().toEpochMilli() / 1000.0);
@@ -1605,16 +1913,30 @@ public class CodeDeployService {
 
             if (stopFlag.get()) { finishStopped(deployment, lambdaTargetMap); return; }
 
+            // Once AllowTraffic starts, a failed partial shift must be restored from the captured snapshot.
+            trafficShifted = true;
             executeAllowTraffic(region, deployment, appSpec, configName, lambdaTargetMap, stopFlag);
 
-            if (stopFlag.get()) { finishStopped(deployment, lambdaTargetMap); return; }
+            if (stopFlag.get()) {
+                rollbackAliasIfEnabled(region, deployment, appSpec, aliasSnapshot, "DEPLOYMENT_STOP_ON_REQUEST");
+                finishStopped(deployment, lambdaTargetMap);
+                return;
+            }
 
             if (appSpec.afterAllowTraffic != null) {
                 boolean ok = invokeHook(region, deployment, appSpec.afterAllowTraffic,
                         "AfterAllowTraffic", lambdaTargetMap, stopFlag);
-                if (stopFlag.get()) { finishStopped(deployment, lambdaTargetMap); return; }
-                if (!ok) { finishFailed(deployment, lambdaTargetMap, "HOOK_EXECUTION_FAILURE",
-                        "Hook function reported failure: AfterAllowTraffic"); return; }
+                if (stopFlag.get()) {
+                    rollbackAliasIfEnabled(region, deployment, appSpec, aliasSnapshot, "DEPLOYMENT_STOP_ON_REQUEST");
+                    finishStopped(deployment, lambdaTargetMap);
+                    return;
+                }
+                if (!ok) {
+                    rollbackAliasIfEnabled(region, deployment, appSpec, aliasSnapshot, "DEPLOYMENT_FAILURE");
+                    finishFailed(deployment, lambdaTargetMap, "HOOK_EXECUTION_FAILURE",
+                            "Hook function reported failure: AfterAllowTraffic");
+                    return;
+                }
             }
 
             updateTargetStatus(lambdaTargetMap, "Succeeded");
@@ -1623,16 +1945,22 @@ public class CodeDeployService {
 
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
+            if (trafficShifted) {
+                rollbackAliasIfEnabled(region, deployment, appSpec, aliasSnapshot, "DEPLOYMENT_STOP_ON_REQUEST");
+            }
             finishStopped(deployment, lambdaTargetMap);
         } catch (Exception e) {
             LOG.warnv("Deployment {0} failed: {1}", deploymentId, e.getMessage());
+            if (trafficShifted) {
+                rollbackAliasIfEnabled(region, deployment, appSpec, aliasSnapshot, "DEPLOYMENT_FAILURE");
+            }
             finishFailed(deployment, lambdaTargetMap, "DeploymentFailed", e.getMessage());
         } finally {
             stopFlags.remove(deploymentId);
         }
     }
 
-    private void executeAllowTraffic(String region, Deployment deployment, AppSpecInfo appSpec,
+    private boolean executeAllowTraffic(String region, Deployment deployment, AppSpecInfo appSpec,
                                        String configName, Map<String, Object> lambdaTargetMap,
                                        AtomicBoolean stopFlag) throws InterruptedException {
         TrafficRoutingInfo trc = getTrafficRoutingInfo(region, configName);
@@ -1647,12 +1975,11 @@ public class CodeDeployService {
                     lambdaService.updateAlias(region, appSpec.functionName, appSpec.aliasName,
                             appSpec.currentVersion, null, routing);
 
-                    // Wait the canary interval (capped at 5s for emulator speed)
-                    long waitMs = Math.min(trc.intervalSeconds * 1000L, 5000L);
+                    long waitMs = trc.intervalSeconds * 1000L;
                     if (waitMs > 0 && !stopFlag.get()) {
-                        Thread.sleep(waitMs);
+                        sleeper.sleep(waitMs);
                     }
-                    if (stopFlag.get()) { return; }
+                    if (stopFlag.get()) { return true; }
 
                     // Step 2: flip 100% to targetVersion
                     lambdaService.updateAlias(region, appSpec.functionName, appSpec.aliasName,
@@ -1669,9 +1996,9 @@ public class CodeDeployService {
                             double weight = pct / 100.0;
                             lambdaService.updateAlias(region, appSpec.functionName, appSpec.aliasName,
                                     appSpec.currentVersion, null, Map.of(appSpec.targetVersion, weight));
-                            long waitMs = Math.min(trc.intervalSeconds * 1000L, 2000L);
+                            long waitMs = trc.intervalSeconds * 1000L;
                             if (waitMs > 0) {
-                                Thread.sleep(waitMs);
+                                sleeper.sleep(waitMs);
                             }
                         }
                     }
@@ -1683,18 +2010,35 @@ public class CodeDeployService {
                 }
             }
             finishLifecycleEvent(event, "Succeeded");
+            return true;
         } catch (Exception e) {
             finishLifecycleEvent(event, "Failed");
             throw e;
         }
     }
 
+    @SuppressWarnings("unchecked")
+    private void rollbackAliasIfEnabled(String region, Deployment deployment, AppSpecInfo appSpec,
+                                        AliasRoutingSnapshot snapshot, String event) {
+        DeploymentGroup group = getDeploymentGroup(region, deployment.getApplicationName(), deployment.getDeploymentGroupName());
+        Map<String, Object> config = group.getAutoRollbackConfiguration();
+        if (config == null || !Boolean.TRUE.equals(config.get("enabled"))) {
+            return;
+        }
+        Object events = config.get("events");
+        if (!(events instanceof List<?> list) || !list.contains(event)) {
+            return;
+        }
+        lambdaService.updateAlias(region, appSpec.functionName, appSpec.aliasName,
+                snapshot.functionVersion, null, snapshot.routingConfig);
+    }
+
     private boolean invokeHook(String region, Deployment deployment, String hookFunctionName,
                                 String lifecycleEventName, Map<String, Object> lambdaTargetMap,
                                 AtomicBoolean stopFlag) throws InterruptedException {
         String executionId = UUID.randomUUID().toString();
         CompletableFuture<String> future = new CompletableFuture<>();
-        hookFutures.put(executionId, future);
+        hookExecutions.put(executionId, new HookExecution(deployment.getDeploymentId(), lifecycleEventName, future));
 
         Map<String, Object> event = addLifecycleEvent(lambdaTargetMap, lifecycleEventName);
 
@@ -1757,7 +2101,7 @@ public class CodeDeployService {
             }
             return "Succeeded".equals(status);
         } finally {
-            hookFutures.remove(executionId);
+            hookExecutions.remove(executionId);
         }
     }
 

src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java

@@ -12,6 +12,9 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageFactory;
 import io.github.hectorvent.floci.services.codebuild.CodeBuildService;
 import io.github.hectorvent.floci.services.codebuild.model.Build;
+import io.github.hectorvent.floci.services.codebuild.model.ProjectArtifacts;
+import io.github.hectorvent.floci.services.cloudformation.CloudFormationService;
+import io.github.hectorvent.floci.services.cloudformation.model.Stack;
 import io.github.hectorvent.floci.services.codedeploy.CodeDeployService;
 import io.github.hectorvent.floci.services.codedeploy.model.Deployment;
 import io.github.hectorvent.floci.services.codepipeline.model.CodePipelineExecution;
@@ -19,6 +22,10 @@ import io.github.hectorvent.floci.services.codepipeline.model.CodePipelineExecut
 import io.github.hectorvent.floci.services.codepipeline.model.CodePipelinePipeline;
 import io.github.hectorvent.floci.services.codepipeline.model.CodePipelinePipeline.TransitionState;
 import io.github.hectorvent.floci.services.codepipeline.model.CodePipelineStoredItem;
+import io.github.hectorvent.floci.services.ecs.EcsService;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.RegisterTaskDefinitionRequest;
+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
 import io.github.hectorvent.floci.services.lambda.LambdaService;
 import io.github.hectorvent.floci.services.lambda.model.InvocationType;
 import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
@@ -30,6 +37,7 @@ import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 import org.jboss.logging.Logger;
 
+import java.io.ByteArrayInputStream;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -42,12 +50,15 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 
 @ApplicationScoped
 public class CodePipelineService {
@@ -59,6 +70,8 @@ public class CodePipelineService {
     private static final long SOURCE_POLL_INTERVAL_MS = 500L;
     private static final String SOURCE_POLL_TYPE = "source-poll";
     private static final String MISSING_SOURCE_REVISION = "missing";
+    private static final int MAX_ARTIFACT_ENTRY_SIZE = 256 * 1024 * 1024;
+    private static final int MAX_EXTRACTED_ARTIFACT_SIZE = 256 * 1024 * 1024;
 
     private final AccountAwareStorageBackend<CodePipelinePipeline> pipelineStore;
     private final AccountAwareStorageBackend<CodePipelineExecution> executionStore;
@@ -66,6 +79,8 @@ public class CodePipelineService {
     private final ObjectMapper mapper;
     private final CodeBuildService codeBuildService;
     private final CodeDeployService codeDeployService;
+    private final EcsService ecsService;
+    private final CloudFormationService cloudFormationService;
     private final LambdaService lambdaService;
     private final S3Service s3Service;
     private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();
@@ -76,13 +91,13 @@ public class CodePipelineService {
     // monitor for its whole run, so counting under that monitor would block StartPipelineExecution
     // until the running execution finished.
     private final KeyedLockPool startLocks = new KeyedLockPool();
-    private final Map<String, byte[]> runtimeArtifacts = new ConcurrentHashMap<>();
 
     @Inject
     @SuppressWarnings("unchecked")
     public CodePipelineService(StorageFactory storageFactory, ObjectMapper mapper,
                                  CodeBuildService codeBuildService, CodeDeployService codeDeployService,
-                               LambdaService lambdaService, S3Service s3Service) {
+                                 LambdaService lambdaService, S3Service s3Service,
+                                 CloudFormationService cloudFormationService, EcsService ecsService) {
         this.pipelineStore = storageFactory.create(
                 "codepipeline", "codepipeline-pipelines.json", new TypeReference<Map<String, CodePipelinePipeline>>() {});
         this.executionStore = storageFactory.create(
@@ -92,10 +107,20 @@ public class CodePipelineService {
         this.mapper = mapper;
         this.codeBuildService = codeBuildService;
         this.codeDeployService = codeDeployService;
+        this.ecsService = ecsService;
+        this.cloudFormationService = cloudFormationService;
         this.lambdaService = lambdaService;
         this.s3Service = s3Service;
     }
 
+    CodePipelineService(StorageFactory storageFactory, ObjectMapper mapper,
+                        CodeBuildService codeBuildService, CodeDeployService codeDeployService,
+                        LambdaService lambdaService, S3Service s3Service,
+                        CloudFormationService cloudFormationService) {
+        this(storageFactory, mapper, codeBuildService, codeDeployService, lambdaService, s3Service,
+                cloudFormationService, null);
+    }
+
     public JsonNode handle(String action, JsonNode request, String region, String account) {
         return switch (action) {
             case "CreatePipeline" -> createPipeline(request, region, account);
@@ -745,22 +770,82 @@ public class CodePipelineService {
         String pipelineName = text(request, "pipelineName");
         CodePipelineExecution source = requireExecution(
                 account, region, pipelineName, text(request, "pipelineExecutionId"));
-        ObjectNode start = mapper.createObjectNode();
-        start.put("name", pipelineName);
-        start.set("sourceRevisions", mapper.valueToTree(source.getSourceRevisions()));
-        ArrayNode vars = start.putArray("variables");
-        source.getVariables().forEach(v -> vars.addObject()
-                .put("name", v.get("name"))
-                .put("value", v.get("resolvedValue")));
-        return startPipelineExecution(start, region, account);
+        CodePipelinePipeline pipeline = requirePipeline(account, region, pipelineName);
+        String stageName = text(request, "stageName");
+        String retryMode = text(request, "retryMode");
+        requireStage(pipeline, stageName);
+        if (!List.of("FAILED_ACTIONS", "ALL_ACTIONS").contains(retryMode)) {
+            throw new AwsException("ValidationException", "retryMode must be FAILED_ACTIONS or ALL_ACTIONS", 400);
+        }
+        if (!"Failed".equals(stageExecutionStatus(source, stageName))) {
+            throw new AwsException("PipelineExecutionNotRetryableException",
+                    "Stage execution is not failed: " + stageName, 400);
+        }
+
+        List<String> actionNames = new ArrayList<>();
+        for (JsonNode stage : pipeline.getDeclaration().path("stages")) {
+            if (!stageName.equals(stage.path("name").asText())) {
+                continue;
+            }
+            for (JsonNode action : stage.path("actions")) {
+                String actionName = action.path("name").asText();
+                if ("ALL_ACTIONS".equals(retryMode) || source.getActionExecutions().stream()
+                        .anyMatch(a -> stageName.equals(a.getStageName()) && actionName.equals(a.getActionName())
+                                && "Failed".equals(a.getStatus()))) {
+                    actionNames.add(actionName);
+                }
+            }
+        }
+        if (actionNames.isEmpty()) {
+            throw new AwsException("PipelineExecutionNotRetryableException",
+                    "Stage execution has no failed actions: " + stageName, 400);
+        }
+
+        CodePipelineExecution retry = new CodePipelineExecution();
+        retry.setAccountId(account);
+        retry.setRegion(region);
+        retry.setPipelineExecutionId(UUID.randomUUID().toString());
+        retry.setPipelineName(pipelineName);
+        retry.setPipelineVersion(pipeline.getVersion());
+        retry.setExecutionMode(pipeline.getDeclaration().path("executionMode").asText(DEFAULT_EXECUTION_MODE));
+        retry.setExecutionType("STANDARD");
+        retry.setStatus("InProgress");
+        retry.setStatusSummary("Stage execution retry started.");
+        retry.setStartTime(now());
+        retry.setLastUpdateTime(retry.getStartTime());
+        retry.setSourceRevisions(new ArrayList<>(source.getSourceRevisions()));
+        retry.setVariables(new ArrayList<>(source.getVariables()));
+        retry.setArtifacts(new LinkedHashMap<>(source.getArtifacts()));
+        retry.setRetryStageName(stageName);
+        retry.setRetryActionNames(actionNames);
+        retry.setTrigger(Map.of("triggerType", "RetryStageExecution", "triggerDetail", stageName));
+        if (!persistExecutionIfSlotAvailable(retry)) {
+            throw new AwsException("ConcurrentPipelineExecutionsLimitExceededException",
+                    "The pipeline has reached the limit for concurrent pipeline executions", 400);
+        }
+        applyExecutionMode(retry);
+        try {
+            executor.submit(() -> runExecution(pipeline, retry));
+        } catch (RejectedExecutionException exception) {
+            retry.setStatus("Failed");
+            retry.setStatusSummary("Pipeline execution could not be scheduled.");
+            retry.setLastUpdateTime(now());
+            putExecution(retry);
+            throw new AwsException("ConflictException", "Pipeline execution could not be scheduled.", 400);
+        }
+        return mapper.createObjectNode().put("pipelineExecutionId", retry.getPipelineExecutionId());
     }
 
     private ObjectNode rollbackStage(JsonNode request, String region, String account) {
         CodePipelineExecution target = requireExecution(
                 account, region, text(request, "pipelineName"), text(request, "targetPipelineExecutionId"));
-        ObjectNode started = retryStageExecution(mapper.createObjectNode()
-                .put("pipelineName", target.getPipelineName())
-                .put("pipelineExecutionId", target.getPipelineExecutionId()), region, account);
+        ObjectNode start = mapper.createObjectNode().put("name", target.getPipelineName());
+        start.set("sourceRevisions", mapper.valueToTree(target.getSourceRevisions()));
+        ArrayNode vars = start.putArray("variables");
+        target.getVariables().forEach(v -> vars.addObject()
+                .put("name", v.get("name"))
+                .put("value", v.get("resolvedValue")));
+        ObjectNode started = startPipelineExecution(start, region, account);
         CodePipelineExecution rollback = requireExecution(
                 account, region, target.getPipelineName(), started.path("pipelineExecutionId").asText());
         rollback.setExecutionType("ROLLBACK");
@@ -977,6 +1062,10 @@ public class CodePipelineService {
         Runnable work = () -> {
             try {
                 for (JsonNode stage : pipeline.getDeclaration().path("stages")) {
+                    if (execution.getRetryStageName() != null
+                            && !execution.getRetryStageName().equals(stage.path("name").asText())) {
+                        continue;
+                    }
                     waitForTransition(pipeline, execution, stage.path("name").asText());
                     if (finishIfStopped(execution)) {
                         return;
@@ -986,7 +1075,7 @@ public class CodePipelineService {
                     execution.getStageExecutionStatuses().put(stageName, "InProgress");
                     execution.setLastUpdateTime(now());
                     putExecution(execution);
-                    runStage(pipeline, execution, stage);
+                    runStage(pipeline, execution, stage, execution.getRetryActionNames());
                     if ("Failed".equals(execution.getStatus())) {
                         execution.getStageExecutionStatuses().put(stageName, "Failed");
                         return;
@@ -1012,7 +1101,6 @@ public class CodePipelineService {
                 execution.setCurrentStage(null);
                 execution.setLastUpdateTime(now());
                 putExecution(execution);
-                clearRuntimeArtifacts(execution);
             }
         };
         if ("QUEUED".equals(execution.getExecutionMode())) {
@@ -1022,9 +1110,14 @@ public class CodePipelineService {
         }
     }
 
-    private void runStage(CodePipelinePipeline pipeline, CodePipelineExecution execution, JsonNode stage) {
+    private void runStage(CodePipelinePipeline pipeline, CodePipelineExecution execution, JsonNode stage,
+                          List<String> retryActionNames) {
         Map<Integer, List<JsonNode>> groups = new LinkedHashMap<>();
         for (JsonNode action : stage.path("actions")) {
+            if (execution.getRetryStageName() != null
+                    && !retryActionNames.contains(action.path("name").asText())) {
+                continue;
+            }
             groups.computeIfAbsent(action.path("runOrder").asInt(1), ignored -> new ArrayList<>()).add(action);
         }
         groups.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> {
@@ -1098,7 +1191,9 @@ public class CodePipelineService {
         switch (provider) {
             case "S3" -> executeS3(execution, action, state);
             case "CodeBuild" -> executeCodeBuild(execution, action, state);
-            case "CodeDeploy" -> executeCodeDeploy(execution, action, state);
+            case "CodeDeploy" -> executeCodeDeploy(pipeline, execution, action, state);
+            case "CodeDeployToECS" -> executeCodeDeployToEcs(execution, action, state);
+            case "CloudFormation" -> executeCloudFormation(pipeline, execution, action, state);
             case "Lambda" -> executeLambda(execution, action, state);
             case "CodePipeline" -> executeNestedPipeline(execution, action, state);
             default -> throw new AwsException("InvalidActionDeclarationException",
@@ -1113,7 +1208,7 @@ public class CodePipelineService {
             String key = config.path("S3ObjectKey").asText(config.path("ObjectKey").asText(null));
             S3Object object = s3Service.getObject(bucket, key);
             for (JsonNode artifact : action.path("outputArtifacts")) {
-                runtimeArtifacts.put(artifactKey(execution, artifact.path("name").asText()), object.getData());
+                execution.getArtifacts().put(artifact.path("name").asText(), object.getData());
             }
             Map<String, Object> revision = new LinkedHashMap<>();
             revision.put("name", state.getActionName());
@@ -1129,7 +1224,7 @@ public class CodePipelineService {
         String bucket = config.path("BucketName").asText(config.path("S3Bucket").asText(null));
         String objectKey = config.path("ObjectKey").asText(state.getActionName() + ".zip");
         JsonNode input = action.path("inputArtifacts").path(0);
-        byte[] data = runtimeArtifacts.get(artifactKey(execution, input.path("name").asText()));
+        byte[] data = execution.getArtifacts().get(input.path("name").asText());
         if (data == null) {
             throw new AwsException("InvalidJobStateException", "Input artifact is not available", 400);
         }
@@ -1139,9 +1234,14 @@ public class CodePipelineService {
 
     private void executeCodeBuild(CodePipelineExecution execution, JsonNode action,
                                   ActionExecution state) throws InterruptedException {
+        for (JsonNode artifact : action.path("outputArtifacts")) {
+            execution.getArtifacts().remove(artifact.path("name").asText());
+        }
         String projectName = action.path("configuration").path("ProjectName").asText(null);
+        ProjectArtifacts artifactsOverride = new ProjectArtifacts();
+        artifactsOverride.setType("CODEPIPELINE");
         Build build = codeBuildService.startBuild(execution.getRegion(), execution.getAccountId(), projectName,
-                null, null, null, null, null, null, null);
+                null, null, artifactsOverride, null, null, null, null);
         state.setExternalExecutionId(build.getId());
         while (!Boolean.TRUE.equals(build.getBuildComplete())) {
             if (abandonExternalActionIfRequested(execution, state)) {
@@ -1160,16 +1260,38 @@ public class CodePipelineService {
             }
             throw new AwsException("ActionExecutionFailed", message, 400);
         }
+        if (!action.path("outputArtifacts").isEmpty()) {
+            byte[] output = codeBuildService.getBuildOutputArtifact(
+                    execution.getRegion(), execution.getAccountId(), build.getId());
+            if (output == null) {
+                throw new AwsException("ActionExecutionFailed",
+                        "CodeBuild build " + build.getId() + " did not produce an output artifact", 400);
+            }
+            for (JsonNode artifact : action.path("outputArtifacts")) {
+                execution.getArtifacts().put(artifact.path("name").asText(), output);
+            }
+        }
     }
 
-    private void executeCodeDeploy(CodePipelineExecution execution, JsonNode action,
+    private void executeCodeDeploy(CodePipelinePipeline pipeline, CodePipelineExecution execution, JsonNode action,
                                    ActionExecution state) throws InterruptedException {
         JsonNode config = action.path("configuration");
         Map<String, Object> revision = null;
         JsonNode input = action.path("inputArtifacts").path(0);
         if (!input.isMissingNode()) {
-            revision = Map.of("revisionType", "S3",
-                    "s3Location", Map.of("bucket", artifactBucket(action), "key", input.path("name").asText()));
+            String artifactName = input.path("name").asText();
+            byte[] artifact = execution.getArtifacts().get(artifactName);
+            if (artifact == null) {
+                throw new AwsException("InvalidJobStateException", "Input artifact is not available", 400);
+            }
+            String bucket = pipeline.getDeclaration().path("artifactStore").path("location").asText(null);
+            if (bucket == null || bucket.isBlank()) {
+                throw new AwsException("InvalidActionDeclarationException", "Pipeline artifact store is required", 400);
+            }
+            String key = execution.getPipelineExecutionId() + "/" + state.getStageName() + "/"
+                    + state.getActionName() + "/" + artifactName + ".zip";
+            s3Service.putObject(bucket, key, artifact, "application/zip", Map.of());
+            revision = Map.of("revisionType", "S3", "s3Location", Map.of("bucket", bucket, "key", key));
         }
         String deploymentId = codeDeployService.createDeployment(
                 execution.getRegion(), config.path("ApplicationName").asText(),
@@ -1195,6 +1317,417 @@ public class CodePipelineService {
         }
     }
 
+    private void executeCodeDeployToEcs(CodePipelineExecution execution, JsonNode action,
+                                        ActionExecution state) throws InterruptedException {
+        if (ecsService == null) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CodeDeployToECS is not available", 400);
+        }
+        JsonNode config = action.path("configuration");
+        String applicationName = requiredCodeDeployToEcsConfiguration(config, "ApplicationName");
+        String deploymentGroupName = requiredCodeDeployToEcsConfiguration(config, "DeploymentGroupName");
+        String taskDefinitionTemplate = readCodeDeployToEcsTemplate(execution, action, config,
+                "TaskDefinitionTemplateArtifact", "TaskDefinitionTemplatePath");
+        String appSpecTemplate = readCodeDeployToEcsTemplate(execution, action, config,
+                "AppSpecTemplateArtifact", "AppSpecTemplatePath");
+
+        RegisterTaskDefinitionRequest taskDefinitionRequest;
+        try {
+            taskDefinitionRequest = mapper.readValue(taskDefinitionTemplate, RegisterTaskDefinitionRequest.class);
+        } catch (java.io.IOException exception) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "TaskDefinitionTemplatePath must reference a valid ECS task definition JSON template", 400);
+        }
+        if (taskDefinitionRequest.getFamily() == null || taskDefinitionRequest.getFamily().isBlank()) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "Task definition template must specify family", 400);
+        }
+        substituteCodeDeployToEcsImage(execution, action, config, taskDefinitionRequest);
+        TaskDefinition taskDefinition = ecsService.registerTaskDefinition(taskDefinitionRequest, execution.getRegion());
+        String appSpec = appSpecTemplate.replace("<TASK_DEFINITION>", taskDefinition.getTaskDefinitionArn());
+        Map<String, Object> revision = Map.of("revisionType", "AppSpecContent",
+                "appSpecContent", Map.of("content", appSpec));
+        String deploymentId = codeDeployService.createDeployment(execution.getRegion(), applicationName,
+                deploymentGroupName, null, revision, "CodePipeline execution");
+        state.setExternalExecutionId(deploymentId);
+        waitForCodeDeployDeployment(execution, state, deploymentId);
+    }
+
+    private String readCodeDeployToEcsTemplate(CodePipelineExecution execution, JsonNode action, JsonNode config,
+                                               String artifactField, String pathField) {
+        String templatePath = requiredCodeDeployToEcsConfiguration(config, pathField);
+        int separator = templatePath.indexOf("::");
+        if (separator >= 0) {
+            if (separator == 0 || separator == templatePath.length() - 2
+                    || templatePath.indexOf("::", separator + 2) >= 0) {
+                throw new AwsException("InvalidActionDeclarationException",
+                        "CodeDeployToECS template paths must use ArtifactName::TemplateFilePath", 400);
+            }
+            return readCodeDeployToEcsTemplate(execution, action, templatePath.substring(0, separator),
+                    templatePath.substring(separator + 2));
+        }
+        String artifactName = requiredCodeDeployToEcsConfiguration(config, artifactField);
+        return readCodeDeployToEcsTemplate(execution, action, artifactName, templatePath);
+    }
+
+    private String readCodeDeployToEcsTemplate(CodePipelineExecution execution, JsonNode action, String artifactName,
+                                               String templatePath) {
+        boolean declaredInput = false;
+        for (JsonNode input : action.path("inputArtifacts")) {
+            if (artifactName.equals(input.path("name").asText())) {
+                declaredInput = true;
+                break;
+            }
+        }
+        if (!declaredInput) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CodeDeployToECS template references an unavailable input artifact: " + artifactName, 400);
+        }
+        byte[] artifact = execution.getArtifacts().get(artifactName);
+        if (artifact == null) {
+            throw new AwsException("InvalidJobStateException", "Input artifact is not available: " + artifactName, 400);
+        }
+        return readTemplateFromArtifact(artifact, templatePath);
+    }
+
+    private static String requiredCodeDeployToEcsConfiguration(JsonNode config, String field) {
+        String value = config.path(field).asText(null);
+        if (value == null || value.isBlank()) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CodeDeployToECS " + field + " is required", 400);
+        }
+        return value;
+    }
+
+    private void substituteCodeDeployToEcsImage(CodePipelineExecution execution, JsonNode action, JsonNode config,
+                                                RegisterTaskDefinitionRequest taskDefinitionRequest) {
+        String imageArtifactName = requiredCodeDeployToEcsConfiguration(config, "Image1ArtifactName");
+        String containerName = requiredCodeDeployToEcsConfiguration(config, "Image1ContainerName");
+        String imageDetail = readCodeDeployToEcsTemplate(execution, action, imageArtifactName, "imageDetail.json");
+        JsonNode imageDetailJson;
+        try {
+            imageDetailJson = mapper.readTree(imageDetail);
+        } catch (java.io.IOException exception) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "imageDetail.json must contain a valid image detail JSON object", 400);
+        }
+        String imageUri = imageDetailJson.path("ImageURI").asText(null);
+        if (imageUri == null || imageUri.isBlank()) {
+            throw new AwsException("InvalidActionDeclarationException", "imageDetail.json must specify ImageURI", 400);
+        }
+        if (taskDefinitionRequest.getContainerDefinitions() == null) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "Task definition template must specify containerDefinitions", 400);
+        }
+        for (ContainerDefinition container : taskDefinitionRequest.getContainerDefinitions()) {
+            if (containerName.equals(container.getName())) {
+                if (container.getImage() == null) {
+                    throw new AwsException("InvalidActionDeclarationException",
+                            "Image1ContainerName must identify a container with an image", 400);
+                }
+                container.setImage(container.getImage().replace("<IMAGE1_NAME>", imageUri));
+                return;
+            }
+        }
+        throw new AwsException("InvalidActionDeclarationException",
+                "Image1ContainerName does not match a task definition container", 400);
+    }
+
+    private void waitForCodeDeployDeployment(CodePipelineExecution execution, ActionExecution state, String deploymentId)
+            throws InterruptedException {
+        Deployment deployment = codeDeployService.getDeployment(execution.getRegion(), deploymentId);
+        while (!List.of("Succeeded", "Failed", "Stopped").contains(deployment.getStatus())) {
+            if (abandonExternalActionIfRequested(execution, state)) {
+                return;
+            }
+            TimeUnit.MILLISECONDS.sleep(POLL_INTERVAL_MS);
+            deployment = codeDeployService.getDeployment(execution.getRegion(), deploymentId);
+        }
+        if (abandonExternalActionIfRequested(execution, state)) {
+            return;
+        }
+        if (!"Succeeded".equals(deployment.getStatus())) {
+            String message = "CodeDeploy deployment finished with " + deployment.getStatus();
+            if (recordExternalActionFailureWhileStopping(execution, state, message)) {
+                return;
+            }
+            throw new AwsException("ActionExecutionFailed", message, 400);
+        }
+    }
+
+    private void executeCloudFormation(CodePipelinePipeline pipeline, CodePipelineExecution execution,
+                                       JsonNode action, ActionExecution state) throws InterruptedException {
+        JsonNode config = action.path("configuration");
+        String stackName = requiredActionConfiguration(config, "StackName");
+        String actionMode = requiredActionConfiguration(config, "ActionMode");
+        if (!"CREATE_UPDATE".equals(actionMode)) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CloudFormation ActionMode must be CREATE_UPDATE", 400);
+        }
+        String templatePath = requiredActionConfiguration(config, "TemplatePath");
+        int separator = templatePath.indexOf("::");
+        if (separator <= 0 || separator == templatePath.length() - 2
+                || templatePath.indexOf("::", separator + 2) >= 0) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CloudFormation TemplatePath must use ArtifactName::TemplateFilePath", 400);
+        }
+        String artifactName = templatePath.substring(0, separator);
+        byte[] artifact = execution.getArtifacts().get(artifactName);
+        if (artifact == null) {
+            throw new AwsException("InvalidJobStateException", "Input artifact is not available: " + artifactName, 400);
+        }
+        String bucket = pipeline.getDeclaration().path("artifactStore").path("location").asText(null);
+        if (bucket == null || bucket.isBlank()) {
+            throw new AwsException("InvalidActionDeclarationException", "Pipeline artifact store is required", 400);
+        }
+        Map<String, ArtifactLocation> artifactLocations = materializeCloudFormationArtifacts(
+                execution, action, state, bucket);
+        String templateBody = readTemplateFromArtifact(artifact, templatePath.substring(separator + 2));
+
+        boolean exists;
+        try {
+            cloudFormationService.describeStacks(stackName, execution.getRegion(), execution.getAccountId());
+            exists = true;
+        } catch (AwsException exception) {
+            if (!"ValidationError".equals(exception.getErrorCode())) {
+                throw exception;
+            }
+            exists = false;
+        }
+        String changeSetName = "codepipeline-" + UUID.randomUUID().toString().substring(0, 8);
+        String changeSetType = exists ? "UPDATE" : "CREATE";
+        cloudFormationService.createChangeSet(stackName, changeSetName, changeSetType, templateBody, null,
+                resolveParameterOverrides(config, execution, action, artifactLocations), List.of(), Map.of(),
+                execution.getRegion(), execution.getAccountId());
+        Future<?> deployment = cloudFormationService.executeChangeSet(
+                stackName, changeSetName, execution.getRegion(), execution.getAccountId());
+        try {
+            deployment.get();
+        } catch (ExecutionException exception) {
+            Throwable cause = exception.getCause();
+            if (cause instanceof AwsException awsException) {
+                throw awsException;
+            }
+            throw new AwsException("ActionExecutionFailed", "CloudFormation deployment failed: "
+                    + Objects.toString(cause.getMessage(), cause.getClass().getSimpleName()), 400);
+        }
+        Stack stack = cloudFormationService.describeStacks(stackName, execution.getRegion(), execution.getAccountId()).get(0);
+        if (!List.of("CREATE_COMPLETE", "UPDATE_COMPLETE").contains(stack.getStatus())) {
+            throw new AwsException("ActionExecutionFailed", "CloudFormation stack " + stackName
+                    + " finished with " + stack.getStatus(), 400);
+        }
+        state.setExternalExecutionId(stack.getStackId());
+    }
+
+    private static String requiredActionConfiguration(JsonNode config, String field) {
+        String value = config.path(field).asText(null);
+        if (value == null || value.isBlank()) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CloudFormation " + field + " is required", 400);
+        }
+        return value;
+    }
+
+    private Map<String, ArtifactLocation> materializeCloudFormationArtifacts(CodePipelineExecution execution,
+                                                                               JsonNode action,
+                                                                               ActionExecution state,
+                                                                               String bucket) {
+        Map<String, ArtifactLocation> locations = new LinkedHashMap<>();
+        for (JsonNode input : action.path("inputArtifacts")) {
+            String artifactName = input.path("name").asText();
+            byte[] artifact = execution.getArtifacts().get(artifactName);
+            if (artifact == null) {
+                throw new AwsException("InvalidJobStateException",
+                        "Input artifact is not available: " + artifactName, 400);
+            }
+            String prefix = execution.getPipelineExecutionId() + "/" + state.getStageName() + "/"
+                    + state.getActionName() + "/" + artifactName;
+            String archiveKey = prefix + ".zip";
+            s3Service.putObject(bucket, archiveKey, artifact, "application/zip", Map.of());
+            List<ArtifactLocation> nestedArtifacts = extractArtifactEntries(bucket, prefix, artifact);
+            ArtifactLocation location = nestedArtifacts.size() == 1 && nestedArtifacts.getFirst().key().endsWith(".zip")
+                    ? nestedArtifacts.getFirst() : new ArtifactLocation(bucket, archiveKey);
+            locations.put(artifactName, location);
+        }
+        return locations;
+    }
+
+    private List<ArtifactLocation> extractArtifactEntries(String bucket, String prefix, byte[] artifact) {
+        List<ArtifactLocation> nestedArtifacts = new ArrayList<>();
+        int extractedSize = 0;
+        try (ZipInputStream zip = new ZipInputStream(new ByteArrayInputStream(artifact))) {
+            ZipEntry entry;
+            while ((entry = zip.getNextEntry()) != null) {
+                if (entry.isDirectory()) {
+                    continue;
+                }
+                String entryName = entry.getName();
+                validateArtifactEntryName(entryName);
+                byte[] data = zip.readNBytes(MAX_ARTIFACT_ENTRY_SIZE + 1);
+                if (data.length > MAX_ARTIFACT_ENTRY_SIZE) {
+                    throw new AwsException("InvalidActionDeclarationException",
+                            "Artifact entry exceeds 256 MB: " + entryName, 400);
+                }
+                if (data.length > MAX_EXTRACTED_ARTIFACT_SIZE - extractedSize) {
+                    throw new AwsException("InvalidActionDeclarationException",
+                            "CloudFormation input artifact expands beyond 256 MB", 400);
+                }
+                extractedSize += data.length;
+                String key = prefix + "/" + entryName;
+                s3Service.putObject(bucket, key, data, "application/octet-stream", Map.of());
+                nestedArtifacts.add(new ArtifactLocation(bucket, key));
+            }
+        } catch (java.io.IOException exception) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "Could not read CloudFormation input artifact", 400);
+        }
+        return nestedArtifacts;
+    }
+
+    private static void validateArtifactEntryName(String entryName) {
+        if (entryName.startsWith("/") || entryName.startsWith("\\") || entryName.contains("\\")) {
+            throw new AwsException("InvalidActionDeclarationException", "Artifact entry path is not safe", 400);
+        }
+        for (String segment : entryName.split("/")) {
+            if (segment.isBlank() || ".".equals(segment) || "..".equals(segment)) {
+                throw new AwsException("InvalidActionDeclarationException", "Artifact entry path is not safe", 400);
+            }
+        }
+    }
+
+    private Map<String, String> resolveParameterOverrides(JsonNode config, CodePipelineExecution execution,
+                                                           JsonNode action,
+                                                           Map<String, ArtifactLocation> artifactLocations) {
+        String overrides = config.path("ParameterOverrides").asText(null);
+        if (overrides == null || overrides.isBlank()) {
+            return Map.of();
+        }
+        try {
+            JsonNode values = mapper.readTree(overrides);
+            if (!values.isObject()) {
+                throw new AwsException("InvalidActionDeclarationException",
+                        "CloudFormation ParameterOverrides must be a JSON object", 400);
+            }
+            Map<String, String> resolved = new LinkedHashMap<>();
+            values.fields().forEachRemaining(entry -> resolved.put(entry.getKey(),
+                    resolveParameterOverride(entry.getValue(), execution, action, artifactLocations)));
+            return resolved;
+        } catch (java.io.IOException exception) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CloudFormation ParameterOverrides must be valid JSON", 400);
+        }
+    }
+
+    private String resolveParameterOverride(JsonNode value, CodePipelineExecution execution, JsonNode action,
+                                            Map<String, ArtifactLocation> artifactLocations) {
+        if (value.isTextual() || value.isNumber() || value.isBoolean()) {
+            return value.asText();
+        }
+        if (!value.isObject()) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CloudFormation parameter override must be a scalar or supported function", 400);
+        }
+        JsonNode artifactAtt = value.path("Fn::GetArtifactAtt");
+        if (!artifactAtt.isMissingNode()) {
+            List<String> arguments = functionArguments(artifactAtt, 2, "Fn::GetArtifactAtt");
+            ArtifactLocation location = requireArtifactLocation(arguments.getFirst(), action, artifactLocations);
+            return switch (arguments.get(1)) {
+                case "BucketName" -> location.bucket();
+                case "ObjectKey" -> location.key();
+                case "URL" -> "https://s3." + execution.getRegion() + ".amazonaws.com/"
+                        + location.bucket() + "/" + location.key();
+                default -> throw new AwsException("InvalidActionDeclarationException",
+                        "Fn::GetArtifactAtt attribute is not supported: " + arguments.get(1), 400);
+            };
+        }
+        JsonNode getParam = value.path("Fn::GetParam");
+        if (!getParam.isMissingNode()) {
+            List<String> arguments = functionArguments(getParam, 3, "Fn::GetParam");
+            requireArtifactLocation(arguments.getFirst(), action, artifactLocations);
+            byte[] artifact = execution.getArtifacts().get(arguments.getFirst());
+            String contents = readTemplateFromArtifact(artifact, arguments.get(1));
+            try {
+                JsonNode parameters = mapper.readTree(contents);
+                JsonNode parameter = parameters.path(arguments.get(2));
+                if (!parameters.isObject() || parameter.isMissingNode() || !parameter.isValueNode()) {
+                    throw new AwsException("InvalidActionDeclarationException",
+                            "Fn::GetParam could not resolve " + arguments.get(2), 400);
+                }
+                return parameter.asText();
+            } catch (java.io.IOException exception) {
+                throw new AwsException("InvalidActionDeclarationException",
+                        "Fn::GetParam file must contain a JSON object", 400);
+            }
+        }
+        throw new AwsException("InvalidActionDeclarationException",
+                "CloudFormation parameter override uses an unsupported function", 400);
+    }
+
+    private static List<String> functionArguments(JsonNode arguments, int expectedCount, String functionName) {
+        if (!arguments.isArray() || arguments.size() != expectedCount) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    functionName + " requires " + expectedCount + " string arguments", 400);
+        }
+        List<String> values = new ArrayList<>();
+        for (JsonNode argument : arguments) {
+            if (!argument.isTextual() || argument.asText().isBlank()) {
+                throw new AwsException("InvalidActionDeclarationException",
+                        functionName + " requires " + expectedCount + " string arguments", 400);
+            }
+            values.add(argument.asText());
+        }
+        return values;
+    }
+
+    private static ArtifactLocation requireArtifactLocation(String artifactName, JsonNode action,
+                                                            Map<String, ArtifactLocation> artifactLocations) {
+        boolean isInputArtifact = false;
+        for (JsonNode input : action.path("inputArtifacts")) {
+            if (artifactName.equals(input.path("name").asText())) {
+                isInputArtifact = true;
+                break;
+            }
+        }
+        ArtifactLocation location = artifactLocations.get(artifactName);
+        if (!isInputArtifact || location == null) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "Parameter override references an unavailable input artifact: " + artifactName, 400);
+        }
+        return location;
+    }
+
+    private static String readTemplateFromArtifact(byte[] artifact, String templatePath) {
+        if (templatePath.startsWith("/") || templatePath.startsWith("\\") || templatePath.contains("\\")) {
+            throw new AwsException("InvalidActionDeclarationException", "TemplatePath is not safe", 400);
+        }
+        for (String segment : templatePath.split("/")) {
+            if (segment.isBlank() || ".".equals(segment) || "..".equals(segment)) {
+                throw new AwsException("InvalidActionDeclarationException", "TemplatePath is not safe", 400);
+            }
+        }
+        try (ZipInputStream zip = new ZipInputStream(new ByteArrayInputStream(artifact))) {
+            ZipEntry entry;
+            while ((entry = zip.getNextEntry()) != null) {
+                if (!entry.isDirectory() && templatePath.equals(entry.getName())) {
+                    byte[] template = zip.readNBytes(51_201);
+                    if (template.length > 51_200) {
+                        throw new AwsException("InvalidActionDeclarationException",
+                                "CloudFormation template exceeds 51200 bytes", 400);
+                    }
+                    return new String(template, StandardCharsets.UTF_8);
+                }
+            }
+        } catch (java.io.IOException exception) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "Could not read CloudFormation template from input artifact", 400);
+        }
+        throw new AwsException("InvalidActionDeclarationException",
+                "TemplatePath does not exist in the input artifact", 400);
+    }
+
+    private record ArtifactLocation(String bucket, String key) {
+    }
+
     private boolean abandonExternalActionIfRequested(CodePipelineExecution execution, ActionExecution state) {
         if (!execution.isStopRequested() || !execution.isAbandon()) {
             return false;
@@ -1518,9 +2051,9 @@ public class CodePipelineService {
 
     private ObjectNode executionNode(CodePipelineExecution execution) {
         ObjectNode node = mapper.valueToTree(execution);
-        node.remove(List.of("accountId", "region", "startTime", "lastUpdateTime",
+        node.remove(List.of("accountId", "region", "startTime", "lastUpdateTime", "artifacts",
                 "sourceRevisions", "actionExecutions", "currentStage", "stopRequested", "abandon",
-                "rollbackTargetPipelineExecutionId"));
+                "rollbackTargetPipelineExecutionId", "retryStageName", "retryActionNames"));
         if (execution.getRollbackTargetPipelineExecutionId() != null) {
             node.putObject("rollbackMetadata").put(
                     "rollbackTargetPipelineExecutionId", execution.getRollbackTargetPipelineExecutionId());
@@ -1728,19 +2261,10 @@ public class CodePipelineService {
         return account + ":" + region + ":" + pipelineName;
     }
 
-    private static String artifactKey(CodePipelineExecution execution, String artifactName) {
-        return execution.getAccountId() + ":" + execution.getPipelineExecutionId() + ":" + artifactName;
-    }
-
     private static String artifactBucket(JsonNode action) {
         return action.path("configuration").path("BucketName").asText("codepipeline-artifacts");
     }
 
-    private void clearRuntimeArtifacts(CodePipelineExecution execution) {
-        String prefix = execution.getAccountId() + ":" + execution.getPipelineExecutionId() + ":";
-        runtimeArtifacts.keySet().removeIf(key -> key.startsWith(prefix));
-    }
-
     private static boolean isTerminal(String status) {
         return List.of("Succeeded", "Failed", "Stopped", "Superseded", "Cancelled").contains(status);
     }

src/main/java/io/github/hectorvent/floci/services/codepipeline/model/CodePipelineExecution.java

@@ -26,6 +26,7 @@ public class CodePipelineExecution {
     private Double startTime;
     private Double lastUpdateTime;
     private List<Map<String, Object>> artifactRevisions = new ArrayList<>();
+    private Map<String, byte[]> artifacts = new LinkedHashMap<>();
     private List<Map<String, Object>> sourceRevisions = new ArrayList<>();
     private List<Map<String, String>> variables = new ArrayList<>();
     private Map<String, String> trigger = new LinkedHashMap<>();
@@ -35,6 +36,8 @@ public class CodePipelineExecution {
     private volatile boolean stopRequested;
     private volatile boolean abandon;
     private String rollbackTargetPipelineExecutionId;
+    private String retryStageName;
+    private List<String> retryActionNames = new ArrayList<>();
 
     public String getAccountId() {
         return accountId;
@@ -132,6 +135,14 @@ public class CodePipelineExecution {
         this.artifactRevisions = artifactRevisions;
     }
 
+    public Map<String, byte[]> getArtifacts() {
+        return artifacts;
+    }
+
+    public void setArtifacts(Map<String, byte[]> artifacts) {
+        this.artifacts = artifacts == null ? new LinkedHashMap<>() : artifacts;
+    }
+
     public List<Map<String, Object>> getSourceRevisions() {
         return sourceRevisions;
     }
@@ -205,6 +216,22 @@ public class CodePipelineExecution {
         this.rollbackTargetPipelineExecutionId = rollbackTargetPipelineExecutionId;
     }
 
+    public String getRetryStageName() {
+        return retryStageName;
+    }
+
+    public void setRetryStageName(String retryStageName) {
+        this.retryStageName = retryStageName;
+    }
+
+    public List<String> getRetryActionNames() {
+        return retryActionNames;
+    }
+
+    public void setRetryActionNames(List<String> retryActionNames) {
+        this.retryActionNames = retryActionNames == null ? new ArrayList<>() : retryActionNames;
+    }
+
     @RegisterForReflection
     @JsonInclude(JsonInclude.Include.NON_NULL)
     public static class ActionExecution {

src/main/java/io/github/hectorvent/floci/services/cognito/CognitoAuthFlowHandler.java

@@ -1363,6 +1363,62 @@ final class CognitoAuthFlowHandler {
         return service.generateAuthResult(user, pool, client, override);
     }
 
+    /**
+     * Checks a password typed into the hosted login page. Same rules and triggers as
+     * USER_PASSWORD_AUTH, minus SECRET_HASH: the page is Cognito's own, not the app client.
+     * Throws {@code NewPasswordRequiredException} when the user still has a temporary password.
+     */
+    CognitoUser authenticateHostedLogin(UserPool pool, UserPoolClient client, String username, String password) {
+        CognitoUser user;
+        try {
+            user = service.adminGetUser(pool.getId(), username);
+        } catch (AwsException ae) {
+            if (!"UserNotFoundException".equals(ae.getErrorCode())) throw ae;
+            user = tryUserMigration(pool, client, username, password, null, Map.of(),
+                    "UserMigration_Authentication");
+            if (user == null) throw new AwsException("NotAuthorizedException", "Incorrect username or password", 400);
+        }
+
+        firePreAuthentication(pool, client, user, null, Map.of(), false);
+
+        if (!user.isEnabled()) throw new AwsException("NotAuthorizedException", "User is disabled", 400);
+        if ("RESET_REQUIRED".equals(user.getUserStatus())) {
+            throw new AwsException("PasswordResetRequiredException", "Password reset required", 400);
+        }
+        if (user.getPasswordHash() == null || !user.getPasswordHash().equals(service.hashPassword(password))) {
+            throw new AwsException("NotAuthorizedException", "Incorrect username or password", 400);
+        }
+        if ("UNCONFIRMED".equals(user.getUserStatus())) {
+            throw new AwsException("UserNotConfirmedException", "User is not confirmed", 400);
+        }
+        if (user.isTemporaryPassword() || "FORCE_CHANGE_PASSWORD".equals(user.getUserStatus())) {
+            throw new AwsException("NewPasswordRequiredException", "Password change required", 400);
+        }
+        firePostAuthentication(pool, client, user, Map.of(), false);
+        return user;
+    }
+
+    /** Tokens for an authorization code from the hosted login or a federated sign-in. */
+    Map<String, Object> issueHostedTokens(UserPool pool, UserPoolClient client, CognitoUser user, String nonce) {
+        CognitoService.ClaimsOverride override = withIdClaim(
+                firePreTokenGeneration(pool, client, user, Map.of(), "TokenGeneration_HostedAuth"), "nonce", nonce);
+        return service.generateAuthResult(user, pool, client, override);
+    }
+
+    private static CognitoService.ClaimsOverride withIdClaim(CognitoService.ClaimsOverride override,
+                                                             String name, String value) {
+        if (value == null || value.isBlank()) return override;
+        CognitoService.ClaimsOverride base = override != null ? override
+                : new CognitoService.ClaimsOverride(null, null, null, null, null, null, null, null, null);
+        Map<String, Object> idClaims = new HashMap<>();
+        if (base.idClaimsToAddOrOverride() != null) idClaims.putAll(base.idClaimsToAddOrOverride());
+        idClaims.put(name, value);
+        return new CognitoService.ClaimsOverride(idClaims, base.idClaimsToSuppress(),
+                base.accessClaimsToAddOrOverride(), base.accessClaimsToSuppress(),
+                base.scopesToAdd(), base.scopesToSuppress(),
+                base.groupsToOverride(), base.iamRolesToOverride(), base.preferredRole());
+    }
+
     CognitoService.ClaimsOverride preTokenGenerationForRefresh(UserPool pool, UserPoolClient client, CognitoUser user) {
         return firePreTokenGeneration(pool, client, user, Map.of(), "TokenGeneration_RefreshTokens");
     }

src/main/java/io/github/hectorvent/floci/services/cognito/CognitoCustomDomainFilter.java

@@ -14,12 +14,13 @@ import org.jboss.logging.Logger;
 
 import java.net.URI;
 import java.util.Optional;
+import java.util.Set;
 
 /**
  * Routes Cognito custom-domain requests by Host. On AWS a custom domain serves
- * {@code https://<domain>/oauth2/token}, {@code /oauth2/userInfo}, {@code /oauth2/authorize}, and
- * {@code /oauth2/idpresponse}; Floci maps those onto
- * the {@code /cognito-idp/oauth2/...} handlers and pins the pool and the account that own the
+ * {@code https://<domain>/oauth2/token}, {@code /oauth2/userInfo}, {@code /oauth2/authorize},
+ * {@code /oauth2/idpresponse} and the hosted {@code /login} and {@code /logout} pages; Floci maps
+ * those onto the {@code /cognito-idp/...} handlers and pins the pool and the account that own the
  * domain, since the request itself carries no AWS credential.
  */
 @Provider
@@ -37,6 +38,8 @@ public class CognitoCustomDomainFilter implements ContainerRequestFilter {
     private static final Logger LOG = Logger.getLogger(CognitoCustomDomainFilter.class);
     private static final String OAUTH_PREFIX = "/oauth2/";
     private static final String TARGET_PREFIX = "/cognito-idp/oauth2/";
+    private static final Set<String> HOSTED_PAGES = Set.of("/login", "/logout");
+    private static final String HOSTED_TARGET_PREFIX = "/cognito-idp/";
 
     private final CognitoService cognitoService;
 
@@ -49,7 +52,11 @@ public class CognitoCustomDomainFilter implements ContainerRequestFilter {
     public void filter(ContainerRequestContext requestContext) {
         URI originalUri = requestContext.getUriInfo().getRequestUri();
         String path = originalUri.getRawPath();
-        if (path == null || !path.startsWith(OAUTH_PREFIX)) {
+        if (path == null) {
+            return;
+        }
+        boolean hostedPage = HOSTED_PAGES.contains(path);
+        if (!path.startsWith(OAUTH_PREFIX) && !hostedPage) {
             return;
         }
         String host = RequestHost.of(requestContext);
@@ -62,7 +69,8 @@ public class CognitoCustomDomainFilter implements ContainerRequestFilter {
         }
 
         URI newUri = UriBuilder.fromUri(originalUri)
-                .replacePath(TARGET_PREFIX + path.substring(OAUTH_PREFIX.length()))
+                .replacePath(hostedPage ? HOSTED_TARGET_PREFIX + path.substring(1)
+                        : TARGET_PREFIX + path.substring(OAUTH_PREFIX.length()))
                 .build();
         LOG.debugv("Cognito custom domain routing: {0}{1} -> {2}", host, path, newUri.getPath());
         requestContext.setProperty(POOL_PROPERTY, domain.get().getUserPoolId());

src/main/java/io/github/hectorvent/floci/services/cognito/CognitoFederationService.java

@@ -47,6 +47,12 @@ public class CognitoFederationService {
 
     public String beginAuthorization(String userPoolId, String clientId, String redirectUri, List<String> scopes,
                                      String nonce, String providerName, String relyingPartyState) {
+        return beginAuthorization(userPoolId, clientId, redirectUri, scopes, nonce, providerName, relyingPartyState, null);
+    }
+
+    public String beginAuthorization(String userPoolId, String clientId, String redirectUri, List<String> scopes,
+                                     String nonce, String providerName, String relyingPartyState,
+                                     String codeChallenge) {
         IdentityProvider provider = cognitoService.describeIdentityProvider(userPoolId, providerName);
         requireOidcProvider(provider);
         String authorizeEndpoint = requiredProviderDetail(provider, "authorize_url", "authorize endpoint");
@@ -54,7 +60,8 @@ public class CognitoFederationService {
         validateAuthorizeEndpoint(authorizeEndpoint, provider.getProviderName());
         Instant expiresAt = clock.instant().plus(TRANSACTION_LIFETIME);
         CognitoAuthorizationTransaction transaction = new CognitoAuthorizationTransaction(
-                userPoolId, clientId, redirectUri, scopes, nonce, providerName, relyingPartyState, expiresAt);
+                userPoolId, clientId, redirectUri, scopes, nonce, providerName, relyingPartyState, expiresAt,
+                codeChallenge);
         String state = stateStore.putTransaction(transaction);
 
         Map<String, String> parameters = new LinkedHashMap<>();
@@ -89,7 +96,8 @@ public class CognitoFederationService {
                 subject, issuer, mapAttributes(provider, claims));
         CognitoAuthorizationCode authorizationCode = new CognitoAuthorizationCode(
                 transaction.userPoolId(), transaction.clientId(), user.getUsername(), transaction.redirectUri(),
-                transaction.scopes(), clock.instant().plus(AUTHORIZATION_CODE_LIFETIME));
+                transaction.scopes(), clock.instant().plus(AUTHORIZATION_CODE_LIFETIME), transaction.nonce(),
+                transaction.codeChallenge());
         return stateStore.putAuthorizationCode(authorizationCode);
     }
 

src/main/java/io/github/hectorvent/floci/services/cognito/CognitoFederationStateStore.java

@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.cognito;
 
 import io.github.hectorvent.floci.services.cognito.model.CognitoAuthorizationCode;
 import io.github.hectorvent.floci.services.cognito.model.CognitoAuthorizationTransaction;
+import io.github.hectorvent.floci.services.cognito.model.CognitoHostedSession;
 import jakarta.enterprise.context.ApplicationScoped;
 
 import java.security.SecureRandom;
@@ -20,6 +21,7 @@ public class CognitoFederationStateStore {
 
     private final ConcurrentHashMap<String, CognitoAuthorizationTransaction> transactions = new ConcurrentHashMap<>();
     private final ConcurrentHashMap<String, CognitoAuthorizationCode> authorizationCodes = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<String, CognitoHostedSession> hostedSessions = new ConcurrentHashMap<>();
     private final Clock clock;
     private final SecureRandom secureRandom = new SecureRandom();
 
@@ -55,6 +57,31 @@ public class CognitoFederationStateStore {
         return Optional.of(authorizationCode);
     }
 
+    public String putHostedSession(CognitoHostedSession session) {
+        return put(hostedSessions, session);
+    }
+
+    public Optional<CognitoHostedSession> findHostedSession(String id) {
+        if (id == null) {
+            return Optional.empty();
+        }
+        CognitoHostedSession session = hostedSessions.get(id);
+        if (session == null) {
+            return Optional.empty();
+        }
+        if (isExpired(session.expiresAt())) {
+            hostedSessions.remove(id, session);
+            return Optional.empty();
+        }
+        return Optional.of(session);
+    }
+
+    public void removeHostedSession(String id) {
+        if (id != null) {
+            hostedSessions.remove(id);
+        }
+    }
+
     private <T> String put(ConcurrentHashMap<String, T> store, T value) {
         String key;
         do {

src/main/java/io/github/hectorvent/floci/services/cognito/CognitoHostedLoginController.java

@@ -0,0 +1,502 @@
+package io.github.hectorvent.floci.services.cognito;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.cognito.model.CognitoAuthorizationCode;
+import io.github.hectorvent.floci.services.cognito.model.CognitoHostedSession;
+import io.github.hectorvent.floci.services.cognito.model.CognitoUser;
+import io.github.hectorvent.floci.services.cognito.model.UserPool;
+import io.github.hectorvent.floci.services.cognito.model.UserPoolClient;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.CookieParam;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.QueryParam;
+import jakarta.ws.rs.container.ContainerRequestContext;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.MultivaluedMap;
+import jakarta.ws.rs.core.NewCookie;
+import jakarta.ws.rs.core.Response;
+
+import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.SecureRandom;
+import java.time.Clock;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Cognito's hosted (managed) login pages: {@code /login} and {@code /logout}, reached from
+ * {@code /oauth2/authorize} when no external identity provider is named. A successful sign-in
+ * starts a browser session and redirects to the client's callback URL with an authorization
+ * code, which {@code /oauth2/token} redeems.
+ */
+@ApplicationScoped
+@Path("/")
+public class CognitoHostedLoginController {
+
+    static final String SESSION_COOKIE = "cognito";
+    static final String CSRF_COOKIE = "XSRF-TOKEN";
+    private static final Duration SESSION_LIFETIME = Duration.ofHours(1);
+    private static final Duration AUTHORIZATION_CODE_LIFETIME = Duration.ofMinutes(5);
+    private static final String COGNITO_PROVIDER = "COGNITO";
+    private static final String NO_PROVIDERS = "This app client has no identity providers to sign in with.";
+
+    private final CognitoService cognitoService;
+    private final CognitoFederationStateStore stateStore;
+    private final Clock clock;
+    private final SecureRandom secureRandom = new SecureRandom();
+
+    @Inject
+    public CognitoHostedLoginController(CognitoService cognitoService, CognitoFederationStateStore stateStore,
+                                        Clock clock) {
+        this.cognitoService = cognitoService;
+        this.stateStore = stateStore;
+        this.clock = clock;
+    }
+
+    /** The parameters of an authorize request, carried through the login form unchanged. */
+    record AuthorizeRequest(String clientId, String redirectUri, String scope, String state, String nonce,
+                            String codeChallenge, String codeChallengeMethod) {
+
+        Map<String, String> asQuery() {
+            Map<String, String> query = new LinkedHashMap<>();
+            query.put("client_id", clientId);
+            query.put("redirect_uri", redirectUri);
+            query.put("response_type", "code");
+            putIfPresent(query, "scope", scope);
+            putIfPresent(query, "state", state);
+            putIfPresent(query, "nonce", nonce);
+            putIfPresent(query, "code_challenge", codeChallenge);
+            putIfPresent(query, "code_challenge_method", codeChallengeMethod);
+            return query;
+        }
+    }
+
+    /**
+     * Continues an authorize request that names no external provider: an existing session gets a
+     * code straight away, anyone else is sent to the login page, which offers the pool's own
+     * password form and a button for each of the client's other identity providers.
+     */
+    Response authorizeWithCognito(ContainerRequestContext requestContext, UserPoolClient client,
+                                  AuthorizeRequest request, String prompt, String sessionId) {
+        if (!offersCognito(client) && externalProviders(client).isEmpty()) {
+            return errorPage(NO_PROVIDERS);
+        }
+        if (offersCognito(client) && !"login".equals(prompt)) {
+            Optional<CognitoUser> signedIn = sessionUser(client, sessionId);
+            if (signedIn.isPresent()) {
+                return redirectWithCode(client, request, signedIn.get(), null);
+            }
+        }
+        return Response.status(Response.Status.FOUND)
+                .location(URI.create(pagePath(requestContext, "login") + "?" + query(request.asQuery())))
+                .build();
+    }
+
+    @GET
+    @Path("/cognito-idp/login")
+    public Response loginPage(@Context ContainerRequestContext requestContext,
+                              @QueryParam("client_id") String clientId,
+                              @QueryParam("redirect_uri") String redirectUri,
+                              @QueryParam("response_type") String responseType,
+                              @QueryParam("scope") String scope,
+                              @QueryParam("state") String state,
+                              @QueryParam("nonce") String nonce,
+                              @QueryParam("code_challenge") String codeChallenge,
+                              @QueryParam("code_challenge_method") String codeChallengeMethod) {
+        AuthorizeRequest request = new AuthorizeRequest(trimToNull(clientId), trimToNull(redirectUri),
+                trimToNull(scope), state, trimToNull(nonce), trimToNull(codeChallenge), trimToNull(codeChallengeMethod));
+        Object validated = validate(requestContext, request, responseType);
+        if (validated instanceof Response error) {
+            return error;
+        }
+        String csrf = randomToken();
+        return loginForm(requestContext, (UserPoolClient) validated, request, csrf, null, null, 200)
+                .cookie(cookie(requestContext, CSRF_COOKIE, csrf, -1))
+                .build();
+    }
+
+    @POST
+    @Path("/cognito-idp/login")
+    @Consumes(MediaType.APPLICATION_FORM_URLENCODED)
+    public Response submitLogin(@Context ContainerRequestContext requestContext,
+                                @CookieParam(CSRF_COOKIE) String csrfCookie,
+                                MultivaluedMap<String, String> form) {
+        AuthorizeRequest request = new AuthorizeRequest(trimToNull(form.getFirst("client_id")),
+                trimToNull(form.getFirst("redirect_uri")), trimToNull(form.getFirst("scope")),
+                form.getFirst("state"), trimToNull(form.getFirst("nonce")),
+                trimToNull(form.getFirst("code_challenge")), trimToNull(form.getFirst("code_challenge_method")));
+        Object validated = validate(requestContext, request, form.getFirst("response_type"));
+        if (validated instanceof Response error) {
+            return error;
+        }
+        UserPoolClient client = (UserPoolClient) validated;
+        if (!offersCognito(client)) {
+            return errorPage("This app client does not offer sign-in with a user name and password "
+                    + "(COGNITO is not one of its identity providers).");
+        }
+        String csrf = form.getFirst("_csrf");
+        if (csrf == null || csrfCookie == null || !constantTimeEquals(csrf, csrfCookie)) {
+            return errorPage("Your sign-in form expired. Go back to the app and sign in again.");
+        }
+
+        String username = trimToNull(form.getFirst("username"));
+        String password = form.getFirst("password");
+        if (username == null || password == null || password.isEmpty()) {
+            return loginForm(requestContext, client, request, csrf, username, "Enter your user name and password.", 400).build();
+        }
+
+        CognitoUser user;
+        try {
+            user = cognitoService.authenticateHostedLogin(client, username, password);
+        } catch (AwsException e) {
+            return loginForm(requestContext, client, request, csrf, username, describe(e), 400).build();
+        }
+
+        String sessionId = stateStore.putHostedSession(new CognitoHostedSession(
+                client.getUserPoolId(), user.getUsername(), clock.instant().plus(SESSION_LIFETIME)));
+        return redirectWithCode(client, request, user,
+                cookie(requestContext, SESSION_COOKIE, sessionId, (int) SESSION_LIFETIME.toSeconds()));
+    }
+
+    @GET
+    @Path("/cognito-idp/logout")
+    public Response logout(@Context ContainerRequestContext requestContext,
+                           @CookieParam(SESSION_COOKIE) String sessionId,
+                           @QueryParam("client_id") String clientId,
+                           @QueryParam("logout_uri") String logoutUri,
+                           @QueryParam("redirect_uri") String redirectUri,
+                           @QueryParam("response_type") String responseType,
+                           @QueryParam("scope") String scope,
+                           @QueryParam("state") String state) {
+        UserPoolClient client = findClient(requestContext, trimToNull(clientId));
+        if (client == null) {
+            return errorPage("The app client was not found.");
+        }
+
+        String location;
+        if (trimToNull(logoutUri) != null) {
+            if (!client.getLogoutURLs().contains(logoutUri.trim())) {
+                return errorPage("This sign-out URL is not registered for the app client.");
+            }
+            location = logoutUri.trim();
+        } else {
+            AuthorizeRequest request = new AuthorizeRequest(client.getClientId(), trimToNull(redirectUri),
+                    trimToNull(scope), state, null, null, null);
+            Object validated = validate(requestContext, request, responseType);
+            if (validated instanceof Response error) {
+                return error;
+            }
+            location = pagePath(requestContext, "login") + "?" + query(request.asQuery());
+        }
+
+        stateStore.removeHostedSession(sessionId);
+        return Response.status(Response.Status.FOUND)
+                .location(URI.create(location))
+                .cookie(cookie(requestContext, SESSION_COOKIE, "", 0))
+                .build();
+    }
+
+    private Optional<CognitoUser> sessionUser(UserPoolClient client, String sessionId) {
+        Optional<CognitoHostedSession> session = stateStore.findHostedSession(sessionId)
+                .filter(s -> s.userPoolId().equals(client.getUserPoolId()));
+        if (session.isEmpty()) {
+            return Optional.empty();
+        }
+        try {
+            CognitoUser user = cognitoService.adminGetUser(client.getUserPoolId(), session.get().username());
+            return user.isEnabled() ? Optional.of(user) : Optional.empty();
+        } catch (AwsException e) {
+            return Optional.empty();
+        }
+    }
+
+    private Response redirectWithCode(UserPoolClient client, AuthorizeRequest request, CognitoUser user,
+                                      NewCookie sessionCookie) {
+        String code = stateStore.putAuthorizationCode(new CognitoAuthorizationCode(
+                client.getUserPoolId(), client.getClientId(), user.getUsername(), request.redirectUri(),
+                splitScopes(request.scope()), clock.instant().plus(AUTHORIZATION_CODE_LIFETIME),
+                request.nonce(), request.codeChallenge()));
+        Map<String, String> parameters = new LinkedHashMap<>();
+        parameters.put("code", code);
+        putIfPresent(parameters, "state", request.state());
+        String redirectUri = request.redirectUri();
+        Response.ResponseBuilder response = Response.status(Response.Status.FOUND)
+                .location(URI.create(redirectUri + (redirectUri.contains("?") ? "&" : "?") + query(parameters)));
+        if (sessionCookie != null) {
+            response.cookie(sessionCookie);
+        }
+        return response.build();
+    }
+
+    /** Returns the client when the request is one Cognito would accept, otherwise an error page. */
+    private Object validate(ContainerRequestContext requestContext, AuthorizeRequest request, String responseType) {
+        if (!"code".equals(trimToNull(responseType))) {
+            return errorPage("Only response_type=code is supported.");
+        }
+        if (request.clientId() == null || request.redirectUri() == null) {
+            return errorPage("client_id and redirect_uri are required.");
+        }
+        UserPoolClient client = findClient(requestContext, request.clientId());
+        if (client == null) {
+            return errorPage("The app client was not found.");
+        }
+        if (!client.isAllowedOAuthFlowsUserPoolClient() || !client.getAllowedOAuthFlows().contains("code")) {
+            return errorPage("This app client is not allowed to use the authorization code flow.");
+        }
+        if (!client.getCallbackURLs().contains(request.redirectUri())) {
+            return errorPage("redirect_uri is not registered for this app client.");
+        }
+        if (!offersCognito(client) && externalProviders(client).isEmpty()) {
+            return errorPage(NO_PROVIDERS);
+        }
+        String pkceError = pkceError(request.codeChallenge(), request.codeChallengeMethod());
+        if (pkceError != null) {
+            return errorPage(pkceError);
+        }
+        return client;
+    }
+
+    private static boolean offersCognito(UserPoolClient client) {
+        return client.getSupportedIdentityProviders().contains(COGNITO_PROVIDER);
+    }
+
+    private static List<String> externalProviders(UserPoolClient client) {
+        return client.getSupportedIdentityProviders().stream()
+                .filter(provider -> !COGNITO_PROVIDER.equals(provider))
+                .toList();
+    }
+
+    static String pkceError(String codeChallenge, String codeChallengeMethod) {
+        if (codeChallenge == null) {
+            return codeChallengeMethod == null ? null : "code_challenge_method was sent without code_challenge.";
+        }
+        if (!"S256".equals(codeChallengeMethod)) {
+            return "code_challenge_method must be S256.";
+        }
+        return null;
+    }
+
+    private UserPoolClient findClient(ContainerRequestContext requestContext, String clientId) {
+        if (clientId == null) {
+            return null;
+        }
+        try {
+            UserPoolClient client = cognitoService.findClientById(clientId);
+            String domainPoolId = (String) requestContext.getProperty(CognitoCustomDomainFilter.POOL_PROPERTY);
+            return domainPoolId == null || domainPoolId.equals(client.getUserPoolId()) ? client : null;
+        } catch (AwsException e) {
+            return null;
+        }
+    }
+
+    private static String describe(AwsException e) {
+        return switch (e.getErrorCode()) {
+            case "NotAuthorizedException" -> "User is disabled".equals(e.getMessage())
+                    ? "This account is disabled." : "Incorrect user name or password.";
+            case "UserNotConfirmedException" -> "This account is not confirmed yet. Confirm it from the app first.";
+            case "PasswordResetRequiredException" -> "This account needs a password reset before it can sign in.";
+            case "NewPasswordRequiredException" -> "This account still has a temporary password. "
+                    + "Set a permanent one before signing in here.";
+            case "UserLambdaValidationException" -> e.getMessage();
+            default -> "Sign-in failed: " + e.getMessage();
+        };
+    }
+
+    private Response.ResponseBuilder loginForm(ContainerRequestContext requestContext, UserPoolClient client,
+                                               AuthorizeRequest request, String csrf, String username,
+                                               String error, int status) {
+        UserPool pool = cognitoService.describeUserPool(client.getUserPoolId());
+        boolean email = pool.getUsernameAttributes() != null && pool.getUsernameAttributes().contains("email");
+        StringBuilder hidden = new StringBuilder();
+        for (Map.Entry<String, String> field : request.asQuery().entrySet()) {
+            hidden.append("<input type=\"hidden\" name=\"").append(escape(field.getKey()))
+                    .append("\" value=\"").append(escape(field.getValue())).append("\">");
+        }
+        hidden.append("<input type=\"hidden\" name=\"_csrf\" value=\"").append(escape(csrf)).append("\">");
+        String appName = client.getClientName() == null || client.getClientName().isBlank()
+                ? pool.getName() : client.getClientName();
+        // Each external provider starts the federated flow with the same authorize parameters.
+        StringBuilder providers = new StringBuilder();
+        for (String provider : externalProviders(client)) {
+            Map<String, String> query = new LinkedHashMap<>(request.asQuery());
+            query.put("identity_provider", provider);
+            providers.append("<a class=\"provider\" href=\"").append(escape(authorizePath(requestContext) + "?" + query(query)))
+                    .append("\">Continue with ").append(escape(provider)).append("</a>");
+        }
+        String passwordForm = !offersCognito(client) ? "" : """
+                <form method="post" action="">
+                  %s
+                  <label>%s<input name="username" type="%s" autocomplete="username" value="%s" required autofocus></label>
+                  <label>Password<input name="password" type="password" autocomplete="current-password" required></label>
+                  <button type="submit">Sign in</button>
+                </form>
+                """.formatted(hidden, email ? "Email" : "User name", email ? "email" : "text",
+                escape(username == null ? "" : username));
+        String divider = providers.isEmpty() || passwordForm.isEmpty() ? "" : "<p class=\"divider\"><span>or</span></p>";
+        String body = """
+                <h1>Sign in</h1>
+                <p class="muted">to continue to <strong>%s</strong></p>
+                %s
+                %s%s<div class="providers">%s</div>
+                """.formatted(escape(appName),
+                error == null ? "" : "<p class=\"error\" role=\"alert\">" + escape(error) + "</p>",
+                passwordForm, divider, providers);
+        return Response.status(status).type(MediaType.TEXT_HTML_TYPE.withCharset("UTF-8"))
+                .entity(page("Sign in", body))
+                .header("Cache-Control", "no-store")
+                .header("X-Frame-Options", "DENY")
+                .header("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; form-action 'self' "
+                        + cspSource(request.redirectUri()) + "; frame-ancestors 'none'");
+    }
+
+    private static String cspSource(String redirectUri) {
+        URI uri = URI.create(redirectUri);
+        return uri.getScheme() + "://" + uri.getRawAuthority();
+    }
+
+    private Response errorPage(String message) {
+        return Response.status(400).type(MediaType.TEXT_HTML_TYPE.withCharset("UTF-8"))
+                .entity(page("Sign-in error", "<h1>Something went wrong</h1><p class=\"error\" role=\"alert\">"
+                        + escape(message) + "</p>"))
+                .header("Cache-Control", "no-store")
+                .header("X-Frame-Options", "DENY")
+                .header("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'")
+                .build();
+    }
+
+    private static String page(String title, String body) {
+        return """
+                <!doctype html>
+                <html lang="en">
+                <head>
+                <meta charset="utf-8">
+                <meta name="viewport" content="width=device-width, initial-scale=1">
+                <title>%s</title>
+                <style>
+                  :root { --bg:#f4f5f7; --card:#fff; --text:#17191c; --muted:#5d6470; --border:#d9dde3; --accent:#2f5fd0; --on-accent:#fff; --error:#b42318; --error-bg:#fdeceb; }
+                  @media (prefers-color-scheme: dark) { :root { --bg:#101215; --card:#181b20; --text:#e8eaed; --muted:#9aa1ac; --border:#2c3139; --accent:#7c9ef0; --on-accent:#0b1020; --error:#f08b82; --error-bg:#3a1714; } }
+                  * { box-sizing: border-box; }
+                  body { margin:0; min-height:100vh; display:grid; place-items:center; padding:24px 16px; background:var(--bg); color:var(--text); font:15px/1.5 system-ui, -apple-system, "Segoe UI", sans-serif; }
+                  main { width:100%%; max-width:380px; background:var(--card); border:1px solid var(--border); border-radius:10px; padding:28px; }
+                  h1 { margin:0 0 4px; font-size:22px; }
+                  .muted { color:var(--muted); margin:0 0 20px; overflow-wrap:anywhere; }
+                  form { display:grid; gap:14px; }
+                  label { display:grid; gap:6px; font-weight:600; font-size:14px; }
+                  input { font:inherit; padding:10px 12px; border:1px solid var(--border); border-radius:6px; background:var(--bg); color:var(--text); }
+                  input:focus { outline:2px solid var(--accent); outline-offset:1px; }
+                  button { font:inherit; font-weight:600; padding:10px 12px; border:0; border-radius:6px; background:var(--accent); color:var(--on-accent); cursor:pointer; }
+                  .providers { display:grid; gap:10px; }
+                  .provider { display:block; padding:10px 12px; border:1px solid var(--border); border-radius:6px; color:var(--text); font-weight:600; text-align:center; text-decoration:none; overflow-wrap:anywhere; }
+                  .provider:hover { border-color:var(--accent); color:var(--accent); }
+                  .divider { display:flex; align-items:center; gap:10px; margin:16px 0; color:var(--muted); font-size:13px; }
+                  .divider::before, .divider::after { content:""; flex:1; border-top:1px solid var(--border); }
+                  .error { margin:0 0 16px; padding:10px 12px; border-radius:6px; background:var(--error-bg); color:var(--error); overflow-wrap:anywhere; }
+                  footer { margin-top:20px; color:var(--muted); font-size:12px; text-align:center; }
+                </style>
+                </head>
+                <body><main>%s<footer>Cognito-compatible sign-in · local emulator</footer></main></body>
+                </html>
+                """.formatted(escape(title), body);
+    }
+
+    private NewCookie cookie(ContainerRequestContext requestContext, String name, String value, int maxAge) {
+        return new NewCookie.Builder(name).value(value).path("/").maxAge(maxAge)
+                .httpOnly(true).secure(isHttps(requestContext)).sameSite(NewCookie.SameSite.LAX).build();
+    }
+
+    private static boolean isHttps(ContainerRequestContext requestContext) {
+        String forwarded = requestContext.getHeaderString("X-Forwarded-Proto");
+        if (forwarded != null) {
+            return "https".equalsIgnoreCase(forwarded.split(",")[0].trim());
+        }
+        return "https".equalsIgnoreCase(requestContext.getUriInfo().getRequestUri().getScheme());
+    }
+
+    private static String authorizePath(ContainerRequestContext requestContext) {
+        boolean customDomain = requestContext.getProperty(CognitoCustomDomainFilter.POOL_PROPERTY) != null;
+        return customDomain ? "/oauth2/authorize" : "/cognito-idp/oauth2/authorize";
+    }
+
+    /** Custom domains serve the pages at /login and /logout; everything else under /cognito-idp/. */
+    private static String pagePath(ContainerRequestContext requestContext, String page) {
+        boolean customDomain = requestContext.getProperty(CognitoCustomDomainFilter.POOL_PROPERTY) != null;
+        return (customDomain ? "/" : "/cognito-idp/") + page;
+    }
+
+    private String randomToken() {
+        byte[] bytes = new byte[32];
+        secureRandom.nextBytes(bytes);
+        return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes);
+    }
+
+    private static boolean constantTimeEquals(String a, String b) {
+        return MessageDigest.isEqual(a.getBytes(StandardCharsets.UTF_8), b.getBytes(StandardCharsets.UTF_8));
+    }
+
+    static String escape(String value) {
+        StringBuilder escaped = new StringBuilder(value.length());
+        for (char c : value.toCharArray()) {
+            switch (c) {
+                case '&' -> escaped.append("&amp;");
+                case '<' -> escaped.append("&lt;");
+                case '>' -> escaped.append("&gt;");
+                case '"' -> escaped.append("&quot;");
+                case '\'' -> escaped.append("&#39;");
+                default -> escaped.append(c);
+            }
+        }
+        return escaped.toString();
+    }
+
+    private static String query(Map<String, String> parameters) {
+        List<String> pairs = new ArrayList<>();
+        for (Map.Entry<String, String> parameter : parameters.entrySet()) {
+            pairs.add(encode(parameter.getKey()) + "=" + encode(parameter.getValue()));
+        }
+        return String.join("&", pairs);
+    }
+
+    /** Query-string encoding with spaces as %20, which every URL parser reads back as a space. */
+    private static String encode(String value) {
+        return URLEncoder.encode(value, StandardCharsets.UTF_8).replace("+", "%20");
+    }
+
+    private static void putIfPresent(Map<String, String> map, String key, String value) {
+        if (value != null) {
+            map.put(key, value);
+        }
+    }
+
+    private static List<String> splitScopes(String scope) {
+        if (scope == null) {
+            return List.of();
+        }
+        List<String> scopes = new ArrayList<>();
+        for (String value : scope.trim().split("\\s+")) {
+            if (!value.isBlank()) {
+                scopes.add(value);
+            }
+        }
+        return scopes;
+    }
+
+    private static String trimToNull(String value) {
+        if (value == null) {
+            return null;
+        }
+        String trimmed = value.trim();
+        return trimmed.isEmpty() ? null : trimmed;
+    }
+}

src/main/java/io/github/hectorvent/floci/services/cognito/CognitoOAuthController.java

@@ -6,11 +6,11 @@ import io.github.hectorvent.floci.core.common.AwsException;
 import io.github.hectorvent.floci.services.cognito.model.CognitoAuthorizationCode;
 import io.github.hectorvent.floci.services.cognito.model.CognitoAuthorizationTransaction;
 import io.github.hectorvent.floci.services.cognito.model.CognitoUser;
-import io.github.hectorvent.floci.services.cognito.model.UserPool;
 import io.github.hectorvent.floci.services.cognito.model.UserPoolClient;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.CookieParam;
 import jakarta.ws.rs.GET;
 import jakarta.ws.rs.HeaderParam;
 import jakarta.ws.rs.POST;
@@ -46,17 +46,37 @@ public class CognitoOAuthController {
     private final ObjectMapper objectMapper;
     private final CognitoFederationService federationService;
     private final CognitoFederationStateStore stateStore;
+    private final CognitoHostedLoginController hostedLogin;
 
     @Inject
     public CognitoOAuthController(CognitoService cognitoService, ObjectMapper objectMapper,
                                   CognitoFederationService federationService,
-                                  CognitoFederationStateStore stateStore) {
+                                  CognitoFederationStateStore stateStore,
+                                  CognitoHostedLoginController hostedLogin) {
         this.cognitoService = cognitoService;
         this.objectMapper = objectMapper;
         this.federationService = federationService;
         this.stateStore = stateStore;
+        this.hostedLogin = hostedLogin;
     }
 
+    public CognitoOAuthController(CognitoService cognitoService, ObjectMapper objectMapper,
+                                  CognitoFederationService federationService,
+                                  CognitoFederationStateStore stateStore) {
+        this(cognitoService, objectMapper, federationService, stateStore, null);
+    }
+
+    public Response authorize(ContainerRequestContext requestContext, String clientId, String redirectUri,
+                              String responseType, String scope, String nonce, String providerName,
+                              String relyingPartyState) {
+        return authorize(requestContext, clientId, redirectUri, responseType, scope, nonce, providerName,
+                relyingPartyState, null, null, null, null);
+    }
+
+    /**
+     * Without {@code identity_provider} (or with {@code COGNITO}) the pool's own users sign in on
+     * the hosted login page; any other provider name starts a federated sign-in.
+     */
     @GET
     @Path("/cognito-idp/oauth2/authorize")
     public Response authorize(@Context ContainerRequestContext requestContext,
@@ -66,13 +86,22 @@ public class CognitoOAuthController {
                               @QueryParam("scope") String scope,
                               @QueryParam("nonce") String nonce,
                               @QueryParam("identity_provider") String providerName,
-                              @QueryParam("state") String relyingPartyState) {
+                              @QueryParam("state") String relyingPartyState,
+                              @QueryParam("code_challenge") String codeChallenge,
+                              @QueryParam("code_challenge_method") String codeChallengeMethod,
+                              @QueryParam("prompt") String prompt,
+                              @CookieParam(CognitoHostedLoginController.SESSION_COOKIE) String sessionId) {
         if (!"code".equals(trimToNull(responseType))) {
             return oauthError("unsupported_response_type", "Only response_type=code is supported");
         }
-        if (trimToNull(clientId) == null || trimToNull(redirectUri) == null || trimToNull(providerName) == null) {
+        boolean hosted = trimToNull(providerName) == null || "COGNITO".equals(providerName.trim());
+        if (trimToNull(clientId) == null || trimToNull(redirectUri) == null || (hosted && hostedLogin == null)) {
             return oauthError("invalid_request", "client_id, redirect_uri, and identity_provider are required");
         }
+        String pkceError = CognitoHostedLoginController.pkceError(trimToNull(codeChallenge), trimToNull(codeChallengeMethod));
+        if (pkceError != null) {
+            return oauthError("invalid_request", pkceError);
+        }
 
         UserPoolClient client;
         try {
@@ -91,9 +120,20 @@ public class CognitoOAuthController {
             return oauthError("invalid_request", "redirect_uri is not registered for this client");
         }
 
+        if (hosted) {
+            return hostedLogin.authorizeWithCognito(requestContext, client,
+                    new CognitoHostedLoginController.AuthorizeRequest(clientId.trim(), redirectUri, trimToNull(scope),
+                            relyingPartyState, trimToNull(nonce), trimToNull(codeChallenge),
+                            trimToNull(codeChallengeMethod)),
+                    trimToNull(prompt), sessionId);
+        }
+
         try {
-            String location = federationService.beginAuthorization(client.getUserPoolId(), clientId, redirectUri,
-                    splitScopes(scope), trimToNull(nonce), providerName, relyingPartyState);
+            String location = trimToNull(codeChallenge) == null
+                    ? federationService.beginAuthorization(client.getUserPoolId(), clientId, redirectUri,
+                    splitScopes(scope), trimToNull(nonce), providerName, relyingPartyState)
+                    : federationService.beginAuthorization(client.getUserPoolId(), clientId, redirectUri,
+                    splitScopes(scope), trimToNull(nonce), providerName, relyingPartyState, codeChallenge.trim());
             return Response.status(Response.Status.FOUND).location(URI.create(location)).build();
         } catch (AwsException e) {
             return oauthError("invalid_request", e.getMessage());
@@ -185,8 +225,12 @@ public class CognitoOAuthController {
         if ("authorization_code".equals(grantType)) {
             return redeemAuthorizationCode(clientId, clientSecret, formParams, domainPoolId);
         }
+        if ("refresh_token".equals(grantType)) {
+            return refreshTokens(clientId, clientSecret, formParams, domainPoolId);
+        }
         if (!"client_credentials".equals(grantType)) {
-            return oauthError("unsupported_grant_type", "Only client_credentials and authorization_code are supported");
+            return oauthError("unsupported_grant_type",
+                    "Only client_credentials, authorization_code and refresh_token are supported");
         }
         String scope = trimToNull(formParams.getFirst("scope"));
 
@@ -249,10 +293,12 @@ public class CognitoOAuthController {
             return oauthError("invalid_grant", "Authorization code is invalid or has expired");
         }
         CognitoAuthorizationCode consumedCode = consumed.get();
+        if (!pkceVerified(consumedCode.codeChallenge(), trimToNull(formParams.getFirst("code_verifier")))) {
+            return oauthError("invalid_grant", "code_verifier does not match the code_challenge");
+        }
         try {
-            UserPool pool = cognitoService.describeUserPool(consumedCode.userPoolId());
             CognitoUser user = cognitoService.adminGetUser(consumedCode.userPoolId(), consumedCode.userId());
-            Map<String, Object> authentication = cognitoService.generateAuthResult(user, pool, client, null);
+            Map<String, Object> authentication = cognitoService.issueHostedTokens(client, user, consumedCode.nonce());
             ObjectNode body = objectMapper.createObjectNode();
             body.put("access_token", (String) authentication.get("AccessToken"));
             body.put("id_token", (String) authentication.get("IdToken"));
@@ -267,6 +313,67 @@ public class CognitoOAuthController {
         }
     }
 
+    private Response refreshTokens(String clientId, String clientSecret, MultivaluedMap<String, String> formParams,
+                                   String domainPoolId) {
+        String refreshToken = trimToNull(formParams.getFirst("refresh_token"));
+        if (refreshToken == null) {
+            return oauthError("invalid_request", "refresh_token is required");
+        }
+        UserPoolClient client;
+        try {
+            client = cognitoService.findClientById(clientId);
+        } catch (AwsException e) {
+            return oauthError("invalid_client", "Client not found");
+        }
+        if (domainPoolId != null && !domainPoolId.equals(client.getUserPoolId())) {
+            return oauthError("invalid_client", "Client not found");
+        }
+        if (client.getClientSecret() != null && !client.getClientSecret().isBlank()
+                && !secretsEqual(client.getClientSecret(), clientSecret)) {
+            return oauthError("invalid_client", "Client secret is invalid");
+        }
+        try {
+            Map<String, Object> result = cognitoService.initiateAuth(clientId, "REFRESH_TOKEN_AUTH",
+                    Map.of("REFRESH_TOKEN", refreshToken));
+            @SuppressWarnings("unchecked")
+            Map<String, Object> authentication = (Map<String, Object>) result.get("AuthenticationResult");
+            ObjectNode body = objectMapper.createObjectNode();
+            body.put("access_token", (String) authentication.get("AccessToken"));
+            body.put("id_token", (String) authentication.get("IdToken"));
+            body.put("expires_in", ((Number) authentication.get("ExpiresIn")).longValue());
+            body.put("token_type", (String) authentication.get("TokenType"));
+            return Response.ok(body).type(MediaType.APPLICATION_JSON)
+                    .header("Cache-Control", "no-store").header("Pragma", "no-cache").build();
+        } catch (AwsException e) {
+            if ("NotAuthorizedException".equals(e.getErrorCode())) {
+                return oauthError("invalid_grant", e.getMessage());
+            }
+            if ("InvalidParameterException".equals(e.getErrorCode())) {
+                return oauthError("unauthorized_client", e.getMessage());
+            }
+            LOG.error("Failed to refresh Cognito tokens", e);
+            return oauthError("invalid_request", e.getMessage());
+        }
+    }
+
+    /** RFC 7636 S256: the challenge is BASE64URL(SHA-256(verifier)). No challenge, nothing to check. */
+    static boolean pkceVerified(String codeChallenge, String codeVerifier) {
+        if (codeChallenge == null) {
+            return true;
+        }
+        if (codeVerifier == null || codeVerifier.length() < 43 || codeVerifier.length() > 128) {
+            return false;
+        }
+        try {
+            byte[] digest = MessageDigest.getInstance("SHA-256").digest(codeVerifier.getBytes(StandardCharsets.US_ASCII));
+            String expected = Base64.getUrlEncoder().withoutPadding().encodeToString(digest);
+            return MessageDigest.isEqual(expected.getBytes(StandardCharsets.US_ASCII),
+                    codeChallenge.getBytes(StandardCharsets.US_ASCII));
+        } catch (java.security.NoSuchAlgorithmException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
     private Response redirect(String redirectUri, Map<String, String> parameters) {
         StringBuilder location = new StringBuilder(redirectUri);
         location.append(redirectUri.contains("?") ? '&' : '?');

src/main/java/io/github/hectorvent/floci/services/cognito/CognitoService.java

@@ -2695,6 +2695,14 @@ public class CognitoService implements ResourceProvider {
 
     // ──────────────────────────── Auth ────────────────────────────
 
+    public CognitoUser authenticateHostedLogin(UserPoolClient client, String username, String password) {
+        return authFlowHandler.authenticateHostedLogin(describeUserPool(client.getUserPoolId()), client, username, password);
+    }
+
+    public Map<String, Object> issueHostedTokens(UserPoolClient client, CognitoUser user, String nonce) {
+        return authFlowHandler.issueHostedTokens(describeUserPool(client.getUserPoolId()), client, user, nonce);
+    }
+
     public Map<String, Object> initiateAuth(String clientId, String authFlow, Map<String, String> authParameters) {
         return authFlowHandler.initiateAuth(clientId, authFlow, authParameters, Map.of());
     }

src/main/java/io/github/hectorvent/floci/services/cognito/CognitoWellKnownController.java

@@ -66,7 +66,7 @@ public class CognitoWellKnownController {
         String authorizationEndpoint = tokenEndpoint.replace("/token", "/authorize");
 
         String body = """
-                {"issuer":"%s","jwks_uri":"%s","authorization_endpoint":"%s","token_endpoint":"%s","userinfo_endpoint":"%s","subject_types_supported":["public"],"response_types_supported":["code"],"grant_types_supported":["client_credentials","authorization_code"],"token_endpoint_auth_methods_supported":["client_secret_basic","client_secret_post"],"id_token_signing_alg_values_supported":["RS256"]}
+                {"issuer":"%s","jwks_uri":"%s","authorization_endpoint":"%s","token_endpoint":"%s","userinfo_endpoint":"%s","subject_types_supported":["public"],"response_types_supported":["code"],"grant_types_supported":["client_credentials","authorization_code","refresh_token"],"token_endpoint_auth_methods_supported":["client_secret_basic","client_secret_post"],"id_token_signing_alg_values_supported":["RS256"]}
                 """.formatted(issuer, jwksUri, authorizationEndpoint, tokenEndpoint, userInfoEndpoint).strip();
         return Response.ok(body).build();
     }

src/main/java/io/github/hectorvent/floci/services/cognito/model/CognitoAuthorizationCode.java

@@ -3,15 +3,27 @@ package io.github.hectorvent.floci.services.cognito.model;
 import java.time.Instant;
 import java.util.List;
 
+/**
+ * A one-time code handed to the client's redirect URI. {@code codeChallenge} is the PKCE S256
+ * challenge from the authorize request, and {@code nonce} is copied into the ID token; both are
+ * null when the client did not send them.
+ */
 public record CognitoAuthorizationCode(
         String userPoolId,
         String clientId,
         String userId,
         String redirectUri,
         List<String> scopes,
-        Instant expiresAt) {
+        Instant expiresAt,
+        String nonce,
+        String codeChallenge) {
 
     public CognitoAuthorizationCode {
         scopes = List.copyOf(scopes);
     }
+
+    public CognitoAuthorizationCode(String userPoolId, String clientId, String userId, String redirectUri,
+                                    List<String> scopes, Instant expiresAt) {
+        this(userPoolId, clientId, userId, redirectUri, scopes, expiresAt, null, null);
+    }
 }

src/main/java/io/github/hectorvent/floci/services/cognito/model/CognitoAuthorizationTransaction.java

@@ -11,9 +11,16 @@ public record CognitoAuthorizationTransaction(
         String nonce,
         String providerName,
         String relyingPartyState,
-        Instant expiresAt) {
+        Instant expiresAt,
+        String codeChallenge) {
 
     public CognitoAuthorizationTransaction {
         scopes = List.copyOf(scopes);
     }
+
+    public CognitoAuthorizationTransaction(String userPoolId, String clientId, String redirectUri, List<String> scopes,
+                                           String nonce, String providerName, String relyingPartyState,
+                                           Instant expiresAt) {
+        this(userPoolId, clientId, redirectUri, scopes, nonce, providerName, relyingPartyState, expiresAt, null);
+    }
 }

src/main/java/io/github/hectorvent/floci/services/cognito/model/CognitoHostedSession.java

@@ -0,0 +1,7 @@
+package io.github.hectorvent.floci.services.cognito.model;
+
+import java.time.Instant;
+
+/** A signed-in browser on the hosted login pages, so the next authorize request skips the form. */
+public record CognitoHostedSession(String userPoolId, String username, Instant expiresAt) {
+}

src/main/java/io/github/hectorvent/floci/services/ec2/AmiImageResolver.java

@@ -53,11 +53,16 @@ public class AmiImageResolver {
         }
         return new ResolvedAmiImage(
                 image.dockerImage,
-                image.guestRuntime == null || image.guestRuntime.isBlank()
+                image.initType == null || image.initType.isBlank()
                         ? ResolvedAmiImage.DEFAULT_RUNTIME
-                        : image.guestRuntime,
+                        : image.initType,
                 Boolean.TRUE.equals(image.cloudInit),
-                dockerPlatform(image.architecture));
+                dockerPlatform(image.architecture),
+                image.fallbackDockerImage,
+                image.fallbackDockerImage == null ? "public" : "enhanced",
+                image.sshUser == null || image.sshUser.isBlank() ? "root" : image.sshUser,
+                Boolean.TRUE.equals(image.sshPreinstalled),
+                image.sshServiceName == null || image.sshServiceName.isBlank() ? "sshd" : image.sshServiceName);
     }
 
     private static String dockerPlatform(String architecture) {

src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java

@@ -12,6 +12,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
 import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
 import io.github.hectorvent.floci.core.common.docker.PortAllocator;
 import io.github.hectorvent.floci.services.ec2.model.Instance;
+import io.github.hectorvent.floci.services.ec2.model.InstanceLaunchEvent;
 import io.github.hectorvent.floci.services.ec2.model.InstanceNetworkInterface;
 import io.github.hectorvent.floci.services.ec2.model.InstanceState;
 import io.github.hectorvent.floci.services.ec2.model.GroupIdentifier;
@@ -38,8 +39,13 @@ import java.io.ByteArrayOutputStream;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.InetSocketAddress;
+import java.net.Socket;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.HashSet;
@@ -60,6 +66,7 @@ import java.util.concurrent.Semaphore;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiPredicate;
+import java.util.function.Consumer;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -161,6 +168,8 @@ public class Ec2ContainerManager {
 
     static int containerBridgeIpAttempts = 30;
     static long containerBridgeIpPollMillis = 500;
+    static int sshBannerAttempts = 30;
+    static long sshBannerPollMillis = 1_000;
 
     private final ContainerBuilder containerBuilder;
     private final ContainerLifecycleManager lifecycleManager;
@@ -180,9 +189,14 @@ public class Ec2ContainerManager {
     private final ExecutorService executor;
     private final Duration userDataExecutionTimeout;
     private final Set<ResultCallback<Frame>> activeUserDataCallbacks = ConcurrentHashMap.newKeySet();
+    private volatile Consumer<Instance> instancePersister = ignored -> {};
 
     private volatile boolean dockerUnavailableLogged;
 
+    public void setInstancePersister(Consumer<Instance> instancePersister) {
+        this.instancePersister = instancePersister == null ? ignored -> {} : instancePersister;
+    }
+
     public void refreshSecurityGroups(String region, Map<String, SecurityGroup> groups,
                                       Map<String, List<String>> prefixLists) {
         if (firewallManager != null && firewallManager.enabled()) {
@@ -240,7 +254,7 @@ public class Ec2ContainerManager {
         InstanceNetworkInterface eni = instance.getNetworkInterfaces().getFirst();
         String logical = instance.getLogicalPrivateIpAddress() != null
                 ? instance.getLogicalPrivateIpAddress() : eni.getPrivateIpAddress();
-        firewallManager.register(new SecurityGroupNftCompiler.Endpoint(regionResolver.getAccountId(), region,
+        firewallManager.register(new SecurityGroupNftCompiler.Endpoint(accountId(instance), region,
                 instance.getVpcId(), eni.getNetworkInterfaceId(), logical, address,
                 instance.getSecurityGroups().stream().map(GroupIdentifier::getGroupId)
                         .collect(Collectors.toSet()), groups), helperId, prefixLists);
@@ -375,6 +389,9 @@ public class Ec2ContainerManager {
                        Set<Integer> appPorts, List<SecurityGroup> groups,
                        Map<String, List<String>> prefixLists) {
         instance.setState(InstanceState.pending());
+        instance.setLaunchStatus("in-progress");
+        instance.setSshStatus("initializing");
+        persist(instance);
 
         // An instance record is metadata: id, addresses, tags and lifecycle state are all
         // served without a container runtime. Only the guest itself (SSH, UserData, SSM
@@ -398,11 +415,13 @@ public class Ec2ContainerManager {
         try {
             executor.execute(() -> {
                 try {
+                    recordLaunchEvent(instance, "selecting-image", "started", "Selecting the launch image");
+                    ResolvedAmiImage launchImage = selectLaunchImage(instance, image);
                     String instanceId = instance.getInstanceId();
                 // IMDS endpoint that this container should use
                 String flociHost = dockerHostResolver.resolve();
                 int imdsPort = config.services().ec2().imdsPort();
-                StartedContainer started = createAndStartContainer(instance, image, region, flociHost, imdsPort,
+                StartedContainer started = createAndStartContainer(instance, launchImage, region, flociHost, imdsPort,
                         leasedPrivateIp, groups, prefixLists);
                 if (started == null) {
                     return;
@@ -485,6 +504,7 @@ public class Ec2ContainerManager {
                     failLaunch(instance, leasedPrivateIp);
                     return;
                 }
+                persist(instance);
 
                 // Set public-facing addresses only for instances whose subnet
                 // opts in via MapPublicIpOnLaunch (#1984). Private-subnet
@@ -508,14 +528,47 @@ public class Ec2ContainerManager {
 
                 // Inject SSH public key, if one was provided
                 if (publicKey != null && !publicKey.isBlank()) {
-                    injectSshKey(containerId, publicKey);
+                    recordLaunchEvent(instance, "injecting-key", "started", "Injecting the selected public key");
+                    OperationResult injection = injectSshKey(containerId, publicKey);
+                    instance.setKeyInjectionStatus(injection.success() ? "succeeded" : "failed");
+                    recordLaunchEvent(instance, "injecting-key", injection.success() ? "completed" : "failed",
+                            injection.message());
+                } else {
+                    instance.setKeyInjectionStatus("skipped");
+                    recordLaunchEvent(instance, "injecting-key", "skipped", "No key pair was selected");
                 }
-                // sshd runs on every instance regardless of whether a key pair was supplied,
-                // matching real AWS AMIs (which start it as part of normal boot, independent of
-                // key-pair association) - starting it only when a key was present meant
-                // run-instances without --key-name produced a "connection refused" instead of AWS's
-                // actual behavior: a running daemon with simply nothing to authenticate against.
-                startSshd(containerId, instanceId);
+                // Systemd-backed AMIs start their enabled SSH unit during boot. Starting a second
+                // daemon here races it for port 22; lightweight guests still need this bootstrap.
+                if (!launchImage.systemd()) {
+                    recordLaunchEvent(instance, "installing-ssh", "started", "Checking and installing SSH packages");
+                    SshSetupResult setup = startSshd(containerId, instanceId);
+                    String installationStatus = setup.installationAttempted()
+                            ? setup.installationSucceeded() ? "completed" : "failed"
+                            : "skipped";
+                    recordLaunchEvent(instance, "installing-ssh", installationStatus,
+                            setup.installationMessage());
+                    recordLaunchEvent(instance, "starting-ssh", setup.success() ? "completed" : "failed",
+                            setup.message());
+                } else {
+                    recordLaunchEvent(instance, "installing-ssh", "skipped",
+                            "SSH is preinstalled in the selected image");
+                    recordLaunchEvent(instance, "starting-ssh", "skipped",
+                            "SSH startup is managed by " + launchImage.sshServiceName());
+                }
+
+                recordLaunchEvent(instance, "ssh-ready", "started", "Waiting for an SSH protocol banner");
+                OperationResult readiness = waitForSshBanner(instance.getContainerBridgeIp(), 22);
+                String readinessMessage = setupFailureMessage(instance, readiness.message());
+                instance.setSshStatus(readiness.success() ? "ok" : "impaired");
+                instance.setSshStatusMessage(readinessMessage);
+                instance.setLaunchStatus(readiness.success() ? "ready" : "failed");
+                if (!readiness.success()) {
+                    instance.setStateReasonCode("Client.SSHUnavailable");
+                    instance.setStateReasonMessage(readinessMessage);
+                    instance.setStateTransitionReason(readinessMessage);
+                }
+                recordLaunchEvent(instance, "ssh-ready", readiness.success() ? "completed" : "failed",
+                        readinessMessage);
 
                 // Execute UserData
                 String userData = instance.getUserData();
@@ -525,14 +578,14 @@ public class Ec2ContainerManager {
 
                 } catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
-                    failLaunch(instance, leasedPrivateIp);
+                    failLaunch(instance, leasedPrivateIp, "Launch interrupted");
                 } catch (Exception e) {
                     LOG.warnv("Failed to launch EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());
                     // The daemon can disappear between the probe above and any of the calls in
                     // this block. Losing Docker is not the instance's fault, so degrade to a
                     // metadata-only instance; a genuine container failure still fails the launch.
                     if ((firewallManager != null && firewallManager.enabled()) || isDockerAvailable()) {
-                        failLaunch(instance, leasedPrivateIp);
+                        failLaunch(instance, leasedPrivateIp, errorMessage(e));
                     } else {
                         markContainerlessRunning(instance);
                     }
@@ -541,10 +594,51 @@ public class Ec2ContainerManager {
         } catch (RejectedExecutionException e) {
             LOG.warnv("Could not schedule EC2 instance {0} launch because the launch executor is saturated or stopping",
                     instance.getInstanceId());
-            failLaunch(instance, leasedPrivateIp);
+            failLaunch(instance, leasedPrivateIp,
+                    "Launch executor is saturated or stopping: " + errorMessage(e));
         }
     }
 
+    ResolvedAmiImage selectLaunchImage(ResolvedAmiImage image) {
+        return selectLaunchImage(null, image);
+    }
+
+    private ResolvedAmiImage selectLaunchImage(Instance instance, ResolvedAmiImage image) {
+        if (image.fallbackDockerImage() == null || image.fallbackDockerImage().isBlank()) {
+            recordSelectedImage(instance, image, "Selected " + image.imageType() + " image");
+            return image;
+        }
+        String preferred = containerBuilder.resolveImage(image.dockerImage());
+        try {
+            dockerClient.inspectImageCmd(preferred).exec();
+            recordSelectedImage(instance, image, "Enhanced AMI image is available locally");
+            return image;
+        } catch (NotFoundException expected) {
+            LOG.warnv("EC2 AMI image {0} is not present locally; using pullable fallback {1}",
+                    image.dockerImage(), image.fallbackDockerImage());
+            ResolvedAmiImage fallback = new ResolvedAmiImage(image.fallbackDockerImage(),
+                    ResolvedAmiImage.DEFAULT_RUNTIME, false, image.dockerPlatform(), null,
+                    "public-fallback", image.sshUser(), false, image.sshServiceName());
+            recordSelectedImage(instance, fallback,
+                    "Enhanced image " + image.dockerImage() + " was not present locally; selected public fallback");
+            return fallback;
+        }
+    }
+
+    private void recordSelectedImage(Instance instance, ResolvedAmiImage image, String reason) {
+        if (instance == null) {
+            return;
+        }
+        instance.setSelectedImageType(image.imageType());
+        instance.setSelectedImageReference(image.dockerImage());
+        instance.setSshUser(image.sshUser());
+        instance.setSshPreinstalled(image.sshPreinstalled());
+        instance.setInitType(image.initType());
+        instance.setSshServiceName(image.sshServiceName());
+        recordLaunchEvent(instance, "selecting-image", "completed",
+                reason + ": " + image.dockerImage());
+    }
+
     private StartedContainer createAndStartContainer(Instance instance, ResolvedAmiImage image, String region,
                                                      String flociHost, int imdsPort, String leasedPrivateIp,
                                                      List<SecurityGroup> groups,
@@ -558,9 +652,20 @@ public class Ec2ContainerManager {
             if (isLaunchCancelled(instance)) {
                 return null;
             }
-            int sshHostPort = portAllocator.allocate(
+            recordLaunchEvent(instance, "allocating-ssh-port", "started",
+                    "Allocating a host port for guest port 22");
+            List<Integer> occupiedPorts = new ArrayList<>();
+            int sshHostPort;
+            try {
+                sshHostPort = portAllocator.allocate(
                         config.services().ec2().sshPortRangeStart(),
-                    config.services().ec2().sshPortRangeEnd());
+                        config.services().ec2().sshPortRangeEnd(),
+                        occupiedPorts::add);
+            } catch (RuntimeException e) {
+                recordOccupiedPorts(instance, occupiedPorts);
+                throw e;
+            }
+            recordOccupiedPorts(instance, occupiedPorts);
             SecurityGroupFirewallManager.Namespace namespace = null;
             String eniId = null;
             String containerId = null;
@@ -574,8 +679,8 @@ public class Ec2ContainerManager {
                     eniId = eni.getNetworkInterfaceId();
                     instance.setLogicalPrivateIpAddress(eni.getPrivateIpAddress());
                     namespace = firewallManager.createNamespace("ec2", instanceId,
-                            regionResolver.getAccountId(), region, Optional.empty(), Map.of(22, sshHostPort));
-                    firewallManager.register(new SecurityGroupNftCompiler.Endpoint(regionResolver.getAccountId(),
+                            accountId(instance), region, Optional.empty(), Map.of(22, sshHostPort));
+                    firewallManager.register(new SecurityGroupNftCompiler.Endpoint(accountId(instance),
                             region, instance.getVpcId(), eniId, eni.getPrivateIpAddress(),
                             namespace.transportAddress(),
                             instance.getSecurityGroups().stream().map(g -> g.getGroupId())
@@ -583,10 +688,34 @@ public class Ec2ContainerManager {
                             namespace.helperId(), prefixLists);
                 }
                 ContainerSpec spec = buildContainerSpec(containerName, image, region, serviceEndpoint, imdsEndpoint,
-                        instanceId, sshHostPort, namespace, instance.getIamInstanceProfileArn() != null);
-                containerId = image.dockerPlatform() == null
-                        ? lifecycleManager.create(spec)
-                        : lifecycleManager.create(spec, image.dockerPlatform());
+                        instanceId, accountId(instance), sshHostPort, namespace,
+                        instance.getIamInstanceProfileArn() != null);
+                recordLaunchEvent(instance, "allocating-ssh-port", "completed",
+                        "Allocated SSH host port " + sshHostPort);
+                containerId = lifecycleManager.create(spec, image.dockerPlatform(),
+                        new ContainerLifecycleManager.ContainerCreationObserver() {
+                            @Override
+                            public void imagePullStarted(String imageUri) {
+                                recordLaunchEvent(instance, "pulling-image", "started", "Pulling " + imageUri);
+                            }
+
+                            @Override
+                            public void imagePullCompleted(String imageUri) {
+                                recordLaunchEvent(instance, "pulling-image", "completed", "Pulled " + imageUri);
+                            }
+
+                            @Override
+                            public void containerCreationStarted() {
+                                recordLaunchEvent(instance, "creating-container", "started",
+                                        "Creating the instance container");
+                            }
+
+                            @Override
+                            public void containerCreated(String createdContainerId) {
+                                recordLaunchEvent(instance, "creating-container", "completed",
+                                        "Created container " + createdContainerId);
+                            }
+                        });
                 if (!recordCreatedContainer(instance, containerId, sshHostPort)) {
                     lifecycleManager.removeIfExists(containerId);
                     if (namespace != null) {
@@ -596,15 +725,20 @@ public class Ec2ContainerManager {
                     portAllocator.release(sshHostPort);
                     return null;
                 }
+                persist(instance);
                 recorded = true;
                 // Join the VPC's Docker network at the address the subnet allocated, before the
                 // container starts, so the guest comes up already holding its private IP. The
                 // default bridge attachment stays: it is what carries the published SSH host
                 // port, which a network mode set at creation time would suppress.
+                recordLaunchEvent(instance, "configuring-network", "started",
+                        "Attaching the instance container to its VPC network");
                 String vpcAddress = namespace == null
                         ? vpcNetworkManager.attach(region, instance.getVpcId(), instance.getSubnetId(),
                                 containerId, leasedPrivateIp).map(network -> leasedPrivateIp).orElse(null)
                         : null;
+                recordLaunchEvent(instance, "configuring-network", "completed",
+                        vpcAddress == null ? "Using the container network namespace" : "Attached at " + vpcAddress);
                 lifecycleManager.startCreated(containerId, spec);
                 return new StartedContainer(containerId, sshHostPort, vpcAddress, namespace, eniId);
             } catch (Exception e) {
@@ -627,6 +761,8 @@ public class Ec2ContainerManager {
                     portAllocator.markReserved(sshHostPort);
                     LOG.warnv("EC2 instance {0} could not use SSH host port {1}; trying another port",
                             instanceId, String.valueOf(sshHostPort));
+                    recordLaunchEvent(instance, "allocating-ssh-port", "info",
+                            "SSH host port " + sshHostPort + " was occupied; selecting another port");
                     continue;
                 }
                 portAllocator.release(sshHostPort);
@@ -637,6 +773,7 @@ public class Ec2ContainerManager {
 
     private ContainerSpec buildContainerSpec(String containerName, ResolvedAmiImage image, String region,
                                              String serviceEndpoint, String imdsEndpoint, String instanceId,
+                                             String accountId,
                                              int sshHostPort, SecurityGroupFirewallManager.Namespace namespace,
                                              boolean hasInstanceProfile) {
         // Minimal images keep the historic tail command, while cloud-image AMI guests can boot their init.
@@ -650,7 +787,7 @@ public class Ec2ContainerManager {
                 .withHostDockerInternalOnLinux()
                 .withLogRotation()
                 .withLabels(ContainerStorageHelper.resourceIdentityLabels(
-                        "ec2", instanceId, regionResolver.getAccountId(), region))
+                        "ec2", instanceId, accountId, region))
                 // Which Floci owns this container, so the startup reconciler cannot reap a
                 // sibling emulator's live instances off a shared daemon. See LABEL_OWNER_PORT.
                 .withLabels(Map.of(LABEL_OWNER_PORT, ownerIdentity()))
@@ -675,7 +812,7 @@ public class Ec2ContainerManager {
     }
 
     private void failLaunch(Instance instance) {
-        failLaunch(instance, instance.getPrivateIpAddress());
+        failLaunch(instance, instance.getPrivateIpAddress(), "EC2 container launch failed");
     }
 
     /**
@@ -688,6 +825,10 @@ public class Ec2ContainerManager {
      * reserved while the endpoint stands, and would refuse the next launch handed the same one.
      */
     private void failLaunch(Instance instance, String leasedPrivateIp) {
+        failLaunch(instance, leasedPrivateIp, "EC2 container launch failed");
+    }
+
+    private void failLaunch(Instance instance, String leasedPrivateIp, String reason) {
         String containerId;
         int sshHostPort;
         String containerIp;
@@ -698,6 +839,13 @@ public class Ec2ContainerManager {
                     && instance.getSshHostPort() <= 0
                     && (instance.getContainerBridgeIp() == null || instance.getContainerBridgeIp().isBlank());
             instance.setState(InstanceState.terminated());
+            instance.setStateReasonCode("Server.InternalError");
+            instance.setStateReasonMessage(reason);
+            instance.setStateTransitionReason(reason);
+            instance.setTerminatedAt(System.currentTimeMillis());
+            instance.setLaunchStatus("failed");
+            instance.setSshStatus("impaired");
+            instance.setSshStatusMessage(reason);
             containerId = instance.getDockerContainerId();
             sshHostPort = instance.getSshHostPort();
             containerIp = instance.getContainerBridgeIp();
@@ -705,6 +853,7 @@ public class Ec2ContainerManager {
             instance.setSshHostPort(0);
             instance.setContainerBridgeIp(null);
         }
+        recordLaunchEvent(instance, currentLaunchStage(instance), "failed", reason);
         if (alreadyCleaned) {
             return;
         }
@@ -743,6 +892,7 @@ public class Ec2ContainerManager {
                 LOG.warnv("Error unregistering failed EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());
             }
         }
+        persist(instance);
     }
 
     /**
@@ -857,7 +1007,27 @@ public class Ec2ContainerManager {
     private void markContainerlessRunning(Instance instance) {
         LOG.infov("EC2 instance {0} is running without a backing container (no Docker daemon reachable)",
                 instance.getInstanceId());
+        String reason = "Docker daemon is unavailable; the instance has no backing container";
         instance.setState(InstanceState.running());
+        instance.setLaunchStatus("failed");
+        instance.setSshStatus("impaired");
+        instance.setSshStatusMessage(reason);
+        instance.setStateReasonCode("Server.InternalError");
+        instance.setStateReasonMessage(reason);
+        instance.setStateTransitionReason(reason);
+        recordLaunchEvent(instance, "creating-container", "failed", reason);
+    }
+
+    private String accountId(Instance instance) {
+        return instance.getAccountId() == null || instance.getAccountId().isBlank()
+                ? regionResolver.getAccountId() : instance.getAccountId();
+    }
+
+    private void recordOccupiedPorts(Instance instance, List<Integer> occupiedPorts) {
+        for (Integer occupiedPort : occupiedPorts) {
+            recordLaunchEvent(instance, "allocating-ssh-port", "retrying",
+                    "SSH host port " + occupiedPort + " is occupied; trying the next port");
+        }
     }
 
     /**
@@ -1297,6 +1467,13 @@ public class Ec2ContainerManager {
         executor.submit(() -> {
             try {
                 dockerClient.restartContainerCmd(containerId).exec();
+                InspectContainerResponse container = dockerClient.inspectContainerCmd(containerId).exec();
+                if (isLegacyTailGuest(container.getConfig().getCmd())) {
+                    // Instances launched before systemd-backed AMIs used tail as PID 1. Their image
+                    // catalog entry now identifies a systemd runtime, but Docker restart retains
+                    // the original command and therefore does not start the SSH service.
+                    startSshd(containerId, instance.getInstanceId());
+                }
                 LOG.infov("Rebooted EC2 container {0}", containerId);
             } catch (Exception e) {
                 LOG.warnv("Error rebooting EC2 container {0}: {1}", containerId, e.getMessage());
@@ -1304,6 +1481,11 @@ public class Ec2ContainerManager {
         });
     }
 
+    static boolean isLegacyTailGuest(String[] command) {
+        return command != null && command.length == 3
+                && "tail".equals(command[0]) && "-f".equals(command[1]) && "/dev/null".equals(command[2]);
+    }
+
     public boolean isContainerRunning(Instance instance) {
         String containerId = instance.getDockerContainerId();
         return containerId != null && lifecycleManager.isContainerRunning(containerId);
@@ -1379,7 +1561,7 @@ public class Ec2ContainerManager {
         }
     }
 
-    private void injectSshKey(String containerId, String publicKey) {
+    private OperationResult injectSshKey(String containerId, String publicKey) {
         try {
             // Ensure .ssh directory exists with correct permissions
             execInContainer(containerId, new String[]{"sh", "-c",
@@ -1395,32 +1577,39 @@ public class Ec2ContainerManager {
 
             execInContainer(containerId, new String[]{"chmod", "600", "/root/.ssh/authorized_keys"}, 5);
             LOG.infov("Injected SSH public key into container {0}", containerId);
+            return new OperationResult(true, "Public key injected into /root/.ssh/authorized_keys");
         } catch (Exception e) {
             LOG.warnv("Could not inject SSH key into container {0}: {1}", containerId, e.getMessage());
+            return new OperationResult(false, errorMessage(e));
         }
     }
 
-    private void startSshd(String containerId, String instanceId) {
+    private SshSetupResult startSshd(String containerId, String instanceId) {
         try {
             // Exit 1 means sshd is absent and there is nothing to start; exit 2 means sshd is
             // there but the client package did not land. Only the first is fatal - see
             // sshdInstallProbeCommand() for why the probe distinguishes them.
             ContainerExecResult install = execInContainerForResult(containerId, sshdInstallProbeCommand(), 120);
+            boolean installationAttempted = install.output().contains("FLOCI_SSH_INSTALL=1");
             if (install.exitCode() == SSH_CLIENT_MISSING_EXIT_CODE) {
                 LOG.warnv("sshd is available on EC2 instance {0} but the OpenSSH client package is not:"
                         + " scp is missing, so provisioners that upload files over scp will fail: {1}",
                         instanceId, install.summary());
             } else if (install.exitCode() != 0) {
+                String message = "Could not install openssh-server: " + install.summary();
                 LOG.warnv("Could not install openssh-server for EC2 instance {0}: {1}",
                         instanceId, install.summary());
-                return;
+                return new SshSetupResult(false, installationAttempted, false,
+                        installationAttempted ? "SSH package installation failed" : "SSH package installation skipped",
+                        message);
             }
             // Generate host keys
             ContainerExecResult keygen = execInContainerForResult(containerId, new String[]{"ssh-keygen", "-A"}, 10);
             if (keygen.exitCode() != 0) {
                 LOG.warnv("Could not generate SSH host keys for EC2 instance {0}: {1}",
                         instanceId, keygen.summary());
-                return;
+                return new SshSetupResult(false, installationAttempted, true, "SSH packages are available",
+                        "Could not generate SSH host keys: " + keygen.summary());
             }
             // Modern OpenSSH refuses to start without its privilege-separation directory, and /run
             // is a fresh tmpfs in most container images, so /run/sshd genuinely isn't there yet.
@@ -1429,7 +1618,8 @@ public class Ec2ContainerManager {
             if (mkdir.exitCode() != 0) {
                 LOG.warnv("Could not create /run/sshd for EC2 instance {0}: {1}",
                         instanceId, mkdir.summary());
-                return;
+                return new SshSetupResult(false, installationAttempted, true, "SSH packages are available",
+                        "Could not create /run/sshd: " + mkdir.summary());
             }
             // Start sshd without -D so it daemonizes itself and survives this exec session. Since sshd
             // requires execution with an absolute path, several paths are tried until it starts
@@ -1440,13 +1630,108 @@ public class Ec2ContainerManager {
                     continue;
                 }
                 LOG.infov("Started sshd in EC2 instance {0}", instanceId);
-                return;
+                return new SshSetupResult(true, installationAttempted, true,
+                        installationAttempted ? "SSH packages installed" : "SSH was already installed",
+                        "Started sshd using " + sshdPath);
             }
+            return new SshSetupResult(false, installationAttempted, true, "SSH packages are available",
+                    "No supported sshd executable started successfully");
         } catch (Exception e) {
             LOG.warnv("Could not start sshd in EC2 instance {0}: {1}", instanceId, e.getMessage());
+            return new SshSetupResult(false, false, false, "SSH package state is unknown", errorMessage(e));
         }
     }
 
+    private OperationResult waitForSshBanner(String host, int port) throws InterruptedException {
+        if (host == null || host.isBlank()) {
+            return new OperationResult(false, "No container address is available for the SSH readiness check");
+        }
+        String lastError = "SSH did not return a protocol banner";
+        for (int attempt = 0; attempt < sshBannerAttempts; attempt++) {
+            OperationResult probe = probeSshBanner(host, port);
+            if (probe.success()) {
+                return probe;
+            }
+            lastError = probe.message();
+            if (attempt + 1 < sshBannerAttempts) {
+                Thread.sleep(sshBannerPollMillis);
+            }
+        }
+        return new OperationResult(false, lastError);
+    }
+
+    private OperationResult probeSshBanner(String host, int port) {
+        try (Socket socket = new Socket()) {
+            socket.connect(new InetSocketAddress(host, port), 1_000);
+            socket.setSoTimeout(1_000);
+            BufferedReader reader = new BufferedReader(
+                    new InputStreamReader(socket.getInputStream(), StandardCharsets.US_ASCII));
+            String banner = reader.readLine();
+            if (banner != null && banner.startsWith("SSH-")) {
+                return new OperationResult(true, "SSH banner received: " + banner);
+            }
+            return new OperationResult(false, banner == null
+                    ? "SSH connection closed without a protocol banner"
+                    : "Endpoint returned a non-SSH banner: " + banner);
+        } catch (IOException e) {
+            return new OperationResult(false, errorMessage(e));
+        }
+    }
+
+    public OperationResult checkSshReadiness(Instance instance) {
+        return probeSshBanner(instance.getContainerBridgeIp(), 22);
+    }
+
+    private void recordLaunchEvent(Instance instance, String stage, String status, String message) {
+        if (instance == null) {
+            return;
+        }
+        synchronized (instance) {
+            instance.getLaunchEvents().add(new InstanceLaunchEvent(stage, status, Instant.now(), message));
+        }
+        persist(instance);
+    }
+
+    private String setupFailureMessage(Instance instance, String readinessMessage) {
+        synchronized (instance) {
+            List<InstanceLaunchEvent> events = instance.getLaunchEvents();
+            for (int i = events.size() - 1; i >= 0; i--) {
+                InstanceLaunchEvent event = events.get(i);
+                if ("starting-ssh".equals(event.getStage()) && "failed".equals(event.getStatus())) {
+                    return event.getMessage() + "; readiness check: " + readinessMessage;
+                }
+            }
+        }
+        return readinessMessage;
+    }
+
+    private String currentLaunchStage(Instance instance) {
+        synchronized (instance) {
+            List<InstanceLaunchEvent> events = instance.getLaunchEvents();
+            if (!events.isEmpty()) {
+                return events.getLast().getStage();
+            }
+        }
+        return "launch";
+    }
+
+    private void persist(Instance instance) {
+        instancePersister.accept(instance);
+    }
+
+    private static String errorMessage(Throwable error) {
+        if (error == null) {
+            return "Unknown launch failure";
+        }
+        String message = error.getMessage();
+        return message == null || message.isBlank() ? error.getClass().getSimpleName() : message;
+    }
+
+    public record OperationResult(boolean success, String message) {}
+
+    private record SshSetupResult(boolean success, boolean installationAttempted, boolean installationSucceeded,
+                                  String installationMessage, String message) {}
+
     private void executeUserData(String containerId, String instanceId, String userData, String region) {
         try {
             String logGroup = "/aws/ec2/" + instanceId;
@@ -1751,7 +2036,43 @@ public class Ec2ContainerManager {
     }
 
     static String[] userDataExecutionCommand() {
-        return new String[]{USER_DATA_SCRIPT_PATH};
+        String command = """
+                policy=/usr/sbin/policy-rc.d
+                backup=/var/lib/floci-policy-rc.d
+                actions=/var/lib/floci-service-actions
+                rm -f "$actions"
+                if [ -f "$policy" ]; then mv "$policy" "$backup"; fi
+                printf '%s\n' '#!/bin/sh' \
+                  'while [ "$#" -gt 0 ]; do case "$1" in --*) shift ;; *) break ;; esac; done' \
+                  'service=$1' \
+                  'action=$2' \
+                  'action=${action#(}' \
+                  'action=${action%)}' \
+                  'case "$action" in start|restart|try-restart|reload|force-reload)' \
+                  'printf "%s %s\\n" "$service" "$action" >> /var/lib/floci-service-actions ;; esac' \
+                  'exit 101' > "$policy"
+                chmod 755 "$policy"
+                restore_policy_and_start_services() {
+                  status=$?
+                  rm -f "$policy"
+                  if [ -f "$backup" ]; then mv "$backup" "$policy"; fi
+                  if [ -f "$actions" ]; then
+                    while read -r service action; do
+                      if command -v service >/dev/null 2>&1; then
+                        service "$service" "$action" || true
+                      elif [ -x "/etc/init.d/$service" ]; then
+                        "/etc/init.d/$service" "$action" || true
+                      fi
+                    done < "$actions"
+                    rm -f "$actions"
+                  fi
+                  trap - EXIT
+                  exit "$status"
+                }
+                trap restore_policy_and_start_services EXIT
+                /var/lib/user-data.sh
+                """;
+        return new String[]{"sh", "-c", command};
     }
 
     /**
@@ -1783,7 +2104,9 @@ public class Ec2ContainerManager {
      */
     static String[] sshdInstallProbeCommand() {
         return new String[]{"sh", "-c",
+            "installed=0;" +
                     "if ! command -v sshd >/dev/null 2>&1 || ! command -v scp >/dev/null 2>&1; then" +
+                    "  installed=1;" +
                     "  if command -v dnf >/dev/null 2>&1; then dnf install -y openssh-server openssh-clients >/dev/null 2>&1;" +
                     // yum is probed after dnf, and the order is load-bearing: Amazon Linux
                     // 2023 and modern Fedora/RHEL ship both, and there dnf is the supported
@@ -1793,9 +2116,11 @@ public class Ec2ContainerManager {
                     // and sshd is never installed.
                     "  elif command -v yum >/dev/null 2>&1; then yum install -y openssh-server openssh-clients >/dev/null 2>&1;" +
                     "  elif command -v apt-get >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server openssh-client >/dev/null 2>&1;" +
+                    "  elif command -v zypper >/dev/null 2>&1; then zypper --non-interactive install -y openssh >/dev/null 2>&1;" +
                     "  elif command -v apk >/dev/null 2>&1; then apk add --no-cache openssh >/dev/null 2>&1;" +
                     "  fi;" +
                     "fi;" +
+                    "printf 'FLOCI_SSH_INSTALL=%s\\n' \"$installed\";" +
                     "command -v sshd >/dev/null 2>&1 || exit 1;" +
                     "command -v scp >/dev/null 2>&1 || exit 2"};
     }

src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ImageCatalog.java

@@ -168,6 +168,7 @@ public class Ec2ImageCatalog {
         public List<String> aliases = List.of();
         public List<String> publicParameterNames = List.of();
         public String dockerImage;
+        public String fallbackDockerImage;
         public String name;
         public String description;
         public String state;
@@ -182,8 +183,11 @@ public class Ec2ImageCatalog {
         public String imageOwnerAlias;
         public String creationDate;
         public Boolean advertised;
-        public String guestRuntime;
+        public String initType;
         public Boolean cloudInit;
+        public String sshUser;
+        public Boolean sshPreinstalled;
+        public String sshServiceName;
         public String provenance;
 
         public boolean advertised() {

src/main/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataProxy.java

@@ -35,6 +35,8 @@ public final class Ec2MetadataProxy {
                 // leaving the instance without a link-local IMDS endpoint.
                 "elif command -v yum >/dev/null 2>&1; then",
                 "  yum install -y iproute iptables socat curl ca-certificates >/dev/null",
+                "elif command -v zypper >/dev/null 2>&1; then",
+                "  zypper --non-interactive install -y iproute2 iptables socat curl ca-certificates >/dev/null",
                 "elif command -v apk >/dev/null 2>&1; then",
                 "  apk add --no-cache iproute2 iptables socat curl ca-certificates >/dev/null",
                 "else",

src/main/java/io/github/hectorvent/floci/services/ec2/Ec2QueryHandler.java

@@ -702,6 +702,15 @@ public class Ec2QueryHandler {
         int networkInterfaceDeviceIndex = parseIntParam(p, "NetworkInterface.1.DeviceIndex", 0);
         String clientToken = p.getFirst("ClientToken");
         List<String> sgIds = getList(p, "SecurityGroupId");
+        if (sgIds.isEmpty()) {
+            sgIds = getList(p, "NetworkInterface.1.SecurityGroupId");
+        }
+        if (sgIds.isEmpty()) {
+            sgIds = getList(p, "NetworkInterface.1.Groups");
+        }
+        if (sgIds.isEmpty()) {
+            sgIds = getList(p, "NetworkInterface.1.GroupId");
+        }
 
         // UserData is base64-encoded in the wire format
         String userDataEncoded = p.getFirst("UserData");
@@ -1285,12 +1294,20 @@ public class Ec2QueryHandler {
 
     private Response handleDescribeInstanceStatus(MultivaluedMap<String, String> p, String region) {
         List<String> ids = getList(p, "InstanceId");
-        List<Instance> runningInstances = service.describeInstanceStatus(region, ids);
+        boolean includeAllInstances = Boolean.TRUE.equals(
+                parseOptionalBoolean(p.getFirst("IncludeAllInstances"), "IncludeAllInstances"));
+        List<Instance> instances = service.describeInstanceStatus(region, ids, includeAllInstances);
         XmlBuilder xml = new XmlBuilder()
                 .start("DescribeInstanceStatusResponse", AwsNamespaces.EC2)
                 .elem("requestId", UUID.randomUUID().toString())
                 .start("instanceStatusSet");
-        for (Instance inst : runningInstances) {
+        for (Instance inst : instances) {
+            String reachability = inst.getSshStatus() == null ? "initializing" : inst.getSshStatus();
+            String detail = switch (reachability) {
+                case "ok" -> "passed";
+                case "impaired" -> "failed";
+                default -> "initializing";
+            };
             xml.start("item")
                     .elem("instanceId", inst.getInstanceId())
                     .elem("availabilityZone", inst.getPlacement() != null ? inst.getPlacement().getAvailabilityZone() : "")
@@ -1305,9 +1322,9 @@ public class Ec2QueryHandler {
                     .end("item").end("details")
                     .end("systemStatus")
                     .start("instanceStatus")
-                    .elem("status", "ok")
+                    .elem("status", reachability)
                     .start("details").start("item")
-                    .elem("name", "reachability").elem("status", "passed")
+                    .elem("name", "reachability").elem("status", detail)
                     .end("item").end("details")
                     .end("instanceStatus")
                     .end("item");

src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java

@@ -61,6 +61,7 @@ import io.github.hectorvent.floci.services.ec2.model.Image;
 import io.github.hectorvent.floci.services.ec2.model.Instance;
 import io.github.hectorvent.floci.services.ec2.model.InstanceCreditSpecification;
 import io.github.hectorvent.floci.services.ec2.model.InstanceCreditSpecificationListResult;
+import io.github.hectorvent.floci.services.ec2.model.InstanceLaunchEvent;
 import io.github.hectorvent.floci.services.ec2.model.InstanceNetworkInterface;
 import io.github.hectorvent.floci.services.ec2.model.InstanceState;
 import io.github.hectorvent.floci.services.ec2.model.NetworkInterface;
@@ -507,13 +508,11 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
 
     @PostConstruct
     void restoreMetadataRegistrations() {
+        containerManager.setInstancePersister(this::persistInstance);
         if (portForwardManager != null) {
-            portForwardManager.setPersister(inst -> {
-                if (inst != null && inst.getRegion() != null && inst.getInstanceId() != null) {
-                    instances.put(key(inst.getRegion(), inst.getInstanceId()), inst);
-                }
-            });
+            portForwardManager.setPersister(this::persistInstance);
         }
+        clearPersistedPrivateKeyMaterial();
         if (config.services().ec2().mock()) {
             return;
         }
@@ -522,8 +521,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
         recoverInterruptedTerminations();
 
         int restored = 0;
-        for (String key : instances.keys()) {
-            Instance instance = instances.get(key).orElse(null);
+        for (AccountAwareStorageBackend.AccountEntry<Instance> entry : allInstanceEntries()) {
+            Instance instance = entry.value();
             if (!needsMetadataRegistration(instance)) {
                 continue;
             }
@@ -533,11 +532,26 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
                 LOG.warnv("Could not restore EC2 firewall for {0}: {1}", instance.getInstanceId(), e.getMessage());
                 containerManager.stopForShutdown(instance);
                 instance.setState(InstanceState.stopped());
-                instances.put(key, instance);
+                persistOwnedInstance(entry, instance);
                 continue;
             }
             if (containerManager.restoreMetadataRegistration(instance)) {
-                instances.put(key, instance);
+                if ("pending".equals(instance.getState().getName())
+                        || "in-progress".equals(instance.getLaunchStatus())) {
+                    instance.setState(InstanceState.running());
+                    instance.setLaunchStatus("unknown");
+                    instance.getLaunchEvents().add(new InstanceLaunchEvent("startup-reconciliation", "unknown",
+                            Instant.now(), "Core restarted before launch completion was recorded"));
+                    Ec2ContainerManager.OperationResult readiness = containerManager.checkSshReadiness(instance);
+                    if (readiness == null) {
+                        instance.setSshStatus("unknown");
+                        instance.setSshStatusMessage("SSH readiness could not be reconstructed after restart");
+                    } else {
+                        instance.setSshStatus(readiness.success() ? "ok" : "impaired");
+                        instance.setSshStatusMessage(readiness.message());
+                    }
+                }
+                persistOwnedInstance(entry, instance);
                 restored++;
                 // Container is running: re-reserve host ports and recreate any missing socat sidecars.
                 if (portForwardManager != null) {
@@ -545,6 +559,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
                 }
             }
         }
+        failUnrecoverablePendingLaunches();
         if (restored > 0) {
             LOG.infov("Restored IMDS metadata registration for {0} EC2 container(s)", restored);
         }
@@ -554,6 +569,68 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
         containerManager.reconcileOrphanedContainers(this::instanceContainerStillWanted);
     }
 
+    private void failUnrecoverablePendingLaunches() {
+        for (AccountAwareStorageBackend.AccountEntry<Instance> entry : allInstanceEntries()) {
+            Instance instance = entry.value();
+            if (instance == null || instance.getState() == null
+                    || (!"pending".equals(instance.getState().getName())
+                    && !"in-progress".equals(instance.getLaunchStatus()))) {
+                continue;
+            }
+            String reason = "Launch state is unknown because core restarted before a backing container was recorded";
+            instance.setState(InstanceState.terminated());
+            instance.setLaunchStatus("failed");
+            instance.setSshStatus("impaired");
+            instance.setSshStatusMessage(reason);
+            instance.setStateReasonCode("Server.InternalError");
+            instance.setStateReasonMessage(reason);
+            instance.setStateTransitionReason(reason);
+            instance.setTerminatedAt(System.currentTimeMillis());
+            instance.getLaunchEvents().add(new InstanceLaunchEvent("startup-reconciliation", "failed",
+                    Instant.now(), reason));
+            persistOwnedInstance(entry, instance);
+        }
+    }
+
+    private List<AccountAwareStorageBackend.AccountEntry<Instance>> allInstanceEntries() {
+        if (instances instanceof AccountAwareStorageBackend<?> rawAccountAware) {
+            @SuppressWarnings("unchecked")
+            AccountAwareStorageBackend<Instance> accountAware =
+                    (AccountAwareStorageBackend<Instance>) rawAccountAware;
+            return accountAware.scanAllAccountEntries(key -> true);
+        }
+        return instances.keys().stream()
+                .map(storageKey -> instances.get(storageKey)
+                        .map(instance -> new AccountAwareStorageBackend.AccountEntry<Instance>(
+                                null, storageKey, instance)))
+                .flatMap(Optional::stream)
+                .toList();
+    }
+
+    private void persistOwnedInstance(AccountAwareStorageBackend.AccountEntry<Instance> entry, Instance instance) {
+        if (entry.accountId() != null && instances instanceof AccountAwareStorageBackend<?> rawAccountAware) {
+            @SuppressWarnings("unchecked")
+            AccountAwareStorageBackend<Instance> accountAware =
+                    (AccountAwareStorageBackend<Instance>) rawAccountAware;
+            accountAware.putForAccount(entry.accountId(), entry.key(), instance);
+            return;
+        }
+        instances.put(entry.key(), instance);
+    }
+
+    private void persistInstance(Instance instance) {
+        if (instance == null || instance.getRegion() == null || instance.getInstanceId() == null) {
+            return;
+        }
+        String storageKey = key(instance.getRegion(), instance.getInstanceId());
+        if (instances instanceof AccountAwareStorageBackend<Instance> accountAware
+                && instance.getAccountId() != null && !instance.getAccountId().isBlank()) {
+            accountAware.putForAccount(instance.getAccountId(), storageKey, instance);
+            return;
+        }
+        instances.put(storageKey, instance);
+    }
+
     /**
      * Whether a container labelled for (region, instanceId) still backs a live instance record.
      * False means the record is gone or already terminated, so the container is an orphan.
@@ -568,13 +645,13 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
     }
 
     private void recoverInterruptedTerminations() {
-        for (String storageKey : instances.keys()) {
-            Instance instance = instances.get(storageKey).orElse(null);
+        for (AccountAwareStorageBackend.AccountEntry<Instance> entry : allInstanceEntries()) {
+            Instance instance = entry.value();
             if (instance == null || instance.getState() == null
                     || !"shutting-down".equals(instance.getState().getName())) {
                 continue;
             }
-            containerManager.terminate(instance, () -> instances.put(storageKey, instance));
+            containerManager.terminate(instance, () -> persistOwnedInstance(entry, instance));
         }
     }
 
@@ -617,8 +694,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
     private void reserveRestoredPrivateIps() {
         int reserved = 0;
         int skipped = 0;
-        for (String storageKey : instances.keys()) {
-            Instance instance = instances.get(storageKey).orElse(null);
+        for (AccountAwareStorageBackend.AccountEntry<Instance> entry : allInstanceEntries()) {
+            Instance instance = entry.value();
             if (instance == null || instance.getSubnetId() == null
                     || instance.getPrivateIpAddress() == null
                     || instance.getState() == null
@@ -2645,6 +2722,10 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
         LaunchTemplateData.MetadataOptions launchMetadataOptions = LaunchTemplateData.MetadataOptions.merge(
                 LaunchTemplateData.MetadataOptions.launchDefaults(), metadataOptions);
         ensureDefaultResources(region);
+        if (keyName != null && findKeyPair(region, keyName) == null) {
+            throw new AwsException("InvalidKeyPair.NotFound",
+                    "The key pair '" + keyName + "' does not exist", 400);
+        }
 
         // floci-kt9: a caller-supplied primary ENI (override-default-eni) governs its own
         // subnet/VPC/security-groups and can only ever back a single instance.
@@ -2739,8 +2820,9 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
                 dockerImage = amiImageResolver.resolveImage(resolveLaunchableImageId(region, imageId));
                 String captured = capturedImageFor(region, imageId);
                 if (captured != null) {
-                    dockerImage = new ResolvedAmiImage(captured, dockerImage.guestRuntime(),
-                            dockerImage.cloudInit(), dockerImage.dockerPlatform());
+                    dockerImage = new ResolvedAmiImage(captured, dockerImage.initType(),
+                            dockerImage.cloudInit(), dockerImage.dockerPlatform(), null, "captured",
+                            dockerImage.sshUser(), dockerImage.sshPreinstalled(), dockerImage.sshServiceName());
                 }
             }
             if (dryRun) {
@@ -2775,6 +2857,11 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
                 inst.setAmiLaunchIndex(i);
                 inst.setClientToken(clientToken);
                 inst.setRegion(region);
+                inst.setAccountId(callerAccountId());
+                inst.setLaunchStatus(config.services().ec2().mock() ? "ready" : "in-progress");
+                inst.setSshStatus(config.services().ec2().mock() ? "ok" : "initializing");
+                inst.getLaunchEvents().add(new InstanceLaunchEvent("launch-requested", "completed",
+                        Instant.now(), "RunInstances accepted the launch"));
                 inst.setUserData(userData);
                 inst.setEncodedUserData(encodedUserData);
                 inst.setIamInstanceProfileArn(iamInstanceProfileArn);
@@ -2854,13 +2941,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
         // Outside the lock: the containers are what the lock protects a reference to, not part of
         // the registry, and a launch is slow.
         if (!config.services().ec2().mock()) {
-            String publicKey = null;
-            if (keyName != null) {
-                KeyPair kp = findKeyPair(region, keyName);
-                if (kp != null) {
-                    publicKey = kp.getPublicKey();
-                }
-            }
+            String publicKey = keyName == null ? null : findKeyPair(region, keyName).getPublicKey();
             for (Instance inst : launched) {
                 List<SecurityGroup> policyGroups = inst.getSecurityGroups().stream()
                         .map(group -> securityGroups.get(key(region, group.getGroupId()))
@@ -3334,7 +3415,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
         }
     }
 
-    public List<Instance> describeInstanceStatus(String region, List<String> instanceIds) {
+    public List<Instance> describeInstanceStatus(String region, List<String> instanceIds,
+                                                  boolean includeAllInstances) {
         ensureDefaultResources(region);
         if (config.services().ec2().mock()) {
             instances.scan(k -> true).stream()
@@ -3345,11 +3427,41 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
                         instances.put(key(i.getRegion(), i.getInstanceId()), i);
                     });
         }
-        return instances.scan(k -> true).stream()
+        List<Instance> matching = instances.scan(k -> true).stream()
                 .filter(i -> i.getRegion().equals(region))
                 .filter(i -> instanceIds.isEmpty() || instanceIds.contains(i.getInstanceId()))
-                .filter(i -> "running".equals(i.getState().getName()))
+                .filter(i -> includeAllInstances || "running".equals(i.getState().getName()))
                 .collect(Collectors.toList());
+        if (!config.services().ec2().mock()) {
+            for (Instance instance : matching) {
+                if (!"running".equals(instance.getState().getName())
+                        || "in-progress".equals(instance.getLaunchStatus())) {
+                    continue;
+                }
+                Ec2ContainerManager.OperationResult result = containerManager.checkSshReadiness(instance);
+                String status = result.success() ? "ok" : "impaired";
+                String message = !result.success()
+                        && "Client.SSHUnavailable".equals(instance.getStateReasonCode())
+                        && instance.getStateReasonMessage() != null
+                        ? instance.getStateReasonMessage() : result.message();
+                if (!Objects.equals(status, instance.getSshStatus())
+                        || !Objects.equals(message, instance.getSshStatusMessage())) {
+                    instance.setSshStatus(status);
+                    instance.setSshStatusMessage(message);
+                    if (result.success() && "failed".equals(instance.getLaunchStatus())
+                            && "Client.SSHUnavailable".equals(instance.getStateReasonCode())) {
+                        instance.setLaunchStatus("ready");
+                        instance.setStateReasonCode(null);
+                        instance.setStateReasonMessage(null);
+                        instance.setStateTransitionReason(null);
+                    }
+                    instance.getLaunchEvents().add(new InstanceLaunchEvent("ssh-status-check",
+                            result.success() ? "completed" : "failed", Instant.now(), message));
+                    persistInstance(instance);
+                }
+            }
+        }
+        return matching;
     }
 
     /**
@@ -4889,6 +5001,38 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
 
     // ─── Key Pairs ─────────────────────────────────────────────────────────────
 
+    private void clearPersistedPrivateKeyMaterial() {
+        int cleared = 0;
+        if (keyPairs instanceof AccountAwareStorageBackend<?> rawAccountAware) {
+            @SuppressWarnings("unchecked")
+            AccountAwareStorageBackend<KeyPair> accountAware =
+                    (AccountAwareStorageBackend<KeyPair>) rawAccountAware;
+            for (AccountAwareStorageBackend.AccountEntry<KeyPair> entry
+                    : accountAware.scanAllAccountEntries(key -> true)) {
+                KeyPair keyPair = entry.value();
+                if (keyPair.getKeyMaterial() == null) {
+                    continue;
+                }
+                keyPair.setKeyMaterial(null);
+                accountAware.putForAccount(entry.accountId(), entry.key(), keyPair);
+                cleared++;
+            }
+        } else {
+            for (String storageKey : keyPairs.keys()) {
+                KeyPair keyPair = keyPairs.get(storageKey).orElse(null);
+                if (keyPair == null || keyPair.getKeyMaterial() == null) {
+                    continue;
+                }
+                keyPair.setKeyMaterial(null);
+                keyPairs.put(storageKey, keyPair);
+                cleared++;
+            }
+        }
+        if (cleared > 0) {
+            LOG.infov("Cleared private material from {0} persisted EC2 key pair record(s)", cleared);
+        }
+    }
+
     public KeyPair createKeyPair(String region, String keyName) {
         ensureDefaultResources(region);
         requireKeyName(keyName);
@@ -4904,7 +5048,13 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
         // Without storing it, a key pair created here could never authenticate anything.
         kp.setPublicKey(generated.openSshPublicKey());
         kp.setRegion(region);
-        keyPairs.put(key(region, keyPairId), kp);
+        KeyPair stored = new KeyPair();
+        stored.setKeyPairId(keyPairId);
+        stored.setKeyName(keyName);
+        stored.setKeyFingerprint(generated.fingerprint());
+        stored.setPublicKey(generated.openSshPublicKey());
+        stored.setRegion(region);
+        keyPairs.put(key(region, keyPairId), stored);
         return kp;
     }
 
@@ -5005,6 +5155,38 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
         return containerManager.isContainerRunning(instance.getDockerContainerId());
     }
 
+    public List<Map<String, Object>> diagnosticInstances() {
+        return instances.scan(key -> true).stream().map(instance -> {
+            boolean containerRunning = isInstanceContainerRunning(instance.getInstanceId());
+            String containerStatus = instance.getDockerContainerId() == null
+                    ? "unavailable" : containerRunning ? "running" : "stopped";
+            Map<String, Object> diagnostic = new LinkedHashMap<>();
+            diagnostic.put("instanceId", instance.getInstanceId());
+            diagnostic.put("sshHost", "localhost");
+            diagnostic.put("sshPort", instance.getSshHostPort());
+            diagnostic.put("containerId", instance.getDockerContainerId());
+            diagnostic.put("containerStatus", containerStatus);
+            diagnostic.put("state", instance.getState() == null ? null : instance.getState().getName());
+            diagnostic.put("launchStatus", instance.getLaunchStatus());
+            synchronized (instance) {
+                diagnostic.put("launchEvents", List.copyOf(instance.getLaunchEvents()));
+            }
+            diagnostic.put("selectedImageType", instance.getSelectedImageType());
+            diagnostic.put("selectedImageReference", instance.getSelectedImageReference());
+            diagnostic.put("sshUser", instance.getSshUser());
+            diagnostic.put("sshPreinstalled", instance.isSshPreinstalled());
+            diagnostic.put("initType", instance.getInitType());
+            diagnostic.put("sshServiceName", instance.getSshServiceName());
+            diagnostic.put("sshStatus", instance.getSshStatus());
+            diagnostic.put("sshStatusMessage", instance.getSshStatusMessage());
+            diagnostic.put("keyName", instance.getKeyName());
+            diagnostic.put("keyInjectionStatus", instance.getKeyInjectionStatus());
+            diagnostic.put("publishedPorts", instance.getPublishedPorts());
+            diagnostic.put("failure", instance.getStateReasonMessage());
+            return diagnostic;
+        }).toList();
+    }
+
     private static void requireKeyName(String keyName) {
         if (keyName == null || keyName.isBlank()) {
             throw new AwsException("MissingParameter", "The request must contain the parameter KeyName", 400);

src/main/java/io/github/hectorvent/floci/services/ec2/ResolvedAmiImage.java

@@ -1,15 +1,17 @@
 package io.github.hectorvent.floci.services.ec2;
 
-public record ResolvedAmiImage(String dockerImage, String guestRuntime, boolean cloudInit,
-                               String dockerPlatform) {
+public record ResolvedAmiImage(String dockerImage, String initType, boolean cloudInit,
+                               String dockerPlatform, String fallbackDockerImage, String imageType,
+                               String sshUser, boolean sshPreinstalled, String sshServiceName) {
     public static final String DEFAULT_RUNTIME = "minimal";
     public static final String SYSTEMD_RUNTIME = "systemd";
 
     public static ResolvedAmiImage minimal(String dockerImage) {
-        return new ResolvedAmiImage(dockerImage, DEFAULT_RUNTIME, false, null);
+        return new ResolvedAmiImage(dockerImage, DEFAULT_RUNTIME, false, null, null,
+                "public", "root", false, "sshd");
     }
 
     public boolean systemd() {
-        return SYSTEMD_RUNTIME.equals(guestRuntime);
+        return SYSTEMD_RUNTIME.equals(initType);
     }
 }

src/main/java/io/github/hectorvent/floci/services/ec2/model/Instance.java

@@ -48,8 +48,21 @@ public class Instance {
     private String stateReasonCode;
     private String stateReasonMessage;
     private String region;
+    private String accountId;
     private List<Tag> tags = new ArrayList<>();
 
+    private String launchStatus;
+    private List<InstanceLaunchEvent> launchEvents = new ArrayList<>();
+    private String selectedImageType;
+    private String selectedImageReference;
+    private String sshUser;
+    private boolean sshPreinstalled;
+    private String initType;
+    private String sshServiceName;
+    private String sshStatus;
+    private String sshStatusMessage;
+    private String keyInjectionStatus;
+
     private String rootVolumeId;
 
     // NOTE: disableApiStop and disableApiTermination are stored but ModifyInstanceAttribute
@@ -192,9 +205,50 @@ public class Instance {
     public String getRegion() { return region; }
     public void setRegion(String region) { this.region = region; }
 
+    public String getAccountId() { return accountId; }
+    public void setAccountId(String accountId) { this.accountId = accountId; }
+
     public List<Tag> getTags() { return tags; }
     public void setTags(List<Tag> tags) { this.tags = tags; }
 
+    public String getLaunchStatus() { return launchStatus; }
+    public void setLaunchStatus(String launchStatus) { this.launchStatus = launchStatus; }
+
+    public List<InstanceLaunchEvent> getLaunchEvents() {
+        if (launchEvents == null) {
+            launchEvents = new ArrayList<>();
+        }
+        return launchEvents;
+    }
+    public void setLaunchEvents(List<InstanceLaunchEvent> launchEvents) { this.launchEvents = launchEvents; }
+
+    public String getSelectedImageType() { return selectedImageType; }
+    public void setSelectedImageType(String selectedImageType) { this.selectedImageType = selectedImageType; }
+
+    public String getSelectedImageReference() { return selectedImageReference; }
+    public void setSelectedImageReference(String selectedImageReference) { this.selectedImageReference = selectedImageReference; }
+
+    public String getSshUser() { return sshUser; }
+    public void setSshUser(String sshUser) { this.sshUser = sshUser; }
+
+    public boolean isSshPreinstalled() { return sshPreinstalled; }
+    public void setSshPreinstalled(boolean sshPreinstalled) { this.sshPreinstalled = sshPreinstalled; }
+
+    public String getInitType() { return initType; }
+    public void setInitType(String initType) { this.initType = initType; }
+
+    public String getSshServiceName() { return sshServiceName; }
+    public void setSshServiceName(String sshServiceName) { this.sshServiceName = sshServiceName; }
+
+    public String getSshStatus() { return sshStatus; }
+    public void setSshStatus(String sshStatus) { this.sshStatus = sshStatus; }
+
+    public String getSshStatusMessage() { return sshStatusMessage; }
+    public void setSshStatusMessage(String sshStatusMessage) { this.sshStatusMessage = sshStatusMessage; }
+
+    public String getKeyInjectionStatus() { return keyInjectionStatus; }
+    public void setKeyInjectionStatus(String keyInjectionStatus) { this.keyInjectionStatus = keyInjectionStatus; }
+
     public String getDockerContainerId() { return dockerContainerId; }
     public void setDockerContainerId(String dockerContainerId) { this.dockerContainerId = dockerContainerId; }
 

src/main/java/io/github/hectorvent/floci/services/ec2/model/InstanceLaunchEvent.java

@@ -0,0 +1,37 @@
+package io.github.hectorvent.floci.services.ec2.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class InstanceLaunchEvent {
+
+    private String stage;
+    private String status;
+    private Instant timestamp;
+    private String message;
+
+    public InstanceLaunchEvent() {}
+
+    public InstanceLaunchEvent(String stage, String status, Instant timestamp, String message) {
+        this.stage = stage;
+        this.status = status;
+        this.timestamp = timestamp;
+        this.message = message;
+    }
+
+    public String getStage() { return stage; }
+    public void setStage(String stage) { this.stage = stage; }
+
+    public String getStatus() { return status; }
+    public void setStatus(String status) { this.status = status; }
+
+    public Instant getTimestamp() { return timestamp; }
+    public void setTimestamp(Instant timestamp) { this.timestamp = timestamp; }
+
+    public String getMessage() { return message; }
+    public void setMessage(String message) { this.message = message; }
+}

src/main/java/io/github/hectorvent/floci/services/ecs/EcsLoadBalancerRegistrar.java

@@ -61,6 +61,11 @@ public class EcsLoadBalancerRegistrar {
                 throw new AwsException("TargetGroupNotFound",
                         "Target group " + loadBalancer.getTargetGroupArn() + " was not found.", 400);
             }
+            if (taskDefinition.isFargateCompatible() && !"ip".equals(targetGroup.getTargetType())) {
+                throw new AwsException("InvalidParameterException",
+                        "The target group " + loadBalancer.getTargetGroupArn()
+                                + " must have target type ip for a Fargate service.", 400);
+            }
             boolean declared = taskDefinition.getContainerDefinitions().stream()
                     .filter(container -> loadBalancer.getContainerName().equals(container.getName()))
                     .flatMap(container -> container.getPortMappings() == null
@@ -78,7 +83,16 @@ public class EcsLoadBalancerRegistrar {
 
     /** Registers the task's load-balanced containers as ELBv2 targets. */
     public void registerTask(EcsTask task, EcsServiceModel svc, String region) {
-        forEachTarget(task, svc, (tgArn, td) -> {
+        register(task, svc.getLoadBalancers(), region);
+    }
+
+    /** Registers a task-set task against the load balancers declared by that task set. */
+    public void registerTaskSetTask(EcsTask task, List<EcsLoadBalancer> loadBalancers, String region) {
+        register(task, loadBalancers, region);
+    }
+
+    private void register(EcsTask task, List<EcsLoadBalancer> loadBalancers, String region) {
+        forEachTarget(task, loadBalancers, (tgArn, td) -> {
             try {
                 elbV2Service.registerTargets(region, tgArn, List.of(td));
                 LOG.infov("Registered ECS task target {0}:{1} into target group {2}",
@@ -91,7 +105,16 @@ public class EcsLoadBalancerRegistrar {
 
     /** Deregisters the task's load-balanced containers from their ELBv2 target groups. */
     public void deregisterTask(EcsTask task, EcsServiceModel svc, String region) {
-        forEachTarget(task, svc, (tgArn, td) -> {
+        deregister(task, svc.getLoadBalancers(), region);
+    }
+
+    /** Deregisters a task-set task from the load balancers declared by that task set. */
+    public void deregisterTaskSetTask(EcsTask task, List<EcsLoadBalancer> loadBalancers, String region) {
+        deregister(task, loadBalancers, region);
+    }
+
+    private void deregister(EcsTask task, List<EcsLoadBalancer> loadBalancers, String region) {
+        forEachTarget(task, loadBalancers, (tgArn, td) -> {
             try {
                 elbV2Service.deregisterTargets(region, tgArn, List.of(td));
                 LOG.infov("Deregistered ECS task target {0}:{1} from target group {2}",
@@ -102,15 +125,15 @@ public class EcsLoadBalancerRegistrar {
         });
     }
 
-    private void forEachTarget(EcsTask task, EcsServiceModel svc,
+    private void forEachTarget(EcsTask task, List<EcsLoadBalancer> loadBalancers,
                                 BiConsumer<String, TargetDescription> action) {
-        if (svc == null || svc.getLoadBalancers() == null || svc.getLoadBalancers().isEmpty()) {
+        if (loadBalancers == null || loadBalancers.isEmpty()) {
             return;
         }
         if (task.getContainers() == null || task.getContainers().isEmpty()) {
             return;
         }
-        for (EcsLoadBalancer lb : svc.getLoadBalancers()) {
+        for (EcsLoadBalancer lb : loadBalancers) {
             if (lb.getTargetGroupArn() == null || lb.getTargetGroupArn().isBlank()) {
                 continue;
             }

src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java

@@ -1166,8 +1166,8 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         }
         EcsCluster cluster = resolveClusterOrDefault(request.getCluster(), region);
         TaskDefinition taskDef = resolveLaunchableTaskDefinition(request.getTaskDefinition(), region);
-        validateEc2BridgeLaunch(cluster, taskDef, request);
-        return launchTasks(cluster, taskDef, request, null, null, region);
+        validateRuntimeLaunch(cluster, taskDef, request);
+        return launchTasks(cluster, taskDef, request, null, null, null, region);
     }
 
     public List<EcsTask> startTask(String clusterRef, List<String> containerInstanceRefs,
@@ -1204,7 +1204,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         for (String instanceRef : instanceRefs) {
             ContainerInstance instance = resolveContainerInstanceOrThrow(cluster.getClusterArn(), instanceRef);
             result.addAll(launchTasks(cluster, taskDef, request,
-                    instance.getContainerInstanceArn(), null, region));
+                    instance.getContainerInstanceArn(), null, null, region));
         }
         return result;
     }
@@ -1218,7 +1218,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
      */
     private List<EcsTask> launchTasks(EcsCluster cluster, TaskDefinition taskDef, RunTaskRequest request,
                                        String containerInstanceArn,
-                                       String owningServiceArn, String region) {
+                                       String owningServiceArn, String taskSetArn, String region) {
         // Fail loudly instead of silently launching zero containers and leaving
         // a task that looks RUNNING with nothing behind it.
         if (taskDef.getContainerDefinitions() == null || taskDef.getContainerDefinitions().isEmpty()) {
@@ -1248,6 +1248,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
             task.setGroup(request.getGroup() != null ? request.getGroup() : "family:" + taskDef.getFamily());
             task.setStartedBy(request.getStartedBy());
             task.setOwningServiceArn(owningServiceArn);
+            task.setTaskSetArn(taskSetArn);
             task.setLastStatus(TaskStatus.PENDING.name());
             task.setDesiredStatus(TaskStatus.RUNNING.name());
             task.setCpu(effectiveCpu(taskDef, request));
@@ -1399,7 +1400,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
             request.setPropagateTags(PROPAGATE_TAGS_TASK_DEFINITION);
         }
         EcsTask task = launchTasks(cluster, taskDef, request, containerInstanceArn,
-                svc.getServiceArn(), region).getFirst();
+                svc.getServiceArn(), null, region).getFirst();
         task.setDeploymentId(deploymentId(svc));
         return task;
     }
@@ -1553,26 +1554,34 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         return new Placement(launchType, capacityProvider);
     }
 
-    /** Floci's Docker-backed service implementation supports EC2 bridge tasks only. */
-    private void validateEc2BridgeLaunch(EcsCluster cluster, TaskDefinition taskDef, RunTaskRequest request) {
-        if (taskDef.getNetworkMode() != NetworkMode.bridge) {
+    /**
+     * Keeps the established EC2 bridge restriction while allowing Docker-backed Fargate tasks to
+     * use their awsvpc task ENI. A capacity-provider strategy determines the effective launch
+     * type, so validate every placement rather than only the explicit launchType member.
+     */
+    private void validateRuntimeLaunch(EcsCluster cluster, TaskDefinition taskDef, RunTaskRequest request) {
+        List<Placement> placements = resolvePlacements(cluster, request, 1);
+        boolean fargate = placements.stream().anyMatch(placement -> placement.launchType() == LaunchType.FARGATE);
+        boolean ec2 = placements.stream().anyMatch(placement -> placement.launchType() == LaunchType.EC2);
+        if (fargate && ec2) {
+            throw new AwsException("ClientException",
+                    "A task definition cannot be placed on both EC2 and FARGATE capacity providers.", 400);
+        }
+        if (fargate) {
+            if (taskDef.getNetworkMode() != NetworkMode.awsvpc) {
+                throw new AwsException("ClientException",
+                        "Fargate tasks require the awsvpc network mode.", 400);
+            }
+            if (!taskDef.isFargateCompatible()) {
+                throw new AwsException("ClientException",
+                        "The task definition is not compatible with the FARGATE launch type.", 400);
+            }
+            return;
+        }
+        if (ec2 && taskDef.getNetworkMode() != NetworkMode.bridge) {
             throw new AwsException("ClientException",
                     "Only bridge network mode is supported for ECS EC2 tasks.", 400);
         }
-        LaunchType launchType = request.getLaunchType();
-        if (launchType != null && launchType != LaunchType.EC2) {
-            throw new AwsException("ClientException",
-                    "Only the EC2 launch type is supported; FARGATE is not available.", 400);
-        }
-        List<CapacityProviderStrategyItem> strategy = request.getCapacityProviderStrategy();
-        if (strategy == null || strategy.isEmpty()) {
-            strategy = clusterDefaultStrategy(cluster);
-        }
-        if (strategy != null && strategy.stream()
-                .anyMatch(item -> FARGATE_CAPACITY_PROVIDERS.contains(item.capacityProvider()))) {
-            throw new AwsException("ClientException",
-                    "FARGATE capacity providers are not supported; use an EC2 capacity provider.", 400);
-        }
     }
 
     /** Accepts the two built-in Fargate providers as well as anything CreateCapacityProvider made. */
@@ -1730,6 +1739,13 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
 
     /** Registers a freshly-started task's containers as ELBv2 targets if its service is load-balanced. */
     private void registerTaskWithLoadBalancers(EcsTask task, EcsCluster cluster, String region) {
+        if (task.getTaskSetArn() != null) {
+            TaskSet taskSet = taskSets.get(task.getTaskSetArn());
+            if (taskSet != null) {
+                lbRegistrar.registerTaskSetTask(task, taskSet.getLoadBalancers(), region);
+            }
+            return;
+        }
         EcsServiceModel svc = owningService(task, cluster);
         if (svc != null && !svc.getLoadBalancers().isEmpty()) {
             lbRegistrar.registerTask(task, svc, region);
@@ -1764,7 +1780,12 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
             return;
         }
         EcsServiceModel svc = owningService(task, cluster);
-        if (svc != null && !svc.getLoadBalancers().isEmpty()) {
+        if (task.getTaskSetArn() != null) {
+            TaskSet taskSet = taskSets.get(task.getTaskSetArn());
+            if (taskSet != null) {
+                lbRegistrar.deregisterTaskSetTask(task, taskSet.getLoadBalancers(), region);
+            }
+        } else if (svc != null && !svc.getLoadBalancers().isEmpty()) {
             lbRegistrar.deregisterTask(task, svc, region);
         }
     }
@@ -2052,7 +2073,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         RunTaskRequest launchValidation = new RunTaskRequest();
         launchValidation.setLaunchType(request.getLaunchType());
         launchValidation.setCapacityProviderStrategy(request.getCapacityProviderStrategy());
-        validateEc2BridgeLaunch(cluster, taskDef, launchValidation);
+        validateRuntimeLaunch(cluster, taskDef, launchValidation);
         lbRegistrar.validateServiceLoadBalancers(request.getLoadBalancers(), taskDef, region);
         String taskDefinition = taskDef.getTaskDefinitionArn();
         String serviceName = request.getServiceName();
@@ -2132,6 +2153,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         cluster.setActiveServicesCount(cluster.getActiveServicesCount() + 1);
         persistCluster(region, cluster);
         recordServiceDeployment(svc, taskDefinition, region);
+        if (svc.getLaunchType() == LaunchType.FARGATE && "CODE_DEPLOY".equals(svc.getDeploymentController())) {
+            createInitialCodeDeployTaskSet(cluster, svc, region);
+        }
         if (eventPublisher != null) {
             eventPublisher.emitDeploymentStateChange(svc, "SERVICE_DEPLOYMENT_STARTED",
                     "ECS deployment " + svc.getDeploymentId() + " started.", region);
@@ -2140,6 +2164,31 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         return svc;
     }
 
+    /**
+     * A Fargate CodeDeploy service starts with its blue deployment already placed. EC2 services
+     * retain their existing controller-owned task-set lifecycle.
+     */
+    private void createInitialCodeDeployTaskSet(EcsCluster cluster, EcsServiceModel svc, String region) {
+        CreateTaskSetRequest request = new CreateTaskSetRequest();
+        request.setCluster(cluster.getClusterName());
+        request.setService(svc.getServiceName());
+        request.setTaskDefinition(svc.getTaskDefinition());
+        if (svc.getCapacityProviderStrategy() != null && !svc.getCapacityProviderStrategy().isEmpty()) {
+            request.setCapacityProviderStrategy(svc.getCapacityProviderStrategy());
+        } else {
+            request.setLaunchType(LaunchType.FARGATE);
+        }
+        request.setPlatformVersion(svc.getPlatformVersion());
+        request.setScaleValue(100.0);
+        request.setNetworkConfiguration(svc.getNetworkConfiguration());
+        request.setLoadBalancers(svc.getLoadBalancers());
+        request.setServiceRegistries(svc.getServiceRegistries());
+
+        TaskSet taskSet = createTaskSet(request, region);
+        taskSet.setStatus("PRIMARY");
+        taskSet.setUpdatedAt(Instant.now());
+    }
+
     public EcsServiceModel updateService(String clusterRef, String serviceName, String taskDefinition,
                                           Integer desiredCount, NetworkConfiguration networkConfiguration,
                                           String region) {
@@ -2436,6 +2485,20 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
                                 t.getTaskArn(), e.getMessage());
                     }
                 });
+        // Service ARNs are reused when a same-named service is recreated. Retaining its task
+        // sets would make the new service reconcile and relaunch the deleted deployment.
+        taskSets.entrySet().removeIf(entry -> svc.getServiceArn().equals(entry.getValue().getServiceArn()));
+        // Inactive services are not reconciled, so retain the final observed task state rather
+        // than leaving the count that was recorded before the service drain.
+        svc.setRunningCount((int) tasks.values().stream()
+                .filter(t -> ownedBy(t, svc, cluster))
+                .filter(t -> TaskStatus.RUNNING.name().equals(t.getLastStatus()))
+                .count());
+        svc.setPendingCount((int) tasks.values().stream()
+                .filter(t -> ownedBy(t, svc, cluster))
+                .filter(t -> TaskStatus.PENDING.name().equals(t.getLastStatus()))
+                .count());
+        services.put(key, svc);
         return svc;
     }
 
@@ -3285,7 +3348,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
     private static final Set<String> TASK_SET_CONTROLLERS = Set.of("EXTERNAL", "CODE_DEPLOY");
     /** The only unit {@code Scale} accepts, so the computed count is always a percentage. */
     private static final String SCALE_UNIT_PERCENT = "PERCENT";
-    /** The stability a task set reports while Floci places no tasks of its own for one. */
+    /** Task sets report steady state when their running task count reaches the computed desired count. */
     private static final String STABILITY_STATUS_STEADY_STATE = "STEADY_STATE";
 
     public TaskSet createTaskSet(String clusterRef, String serviceRef, String taskDefinitionRef,
@@ -3363,7 +3426,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         ts.setUpdatedAt(Instant.now());
         applyTaskSetScale(ts, svc);
 
+        lbRegistrar.validateServiceLoadBalancers(ts.getLoadBalancers(), taskDef, region);
         taskSets.put(taskSetArn, ts);
+        reconcileTaskSet(cluster, svc, ts, region);
         return ts;
     }
 
@@ -3391,15 +3456,54 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
      * Stamps {@code computedDesiredCount}, which AWS derives from the service's desired count and
      * the task set's scale percentage, always rounding up: a computed 1.2 becomes 2 tasks.
      *
-     * <p>Floci places no tasks for a task set, so its running and pending counts stay at zero. A
-     * stability status derived from them would leave every set on a service with a desired count
-     * above zero in {@code STABILIZING} for ever, hanging anything that waits for the set to
-     * stabilise, so the set is reported {@code STEADY_STATE} until Floci tracks tasks for it.
      */
     private void applyTaskSetScale(TaskSet ts, EcsServiceModel svc) {
         ts.setComputedDesiredCount(
                 (int) Math.ceil(svc.getDesiredCount() * ts.getScaleValue() / 100.0));
-        ts.setStabilityStatus(STABILITY_STATUS_STEADY_STATE);
+    }
+
+    /** Places and drains task-set tasks with the same runtime launcher used by the service scheduler. */
+    private void reconcileTaskSet(EcsCluster cluster, EcsServiceModel svc, TaskSet ts, String region) {
+        List<EcsTask> running = tasks.values().stream()
+                .filter(task -> ts.getTaskSetArn().equals(task.getTaskSetArn()))
+                .filter(task -> TaskStatus.RUNNING.name().equals(task.getLastStatus()))
+                .toList();
+        int toStart = ts.getComputedDesiredCount() - running.size();
+        if (toStart > 0) {
+            TaskDefinition taskDef = resolveTaskDefinitionOrThrow(ts.getTaskDefinition(), region);
+            for (int i = 0; i < toStart; i++) {
+                RunTaskRequest request = new RunTaskRequest();
+                request.setCount(1);
+                request.setGroup("service:" + svc.getServiceName());
+                request.setStartedBy(ts.getStartedBy());
+                request.setNetworkConfiguration(ts.getNetworkConfiguration());
+                request.setCapacityProviderStrategy(ts.getCapacityProviderStrategy());
+                if (ts.getCapacityProviderStrategy() == null || ts.getCapacityProviderStrategy().isEmpty()) {
+                    request.setLaunchType(ts.getLaunchType());
+                }
+                validateRuntimeLaunch(cluster, taskDef, request);
+                launchTasks(cluster, taskDef, request, null, svc.getServiceArn(), ts.getTaskSetArn(), region);
+            }
+        } else if (toStart < 0) {
+            running.stream().limit(-toStart).forEach(task -> stopTask(cluster.getClusterName(), task.getTaskArn(),
+                    "Task set scale-in", STOP_CODE_SERVICE_SCHEDULER_INITIATED, region));
+        }
+        updateTaskSetRuntime(ts);
+    }
+
+    private void updateTaskSetRuntime(TaskSet ts) {
+        int running = (int) tasks.values().stream()
+                .filter(task -> ts.getTaskSetArn().equals(task.getTaskSetArn()))
+                .filter(task -> TaskStatus.RUNNING.name().equals(task.getLastStatus()))
+                .count();
+        int pending = (int) tasks.values().stream()
+                .filter(task -> ts.getTaskSetArn().equals(task.getTaskSetArn()))
+                .filter(task -> TaskStatus.PENDING.name().equals(task.getLastStatus()))
+                .count();
+        ts.setRunningCount(running);
+        ts.setPendingCount(pending);
+        ts.setStabilityStatus(running == ts.getComputedDesiredCount() && pending == 0
+                ? STABILITY_STATUS_STEADY_STATE : "STABILIZING");
         ts.setStabilityStatusAt(Instant.now());
     }
 
@@ -3416,6 +3520,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         ts.setScaleUnit(SCALE_UNIT_PERCENT);
         ts.setUpdatedAt(Instant.now());
         applyTaskSetScale(ts, svc);
+        reconcileTaskSet(cluster, svc, ts, region);
         return ts;
     }
 
@@ -3436,6 +3541,11 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         }
         ts.setStatus("DRAINING");
         ts.setUpdatedAt(Instant.now());
+        tasks.values().stream()
+                .filter(task -> ts.getTaskSetArn().equals(task.getTaskSetArn()))
+                .filter(task -> !TaskStatus.STOPPED.name().equals(task.getLastStatus()))
+                .forEach(task -> stopTask(cluster.getClusterName(), task.getTaskArn(),
+                        "Task set deleted", STOP_CODE_SERVICE_SCHEDULER_INITIATED, region));
         taskSets.remove(ts.getTaskSetArn());
         return ts;
     }
@@ -4128,9 +4238,20 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         if (!"ACTIVE".equals(svc.getStatus())) {
             return;
         }
-
+        String deploymentController = svc.getDeploymentController() != null
+                ? svc.getDeploymentController() : DEFAULT_DEPLOYMENT_CONTROLLER;
         String region = extractRegionFromServiceKey(key);
         String clusterName = extractClusterNameFromServiceKey(key);
+        if (!DEFAULT_DEPLOYMENT_CONTROLLER.equals(deploymentController)) {
+            EcsCluster cluster = resolveClusterOrDefault(clusterName, region);
+            for (TaskSet taskSet : taskSetsFor(svc)) {
+                reconcileTaskSet(cluster, svc, taskSet, region);
+            }
+            // Task sets are scheduled by their CodeDeploy or external controller, not by the
+            // normal service reconciler. Otherwise green tasks would be supplemented by a second
+            // desiredCount-worth of regular service tasks on every tick.
+            return;
+        }
 
         if (SCHEDULING_DAEMON.equals(svc.getSchedulingStrategy())) {
             reconcileDaemonService(key, svc, clusterName, region);
@@ -4140,6 +4261,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         EcsCluster cluster = resolveClusterOrDefault(clusterName, region);
         List<EcsTask> runningTasks = tasks.values().stream()
                 .filter(t -> ownedBy(t, svc, cluster))
+                // Task sets own their desired count. Including them here lets the rolling-service
+                // reconciler replace a CodeDeploy green task before traffic is switched.
+                .filter(t -> t.getTaskSetArn() == null)
                 .filter(t -> TaskStatus.RUNNING.name().equals(t.getLastStatus()))
                 .toList();
         long running = runningTasks.size();

src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java

@@ -226,6 +226,9 @@ public class EcsContainerManager {
                     metadataId));
             imagesByContainer.put(def, ecrRegistryManager.rewriteImageUri(def.getImage()));
         }
+        for (String image : new HashSet<>(imagesByContainer.values())) {
+            lifecycleManager.refreshImage(image);
+        }
 
         PreparedNetwork protectedNetwork = prepareNetwork(task, taskDef, region, taskId);
 

src/main/java/io/github/hectorvent/floci/services/ecs/model/EcsTask.java

@@ -21,6 +21,8 @@ public class EcsTask {
      * caller-supplied label and therefore cannot be trusted to establish ownership.
      */
     private String owningServiceArn;
+    /** Task set that launched this service-owned task, or null for a rolling-service task. */
+    private String taskSetArn;
     private LaunchType launchType;
     private String lastStatus;
     private String desiredStatus;
@@ -81,6 +83,7 @@ public class EcsTask {
         this.taskDefinitionArn = other.taskDefinitionArn;
         this.group = other.group;
         this.owningServiceArn = other.owningServiceArn;
+        this.taskSetArn = other.taskSetArn;
         this.launchType = other.launchType;
         this.lastStatus = other.lastStatus;
         this.desiredStatus = other.desiredStatus;
@@ -138,6 +141,9 @@ public class EcsTask {
     public String getOwningServiceArn() { return owningServiceArn; }
     public void setOwningServiceArn(String owningServiceArn) { this.owningServiceArn = owningServiceArn; }
 
+    public String getTaskSetArn() { return taskSetArn; }
+    public void setTaskSetArn(String taskSetArn) { this.taskSetArn = taskSetArn; }
+
     public LaunchType getLaunchType() { return launchType; }
     public void setLaunchType(LaunchType launchType) { this.launchType = launchType; }
 

src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2Service.java

@@ -148,10 +148,13 @@ public class ElbV2Service implements ResourceProvider {
             }
         }
         for (Map.Entry<String, Map<String, TargetGroup>> regionEntry : targetGroups.entrySet()) {
+            Set<String> existingLoadBalancers = loadBalancers.getOrDefault(regionEntry.getKey(), Map.of()).keySet();
             for (TargetGroup targetGroup : regionEntry.getValue().values()) {
+                targetGroup.getLoadBalancerArns().removeIf(lbArn -> !existingLoadBalancers.contains(lbArn));
                 tgToLbs.computeIfAbsent(targetGroup.getTargetGroupArn(), k -> ConcurrentHashMap.newKeySet())
                         .addAll(targetGroup.getLoadBalancerArns());
             }
+            targetGroups.put(regionEntry.getKey(), regionEntry.getValue());
         }
     }
 
@@ -287,8 +290,10 @@ public class ElbV2Service implements ResourceProvider {
             listeners.put(region, regionListeners);
             rules.put(region, regionRules);
         }
-        // remove from TG index
-        tgToLbs.values().forEach(lbSet -> lbSet.remove(arn));
+        // A deleted load balancer cannot keep a target group in use. Persist the removal too,
+        // otherwise restart rebuilds this index from stale LoadBalancerArns on the target group.
+        unlinkUnreferencedTargetGroups(region, arn);
+        persistRegion(targetGroups, region);
         tags.remove(arn);
         loadBalancers.put(region, regionLbs);
     }
@@ -1099,6 +1104,7 @@ public class ElbV2Service implements ResourceProvider {
         if (defaultRule == null) {
             return;
         }
+        Listener listener = requireListener(region, listenerArn);
         Action action = new Action();
         action.setType("forward");
         if (greenWeightPct >= 100) {
@@ -1113,6 +1119,9 @@ public class ElbV2Service implements ResourceProvider {
             action.setTargetGroups(List.of(blueTuple, greenTuple));
         }
         defaultRule.setActions(List.of(action));
+        listener.setDefaultActions(List.of(action));
+        persistRegion(rules, region);
+        persistRegion(listeners, region);
         dataPlane.recompileRules(listenerArn, getListenerRules(region, listenerArn));
     }
 

src/main/java/io/github/hectorvent/floci/services/kafkaconnect/KafkaConnectController.java

@@ -0,0 +1,73 @@
+package io.github.hectorvent.floci.services.kafkaconnect;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.services.kafkaconnect.model.KafkaConnectConnector;
+import io.github.hectorvent.floci.services.kafkaconnect.model.KafkaConnectCustomPlugin;
+import io.github.hectorvent.floci.services.kafkaconnect.model.KafkaConnectWorkerConfiguration;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/** AWS MSK Connect REST JSON control plane. */
+@Path("/")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public class KafkaConnectController {
+    private final KafkaConnectService service;
+    private final ObjectMapper objectMapper;
+
+    @Inject
+    public KafkaConnectController(KafkaConnectService service, ObjectMapper objectMapper) { this.service = service; this.objectMapper = objectMapper; }
+
+    @POST @Path("/v1/custom-plugins")
+    public Response createCustomPlugin(JsonNode request) {
+        JsonNode location = request.path("location").path("s3Location");
+        KafkaConnectCustomPlugin plugin = service.createPlugin(text(request, "name"), text(request, "contentType"),
+                text(location, "bucketArn"), text(location, "fileKey"), strings(request.get("tags")));
+        return Response.ok(pluginView(plugin)).build();
+    }
+    @GET @Path("/v1/custom-plugins") public Response listCustomPlugins() { return Response.ok(Map.of("customPlugins", service.plugins().stream().map(this::pluginView).toList())).build(); }
+    @GET @Path("/v1/custom-plugins/{arn}") public Response describeCustomPlugin(@PathParam("arn") String arn) { return Response.ok(pluginView(service.plugin(arn))).build(); }
+    @DELETE @Path("/v1/custom-plugins/{arn}") public Response deleteCustomPlugin(@PathParam("arn") String arn) { service.deletePlugin(arn); return Response.ok(Map.of("customPluginArn", arn)).build(); }
+
+    @POST @Path("/v1/worker-configurations") public Response createWorkerConfiguration(JsonNode request) { return Response.ok(workerView(service.createWorkerConfiguration(text(request, "name"), text(request, "propertiesFileContent")))).build(); }
+    @GET @Path("/v1/worker-configurations") public Response listWorkerConfigurations() { return Response.ok(Map.of("workerConfigurations", service.workers().stream().map(this::workerView).toList())).build(); }
+    @GET @Path("/v1/worker-configurations/{arn}") public Response describeWorkerConfiguration(@PathParam("arn") String arn) { return Response.ok(workerView(service.worker(arn))).build(); }
+    @DELETE @Path("/v1/worker-configurations/{arn}") public Response deleteWorkerConfiguration(@PathParam("arn") String arn) { service.deleteWorker(arn); return Response.ok(Map.of("workerConfigurationArn", arn)).build(); }
+
+    @POST @Path("/v1/connectors")
+    public Response createConnector(JsonNode request) {
+        JsonNode cluster = request.path("kafkaCluster").path("apacheKafkaCluster");
+        JsonNode plugin = request.path("plugins");
+        String pluginArn = plugin.isArray() && !plugin.isEmpty() ? text(plugin.get(0).path("customPlugin"), "customPluginArn") : null;
+        JsonNode worker = request.path("workerConfiguration");
+        String bootstrapServers = cluster.path("bootstrapServers").isArray()
+                ? cluster.path("bootstrapServers").path(0).asText(null) : text(cluster, "bootstrapServers");
+        KafkaConnectConnector connector = service.createConnector(text(request, "connectorName"), bootstrapServers, pluginArn,
+                text(worker, "workerConfigurationArn"), strings(request.get("connectorConfiguration")), strings(request.get("tags")));
+        return Response.ok(connectorView(connector)).build();
+    }
+    @GET @Path("/v1/connectors") public Response listConnectors() { return Response.ok(Map.of("connectors", service.connectors().stream().map(this::connectorView).toList())).build(); }
+    @GET @Path("/v1/connectors/{arn}") public Response describeConnector(@PathParam("arn") String arn) { return Response.ok(connectorView(service.connector(arn))).build(); }
+    @PUT @Path("/v1/connectors/{arn}") public Response updateConnector(@PathParam("arn") String arn, JsonNode request) { return Response.ok(connectorView(service.updateConnector(arn, strings(request.get("connectorConfiguration"))))).build(); }
+    @DELETE @Path("/v1/connectors/{arn}") public Response deleteConnector(@PathParam("arn") String arn) { service.deleteConnector(arn); return Response.ok(Map.of("connectorArn", arn)).build(); }
+
+    private Map<String, Object> pluginView(KafkaConnectCustomPlugin plugin) { return Map.of("customPluginArn", plugin.getArn(), "name", plugin.getName(), "customPluginName", plugin.getName(), "contentType", plugin.getContentType(), "creationTime", plugin.getCreationTime(), "state", "ACTIVE", "contentLocation", Map.of("s3Location", Map.of("bucketArn", plugin.getBucketArn(), "fileKey", plugin.getFileKey()))); }
+    private Map<String, Object> workerView(KafkaConnectWorkerConfiguration worker) { return Map.of("workerConfigurationArn", worker.getArn(), "name", worker.getName(), "revision", worker.getRevision(), "creationTime", worker.getCreationTime(), "propertiesFileContent", worker.getPropertiesFileContent()); }
+    private Map<String, Object> connectorView(KafkaConnectConnector connector) { Map<String, Object> view = new LinkedHashMap<>(); view.put("connectorArn", connector.getArn()); view.put("connectorName", connector.getName()); view.put("connectorState", connector.getState()); view.put("currentVersion", String.valueOf(connector.getCurrentVersion())); view.put("creationTime", connector.getCreationTime()); view.put("connectorConfiguration", connector.getConnectorConfiguration()); return view; }
+    private String text(JsonNode node, String name) { JsonNode value = node.get(name); return value == null || value.isNull() ? null : value.asText(); }
+    private Map<String, String> strings(JsonNode node) { Map<String, String> values = new LinkedHashMap<>(); if (node != null && node.isObject()) node.fields().forEachRemaining(entry -> values.put(entry.getKey(), entry.getValue().asText())); return values; }
+}

src/main/java/io/github/hectorvent/floci/services/kafkaconnect/KafkaConnectService.java

@@ -0,0 +1,125 @@
+package io.github.hectorvent.floci.services.kafkaconnect;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.kafkaconnect.model.KafkaConnectConnector;
+import io.github.hectorvent.floci.services.kafkaconnect.model.KafkaConnectCustomPlugin;
+import io.github.hectorvent.floci.services.kafkaconnect.model.KafkaConnectWorkerConfiguration;
+import io.github.hectorvent.floci.services.msk.MskService;
+import io.github.hectorvent.floci.services.msk.model.MskCluster;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+@ApplicationScoped
+public class KafkaConnectService {
+    private static final Logger LOG = Logger.getLogger(KafkaConnectService.class);
+    private final StorageBackend<String, KafkaConnectCustomPlugin> plugins;
+    private final StorageBackend<String, KafkaConnectConnector> connectors;
+    private final StorageBackend<String, KafkaConnectWorkerConfiguration> workers;
+    private final RegionResolver regionResolver;
+    private final S3Service s3Service;
+    private final MskService mskService;
+    private final KafkaConnectWorkerManager workerManager;
+    private final EmulatorConfig config;
+
+    @Inject
+    public KafkaConnectService(StorageFactory storageFactory, RegionResolver regionResolver, S3Service s3Service,
+                               MskService mskService, KafkaConnectWorkerManager workerManager, EmulatorConfig config) {
+        this.plugins = storageFactory.create("kafkaconnect", "custom-plugins.json", new TypeReference<Map<String, KafkaConnectCustomPlugin>>() { });
+        this.connectors = storageFactory.create("kafkaconnect", "connectors.json", new TypeReference<Map<String, KafkaConnectConnector>>() { });
+        this.workers = storageFactory.create("kafkaconnect", "worker-configurations.json", new TypeReference<Map<String, KafkaConnectWorkerConfiguration>>() { });
+        this.regionResolver = regionResolver;
+        this.s3Service = s3Service;
+        this.mskService = mskService;
+        this.workerManager = workerManager;
+        this.config = config;
+    }
+
+    public KafkaConnectCustomPlugin createPlugin(String name, String contentType, String bucketArn, String fileKey, Map<String, String> tags) {
+        require(name, "customPluginName"); require(bucketArn, "contentLocation.s3Location.bucketArn"); require(fileKey, "contentLocation.s3Location.fileKey");
+        if (!"JAR".equals(contentType)) { throw badRequest("contentType must be JAR"); }
+        String bucket = bucketName(bucketArn);
+        s3Service.getObject(bucket, fileKey);
+        KafkaConnectCustomPlugin plugin = new KafkaConnectCustomPlugin();
+        plugin.setArn(arn("custom-plugin", name)); plugin.setName(name); plugin.setContentType(contentType);
+        plugin.setBucketArn(bucketArn); plugin.setFileKey(fileKey); plugin.setCreationTime(Instant.now()); plugin.setTags(tags);
+        plugins.put(plugin.getArn(), plugin);
+        return plugin;
+    }
+
+    public KafkaConnectWorkerConfiguration createWorkerConfiguration(String name, String properties) {
+        require(name, "name"); require(properties, "propertiesFileContent");
+        KafkaConnectWorkerConfiguration worker = new KafkaConnectWorkerConfiguration();
+        worker.setArn(arn("worker-configuration", name)); worker.setName(name); worker.setPropertiesFileContent(properties);
+        worker.setRevision(1); worker.setCreationTime(Instant.now()); workers.put(worker.getArn(), worker);
+        return worker;
+    }
+
+    public KafkaConnectConnector createConnector(String name, String bootstrapServers, String pluginArn, String workerArn,
+                                                  Map<String, String> connectorConfiguration, Map<String, String> tags) {
+        require(name, "connectorName"); require(bootstrapServers, "kafkaCluster.apacheKafkaCluster.bootstrapServers"); require(pluginArn, "plugins");
+        KafkaConnectCustomPlugin plugin = plugin(pluginArn);
+        MskCluster cluster = mskService.listClusters().stream()
+                .filter(candidate -> bootstrapServers.equals(candidate.getBootstrapBrokers()))
+                .findFirst().orElseThrow(() -> badRequest("bootstrapServers must name an MSK cluster managed by Floci"));
+        if (cluster.getInternalBootstrapBrokers() == null && !config.services().kafkaconnect().mock()) {
+            throw badRequest("The MSK cluster does not have an internal Redpanda bootstrap endpoint");
+        }
+        KafkaConnectConnector connector = new KafkaConnectConnector();
+        applyS3Defaults(connectorConfiguration);
+        connector.setArn(arn("connector", name)); connector.setName(name); connector.setState("RUNNING");
+        connector.setClusterArn(cluster.getClusterArn()); connector.setPluginArn(pluginArn); connector.setWorkerConfigurationArn(workerArn);
+        connector.setCurrentVersion(1); connector.setCreationTime(Instant.now()); connector.setConnectorConfiguration(connectorConfiguration); connector.setTags(tags);
+        if (!config.services().kafkaconnect().mock()) {
+            String properties = workerArn == null ? null : worker(workerArn).getPropertiesFileContent();
+            byte[] archive = s3Service.getObject(bucketName(plugin.getBucketArn()), plugin.getFileKey()).getData();
+            workerManager.start(connector, plugin, archive, cluster.getInternalBootstrapBrokers(), properties,
+                    regionResolver.getRegion(), regionResolver.getAccountId());
+        }
+        connectors.put(connector.getArn(), connector);
+        return connector;
+    }
+
+    public KafkaConnectConnector updateConnector(String arn, Map<String, String> configuration) {
+        KafkaConnectConnector connector = connector(arn);
+        Map<String, String> updatedConfiguration = new LinkedHashMap<>(connector.getConnectorConfiguration());
+        updatedConfiguration.putAll(configuration);
+        applyS3Defaults(updatedConfiguration);
+        connector.setConnectorConfiguration(updatedConfiguration); connector.setCurrentVersion(connector.getCurrentVersion() + 1);
+        connectors.put(arn, connector); if (!config.services().kafkaconnect().mock()) { workerManager.configure(connector); } return connector;
+    }
+    public void deleteConnector(String arn) { connector(arn); workerManager.stop(arn); connectors.delete(arn); }
+    public void deletePlugin(String arn) { plugin(arn); plugins.delete(arn); }
+    public void deleteWorker(String arn) { worker(arn); workers.delete(arn); }
+    public KafkaConnectCustomPlugin plugin(String arn) { return plugins.get(arn).orElseThrow(() -> notFound("Custom plugin", arn)); }
+    public KafkaConnectConnector connector(String arn) { return connectors.get(arn).orElseThrow(() -> notFound("Connector", arn)); }
+    public KafkaConnectWorkerConfiguration worker(String arn) { return workers.get(arn).orElseThrow(() -> notFound("Worker configuration", arn)); }
+    public List<KafkaConnectCustomPlugin> plugins() { return plugins.scan(key -> true); }
+    public List<KafkaConnectConnector> connectors() { return connectors.scan(key -> true); }
+    public List<KafkaConnectWorkerConfiguration> workers() { return workers.scan(key -> true); }
+    private void applyS3Defaults(Map<String, String> configuration) {
+        configuration.putIfAbsent("store.url", config.baseUrl());
+        configuration.putIfAbsent("aws.access.key.id", "test");
+        configuration.putIfAbsent("aws.secret.access.key", "test");
+        configuration.putIfAbsent("format.class", "io.confluent.connect.s3.format.json.JsonFormat");
+        configuration.putIfAbsent("storage.class", "io.confluent.connect.s3.storage.S3Storage");
+    }
+    private String arn(String type, String name) { return regionResolver.buildArn("kafkaconnect", regionResolver.getRegion(), type + "/" + name + "/" + UUID.randomUUID()); }
+    private String bucketName(String bucketArn) { try { String resource = AwsArnUtils.parse(bucketArn).resource(); if (!"s3".equals(AwsArnUtils.parse(bucketArn).service()) || resource.isBlank()) throw new IllegalArgumentException(); return resource; } catch (IllegalArgumentException e) { throw badRequest("contentLocation.s3Location.bucketArn must be an S3 bucket ARN"); } }
+    private void require(String value, String field) { if (value == null || value.isBlank()) throw badRequest(field + " is required"); }
+    private AwsException badRequest(String message) { return new AwsException("BadRequestException", message, 400); }
+    private AwsException notFound(String type, String arn) { return new AwsException("NotFoundException", type + " " + arn + " does not exist.", 404); }
+}

src/main/java/io/github/hectorvent/floci/services/kafkaconnect/KafkaConnectWorkerManager.java

@@ -0,0 +1,197 @@
+package io.github.hectorvent.floci.services.kafkaconnect;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.services.kafkaconnect.model.KafkaConnectConnector;
+import io.github.hectorvent.floci.services.kafkaconnect.model.KafkaConnectCustomPlugin;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.zip.ZipInputStream;
+
+/** Runs stock Kafka Connect workers. Connector implementations are supplied exclusively by S3 ZIPs. */
+@ApplicationScoped
+public class KafkaConnectWorkerManager {
+    private static final Logger LOG = Logger.getLogger(KafkaConnectWorkerManager.class);
+    private static final int REST_PORT = 8083;
+    private final ContainerBuilder containerBuilder;
+    private final ContainerLifecycleManager lifecycleManager;
+    private final ContainerLogStreamer logStreamer;
+    private final EmulatorConfig config;
+    private final ObjectMapper objectMapper;
+    private final Map<String, Worker> workers = new ConcurrentHashMap<>();
+
+    @Inject
+    public KafkaConnectWorkerManager(ContainerBuilder containerBuilder, ContainerLifecycleManager lifecycleManager,
+                                     ContainerLogStreamer logStreamer, EmulatorConfig config, ObjectMapper objectMapper) {
+        this.containerBuilder = containerBuilder;
+        this.lifecycleManager = lifecycleManager;
+        this.logStreamer = logStreamer;
+        this.config = config;
+        this.objectMapper = objectMapper;
+    }
+
+    public void start(KafkaConnectConnector connector, KafkaConnectCustomPlugin plugin, byte[] archive,
+                      String internalBootstrap, String workerProperties, String region, String account) {
+        try {
+            Path directory = Files.createTempDirectory("floci-kafka-connect-");
+            Path plugins = Files.createDirectories(directory.resolve("plugins"));
+            extractZip(archive, plugins);
+            Path properties = directory.resolve("connect-distributed.properties");
+            Files.writeString(properties, distributedProperties(internalBootstrap, workerProperties));
+            String name = containerName(connector.getArn());
+            lifecycleManager.removeIfExists(name);
+            ContainerSpec spec = containerBuilder.newContainer(config.services().kafkaconnect().workerImage())
+                    .withName(name).withDockerNetwork(config.services().dockerNetwork()).withLogRotation()
+                    .withDynamicPort(REST_PORT)
+                    .withCmd(List.of("/opt/kafka/bin/connect-distributed.sh", "/opt/floci/connect-distributed.properties"))
+                    .build();
+            String containerId = lifecycleManager.create(spec);
+            copyWorkDirectory(containerId, directory);
+            ContainerLifecycleManager.ContainerInfo info = lifecycleManager.startCreated(containerId, spec);
+            String endpoint = "http://" + info.getEndpoint(REST_PORT).host() + ":" + info.getEndpoint(REST_PORT).port();
+            Closeable logs = logStreamer.attachForAccount(account, info.containerId(), "/aws/msk-connect/" + connector.getName(),
+                    logStreamer.generateLogStreamName(info.containerId().substring(0, 8)), region, "kafkaconnect:" + connector.getName());
+            workers.put(connector.getArn(), new Worker(info.containerId(), endpoint, directory, logs));
+            try {
+                configure(connector);
+            } catch (RuntimeException e) {
+                stop(connector.getArn());
+                throw e;
+            }
+        } catch (IOException e) {
+            throw new IllegalStateException("Unable to prepare Kafka Connect plugin archive", e);
+        }
+    }
+
+    public void configure(KafkaConnectConnector connector) {
+        Worker worker = workers.get(connector.getArn());
+        if (worker == null) {
+            return;
+        }
+        IOException lastFailure = null;
+        for (int attempt = 0; attempt < 30; attempt++) {
+            try {
+                HttpURLConnection connection = (HttpURLConnection) URI.create(worker.endpoint() + "/connectors/" + connector.getName() + "/config")
+                        .toURL().openConnection();
+                connection.setRequestMethod("PUT");
+                connection.setDoOutput(true);
+                connection.setConnectTimeout(5000);
+                connection.setReadTimeout(10000);
+                connection.setRequestProperty("Content-Type", "application/json");
+                objectMapper.writeValue(connection.getOutputStream(), connector.getConnectorConfiguration());
+                int responseCode = connection.getResponseCode();
+                if (responseCode == 404) {
+                    throw new IOException("Kafka Connect worker has not registered its connector API yet");
+                }
+                if (responseCode >= 300) {
+                    throw new IllegalStateException("Kafka Connect rejected connector configuration: " + responseCode);
+                }
+                return;
+            } catch (IOException e) {
+                lastFailure = e;
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException interrupted) {
+                    Thread.currentThread().interrupt();
+                    throw new IllegalStateException("Interrupted while waiting for Kafka Connect worker", interrupted);
+                }
+            }
+        }
+        throw new IllegalStateException("Kafka Connect worker is not ready for connector " + connector.getName(), lastFailure);
+    }
+
+    public void stop(String connectorArn) {
+        Worker worker = workers.remove(connectorArn);
+        if (worker == null) {
+            lifecycleManager.removeIfExists(containerName(connectorArn));
+            return;
+        }
+        lifecycleManager.stopAndRemove(worker.containerId(), worker.logs());
+        try {
+            Files.walk(worker.directory()).sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
+                try {
+                    Files.deleteIfExists(path);
+                } catch (IOException ignored) {
+                    // A failed cleanup only leaves a disposable plugin extraction directory.
+                }
+            });
+        } catch (IOException e) {
+            LOG.debugv("Could not remove Kafka Connect work directory {0}: {1}", worker.directory(), e.getMessage());
+        }
+    }
+
+    private String distributedProperties(String bootstrap, String extra) {
+        return "bootstrap.servers=" + bootstrap + "\n"
+                + "group.id=floci-kafka-connect\nconfig.storage.topic=floci-connect-configs\n"
+                + "offset.storage.topic=floci-connect-offsets\nstatus.storage.topic=floci-connect-status\n"
+                + "config.storage.replication.factor=1\noffset.storage.replication.factor=1\n"
+                + "status.storage.replication.factor=1\nkey.converter=org.apache.kafka.connect.json.JsonConverter\n"
+                + "value.converter=org.apache.kafka.connect.json.JsonConverter\nplugin.path=/opt/floci/plugins\n"
+                + (extra == null ? "" : extra + "\n");
+    }
+
+    private String containerName(String connectorArn) {
+        return "floci-kafkaconnect-" + connectorArn.substring(connectorArn.lastIndexOf('/') + 1);
+    }
+
+    private void extractZip(byte[] archive, Path output) throws IOException {
+        try (ZipInputStream zip = new ZipInputStream(new java.io.ByteArrayInputStream(archive))) {
+            java.util.zip.ZipEntry entry;
+            while ((entry = zip.getNextEntry()) != null) {
+                Path target = output.resolve(entry.getName()).normalize();
+                if (!target.startsWith(output)) {
+                    throw new IOException("Plugin ZIP contains an invalid path");
+                }
+                if (entry.isDirectory()) { Files.createDirectories(target); }
+                else { Files.createDirectories(target.getParent()); Files.copy(zip, target); }
+            }
+        }
+    }
+
+    private void copyWorkDirectory(String containerId, Path directory) throws IOException {
+        ByteArrayOutputStream archive = new ByteArrayOutputStream();
+        try (TarArchiveOutputStream tar = new TarArchiveOutputStream(archive);
+             java.util.stream.Stream<Path> paths = Files.walk(directory)) {
+            tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
+            tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
+            for (Path path : paths.sorted(Comparator.naturalOrder()).toList()) {
+                String relative = directory.relativize(path).toString();
+                if (relative.isEmpty()) {
+                    continue;
+                }
+                TarArchiveEntry entry = new TarArchiveEntry(path.toFile(), "floci/" + relative);
+                tar.putArchiveEntry(entry);
+                if (Files.isRegularFile(path)) {
+                    Files.copy(path, tar);
+                }
+                tar.closeArchiveEntry();
+            }
+        }
+        lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)
+                .withRemotePath("/opt")
+                .withTarInputStream(new ByteArrayInputStream(archive.toByteArray()))
+                .exec();
+    }
+
+    private record Worker(String containerId, String endpoint, Path directory, Closeable logs) { }
+}

src/main/java/io/github/hectorvent/floci/services/kafkaconnect/model/KafkaConnectConnector.java

@@ -0,0 +1,41 @@
+package io.github.hectorvent.floci.services.kafkaconnect.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+@RegisterForReflection
+public class KafkaConnectConnector {
+    private String arn;
+    private String name;
+    private String state;
+    private String clusterArn;
+    private String pluginArn;
+    private String workerConfigurationArn;
+    private long currentVersion;
+    private Instant creationTime;
+    private Map<String, String> connectorConfiguration = new LinkedHashMap<>();
+    private Map<String, String> tags = new LinkedHashMap<>();
+
+    public String getArn() { return arn; }
+    public void setArn(String arn) { this.arn = arn; }
+    public String getName() { return name; }
+    public void setName(String name) { this.name = name; }
+    public String getState() { return state; }
+    public void setState(String state) { this.state = state; }
+    public String getClusterArn() { return clusterArn; }
+    public void setClusterArn(String clusterArn) { this.clusterArn = clusterArn; }
+    public String getPluginArn() { return pluginArn; }
+    public void setPluginArn(String pluginArn) { this.pluginArn = pluginArn; }
+    public String getWorkerConfigurationArn() { return workerConfigurationArn; }
+    public void setWorkerConfigurationArn(String workerConfigurationArn) { this.workerConfigurationArn = workerConfigurationArn; }
+    public long getCurrentVersion() { return currentVersion; }
+    public void setCurrentVersion(long currentVersion) { this.currentVersion = currentVersion; }
+    public Instant getCreationTime() { return creationTime; }
+    public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }
+    public Map<String, String> getConnectorConfiguration() { return connectorConfiguration; }
+    public void setConnectorConfiguration(Map<String, String> connectorConfiguration) { this.connectorConfiguration = connectorConfiguration; }
+    public Map<String, String> getTags() { return tags; }
+    public void setTags(Map<String, String> tags) { this.tags = tags; }
+}

src/main/java/io/github/hectorvent/floci/services/kafkaconnect/model/KafkaConnectCustomPlugin.java

@@ -0,0 +1,32 @@
+package io.github.hectorvent.floci.services.kafkaconnect.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+@RegisterForReflection
+public class KafkaConnectCustomPlugin {
+    private String arn;
+    private String name;
+    private String contentType;
+    private String bucketArn;
+    private String fileKey;
+    private Instant creationTime;
+    private Map<String, String> tags = new LinkedHashMap<>();
+
+    public String getArn() { return arn; }
+    public void setArn(String arn) { this.arn = arn; }
+    public String getName() { return name; }
+    public void setName(String name) { this.name = name; }
+    public String getContentType() { return contentType; }
+    public void setContentType(String contentType) { this.contentType = contentType; }
+    public String getBucketArn() { return bucketArn; }
+    public void setBucketArn(String bucketArn) { this.bucketArn = bucketArn; }
+    public String getFileKey() { return fileKey; }
+    public void setFileKey(String fileKey) { this.fileKey = fileKey; }
+    public Instant getCreationTime() { return creationTime; }
+    public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }
+    public Map<String, String> getTags() { return tags; }
+    public void setTags(Map<String, String> tags) { this.tags = tags; }
+}

src/main/java/io/github/hectorvent/floci/services/kafkaconnect/model/KafkaConnectWorkerConfiguration.java

@@ -0,0 +1,24 @@
+package io.github.hectorvent.floci.services.kafkaconnect.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.time.Instant;
+
+@RegisterForReflection
+public class KafkaConnectWorkerConfiguration {
+    private String arn;
+    private String name;
+    private String propertiesFileContent;
+    private long revision;
+    private Instant creationTime;
+
+    public String getArn() { return arn; }
+    public void setArn(String arn) { this.arn = arn; }
+    public String getName() { return name; }
+    public void setName(String name) { this.name = name; }
+    public String getPropertiesFileContent() { return propertiesFileContent; }
+    public void setPropertiesFileContent(String propertiesFileContent) { this.propertiesFileContent = propertiesFileContent; }
+    public long getRevision() { return revision; }
+    public void setRevision(long revision) { this.revision = revision; }
+    public Instant getCreationTime() { return creationTime; }
+    public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }
+}

src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java

@@ -110,7 +110,11 @@ public class KinesisJsonHandler {
 
     private Response handleCreateStream(JsonNode request, String region) {
         String streamName = request.path("StreamName").asText();
-        int shardCount = request.path("ShardCount").asInt(1);
+        JsonNode shardCountNode = request.path("ShardCount");
+        boolean hasShardCount = !shardCountNode.isMissingNode() && !shardCountNode.isNull();
+        if (hasShardCount && (!shardCountNode.isIntegralNumber() || !shardCountNode.canConvertToInt())) {
+            throw new AwsException("InvalidArgumentException", "ShardCount must be an integer.", 400);
+        }
         String streamMode = null;
         JsonNode modeDetails = request.path("StreamModeDetails");
         if (modeDetails.isObject()) {
@@ -119,6 +123,18 @@ public class KinesisJsonHandler {
                 streamMode = mode;
             }
         }
+        if (hasShardCount && streamMode != null) {
+            throw new AwsException("InvalidArgumentException",
+                    "Specify either ShardCount or StreamModeDetails, not both.", 400);
+        }
+        if (!hasShardCount && streamMode == null) {
+            throw new AwsException("InvalidArgumentException",
+                    "Either ShardCount or StreamModeDetails must be specified.", 400);
+        }
+        if (hasShardCount && shardCountNode.intValue() < 1) {
+            throw new AwsException("InvalidArgumentException", "ShardCount must be at least 1.", 400);
+        }
+        int shardCount = hasShardCount ? shardCountNode.intValue() : 1;
         // CreateStream's optional Tags member was being dropped. That is invisible to a
         // hand-written script but not to Terraform: aws_kinesis_stream sets tags at create
         // time and then reads them back with ListTagsForStream, so an empty read produces a
@@ -633,6 +649,15 @@ public class KinesisJsonHandler {
         String shardId = request.path("ShardId").asText();
         String type = request.path("ShardIteratorType").asText();
         String seq = request.has("StartingSequenceNumber") ? request.path("StartingSequenceNumber").asText() : null;
+        if (!Set.of("AT_SEQUENCE_NUMBER", "AFTER_SEQUENCE_NUMBER", "AT_TIMESTAMP", "LATEST", "TRIM_HORIZON")
+                .contains(type)) {
+            throw new AwsException("InvalidArgumentException", "Invalid ShardIteratorType: " + type, 400);
+        }
+        if (("AT_SEQUENCE_NUMBER".equals(type) || "AFTER_SEQUENCE_NUMBER".equals(type))
+                && (seq == null || seq.isBlank())) {
+            throw new AwsException("InvalidArgumentException",
+                    "ShardIteratorType " + type + " requires a StartingSequenceNumber", 400);
+        }
         // AWS sends Timestamp as epoch seconds (double with fractional ms).
         // Convert to long millis at the boundary; the emulator stores time in ms everywhere.
         // Use Math.round to avoid 1ms drift from FP multiplication (e.g. X.999...).
@@ -661,6 +686,9 @@ public class KinesisJsonHandler {
     private Response handleGetRecords(JsonNode request, String region) {
         String iterator = request.path("ShardIterator").asText();
         Integer limit = request.has("Limit") ? request.path("Limit").asInt() : null;
+        if (limit != null && (limit < 1 || limit > 10000)) {
+            throw new AwsException("InvalidArgumentException", "Limit must be between 1 and 10000.", 400);
+        }
 
         Map<String, Object> result = service.getRecords(iterator, limit, region);
         List<KinesisRecord> records = (List<KinesisRecord>) result.get("Records");

src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java

@@ -906,7 +906,7 @@ public class KinesisService implements ResourceProvider {
             }
         }
 
-        int max = limit != null ? Math.min(limit, 1000) : 1000;
+        int max = limit != null ? Math.min(limit, 10000) : 10000;
         List<KinesisRecord> result = new ArrayList<>();
         int nextIndex = startIndex;
         for (int i = startIndex; i < allRecords.size() && result.size() < max; i++) {

src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java

@@ -48,6 +48,10 @@ public class ImageCacheService {
     }
 
     public String ensureImageExists(String imageUri, String platform) {
+        return ensureImageExists(imageUri, platform, ImagePullObserver.NOOP);
+    }
+
+    public String ensureImageExists(String imageUri, String platform, ImagePullObserver observer) {
         boolean explicitPlatform = platform != null && !platform.isBlank();
         String requestedPlatform = explicitPlatform ? platform.trim() : daemonPlatform();
         ImageKey imageKey = new ImageKey(imageUri, requestedPlatform);
@@ -68,6 +72,27 @@ public class ImageCacheService {
                 LOG.infov("Image already present locally, skipping pull: {0}", imageUri);
                 return resolvedImage;
             }
+            return pullImage(imageUri, requestedPlatform, explicitPlatform, imageKey, observer);
+        }
+    }
+
+    public String refreshImage(String imageUri) {
+        return refreshImage(imageUri, null);
+    }
+
+    public String refreshImage(String imageUri, String platform) {
+        boolean explicitPlatform = platform != null && !platform.isBlank();
+        String requestedPlatform = explicitPlatform ? platform.trim() : daemonPlatform();
+        ImageKey imageKey = new ImageKey(imageUri, requestedPlatform);
+        Object lock = locks.computeIfAbsent(imageUri, k -> new Object());
+        synchronized (lock) {
+            return pullImage(imageUri, requestedPlatform, explicitPlatform, imageKey, ImagePullObserver.NOOP);
+        }
+    }
+
+    private String pullImage(String imageUri, String requestedPlatform, boolean explicitPlatform, ImageKey imageKey,
+                             ImagePullObserver observer) {
+        observer.started(imageUri);
         LOG.infov("Pulling image: {0}", imageUri);
         try {
             runWithRetry(imageUri, MAX_PULL_ATTEMPTS, INITIAL_BACKOFF_MS, () -> {
@@ -79,15 +104,23 @@ public class ImageCacheService {
                 pullImage.exec(new PullImageResultCallback())
                         .awaitCompletion(5, TimeUnit.MINUTES);
             });
-                resolvedImage = resolvedImageReference(imageUri, inspectLocalImage(imageUri));
+            String resolvedImage = resolvedImageReference(imageUri, inspectLocalImage(imageUri));
             resolvedImages.put(imageKey, resolvedImage);
             LOG.infov("Image pulled successfully: {0}", imageUri);
+            observer.completed(imageUri);
             return resolvedImage;
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
             throw new RuntimeException("Interrupted while pulling image: " + imageUri, e);
         }
     }
+
+    public interface ImagePullObserver {
+        ImagePullObserver NOOP = new ImagePullObserver() {};
+
+        default void started(String imageUri) {}
+
+        default void completed(String imageUri) {}
     }
 
     /**

src/main/java/io/github/hectorvent/floci/services/lightsail/LightsailContainerManager.java

@@ -0,0 +1,703 @@
+package io.github.hectorvent.floci.services.lightsail;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.dockerjava.api.async.ResultCallback;
+import com.github.dockerjava.api.command.InspectExecResponse;
+import com.github.dockerjava.api.model.Frame;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.core.common.docker.PortAllocator;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.jboss.logging.Logger;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Base64;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+@ApplicationScoped
+public class LightsailContainerManager {
+
+    private static final Logger LOG = Logger.getLogger(LightsailContainerManager.class);
+    private static final String WORDPRESS_IMAGE = "wordpress:6.7.2-apache";
+    private static final String MARIADB_IMAGE = "mariadb:11.7";
+    private static final String WORDPRESS_DATABASE = "wordpress";
+    private static final String WORDPRESS_DATABASE_USER = "wordpress";
+    private static final String WORDPRESS_DATABASE_PASSWORD = "wordpress";
+    private static final String NODE_APPLICATION_PATH = "/opt/lightsail/app";
+    private static final String NODE_APPLICATION_USER = "node";
+    private final ContainerBuilder containerBuilder;
+    private final ContainerLifecycleManager lifecycleManager;
+    private final EmulatorConfig config;
+    private final RegionResolver regionResolver;
+    private final PortAllocator portAllocator;
+    private final ObjectMapper mapper;
+
+    @Inject
+    public LightsailContainerManager(ContainerBuilder containerBuilder, ContainerLifecycleManager lifecycleManager,
+                                       EmulatorConfig config, RegionResolver regionResolver, PortAllocator portAllocator) {
+        this(containerBuilder, lifecycleManager, config, regionResolver, portAllocator, new ObjectMapper());
+    }
+
+    LightsailContainerManager(ContainerBuilder containerBuilder, ContainerLifecycleManager lifecycleManager,
+                              EmulatorConfig config, RegionResolver regionResolver, PortAllocator portAllocator,
+                              ObjectMapper mapper) {
+        this.containerBuilder = containerBuilder;
+        this.lifecycleManager = lifecycleManager;
+        this.config = config;
+        this.regionResolver = regionResolver;
+        this.portAllocator = portAllocator;
+        this.mapper = mapper;
+    }
+
+    public Launch launch(String instanceName, String region, String blueprintId, String bundleId, String image) {
+        return launch(instanceName, region, blueprintId, bundleId, image, null);
+    }
+
+    public Launch launch(String instanceName, String region, String blueprintId, String bundleId, String image, String publicKey) {
+        if (!isDockerAvailable()) {
+            return null;
+        }
+        Runtime runtime = runtimeFor(blueprintId);
+        if (runtime == null && (image == null || image.isBlank())) {
+            throw new IllegalArgumentException("Blueprint " + blueprintId + " has no local runtime mapping");
+        }
+        String containerName = ContainerStorageHelper.dockerName(config, "floci-lightsail-" + instanceName);
+        lifecycleManager.removeIfExists(containerName);
+        if ("wordpress".equals(blueprintId) && image == null) {
+            return launchWordPress(instanceName, region, bundleId, containerName);
+        }
+        if ("nodejs".equals(blueprintId) && image == null) {
+            return launchNode(instanceName, region, bundleId, containerName, publicKey);
+        }
+        boolean supportsSsh = image == null && runtime.sshUser() != null;
+        int sshHostPort = supportsSsh ? portAllocator.allocate(config.services().ec2().sshPortRangeStart(),
+                config.services().ec2().sshPortRangeEnd()) : 0;
+        ContainerBuilder.Builder builder = containerBuilder.newContainer(image == null ? runtime.image() : image)
+                .withName(containerName)
+                .withLogRotation()
+                .withLabels(ContainerStorageHelper.resourceIdentityLabels(
+                        "lightsail", instanceName, regionResolver.getAccountId(), region))
+                .withMemoryMb(memoryMb(bundleId))
+                .withCpuUnits(cpuUnits(bundleId));
+        if (supportsSsh) {
+            builder.withPortBinding(22, sshHostPort);
+        }
+        if (image == null && runtime.command() != null) {
+            builder.withCmd(runtime.command());
+        }
+        ContainerSpec spec = builder.build();
+        try {
+            ContainerInfo container = lifecycleManager.createAndStart(spec);
+            if (supportsSsh && !bootstrapSsh(container.containerId(), runtime.sshUser(), publicKey, sshHostPort)) {
+                lifecycleManager.stopAndRemove(container.containerId(), null);
+                portAllocator.release(sshHostPort);
+                return null;
+            }
+            return new Launch(container.containerId(), null, null, supportsSsh ? sshHostPort : null);
+        } catch (RuntimeException e) {
+            if (sshHostPort > 0) {
+                portAllocator.release(sshHostPort);
+            }
+            LOG.warnv("Lightsail instance {0} could not start a backing container: {1}", instanceName, e.getMessage());
+            return null;
+        }
+    }
+
+    public static Runtime runtimeFor(String blueprintId) {
+        return switch (blueprintId) {
+            case "ubuntu_22_04" -> new Runtime("ubuntu:22.04", List.of("tail", "-f", "/dev/null"), "ubuntu", null);
+            case "ubuntu_24_04" -> new Runtime("ubuntu:24.04", List.of("tail", "-f", "/dev/null"), "ubuntu", null);
+            case "amazon_linux_2023" -> new Runtime("public.ecr.aws/amazonlinux/amazonlinux:2023",
+                    List.of("tail", "-f", "/dev/null"), "ec2-user", null);
+            case "nginx" -> new Runtime("nginx:1.27-alpine", null, null, 80);
+            case "wordpress" -> new Runtime(WORDPRESS_IMAGE, null, null, 80);
+            case "nodejs" -> new Runtime("node:24-bookworm", null, NODE_APPLICATION_USER, 3000);
+            default -> null;
+        };
+    }
+
+    private Launch launchNode(String instanceName, String region, String bundleId, String containerName, String publicKey) {
+        int sshHostPort = portAllocator.allocate(config.services().ec2().sshPortRangeStart(),
+                config.services().ec2().sshPortRangeEnd());
+        String applicationVolume = containerName + "-app";
+        String releaseId = Long.toString(System.currentTimeMillis());
+        lifecycleManager.ensureVolume(applicationVolume);
+        ContainerSpec spec = containerBuilder.newContainer(runtimeFor("nodejs").image())
+                .withName(containerName)
+                .withLogRotation()
+                .withEnv("FLOCI_LIGHTSAIL_RELEASE_ID", releaseId)
+                .withNamedVolume(applicationVolume, NODE_APPLICATION_PATH)
+                .withPortBinding(22, sshHostPort)
+                .withLabels(ContainerStorageHelper.resourceIdentityLabels(
+                        "lightsail", instanceName, regionResolver.getAccountId(), region))
+                .withMemoryMb(memoryMb(bundleId))
+                .withCpuUnits(cpuUnits(bundleId))
+                .withCmd(nodeSupervisorCommand())
+                .build();
+        try {
+            ContainerInfo container = lifecycleManager.createAndStart(spec);
+            if (!bootstrapSsh(container.containerId(), NODE_APPLICATION_USER, publicKey, sshHostPort)) {
+                lifecycleManager.stopAndRemove(container.containerId(), null);
+                lifecycleManager.removeVolume(applicationVolume);
+                portAllocator.release(sshHostPort);
+                return null;
+            }
+            return new Launch(container.containerId(), null, null, null, sshHostPort, applicationVolume, releaseId);
+        } catch (RuntimeException e) {
+            lifecycleManager.removeVolume(applicationVolume);
+            portAllocator.release(sshHostPort);
+            LOG.warnv("Lightsail Node.js instance {0} could not start: {1}", instanceName, e.getMessage());
+            return null;
+        }
+    }
+
+    static List<String> nodeSupervisorCommand() {
+        String script = "app=" + NODE_APPLICATION_PATH + ";"
+                + "mkdir -p \"$app/releases\" \"$app/logs\";"
+                + "if [ ! -L \"$app/current\" ]; then mkdir -p \"$app/releases/$FLOCI_LIGHTSAIL_RELEASE_ID\";"
+                + "ln -s \"releases/$FLOCI_LIGHTSAIL_RELEASE_ID\" \"$app/current\"; fi;"
+                + "chown -R " + NODE_APPLICATION_USER + ":" + NODE_APPLICATION_USER + " \"$app\";"
+                + "trap 'exit 0' INT TERM;delay=1;"
+                + "while true; do "
+                + "ls -1dt \"$app\"/releases/* 2>/dev/null | tail -n +4 | xargs -r rm -rf;"
+                + "if [ -f \"$app/current/package.json\" ]; then "
+                + "su -s /bin/sh " + NODE_APPLICATION_USER
+                + " -c 'cd /opt/lightsail/app/current && PORT=$(cat .floci-port 2>/dev/null || printf 3000) HOST=0.0.0.0;"
+                + "if [ -f .floci-start ]; then exec sh -c \"$(base64 -d .floci-start)\"; else exec npm start; fi' >>\"$app/logs/app.stdout.log\" "
+                + "2>>\"$app/logs/app.stderr.log\";status=$?;"
+                + "printf '%s app exited with status %s\\n' \"$(date -u +%FT%TZ)\" \"$status\" >>\"$app/logs/supervisor.log\";"
+                + "sleep \"$delay\";[ \"$delay\" -lt 30 ] && delay=$((delay * 2));"
+                + "else delay=1;sleep 2;fi;"
+                + "done";
+        return List.of("sh", "-c", script);
+    }
+
+    private Launch launchWordPress(String instanceName, String region, String bundleId, String containerName) {
+        String databaseContainerName = containerName + "-mariadb";
+        String databaseVolume = databaseContainerName + "-data";
+        String wordpressContentVolume = containerName + "-wp-content";
+        lifecycleManager.removeIfExists(databaseContainerName);
+        lifecycleManager.ensureVolume(databaseVolume);
+        lifecycleManager.ensureVolume(wordpressContentVolume);
+        ContainerSpec databaseSpec = containerBuilder.newContainer(MARIADB_IMAGE)
+                .withName(databaseContainerName)
+                .withLogRotation()
+                .withEnv("MARIADB_DATABASE", WORDPRESS_DATABASE)
+                .withEnv("MARIADB_USER", WORDPRESS_DATABASE_USER)
+                .withEnv("MARIADB_PASSWORD", WORDPRESS_DATABASE_PASSWORD)
+                .withEnv("MARIADB_ROOT_PASSWORD", WORDPRESS_DATABASE_PASSWORD)
+                .withNamedVolume(databaseVolume, "/var/lib/mysql")
+                .withLabels(ContainerStorageHelper.resourceIdentityLabels(
+                        "lightsail", instanceName, regionResolver.getAccountId(), region))
+                .withMemoryMb(Math.max(256, memoryMb(bundleId) / 2))
+                .withCpuUnits(cpuUnits(bundleId))
+                .build();
+        String databaseContainerId = null;
+        try {
+            databaseContainerId = lifecycleManager.createAndStart(databaseSpec).containerId();
+            ContainerSpec wordpressSpec = containerBuilder.newContainer(WORDPRESS_IMAGE)
+                    .withName(containerName)
+                    .withLogRotation()
+                    .withEnv("WORDPRESS_DB_HOST", "127.0.0.1:3306")
+                    .withEnv("WORDPRESS_DB_NAME", WORDPRESS_DATABASE)
+                    .withEnv("WORDPRESS_DB_USER", WORDPRESS_DATABASE_USER)
+                    .withEnv("WORDPRESS_DB_PASSWORD", WORDPRESS_DATABASE_PASSWORD)
+                    .withNetworkMode("container:" + databaseContainerId)
+                    .withNamedVolume(wordpressContentVolume, "/var/www/html/wp-content")
+                    .withLabels(ContainerStorageHelper.resourceIdentityLabels(
+                            "lightsail", instanceName, regionResolver.getAccountId(), region))
+                    .withMemoryMb(memoryMb(bundleId))
+                    .withCpuUnits(cpuUnits(bundleId))
+                    .build();
+            ContainerInfo wordpress = lifecycleManager.createAndStart(wordpressSpec);
+            return new Launch(wordpress.containerId(), databaseContainerId, databaseVolume, wordpressContentVolume, null);
+        } catch (RuntimeException e) {
+            if (databaseContainerId != null) {
+                lifecycleManager.stopAndRemove(databaseContainerId, null);
+            }
+            lifecycleManager.removeVolume(databaseVolume);
+            lifecycleManager.removeVolume(wordpressContentVolume);
+            LOG.warnv("Lightsail WordPress instance {0} could not start: {1}", instanceName, e.getMessage());
+            return null;
+        }
+    }
+
+    private static int memoryMb(String bundleId) {
+        return switch (bundleId) {
+            case "nano_3_0" -> 512;
+            case "micro_3_0" -> 1024;
+            case "small_3_0" -> 2048;
+            case "medium_3_0" -> 4096;
+            default -> 1024;
+        };
+    }
+
+    private static int cpuUnits(String bundleId) {
+        return "nano_3_0".equals(bundleId) ? 1024 : 2048;
+    }
+
+    public record Runtime(String image, List<String> command, String sshUser, Integer applicationPort) {
+    }
+
+    public record Launch(String containerId, String databaseContainerId, String databaseVolume,
+                          String wordpressContentVolume, Integer sshHostPort, String applicationVolume,
+                          String applicationRelease) {
+        public Launch(String containerId, String databaseContainerId, String databaseVolume,
+                      String wordpressContentVolume, Integer sshHostPort) {
+            this(containerId, databaseContainerId, databaseVolume, wordpressContentVolume, sshHostPort, null, null);
+        }
+
+        public Launch(String containerId, String databaseContainerId, String databaseVolume) {
+            this(containerId, databaseContainerId, databaseVolume, null, null, null, null);
+        }
+
+        public Launch(String containerId, String databaseContainerId, String databaseVolume, Integer sshHostPort) {
+            this(containerId, databaseContainerId, databaseVolume, null, sshHostPort, null, null);
+        }
+    }
+
+    public void stop(String containerId, String databaseContainerId) {
+        if (containerId != null && !containerId.isBlank()) {
+            lifecycleManager.getDockerClient().stopContainerCmd(containerId).exec();
+        }
+        if (databaseContainerId != null && !databaseContainerId.isBlank()) {
+            lifecycleManager.getDockerClient().stopContainerCmd(databaseContainerId).exec();
+        }
+    }
+
+    public void start(String containerId, String databaseContainerId) {
+        start(containerId, databaseContainerId, false);
+    }
+
+    public void start(String containerId, String databaseContainerId, boolean sshEnabled) {
+        if (databaseContainerId != null && !databaseContainerId.isBlank()) {
+            lifecycleManager.getDockerClient().startContainerCmd(databaseContainerId).exec();
+        }
+        if (containerId != null && !containerId.isBlank()) {
+            lifecycleManager.getDockerClient().startContainerCmd(containerId).exec();
+            if (sshEnabled) {
+                startSshd(containerId);
+            }
+        }
+    }
+
+    public void reboot(String containerId, String databaseContainerId) {
+        reboot(containerId, databaseContainerId, false);
+    }
+
+    public void reboot(String containerId, String databaseContainerId, boolean sshEnabled) {
+        if (containerId != null && !containerId.isBlank()) {
+            lifecycleManager.getDockerClient().restartContainerCmd(containerId).exec();
+            if (sshEnabled) {
+                startSshd(containerId);
+            }
+        }
+        if (databaseContainerId != null && !databaseContainerId.isBlank()) {
+            lifecycleManager.getDockerClient().restartContainerCmd(databaseContainerId).exec();
+        }
+    }
+
+    public void terminate(String containerId, String databaseContainerId, String databaseVolume,
+                           String wordpressContentVolume) {
+        terminate(containerId, databaseContainerId, databaseVolume, wordpressContentVolume, null);
+    }
+
+    public void terminate(String containerId, String databaseContainerId, String databaseVolume,
+                           String wordpressContentVolume, String applicationVolume) {
+        if (containerId != null && !containerId.isBlank()) {
+            lifecycleManager.stopAndRemoveStrict(containerId, null);
+        }
+        if (databaseContainerId != null && !databaseContainerId.isBlank()) {
+            lifecycleManager.stopAndRemoveStrict(databaseContainerId, null);
+        }
+        if (databaseVolume != null && !databaseVolume.isBlank()) {
+            lifecycleManager.removeVolumeStrict(databaseVolume);
+        }
+        if (wordpressContentVolume != null && !wordpressContentVolume.isBlank()) {
+            lifecycleManager.removeVolumeStrict(wordpressContentVolume);
+        }
+        if (applicationVolume != null && !applicationVolume.isBlank()) {
+            lifecycleManager.removeVolumeStrict(applicationVolume);
+        }
+    }
+
+    public String capture(String containerId, String snapshotName) {
+        if (containerId == null || containerId.isBlank()) {
+            return null;
+        }
+        String image = ContainerStorageHelper.dockerName(config, "floci-lightsail-snapshot-" + snapshotName) + ":capture";
+        try {
+            lifecycleManager.getDockerClient().commitCmd(containerId)
+                    .withRepository(image.substring(0, image.indexOf(':')))
+                    .withTag("capture")
+                    .exec();
+            return image;
+        } catch (RuntimeException e) {
+            throw new IllegalStateException("Could not capture Lightsail instance container " + containerId, e);
+        }
+    }
+
+    public void removeCapture(String image) {
+        if (image == null || image.isBlank()) {
+            return;
+        }
+        try {
+            lifecycleManager.getDockerClient().removeImageCmd(image).withForce(true).exec();
+        } catch (RuntimeException e) {
+            LOG.warnv("Could not remove Lightsail snapshot image {0}: {1}", image, e.getMessage());
+        }
+    }
+
+    public void deployStaticFiles(String containerId, List<StaticFile> files) {
+        if (containerId == null || containerId.isBlank()) {
+            throw new IllegalArgumentException("The Lightsail instance has no running container.");
+        }
+        if (files == null || files.isEmpty()) {
+            throw new IllegalArgumentException("At least one static file is required.");
+        }
+        for (StaticFile file : files) {
+            validateDeploymentPath(file.path());
+        }
+        try {
+            ByteArrayOutputStream archive = new ByteArrayOutputStream();
+            try (TarArchiveOutputStream tar = new TarArchiveOutputStream(archive)) {
+                tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
+                for (StaticFile file : files) {
+                    TarArchiveEntry entry = new TarArchiveEntry(file.path());
+                    entry.setMode(0644);
+                    entry.setSize(Files.size(file.source()));
+                    tar.putArchiveEntry(entry);
+                    Files.copy(file.source(), tar);
+                    tar.closeArchiveEntry();
+                }
+                tar.finish();
+            }
+            lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)
+                    .withTarInputStream(new ByteArrayInputStream(archive.toByteArray()))
+                    .withRemotePath("/usr/share/nginx/html")
+                    .exec();
+        } catch (IOException | RuntimeException e) {
+            throw new IllegalStateException("Could not deploy static files to the Lightsail container", e);
+        }
+    }
+
+    public NodeDeployment deployNodeApplication(String containerId, List<StaticFile> uploads, String explicitStart,
+                                                  int port) {
+        if (containerId == null || containerId.isBlank()) {
+            throw new IllegalArgumentException("The Lightsail instance has no running container.");
+        }
+        if (port < 1 || port > 65535) {
+            throw new IllegalArgumentException("PORT must be between 1 and 65535.");
+        }
+        Path staging = null;
+        try {
+            staging = Files.createTempDirectory("floci-node-deploy-");
+            int files = unpackNodeUploads(uploads, staging);
+            if (files == 0) {
+                throw new IllegalArgumentException("At least one application file is required.");
+            }
+            Path packageJson = staging.resolve("package.json");
+            if (!Files.isRegularFile(packageJson)) {
+                throw new IllegalArgumentException("package.json is required before deployment files are written.");
+            }
+            JsonNode packageDefinition = mapper.readTree(packageJson.toFile());
+            String start = explicitStart == null || explicitStart.isBlank()
+                    ? packageDefinition.path("scripts").path("start").asText(null) : explicitStart;
+            if (start == null || start.isBlank()) {
+                throw new IllegalArgumentException("Specify a start command or add scripts.start to package.json.");
+            }
+            String releaseId = UUID.randomUUID().toString();
+            Files.writeString(staging.resolve(".floci-start"), Base64.getEncoder().encodeToString(start.getBytes(StandardCharsets.UTF_8)));
+            Files.writeString(staging.resolve(".floci-port"), Integer.toString(port));
+            CommandResult releaseDirectory = execOutput(containerId, new String[]{"sh", "-c",
+                    "mkdir -p " + NODE_APPLICATION_PATH + "/releases/" + releaseId}, 15);
+            if (!releaseDirectory.success()) {
+                throw new IllegalStateException("Could not prepare the application release directory. " + releaseDirectory.output());
+            }
+            copyNodeRelease(containerId, staging, releaseId);
+            CommandResult current = execOutput(containerId, new String[]{"sh", "-c", "readlink " + NODE_APPLICATION_PATH + "/current || true"}, 15);
+            String install = "set -eu; release=" + NODE_APPLICATION_PATH + "/releases/" + releaseId + ";"
+                    + "chown -R " + NODE_APPLICATION_USER + ":" + NODE_APPLICATION_USER + " \"$release\";"
+                    + "su -s /bin/sh " + NODE_APPLICATION_USER + " -c 'cd \"$0\" && "
+                    + (Files.exists(staging.resolve("package-lock.json")) ? "npm ci" : "npm install") + "' \"$release\";"
+                    + "ln -s \"releases/" + releaseId + "\" " + NODE_APPLICATION_PATH + "/current.next;"
+                    + "mv -Tf " + NODE_APPLICATION_PATH + "/current.next " + NODE_APPLICATION_PATH + "/current";
+            CommandResult result = execOutput(containerId, new String[]{"sh", "-c", install}, 300);
+            if (!result.success()) {
+                throw new IllegalStateException("Dependency installation failed. " + result.output());
+            }
+            lifecycleManager.getDockerClient().restartContainerCmd(containerId).exec();
+            CommandResult health = execOutput(containerId, new String[]{"sh", "-c",
+                    "for n in $(seq 1 30); do node -e 'fetch(\"http://127.0.0.1:" + port
+                            + "/\").then(() => process.exit(0)).catch(() => process.exit(1))' && exit 0; sleep 1; done; exit 1"}, 35);
+            if (!health.success()) {
+                if (current.success() && !current.output().isBlank()) {
+                    String previous = current.output().trim();
+                    execOutput(containerId, new String[]{"sh", "-c", "ln -s \"" + previous + "\" "
+                            + NODE_APPLICATION_PATH + "/current.rollback && mv -Tf " + NODE_APPLICATION_PATH
+                            + "/current.rollback " + NODE_APPLICATION_PATH + "/current"}, 15);
+                    lifecycleManager.getDockerClient().restartContainerCmd(containerId).exec();
+                }
+                throw new IllegalStateException("Application health check on 127.0.0.1:" + port
+                        + " failed. Ensure the start command listens on PORT and binds 0.0.0.0, not localhost. " + health.output());
+            }
+            return new NodeDeployment(releaseId, files, port, start);
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Could not read the Node.js deployment: " + e.getMessage(), e);
+        } finally {
+            if (staging != null) {
+                deleteTree(staging);
+            }
+        }
+    }
+
+    private int unpackNodeUploads(List<StaticFile> uploads, Path staging) throws IOException {
+        if (uploads == null || uploads.isEmpty() || uploads.size() > 1_000) {
+            throw new IllegalArgumentException("Provide between 1 and 1000 deployment files.");
+        }
+        long[] size = {0};
+        int[] files = {0};
+        for (StaticFile upload : uploads) {
+            if (upload.path().toLowerCase().endsWith(".zip")) {
+                try (ZipArchiveInputStream zip = new ZipArchiveInputStream(Files.newInputStream(upload.source()))) {
+                    ZipArchiveEntry entry;
+                    while ((entry = zip.getNextZipEntry()) != null) {
+                        if (entry.isUnixSymlink()) {
+                            throw new IllegalArgumentException("Deployment archives must not contain symbolic links.");
+                        }
+                        if (!entry.isDirectory()) {
+                            writeNodeFile(staging, entry.getName(), zip, size, files);
+                        } else {
+                            validateNodePath(entry.getName());
+                        }
+                    }
+                }
+            } else {
+                try (java.io.InputStream input = Files.newInputStream(upload.source())) {
+                    writeNodeFile(staging, upload.path(), input, size, files);
+                }
+            }
+        }
+        return files[0];
+    }
+
+    private void writeNodeFile(Path staging, String name, java.io.InputStream input, long[] size, int[] files) throws IOException {
+        validateNodePath(name);
+        if (name.equals("node_modules") || name.startsWith("node_modules/")) {
+            return;
+        }
+        if (++files[0] > 1_000) {
+            throw new IllegalArgumentException("Deployment archive contains more than 1000 files.");
+        }
+        Path target = staging.resolve(name).normalize();
+        if (!target.startsWith(staging)) {
+            throw new IllegalArgumentException("Deployment archive contains a traversal path.");
+        }
+        Files.createDirectories(target.getParent());
+        try (OutputStream output = Files.newOutputStream(target)) {
+            byte[] buffer = new byte[8192];
+            int read;
+            while ((read = input.read(buffer)) >= 0) {
+                size[0] += read;
+                if (size[0] > 50L * 1024 * 1024) {
+                    throw new IllegalArgumentException("Deployment archive exceeds the 50 MiB uncompressed limit.");
+                }
+                output.write(buffer, 0, read);
+            }
+        }
+    }
+
+    private void copyNodeRelease(String containerId, Path staging, String releaseId) throws IOException {
+        ByteArrayOutputStream archive = new ByteArrayOutputStream();
+        try (TarArchiveOutputStream tar = new TarArchiveOutputStream(archive)) {
+            tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
+            try (java.util.stream.Stream<Path> paths = Files.walk(staging)) {
+                for (Path path : paths.filter(Files::isRegularFile).toList()) {
+                    String name = staging.relativize(path).toString().replace('\\', '/');
+                    TarArchiveEntry entry = new TarArchiveEntry(name);
+                    entry.setMode(0644);
+                    entry.setSize(Files.size(path));
+                    tar.putArchiveEntry(entry);
+                    Files.copy(path, tar);
+                    tar.closeArchiveEntry();
+                }
+            }
+            tar.finish();
+        }
+        lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)
+                .withTarInputStream(new ByteArrayInputStream(archive.toByteArray()))
+                .withRemotePath(NODE_APPLICATION_PATH + "/releases/" + releaseId).exec();
+    }
+
+    static void validateNodePath(String path) {
+        validateDeploymentPath(path);
+        if (path.startsWith(".floci-")) {
+            throw new IllegalArgumentException("Deployment contains a reserved path.");
+        }
+    }
+
+    private CommandResult execOutput(String containerId, String[] command, int timeoutSeconds) {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        try {
+            String execId = lifecycleManager.getDockerClient().execCreateCmd(containerId).withCmd(command)
+                    .withAttachStdout(true).withAttachStderr(true).exec().getId();
+            CountDownLatch complete = new CountDownLatch(1);
+            lifecycleManager.getDockerClient().execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>() {
+                @Override public void onNext(Frame frame) { output.writeBytes(frame.getPayload()); }
+                @Override public void onComplete() { complete.countDown(); }
+                @Override public void onError(Throwable error) { complete.countDown(); }
+            });
+            if (!complete.await(timeoutSeconds, TimeUnit.SECONDS)) {
+                return new CommandResult(false, "Command timed out.");
+            }
+            InspectExecResponse result = lifecycleManager.getDockerClient().inspectExecCmd(execId).exec();
+            return new CommandResult(result.getExitCodeLong() != null && result.getExitCodeLong() == 0,
+                    output.toString(StandardCharsets.UTF_8).substring(0, Math.min(output.size(), 8_000)));
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return new CommandResult(false, "Command interrupted.");
+        } catch (RuntimeException e) {
+            return new CommandResult(false, e.getMessage());
+        }
+    }
+
+    private static void deleteTree(Path root) {
+        try (java.util.stream.Stream<Path> paths = Files.walk(root)) {
+            paths.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
+                try {
+                    Files.deleteIfExists(path);
+                } catch (IOException ignored) {
+                    // The staging directory is outside persisted state and the OS will eventually clean it.
+                }
+            });
+        } catch (IOException ignored) {
+            // The staging directory is outside persisted state and the OS will eventually clean it.
+        }
+    }
+
+    static void validateDeploymentPath(String path) {
+        if (path == null || path.isBlank() || path.startsWith("/") || path.startsWith("\\")
+                || path.contains("\\") || path.indexOf('\u0000') >= 0) {
+            throw new IllegalArgumentException("Static file paths must be non-empty relative paths using forward slashes.");
+        }
+        for (String segment : path.split("/")) {
+            if (segment.isEmpty() || ".".equals(segment) || "..".equals(segment)) {
+                throw new IllegalArgumentException("Static file paths must not contain traversal segments.");
+            }
+        }
+    }
+
+    public record StaticFile(String path, Path source) {
+    }
+
+    public record NodeDeployment(String releaseId, int files, int port, String startCommand) {
+    }
+
+    private record CommandResult(boolean success, String output) {
+    }
+
+    private boolean bootstrapSsh(String containerId, String user, String publicKey, int hostPort) {
+        if (publicKey == null || publicKey.isBlank()) {
+            return false;
+        }
+        String encodedKey = Base64.getEncoder().encodeToString(publicKey.trim().getBytes(StandardCharsets.UTF_8));
+        String setupUser = "id -u " + user + " >/dev/null 2>&1 || useradd -m -s /bin/bash " + user + ";"
+                + "mkdir -p /home/" + user + "/.ssh;chmod 700 /home/" + user + "/.ssh;"
+                + "printf %s '" + encodedKey + "' | base64 -d > /home/" + user + "/.ssh/authorized_keys;"
+                + "chown -R " + user + ":" + user + " /home/" + user + "/.ssh;chmod 600 /home/" + user + "/.ssh/authorized_keys";
+        return exec(containerId, new String[]{"sh", "-c", setupUser}, 15)
+                && startSshd(containerId)
+                && sshBanner(hostPort);
+    }
+
+    private boolean startSshd(String containerId) {
+        String install = "if command -v dnf >/dev/null 2>&1; then dnf install -y openssh-server openssh-clients; "
+                + "elif command -v apt-get >/dev/null 2>&1; then apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server openssh-client; "
+                + "else exit 1; fi";
+        return exec(containerId, new String[]{"sh", "-c", "command -v sshd >/dev/null 2>&1 || (" + install + ")"}, 120)
+                && exec(containerId, new String[]{"sh", "-c", "ssh-keygen -A && mkdir -p /run/sshd && /usr/sbin/sshd"}, 15);
+    }
+
+    private boolean exec(String containerId, String[] command, int timeoutSeconds) {
+        try {
+            String execId = lifecycleManager.getDockerClient().execCreateCmd(containerId).withCmd(command)
+                    .withAttachStdout(true).withAttachStderr(true).exec().getId();
+            CountDownLatch complete = new CountDownLatch(1);
+            lifecycleManager.getDockerClient().execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>() {
+                @Override public void onComplete() { complete.countDown(); }
+                @Override public void onError(Throwable error) { complete.countDown(); }
+            });
+            if (!complete.await(timeoutSeconds, TimeUnit.SECONDS)) {
+                return false;
+            }
+            InspectExecResponse result = lifecycleManager.getDockerClient().inspectExecCmd(execId).exec();
+            return result.getExitCodeLong() != null && result.getExitCodeLong() == 0;
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return false;
+        } catch (RuntimeException e) {
+            LOG.warnv("Could not execute SSH bootstrap in container {0}: {1}", containerId, e.getMessage());
+            return false;
+        }
+    }
+
+    private boolean sshBanner(int hostPort) {
+        List<String> hosts = List.of("127.0.0.1", "host.docker.internal");
+        for (int attempt = 0; attempt < 20; attempt++) {
+            for (String host : hosts) {
+                try (Socket socket = new Socket()) {
+                    socket.connect(new InetSocketAddress(host, hostPort), 1_000);
+                    socket.setSoTimeout(1_000);
+                    BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.US_ASCII));
+                    String banner = reader.readLine();
+                    if (banner != null && banner.startsWith("SSH-")) {
+                        return true;
+                    }
+                } catch (IOException ignored) {
+                    // Docker Desktop publishes container ports on the host gateway, unavailable during early startup.
+                }
+            }
+            try {
+                Thread.sleep(250);
+            } catch (InterruptedException interrupted) {
+                Thread.currentThread().interrupt();
+                return false;
+            }
+        }
+        return false;
+    }
+
+    private boolean isDockerAvailable() {
+        try {
+            lifecycleManager.getDockerClient().pingCmd().exec();
+            return true;
+        } catch (RuntimeException e) {
+            LOG.debugv("Docker is unavailable for Lightsail containers: {0}", e.getMessage());
+            return false;
+        }
+    }
+}

src/main/java/io/github/hectorvent/floci/services/lightsail/LightsailJsonHandler.java

@@ -81,7 +81,6 @@ public class LightsailJsonHandler {
             Map.entry("GetDomains", "domains"),
             Map.entry("GetExportSnapshotRecords", "exportSnapshotRecords"),
             Map.entry("GetInstanceMetricData", "metricData"),
-            Map.entry("GetInstanceSnapshots", "instanceSnapshots"),
             Map.entry("GetLoadBalancerMetricData", "metricData"),
             Map.entry("GetLoadBalancerTlsCertificates", "tlsCertificates"),
             Map.entry("GetLoadBalancerTlsPolicies", "tlsPolicies"),
@@ -115,6 +114,11 @@ public class LightsailJsonHandler {
             case "StopInstance" -> ok(service.stopInstance(region, required(request, "instanceName")));
             case "RebootInstance" -> ok(service.rebootInstance(region, required(request, "instanceName")));
             case "GetInstanceState" -> ok(service.getInstanceState(region, required(request, "instanceName")));
+            case "CreateInstanceSnapshot" -> ok(service.createInstanceSnapshot(region, request));
+            case "CreateInstancesFromSnapshot" -> ok(service.createInstancesFromSnapshot(region, request));
+            case "GetInstanceSnapshots" -> ok(service.getInstanceSnapshots(region));
+            case "GetInstanceSnapshot" -> ok(service.getInstanceSnapshot(region, required(request, "instanceSnapshotName")));
+            case "DeleteInstanceSnapshot" -> ok(service.deleteInstanceSnapshot(region, required(request, "instanceSnapshotName")));
             case "GetInstancePortStates" -> ok(service.getInstancePortStates(region, required(request, "instanceName")));
             case "OpenInstancePublicPorts" -> ok(service.putInstancePorts(region, requiredInstanceName(request), request, true));
             case "PutInstancePublicPorts" -> ok(service.putInstancePorts(region, requiredInstanceName(request), request, false));

src/main/java/io/github/hectorvent/floci/services/lightsail/LightsailPortForwardManager.java

@@ -0,0 +1,197 @@
+package io.github.hectorvent.floci.services.lightsail;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.InspectContainerResponse;
+import com.github.dockerjava.api.model.ContainerNetwork;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.core.common.docker.PortAllocator;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@ApplicationScoped
+public class LightsailPortForwardManager {
+
+    static final String PORT_FORWARDS_FIELD = "_portForwards";
+    private static final Logger LOG = Logger.getLogger(LightsailPortForwardManager.class);
+
+    private final DockerClient dockerClient;
+    private final ContainerBuilder containerBuilder;
+    private final ContainerLifecycleManager lifecycleManager;
+    private final PortAllocator portAllocator;
+    private final EmulatorConfig config;
+
+    @Inject
+    public LightsailPortForwardManager(DockerClient dockerClient, ContainerBuilder containerBuilder,
+                                       ContainerLifecycleManager lifecycleManager, PortAllocator portAllocator,
+                                       EmulatorConfig config) {
+        this.dockerClient = dockerClient;
+        this.containerBuilder = containerBuilder;
+        this.lifecycleManager = lifecycleManager;
+        this.portAllocator = portAllocator;
+        this.config = config;
+    }
+
+    public void reconcile(ObjectNode instance) {
+        if (instance == null || !"running".equals(instance.path("state").path("name").asText())
+                || !instance.hasNonNull("containerId")) {
+            return;
+        }
+        synchronized (instance) {
+            Set<Integer> desiredPorts = extractPublishablePorts(instance.path("networking").path("ports"),
+                    config.services().ec2().maxPublishedPortsPerInstance());
+            // OS runtimes publish SSH directly on their container, not through a sidecar.
+            desiredPorts.remove(22);
+            ObjectNode forwards = forwards(instance);
+            for (Integer appPort : desiredPorts) {
+                if (!forwards.has(String.valueOf(appPort))) {
+                    publish(instance, appPort, forwards);
+                }
+            }
+            for (String appPort : new ArrayList<>(forwards.propertyStream().map(Map.Entry::getKey).toList())) {
+                if (!desiredPorts.contains(Integer.parseInt(appPort))) {
+                    unpublish(instance, appPort, forwards);
+                }
+            }
+        }
+    }
+
+    public void unpublishAll(ObjectNode instance) {
+        if (instance == null) {
+            return;
+        }
+        synchronized (instance) {
+            ObjectNode forwards = forwards(instance);
+            for (String appPort : new ArrayList<>(forwards.propertyStream().map(Map.Entry::getKey).toList())) {
+                unpublish(instance, appPort, forwards);
+            }
+        }
+    }
+
+    public void releaseSshPort(ObjectNode instance) {
+        if (instance != null && instance.has("_sshHostPort")) {
+            portAllocator.release(instance.path("_sshHostPort").asInt());
+            instance.remove("_sshHostPort");
+        }
+    }
+
+    static Set<Integer> extractPublishablePorts(JsonNode portRules, int maxPerInstance) {
+        Set<Integer> ports = new LinkedHashSet<>();
+        if (!portRules.isArray()) {
+            return ports;
+        }
+        for (JsonNode rule : portRules) {
+            if (!"tcp".equalsIgnoreCase(rule.path("protocol").asText()) || !hasCidrSource(rule)) {
+                continue;
+            }
+            int from = rule.path("fromPort").asInt(-1);
+            int to = rule.path("toPort").asInt(-1);
+            if (from < 0 || to < 0) {
+                continue;
+            }
+            int low = Math.min(from, to);
+            int high = Math.max(from, to);
+            if ((long) high - low + 1 > maxPerInstance) {
+                LOG.warnv("Skipping wide Lightsail public-port range {0}-{1} (span exceeds max {2})",
+                        low, high, maxPerInstance);
+                continue;
+            }
+            for (int port = low; port <= high && ports.size() < maxPerInstance; port++) {
+                ports.add(port);
+            }
+        }
+        return ports;
+    }
+
+    static String forwardContainerName(String instanceName, int appPort) {
+        return "floci-lightsail-fwd-" + instanceName + "-" + appPort;
+    }
+
+    private void publish(ObjectNode instance, int appPort, ObjectNode forwards) {
+        int hostPort = portAllocator.allocate(config.services().ec2().appPortRangeStart(),
+                config.services().ec2().appPortRangeEnd());
+        try {
+            NetworkTarget target = resolveTarget(instance.path("containerId").asText());
+            if (target == null) {
+                portAllocator.release(hostPort);
+                return;
+            }
+            String name = forwardContainerName(instance.path("name").asText(), appPort);
+            lifecycleManager.removeIfExists(name);
+            ContainerBuilder.Builder builder = containerBuilder.newContainer(config.services().ec2().socatImage())
+                    .withName(name)
+                    .withEntrypoint(List.of("socat"))
+                    .withCmd(List.of("TCP-LISTEN:" + appPort + ",fork,reuseaddr", "TCP:" + target.ip() + ":" + appPort))
+                    .withPortBinding(appPort, hostPort)
+                    .withLogRotation();
+            if (target.network() != null) {
+                builder.withNetworkMode(target.network());
+            }
+            ContainerSpec spec = builder.build();
+            String containerId = lifecycleManager.create(spec);
+            lifecycleManager.startCreated(containerId, spec);
+            forwards.put(String.valueOf(appPort), hostPort);
+            LOG.infov("Published Lightsail instance {0} app port {1} on host port {2}",
+                    instance.path("name").asText(), appPort, hostPort);
+        } catch (Exception e) {
+            portAllocator.release(hostPort);
+            LOG.warnv("Failed to publish Lightsail instance {0} app port {1}: {2}",
+                    instance.path("name").asText(), appPort, e.getMessage());
+        }
+    }
+
+    private void unpublish(ObjectNode instance, String appPort, ObjectNode forwards) {
+        lifecycleManager.removeIfExists(forwardContainerName(instance.path("name").asText(), Integer.parseInt(appPort)));
+        int hostPort = forwards.path(appPort).asInt();
+        forwards.remove(appPort);
+        portAllocator.release(hostPort);
+    }
+
+    private ObjectNode forwards(ObjectNode instance) {
+        JsonNode node = instance.get(PORT_FORWARDS_FIELD);
+        if (node instanceof ObjectNode objectNode) {
+            return objectNode;
+        }
+        return instance.putObject(PORT_FORWARDS_FIELD);
+    }
+
+    private NetworkTarget resolveTarget(String containerId) {
+        try {
+            InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();
+            Map<String, ContainerNetwork> networks = inspect.getNetworkSettings().getNetworks();
+            String networkMode = inspect.getHostConfig().getNetworkMode();
+            if (networks.isEmpty() && networkMode != null && networkMode.startsWith("container:")) {
+                inspect = dockerClient.inspectContainerCmd(networkMode.substring("container:".length())).exec();
+                networks = inspect.getNetworkSettings().getNetworks();
+            }
+            for (Map.Entry<String, ContainerNetwork> entry : networks.entrySet()) {
+                String ip = entry.getValue().getIpAddress();
+                if (ip != null && !ip.isBlank()) {
+                    return new NetworkTarget("bridge".equals(entry.getKey()) ? null : entry.getKey(), ip);
+                }
+            }
+        } catch (Exception e) {
+            LOG.warnv("Could not inspect Lightsail instance container {0}: {1}", containerId, e.getMessage());
+        }
+        return null;
+    }
+
+    private static boolean hasCidrSource(JsonNode rule) {
+        return rule.path("cidrs").isArray() && !rule.path("cidrs").isEmpty()
+                || rule.path("ipv6Cidrs").isArray() && !rule.path("ipv6Cidrs").isEmpty();
+    }
+
+    private record NetworkTarget(String network, String ip) {
+    }
+}

src/main/java/io/github/hectorvent/floci/services/lightsail/LightsailService.java

@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.core.resource.ResourceProvider;
 import io.github.hectorvent.floci.core.resource.SupportedResourceType;
 import io.github.hectorvent.floci.core.storage.StorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.ec2.Ec2KeyMaterial;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 
@@ -35,20 +36,30 @@ public class LightsailService implements ResourceProvider {
     private static final String RESOURCE_DISK = "Disk";
     private static final String RESOURCE_STATIC_IP = "StaticIp";
     private static final String RESOURCE_KEY_PAIR = "KeyPair";
+    private static final String RESOURCE_INSTANCE_SNAPSHOT = "InstanceSnapshot";
 
     private final StorageBackend<String, ObjectNode> resourceStore;
     private final StorageBackend<String, ObjectNode> operationStore;
     private final ObjectMapper mapper;
     private final RegionResolver regionResolver;
+    private final LightsailContainerManager containerManager;
+    private final LightsailPortForwardManager portForwardManager;
 
     @Inject
-    public LightsailService(StorageFactory storageFactory, ObjectMapper mapper, RegionResolver regionResolver) {
+    public LightsailService(StorageFactory storageFactory, ObjectMapper mapper, RegionResolver regionResolver,
+                             LightsailContainerManager containerManager, LightsailPortForwardManager portForwardManager) {
         this.resourceStore = storageFactory.create(SERVICE, "lightsail-resources.json",
                 new TypeReference<Map<String, ObjectNode>>() {});
         this.operationStore = storageFactory.create(SERVICE, "lightsail-operations.json",
                 new TypeReference<Map<String, ObjectNode>>() {});
         this.mapper = mapper;
         this.regionResolver = regionResolver;
+        this.containerManager = containerManager;
+        this.portForwardManager = portForwardManager;
+    }
+
+    LightsailService(StorageFactory storageFactory, ObjectMapper mapper, RegionResolver regionResolver) {
+        this(storageFactory, mapper, regionResolver, null, null);
     }
 
     public ObjectNode createInstances(String region, JsonNode request) {
@@ -56,6 +67,11 @@ public class LightsailService implements ResourceProvider {
         requireText(request, "availabilityZone");
         String blueprintId = requireText(request, "blueprintId");
         String bundleId = requireText(request, "bundleId");
+        LightsailContainerManager.Runtime runtime = LightsailContainerManager.runtimeFor(blueprintId);
+        if (runtime == null && textOrNull(request, "_containerImage") == null) {
+            throw new AwsException("InvalidInputException", "Blueprint " + blueprintId
+                    + " is not available locally because it has no runtime mapping.", 400);
+        }
         String availabilityZone = request.path("availabilityZone").asText();
         ArrayNode operations = mapper.createArrayNode();
 
@@ -75,12 +91,12 @@ public class LightsailService implements ResourceProvider {
             instance.set("ipv6Addresses", mapper.createArrayNode());
             instance.put("ipAddressType", request.path("ipAddressType").asText("ipv4"));
             instance.set("hardware", hardware(bundleId));
-            instance.set("networking", defaultNetworking());
+            instance.set("networking", defaultNetworking(runtime == null ? null : runtime.applicationPort()));
             instance.set("state", instanceState("running"));
             instance.put("username", defaultUsername(blueprintId));
-            if (request.hasNonNull("keyPairName")) {
-                instance.put("sshKeyName", request.path("keyPairName").asText());
-            }
+            String keyPairName = request.path("keyPairName").asText("LightsailDefaultKeyPair");
+            ObjectNode keyPair = selectedKeyPair(region, keyPairName);
+            instance.put("sshKeyName", keyPairName);
             instance.set("tags", tags(request.path("tags")));
             instance.set("metadataOptions", mapper.createObjectNode()
                     .put("state", "applied")
@@ -88,7 +104,49 @@ public class LightsailService implements ResourceProvider {
                     .put("httpEndpoint", "enabled")
                     .put("httpPutResponseHopLimit", 1));
 
+            String snapshotImage = textOrNull(request, "_containerImage");
+            LightsailContainerManager.Launch launch = containerManager == null ? null
+                    : containerManager.launch(name, region, blueprintId, bundleId, snapshotImage,
+                    decodePublicKey(keyPair.path("publicKeyBase64").asText()));
+            String containerId = launch == null ? null : launch.containerId();
+            instance.put("runtimeStatus", containerId == null ? "container-unavailable" : "running");
+            if (runtime != null) {
+                instance.put("runtimeImage", runtime.image());
+                if (runtime.applicationPort() != null) {
+                    instance.put("applicationPort", runtime.applicationPort());
+                }
+            }
+            if (containerId != null) {
+                instance.put("containerId", containerId);
+            }
+            if (launch != null && launch.sshHostPort() != null) {
+                instance.put("_sshHostPort", launch.sshHostPort());
+            }
+            if (launch != null && launch.databaseContainerId() != null) {
+                instance.put("databaseContainerId", launch.databaseContainerId());
+                instance.put("databaseVolume", launch.databaseVolume());
+            }
+            if (launch != null && launch.wordpressContentVolume() != null) {
+                instance.put("wordpressContentVolume", launch.wordpressContentVolume());
+            }
+            if (launch != null && launch.applicationVolume() != null) {
+                instance.put("applicationVolume", launch.applicationVolume());
+                instance.put("applicationRelease", launch.applicationRelease());
+                instance.set("runtimeMetadata", mapper.createObjectNode()
+                        .put("launchedAt", epochSeconds(Instant.now()))
+                        .put("applicationPath", "/opt/lightsail/app")
+                        .put("currentRelease", launch.applicationRelease())
+                        .put("releaseRetention", 3)
+                        .put("stdoutLog", "/opt/lightsail/app/logs/app.stdout.log")
+                        .put("stderrLog", "/opt/lightsail/app/logs/app.stderr.log")
+                        .put("supervisorLog", "/opt/lightsail/app/logs/supervisor.log"));
+            }
+
             resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
+            if (portForwardManager != null) {
+                portForwardManager.reconcile(instance);
+                resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
+            }
             operations.add(operation(region, RESOURCE_INSTANCE, name, "CreateInstance"));
         }
 
@@ -98,16 +156,33 @@ public class LightsailService implements ResourceProvider {
     public ObjectNode getInstances(String region) {
         ObjectNode response = mapper.createObjectNode();
         ArrayNode instances = response.putArray("instances");
-        listResources(region, RESOURCE_INSTANCE).forEach(instances::add);
+        for (ObjectNode instance : listResources(region, RESOURCE_INSTANCE)) {
+            reconcileForwards(region, instance);
+            instances.add(instanceResponse(instance));
+        }
         return response;
     }
 
     public ObjectNode getInstance(String region, String name) {
-        return mapper.createObjectNode().set("instance", requireResource(region, RESOURCE_INSTANCE, name));
+        return mapper.createObjectNode().set("instance", instanceResponse(requireResource(region, RESOURCE_INSTANCE, name)));
     }
 
     public ObjectNode deleteInstance(String region, String name) {
-        requireResource(region, RESOURCE_INSTANCE, name);
+        ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
+        if (portForwardManager != null) {
+            portForwardManager.unpublishAll(instance);
+        }
+        try {
+            if (containerManager != null) {
+                containerManager.terminate(instance.path("containerId").asText(null),
+                        instance.path("databaseContainerId").asText(null), instance.path("databaseVolume").asText(null),
+                        instance.path("wordpressContentVolume").asText(null), instance.path("applicationVolume").asText(null));
+            }
+        } finally {
+            if (portForwardManager != null) {
+                portForwardManager.releaseSshPort(instance);
+            }
+        }
         resourceStore.delete(resourceKey(region, RESOURCE_INSTANCE, name));
         detachStaticIpsFrom(name);
         detachDisksFrom(name);
@@ -116,21 +191,42 @@ public class LightsailService implements ResourceProvider {
 
     public ObjectNode startInstance(String region, String name) {
         ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
+        if (containerManager != null) {
+            containerManager.start(instance.path("containerId").asText(null),
+                    instance.path("databaseContainerId").asText(null), instance.has("_sshHostPort"));
+        }
         instance.set("state", instanceState("running"));
+        instance.put("runtimeStatus", instance.hasNonNull("containerId") ? "running" : "container-unavailable");
+        if (portForwardManager != null) {
+            portForwardManager.reconcile(instance);
+        }
         resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
         return operations(region, RESOURCE_INSTANCE, name, "StartInstance");
     }
 
     public ObjectNode stopInstance(String region, String name) {
         ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
+        if (portForwardManager != null) {
+            portForwardManager.unpublishAll(instance);
+        }
+        if (containerManager != null) {
+            containerManager.stop(instance.path("containerId").asText(null),
+                    instance.path("databaseContainerId").asText(null));
+        }
         instance.set("state", instanceState("stopped"));
+        instance.put("runtimeStatus", instance.hasNonNull("containerId") ? "stopped" : "container-unavailable");
         resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
         return operations(region, RESOURCE_INSTANCE, name, "StopInstance");
     }
 
     public ObjectNode rebootInstance(String region, String name) {
         ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
+        if (containerManager != null) {
+            containerManager.reboot(instance.path("containerId").asText(null),
+                    instance.path("databaseContainerId").asText(null), instance.has("_sshHostPort"));
+        }
         instance.set("state", instanceState("running"));
+        instance.put("runtimeStatus", instance.hasNonNull("containerId") ? "running" : "container-unavailable");
         resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
         return operations(region, RESOURCE_INSTANCE, name, "RebootInstance");
     }
@@ -140,6 +236,135 @@ public class LightsailService implements ResourceProvider {
         return mapper.createObjectNode().set("state", instance.path("state").deepCopy());
     }
 
+    public List<Map<String, Object>> hostPortMappings(String region, String name) {
+        ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
+        List<Map<String, Object>> mappings = new ArrayList<>();
+        instance.path(LightsailPortForwardManager.PORT_FORWARDS_FIELD).properties().forEach(entry -> {
+            Map<String, Object> mapping = new LinkedHashMap<>();
+            mapping.put("instancePort", Integer.parseInt(entry.getKey()));
+            mapping.put("host", "localhost");
+            mapping.put("hostPort", entry.getValue().asInt());
+            mapping.put("protocol", "tcp");
+            mappings.add(mapping);
+        });
+        if (instance.has("_sshHostPort")) {
+            Map<String, Object> mapping = new LinkedHashMap<>();
+            mapping.put("instancePort", 22);
+            mapping.put("host", "localhost");
+            mapping.put("hostPort", instance.path("_sshHostPort").asInt());
+            mapping.put("protocol", "tcp");
+            mappings.add(mapping);
+        }
+        return mappings;
+    }
+
+    public int deployStaticFiles(String region, String name, List<LightsailContainerManager.StaticFile> files) {
+        ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
+        if (!"nginx".equals(instance.path("blueprintId").asText())) {
+            throw new AwsException("InvalidInputException", "Static file deployment is available only for Nginx Lightsail instances.", 400);
+        }
+        if (containerManager == null) {
+            throw new AwsException("InternalError", "Lightsail container support is unavailable.", 503);
+        }
+        try {
+            containerManager.deployStaticFiles(instance.path("containerId").asText(null), files);
+            return files.size();
+        } catch (IllegalArgumentException e) {
+            throw new AwsException("InvalidInputException", e.getMessage(), 400);
+        } catch (IllegalStateException e) {
+            throw new AwsException("InternalError", e.getMessage(), 502);
+        }
+    }
+
+    public LightsailContainerManager.NodeDeployment deployNodeApplication(String region, String name,
+                                                                            List<LightsailContainerManager.StaticFile> files,
+                                                                            String startCommand, int port) {
+        ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
+        if (!"nodejs".equals(instance.path("blueprintId").asText())) {
+            throw new AwsException("InvalidInputException", "Node.js deployment is available only for Node.js Lightsail instances.", 400);
+        }
+        if (containerManager == null) {
+            throw new AwsException("InternalError", "Lightsail container support is unavailable.", 503);
+        }
+        try {
+            LightsailContainerManager.NodeDeployment deployment = containerManager.deployNodeApplication(
+                    instance.path("containerId").asText(null), files, startCommand, port);
+            instance.put("applicationPort", port);
+            instance.put("applicationRelease", deployment.releaseId());
+            ((ObjectNode) instance.path("runtimeMetadata")).put("currentRelease", deployment.releaseId());
+            resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
+            return deployment;
+        } catch (IllegalArgumentException e) {
+            throw new AwsException("InvalidInputException", e.getMessage(), 400);
+        } catch (IllegalStateException e) {
+            throw new AwsException("InternalError", e.getMessage(), 502);
+        }
+    }
+
+    public ObjectNode createInstanceSnapshot(String region, JsonNode request) {
+        String instanceName = requireText(request, "instanceName");
+        String snapshotName = requireText(request, "instanceSnapshotName");
+        ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, instanceName);
+        ensureNotExists(region, RESOURCE_INSTANCE_SNAPSHOT, snapshotName);
+
+        ObjectNode snapshot = baseResource(region, instance.path("location").path("availabilityZone").asText(),
+                RESOURCE_INSTANCE_SNAPSHOT, snapshotName);
+        snapshot.put("fromInstanceName", instanceName);
+        snapshot.put("blueprintId", instance.path("blueprintId").asText());
+        snapshot.put("bundleId", instance.path("bundleId").asText());
+        snapshot.put("state", "available");
+        snapshot.set("tags", tags(request.path("tags")));
+        try {
+            String capturedImage = containerManager == null ? null
+                    : containerManager.capture(instance.path("containerId").asText(null), snapshotName);
+            if (capturedImage != null) {
+                snapshot.put("containerImage", capturedImage);
+            }
+        } catch (IllegalStateException e) {
+            throw new AwsException("InternalError", e.getMessage(), 500);
+        }
+        resourceStore.put(resourceKey(region, RESOURCE_INSTANCE_SNAPSHOT, snapshotName), snapshot);
+        return operations(region, RESOURCE_INSTANCE_SNAPSHOT, snapshotName, "CreateInstanceSnapshot");
+    }
+
+    public ObjectNode getInstanceSnapshots(String region) {
+        ObjectNode response = mapper.createObjectNode();
+        ArrayNode snapshots = response.putArray("instanceSnapshots");
+        listResources(region, RESOURCE_INSTANCE_SNAPSHOT).stream()
+                .map(this::snapshotResponse)
+                .forEach(snapshots::add);
+        return response;
+    }
+
+    public ObjectNode getInstanceSnapshot(String region, String name) {
+        return mapper.createObjectNode().set("instanceSnapshot", snapshotResponse(
+                requireResource(region, RESOURCE_INSTANCE_SNAPSHOT, name)));
+    }
+
+    public ObjectNode deleteInstanceSnapshot(String region, String name) {
+        ObjectNode snapshot = requireResource(region, RESOURCE_INSTANCE_SNAPSHOT, name);
+        if (containerManager != null) {
+            containerManager.removeCapture(snapshot.path("containerImage").asText(null));
+        }
+        resourceStore.delete(resourceKey(region, RESOURCE_INSTANCE_SNAPSHOT, name));
+        return operations(region, RESOURCE_INSTANCE_SNAPSHOT, name, "DeleteInstanceSnapshot");
+    }
+
+    public ObjectNode createInstancesFromSnapshot(String region, JsonNode request) {
+        String snapshotName = requireText(request, "instanceSnapshotName");
+        ObjectNode snapshot = requireResource(region, RESOURCE_INSTANCE_SNAPSHOT, snapshotName);
+        ObjectNode createRequest = request.deepCopy();
+        createRequest.put("blueprintId", snapshot.path("blueprintId").asText());
+        createRequest.put("bundleId", snapshot.path("bundleId").asText());
+        if (snapshot.hasNonNull("containerImage")) {
+            createRequest.put("_containerImage", snapshot.path("containerImage").asText());
+        }
+        if (!createRequest.hasNonNull("availabilityZone")) {
+            createRequest.put("availabilityZone", snapshot.path("location").path("availabilityZone").asText());
+        }
+        return createInstances(region, createRequest);
+    }
+
     public ObjectNode getInstancePortStates(String region, String name) {
         ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
         ObjectNode response = mapper.createObjectNode();
@@ -169,6 +394,9 @@ public class LightsailService implements ResourceProvider {
             ports.add(portInfo.deepCopy());
         }
         ((ObjectNode) instance.path("networking")).set("ports", ports);
+        if (portForwardManager != null) {
+            portForwardManager.reconcile(instance);
+        }
         resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
         return mapper.createObjectNode().set("operation", operation(region, RESOURCE_INSTANCE, name,
                 merge ? "OpenInstancePublicPorts" : "PutInstancePublicPorts"));
@@ -187,6 +415,9 @@ public class LightsailService implements ResourceProvider {
             }
         }
         ((ObjectNode) instance.path("networking")).set("ports", remaining);
+        if (portForwardManager != null) {
+            portForwardManager.reconcile(instance);
+        }
         resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
         return mapper.createObjectNode().set("operation", operation(region, RESOURCE_INSTANCE, name,
                 "CloseInstancePublicPorts"));
@@ -349,9 +580,11 @@ public class LightsailService implements ResourceProvider {
         resourceStore.put(resourceKey(region, RESOURCE_KEY_PAIR, name), keyPair);
 
         ObjectNode response = mapper.createObjectNode();
-        response.set("keyPair", keyPair.deepCopy());
+        ObjectNode keyPairResponse = keyPair.deepCopy();
+        keyPairResponse.remove("privateKeyBase64");
+        response.set("keyPair", keyPairResponse);
         response.put("publicKeyBase64", keyPair.path("publicKeyBase64").asText());
-        response.put("privateKeyBase64", privateKeyBase64(name));
+        response.put("privateKeyBase64", keyPair.path("privateKeyBase64").asText());
         response.set("operation", operation(region, RESOURCE_KEY_PAIR, name, "CreateKeyPair"));
         return response;
     }
@@ -374,8 +607,9 @@ public class LightsailService implements ResourceProvider {
                     return created;
                 });
         ObjectNode response = mapper.createObjectNode();
-        response.put("publicKeyBase64", publicKeyBase64(name));
-        response.put("privateKeyBase64", privateKeyBase64(name));
+        ObjectNode keyPair = requireResource(region, RESOURCE_KEY_PAIR, name);
+        response.put("publicKeyBase64", keyPair.path("publicKeyBase64").asText());
+        response.put("privateKeyBase64", keyPair.path("privateKeyBase64").asText());
         response.put("createdAt", epochSeconds(Instant.now()));
         return response;
     }
@@ -420,7 +654,9 @@ public class LightsailService implements ResourceProvider {
         blueprints.add(blueprint("ubuntu_22_04", "Ubuntu", "linux", "Ubuntu 22.04 LTS"));
         blueprints.add(blueprint("ubuntu_24_04", "Ubuntu", "linux", "Ubuntu 24.04 LTS"));
         blueprints.add(blueprint("amazon_linux_2023", "Amazon Linux", "linux", "Amazon Linux 2023"));
+        blueprints.add(blueprint("nginx", "Nginx", "nginx", "Nginx web server"));
         blueprints.add(blueprint("wordpress", "WordPress", "wordpress", "WordPress"));
+        blueprints.add(blueprint("nodejs", "Node.js", "nodejs", "Node.js current LTS application server"));
         return response;
     }
 
@@ -530,6 +766,7 @@ public class LightsailService implements ResourceProvider {
                 new SupportedResourceType(SERVICE + ":" + RESOURCE_INSTANCE, SERVICE, true),
                 new SupportedResourceType(SERVICE + ":" + RESOURCE_DISK, SERVICE, true),
                 new SupportedResourceType(SERVICE + ":" + RESOURCE_KEY_PAIR, SERVICE, true),
+                new SupportedResourceType(SERVICE + ":" + RESOURCE_INSTANCE_SNAPSHOT, SERVICE, true),
                 // Lightsail static IPs are not taggable in AWS.
                 new SupportedResourceType(SERVICE + ":" + RESOURCE_STATIC_IP, SERVICE, false));
     }
@@ -562,6 +799,12 @@ public class LightsailService implements ResourceProvider {
         return resource;
     }
 
+    private ObjectNode snapshotResponse(ObjectNode snapshot) {
+        ObjectNode response = snapshot.deepCopy();
+        response.remove("containerImage");
+        return response;
+    }
+
     private ObjectNode operation(String region, String resourceType, String resourceName, String operationType) {
         Instant now = Instant.now();
         String id = UUID.randomUUID().toString();
@@ -627,6 +870,21 @@ public class LightsailService implements ResourceProvider {
                 .toList();
     }
 
+    private ObjectNode instanceResponse(ObjectNode instance) {
+        ObjectNode response = instance.deepCopy();
+        response.remove(List.of("containerId", "databaseContainerId", "databaseVolume", "wordpressContentVolume", "applicationVolume", "applicationRelease", "runtimeMetadata", "runtimeStatus", "runtimeImage", "applicationPort", "_sshHostPort",
+                LightsailPortForwardManager.PORT_FORWARDS_FIELD));
+        return response;
+    }
+
+    private void reconcileForwards(String region, ObjectNode instance) {
+        if (portForwardManager == null) {
+            return;
+        }
+        portForwardManager.reconcile(instance);
+        resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, instance.path("name").asText()), instance);
+    }
+
     private void ensureNotExists(String region, String resourceType, String name) {
         if (resourceStore.get(resourceKey(region, resourceType, name)).isPresent()) {
             throw new AwsException("InvalidInputException", resourceType + " " + name + " already exists", 400);
@@ -666,11 +924,44 @@ public class LightsailService implements ResourceProvider {
     private ObjectNode keyPair(String region, String name, String publicKeyBase64, ArrayNode tags) {
         ObjectNode keyPair = baseResource(region, region + "a", RESOURCE_KEY_PAIR, name);
         keyPair.set("tags", tags);
-        keyPair.put("fingerprint", Base64.getEncoder().encodeToString(("fingerprint:" + name).getBytes(StandardCharsets.UTF_8)));
-        keyPair.put("publicKeyBase64", publicKeyBase64 == null ? publicKeyBase64(name) : publicKeyBase64);
+        if (publicKeyBase64 == null || publicKeyBase64.isBlank()) {
+            Ec2KeyMaterial.Generated generated = Ec2KeyMaterial.generateRsa();
+            keyPair.put("fingerprint", generated.fingerprint());
+            keyPair.put("publicKeyBase64", Base64.getEncoder().encodeToString(
+                    generated.openSshPublicKey().getBytes(StandardCharsets.UTF_8)));
+            keyPair.put("privateKeyBase64", Base64.getEncoder().encodeToString(
+                    generated.privateKeyPem().getBytes(StandardCharsets.UTF_8)));
+        } else {
+            String publicKey = decodePublicKey(publicKeyBase64);
+            String fingerprint = Ec2KeyMaterial.fingerprintOf(publicKey);
+            if (fingerprint == null) {
+                throw new AwsException("InvalidInputException", "publicKeyBase64 must contain a valid OpenSSH public key", 400);
+            }
+            keyPair.put("fingerprint", fingerprint);
+            keyPair.put("publicKeyBase64", publicKeyBase64);
+        }
         return keyPair;
     }
 
+    private ObjectNode selectedKeyPair(String region, String name) {
+        if ("LightsailDefaultKeyPair".equals(name)) {
+            return resourceStore.get(resourceKey(region, RESOURCE_KEY_PAIR, name)).orElseGet(() -> {
+                ObjectNode created = keyPair(region, name, null, mapper.createArrayNode());
+                resourceStore.put(resourceKey(region, RESOURCE_KEY_PAIR, name), created);
+                return created;
+            });
+        }
+        return requireResource(region, RESOURCE_KEY_PAIR, name);
+    }
+
+    private static String decodePublicKey(String publicKeyBase64) {
+        try {
+            return new String(Base64.getDecoder().decode(publicKeyBase64), StandardCharsets.UTF_8);
+        } catch (IllegalArgumentException e) {
+            throw new AwsException("InvalidInputException", "publicKeyBase64 must be base64 encoded", 400);
+        }
+    }
+
     private ObjectNode regionNode(String name, String displayName, String continentCode, boolean includeAzs) {
         ObjectNode region = mapper.createObjectNode();
         region.put("continentCode", continentCode);
@@ -746,7 +1037,7 @@ public class LightsailService implements ResourceProvider {
         return hardware;
     }
 
-    private ObjectNode defaultNetworking() {
+    private ObjectNode defaultNetworking(Integer applicationPort) {
         ObjectNode networking = mapper.createObjectNode();
         ArrayNode ports = networking.putArray("ports");
         ObjectNode ssh = mapper.createObjectNode();
@@ -757,6 +1048,16 @@ public class LightsailService implements ResourceProvider {
         ssh.putArray("ipv6Cidrs");
         ssh.putArray("cidrListAliases");
         ports.add(ssh);
+        if (applicationPort != null) {
+            ObjectNode application = mapper.createObjectNode();
+            application.put("fromPort", applicationPort);
+            application.put("toPort", applicationPort);
+            application.put("protocol", "tcp");
+            application.putArray("cidrs").add("0.0.0.0/0");
+            application.putArray("ipv6Cidrs");
+            application.putArray("cidrListAliases");
+            ports.add(application);
+        }
         networking.put("monthlyTransfer", mapper.createObjectNode()
                 .put("gbPerMonthAllocated", 1024));
         return networking;
@@ -892,21 +1193,19 @@ public class LightsailService implements ResourceProvider {
             case "ubuntu_24_04" -> "Ubuntu 24.04 LTS";
             case "ubuntu_22_04" -> "Ubuntu 22.04 LTS";
             case "amazon_linux_2023" -> "Amazon Linux 2023";
+            case "nginx" -> "Nginx";
             case "wordpress" -> "WordPress";
+            case "nodejs" -> "Node.js";
             default -> blueprintId;
         };
     }
 
     private static String defaultUsername(String blueprintId) {
-        return blueprintId != null && blueprintId.startsWith("amazon_linux") ? "ec2-user" : "ubuntu";
+        return switch (blueprintId) {
+            case "amazon_linux_2023" -> "ec2-user";
+            case "nodejs" -> "node";
+            default -> "ubuntu";
+        };
     }
 
-    private static String publicKeyBase64(String name) {
-        return Base64.getEncoder().encodeToString(("ssh-rsa FLOCI-" + name).getBytes(StandardCharsets.UTF_8));
-    }
-
-    private static String privateKeyBase64(String name) {
-        return Base64.getEncoder().encodeToString(("-----BEGIN PRIVATE KEY-----\nfloci-" + name
-                + "\n-----END PRIVATE KEY-----\n").getBytes(StandardCharsets.UTF_8));
-    }
 }

src/main/java/io/github/hectorvent/floci/services/msk/MskController.java

@@ -91,8 +91,15 @@ public class MskController {
 
     @GET
     @Path("/v1/clusters/{clusterArn}/bootstrap-brokers")
-    public Response getBootstrapBrokers(@PathParam("clusterArn") String clusterArn) {
+    public Response getBootstrapBrokers(@PathParam("clusterArn") String clusterArn,
+                                        @HeaderParam("X-Floci-Internal-Client") String internalClient) {
         String bootstrapBrokers = mskService.getBootstrapBrokers(clusterArn);
+        if ("true".equals(internalClient)) {
+            MskCluster cluster = mskService.describeCluster(clusterArn);
+            if (cluster.getInternalBootstrapBrokers() != null) {
+                bootstrapBrokers = cluster.getInternalBootstrapBrokers();
+            }
+        }
         return Response.ok(Map.of("bootstrapBrokerString", bootstrapBrokers)).build();
     }
 

src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java

@@ -7,7 +7,6 @@ import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
 import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
 import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
 import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;
-import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;
 import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
 import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
 import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
@@ -32,6 +31,7 @@ public class RedpandaManager {
 
     private static final Logger LOG = Logger.getLogger(RedpandaManager.class);
     private static final int KAFKA_PORT = 9092;
+    private static final int INTERNAL_KAFKA_PORT = 9093;
     static final int ADMIN_PORT = 9644;
 
     // Redpanda's Admin API exposes readiness at /v1/status/ready. /ready always returns 404
@@ -124,28 +124,22 @@ public class RedpandaManager {
         // reachable address up front. The host port has to be known before the
         // container starts (the flag is a startup arg), so in native mode we
         // pre-allocate it ourselves instead of letting Docker assign one dynamically.
-        String kafkaAdvertiseAddr;
-        int kafkaHostPort = 0;
-        if (!containerDetector.isRunningInContainer()) {
-            kafkaHostPort = portAllocator.allocate(
+        int kafkaHostPort = portAllocator.allocate(
                 config.services().msk().kafkaHostPortBase(),
                 config.services().msk().kafkaHostPortMax());
-            kafkaAdvertiseAddr = "localhost:" + kafkaHostPort;
-        } else {
-            // Sibling containers on the docker network resolve the broker via its
-            // container name through Docker's embedded DNS.
-            kafkaAdvertiseAddr = containerName + ":" + KAFKA_PORT;
-        }
+        String externalKafkaAdvertiseAddr = "external://localhost:" + kafkaHostPort;
+        String internalKafkaAdvertiseAddr = "internal://" + containerName + ":" + INTERNAL_KAFKA_PORT;
 
         // Build command
         List<String> cmd = new ArrayList<>(List.of(
                 "redpanda", "start", "--overprovisioned", "--smp", "1",
                 "--memory", "512M", "--reserve-memory", "0M",
-                "--advertise-kafka-addr", kafkaAdvertiseAddr));
+                "--kafka-addr", "internal://0.0.0.0:" + INTERNAL_KAFKA_PORT + ",external://0.0.0.0:" + KAFKA_PORT,
+                "--advertise-kafka-addr", internalKafkaAdvertiseAddr + "," + externalKafkaAdvertiseAddr));
 
-        // Build container spec. Publish Kafka/admin ports to the host only in
-        // native mode; in Docker mode producers/consumers reach the broker via
-        // the docker network IP resolved from container inspect.
+        // MSK clients run outside Floci's Docker network as often as inside it. Publish and
+        // advertise the Kafka listener in both topologies so GetBootstrapBrokers is usable by
+        // real host clients, not merely sibling containers.
         ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image)
                 .withName(containerName)
                 .withDockerNetwork(config.services().dockerNetwork())
@@ -155,11 +149,7 @@ public class RedpandaManager {
                         AwsArnUtils.accountOrDefault(cluster.getClusterArn(), regionResolver.getAccountId()),
                         AwsArnUtils.regionOrDefault(cluster.getClusterArn(), regionResolver.getDefaultRegion())));
 
-        if (!containerDetector.isRunningInContainer()) {
         specBuilder.withPortBinding(KAFKA_PORT, kafkaHostPort).withDynamicPort(ADMIN_PORT);
-        } else {
-            specBuilder.withExposedPort(KAFKA_PORT).withExposedPort(ADMIN_PORT);
-        }
 
         // Handle persistence mounting
         if (ContainerStorageHelper.isNamedVolumeMode(config)) {
@@ -183,17 +173,14 @@ public class RedpandaManager {
         try {
             info = lifecycleManager.createAndStart(spec);
         } catch (RuntimeException e) {
-            if (!containerDetector.isRunningInContainer()) {
             portAllocator.release(kafkaHostPort);
-            }
             throw e;
         }
         cluster.setContainerId(info.containerId());
 
         // Resolve endpoints
-        EndpointInfo kafkaEndpoint = info.getEndpoint(KAFKA_PORT);
-
-        cluster.setBootstrapBrokers(kafkaEndpoint.host() + ":" + kafkaEndpoint.port());
+        cluster.setBootstrapBrokers("localhost:" + kafkaHostPort);
+        cluster.setInternalBootstrapBrokers(containerName + ":" + INTERNAL_KAFKA_PORT);
         LOG.infov("Redpanda container {0} started. Bootstrap: {1}", info.containerId(), cluster.getBootstrapBrokers());
 
         // Attach log streaming (new feature)
@@ -231,7 +218,13 @@ public class RedpandaManager {
                 return false;
             }
         } else {
-            String containerIp = bootstrap.split(":")[0];
+            String containerIp = lifecycleManager.getDockerClient().inspectContainerCmd(cluster.getContainerId())
+                    .exec().getNetworkSettings().getNetworks().values().stream()
+                    .map(network -> network.getIpAddress()).filter(ip -> ip != null && !ip.isBlank())
+                    .findFirst().orElse(null);
+            if (containerIp == null) {
+                return false;
+            }
             adminUrl = "http://" + containerIp + ":" + ADMIN_PORT + ADMIN_READY_PATH;
         }
 

src/main/java/io/github/hectorvent/floci/services/msk/model/MskCluster.java

@@ -89,6 +89,10 @@ public class MskCluster {
     @JsonProperty("bootstrapBrokers")
     private String bootstrapBrokers;
 
+    // Broker address for Floci services sharing the Docker network.
+    @JsonProperty("internalBootstrapBrokers")
+    private String internalBootstrapBrokers;
+
     // Docker container ID for mock=false
     @JsonProperty("containerId")
     private String containerId;
@@ -178,6 +182,11 @@ public class MskCluster {
     public String getBootstrapBrokers() { return bootstrapBrokers; }
     public void setBootstrapBrokers(String bootstrapBrokers) { this.bootstrapBrokers = bootstrapBrokers; }
 
+    public String getInternalBootstrapBrokers() { return internalBootstrapBrokers; }
+    public void setInternalBootstrapBrokers(String internalBootstrapBrokers) {
+        this.internalBootstrapBrokers = internalBootstrapBrokers;
+    }
+
     public String getContainerId() { return containerId; }
     public void setContainerId(String containerId) { this.containerId = containerId; }
 

src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java

@@ -764,6 +764,9 @@ public class RdsService implements Resettable, ResourceProvider {
                 } catch (RuntimeException rollbackFailure) {
                     e.addSuppressed(rollbackFailure);
                 }
+                if (!persistedResourceUsesProxyPort(proxyPort)) {
+                    releaseProxyPort(proxyPort);
+                }
                 throw e;
             }
         }
@@ -6030,7 +6033,7 @@ public class RdsService implements Resettable, ResourceProvider {
         int base = config.services().rds().proxyBasePort();
         int max = config.services().rds().proxyMaxPort();
         for (int port = base; port <= max; port++) {
-            if (usedPorts.add(port)) {
+            if (!persistedResourceUsesProxyPort(port) && usedPorts.add(port)) {
                 return port;
             }
         }
@@ -6038,6 +6041,12 @@ public class RdsService implements Resettable, ResourceProvider {
                 "No available proxy ports in range " + base + "-" + max, 503);
     }
 
+    private boolean persistedResourceUsesProxyPort(int port) {
+        return allInstances().stream().anyMatch(instance -> instance.getProxyPort() == port)
+                || allClusters().stream().anyMatch(cluster -> cluster.getProxyPort() == port)
+                || allProxies().stream().anyMatch(proxy -> proxy.getProxyPort() == port);
+    }
+
     private void releaseProxyPort(int port) {
         usedPorts.remove(port);
     }

src/main/java/io/github/hectorvent/floci/tools/ami/AmiImageTool.java

@@ -134,8 +134,11 @@ public final class AmiImageTool {
             staged.creationDate = image.aws.creationDate;
             staged.rootDeviceType = image.aws.rootDeviceType;
             staged.virtualizationType = image.aws.virtualizationType;
-            staged.guestRuntime = image.guest.runtime;
+            staged.initType = image.guest.runtime;
             staged.cloudInit = image.guest.cloudInit;
+            staged.sshUser = "root";
+            staged.sshPreinstalled = true;
+            staged.sshServiceName = "ssh";
             staged.provenance = contextDir(outputRoot, image).resolve("provenance.yaml").toString();
             if (write) {
                 YAML.writerWithDefaultPrettyPrinter().writeValue(catalogPath.toFile(), catalog);
@@ -442,8 +445,11 @@ public final class AmiImageTool {
         public String rootDeviceType;
         public String virtualizationType;
         public String creationDate;
-        public String guestRuntime;
+        public String initType;
         public Boolean cloudInit;
+        public String sshUser;
+        public Boolean sshPreinstalled;
+        public String sshServiceName;
         public String provenance;
     }
 

src/main/resources/application.yml

@@ -155,6 +155,8 @@ floci:
     services:
       ssm:
         flush-interval-ms: 5000
+      kafkaconnect:
+        flush-interval-ms: 5000
       dynamodb:
         flush-interval-ms: 5000
       sns:
@@ -357,6 +359,10 @@ floci:
       default-image: "redpandadata/redpanda:latest"
       kafka-host-port-base: 9300
       kafka-host-port-max: 9399
+    kafkaconnect:
+      enabled: true
+      mock: false
+      worker-image: apache/kafka:3.9.0
     amazonmq:
       enabled: true
       mock: false

src/main/resources/ec2/image-catalog.yaml

@@ -9,6 +9,10 @@ images:
       - /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs
       - /aws/service/ami-amazon-linux-latest/amzn2-ami-kernel-5.10-hvm-x86_64-gp2
     dockerImage: public.ecr.aws/amazonlinux/amazonlinux:2
+    sshUser: root
+    sshPreinstalled: false
+    initType: minimal
+    sshServiceName: sshd
     name: amzn2-ami-hvm-2.0.20230404.0-x86_64-gp2
     description: Amazon Linux 2 AMI
     ownerId: "137112412989"
@@ -24,27 +28,40 @@ images:
       - /aws/service/ami-amazon-linux-latest/al2023-ami-minimal-kernel-default-x86_64
       - /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-x86_64
     dockerImage: floci/ami-amazonlinux:2023
+    fallbackDockerImage: public.ecr.aws/amazonlinux/amazonlinux:2023
     name: al2023-ami-2023.0.20230315.0-kernel-6.1-x86_64
     description: Amazon Linux 2023 AMI
     ownerId: "137112412989"
     imageOwnerAlias: amazon
     architecture: x86_64
     creationDate: "2023-03-15T00:00:00.000Z"
-    guestRuntime: systemd
+    sshUser: root
+    sshPreinstalled: true
+    initType: systemd
+    sshServiceName: sshd
 
   - imageId: ami-0abcdef1234567892
     aliases:
       - ami-ubuntu2004
-    dockerImage: public.ecr.aws/docker/library/ubuntu:20.04
+    dockerImage: floci/ami-ubuntu:20.04
+    fallbackDockerImage: public.ecr.aws/docker/library/ubuntu:20.04
     name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230324
     description: Canonical, Ubuntu, 20.04 LTS
     ownerId: "099720109477"
     imageOwnerAlias: canonical
     architecture: x86_64
     creationDate: "2023-03-24T00:00:00.000Z"
+    sshUser: root
+    sshPreinstalled: true
+    initType: systemd
+    sshServiceName: ssh
 
   - imageId: ami-ubuntu2204
     dockerImage: public.ecr.aws/docker/library/ubuntu:22.04
+    sshUser: root
+    sshPreinstalled: false
+    initType: minimal
+    sshServiceName: ssh
     name: ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20230420
     description: Canonical, Ubuntu, 22.04 LTS
     ownerId: "099720109477"
@@ -56,6 +73,10 @@ images:
     aliases:
       - ami-ubuntu2404
     dockerImage: public.ecr.aws/docker/library/ubuntu:24.04
+    sshUser: root
+    sshPreinstalled: false
+    initType: minimal
+    sshServiceName: ssh
     name: ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-server-20260515
     description: Canonical, Ubuntu, 24.04 LTS
     ownerId: "099720109477"
@@ -65,6 +86,10 @@ images:
 
   - imageId: ami-ubuntu2404-amd64
     dockerImage: public.ecr.aws/docker/library/ubuntu:24.04
+    sshUser: root
+    sshPreinstalled: false
+    initType: minimal
+    sshServiceName: ssh
     name: ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-20260515
     description: Canonical, Ubuntu, 24.04 LTS
     ownerId: "099720109477"
@@ -76,6 +101,7 @@ images:
     aliases:
       - ami-ubuntu2404-cloud
     dockerImage: floci/ami-ubuntu:24.04-arm64
+    fallbackDockerImage: public.ecr.aws/docker/library/ubuntu:24.04
     name: ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-server-20260515
     description: Canonical, Ubuntu, 24.04 LTS cloud image
     ownerId: "099720109477"
@@ -84,12 +110,19 @@ images:
     rootDeviceType: ebs
     virtualizationType: hvm
     creationDate: "2026-05-15T00:00:00.000Z"
-    guestRuntime: systemd
+    sshUser: root
+    sshPreinstalled: true
+    initType: systemd
+    sshServiceName: ssh
     cloudInit: true
     provenance: target/ami-images/ubuntu-24.04-arm64/provenance.yaml
 
   - imageId: ami-debian12
     dockerImage: public.ecr.aws/docker/library/debian:12
+    sshUser: root
+    sshPreinstalled: false
+    initType: minimal
+    sshServiceName: ssh
     name: debian-12-amd64-20230411
     description: Debian 12 AMI
     architecture: x86_64
@@ -97,20 +130,34 @@ images:
 
   - imageId: ami-alpine
     dockerImage: public.ecr.aws/docker/library/alpine:latest
+    sshUser: root
+    sshPreinstalled: false
+    initType: minimal
+    sshServiceName: sshd
     name: alpine-linux-latest-x86_64
     description: Alpine Linux AMI
     architecture: x86_64
     creationDate: "2023-04-01T00:00:00.000Z"
 
   - imageId: ami-fedora42
-    dockerImage: registry.fedoraproject.org/fedora:42
+    dockerImage: floci/ami-fedora:42
+    fallbackDockerImage: registry.fedoraproject.org/fedora:42
     name: Fedora-Cloud-Base-42-x86_64
     description: Fedora Linux 42 AMI
     architecture: x86_64
     creationDate: "2025-04-15T00:00:00.000Z"
+    sshUser: root
+    sshPreinstalled: true
+    initType: systemd
+    sshServiceName: sshd
 
   - imageId: ami-opensuse-leap156
-    dockerImage: registry.opensuse.org/opensuse/leap:15.6
+    dockerImage: floci/ami-opensuse:15.6
+    fallbackDockerImage: registry.opensuse.org/opensuse/leap:15.6
+    sshUser: root
+    sshPreinstalled: true
+    initType: systemd
+    sshServiceName: sshd
     name: openSUSE-Leap-15.6-x86_64
     description: openSUSE Leap 15.6 AMI
     architecture: x86_64

src/test/java/io/github/hectorvent/floci/core/common/docker/CurrentContainerNetworkResolverTest.java

@@ -6,6 +6,7 @@ import com.github.dockerjava.api.command.InspectContainerResponse;
 import com.github.dockerjava.api.model.ContainerNetwork;
 import com.github.dockerjava.api.model.ExposedPort;
 import com.github.dockerjava.api.model.Ports;
+import io.github.hectorvent.floci.config.EmulatorConfig;
 import org.junit.jupiter.api.Test;
 
 import java.util.LinkedHashMap;
@@ -43,6 +44,55 @@ class CurrentContainerNetworkResolverTest {
         assertEquals(Optional.of("172.24.0.2"), resolver.resolveContainerIp());
     }
 
+    @Test
+    void resolveNetworkName_prefersConfiguredDockerNetworkOverOtherAttachedNetworks() {
+        DockerClient dockerClient = mock(DockerClient.class);
+        ContainerDetector containerDetector = mock(ContainerDetector.class);
+        when(containerDetector.isRunningInContainer()).thenReturn(true);
+
+        InspectContainerCmd inspectCmd = mock(InspectContainerCmd.class);
+        InspectContainerResponse inspect = mock(InspectContainerResponse.class, RETURNS_DEEP_STUBS);
+        Map<String, ContainerNetwork> networks = networks(
+                "floci-vpc-4566-us-east-1-vpc-default-us-east-1", "172.31.0.3",
+                "floci_default", "172.22.0.4");
+        when(dockerClient.inspectContainerCmd("floci-container")).thenReturn(inspectCmd);
+        when(inspectCmd.exec()).thenReturn(inspect);
+        when(inspect.getNetworkSettings().getNetworks()).thenReturn(networks);
+
+        EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+        when(config.services().dockerNetwork()).thenReturn(Optional.of("floci_default"));
+
+        CurrentContainerNetworkResolver resolver =
+                new TestResolver(dockerClient, containerDetector, config, "floci-container");
+
+        assertEquals(Optional.of("floci_default"), resolver.resolveNetworkName());
+        assertEquals(Optional.of("172.22.0.4"), resolver.resolveContainerIp());
+    }
+
+    @Test
+    void resolveNetworkName_fallsBackWhenConfiguredNetworkIsNotAttached() {
+        DockerClient dockerClient = mock(DockerClient.class);
+        ContainerDetector containerDetector = mock(ContainerDetector.class);
+        when(containerDetector.isRunningInContainer()).thenReturn(true);
+
+        InspectContainerCmd inspectCmd = mock(InspectContainerCmd.class);
+        InspectContainerResponse inspect = mock(InspectContainerResponse.class, RETURNS_DEEP_STUBS);
+        Map<String, ContainerNetwork> networks = networks(
+                "bridge", "172.17.0.2",
+                "avoxx-network", "172.24.0.2");
+        when(dockerClient.inspectContainerCmd("floci-container")).thenReturn(inspectCmd);
+        when(inspectCmd.exec()).thenReturn(inspect);
+        when(inspect.getNetworkSettings().getNetworks()).thenReturn(networks);
+
+        EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+        when(config.services().dockerNetwork()).thenReturn(Optional.of("floci_default"));
+
+        CurrentContainerNetworkResolver resolver =
+                new TestResolver(dockerClient, containerDetector, config, "floci-container");
+
+        assertEquals(Optional.of("avoxx-network"), resolver.resolveNetworkName());
+    }
+
     @Test
     void resolveNetworkName_returnsEmptyOutsideContainer() {
         DockerClient dockerClient = mock(DockerClient.class);
@@ -126,7 +176,12 @@ class CurrentContainerNetworkResolverTest {
         private final String containerId;
 
         TestResolver(DockerClient dockerClient, ContainerDetector containerDetector, String containerId) {
-            super(dockerClient, containerDetector);
+            this(dockerClient, containerDetector, null, containerId);
+        }
+
+        TestResolver(DockerClient dockerClient, ContainerDetector containerDetector, EmulatorConfig config,
+                     String containerId) {
+            super(dockerClient, containerDetector, config);
             this.containerId = containerId;
         }
 

src/test/java/io/github/hectorvent/floci/core/common/docker/PortAllocatorTest.java

@@ -34,6 +34,20 @@ class PortAllocatorTest {
         allocator.release(second);
     }
 
+    @Test
+    void reportsOccupiedPortsSkippedDuringAllocation() {
+        PortAllocator allocator = new PortAllocator();
+        int first = allocator.allocate(19900, 19900);
+        List<Integer> occupied = new ArrayList<>();
+
+        int second = allocator.allocate(19900, 19901, occupied::add);
+
+        assertEquals(19901, second);
+        assertEquals(List.of(19900), occupied);
+        allocator.release(first);
+        allocator.release(second);
+    }
+
     @Test
     void releasedPortBecomesAvailableAgain() {
         PortAllocator allocator = new PortAllocator();

src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerTest.java

@@ -1,6 +1,8 @@
 package io.github.hectorvent.floci.lifecycle;
 
 import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.services.lightsail.LightsailService;
 import io.github.hectorvent.floci.config.FlociCertificateAuthority;
 import io.github.hectorvent.floci.core.common.ContainerTeardown;
 import io.github.hectorvent.floci.core.common.Resettable;
@@ -36,6 +38,8 @@ class EmulatorInfoControllerTest {
     @Mock private Instance<ContainerTeardown> containerTeardowns;
     @Mock private FlociCertificateAuthority certificateAuthority;
     @Mock private EmulatorConfig config;
+    @Mock private Ec2Service ec2Service;
+    @Mock private LightsailService lightsailService;
     @Mock private ContainerTeardown sageMakerTeardown;
     @Mock private ContainerTeardown batchTeardown;
     @Mock private Resettable resettable;
@@ -45,7 +49,7 @@ class EmulatorInfoControllerTest {
     @BeforeEach
     void setUp() {
         controller = new EmulatorInfoController(serviceRegistry, initLifecycleState, storageFactory,
-                resettables, containerTeardowns, certificateAuthority, config);
+                resettables, containerTeardowns, certificateAuthority, config, ec2Service, lightsailService);
     }
 
     @Test

src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationLambdaAddressingIntegrationTest.java

@@ -3,8 +3,16 @@ package io.github.hectorvent.floci.services.cloudformation;
 import io.quarkus.test.junit.QuarkusTest;
 import org.junit.jupiter.api.Test;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
 import static io.restassured.RestAssured.given;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 
@@ -259,6 +267,92 @@ class CloudFormationLambdaAddressingIntegrationTest {
         awaitStackGone(stackName);
     }
 
+    @Test
+    void updatePublishesVersionAfterDependentFunctionS3CodeChanges() throws IOException {
+        String suffix = Long.toString(System.nanoTime(), 36);
+        String bucket = "cfn-version-code-" + suffix;
+        String stackName = "cfn-version-code-" + suffix;
+        String fnName = "version-code-fn-" + suffix;
+
+        given().when().put("/" + bucket).then().statusCode(200);
+        given().contentType("application/zip").body(handlerZip("v1"))
+                .when().put("/" + bucket + "/v1.zip").then().statusCode(200);
+
+        createStack(stackName, versionCodeTemplate(fnName, bucket, "v1.zip"));
+
+        String versionOneSha = given()
+                .queryParam("Qualifier", "1")
+                .when().get("/2015-03-31/functions/" + fnName + "/configuration")
+                .then().statusCode(200)
+                .extract().path("CodeSha256");
+
+        given().contentType("application/zip").body(handlerZip("v2"))
+                .when().put("/" + bucket + "/v2.zip").then().statusCode(200);
+
+        updateStack(stackName, versionCodeTemplate(fnName, bucket, "v2.zip"));
+
+        String latestSha = given()
+                .when().get("/2015-03-31/functions/" + fnName + "/configuration")
+                .then().statusCode(200)
+                .extract().path("CodeSha256");
+
+        given()
+                .queryParam("Qualifier", "2")
+                .when().get("/2015-03-31/functions/" + fnName + "/configuration")
+                .then().statusCode(200)
+                .body("CodeSha256", equalTo(latestSha))
+                .body("CodeSha256", not(equalTo(versionOneSha)));
+    }
+
+    private static void createStack(String stackName, String template) {
+        given()
+                .contentType("application/x-www-form-urlencoded")
+                .header("Authorization", CFN_AUTH)
+                .formParam("Action", "CreateStack")
+                .formParam("StackName", stackName)
+                .formParam("TemplateBody", template)
+        .when()
+                .post("/")
+        .then()
+                .statusCode(200);
+    }
+
+    private static void updateStack(String stackName, String template) {
+        given()
+                .contentType("application/x-www-form-urlencoded")
+                .header("Authorization", CFN_AUTH)
+                .formParam("Action", "UpdateStack")
+                .formParam("StackName", stackName)
+                .formParam("TemplateBody", template)
+        .when()
+                .post("/")
+        .then()
+                .statusCode(200);
+    }
+
+    private static String versionCodeTemplate(String fnName, String bucket, String key) {
+        return """
+                {"Resources": {
+                  "Ver": {"Type": "AWS::Lambda::Version", "DependsOn": "Fn",
+                    "Properties": {"FunctionName": {"Ref": "Fn"}}},
+                  "Fn": {"Type": "AWS::Lambda::Function", "Properties": {
+                    "FunctionName": "%s", "Runtime": "nodejs20.x", "Handler": "index.handler",
+                    "Role": "arn:aws:iam::000000000000:role/r",
+                    "Code": {"S3Bucket": "%s", "S3Key": "%s"}}}
+                }}
+                """.formatted(fnName, bucket, key);
+    }
+
+    private static byte[] handlerZip(String response) throws IOException {
+        ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+        try (ZipOutputStream zip = new ZipOutputStream(bytes)) {
+            zip.putNextEntry(new ZipEntry("index.js"));
+            zip.write(("exports.handler = async () => '" + response + "';").getBytes(StandardCharsets.UTF_8));
+            zip.closeEntry();
+        }
+        return bytes.toByteArray();
+    }
+
     private static void awaitStackGone(String stackName) throws InterruptedException {
         for (int i = 0; i < 200; i++) {
             var response = given()

src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LambdaVersionAliasCfnProvisionerTest.java

@@ -90,6 +90,21 @@ class LambdaVersionAliasCfnProvisionerTest {
                 r.getAttributes().get("FunctionArn"));
     }
 
+    @Test
+    void updatePublishesThroughLambdaInsteadOfRetainingThePriorVersion() {
+        when(lambda.publishVersion(eq("us-east-1"), eq("my-fn"), isNull())).thenReturn(publishedVersion());
+
+        ObjectNode props = mapper.createObjectNode();
+        props.put("FunctionName", "my-fn");
+        StackResource r = resource("AWS::Lambda::Version", "MyFuncVersion");
+        r.setPhysicalId("arn:aws:lambda:us-east-1:000000000000:function:my-fn:2");
+
+        provisioner.provision(r, props, ctx());
+
+        verify(lambda).publishVersion("us-east-1", "my-fn", null);
+        assertEquals("arn:aws:lambda:us-east-1:000000000000:function:my-fn:3", r.getPhysicalId());
+    }
+
     @Test
     void versionWithoutFunctionNameFails() {
         StackResource r = resource("AWS::Lambda::Version", "MyFuncVersion");

src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsMetricFilterResetTest.java

@@ -115,6 +115,6 @@ class CloudWatchLogsMetricFilterResetTest {
         when(resettables.iterator()).thenAnswer(call -> List.of(resettable).iterator());
         Instance<ContainerTeardown> teardowns = mock(Instance.class);
         when(teardowns.iterator()).thenAnswer(call -> List.<ContainerTeardown>of().iterator());
-        return new EmulatorInfoController(null, null, f.factory, resettables, teardowns, null, null);
+        return new EmulatorInfoController(null, null, f.factory, resettables, teardowns, null, null, null, null);
     }
 }

src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildDockerIntegrationTest.java

@@ -0,0 +1,104 @@
+package io.github.hectorvent.floci.services.codebuild;
+
+import com.github.dockerjava.api.DockerClient;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@QuarkusTest
+class CodeBuildDockerIntegrationTest {
+
+    private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+
+    @Inject
+    DockerClient dockerClient;
+
+    @BeforeAll
+    static void configureRestAssured() {
+        RestAssuredJsonUtils.configureAwsContentTypes();
+    }
+
+    @BeforeEach
+    void requireDocker() {
+        try {
+            dockerClient.pingCmd().exec();
+        } catch (Exception e) {
+            Assumptions.assumeTrue(false,
+                    "Docker daemon must be available for CodeBuild Docker integration tests");
+        }
+    }
+
+    @Test
+    void privilegedBuildCanUseMountedDockerSocket() throws InterruptedException {
+        String projectName = "codebuild-docker-" + UUID.randomUUID().toString().substring(0, 8);
+        given()
+                .header("X-Amz-Target", "CodeBuild_20161006.CreateProject")
+                .contentType(CONTENT_TYPE)
+                .body("""
+                        {
+                          "name": "%s",
+                          "source": {"type": "NO_SOURCE"},
+                          "artifacts": {"type": "NO_ARTIFACTS"},
+                          "environment": {
+                            "type": "LINUX_CONTAINER",
+                            "image": "docker:27-cli",
+                            "computeType": "BUILD_GENERAL1_SMALL",
+                            "privilegedMode": true
+                          },
+                          "serviceRole": "arn:aws:iam::000000000000:role/codebuild-role"
+                        }
+                        """.formatted(projectName))
+                .when()
+                .post("/")
+                .then()
+                .statusCode(200);
+
+        String buildId = given()
+                .header("X-Amz-Target", "CodeBuild_20161006.StartBuild")
+                .contentType(CONTENT_TYPE)
+                .body("""
+                        {
+                          "projectName": "%s",
+                          "buildspecOverride": "version: 0.2\\nphases:\\n  build:\\n    commands:\\n      - docker version\\n      - printf 'FROM scratch\\\\n' > Dockerfile\\n      - docker build -t floci-codebuild-socket-test .\\n      - docker image rm floci-codebuild-socket-test\\n"
+                        }
+                        """.formatted(projectName))
+                .when()
+                .post("/")
+                .then()
+                .statusCode(200)
+                .extract()
+                .path("build.id");
+
+        String status = waitForBuild(buildId);
+        assertEquals("SUCCEEDED", status);
+    }
+
+    private String waitForBuild(String buildId) throws InterruptedException {
+        for (int attempts = 0; attempts < 120; attempts++) {
+            String status = given()
+                    .header("X-Amz-Target", "CodeBuild_20161006.BatchGetBuilds")
+                    .contentType(CONTENT_TYPE)
+                    .body("{\"ids\":[\"" + buildId + "\"]}")
+                    .when()
+                    .post("/")
+                    .then()
+                    .statusCode(200)
+                    .extract()
+                    .path("builds[0].buildStatus");
+            if (!"IN_PROGRESS".equals(status)) {
+                return status;
+            }
+            Thread.sleep(500);
+        }
+        return "TIMED_OUT";
+    }
+}

src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunnerTest.java

@@ -0,0 +1,52 @@
+package io.github.hectorvent.floci.services.codebuild;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+class CodeBuildRunnerTest {
+
+    @Test
+    void privilegedBuildMountsConfiguredDockerSocket() {
+        ContainerBuilder.Builder container = mock(ContainerBuilder.Builder.class);
+
+        CodeBuildRunner.exposeDockerSocketIfPrivileged(container, true,
+                dockerConfig("unix:///run/user/1000/docker.sock"));
+
+        verify(container).withBind("/run/user/1000/docker.sock", "/var/run/docker.sock");
+    }
+
+    @Test
+    void unprivilegedBuildCannotAccessDockerSocket() {
+        ContainerBuilder.Builder container = mock(ContainerBuilder.Builder.class);
+
+        CodeBuildRunner.exposeDockerSocketIfPrivileged(container, false,
+                dockerConfig("unix:///run/user/1000/docker.sock"));
+
+        verifyNoInteractions(container);
+    }
+
+    @Test
+    void privilegedBuildDoesNotInventSocketForTcpDockerHost() {
+        ContainerBuilder.Builder container = mock(ContainerBuilder.Builder.class);
+
+        CodeBuildRunner.exposeDockerSocketIfPrivileged(container, true,
+                dockerConfig("tcp://docker.example.test:2376"));
+
+        verifyNoInteractions(container);
+    }
+
+    private EmulatorConfig.DockerConfig dockerConfig(String dockerHost) {
+        EmulatorConfig.DockerConfig dockerConfig = mock(EmulatorConfig.DockerConfig.class);
+        when(dockerConfig.dockerHost()).thenReturn(dockerHost);
+        when(dockerConfig.dockerConfigPath()).thenReturn(Optional.empty());
+        return dockerConfig;
+    }
+}

src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployEcsIntegrationTest.java

@@ -281,7 +281,7 @@ class CodeDeployEcsIntegrationTest {
 
     @Test
     @Order(11)
-    void deploymentCompletesSuccessfully() throws InterruptedException {
+    void deploymentFailsWithoutRealGreenTargets() throws InterruptedException {
         Assumptions.assumeTrue(deploymentId != null, "deploymentId must be set");
 
         String status = null;
@@ -302,7 +302,7 @@ class CodeDeployEcsIntegrationTest {
             TimeUnit.MILLISECONDS.sleep(500);
         }
 
-        Assertions.assertEquals("Succeeded", status, "ECS deployment should succeed");
+        Assertions.assertEquals("Failed", status, "ECS deployment without green targets must fail");
     }
 
     @Test
@@ -350,14 +350,14 @@ class CodeDeployEcsIntegrationTest {
         .then()
             .statusCode(200)
             .body("deploymentTargets[0].deploymentTargetType", equalTo("ECSTarget"))
-            .body("deploymentTargets[0].ecsTarget.status", equalTo("Succeeded"));
+            .body("deploymentTargets[0].ecsTarget.status", equalTo("Failed"));
     }
 
     @Test
     @Order(14)
-    void listenerNowPointsToGreen() {
+    void listenerRemainsOnBlueAfterFailedDeployment() {
         Assumptions.assumeTrue(listenerArn != null, "listenerArn must be set");
-        Assumptions.assumeTrue(greenTgArn != null, "greenTgArn must be set");
+        Assumptions.assumeTrue(blueTgArn != null, "blueTgArn must be set");
 
         given()
             .contentType(ELBV2_CONTENT_TYPE)
@@ -369,6 +369,6 @@ class CodeDeployEcsIntegrationTest {
         .then()
             .statusCode(200)
             .body("DescribeRulesResponse.DescribeRulesResult.Rules.member.Actions.member.TargetGroupArn",
-                    equalTo(greenTgArn));
+                    equalTo(blueTgArn));
     }
 }

src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServiceTest.java

@@ -5,32 +5,49 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import io.github.hectorvent.floci.core.common.RegionResolver;
 import io.github.hectorvent.floci.services.codedeploy.model.Deployment;
 import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.services.ec2.model.Instance;
+import io.github.hectorvent.floci.services.ec2.model.Reservation;
 import io.github.hectorvent.floci.services.ecs.EcsService;
+import io.github.hectorvent.floci.services.ecs.model.EcsLoadBalancer;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import io.github.hectorvent.floci.services.ecs.model.TaskSet;
 import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
+import io.github.hectorvent.floci.services.elbv2.model.Action;
+import io.github.hectorvent.floci.services.elbv2.model.Listener;
+import io.github.hectorvent.floci.services.elbv2.model.Rule;
+import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;
 import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.model.LambdaAlias;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
 import io.github.hectorvent.floci.services.lambda.model.InvocationType;
 import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
 import io.github.hectorvent.floci.services.ssm.SsmCommandService;
 import io.github.hectorvent.floci.testing.MutableClock;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
 
 import java.time.Clock;
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.awaitility.Awaitility.await;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -48,6 +65,14 @@ class CodeDeployServiceTest {
     @BeforeEach
     void setUp() {
         lambdaService = mock(LambdaService.class);
+        LambdaAlias alias = new LambdaAlias();
+        alias.setFunctionVersion("1");
+        when(lambdaService.getAlias(eq(REGION), anyString(), anyString())).thenReturn(alias);
+        LambdaFunction versionOne = new LambdaFunction();
+        versionOne.setVersion("1");
+        LambdaFunction versionTwo = new LambdaFunction();
+        versionTwo.setVersion("2");
+        when(lambdaService.listVersionsByFunction(eq(REGION), anyString())).thenReturn(List.of(versionOne, versionTwo));
         ssmCommandService = mock(SsmCommandService.class);
         service = newService(Duration.ofHours(1));
     }
@@ -57,9 +82,13 @@ class CodeDeployServiceTest {
     }
 
     private CodeDeployService newService(Duration hookCallbackTimeout, Clock clock) {
+        return newService(hookCallbackTimeout, clock, Thread::sleep);
+    }
+
+    private CodeDeployService newService(Duration hookCallbackTimeout, Clock clock, CodeDeployService.Sleeper sleeper) {
         CodeDeployService svc = new CodeDeployService(lambdaService, mock(EcsService.class),
                 mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
-                new RegionResolver(REGION, ACCOUNT_ID), null, hookCallbackTimeout, clock);
+                new RegionResolver(REGION, ACCOUNT_ID), null, hookCallbackTimeout, clock, null, sleeper);
         svc.initializeStorage();
         return svc;
     }
@@ -155,6 +184,43 @@ class CodeDeployServiceTest {
         assertEquals("Succeeded", deployment.getStatus());
     }
 
+    @Test
+    void ec2TagFilterTargetsRegisteredInstanceAndRunsHook() {
+        Ec2Service ec2Service = mock(Ec2Service.class);
+        CodeDeployService ec2TargetService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                mock(ElbV2Service.class), ssmCommandService, ec2Service, new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        ec2TargetService.initializeStorage();
+
+        Instance instance = new Instance();
+        instance.setInstanceId("i-tagged");
+        Reservation reservation = new Reservation();
+        reservation.setInstances(List.of(instance));
+        when(ec2Service.describeInstances(eq(REGION), eq(List.of()), any())).thenReturn(List.of(reservation));
+        when(ssmCommandService.isInstanceRegistered("i-tagged", REGION)).thenReturn(true);
+        when(ssmCommandService.sendCommandToInstance(eq("i-tagged"), eq("AWS-RunShellScript"), any(), eq(30), eq(REGION)))
+                .thenReturn("cmd-tagged");
+        when(ssmCommandService.getCommandInvocationStatus("cmd-tagged", "i-tagged", REGION)).thenReturn("Success");
+
+        ec2TargetService.createApplication(REGION, "ec2-tag-app", "Server", null);
+        ec2TargetService.createDeploymentGroup(REGION, "ec2-tag-app", "ec2-tag-group",
+                "CodeDeployDefault.AllAtOnce", ROLE_ARN,
+                Map.of("ec2TagFilters", List.of(Map.of("Key", "Environment", "Value", "test", "Type", "KEY_AND_VALUE"))));
+
+        String deploymentId = ec2TargetService.createDeployment(REGION, "ec2-tag-app", "ec2-tag-group", null,
+                serverRevision(30), "EC2 tag target");
+        Deployment deployment = awaitTerminal(ec2TargetService, deploymentId, Duration.ofSeconds(5));
+
+        assertEquals("Succeeded", deployment.getStatus());
+        assertEquals("i-tagged", ec2TargetService.listDeploymentTargets(REGION, deploymentId).getFirst());
+        ArgumentCaptor<Map<String, List<String>>> filters = ArgumentCaptor.forClass(Map.class);
+        verify(ec2Service).describeInstances(eq(REGION), eq(List.of()), filters.capture());
+        assertEquals(Map.of("tag:Environment", List.of("test")), filters.getValue());
+        verify(ssmCommandService).sendCommandToInstance(eq("i-tagged"), eq("AWS-RunShellScript"),
+                eq(Map.of("commands", List.of("sh scripts/start_server.sh"))), eq(30), eq(REGION));
+    }
+
     // ---- Lambda platform hooks ----------------------------------------------------------
 
     @Test
@@ -205,7 +271,7 @@ class CodeDeployServiceTest {
         when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
                 .thenAnswer(invocation -> {
                     byte[] payload = invocation.getArgument(2);
-                    service.putLifecycleEventHookExecutionStatus("ignored", executionIdFrom(payload), "Succeeded");
+                    service.putLifecycleEventHookExecutionStatus(deploymentIdFrom(payload), executionIdFrom(payload), "Succeeded");
                     return cleanInvoke;
                 });
 
@@ -270,7 +336,7 @@ class CodeDeployServiceTest {
         when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
                 .thenAnswer(invocation -> {
                     byte[] payload = invocation.getArgument(2);
-                    service.putLifecycleEventHookExecutionStatus("ignored", executionIdFrom(payload), "Failed");
+                    service.putLifecycleEventHookExecutionStatus(deploymentIdFrom(payload), executionIdFrom(payload), "Failed");
                     return cleanInvoke;
                 });
 
@@ -304,7 +370,7 @@ class CodeDeployServiceTest {
                     byte[] payload = invocation.getArgument(2);
                     String executionId = executionIdFrom(payload);
                     capturedExecutionId.set(executionId);
-                    service.putLifecycleEventHookExecutionStatus("ignored", executionId, "Succeeded");
+                    service.putLifecycleEventHookExecutionStatus(deploymentIdFrom(payload), executionId, "Succeeded");
                     return cleanInvoke;
                 });
 
@@ -312,11 +378,372 @@ class CodeDeployServiceTest {
         Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));
         assertEquals("Succeeded", deployment.getStatus());
 
-        service.putLifecycleEventHookExecutionStatus("ignored", capturedExecutionId.get(), "Failed");
+        org.junit.jupiter.api.Assertions.assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,
+                () -> service.putLifecycleEventHookExecutionStatus(deploymentId, capturedExecutionId.get(), "Failed"));
 
         assertEquals("Succeeded", service.getDeployment(REGION, deploymentId).getStatus());
     }
 
+    @Test
+    void failedPostTrafficHookRestoresExactAliasRoutingWhenRollbackIsEnabled() {
+        service.createApplication(REGION, "app-rollback", "Lambda", null);
+        service.createDeploymentGroup(REGION, "app-rollback", "group-rollback",
+                "CodeDeployDefault.LambdaAllAtOnce", ROLE_ARN,
+                Map.of("autoRollbackConfiguration", Map.of("enabled", true,
+                        "events", List.of("DEPLOYMENT_FAILURE"))));
+        LambdaAlias alias = new LambdaAlias();
+        alias.setFunctionVersion("1");
+        alias.setRoutingConfig(Map.of("3", 0.2));
+        when(lambdaService.getAlias(eq(REGION), anyString(), anyString())).thenReturn(alias);
+        InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-rollback");
+        when(lambdaService.invoke(eq(REGION), eq("afterHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
+                .thenAnswer(invocation -> {
+                    byte[] payload = invocation.getArgument(2);
+                    service.putLifecycleEventHookExecutionStatus(deploymentIdFrom(payload), executionIdFrom(payload), "Failed");
+                    return cleanInvoke;
+                });
+
+        String deploymentId = createLambdaDeployment("app-rollback", "group-rollback", null, "afterHook");
+
+        assertEquals("Failed", awaitTerminal(deploymentId, Duration.ofSeconds(5)).getStatus());
+        ArgumentCaptor<Map<String, Double>> routing = ArgumentCaptor.forClass(Map.class);
+        ArgumentCaptor<String> version = ArgumentCaptor.forClass(String.class);
+        verify(lambdaService, times(2)).updateAlias(eq(REGION), eq("myFunc"), eq("live"), version.capture(),
+                any(), routing.capture());
+        assertEquals("2", version.getAllValues().get(0));
+        assertEquals("1", version.getAllValues().get(1));
+        assertEquals(Map.of("3", 0.2), routing.getAllValues().get(1));
+    }
+
+    @Test
+    void lambdaAppSpecRequiresCurrentVersionAndMatchingAlias() {
+        createLambdaAppAndGroup("app-invalid-appspec", "group-invalid-appspec");
+        Map<String, Object> revision = Map.of("revisionType", "AppSpecContent", "appSpecContent", Map.of("content", """
+                version: '0.0'
+                Resources:
+                  - myFunc:
+                      Type: AWS::Lambda::Function
+                      Properties:
+                        Name: myFunc
+                        Alias: live
+                        TargetVersion: '2'
+                """));
+
+        io.github.hectorvent.floci.core.common.AwsException error = org.junit.jupiter.api.Assertions.assertThrows(
+                io.github.hectorvent.floci.core.common.AwsException.class,
+                () -> service.createDeployment(REGION, "app-invalid-appspec", "group-invalid-appspec", null, revision, null));
+
+        assertEquals("InvalidRevisionException", error.getErrorCode());
+    }
+
+    @Test
+    void lambdaCanaryUsesFullNamedConfigurationInterval() {
+        AtomicLong sleptMillis = new AtomicLong();
+        CodeDeployService noWaitService = newService(Duration.ofHours(1), Clock.systemUTC(), sleptMillis::set);
+        noWaitService.createApplication(REGION, "app-canary", "Lambda", null);
+        noWaitService.createDeploymentGroup(REGION, "app-canary", "group-canary",
+                "CodeDeployDefault.LambdaCanary10Percent5Minutes", ROLE_ARN, null);
+
+        String deploymentId = noWaitService.createDeployment(REGION, "app-canary", "group-canary", null,
+                lambdaRevision(null, null), null);
+
+        assertEquals("Succeeded", awaitTerminal(noWaitService, deploymentId, Duration.ofSeconds(5)).getStatus());
+        assertEquals(Duration.ofMinutes(5).toMillis(), sleptMillis.get());
+    }
+
+    @Test
+    void lambdaLinearUsesFullNamedConfigurationInterval() {
+        AtomicLong sleptMillis = new AtomicLong();
+        CodeDeployService noWaitService = newService(Duration.ofHours(1), Clock.systemUTC(), sleptMillis::addAndGet);
+        noWaitService.createApplication(REGION, "app-linear", "Lambda", null);
+        noWaitService.createDeploymentGroup(REGION, "app-linear", "group-linear",
+                "CodeDeployDefault.LambdaLinear10PercentEvery1Minute", ROLE_ARN, null);
+
+        String deploymentId = noWaitService.createDeployment(REGION, "app-linear", "group-linear", null,
+                lambdaRevision(null, null), null);
+
+        assertEquals("Succeeded", awaitTerminal(noWaitService, deploymentId, Duration.ofSeconds(5)).getStatus());
+        assertEquals(Duration.ofMinutes(9).toMillis(), sleptMillis.get());
+    }
+
+    @Test
+    void ecsCanaryUsesFullNamedConfigurationInterval() throws InterruptedException {
+        AtomicLong sleptMillis = new AtomicLong();
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        CodeDeployService noWaitService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                sleptMillis::set);
+        noWaitService.initializeStorage();
+
+        noWaitService.executeEcsAllowTraffic(REGION, new Deployment(),
+                "CodeDeployDefault.ECSCanary10Percent5Minutes", new ConcurrentHashMap<>(), List.of("listener"),
+                "blue", "green", new java.util.concurrent.atomic.AtomicBoolean());
+
+        assertEquals(Duration.ofMinutes(5).toMillis(), sleptMillis.get());
+        verify(elbV2Service).shiftListenerForward(REGION, "listener", "blue", "green", 10);
+        verify(elbV2Service).shiftListenerForward(REGION, "listener", "blue", "green", 100);
+    }
+
+    @Test
+    void ecsLinearUsesFullNamedConfigurationInterval() throws InterruptedException {
+        AtomicLong sleptMillis = new AtomicLong();
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        CodeDeployService noWaitService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                sleptMillis::addAndGet);
+        noWaitService.initializeStorage();
+
+        noWaitService.executeEcsAllowTraffic(REGION, new Deployment(),
+                "CodeDeployDefault.ECSLinear10PercentEvery1Minutes", new ConcurrentHashMap<>(), List.of("listener"),
+                "blue", "green", new java.util.concurrent.atomic.AtomicBoolean());
+
+        assertEquals(Duration.ofMinutes(9).toMillis(), sleptMillis.get());
+        verify(elbV2Service).shiftListenerForward(REGION, "listener", "blue", "green", 100);
+    }
+
+    @Test
+    void ecsGreenTaskSetWaitsForFargateTasksToStart() throws InterruptedException {
+        EcsService ecsService = mock(EcsService.class);
+        AtomicInteger sleeps = new AtomicInteger();
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, ecsService,
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> sleeps.incrementAndGet());
+        TaskSet pending = greenTaskSet();
+        pending.setRunningCount(0);
+        TaskSet running = greenTaskSet();
+        when(ecsService.describeTaskSets("cluster", "service", List.of("green"), REGION))
+                .thenReturn(List.of(pending), List.of(running));
+
+        TaskSet actual = ecsDeploymentService.waitForGreenTaskSetRunning(REGION, "cluster", "service", pending,
+                new java.util.concurrent.atomic.AtomicBoolean());
+
+        assertEquals(running, actual);
+        assertEquals(1, sleeps.get());
+    }
+
+    @Test
+    void ecsGreenTaskSetFailsAfterFargateStartupTimeout() {
+        EcsService ecsService = mock(EcsService.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, ecsService,
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> { });
+        TaskSet pending = greenTaskSet();
+        pending.setRunningCount(0);
+        when(ecsService.describeTaskSets("cluster", "service", List.of("green"), REGION)).thenReturn(List.of(pending));
+
+        IllegalStateException exception = assertThrows(IllegalStateException.class,
+                () -> ecsDeploymentService.waitForGreenTaskSetRunning(REGION, "cluster", "service", pending,
+                        new java.util.concurrent.atomic.AtomicBoolean()));
+
+        assertEquals("ECS green task set has no running tasks", exception.getMessage());
+    }
+
+    @Test
+    void ecsGreenTaskSetMustMatchServiceLaunchTypeAndTargetGroup() {
+        EcsService ecsService = mock(EcsService.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, ecsService,
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        TaskSet green = greenTaskSet();
+        green.setLaunchType(LaunchType.EC2);
+        when(ecsService.describeTaskSets("cluster", "service", List.of("green"), REGION)).thenReturn(List.of(green));
+
+        assertThrows(IllegalStateException.class, () -> ecsDeploymentService.verifyGreenTaskSet(REGION,
+                "cluster", "service", "deployment", fargateService(), green, "green-target-group"));
+    }
+
+    @Test
+    void ecsGreenTaskSetAcceptsTheExistingEc2Path() {
+        EcsService ecsService = mock(EcsService.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, ecsService,
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        TaskSet green = greenTaskSet();
+        green.setLaunchType(LaunchType.EC2);
+        EcsServiceModel ec2Service = new EcsServiceModel();
+        ec2Service.setLaunchType(LaunchType.EC2);
+        when(ecsService.describeTaskSets("cluster", "service", List.of("green"), REGION)).thenReturn(List.of(green));
+
+        ecsDeploymentService.verifyGreenTaskSet(REGION, "cluster", "service", "deployment", ec2Service,
+                green, "green-target-group");
+    }
+
+    @Test
+    void ecsGreenTaskSetRejectsEc2WithoutRunningTasksImmediately() {
+        EcsService ecsService = mock(EcsService.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, ecsService,
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> { });
+        TaskSet green = greenTaskSet();
+        green.setLaunchType(LaunchType.EC2);
+        green.setRunningCount(0);
+        EcsServiceModel ec2Service = new EcsServiceModel();
+        ec2Service.setLaunchType(LaunchType.EC2);
+        when(ecsService.describeTaskSets("cluster", "service", List.of("green"), REGION)).thenReturn(List.of(green));
+
+        IllegalStateException exception = assertThrows(IllegalStateException.class,
+                () -> ecsDeploymentService.verifyGreenTaskSet(REGION, "cluster", "service", "deployment",
+                        ec2Service, green, "green-target-group"));
+
+        assertEquals("ECS green task set has no running tasks", exception.getMessage());
+    }
+
+    @Test
+    void ecsDeploymentVerifiesTheAuthoritativeDefaultRuleInsteadOfStaleListenerActions() {
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        Listener staleListener = new Listener();
+        staleListener.setDefaultActions(List.of(forwardAction("blue-target-group")));
+        when(elbV2Service.describeListeners(REGION, null, List.of("listener"))).thenReturn(List.of(staleListener));
+        when(elbV2Service.describeRules(REGION, "listener", null))
+                .thenReturn(List.of(defaultRule(forwardAction("green-target-group"))));
+
+        ecsDeploymentService.verifyListenersForwardToGreen(REGION, List.of("listener"), "green-target-group");
+
+        verify(elbV2Service, never()).describeListeners(anyString(), any(), any());
+    }
+
+    @Test
+    void ecsDeploymentAcceptsWeightedDefaultRuleWithAllTrafficOnGreen() {
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        Action weightedForward = new Action();
+        weightedForward.setType("forward");
+        weightedForward.setTargetGroups(List.of(targetGroup("blue-target-group", 0),
+                targetGroup("green-target-group", 100)));
+        when(elbV2Service.describeRules(REGION, "listener", null))
+                .thenReturn(List.of(defaultRule(weightedForward)));
+
+        ecsDeploymentService.verifyListenersForwardToGreen(REGION, List.of("listener"), "green-target-group");
+    }
+
+    @Test
+    void ecsDeploymentRejectsDefaultRuleWithResidualBlueTraffic() {
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        Action weightedForward = new Action();
+        weightedForward.setType("forward");
+        weightedForward.setTargetGroups(List.of(targetGroup("blue-target-group", 1),
+                targetGroup("green-target-group", 99)));
+        when(elbV2Service.describeRules(REGION, "listener", null))
+                .thenReturn(List.of(defaultRule(weightedForward)));
+
+        IllegalStateException exception = assertThrows(IllegalStateException.class,
+                () -> ecsDeploymentService.verifyListenersForwardToGreen(REGION,
+                        List.of("listener"), "green-target-group"));
+
+        assertEquals("Production listener did not switch to the green target group", exception.getMessage());
+    }
+
+    @Test
+    void ecsDeploymentRejectsAnEmptyGreenTargetGroup() {
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> { });
+
+        assertThrows(IllegalStateException.class, () -> ecsDeploymentService.waitForGreenTargets(REGION,
+                "green-target-group", new java.util.concurrent.atomic.AtomicBoolean()));
+    }
+
+    @Test
+    void ecsDeploymentWaitsForHealthyGreenTargets() throws InterruptedException {
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        AtomicInteger sleeps = new AtomicInteger();
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> sleeps.incrementAndGet());
+        TargetHealth target = new TargetHealth();
+        target.setState("healthy");
+        when(elbV2Service.describeTargetHealth(REGION, "green-target-group", null))
+                .thenReturn(List.of(), List.of(target));
+
+        ecsDeploymentService.waitForGreenTargets(REGION, "green-target-group",
+                new java.util.concurrent.atomic.AtomicBoolean());
+
+        assertEquals(1, sleeps.get());
+    }
+
+    @Test
+    void ecsDeploymentWaitsForDefaultTargetGroupHealthThreshold() throws InterruptedException {
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        AtomicInteger polls = new AtomicInteger();
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> { });
+        TargetHealth initial = new TargetHealth();
+        initial.setState("initial");
+        TargetHealth healthy = new TargetHealth();
+        healthy.setState("healthy");
+        when(elbV2Service.describeTargetHealth(REGION, "green-target-group", null))
+                .thenAnswer(ignored -> List.of(polls.incrementAndGet() > 150 ? healthy : initial));
+
+        ecsDeploymentService.waitForGreenTargets(REGION, "green-target-group",
+                new java.util.concurrent.atomic.AtomicBoolean());
+
+        assertEquals(151, polls.get());
+    }
+
+    private TaskSet greenTaskSet() {
+        EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+        loadBalancer.setTargetGroupArn("green-target-group");
+        TaskSet taskSet = new TaskSet();
+        taskSet.setTaskSetArn("green");
+        taskSet.setExternalId("deployment");
+        taskSet.setStartedBy("CODE_DEPLOY");
+        taskSet.setStatus("ACTIVE");
+        taskSet.setLaunchType(LaunchType.FARGATE);
+        taskSet.setComputedDesiredCount(1);
+        taskSet.setRunningCount(1);
+        taskSet.setLoadBalancers(List.of(loadBalancer));
+        return taskSet;
+    }
+
+    private EcsServiceModel fargateService() {
+        EcsServiceModel ecsService = new EcsServiceModel();
+        ecsService.setLaunchType(LaunchType.FARGATE);
+        return ecsService;
+    }
+
+    private Rule defaultRule(Action action) {
+        Rule rule = new Rule();
+        rule.setDefault(true);
+        rule.setActions(List.of(action));
+        return rule;
+    }
+
+    private Action forwardAction(String targetGroupArn) {
+        Action action = new Action();
+        action.setType("forward");
+        action.setTargetGroupArn(targetGroupArn);
+        return action;
+    }
+
+    private Action.TargetGroupTuple targetGroup(String targetGroupArn, int weight) {
+        Action.TargetGroupTuple target = new Action.TargetGroupTuple();
+        target.setTargetGroupArn(targetGroupArn);
+        target.setWeight(weight);
+        return target;
+    }
+
     // ---- Helpers --------------------------------------------------------------------------
 
     private String createServerDeployment(String appName, String groupName, int timeoutSeconds) {
@@ -327,6 +754,10 @@ class CodeDeployServiceTest {
                                            int timeoutSeconds) {
         target.createApplication(REGION, appName, "Server", null);
         target.createDeploymentGroup(REGION, appName, groupName, "CodeDeployDefault.AllAtOnce", ROLE_ARN, null);
+        return target.createDeployment(REGION, appName, groupName, null, serverRevision(timeoutSeconds), "server hook test");
+    }
+
+    private Map<String, Object> serverRevision(int timeoutSeconds) {
         String appSpec = """
                 os: linux
                 hooks:
@@ -334,9 +765,8 @@ class CodeDeployServiceTest {
                     - location: scripts/start_server.sh
                       timeout: %d
         """.formatted(timeoutSeconds);
-        Map<String, Object> revision = Map.of("revisionType", "AppSpecContent",
+        return Map.of("revisionType", "AppSpecContent",
                 "appSpecContent", Map.of("content", appSpec));
-        return target.createDeployment(REGION, appName, groupName, null, revision, "server hook test");
     }
 
     private void createLambdaAppAndGroup(String appName, String groupName) {
@@ -385,6 +815,14 @@ class CodeDeployServiceTest {
         }
     }
 
+    private static String deploymentIdFrom(byte[] payload) {
+        try {
+            return JSON.readTree(payload).get("DeploymentId").asText();
+        } catch (Exception e) {
+            throw new IllegalStateException("Could not parse hook payload", e);
+        }
+    }
+
     private Deployment awaitTerminal(String deploymentId, Duration atMost) {
         return awaitTerminal(service, deploymentId, atMost);
     }

src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineServiceTest.java

@@ -8,29 +8,45 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageFactory;
 import io.github.hectorvent.floci.services.codebuild.CodeBuildService;
 import io.github.hectorvent.floci.services.codebuild.model.Build;
+import io.github.hectorvent.floci.services.cloudformation.CloudFormationService;
+import io.github.hectorvent.floci.services.cloudformation.model.Stack;
 import io.github.hectorvent.floci.services.codedeploy.CodeDeployService;
 import io.github.hectorvent.floci.services.codedeploy.model.Deployment;
 import io.github.hectorvent.floci.services.codepipeline.model.CodePipelinePipeline;
+import io.github.hectorvent.floci.services.ecs.EcsService;
+import io.github.hectorvent.floci.services.ecs.model.RegisterTaskDefinitionRequest;
+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
 import io.github.hectorvent.floci.services.lambda.LambdaService;
 import io.github.hectorvent.floci.services.s3.S3Service;
 import io.github.hectorvent.floci.services.s3.model.S3Object;
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
 
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -46,7 +62,8 @@ class CodePipelineServiceTest {
             mock(CodeBuildService.class),
             mock(CodeDeployService.class),
             mock(LambdaService.class),
-            mock(S3Service.class));
+            mock(S3Service.class),
+            mock(CloudFormationService.class));
 
     @Test
     void getPipelineTreatsMissingStoredVersionAsVersionOne() throws Exception {
@@ -57,7 +74,8 @@ class CodePipelineServiceTest {
                 mock(CodeBuildService.class),
                 mock(CodeDeployService.class),
                 mock(LambdaService.class),
-                mock(S3Service.class));
+                mock(S3Service.class),
+                mock(CloudFormationService.class));
         CodePipelinePipeline pipeline = mapper.readValue("""
                 {
                     "accountId": "000000000000",
@@ -117,7 +135,8 @@ class CodePipelineServiceTest {
                 mock(CodeBuildService.class),
                 mock(CodeDeployService.class),
                 mock(LambdaService.class),
-                s3Service);
+                s3Service,
+                mock(CloudFormationService.class));
         String pipelineRequest = """
                 {
                     "pipeline": {
@@ -224,6 +243,450 @@ class CodePipelineServiceTest {
         }
     }
 
+    @Test
+    void codeBuildOutputArtifactIsMaterializedForCodeDeploy() throws Exception {
+        CodeBuildService codeBuildService = mock(CodeBuildService.class);
+        CodeDeployService codeDeployService = mock(CodeDeployService.class);
+        S3Service s3Service = mock(S3Service.class);
+        byte[] sourceBytes = "source artifact".getBytes();
+        byte[] buildOutput = "CodeBuild-produced artifact".getBytes();
+        S3Object source = new S3Object();
+        source.setData(sourceBytes);
+        source.setETag("source-etag");
+        source.setVersionId("source-version");
+        source.setLastModified(Instant.now());
+        Build build = build("SUCCEEDED", true);
+        build.setId("build-42");
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(codeBuildService.startBuild(anyString(), anyString(), anyString(),
+                any(), any(), any(), any(), any(), any(), any())).thenReturn(build);
+        when(codeBuildService.getBuildOutputArtifact(REGION, ACCOUNT, "build-42")).thenReturn(buildOutput);
+        when(codeDeployService.createDeployment(eq(REGION), eq("lambda-application"), eq("lambda-group"),
+                any(), any(), anyString())).thenReturn("deployment-42");
+        when(codeDeployService.getDeployment(REGION, "deployment-42")).thenReturn(deployment("Succeeded"));
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, codeBuildService, codeDeployService,
+                mock(LambdaService.class), s3Service, mock(CloudFormationService.class));
+
+        try {
+            handle(pipelineService, "CreatePipeline", """
+                    {
+                        "pipeline": {
+                            "name": "source-build-deploy",
+                            "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",
+                            "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},
+                            "stages": [{
+                                "name": "Source", "actions": [{
+                                    "name": "SourceAction",
+                                    "actionTypeId": {"category": "Source", "owner": "AWS", "provider": "S3", "version": "1"},
+                                    "configuration": {"S3Bucket": "source-bucket", "S3ObjectKey": "source.zip"},
+                                    "outputArtifacts": [{"name": "SourceOutput"}]
+                                }]
+                            }, {
+                                "name": "Build", "actions": [{
+                                    "name": "BuildAction",
+                                    "actionTypeId": {"category": "Build", "owner": "AWS", "provider": "CodeBuild", "version": "1"},
+                                    "configuration": {"ProjectName": "build-project"},
+                                    "inputArtifacts": [{"name": "SourceOutput"}],
+                                    "outputArtifacts": [{"name": "BuildOutput"}]
+                                }]
+                            }, {
+                                "name": "Deploy", "actions": [{
+                                    "name": "DeployAction",
+                                    "actionTypeId": {"category": "Deploy", "owner": "AWS", "provider": "CodeDeploy", "version": "1"},
+                                    "configuration": {"ApplicationName": "lambda-application", "DeploymentGroupName": "lambda-group"},
+                                    "inputArtifacts": [{"name": "BuildOutput"}]
+                                }]
+                            }]
+                        }
+                    }
+                    """);
+            String executionId = startExecution(pipelineService, "source-build-deploy");
+
+            waitForExecutionStatus(pipelineService, "source-build-deploy", executionId, "Succeeded");
+            ArgumentCaptor<byte[]> deployed = ArgumentCaptor.forClass(byte[].class);
+            verify(s3Service).putObject(eq("codepipeline-artifacts"), anyString(), deployed.capture(),
+                    eq("application/zip"), any());
+            assertArrayEquals(buildOutput, deployed.getValue());
+            ArgumentCaptor<Map<String, Object>> revision = ArgumentCaptor.forClass(Map.class);
+            verify(codeDeployService).createDeployment(eq(REGION), eq("lambda-application"), eq("lambda-group"),
+                    any(), revision.capture(), anyString());
+            assertEquals("codepipeline-artifacts", ((Map<?, ?>) revision.getValue().get("s3Location")).get("bucket"));
+            verify(codeBuildService, times(1)).getBuildOutputArtifact(REGION, ACCOUNT, "build-42");
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void cloudFormationCreateUpdateMaterializesArtifactAndWaitsForStack() throws Exception {
+        S3Service s3Service = mock(S3Service.class);
+        CloudFormationService cloudFormationService = mock(CloudFormationService.class);
+        S3Object source = new S3Object();
+        source.setData(zipArtifact("templates/application.yaml", """
+                AWSTemplateFormatVersion: '2010-09-09'
+                Resources: {}
+                """));
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        Stack stack = new Stack();
+        stack.setStackId("arn:aws:cloudformation:us-east-1:000000000000:stack/application/id");
+        stack.setStatus("CREATE_COMPLETE");
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(cloudFormationService.describeStacks("application", REGION, ACCOUNT))
+                .thenThrow(new AwsException("ValidationError", "Stack does not exist", 400))
+                .thenReturn(List.of(stack));
+        when(cloudFormationService.executeChangeSet(eq("application"), anyString(), eq(REGION), eq(ACCOUNT)))
+                .thenReturn(CompletableFuture.completedFuture(null));
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, mock(CodeBuildService.class), mock(CodeDeployService.class),
+                mock(LambdaService.class), s3Service, cloudFormationService);
+
+        try {
+            handle(pipelineService, "CreatePipeline", """
+                    {
+                        "pipeline": {
+                            "name": "cloudformation-pipeline",
+                            "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",
+                            "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},
+                            "stages": [{
+                                "name": "Source", "actions": [{
+                                    "name": "SourceAction",
+                                    "actionTypeId": {"category": "Source", "owner": "AWS", "provider": "S3", "version": "1"},
+                                    "configuration": {"S3Bucket": "source-bucket", "S3ObjectKey": "source.zip"},
+                                    "outputArtifacts": [{"name": "SourceOutput"}]
+                                }]
+                            }, {
+                                "name": "Deploy", "actions": [{
+                                    "name": "CloudFormationAction",
+                                    "actionTypeId": {"category": "Deploy", "owner": "AWS", "provider": "CloudFormation", "version": "1"},
+                                    "configuration": {"StackName": "application", "ActionMode": "CREATE_UPDATE", "TemplatePath": "SourceOutput::templates/application.yaml"},
+                                    "inputArtifacts": [{"name": "SourceOutput"}]
+                                }]
+                            }]
+                        }
+                    }
+                    """);
+            String executionId = startExecution(pipelineService, "cloudformation-pipeline");
+
+            waitForExecutionStatus(pipelineService, "cloudformation-pipeline", executionId, "Succeeded");
+            ArgumentCaptor<byte[]> artifact = ArgumentCaptor.forClass(byte[].class);
+            verify(s3Service).putObject(eq("codepipeline-artifacts"), anyString(), artifact.capture(),
+                    eq("application/zip"), any());
+            assertArrayEquals(source.getData(), artifact.getValue());
+            ArgumentCaptor<String> template = ArgumentCaptor.forClass(String.class);
+            verify(cloudFormationService).createChangeSet(eq("application"), anyString(), eq("CREATE"),
+                    template.capture(), eq(null), eq(Map.of()), eq(List.of()), eq(Map.of()),
+                    eq(REGION), eq(ACCOUNT));
+            assertEquals("AWSTemplateFormatVersion: '2010-09-09'\nResources: {}\n", template.getValue());
+            verify(cloudFormationService).executeChangeSet(eq("application"), anyString(), eq(REGION), eq(ACCOUNT));
+
+            stack.setStatus("UPDATE_COMPLETE");
+            String updateExecutionId = startExecution(pipelineService, "cloudformation-pipeline");
+            waitForExecutionStatus(pipelineService, "cloudformation-pipeline", updateExecutionId, "Succeeded");
+            verify(cloudFormationService).createChangeSet(eq("application"), anyString(), eq("UPDATE"),
+                    anyString(), eq(null), eq(Map.of()), eq(List.of()), eq(Map.of()), eq(REGION), eq(ACCOUNT));
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void codeDeployToEcsSubstitutesFargateTaskDefinitionImageFromImageDetail() throws Exception {
+        S3Service s3Service = mock(S3Service.class);
+        CodeDeployService codeDeployService = mock(CodeDeployService.class);
+        EcsService ecsService = mock(EcsService.class);
+        S3Object source = new S3Object();
+        source.setData(zipArtifact(Map.of(
+                "taskdef.json", """
+                        {"family":"pipeline-task","networkMode":"awsvpc","requiresCompatibilities":["FARGATE"],"containerDefinitions":[{"name":"app","image":"<IMAGE1_NAME>"}]}
+                        """,
+                "imageDetail.json", """
+                        {"ImageURI":"000000000000.dkr.ecr.us-east-1.amazonaws.com/pipeline-app:build-5"}
+                        """,
+                "appspec.json", """
+                        {"version":0.0,"Resources":[{"TargetService":{"Type":"AWS::ECS::Service","Properties":{"TaskDefinition":"<TASK_DEFINITION>","LoadBalancerInfo":{"ContainerName":"app","ContainerPort":80}}}}]}
+                        """)));
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        TaskDefinition taskDefinition = new TaskDefinition();
+        taskDefinition.setTaskDefinitionArn("arn:aws:ecs:us-east-1:000000000000:task-definition/pipeline-task:2");
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(ecsService.registerTaskDefinition(any(RegisterTaskDefinitionRequest.class), eq(REGION)))
+                .thenReturn(taskDefinition);
+        when(codeDeployService.createDeployment(eq(REGION), eq("ecs-app"), eq("ecs-group"),
+                any(), any(), anyString())).thenReturn("deployment-ecs");
+        when(codeDeployService.getDeployment(REGION, "deployment-ecs")).thenReturn(deployment("Succeeded"));
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, mock(CodeBuildService.class), codeDeployService,
+                mock(LambdaService.class), s3Service, mock(CloudFormationService.class), ecsService);
+
+        try {
+            createCodeDeployToEcsPipeline(pipelineService, "ecs-success-pipeline");
+            String executionId = startExecution(pipelineService, "ecs-success-pipeline");
+
+            waitForExecutionStatus(pipelineService, "ecs-success-pipeline", executionId, "Succeeded");
+            ArgumentCaptor<RegisterTaskDefinitionRequest> request = ArgumentCaptor.forClass(RegisterTaskDefinitionRequest.class);
+            verify(ecsService).registerTaskDefinition(request.capture(), eq(REGION));
+            assertEquals("pipeline-task", request.getValue().getFamily());
+            assertTrue(request.getValue().isFargate());
+            assertEquals("000000000000.dkr.ecr.us-east-1.amazonaws.com/pipeline-app:build-5",
+                    request.getValue().getContainerDefinitions().getFirst().getImage());
+            ArgumentCaptor<Map<String, Object>> revision = ArgumentCaptor.forClass(Map.class);
+            verify(codeDeployService).createDeployment(eq(REGION), eq("ecs-app"), eq("ecs-group"),
+                    eq(null), revision.capture(), anyString());
+            String appSpec = (String) ((Map<?, ?>) revision.getValue().get("appSpecContent")).get("content");
+            assertTrue(appSpec.contains(taskDefinition.getTaskDefinitionArn()));
+            assertFalse(appSpec.contains("<TASK_DEFINITION>"));
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void codeDeployToEcsPropagatesDeploymentFailure() throws Exception {
+        S3Service s3Service = mock(S3Service.class);
+        CodeDeployService codeDeployService = mock(CodeDeployService.class);
+        EcsService ecsService = mock(EcsService.class);
+        S3Object source = new S3Object();
+        source.setData(zipArtifact(Map.of(
+                "taskdef.json", "{\"family\":\"pipeline-task\",\"containerDefinitions\":[{\"name\":\"app\",\"image\":\"<IMAGE1_NAME>\"}]}",
+                "imageDetail.json", "{\"ImageURI\":\"000000000000.dkr.ecr.us-east-1.amazonaws.com/pipeline-app:build-5\"}",
+                "appspec.json", "{\"version\":0.0,\"Resources\":[{\"TargetService\":{\"Properties\":{\"TaskDefinition\":\"<TASK_DEFINITION>\"}}}]}")));
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        TaskDefinition taskDefinition = new TaskDefinition();
+        taskDefinition.setTaskDefinitionArn("arn:aws:ecs:us-east-1:000000000000:task-definition/pipeline-task:1");
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(ecsService.registerTaskDefinition(any(RegisterTaskDefinitionRequest.class), eq(REGION)))
+                .thenReturn(taskDefinition);
+        when(codeDeployService.createDeployment(anyString(), anyString(), anyString(), any(), any(), anyString()))
+                .thenReturn("deployment-ecs");
+        when(codeDeployService.getDeployment(REGION, "deployment-ecs")).thenReturn(deployment("Failed"));
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, mock(CodeBuildService.class), codeDeployService,
+                mock(LambdaService.class), s3Service, mock(CloudFormationService.class), ecsService);
+
+        try {
+            createCodeDeployToEcsPipeline(pipelineService, "ecs-failure-pipeline");
+            String executionId = startExecution(pipelineService, "ecs-failure-pipeline");
+
+            waitForExecutionStatus(pipelineService, "ecs-failure-pipeline", executionId, "Failed");
+            JsonNode state = pipelineState(pipelineService, "ecs-failure-pipeline");
+            assertEquals("Failed", state.path("stageStates").path(1).path("actionStates").path(0)
+                    .path("latestExecution").path("status").asText());
+            assertTrue(state.path("stageStates").path(1).path("actionStates").path(0)
+                    .path("latestExecution").path("summary").asText().contains("Failed"));
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void cloudFormationExtractsCodeBuildZipAndResolvesArtifactParameterOverrides() throws Exception {
+        CodeBuildService codeBuildService = mock(CodeBuildService.class);
+        S3Service s3Service = mock(S3Service.class);
+        CloudFormationService cloudFormationService = mock(CloudFormationService.class);
+        S3Object source = new S3Object();
+        source.setData(zipArtifact("template.yaml", """
+                Parameters:
+                  CodeS3Bucket: {Type: String}
+                  CodeS3Key: {Type: String}
+                Resources: {}
+                """));
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        byte[] functionZip = "function bytes".getBytes(StandardCharsets.UTF_8);
+        Build build = build("SUCCEEDED", true);
+        build.setId("build-function");
+        Stack stack = new Stack();
+        stack.setStackId("arn:aws:cloudformation:us-east-1:000000000000:stack/application/id");
+        stack.setStatus("CREATE_COMPLETE");
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(codeBuildService.startBuild(anyString(), anyString(), anyString(),
+                any(), any(), any(), any(), any(), any(), any())).thenReturn(build);
+        when(codeBuildService.getBuildOutputArtifact(REGION, ACCOUNT, "build-function"))
+                .thenReturn(zipArtifact("function.zip", functionZip));
+        when(cloudFormationService.describeStacks("application", REGION, ACCOUNT))
+                .thenThrow(new AwsException("ValidationError", "Stack does not exist", 400))
+                .thenReturn(List.of(stack));
+        when(cloudFormationService.executeChangeSet(eq("application"), anyString(), eq(REGION), eq(ACCOUNT)))
+                .thenReturn(CompletableFuture.completedFuture(null));
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, codeBuildService, mock(CodeDeployService.class),
+                mock(LambdaService.class), s3Service, cloudFormationService);
+
+        try {
+            handle(pipelineService, "CreatePipeline", """
+                    {"pipeline":{"name":"build-cloudformation","roleArn":"arn:aws:iam::000000000000:role/codepipeline-role",
+                    "artifactStore":{"type":"S3","location":"codepipeline-artifacts"},"stages":[
+                    {"name":"Source","actions":[{"name":"SourceAction","actionTypeId":{"category":"Source","owner":"AWS","provider":"S3","version":"1"},
+                    "configuration":{"S3Bucket":"source-bucket","S3ObjectKey":"source.zip"},"outputArtifacts":[{"name":"SourceOutput"}]}]},
+                    {"name":"Build","actions":[{"name":"BuildAction","actionTypeId":{"category":"Build","owner":"AWS","provider":"CodeBuild","version":"1"},
+                    "configuration":{"ProjectName":"build-project"},"inputArtifacts":[{"name":"SourceOutput"}],"outputArtifacts":[{"name":"BuildOutput"}]}]},
+                    {"name":"Deploy","actions":[{"name":"CloudFormationAction","actionTypeId":{"category":"Deploy","owner":"AWS","provider":"CloudFormation","version":"1"},
+                    "configuration":{"StackName":"application","ActionMode":"CREATE_UPDATE","TemplatePath":"SourceOutput::template.yaml",
+                    "ParameterOverrides":"{\\"CodeS3Bucket\\":{\\"Fn::GetArtifactAtt\\":[\\"BuildOutput\\",\\"BucketName\\"]},\\"CodeS3Key\\":{\\"Fn::GetArtifactAtt\\":[\\"BuildOutput\\",\\"ObjectKey\\"]}}"},
+                    "inputArtifacts":[{"name":"SourceOutput"},{"name":"BuildOutput"}]}]}]}}
+                    """);
+            String executionId = startExecution(pipelineService, "build-cloudformation");
+
+            waitForExecutionStatus(pipelineService, "build-cloudformation", executionId, "Succeeded");
+            ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);
+            ArgumentCaptor<byte[]> contents = ArgumentCaptor.forClass(byte[].class);
+            verify(s3Service, times(4)).putObject(eq("codepipeline-artifacts"), keys.capture(), contents.capture(),
+                    anyString(), any());
+            int functionIndex = keys.getAllValues().indexOf(executionId + "/Deploy/CloudFormationAction/BuildOutput/function.zip");
+            assertTrue(functionIndex >= 0);
+            assertArrayEquals(functionZip, contents.getAllValues().get(functionIndex));
+            ArgumentCaptor<Map<String, String>> parameters = ArgumentCaptor.forClass(Map.class);
+            verify(cloudFormationService).createChangeSet(eq("application"), anyString(), eq("CREATE"),
+                    anyString(), eq(null), parameters.capture(), eq(List.of()), eq(Map.of()), eq(REGION), eq(ACCOUNT));
+            assertEquals(Map.of("CodeS3Bucket", "codepipeline-artifacts",
+                    "CodeS3Key", executionId + "/Deploy/CloudFormationAction/BuildOutput/function.zip"),
+                    parameters.getValue());
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void cloudFormationRejectsUnsafeTemplatePath() throws Exception {
+        S3Service s3Service = mock(S3Service.class);
+        S3Object source = new S3Object();
+        source.setData(zipArtifact("template.yaml", "Resources: {}"));
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, mock(CodeBuildService.class), mock(CodeDeployService.class),
+                mock(LambdaService.class), s3Service, mock(CloudFormationService.class));
+
+        try {
+            handle(pipelineService, "CreatePipeline", """
+                    {"pipeline":{"name":"unsafe-template-path","roleArn":"arn:aws:iam::000000000000:role/codepipeline-role",
+                    "artifactStore":{"type":"S3","location":"codepipeline-artifacts"},"stages":[
+                    {"name":"Source","actions":[{"name":"SourceAction","actionTypeId":{"category":"Source","owner":"AWS","provider":"S3","version":"1"},
+                    "configuration":{"S3Bucket":"source-bucket","S3ObjectKey":"source.zip"},"outputArtifacts":[{"name":"SourceOutput"}]}]},
+                    {"name":"Deploy","actions":[{"name":"CloudFormationAction","actionTypeId":{"category":"Deploy","owner":"AWS","provider":"CloudFormation","version":"1"},
+                    "configuration":{"StackName":"application","ActionMode":"CREATE_UPDATE","TemplatePath":"SourceOutput::../template.yaml"},"inputArtifacts":[{"name":"SourceOutput"}]}]}]}}
+                    """);
+            String executionId = startExecution(pipelineService, "unsafe-template-path");
+
+            JsonNode execution = waitForExecutionStatus(pipelineService, "unsafe-template-path", executionId, "Failed");
+            assertEquals("Action CloudFormationAction failed: TemplatePath is not safe",
+                    execution.path("statusSummary").asText());
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void retryFailedActionsRerunsOnlyFailedStageActionsAndKeepsUpstreamArtifacts() throws Exception {
+        retryStageExecutionRerunsRequestedActionsOnly("FAILED_ACTIONS", 1, 2);
+    }
+
+    @Test
+    void retryAllActionsRerunsEveryRequestedStageActionAndKeepsUpstreamArtifacts() throws Exception {
+        retryStageExecutionRerunsRequestedActionsOnly("ALL_ACTIONS", 2, 2);
+    }
+
+    private void retryStageExecutionRerunsRequestedActionsOnly(String retryMode, int expectedGoodDeploys,
+                                                                int expectedFailingDeploys) throws Exception {
+        CodeBuildService codeBuildService = mock(CodeBuildService.class);
+        S3Service s3Service = mock(S3Service.class);
+        S3Object source = new S3Object();
+        source.setData("source artifact".getBytes());
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        Build build = build("SUCCEEDED", true);
+        build.setId("build-retry");
+        AtomicInteger failingDeploys = new AtomicInteger();
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(codeBuildService.startBuild(anyString(), anyString(), anyString(),
+                any(), any(), any(), any(), any(), any(), any())).thenReturn(build);
+        when(codeBuildService.getBuildOutputArtifact(REGION, ACCOUNT, "build-retry"))
+                .thenReturn("build artifact".getBytes());
+        doAnswer(invocation -> {
+            if ("fail.zip".equals(invocation.getArgument(1)) && failingDeploys.incrementAndGet() == 1) {
+                throw new AwsException("ActionExecutionFailed", "first deploy fails", 400);
+            }
+            return null;
+        }).when(s3Service).putObject(anyString(), anyString(), any(), anyString(), any());
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, codeBuildService, mock(CodeDeployService.class),
+                mock(LambdaService.class), s3Service, mock(CloudFormationService.class));
+
+        try {
+            handle(pipelineService, "CreatePipeline", """
+                    {
+                        "pipeline": {
+                            "name": "retry-stage-pipeline",
+                            "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",
+                            "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},
+                            "stages": [{
+                                "name": "Source", "actions": [{
+                                    "name": "SourceAction",
+                                    "actionTypeId": {"category": "Source", "owner": "AWS", "provider": "S3", "version": "1"},
+                                    "configuration": {"S3Bucket": "source-bucket", "S3ObjectKey": "source.zip"},
+                                    "outputArtifacts": [{"name": "SourceOutput"}]
+                                }]
+                            }, {
+                                "name": "Build", "actions": [{
+                                    "name": "BuildAction",
+                                    "actionTypeId": {"category": "Build", "owner": "AWS", "provider": "CodeBuild", "version": "1"},
+                                    "configuration": {"ProjectName": "build-project"},
+                                    "inputArtifacts": [{"name": "SourceOutput"}],
+                                    "outputArtifacts": [{"name": "BuildOutput"}]
+                                }]
+                            }, {
+                                "name": "Deploy", "actions": [{
+                                    "name": "GoodDeploy",
+                                    "actionTypeId": {"category": "Deploy", "owner": "AWS", "provider": "S3", "version": "1"},
+                                    "configuration": {"BucketName": "deploy-bucket", "ObjectKey": "good.zip"},
+                                    "inputArtifacts": [{"name": "BuildOutput"}]
+                                }, {
+                                    "name": "FailingDeploy",
+                                    "actionTypeId": {"category": "Deploy", "owner": "AWS", "provider": "S3", "version": "1"},
+                                    "configuration": {"BucketName": "deploy-bucket", "ObjectKey": "fail.zip"},
+                                    "inputArtifacts": [{"name": "BuildOutput"}]
+                                }]
+                            }]
+                        }
+                    }
+                    """);
+            String executionId = startExecution(pipelineService, "retry-stage-pipeline");
+            waitForExecutionStatus(pipelineService, "retry-stage-pipeline", executionId, "Failed");
+
+            String retryExecutionId = handle(pipelineService, "RetryStageExecution", """
+                    {
+                        "pipelineName": "retry-stage-pipeline",
+                        "pipelineExecutionId": "%s",
+                        "stageName": "Deploy",
+                        "retryMode": "%s"
+                    }
+                    """.formatted(executionId, retryMode)).path("pipelineExecutionId").asText();
+            waitForExecutionStatus(pipelineService, "retry-stage-pipeline", retryExecutionId, "Succeeded");
+
+            verify(s3Service, times(1)).getObject("source-bucket", "source.zip");
+            verify(codeBuildService, times(1)).startBuild(anyString(), anyString(), anyString(),
+                    any(), any(), any(), any(), any(), any(), any());
+            verify(s3Service, times(expectedGoodDeploys)).putObject(eq("deploy-bucket"), eq("good.zip"),
+                    any(), eq("application/zip"), any());
+            verify(s3Service, times(expectedFailingDeploys)).putObject(eq("deploy-bucket"), eq("fail.zip"),
+                    any(), eq("application/zip"), any());
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
     @Test
     void stoppingCodeDeployWaitsForFailureWithoutFailingThePipeline() throws Exception {
         CodeDeployService codeDeployService = mock(CodeDeployService.class);
@@ -395,7 +858,7 @@ class CodePipelineServiceTest {
                                             CodeDeployService codeDeployService) {
         return new CodePipelineService(
                 new InMemoryStorageFactory(), mapper, codeBuildService, codeDeployService,
-                mock(LambdaService.class), mock(S3Service.class));
+                mock(LambdaService.class), mock(S3Service.class), mock(CloudFormationService.class));
     }
 
     private void createExternalActionPipeline(CodePipelineService pipelineService,
@@ -485,6 +948,76 @@ class CodePipelineServiceTest {
         return deployment;
     }
 
+    private void createCodeDeployToEcsPipeline(CodePipelineService pipelineService, String pipelineName) throws Exception {
+        handle(pipelineService, "CreatePipeline", """
+                {
+                    "pipeline": {
+                        "name": "%s",
+                        "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",
+                        "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},
+                        "stages": [{
+                            "name": "Source",
+                            "actions": [{
+                                "name": "SourceAction",
+                                "actionTypeId": {"category": "Source", "owner": "AWS", "provider": "S3", "version": "1"},
+                                "configuration": {"S3Bucket": "source-bucket", "S3ObjectKey": "source.zip"},
+                                "outputArtifacts": [{"name": "SourceOutput"}]
+                            }]
+                        }, {
+                            "name": "Deploy",
+                            "actions": [{
+                                "name": "DeployToEcs",
+                                "actionTypeId": {"category": "Deploy", "owner": "AWS", "provider": "CodeDeployToECS", "version": "1"},
+                                "configuration": {
+                                    "ApplicationName": "ecs-app",
+                                    "DeploymentGroupName": "ecs-group",
+                                    "TaskDefinitionTemplateArtifact": "SourceOutput",
+                                    "TaskDefinitionTemplatePath": "taskdef.json",
+                                    "AppSpecTemplateArtifact": "SourceOutput",
+                                    "AppSpecTemplatePath": "appspec.json",
+                                    "Image1ArtifactName": "SourceOutput",
+                                    "Image1ContainerName": "app"
+                                },
+                                "inputArtifacts": [{"name": "SourceOutput"}]
+                            }]
+                        }]
+                    }
+                }
+                """.formatted(pipelineName));
+    }
+
+    private byte[] zipArtifact(String entryName, String contents) throws Exception {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        try (ZipOutputStream zip = new ZipOutputStream(output)) {
+            zip.putNextEntry(new ZipEntry(entryName));
+            zip.write(contents.getBytes(StandardCharsets.UTF_8));
+            zip.closeEntry();
+        }
+        return output.toByteArray();
+    }
+
+    private byte[] zipArtifact(String entryName, byte[] contents) throws Exception {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        try (ZipOutputStream zip = new ZipOutputStream(output)) {
+            zip.putNextEntry(new ZipEntry(entryName));
+            zip.write(contents);
+            zip.closeEntry();
+        }
+        return output.toByteArray();
+    }
+
+    private byte[] zipArtifact(Map<String, String> entries) throws Exception {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        try (ZipOutputStream zip = new ZipOutputStream(output)) {
+            for (Map.Entry<String, String> entry : entries.entrySet()) {
+                zip.putNextEntry(new ZipEntry(entry.getKey()));
+                zip.write(entry.getValue().getBytes(StandardCharsets.UTF_8));
+                zip.closeEntry();
+            }
+        }
+        return output.toByteArray();
+    }
+
     private static final class InMemoryStorageFactory extends StorageFactory {
         private InMemoryStorageFactory() {
             super(null, null);

src/test/java/io/github/hectorvent/floci/services/cognito/CognitoCustomDomainFilterTest.java

@@ -36,6 +36,20 @@ class CognitoCustomDomainFilterTest {
         verify(request).setRequestUri(URI.create("http://" + DOMAIN + ":4566/cognito-idp/oauth2/token?x=1"));
     }
 
+    @Test
+    void servesTheHostedLoginAndLogoutPagesOnACustomDomain() {
+        when(cognitoService.findCustomDomain(DOMAIN)).thenReturn(Optional.of(domain("us-east-1_abc")));
+        ContainerRequestContext login = request("https://" + DOMAIN + "/login?client_id=c", DOMAIN);
+        ContainerRequestContext logout = request("https://" + DOMAIN + "/logout?client_id=c", DOMAIN);
+
+        filter.filter(login);
+        filter.filter(logout);
+
+        verify(login).setRequestUri(URI.create("https://" + DOMAIN + "/cognito-idp/login?client_id=c"));
+        verify(logout).setRequestUri(URI.create("https://" + DOMAIN + "/cognito-idp/logout?client_id=c"));
+        verify(login).setProperty(CognitoCustomDomainFilter.POOL_PROPERTY, "us-east-1_abc");
+    }
+
     /** HTTP/2 carries no Host header; the authority is on the request URI. */
     @Test
     void fallsBackToTheRequestAuthorityWithoutAHostHeader() {

src/test/java/io/github/hectorvent/floci/services/cognito/CognitoHostedLoginIntegrationTest.java

@@ -0,0 +1,292 @@
+package io.github.hectorvent.floci.services.cognito;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.net.URI;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.util.Base64;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static io.github.hectorvent.floci.services.cognito.CognitoRestAssuredUtils.cognitoJson;
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@QuarkusTest
+class CognitoHostedLoginIntegrationTest {
+
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+    private static final String CALLBACK = "https://app.example.test/callback";
+    private static final String SIGNED_OUT = "https://app.example.test/signed-out";
+    private static final String PASSWORD = "Passw0rd!";
+    private static final String VERIFIER = "dBjftJeZ4CVP-mB92K27uhbUWU1p1r_wW1gFWFOEjXk-local-verifier";
+
+    private String poolId;
+    private String clientId;
+    private String email;
+
+    @BeforeAll
+    static void configureRestAssured() {
+        RestAssuredJsonUtils.configureAwsContentTypes();
+    }
+
+    @BeforeEach
+    void createPoolClientAndUser() throws Exception {
+        poolId = cognitoJson("CreateUserPool", """
+                {"PoolName":"HostedLogin","UsernameAttributes":["email"],"AutoVerifiedAttributes":["email"]}
+                """).path("UserPool").path("Id").asText();
+        clientId = cognitoJson("CreateUserPoolClient", """
+                {"UserPoolId":"%s","ClientName":"hosted-web","AllowedOAuthFlowsUserPoolClient":true,
+                 "AllowedOAuthFlows":["code"],"AllowedOAuthScopes":["openid","email"],
+                 "CallbackURLs":["%s"],"LogoutURLs":["%s"],"SupportedIdentityProviders":["COGNITO"],
+                 "ExplicitAuthFlows":["ALLOW_REFRESH_TOKEN_AUTH"]}
+                """.formatted(poolId, CALLBACK, SIGNED_OUT)).path("UserPoolClient").path("ClientId").asText();
+        email = "hosted-" + System.nanoTime() + "@example.test";
+        cognitoJson("AdminCreateUser", """
+                {"UserPoolId":"%s","Username":"%s","MessageAction":"SUPPRESS",
+                 "UserAttributes":[{"Name":"email","Value":"%s"},{"Name":"email_verified","Value":"true"}]}
+                """.formatted(poolId, email, email));
+        cognitoJson("AdminSetUserPassword", """
+                {"UserPoolId":"%s","Username":"%s","Password":"%s","Permanent":true}
+                """.formatted(poolId, email, PASSWORD));
+    }
+
+    @Test
+    void signsInOnTheHostedPageAndRedeemsThePkceCodeForTokens() throws Exception {
+        String loginLocation = authorize(null, "openid email", "nonce-123")
+                .then().statusCode(302).extract().header("Location");
+        assertTrue(URI.create(loginLocation).getPath().endsWith("/cognito-idp/login"), loginLocation);
+        assertEquals("app-state", queryParameter(loginLocation, "state"));
+
+        Response page = given().redirects().follow(false).urlEncodingEnabled(false).get(loginLocation)
+                .then().statusCode(200)
+                .header("Content-Type", containsString("text/html"))
+                .body(containsString("to continue to <strong>hosted-web</strong>"))
+                .body(containsString("type=\"email\""))
+                .extract().response();
+        String csrf = page.getCookie(CognitoHostedLoginController.CSRF_COOKIE);
+        assertNotNull(csrf);
+        assertEquals(csrf, hiddenValue(page.asString(), "_csrf"));
+
+        Response signedIn = submit(page, email, PASSWORD).then().statusCode(302).extract().response();
+        String callback = signedIn.getHeader("Location");
+        assertTrue(callback.startsWith(CALLBACK + "?code="), callback);
+        assertEquals("app-state", queryParameter(callback, "state"));
+        assertTrue(signedIn.getDetailedCookie(CognitoHostedLoginController.SESSION_COOKIE).isHttpOnly());
+
+        given().formParam("grant_type", "authorization_code").formParam("client_id", clientId)
+                .formParam("code", queryParameter(callback, "code")).formParam("redirect_uri", CALLBACK)
+                .formParam("code_verifier", VERIFIER + "-wrong")
+                .when().post("/cognito-idp/oauth2/token")
+                .then().statusCode(400).body("error", equalTo("invalid_grant"));
+
+        String sessionId = signedIn.getCookie(CognitoHostedLoginController.SESSION_COOKIE);
+        String secondCallback = authorize(sessionId, "openid email", "nonce-456")
+                .then().statusCode(302).extract().header("Location");
+        assertTrue(secondCallback.startsWith(CALLBACK + "?code="), "an open session skips the form: " + secondCallback);
+
+        Response tokens = given().formParam("grant_type", "authorization_code").formParam("client_id", clientId)
+                .formParam("code", queryParameter(secondCallback, "code")).formParam("redirect_uri", CALLBACK)
+                .formParam("code_verifier", VERIFIER)
+                .when().post("/cognito-idp/oauth2/token")
+                .then().statusCode(200).body("token_type", equalTo("Bearer")).extract().response();
+        JsonNode idToken = claims(tokens.path("id_token"));
+        assertEquals("nonce-456", idToken.path("nonce").asText());
+        assertEquals(email, idToken.path("email").asText());
+        assertEquals("id", idToken.path("token_use").asText());
+
+        given().formParam("grant_type", "refresh_token").formParam("client_id", clientId)
+                .formParam("refresh_token", tokens.path("refresh_token").toString())
+                .when().post("/cognito-idp/oauth2/token")
+                .then().statusCode(200).body("access_token", not(equalTo(tokens.path("access_token").toString())));
+    }
+
+    @Test
+    void wrongPasswordUnknownUserAndMissingCsrfAreRefused() {
+        Response page = loginPage();
+        submit(page, email, "Wrong-passw0rd").then().statusCode(400)
+                .body(containsString("Incorrect user name or password."))
+                .body(containsString("value=\"" + email + "\""));
+        submit(page, "nobody@example.test", PASSWORD).then().statusCode(400)
+                .body(containsString("Incorrect user name or password."));
+
+        given().redirects().follow(false)
+                .formParam("client_id", clientId).formParam("redirect_uri", CALLBACK).formParam("response_type", "code")
+                .formParam("username", email).formParam("password", PASSWORD)
+                .when().post("/cognito-idp/login")
+                .then().statusCode(400).body(containsString("Your sign-in form expired"));
+    }
+
+    @Test
+    void promptLoginShowsTheFormAndLogoutEndsTheSession() {
+        Response signedIn = submit(loginPage(), email, PASSWORD).then().statusCode(302).extract().response();
+        String sessionId = signedIn.getCookie(CognitoHostedLoginController.SESSION_COOKIE);
+
+        String forced = given().redirects().follow(false).cookie(CognitoHostedLoginController.SESSION_COOKIE, sessionId)
+                .queryParam("client_id", clientId).queryParam("redirect_uri", CALLBACK)
+                .queryParam("response_type", "code").queryParam("prompt", "login")
+                .when().get("/cognito-idp/oauth2/authorize")
+                .then().statusCode(302).extract().header("Location");
+        assertTrue(URI.create(forced).getPath().endsWith("/cognito-idp/login"), forced);
+
+        given().redirects().follow(false).cookie(CognitoHostedLoginController.SESSION_COOKIE, sessionId)
+                .queryParam("client_id", clientId).queryParam("logout_uri", "https://evil.example.test/")
+                .when().get("/cognito-idp/logout")
+                .then().statusCode(400).body(containsString("not registered"));
+
+        Response loggedOut = given().redirects().follow(false).cookie(CognitoHostedLoginController.SESSION_COOKIE, sessionId)
+                .queryParam("client_id", clientId).queryParam("logout_uri", SIGNED_OUT)
+                .when().get("/cognito-idp/logout")
+                .then().statusCode(302).header("Location", equalTo(SIGNED_OUT)).extract().response();
+        assertEquals(0, loggedOut.getDetailedCookie(CognitoHostedLoginController.SESSION_COOKIE).getMaxAge());
+
+        String afterLogout = authorize(sessionId, "openid", null).then().statusCode(302).extract().header("Location");
+        assertTrue(URI.create(afterLogout).getPath().endsWith("/cognito-idp/login"), afterLogout);
+    }
+
+    @Test
+    void clientsWithoutCognitoAsAProviderAndBadPkceAreRejected() throws Exception {
+        cognitoJson("CreateIdentityProvider", """
+                {"UserPoolId":"%s","ProviderName":"CompanyIdp","ProviderType":"OIDC",
+                 "ProviderDetails":{"authorize_url":"https://idp.example.test/authorize","token_url":"https://idp.example.test/token",
+                  "attributes_url":"https://idp.example.test/userinfo","client_id":"idp-client","client_secret":"idp-secret"},
+                 "AttributeMapping":{"email":"email","username":"sub"}}
+                """.formatted(poolId));
+        String federatedOnly = cognitoJson("CreateUserPoolClient", """
+                {"UserPoolId":"%s","ClientName":"federated-only","AllowedOAuthFlowsUserPoolClient":true,
+                 "AllowedOAuthFlows":["code"],"AllowedOAuthScopes":["openid"],"CallbackURLs":["%s"],
+                 "SupportedIdentityProviders":["CompanyIdp"]}
+                """.formatted(poolId, CALLBACK)).path("UserPoolClient").path("ClientId").asText();
+        String federatedLogin = given().redirects().follow(false)
+                .queryParam("client_id", federatedOnly).queryParam("redirect_uri", CALLBACK).queryParam("response_type", "code")
+                .queryParam("state", "app-state")
+                .when().get("/cognito-idp/oauth2/authorize")
+                .then().statusCode(302).extract().header("Location");
+        String federatedPage = given().redirects().follow(false).urlEncodingEnabled(false).get(federatedLogin)
+                .then().statusCode(200).extract().asString();
+        assertTrue(federatedPage.contains(">Continue with CompanyIdp</a>"), federatedPage);
+        assertTrue(federatedPage.contains("identity_provider=CompanyIdp"), "the button starts the federated flow");
+        assertTrue(federatedPage.contains("state=app-state"), "the button keeps the app's authorize parameters");
+        assertFalse(federatedPage.contains("name=\"password\""), "no password form without COGNITO");
+        given().redirects().follow(false)
+                .formParam("client_id", federatedOnly).formParam("redirect_uri", CALLBACK).formParam("response_type", "code")
+                .formParam("username", email).formParam("password", PASSWORD)
+                .when().post("/cognito-idp/login")
+                .then().statusCode(400).body(containsString("COGNITO is not one of its identity providers"));
+
+        cognitoJson("UpdateUserPoolClient", """
+                {"UserPoolId":"%s","ClientId":"%s","ClientName":"hosted-web","AllowedOAuthFlowsUserPoolClient":true,
+                 "AllowedOAuthFlows":["code"],"AllowedOAuthScopes":["openid","email"],
+                 "CallbackURLs":["%s"],"LogoutURLs":["%s"],"SupportedIdentityProviders":["COGNITO","CompanyIdp"],
+                 "ExplicitAuthFlows":["ALLOW_REFRESH_TOKEN_AUTH"]}
+                """.formatted(poolId, clientId, CALLBACK, SIGNED_OUT));
+        String both = loginPage().asString();
+        assertTrue(both.contains("name=\"password\"") && both.contains(">Continue with CompanyIdp</a>"), both);
+
+        given().redirects().follow(false)
+                .queryParam("client_id", clientId).queryParam("redirect_uri", CALLBACK).queryParam("response_type", "code")
+                .queryParam("code_challenge", "abc").queryParam("code_challenge_method", "plain")
+                .when().get("/cognito-idp/oauth2/authorize")
+                .then().statusCode(400).body("error", equalTo("invalid_request"));
+    }
+
+    @Test
+    void unconfirmedUsersLearnWhyOnlyWithTheRightPassword() throws Exception {
+        String pending = "pending-" + System.nanoTime() + "@example.test";
+        cognitoJson("SignUp", """
+                {"ClientId":"%s","Username":"%s","Password":"%s","UserAttributes":[{"Name":"email","Value":"%s"}]}
+                """.formatted(clientId, pending, PASSWORD, pending));
+
+        Response page = loginPage();
+        submit(page, pending, "Wrong-passw0rd").then().statusCode(400)
+                .body(containsString("Incorrect user name or password."));
+        submit(page, pending, PASSWORD).then().statusCode(400)
+                .body(containsString("not confirmed yet"));
+    }
+
+    private Response authorize(String sessionId, String scope, String nonce) {
+        var request = given().redirects().follow(false)
+                .queryParam("client_id", clientId).queryParam("redirect_uri", CALLBACK)
+                .queryParam("response_type", "code").queryParam("scope", scope).queryParam("state", "app-state")
+                .queryParam("code_challenge", challenge(VERIFIER)).queryParam("code_challenge_method", "S256");
+        if (nonce != null) {
+            request.queryParam("nonce", nonce);
+        }
+        if (sessionId != null) {
+            request.cookie(CognitoHostedLoginController.SESSION_COOKIE, sessionId);
+        }
+        return request.when().get("/cognito-idp/oauth2/authorize");
+    }
+
+    private Response loginPage() {
+        return given().redirects().follow(false)
+                .queryParam("client_id", clientId).queryParam("redirect_uri", CALLBACK).queryParam("response_type", "code")
+                .queryParam("state", "app-state")
+                .when().get("/cognito-idp/login")
+                .then().statusCode(200).extract().response();
+    }
+
+    private Response submit(Response page, String username, String password) {
+        String html = page.asString();
+        var request = given().redirects().follow(false)
+                .cookie(CognitoHostedLoginController.CSRF_COOKIE, page.getCookie(CognitoHostedLoginController.CSRF_COOKIE))
+                .formParam("username", username).formParam("password", password);
+        Matcher hidden = Pattern.compile("<input type=\"hidden\" name=\"([^\"]+)\" value=\"([^\"]*)\">").matcher(html);
+        while (hidden.find()) {
+            request.formParam(hidden.group(1), unescape(hidden.group(2)));
+        }
+        return request.when().post("/cognito-idp/login");
+    }
+
+    private static String hiddenValue(String html, String name) {
+        Matcher matcher = Pattern.compile("name=\"" + Pattern.quote(name) + "\" value=\"([^\"]*)\"").matcher(html);
+        return matcher.find() ? unescape(matcher.group(1)) : null;
+    }
+
+    private static String unescape(String value) {
+        return value.replace("&quot;", "\"").replace("&#39;", "'").replace("&lt;", "<").replace("&gt;", ">")
+                .replace("&amp;", "&");
+    }
+
+    private static String challenge(String verifier) {
+        try {
+            byte[] digest = MessageDigest.getInstance("SHA-256").digest(verifier.getBytes(StandardCharsets.US_ASCII));
+            return Base64.getUrlEncoder().withoutPadding().encodeToString(digest);
+        } catch (Exception e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
+    private static JsonNode claims(String jwt) throws Exception {
+        return OBJECT_MAPPER.readTree(Base64.getUrlDecoder().decode(jwt.split("\\.")[1]));
+    }
+
+    private static String queryParameter(String location, String name) {
+        String query = URI.create(location).getRawQuery();
+        if (query == null) {
+            return null;
+        }
+        for (String parameter : query.split("&")) {
+            String[] pair = parameter.split("=", 2);
+            if (pair[0].equals(name)) {
+                return URLDecoder.decode(pair[1], StandardCharsets.UTF_8);
+            }
+        }
+        return null;
+    }
+}

src/test/java/io/github/hectorvent/floci/services/cognito/CognitoOAuthControllerTest.java

@@ -127,7 +127,7 @@ class CognitoOAuthControllerTest {
                 POOL_ID, CLIENT_ID, "federated-user", CALLBACK_URI, List.of("openid"), CLOCK.instant().plusSeconds(60)));
         when(cognitoService.describeUserPool(POOL_ID)).thenReturn(pool());
         when(cognitoService.adminGetUser(POOL_ID, "federated-user")).thenReturn(user());
-        when(cognitoService.generateAuthResult(any(CognitoUser.class), any(UserPool.class), eq(client), eq(null)))
+        when(cognitoService.issueHostedTokens(eq(client), any(CognitoUser.class), eq(null)))
                 .thenReturn(Map.of("AccessToken", "access-token", "IdToken", "id-token", "RefreshToken", "refresh-token",
                         "ExpiresIn", 3600, "TokenType", "Bearer"));
 
@@ -226,7 +226,7 @@ class CognitoOAuthControllerTest {
                 POOL_ID, CLIENT_ID, "federated-user", CALLBACK_URI, List.of("openid"), CLOCK.instant().plusSeconds(60)));
         when(cognitoService.describeUserPool(POOL_ID)).thenReturn(pool());
         when(cognitoService.adminGetUser(POOL_ID, "federated-user")).thenReturn(user());
-        when(cognitoService.generateAuthResult(any(CognitoUser.class), any(UserPool.class), eq(client), eq(null)))
+        when(cognitoService.issueHostedTokens(eq(client), any(CognitoUser.class), eq(null)))
                 .thenReturn(Map.of("AccessToken", "access-token", "IdToken", "id-token", "RefreshToken", "refresh-token",
                         "ExpiresIn", 3600, "TokenType", "Bearer"));
         return code;

src/test/java/io/github/hectorvent/floci/services/cognito/CognitoOAuthTokenIntegrationTest.java

@@ -373,7 +373,7 @@ class CognitoOAuthTokenIntegrationTest {
     @Order(16)
     void unsupportedGrantTypeReturnsUnsupportedGrantType() {
         given()
-                .formParam("grant_type", "refresh_token")
+                .formParam("grant_type", "password")
                 .formParam("client_id", clientId)
         .when()
                 .post("/cognito-idp/oauth2/token")

src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java

@@ -10,9 +10,12 @@ import com.github.dockerjava.api.command.InspectContainerCmd;
 import com.github.dockerjava.api.command.InspectContainerResponse;
 import com.github.dockerjava.api.command.InspectExecCmd;
 import com.github.dockerjava.api.command.InspectExecResponse;
+import com.github.dockerjava.api.command.InspectImageCmd;
+import com.github.dockerjava.api.command.InspectImageResponse;
 import com.github.dockerjava.api.command.ListContainersCmd;
 import com.github.dockerjava.api.command.StartContainerCmd;
 import com.github.dockerjava.api.exception.DockerException;
+import com.github.dockerjava.api.exception.NotFoundException;
 import com.github.dockerjava.api.model.Container;
 import com.github.dockerjava.api.model.ContainerConfig;
 import com.github.dockerjava.api.model.ContainerNetwork;
@@ -56,8 +59,10 @@ import java.util.concurrent.Future;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BooleanSupplier;
+import java.util.function.IntConsumer;
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -69,6 +74,7 @@ import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.anySet;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.nullable;
 import static org.mockito.Answers.RETURNS_SELF;
 import static org.mockito.Answers.RETURNS_DEEP_STUBS;
 import static org.mockito.Mockito.after;
@@ -97,6 +103,8 @@ class Ec2ContainerManagerTest {
     void resetBridgeIpPolling() {
         Ec2ContainerManager.containerBridgeIpAttempts = 30;
         Ec2ContainerManager.containerBridgeIpPollMillis = 500;
+        Ec2ContainerManager.sshBannerAttempts = 30;
+        Ec2ContainerManager.sshBannerPollMillis = 1_000;
     }
 
     @Test
@@ -436,7 +444,20 @@ class Ec2ContainerManagerTest {
 
     @Test
     void userDataExecutionCommandRunsScriptDirectlySoShebangIsHonored() {
-        assertArrayEquals(new String[]{"/var/lib/user-data.sh"}, Ec2ContainerManager.userDataExecutionCommand());
+        String[] command = Ec2ContainerManager.userDataExecutionCommand();
+
+        assertArrayEquals(new String[]{"sh", "-c"}, Arrays.copyOf(command, 2));
+        assertTrue(command[2].contains("/var/lib/user-data.sh"));
+        assertFalse(command[2].contains("sh /var/lib/user-data.sh"));
+    }
+
+    @Test
+    void userDataExecutionCommandRestoresPolicyAndStartsPackageServices() {
+        String command = Ec2ContainerManager.userDataExecutionCommand()[2];
+
+        assertTrue(command.contains("mv \"$backup\" \"$policy\""));
+        assertTrue(command.contains("service \"$service\" \"$action\""));
+        assertTrue(command.contains("exit \"$status\""));
     }
 
     @Test
@@ -731,6 +752,8 @@ class Ec2ContainerManagerTest {
     void launchReturnsTheLeaseWhenTheLaunchThrows() throws Exception {
         LaunchHarness harness = launchHarness();
         when(harness.builder.build()).thenThrow(new IllegalStateException("daemon went away"));
+        AtomicInteger persistenceCalls = new AtomicInteger();
+        harness.manager.setInstancePersister(ignored -> persistenceCalls.incrementAndGet());
 
         Instance instance = leasedInstance("i-throws");
 
@@ -742,6 +765,13 @@ class Ec2ContainerManagerTest {
         verify(harness.vpcNetworkManager, timeout(2000)).detach("us-west-2", "vpc-lease", null);
         verify(harness.vpcNetworkManager, timeout(2000))
                 .releasePrivateIp("us-west-2", "subnet-lease", "10.0.1.10");
+        assertEquals("failed", instance.getLaunchStatus());
+        assertEquals("impaired", instance.getSshStatus());
+        assertEquals("daemon went away", instance.getStateReasonMessage());
+        assertTrue(instance.getLaunchEvents().stream()
+                .anyMatch(event -> "failed".equals(event.getStatus())
+                        && "daemon went away".equals(event.getMessage())));
+        assertTrue(persistenceCalls.get() >= instance.getLaunchEvents().size());
     }
 
     private static Instance leasedInstance(String instanceId) {
@@ -789,13 +819,14 @@ class Ec2ContainerManagerTest {
         when(inspect.exec()).thenReturn(response);
 
         Instance instance = instance("i-labeltest");
+        instance.setAccountId("123456789012");
         harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");
 
         verify(harness.builder, timeout(2000)).withLabels(Map.of(
                 "io.floci", "aws",
                 "io.floci.service", "ec2",
                 "io.floci.resource-id", "i-labeltest",
-                "io.floci.account", "000000000000",
+                "io.floci.account", "123456789012",
                 "io.floci.region", "us-west-2"));
     }
 
@@ -854,6 +885,14 @@ class Ec2ContainerManagerTest {
                 "dnf must be probed before yum");
     }
 
+    @Test
+    void sshdInstallProbeHandlesOpenSuse() {
+        String script = Ec2ContainerManager.sshdInstallProbeCommand()[2];
+
+        assertTrue(script.contains("command -v zypper"), script);
+        assertTrue(script.contains("zypper --non-interactive install -y openssh"), script);
+    }
+
     @Test
     void metadataProxyInstallCommandHandlesYumForAmazonLinux2() {
         // The IMDS proxy dependency install had the same gap, and unlike the sshd probe it
@@ -866,6 +905,15 @@ class Ec2ContainerManagerTest {
         assertTrue(script.contains("yum install -y iproute iptables socat curl ca-certificates"), script);
     }
 
+    @Test
+    void metadataProxyInstallCommandHandlesOpenSuse() {
+        String script = Ec2ContainerManager.metadataProxyInstallCommand()[2];
+
+        assertTrue(script.contains("command -v zypper"), script);
+        assertTrue(script.contains("zypper --non-interactive install -y iproute2 iptables socat curl ca-certificates"),
+                script);
+    }
+
     @Test
     void metadataProxyInstallCommandLetsDnfReplaceCurlMinimalWithCurl() {
         // public.ecr.aws/amazonlinux/amazonlinux:2023 -- the fallback image AmiImageResolver
@@ -951,7 +999,8 @@ class Ec2ContainerManagerTest {
         Ec2ContainerManager.containerBridgeIpAttempts = 1;
         Ec2ContainerManager.containerBridgeIpPollMillis = 1;
         LaunchHarness harness = launchHarness();
-        when(harness.lifecycleManager.create(any(ContainerSpec.class), eq("linux/arm64")))
+        when(harness.lifecycleManager.create(any(ContainerSpec.class), eq("linux/arm64"),
+                any(ContainerLifecycleManager.ContainerCreationObserver.class)))
                 .thenReturn(TEST_CONTAINER_ID);
         InspectContainerCmd inspect = mock(InspectContainerCmd.class);
         InspectContainerResponse withIp = inspectResponse("172.18.0.11");
@@ -962,7 +1011,7 @@ class Ec2ContainerManagerTest {
 
         harness.manager.launch(instance,
                 new ResolvedAmiImage("floci/ami-ubuntu:24.04-arm64", ResolvedAmiImage.SYSTEMD_RUNTIME, true,
-                        "linux/arm64"),
+                        "linux/arm64", null, "enhanced", "root", true, "ssh"),
                 null,
                 "us-west-2");
 
@@ -970,7 +1019,49 @@ class Ec2ContainerManagerTest {
         verify(harness.builder).withCmd(List.of("/sbin/init"));
         verify(harness.builder).withCgroupnsMode("host");
         verify(harness.builder).withBind("/sys/fs/cgroup", "/sys/fs/cgroup");
-        verify(harness.lifecycleManager).create(any(ContainerSpec.class), eq("linux/arm64"));
+        verify(harness.lifecycleManager).create(any(ContainerSpec.class), eq("linux/arm64"),
+                any(ContainerLifecycleManager.ContainerCreationObserver.class));
+    }
+
+    @Test
+    void legacyTailGuestIsRecognizedForSshBootstrapAfterReboot() {
+        assertTrue(Ec2ContainerManager.isLegacyTailGuest(new String[]{"tail", "-f", "/dev/null"}));
+        assertFalse(Ec2ContainerManager.isLegacyTailGuest(new String[]{"/sbin/init"}));
+    }
+
+    @Test
+    void missingLocalAmiUsesPullableMinimalFallback() {
+        LaunchHarness harness = launchHarness();
+        InspectImageCmd inspect = mock(InspectImageCmd.class);
+        when(harness.containerBuilder.resolveImage("floci/ami-ubuntu:20.04"))
+                .thenReturn("floci/ami-ubuntu:20.04");
+        when(harness.dockerClient.inspectImageCmd("floci/ami-ubuntu:20.04")).thenReturn(inspect);
+        when(inspect.exec()).thenThrow(new NotFoundException("missing"));
+
+        ResolvedAmiImage selected = harness.manager.selectLaunchImage(new ResolvedAmiImage(
+                "floci/ami-ubuntu:20.04", ResolvedAmiImage.SYSTEMD_RUNTIME, false,
+                "linux/amd64", "public.ecr.aws/docker/library/ubuntu:20.04",
+                "enhanced", "root", true, "ssh"));
+
+        assertEquals("public.ecr.aws/docker/library/ubuntu:20.04", selected.dockerImage());
+        assertFalse(selected.systemd());
+        assertEquals("linux/amd64", selected.dockerPlatform());
+    }
+
+    @Test
+    void availableLocalAmiKeepsItsSystemdRuntime() {
+        LaunchHarness harness = launchHarness();
+        InspectImageCmd inspect = mock(InspectImageCmd.class);
+        when(harness.containerBuilder.resolveImage("floci/ami-ubuntu:20.04"))
+                .thenReturn("floci/ami-ubuntu:20.04");
+        when(harness.dockerClient.inspectImageCmd("floci/ami-ubuntu:20.04")).thenReturn(inspect);
+        when(inspect.exec()).thenReturn(mock(InspectImageResponse.class));
+        ResolvedAmiImage image = new ResolvedAmiImage(
+                "floci/ami-ubuntu:20.04", ResolvedAmiImage.SYSTEMD_RUNTIME, false,
+                "linux/amd64", "public.ecr.aws/docker/library/ubuntu:20.04",
+                "enhanced", "root", true, "ssh");
+
+        assertEquals(image, harness.manager.selectLaunchImage(image));
     }
 
     @Test
@@ -1074,8 +1165,9 @@ class Ec2ContainerManagerTest {
         Ec2ContainerManager.containerBridgeIpAttempts = 1;
         Ec2ContainerManager.containerBridgeIpPollMillis = 1;
         LaunchHarness harness = launchHarness();
-        when(harness.portAllocator.allocate(anyInt(), anyInt())).thenReturn(2201, 2202);
-        when(harness.lifecycleManager.create(any(ContainerSpec.class)))
+        when(harness.portAllocator.allocate(anyInt(), anyInt(), any(IntConsumer.class))).thenReturn(2201, 2202);
+        when(harness.lifecycleManager.create(any(ContainerSpec.class), nullable(String.class),
+                any(ContainerLifecycleManager.ContainerCreationObserver.class)))
                 .thenReturn("container-conflict", TEST_CONTAINER_ID);
         when(harness.lifecycleManager.startCreated(anyString(), any(ContainerSpec.class)))
                 .thenThrow(new RuntimeException("Bind for 0.0.0.0:2201 failed: port is already allocated"))
@@ -1098,6 +1190,28 @@ class Ec2ContainerManagerTest {
         verify(harness.builder).withPortBinding(22, 2202);
     }
 
+    @Test
+    void launchRecordsOccupiedCandidatesWhenTheSshPortRangeIsExhausted() throws Exception {
+        LaunchHarness harness = launchHarness();
+        when(harness.portAllocator.allocate(anyInt(), anyInt(), any(IntConsumer.class)))
+                .thenAnswer(invocation -> {
+                    IntConsumer observer = invocation.getArgument(2);
+                    observer.accept(2200);
+                    observer.accept(2201);
+                    throw new RuntimeException("No free port available in range 2200-2201");
+                });
+        Instance instance = instance("i-ports-exhausted");
+
+        harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");
+
+        awaitUntil(() -> "terminated".equals(instance.getState().getName()), Duration.ofSeconds(2));
+        assertTrue(instance.getLaunchEvents().stream().anyMatch(event -> "retrying".equals(event.getStatus())
+                && event.getMessage().contains("2200")));
+        assertTrue(instance.getLaunchEvents().stream().anyMatch(event -> "retrying".equals(event.getStatus())
+                && event.getMessage().contains("2201")));
+        assertEquals("No free port available in range 2200-2201", instance.getStateReasonMessage());
+    }
+
     @Test
     void launchReleasesPortAndCleansUpContainerAfterNonRetryableStartFailure() throws Exception {
         LaunchHarness harness = launchHarness();
@@ -1155,7 +1269,8 @@ class Ec2ContainerManagerTest {
         LaunchHarness harness = launchHarness();
         CountDownLatch createEntered = new CountDownLatch(1);
         CountDownLatch allowCreate = new CountDownLatch(1);
-        when(harness.lifecycleManager.create(any(ContainerSpec.class))).thenAnswer(invocation -> {
+        when(harness.lifecycleManager.create(any(ContainerSpec.class), nullable(String.class),
+                any(ContainerLifecycleManager.ContainerCreationObserver.class))).thenAnswer(invocation -> {
             createEntered.countDown();
             assertTrue(allowCreate.await(2, TimeUnit.SECONDS));
             return TEST_CONTAINER_ID;
@@ -1218,7 +1333,7 @@ class Ec2ContainerManagerTest {
         verify(copy).withRemotePath("/var/lib");
         verify(copy, never()).withRemotePath("/tmp");
         assertTrue(harness.executedCommands.stream()
-                .anyMatch(command -> Arrays.equals(command, new String[]{"/var/lib/user-data.sh"})));
+                .anyMatch(command -> Arrays.equals(command, Ec2ContainerManager.userDataExecutionCommand())));
     }
 
     @Test
@@ -1230,7 +1345,8 @@ class Ec2ContainerManagerTest {
         ExecutorService callerExecutor = Executors.newSingleThreadExecutor();
         CountDownLatch createEntered = new CountDownLatch(1);
         CountDownLatch releaseCreate = new CountDownLatch(1);
-        when(harness.lifecycleManager().create(any(ContainerSpec.class))).thenAnswer(invocation -> {
+        when(harness.lifecycleManager().create(any(ContainerSpec.class), nullable(String.class),
+                any(ContainerLifecycleManager.ContainerCreationObserver.class))).thenAnswer(invocation -> {
             createEntered.countDown();
             releaseCreate.await(2, TimeUnit.SECONDS);
             throw new RuntimeException("test launch failure");
@@ -1351,16 +1467,7 @@ class Ec2ContainerManagerTest {
         verify(harness.dockerClient, never()).copyArchiveToContainerCmd(TEST_CONTAINER_ID);
     }
 
-    private static final String[] SSHD_INSTALL_PROBE_CMD = {"sh", "-c",
-            "if ! command -v sshd >/dev/null 2>&1 || ! command -v scp >/dev/null 2>&1; then"
-            + "  if command -v dnf >/dev/null 2>&1; then dnf install -y openssh-server openssh-clients >/dev/null 2>&1;"
-            + "  elif command -v yum >/dev/null 2>&1; then yum install -y openssh-server openssh-clients >/dev/null 2>&1;"
-            + "  elif command -v apt-get >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server openssh-client >/dev/null 2>&1;"
-            + "  elif command -v apk >/dev/null 2>&1; then apk add --no-cache openssh >/dev/null 2>&1;"
-            + "  fi;"
-            + "fi;"
-            + "command -v sshd >/dev/null 2>&1 || exit 1;"
-            + "command -v scp >/dev/null 2>&1 || exit 2"};
+    private static final String[] SSHD_INSTALL_PROBE_CMD = Ec2ContainerManager.sshdInstallProbeCommand();
 
     @Test
     void launchWhenSshdInstallFails_doesNotAttemptKeygenOrStart() throws Exception {
@@ -1459,6 +1566,8 @@ class Ec2ContainerManagerTest {
     }
 
     private static LaunchHarness launchHarness(ExecutorService executor, Duration userDataTimeout) {
+        Ec2ContainerManager.sshBannerAttempts = 1;
+        Ec2ContainerManager.sshBannerPollMillis = 1;
         ContainerBuilder containerBuilder = mock(ContainerBuilder.class);
         ContainerBuilder.Builder builder = mock(ContainerBuilder.Builder.class, withSettings().defaultAnswer(RETURNS_SELF));
         when(containerBuilder.newContainer(anyString())).thenReturn(builder);
@@ -1466,6 +1575,13 @@ class Ec2ContainerManagerTest {
 
         ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
         when(lifecycleManager.create(any(ContainerSpec.class))).thenReturn(TEST_CONTAINER_ID);
+        when(lifecycleManager.create(any(ContainerSpec.class), nullable(String.class),
+                any(ContainerLifecycleManager.ContainerCreationObserver.class))).thenAnswer(invocation -> {
+                    ContainerLifecycleManager.ContainerCreationObserver observer = invocation.getArgument(2);
+                    observer.containerCreationStarted();
+                    observer.containerCreated(TEST_CONTAINER_ID);
+                    return TEST_CONTAINER_ID;
+                });
         when(lifecycleManager.isContainerRunning(TEST_CONTAINER_ID)).thenReturn(true);
 
         DockerHostResolver dockerHostResolver = mock(DockerHostResolver.class);
@@ -1473,7 +1589,7 @@ class Ec2ContainerManagerTest {
         ContainerReachableEndpoint reachableEndpoint = mock(ContainerReachableEndpoint.class);
         when(reachableEndpoint.baseUrl()).thenReturn("http://localhost.floci.io:4680");
         PortAllocator portAllocator = mock(PortAllocator.class);
-        when(portAllocator.allocate(anyInt(), anyInt())).thenReturn(2201);
+        when(portAllocator.allocate(anyInt(), anyInt(), any(IntConsumer.class))).thenReturn(2201);
 
         EmulatorConfig config = mock(EmulatorConfig.class);
         EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);
@@ -1525,7 +1641,8 @@ class Ec2ContainerManagerTest {
                         reachableEndpoint,
                         executor,
                         userDataTimeout);
-        return new LaunchHarness(manager, lifecycleManager, dockerClient, metadataServer, logStreamer, builder,
+        return new LaunchHarness(manager, lifecycleManager, dockerClient, metadataServer, logStreamer,
+                containerBuilder, builder,
                 portAllocator, portForwardManager, config, vpcNetworkManager, new CopyOnWriteArrayList<>());
     }
 
@@ -1676,7 +1793,7 @@ class Ec2ContainerManagerTest {
         });
         when(execCreate.exec()).thenAnswer(invocation -> {
             String[] command = currentCommand.get();
-            return command != null && command.length == 1 && "/var/lib/user-data.sh".equals(command[0])
+            return Arrays.equals(command, Ec2ContainerManager.userDataExecutionCommand())
                     ? userDataExec : metadataExec;
         });
 
@@ -1753,12 +1870,19 @@ class Ec2ContainerManagerTest {
     void launchRunsInstanceAsMetadataOnlyWhenNoDockerDaemonIsReachable() throws Exception {
         LaunchHarness harness = launchHarness();
         stubDockerPing(harness.dockerClient(), false);
+        AtomicInteger persistenceCalls = new AtomicInteger();
+        harness.manager.setInstancePersister(ignored -> persistenceCalls.incrementAndGet());
 
         Instance instance = instance("i-nodocker");
 
         harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");
 
         assertEquals("running", instance.getState().getName());
+        assertEquals("failed", instance.getLaunchStatus());
+        assertEquals("impaired", instance.getSshStatus());
+        assertTrue(instance.getSshStatusMessage().contains("Docker daemon is unavailable"));
+        assertEquals("creating-container", instance.getLaunchEvents().getLast().getStage());
+        assertTrue(persistenceCalls.get() >= 2);
         assertNull(instance.getDockerContainerId());
         verify(harness.lifecycleManager(), never()).create(any(ContainerSpec.class));
         verify(harness.metadataServer(), never()).registerContainer(anyString(), anyString(), any());
@@ -1770,7 +1894,8 @@ class Ec2ContainerManagerTest {
         PingCmd ping = mock(PingCmd.class);
         when(harness.dockerClient().pingCmd()).thenReturn(ping);
         doNothing().doThrow(new RuntimeException("No such file or directory")).when(ping).exec();
-        when(harness.lifecycleManager().create(any(ContainerSpec.class)))
+        when(harness.lifecycleManager().create(any(ContainerSpec.class), nullable(String.class),
+                any(ContainerLifecycleManager.ContainerCreationObserver.class)))
                 .thenThrow(new RuntimeException("No such file or directory"));
 
         Instance instance = instance("i-dockergone");
@@ -1830,6 +1955,7 @@ class Ec2ContainerManagerTest {
                                  DockerClient dockerClient,
                                   Ec2MetadataServer metadataServer,
                                   ContainerLogStreamer logStreamer,
+                                  ContainerBuilder containerBuilder,
                                   ContainerBuilder.Builder builder,
                                  PortAllocator portAllocator,
                                  Ec2PortForwardManager portForwardManager,
@@ -1856,7 +1982,7 @@ class Ec2ContainerManagerTest {
             });
             when(execCreate.exec()).thenAnswer(invocation -> {
                 String[] command = currentCommand.get();
-                if (command != null && command.length == 1 && "/var/lib/user-data.sh".equals(command[0])) {
+                if (Arrays.equals(command, Ec2ContainerManager.userDataExecutionCommand())) {
                     return userDataExec;
                 }
                 return metadataExec;

src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ImageCatalogTest.java

@@ -66,9 +66,21 @@ class Ec2ImageCatalogTest {
         ResolvedAmiImage image = amiImageResolver.resolveImage("ami-amazonlinux2023");
 
         assertEquals("floci/ami-amazonlinux:2023", image.dockerImage());
-        assertEquals(ResolvedAmiImage.SYSTEMD_RUNTIME, image.guestRuntime());
+        assertEquals(ResolvedAmiImage.SYSTEMD_RUNTIME, image.initType());
+        assertEquals("root", image.sshUser());
+        assertTrue(image.sshPreinstalled());
+        assertEquals("sshd", image.sshServiceName());
         assertFalse(image.cloudInit());
         assertTrue(image.systemd());
+        assertEquals("public.ecr.aws/amazonlinux/amazonlinux:2023", image.fallbackDockerImage());
+    }
+
+    @Test
+    void everyLocalCatalogImageHasAPullableFallback() {
+        imageCatalog.images().stream()
+                .filter(image -> image.dockerImage.startsWith("floci/"))
+                .forEach(image -> assertTrue(image.fallbackDockerImage != null
+                        && !image.fallbackDockerImage.isBlank(), image.imageId));
     }
 
     @Test
@@ -78,9 +90,12 @@ class Ec2ImageCatalogTest {
     }
 
     @Test
-    void resolverUsesFedoraAndOpenSuseImages() {
-        assertEquals("registry.fedoraproject.org/fedora:42", amiImageResolver.resolve("ami-fedora42"));
-        assertEquals("registry.opensuse.org/opensuse/leap:15.6", amiImageResolver.resolve("ami-opensuse-leap156"));
+    void resolverUsesFedoraAndLocalSshReadyImages() {
+        assertEquals("floci/ami-ubuntu:20.04", amiImageResolver.resolve("ami-0abcdef1234567892"));
+        assertEquals("floci/ami-fedora:42", amiImageResolver.resolve("ami-fedora42"));
+        assertEquals("floci/ami-opensuse:15.6", amiImageResolver.resolve("ami-opensuse-leap156"));
+        assertTrue(amiImageResolver.resolveImage("ami-0abcdef1234567892").systemd());
+        assertTrue(amiImageResolver.resolveImage("ami-fedora42").systemd());
     }
 
     @Test

src/test/java/io/github/hectorvent/floci/services/ec2/Ec2IntegrationTest.java

@@ -2215,7 +2215,7 @@ class Ec2IntegrationTest {
             .formParam("LaunchTemplateName", "sample-template")
             .formParam("LaunchTemplateData.ImageId", "ami-0abcdef1234567890")
             .formParam("LaunchTemplateData.InstanceType", "t3.micro")
-            .formParam("LaunchTemplateData.KeyName", "test-key")
+            .formParam("LaunchTemplateData.KeyName", keyPairName)
             .formParam("LaunchTemplateData.IamInstanceProfile.Name", "sample-profile")
             .formParam("LaunchTemplateData.SecurityGroupId.1", securityGroupId)
             .formParam("LaunchTemplateData.UserData", launchTemplateUserData)
@@ -2328,7 +2328,7 @@ class Ec2IntegrationTest {
             .formParam("VersionDescription", "updated by test")
             .formParam("LaunchTemplateData.ImageId", "ami-0abcdef1234567890")
             .formParam("LaunchTemplateData.InstanceType", "t3.small")
-            .formParam("LaunchTemplateData.KeyName", "test-key")
+            .formParam("LaunchTemplateData.KeyName", keyPairName)
             .formParam("LaunchTemplateData.IamInstanceProfile.Name", "sample-profile-v2")
             .formParam("LaunchTemplateData.SecurityGroupId.1", securityGroupId)
             .formParam("LaunchTemplateData.UserData", launchTemplateVersionUserData)
@@ -2468,7 +2468,7 @@ class Ec2IntegrationTest {
             .statusCode(200)
             .body("RunInstancesResponse.instancesSet.item.imageId", equalTo("ami-sample-override"))
             .body("RunInstancesResponse.instancesSet.item.instanceType", equalTo("c7g.large"))
-            .body("RunInstancesResponse.instancesSet.item.keyName", equalTo("test-key"))
+            .body("RunInstancesResponse.instancesSet.item.keyName", equalTo(keyPairName))
             .body("RunInstancesResponse.instancesSet.item.iamInstanceProfile.arn",
                     equalTo("arn:aws:iam::000000000000:instance-profile/sample-profile-v2"))
             .body("RunInstancesResponse.instancesSet.item.groupSet.item.groupId", equalTo(securityGroupId))
@@ -2861,7 +2861,7 @@ class Ec2IntegrationTest {
             .formParam("InstanceType", "t2.micro")
             .formParam("MinCount", "1")
             .formParam("MaxCount", "1")
-            .formParam("KeyName", "test-key")
+            .formParam("KeyName", keyPairName)
             .formParam("SubnetId", subnetId)
             .formParam("SecurityGroupId.1", securityGroupId)
             .header("Authorization", AUTH_HEADER)
@@ -2872,10 +2872,30 @@ class Ec2IntegrationTest {
             .body("RunInstancesResponse.instancesSet.item.instanceId", startsWith("i-"))
             .body("RunInstancesResponse.instancesSet.item.instanceState.name", equalTo("pending"))
             .body("RunInstancesResponse.instancesSet.item.instanceType", equalTo("t2.micro"))
-            .body("RunInstancesResponse.instancesSet.item.keyName", equalTo("test-key"))
+            .body("RunInstancesResponse.instancesSet.item.keyName", equalTo(keyPairName))
             .extract().path("RunInstancesResponse.instancesSet.item.instanceId");
     }
 
+    @Test
+    @Order(80)
+    void runInstancesUsesSecurityGroupsFromNetworkInterface() {
+        given()
+            .formParam("Action", "RunInstances")
+            .formParam("ImageId", "ami-0abcdef1234567890")
+            .formParam("InstanceType", "t2.micro")
+            .formParam("MinCount", "1")
+            .formParam("MaxCount", "1")
+            .formParam("NetworkInterface.1.DeviceIndex", "0")
+            .formParam("NetworkInterface.1.SubnetId", subnetId)
+            .formParam("NetworkInterface.1.SecurityGroupId.1", securityGroupId)
+            .header("Authorization", AUTH_HEADER)
+        .when()
+            .post("/")
+        .then()
+            .statusCode(200)
+            .body("RunInstancesResponse.instancesSet.item.groupSet.item.groupId", equalTo(securityGroupId));
+    }
+
     @Test
     @Order(80)
     void runInstancesWithArm64ImageDescribesArm64Architecture() {
@@ -3692,7 +3712,7 @@ class Ec2IntegrationTest {
             .formParam("InstanceType", "t2.micro")
             .formParam("MinCount", "5")
             .formParam("MaxCount", "5")
-            .formParam("KeyName", "test-key")
+            .formParam("KeyName", keyPairName)
             .formParam("SubnetId", subnetId)
             .formParam("SecurityGroupId.1", securityGroupId)
             .header("Authorization", AUTH_HEADER)

src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java

@@ -1510,6 +1510,31 @@ class Ec2ServiceTest {
         assertNotEquals(first.getKeyMaterial(), second.getKeyMaterial());
         assertNotEquals(first.getPublicKey(), second.getPublicKey());
         assertNotEquals(first.getKeyFingerprint(), second.getKeyFingerprint());
+        KeyPair persisted = service.describeKeyPairs("us-east-1", List.of("usable-key"), List.of()).getFirst();
+        assertNull(persisted.getKeyMaterial());
+        assertEquals(first.getPublicKey(), persisted.getPublicKey());
+    }
+
+    @Test
+    void startupClearsLegacyPersistedPrivateKeyMaterialAcrossAccounts() {
+        AccountAwareStorageBackend<KeyPair> keyPairStore = AccountAwareStorageBackend.inMemory("000000000000");
+        KeyPair legacy = new KeyPair();
+        legacy.setKeyPairId("key-legacy");
+        legacy.setKeyName("legacy-key");
+        legacy.setKeyMaterial("-----BEGIN RSA PRIVATE KEY-----\nsecret\n-----END RSA PRIVATE KEY-----");
+        legacy.setPublicKey("ssh-rsa public");
+        legacy.setRegion("us-east-1");
+        keyPairStore.putForAccount("111111111111", "us-east-1::key-legacy", legacy);
+        Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),
+                mock(Ec2PortForwardManager.class),
+                mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class), new Ec2InstanceTypeCatalog(),
+                new InMemoryStorageFactory(Map.of("ec2-key-pairs.json", keyPairStore)));
+
+        service.restoreMetadataRegistrations();
+
+        KeyPair restored = keyPairStore.getForAccount("111111111111", "us-east-1::key-legacy").orElseThrow();
+        assertNull(restored.getKeyMaterial());
+        assertEquals("ssh-rsa public", restored.getPublicKey());
     }
 
     @Test
@@ -1856,7 +1881,7 @@ class Ec2ServiceTest {
         AmiImageResolver resolver = mock(AmiImageResolver.class);
         when(resolver.resolveImage("ami-arm-source"))
                 .thenReturn(new ResolvedAmiImage("arm-image", ResolvedAmiImage.DEFAULT_RUNTIME, false,
-                        "linux/arm64"));
+                        "linux/arm64", null, "public", "root", false, "sshd"));
         Ec2Service service = liveService(capturingContainerManager(), resolver, catalog);
         Reservation sourceReservation = service.runInstances("us-east-1", "ami-arm-source", "t4g.medium",
                 1, 1, null, List.of(), null, null, List.of(), null, null);
@@ -2065,6 +2090,135 @@ class Ec2ServiceTest {
         verify(vpcNetworks, times(2)).reservePrivateIp("us-east-1", "subnet-a", "10.0.1.10");
     }
 
+    @Test
+    void restorePromotesAPendingInstanceWhoseContainerIsAlreadyRunning() {
+        AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
+        Instance instance = persistedInstance("i-pending", "10.0.1.10", InstanceState.pending());
+        instance.setDockerContainerId("container-id");
+        instanceStore.put("us-east-1::i-pending", instance);
+
+        Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);
+        when(containerManager.restoreMetadataRegistration(instance)).thenReturn(true);
+        VpcNetworkManager vpcNetworks = mock(VpcNetworkManager.class);
+        when(vpcNetworks.enabled()).thenReturn(true);
+        Ec2Service service = new Ec2Service(mockConfig(false), containerManager,
+                mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+                new Ec2InstanceTypeCatalog(),
+                new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), vpcNetworks);
+
+        service.restoreMetadataRegistrations();
+
+        assertEquals("running", instanceStore.get("us-east-1::i-pending").orElseThrow().getState().getName());
+    }
+
+    @Test
+    void restoreFailsAPendingInstanceWithoutARecordedContainer() {
+        AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
+        Instance instance = persistedInstance("i-pending", "10.0.1.10", InstanceState.pending());
+        instanceStore.put("us-east-1::i-pending", instance);
+        Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),
+                mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+                new Ec2InstanceTypeCatalog(),
+                new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)),
+                mock(VpcNetworkManager.class));
+
+        service.restoreMetadataRegistrations();
+
+        Instance restored = instanceStore.get("us-east-1::i-pending").orElseThrow();
+        assertEquals("terminated", restored.getState().getName());
+        assertEquals("failed", restored.getLaunchStatus());
+        assertEquals("impaired", restored.getSshStatus());
+        assertEquals("startup-reconciliation", restored.getLaunchEvents().getLast().getStage());
+        assertEquals("failed", restored.getLaunchEvents().getLast().getStatus());
+    }
+
+    @Test
+    void restoreFailsPendingInstancesAcrossAccountPartitions() {
+        AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
+        Instance instance = persistedInstance("i-other-account", "10.0.1.10", InstanceState.pending());
+        instance.setAccountId("111111111111");
+        instanceStore.putForAccount("111111111111", "us-east-1::i-other-account", instance);
+        Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),
+                mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+                new Ec2InstanceTypeCatalog(),
+                new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)),
+                mock(VpcNetworkManager.class));
+
+        service.restoreMetadataRegistrations();
+
+        Instance restored = instanceStore.getForAccount(
+                "111111111111", "us-east-1::i-other-account").orElseThrow();
+        assertEquals("terminated", restored.getState().getName());
+        assertEquals("failed", restored.getLaunchStatus());
+        assertTrue(instanceStore.getForAccount(
+                "000000000000", "us-east-1::i-other-account").isEmpty());
+    }
+
+    @Test
+    void restoreFailsARunningInstanceWhoseLaunchWasInterrupted() {
+        AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
+        Instance instance = persistedInstance("i-interrupted", "10.0.1.10", InstanceState.running());
+        instance.setLaunchStatus("in-progress");
+        instanceStore.put("us-east-1::i-interrupted", instance);
+        Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),
+                mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+                new Ec2InstanceTypeCatalog(),
+                new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)),
+                mock(VpcNetworkManager.class));
+
+        service.restoreMetadataRegistrations();
+
+        Instance restored = instanceStore.get("us-east-1::i-interrupted").orElseThrow();
+        assertEquals("terminated", restored.getState().getName());
+        assertEquals("failed", restored.getLaunchStatus());
+        assertEquals("startup-reconciliation", restored.getLaunchEvents().getLast().getStage());
+    }
+
+    @Test
+    void describeInstanceStatusIncludesFailedLaunchOnlyWhenRequested() {
+        AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
+        Instance instance = persistedInstance("i-failed", "10.0.1.10", InstanceState.terminated());
+        instance.setLaunchStatus("failed");
+        instance.setSshStatus("impaired");
+        instance.setSshStatusMessage("No free port available in range 2200-2209");
+        instanceStore.put("us-east-1::i-failed", instance);
+        Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),
+                mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+                new Ec2InstanceTypeCatalog(),
+                new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)));
+
+        assertTrue(service.describeInstanceStatus("us-east-1", List.of("i-failed"), false).isEmpty());
+        List<Instance> statuses = service.describeInstanceStatus("us-east-1", List.of("i-failed"), true);
+        assertEquals(1, statuses.size());
+        assertEquals("impaired", statuses.getFirst().getSshStatus());
+        assertEquals("No free port available in range 2200-2209", statuses.getFirst().getSshStatusMessage());
+    }
+
+    @Test
+    void describeInstanceStatusClearsRecoveredSshLaunchFailure() {
+        AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
+        Instance instance = persistedInstance("i-recovered", "10.0.1.10", InstanceState.running());
+        instance.setLaunchStatus("failed");
+        instance.setSshStatus("impaired");
+        instance.setStateReasonCode("Client.SSHUnavailable");
+        instance.setStateReasonMessage("connection refused");
+        instanceStore.put("us-east-1::i-recovered", instance);
+        Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);
+        when(containerManager.checkSshReadiness(instance))
+                .thenReturn(new Ec2ContainerManager.OperationResult(true, "SSH banner received: SSH-2.0-test"));
+        Ec2Service service = new Ec2Service(mockConfig(false), containerManager,
+                mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+                new Ec2InstanceTypeCatalog(),
+                new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)));
+
+        Instance status = service.describeInstanceStatus("us-east-1", List.of("i-recovered"), false).getFirst();
+
+        assertEquals("ready", status.getLaunchStatus());
+        assertEquals("ok", status.getSshStatus());
+        assertNull(status.getStateReasonCode());
+        assertNull(status.getStateReasonMessage());
+    }
+
     private static Instance persistedInstance(String instanceId, String privateIp, InstanceState state) {
         Instance instance = new Instance();
         instance.setInstanceId(instanceId);

src/test/java/io/github/hectorvent/floci/services/ecs/EcsEc2BridgeIntegrationTest.java

@@ -21,17 +21,17 @@ class EcsEc2BridgeIntegrationTest {
     }
 
     @Test
-    void runTaskAndCreateServiceRejectFargateAndNonBridgeTasks() {
+    void runTaskAndCreateServiceKeepEc2BridgeRestrictions() {
         call("RegisterTaskDefinition", "{\"family\":\"ec2-bridge\",\"networkMode\":\"bridge\","
                 + "\"containerDefinitions\":[{\"name\":\"app\",\"image\":\"nginx\"}]}", 200);
         call("RunTask", "{\"taskDefinition\":\"ec2-bridge\",\"launchType\":\"FARGATE\"}", 400)
-                .body("message", containsString("EC2 launch type"));
+                .body("message", containsString("awsvpc"));
         call("RunTask", "{\"taskDefinition\":\"ec2-bridge\",\"capacityProviderStrategy\":["
                 + "{\"capacityProvider\":\"FARGATE\",\"weight\":1}]}", 400)
-                .body("message", containsString("FARGATE capacity providers"));
+                .body("message", containsString("awsvpc"));
         call("CreateService", "{\"serviceName\":\"ec2-bridge\",\"taskDefinition\":\"ec2-bridge\","
                 + "\"desiredCount\":0,\"launchType\":\"FARGATE\"}", 400)
-                .body("message", containsString("EC2 launch type"));
+                .body("message", containsString("awsvpc"));
 
         call("RegisterTaskDefinition", "{\"family\":\"not-bridge\",\"networkMode\":\"awsvpc\","
                 + "\"containerDefinitions\":[{\"name\":\"app\",\"image\":\"nginx\"}]}", 200);

src/test/java/io/github/hectorvent/floci/services/ecs/EcsEc2ServiceReconciliationTest.java

@@ -0,0 +1,159 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.CreateTaskSetRequest;
+import io.github.hectorvent.floci.services.ecs.model.EcsTask;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.ecs.model.TaskSet;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class EcsEc2ServiceReconciliationTest {
+
+    private static final String REGION = "us-east-1";
+
+    @Test
+    void taskDefinitionUpdateReplacesAndDrainsEc2ServiceTasks() {
+        EcsService service = newMockModeService();
+        service.createCluster("roll-cluster", REGION);
+        TaskDefinition revisionOne = registerTaskDefinition(service, "roll", "app:1");
+        service.createService("roll-cluster", "web", revisionOne.getTaskDefinitionArn(), 2,
+                LaunchType.EC2, List.of(), null, REGION);
+        service.reconcileServices();
+
+        TaskDefinition revisionTwo = registerTaskDefinition(service, "roll", "app:2");
+        service.updateService("roll-cluster", "web", revisionTwo.getTaskDefinitionArn(), null, null, REGION);
+
+        service.reconcileServices();
+        assertEquals(4, runningTasks(service).size(), "replacements start before old tasks drain");
+        service.reconcileServices();
+
+        List<EcsTask> running = runningTasks(service);
+        assertEquals(2, running.size());
+        assertTrue(running.stream().allMatch(task -> revisionTwo.getTaskDefinitionArn()
+                .equals(task.getTaskDefinitionArn())));
+    }
+
+    @Test
+    void forceDeleteDrainsAllEc2ServiceTasks() {
+        EcsService service = newMockModeService();
+        service.createCluster("delete-cluster", REGION);
+        TaskDefinition taskDefinition = registerTaskDefinition(service, "delete", "app:1");
+        service.createService("delete-cluster", "worker", taskDefinition.getTaskDefinitionArn(), 2,
+                LaunchType.EC2, List.of(), null, REGION);
+        service.reconcileServices();
+
+        service.deleteService("delete-cluster", "worker", true, REGION);
+
+        assertEquals(0, runningTasks(service).size());
+    }
+
+    @Test
+    void codeDeployTaskSetsAreNotSupplementedByNormalEc2ServiceTasks() {
+        EcsService service = newMockModeService();
+        service.createCluster("blue-green-cluster", REGION);
+        TaskDefinition taskDefinition = registerTaskDefinition(service, "blue-green", "app:1");
+        service.createService("blue-green-cluster", "web", taskDefinition.getTaskDefinitionArn(), 2,
+                LaunchType.EC2, List.of(), null, null, null, "CODE_DEPLOY", null, REGION);
+        CreateTaskSetRequest request = new CreateTaskSetRequest();
+        request.setCluster("blue-green-cluster");
+        request.setService("web");
+        request.setTaskDefinition(taskDefinition.getTaskDefinitionArn());
+        request.setLaunchType(LaunchType.EC2);
+        TaskSet taskSet = service.createTaskSet(request, REGION);
+
+        service.reconcileServices();
+
+        assertEquals(2, taskSet.getRunningCount());
+        assertEquals(2, runningTasks(service).size());
+
+        service.deleteService("blue-green-cluster", "web", true, REGION);
+
+        assertEquals(0, runningTasks(service).size(), "force delete drains CodeDeploy task-set tasks");
+    }
+
+    @Test
+    void recreatingAnEc2CodeDeployServiceDoesNotReuseDeletedTaskSets() {
+        EcsService service = newMockModeService();
+        service.createCluster("ec2-recreate-cluster", REGION);
+        TaskDefinition taskDefinition = registerTaskDefinition(service, "ec2-recreate", "app:1");
+        service.createService("ec2-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.EC2, List.of(), null, null, null, "CODE_DEPLOY", null, REGION);
+        CreateTaskSetRequest request = new CreateTaskSetRequest();
+        request.setCluster("ec2-recreate-cluster");
+        request.setService("blue");
+        request.setTaskDefinition(taskDefinition.getTaskDefinitionArn());
+        request.setLaunchType(LaunchType.EC2);
+        service.createTaskSet(request, REGION);
+
+        service.deleteService("ec2-recreate-cluster", "blue", true, REGION);
+        service.createService("ec2-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.EC2, List.of(), null, null, null, "CODE_DEPLOY", null, REGION);
+        service.reconcileServices();
+
+        assertEquals(List.of(), service.describeTaskSets("ec2-recreate-cluster", "blue", null, REGION));
+        assertEquals(List.of(), runningTasks(service));
+    }
+
+    private static List<EcsTask> runningTasks(EcsService service) {
+        return service.describeTasks(null, service.listTasks(null, null, null, null, REGION), REGION).stream()
+                .filter(task -> "RUNNING".equals(task.getLastStatus()))
+                .toList();
+    }
+
+    private static TaskDefinition registerTaskDefinition(EcsService service, String family, String image) {
+        ContainerDefinition container = new ContainerDefinition();
+        container.setName("app");
+        container.setImage(image);
+        return service.registerTaskDefinition(family, List.of(container), null, null, null,
+                null, null, List.of(), REGION);
+    }
+
+    private static EcsService newMockModeService() {
+        EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+        when(config.services().ecs().mock()).thenReturn(true);
+        when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+        EcsService service = new EcsService(
+                new RegionResolver(REGION, "000000000000"),
+                mock(EcsContainerManager.class),
+                config,
+                mock(EcsLoadBalancerRegistrar.class),
+                new InMemoryStorageFactory(),
+                null);
+        service.initializeStorage();
+        return service;
+    }
+
+    private static final class InMemoryStorageFactory extends StorageFactory {
+        private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
+
+        private InMemoryStorageFactory() {
+            super(null, null);
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+                                                          TypeReference<Map<String, V>> typeReference) {
+            return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,
+                    ignored -> AccountAwareStorageBackend.inMemory("000000000000"));
+        }
+    }
+}

src/test/java/io/github/hectorvent/floci/services/ecs/EcsLoadBalancersIntegrationTest.java

@@ -56,6 +56,13 @@ class EcsLoadBalancersIntegrationTest {
         return targetGroup.getTargetGroupArn();
     }
 
+    private String createInstanceTargetGroup(String name) {
+        TargetGroup targetGroup = elbV2Service.createTargetGroup("us-east-1", name, "HTTP", "HTTP1",
+                8080, "vpc-default", "instance", null, null, false, null, null, null, null, null, null,
+                null, Map.of());
+        return targetGroup.getTargetGroupArn();
+    }
+
     @Test
     void createServiceParsesAndEchoesLoadBalancers() {
         seedClusterAndTaskDef("lb-cluster-1", "lb-td-1");
@@ -147,6 +154,29 @@ class EcsLoadBalancersIntegrationTest {
                 .body("message", org.hamcrest.Matchers.containsString("does not expose port 9999"));
     }
 
+    @Test
+    void fargateServiceRequiresAnIpTargetGroup() {
+        String cluster = "fargate-lb-cluster";
+        String family = "fargate-lb-task";
+        call("CreateCluster", "{\"clusterName\":\"" + cluster + "\"}");
+        call("RegisterTaskDefinition", "{\"family\":\"" + family + "\","
+                + "\"requiresCompatibilities\":[\"FARGATE\"],\"networkMode\":\"awsvpc\","
+                + "\"cpu\":\"256\",\"memory\":\"512\",\"containerDefinitions\":[{"
+                + "\"name\":\"web\",\"image\":\"nginx\",\"portMappings\":[{\"containerPort\":8080}]}]}");
+        String tgArn = createInstanceTargetGroup("fargate-instance-tg");
+
+        given().contentType(CT).header("X-Amz-Target", TARGET + "CreateService")
+                .body("{\"cluster\":\"" + cluster + "\",\"serviceName\":\"fargate-lb\","
+                        + "\"taskDefinition\":\"" + family + "\",\"desiredCount\":0,"
+                        + "\"launchType\":\"FARGATE\",\"networkConfiguration\":{\"awsvpcConfiguration\":{"
+                        + "\"subnets\":[\"subnet-default-us-east-1-a\"]}},\"loadBalancers\":[{"
+                        + "\"targetGroupArn\":\"" + tgArn + "\",\"containerName\":\"web\","
+                        + "\"containerPort\":8080}]}")
+                .when().post("/")
+                .then().statusCode(400)
+                .body("message", org.hamcrest.Matchers.containsString("target type ip"));
+    }
+
     @Test
     void serviceWithLoadBalancersIsStableInMockMode() {
         // Mock mode launches no real containers; the registrar must no-op cleanly.

src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRolloutTest.java

@@ -7,11 +7,18 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageFactory;
 import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;
+import io.github.hectorvent.floci.services.ecs.model.AwsVpcConfiguration;
 import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.CreateTaskSetRequest;
+import io.github.hectorvent.floci.services.ecs.model.EcsLoadBalancer;
 import io.github.hectorvent.floci.services.ecs.model.EcsTask;
 import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;
+import io.github.hectorvent.floci.services.ecs.model.NetworkMode;
 import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
 import io.github.hectorvent.floci.services.ecs.model.TaskStatus;
+import io.github.hectorvent.floci.services.ecs.model.TaskSet;
 import org.junit.jupiter.api.Test;
 
 import java.util.HashMap;
@@ -23,9 +30,12 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
 import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -92,6 +102,206 @@ class EcsServiceRolloutTest {
         assertEquals(rev1.getTaskDefinitionArn(), svc.getTaskDefinition());
     }
 
+    @Test
+    void ec2TaskSetPlacesTasksRegistersGreenTargetsAndDrainsThem() {
+        EcsLoadBalancerRegistrar registrar = mock(EcsLoadBalancerRegistrar.class);
+        EcsService service = newDockerModeService(registrar);
+        service.createCluster("blue-green-cluster", REGION);
+        TaskDefinition taskDefinition = registerTaskDef(service, "blue-green-task", "app:1");
+        service.createService("blue-green-cluster", "blue-green-service", taskDefinition.getTaskDefinitionArn(),
+                2, LaunchType.EC2, List.of(), null, null, null, "CODE_DEPLOY", null, REGION);
+
+        EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+        loadBalancer.setTargetGroupArn("arn:aws:elasticloadbalancing:us-east-1:000000000000:targetgroup/green/id");
+        loadBalancer.setContainerName("app");
+        loadBalancer.setContainerPort(80);
+        CreateTaskSetRequest request = new CreateTaskSetRequest();
+        request.setCluster("blue-green-cluster");
+        request.setService("blue-green-service");
+        request.setTaskDefinition(taskDefinition.getTaskDefinitionArn());
+        request.setLaunchType(LaunchType.EC2);
+        request.setScaleValue(100.0);
+        request.setLoadBalancers(List.of(loadBalancer));
+
+        TaskSet taskSet = service.createTaskSet(request, REGION);
+
+        assertEquals(2, taskSet.getRunningCount());
+        assertEquals("STEADY_STATE", taskSet.getStabilityStatus());
+        verify(registrar, times(2)).registerTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+
+        TaskSet scaled = service.updateTaskSet("blue-green-cluster", "blue-green-service",
+                taskSet.getTaskSetArn(), 0.0, "PERCENT", REGION);
+        assertEquals(0, scaled.getRunningCount());
+        verify(registrar, times(2)).deregisterTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+    }
+
+    @Test
+    void fargateTaskSetRunsRegistersGreenTargetsAndReconcilesStoppedTasks() {
+        EcsLoadBalancerRegistrar registrar = mock(EcsLoadBalancerRegistrar.class);
+        EcsService service = newDockerModeService(registrar);
+        service.createCluster("fargate-blue-green-cluster", REGION);
+        TaskDefinition taskDefinition = registerFargateTaskDef(service, "fargate-blue-green-task", "app:1");
+        NetworkConfiguration networkConfiguration = fargateNetworkConfiguration();
+        service.createService("fargate-blue-green-cluster", "fargate-blue-green-service",
+                taskDefinition.getTaskDefinitionArn(), 2, LaunchType.FARGATE, List.of(), networkConfiguration,
+                null, null, "CODE_DEPLOY", null, REGION);
+
+        EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+        loadBalancer.setTargetGroupArn("arn:aws:elasticloadbalancing:us-east-1:000000000000:targetgroup/green/id");
+        loadBalancer.setContainerName("app");
+        loadBalancer.setContainerPort(80);
+        CreateTaskSetRequest request = new CreateTaskSetRequest();
+        request.setCluster("fargate-blue-green-cluster");
+        request.setService("fargate-blue-green-service");
+        request.setTaskDefinition(taskDefinition.getTaskDefinitionArn());
+        request.setLaunchType(LaunchType.FARGATE);
+        request.setScaleValue(100.0);
+        request.setLoadBalancers(List.of(loadBalancer));
+
+        TaskSet taskSet = service.createTaskSet(request, REGION);
+
+        assertEquals(2, taskSet.getRunningCount());
+        assertEquals("STEADY_STATE", taskSet.getStabilityStatus());
+        verify(registrar, times(4)).registerTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+
+        EcsTask firstTask = runningTasks(service).getFirst();
+        service.stopTask("fargate-blue-green-cluster", firstTask.getTaskArn(), "test stop", REGION);
+        service.reconcileServices();
+
+        assertEquals(2, taskSet.getRunningCount());
+        verify(registrar, times(5)).registerTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+        verify(registrar).deregisterTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+
+        service.deleteTaskSet("fargate-blue-green-cluster", "fargate-blue-green-service",
+                taskSet.getTaskSetArn(), true, REGION);
+        assertEquals(2, runningTasks(service).size());
+        verify(registrar, times(3)).deregisterTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+    }
+
+    @Test
+    void createFargateCodeDeployServiceBootstrapsPrimaryBlueTaskSet() {
+        EcsLoadBalancerRegistrar registrar = mock(EcsLoadBalancerRegistrar.class);
+        EcsService service = newDockerModeService(registrar);
+        service.createCluster("fargate-bootstrap-cluster", REGION);
+        TaskDefinition taskDefinition = registerFargateTaskDef(service, "fargate-bootstrap-task", "app:1");
+        EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+        loadBalancer.setTargetGroupArn("arn:aws:elasticloadbalancing:us-east-1:000000000000:targetgroup/blue/id");
+        loadBalancer.setContainerName("app");
+        loadBalancer.setContainerPort(80);
+
+        service.createService("fargate-bootstrap-cluster", "fargate-bootstrap-service",
+                taskDefinition.getTaskDefinitionArn(), 2, LaunchType.FARGATE, List.of(loadBalancer),
+                fargateNetworkConfiguration(), null, null, "CODE_DEPLOY", null, REGION);
+
+        List<TaskSet> taskSets = service.describeTaskSets("fargate-bootstrap-cluster",
+                "fargate-bootstrap-service", null, REGION);
+        assertEquals(1, taskSets.size());
+        TaskSet taskSet = taskSets.getFirst();
+        assertEquals("PRIMARY", taskSet.getStatus());
+        assertEquals(2, taskSet.getRunningCount());
+        assertEquals("STEADY_STATE", taskSet.getStabilityStatus());
+        assertEquals(List.of(loadBalancer), taskSet.getLoadBalancers());
+        verify(registrar, times(2)).registerTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+    }
+
+    @Test
+    void recreatingAFargateCodeDeployServiceDoesNotReuseDeletedTaskSets() {
+        EcsService service = newDockerModeService(mock(EcsLoadBalancerRegistrar.class));
+        service.createCluster("fargate-recreate-cluster", REGION);
+        TaskDefinition taskDefinition = registerFargateTaskDef(service, "fargate-recreate-task", "app:1");
+        NetworkConfiguration networkConfiguration = fargateNetworkConfiguration();
+        service.createService("fargate-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.FARGATE, List.of(), networkConfiguration, null, null, "CODE_DEPLOY", null, REGION);
+
+        TaskSet oldPrimary = service.describeTaskSets("fargate-recreate-cluster", "blue", null, REGION).getFirst();
+        EcsTask oldTask = runningTasks(service).getFirst();
+        service.deleteService("fargate-recreate-cluster", "blue", true, REGION);
+        service.createService("fargate-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.FARGATE, List.of(), networkConfiguration, null, null, "CODE_DEPLOY", null, REGION);
+
+        List<TaskSet> taskSets = service.describeTaskSets("fargate-recreate-cluster", "blue", null, REGION);
+        assertEquals(1, taskSets.size());
+        assertNotEquals(oldPrimary.getTaskSetArn(), taskSets.getFirst().getTaskSetArn());
+        assertEquals("PRIMARY", taskSets.getFirst().getStatus());
+        assertEquals(1, taskSets.getFirst().getRunningCount());
+        assertEquals(1, runningTasks(service).size());
+        assertEquals("STOPPED", service.describeTasks("fargate-recreate-cluster",
+                List.of(oldTask.getTaskArn()), REGION).getFirst().getLastStatus());
+    }
+
+    @Test
+    void forceDeletingCodeDeployServiceStopsAllTaskSetContainersAndDeregistersTheirTargetsBeforeRecreate() {
+        EcsLoadBalancerRegistrar registrar = mock(EcsLoadBalancerRegistrar.class);
+        EcsContainerManager containerManager = mock(EcsContainerManager.class);
+        EcsTaskHandle blueHandle = mock(EcsTaskHandle.class);
+        EcsTaskHandle greenFirstHandle = mock(EcsTaskHandle.class);
+        EcsTaskHandle greenSecondHandle = mock(EcsTaskHandle.class);
+        EcsTaskHandle otherServiceHandle = mock(EcsTaskHandle.class);
+        EcsTaskHandle recreatedHandle = mock(EcsTaskHandle.class);
+        when(containerManager.startTask(any(), any(), any(), eq(REGION))).thenReturn(
+                blueHandle, greenFirstHandle, greenSecondHandle, otherServiceHandle, recreatedHandle);
+        EcsService service = newDockerModeService(registrar, containerManager);
+        service.createCluster("delete-recreate-cluster", REGION);
+        TaskDefinition taskDefinition = registerFargateTaskDef(service, "delete-recreate-task", "app:1");
+        NetworkConfiguration networkConfiguration = fargateNetworkConfiguration();
+
+        EcsLoadBalancer blueLoadBalancer = loadBalancer("blue");
+        EcsLoadBalancer greenLoadBalancer = loadBalancer("green");
+        EcsLoadBalancer otherLoadBalancer = loadBalancer("other");
+        service.createService("delete-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.FARGATE, List.of(blueLoadBalancer), networkConfiguration, null, null,
+                "CODE_DEPLOY", null, REGION);
+        CreateTaskSetRequest greenTaskSetRequest = new CreateTaskSetRequest();
+        greenTaskSetRequest.setCluster("delete-recreate-cluster");
+        greenTaskSetRequest.setService("blue");
+        greenTaskSetRequest.setTaskDefinition(taskDefinition.getTaskDefinitionArn());
+        greenTaskSetRequest.setLaunchType(LaunchType.FARGATE);
+        greenTaskSetRequest.setNetworkConfiguration(networkConfiguration);
+        greenTaskSetRequest.setLoadBalancers(List.of(greenLoadBalancer));
+        greenTaskSetRequest.setScaleValue(200.0);
+        service.createTaskSet(greenTaskSetRequest, REGION);
+        service.createService("delete-recreate-cluster", "unaffected", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.FARGATE, List.of(otherLoadBalancer), networkConfiguration, null, null,
+                "CODE_DEPLOY", null, REGION);
+
+        service.deleteService("delete-recreate-cluster", "blue", true, REGION);
+        service.createService("delete-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.FARGATE, List.of(blueLoadBalancer), networkConfiguration, null, null,
+                "CODE_DEPLOY", null, REGION);
+
+        verify(containerManager).stopTaskAndCollectExitCodes(blueHandle);
+        verify(containerManager).stopTaskAndCollectExitCodes(greenFirstHandle);
+        verify(containerManager).stopTaskAndCollectExitCodes(greenSecondHandle);
+        verify(containerManager, never()).stopTaskAndCollectExitCodes(otherServiceHandle);
+        verify(containerManager, never()).stopTaskAndCollectExitCodes(recreatedHandle);
+        verify(registrar).deregisterTaskSetTask(any(EcsTask.class),
+                argThat(loadBalancers -> targets(loadBalancers, "blue")), eq(REGION));
+        verify(registrar, times(2)).deregisterTaskSetTask(any(EcsTask.class),
+                argThat(loadBalancers -> targets(loadBalancers, "green")), eq(REGION));
+        verify(registrar, never()).deregisterTask(any(EcsTask.class), any(), eq(REGION));
+        assertEquals(1, service.describeTaskSets("delete-recreate-cluster", "blue", null, REGION).size(),
+                "only the recreated service's primary task set remains");
+        assertEquals(2, runningTasks(service).size(),
+                "the recreated service and the unrelated service retain one task each");
+    }
+
+    @Test
+    void createEc2CodeDeployServiceDoesNotBootstrapATaskSet() {
+        EcsLoadBalancerRegistrar registrar = mock(EcsLoadBalancerRegistrar.class);
+        EcsService service = newDockerModeService(registrar);
+        service.createCluster("ec2-bootstrap-cluster", REGION);
+        TaskDefinition taskDefinition = registerTaskDef(service, "ec2-bootstrap-task", "app:1");
+
+        service.createService("ec2-bootstrap-cluster", "ec2-bootstrap-service",
+                taskDefinition.getTaskDefinitionArn(), 2, LaunchType.EC2, List.of(), null,
+                null, null, "CODE_DEPLOY", null, REGION);
+
+        assertEquals(List.of(), service.describeTaskSets("ec2-bootstrap-cluster",
+                "ec2-bootstrap-service", null, REGION));
+        assertEquals(List.of(), runningTasks(service));
+        verify(registrar, times(0)).registerTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+    }
+
     @Test
     void serviceHoldingARawReferenceIsPinnedOnItsFirstTickInsteadOfRelaunchingForever() {
         EcsService service = newMockModeService();
@@ -219,6 +429,36 @@ class EcsServiceRolloutTest {
                 null, null, List.of(), REGION);
     }
 
+    private static TaskDefinition registerFargateTaskDef(EcsService service, String family, String image) {
+        ContainerDefinition cd = new ContainerDefinition();
+        cd.setName("app");
+        cd.setImage(image);
+        return service.registerTaskDefinition(family, List.of(cd), NetworkMode.awsvpc, "256", "512",
+                null, null, List.of("FARGATE"), REGION);
+    }
+
+    private static NetworkConfiguration fargateNetworkConfiguration() {
+        AwsVpcConfiguration awsvpc = new AwsVpcConfiguration();
+        awsvpc.setSubnets(List.of("subnet-12345678"));
+        NetworkConfiguration networkConfiguration = new NetworkConfiguration();
+        networkConfiguration.setAwsvpcConfiguration(awsvpc);
+        return networkConfiguration;
+    }
+
+    private static EcsLoadBalancer loadBalancer(String name) {
+        EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+        loadBalancer.setTargetGroupArn("arn:aws:elasticloadbalancing:us-east-1:000000000000:targetgroup/"
+                + name + "/id");
+        loadBalancer.setContainerName("app");
+        loadBalancer.setContainerPort(80);
+        return loadBalancer;
+    }
+
+    private static boolean targets(List<EcsLoadBalancer> loadBalancers, String name) {
+        return loadBalancers.size() == 1
+                && loadBalancers.getFirst().getTargetGroupArn().endsWith("/" + name + "/id");
+    }
+
     private static EcsService newMockModeService() {
         return newMockModeService(null);
     }
@@ -238,6 +478,29 @@ class EcsServiceRolloutTest {
         return service;
     }
 
+    private static EcsService newDockerModeService(EcsLoadBalancerRegistrar registrar) {
+        EcsContainerManager containerManager = mock(EcsContainerManager.class);
+        EcsTaskHandle taskHandle = mock(EcsTaskHandle.class);
+        when(containerManager.startTask(any(), any(), any(), eq(REGION))).thenReturn(taskHandle);
+        return newDockerModeService(registrar, containerManager);
+    }
+
+    private static EcsService newDockerModeService(EcsLoadBalancerRegistrar registrar,
+                                                    EcsContainerManager containerManager) {
+        EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+        when(config.services().ecs().mock()).thenReturn(false);
+        when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+        EcsService service = new EcsService(
+                new RegionResolver(REGION, "000000000000"),
+                containerManager,
+                config,
+                registrar,
+                new InMemoryStorageFactory(),
+                null);
+        service.initializeStorage();
+        return service;
+    }
+
     private static final class InMemoryStorageFactory extends StorageFactory {
         private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
 

src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTaskOwnershipTest.java

@@ -7,10 +7,14 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageFactory;
 import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.model.AwsVpcConfiguration;
 import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
 import io.github.hectorvent.floci.services.ecs.model.ContainerInstance;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
 import io.github.hectorvent.floci.services.ecs.model.EcsTask;
 import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import io.github.hectorvent.floci.services.ecs.model.NetworkMode;
+import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;
 import org.junit.jupiter.api.Test;
 
 import java.util.HashMap;
@@ -130,6 +134,28 @@ class EcsServiceTaskOwnershipTest {
                 "service teardown stops only the service's own tasks");
     }
 
+    @Test
+    void forceDeletingAFargateServiceClearsInactiveServiceTaskCounts() {
+        EcsService service = newMockModeService();
+        service.createCluster("fargate-delete-cluster", REGION);
+        registerFargateTaskDef(service);
+        service.createService("fargate-delete-cluster", "worker", "own-fam", 1, LaunchType.FARGATE,
+                List.of(), fargateNetworkConfiguration(), REGION);
+        service.reconcileServices();
+        service.reconcileServices();
+        EcsTask task = allTasks(service).getFirst();
+
+        service.deleteService("fargate-delete-cluster", "worker", true, REGION);
+
+        assertEquals("STOPPED", reload(service, task).getLastStatus());
+        EcsServiceModel deleted = service.describeServices("fargate-delete-cluster", List.of("worker"), REGION)
+                .getFirst();
+        assertEquals("INACTIVE", deleted.getStatus());
+        assertEquals(0, deleted.getDesiredCount());
+        assertEquals(0, deleted.getRunningCount());
+        assertEquals(0, deleted.getPendingCount());
+    }
+
     @Test
     void listTasksByServiceNameReturnsOnlyTasksTheServiceLaunched() {
         EcsService service = newMockModeService();
@@ -216,6 +242,22 @@ class EcsServiceTaskOwnershipTest {
         service.registerTaskDefinition("own-fam", List.of(cd), null, null, null, null, null, List.of(), region);
     }
 
+    private static void registerFargateTaskDef(EcsService service) {
+        ContainerDefinition cd = new ContainerDefinition();
+        cd.setName("app");
+        cd.setImage("app:1");
+        service.registerTaskDefinition("own-fam", List.of(cd), NetworkMode.awsvpc, "256", "512",
+                null, null, List.of("FARGATE"), REGION);
+    }
+
+    private static NetworkConfiguration fargateNetworkConfiguration() {
+        AwsVpcConfiguration awsvpc = new AwsVpcConfiguration();
+        awsvpc.setSubnets(List.of("subnet-12345678"));
+        NetworkConfiguration network = new NetworkConfiguration();
+        network.setAwsvpcConfiguration(awsvpc);
+        return network;
+    }
+
     private static EcsService newMockModeService() {
         EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
         when(config.services().ecs().mock()).thenReturn(true);