feat: Fargate, ECS, ALB,Codebuild, CodePipeline

src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LambdaCfnProvisioner.java

@@ -111,7 +111,7 @@ public class LambdaCfnProvisioner implements CfnResourceProvisioner {
                         "Cannot replace Lambda function " + r.getPhysicalId()
                                 + " without a new FunctionName", 400);
             }
-            func = createLambdaFunction(region, desired, !replacement);
+            func = createLambdaFunction(region, desired, !replacement, r);
             if (replacement && r.getPhysicalId() != null) {
                 deleteReplacedLambda(region, r.getPhysicalId());
             }
@@ -343,13 +343,15 @@ public class LambdaCfnProvisioner implements CfnResourceProvisioner {
         return !Objects.equals(existingPackageType, desired.packageType());
     }
 
-    private LambdaFunction createLambdaFunction(String region, LambdaDesiredState desired, boolean allowAdopt) {
+    private LambdaFunction createLambdaFunction(String region, LambdaDesiredState desired, boolean allowAdopt,
+                                                StackResource resource) {
         try {
             return lambdaService.createFunction(region, desired.createRequest());
         } catch (AwsException e) {
             if (allowAdopt && ("ResourceConflictException".equals(e.getErrorCode())
                     || (e.getMessage() != null && e.getMessage().contains("Function already exist")))) {
-                return lambdaService.getFunction(region, desired.functionName());
+                LambdaFunction existing = lambdaService.getFunction(region, desired.functionName());
+                return updateLambdaFunction(region, existing, desired, resource);
             }
             throw e;
         }

src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LambdaVersionAliasCfnProvisioner.java

@@ -85,16 +85,9 @@ public class LambdaVersionAliasCfnProvisioner implements CfnResourceProvisioner
         }
         String description = ctx.resolveOptional(props, "Description");
 
-        // Version resources are immutable in CloudFormation. An update re-invokes provision with
-        // the prior physical id, and publishing again would mint a new version and repoint every
-        // Ref to it, so the version this resource already published is kept while it still exists.
-        String previous = r.getPhysicalId();
-        if (previous != null && existingVersionIsCurrent(ctx.region(), functionName, previous)) {
-            r.getAttributes().put("Version", previous.substring(previous.lastIndexOf(':') + 1));
-            r.getAttributes().put("FunctionArn", previous);
-            return;
-        }
-
+        // Lambda deduplicates an unchanged publish by $LATEST revision. Calling it here is
+        // essential on UpdateStack: after a dependent function updates its code, the Version
+        // must snapshot that new revision rather than retaining its prior physical id.
         LambdaFunction version = lambdaService.publishVersion(ctx.region(), functionName, description);
 
         // Matches real CloudFormation: Ref returns the qualified ARN, GetAtt Version the version
@@ -104,26 +97,6 @@ public class LambdaVersionAliasCfnProvisioner implements CfnResourceProvisioner
         r.getAttributes().put("FunctionArn", version.getFunctionArn());
     }
 
-    /** True when {@code physicalId} names a still-published version of the function in the template. */
-    private boolean existingVersionIsCurrent(String region, String functionName, String physicalId) {
-        int sep = physicalId.lastIndexOf(':');
-        if (sep <= 0) {
-            return false;
-        }
-        String version = physicalId.substring(sep + 1);
-        String baseArn = physicalId.substring(0, sep);
-        try {
-            LambdaFunction fn = lambdaService.getFunction(region, functionName);
-            if (!baseArn.equals(fn.getFunctionArn().replace(":$LATEST", ""))) {
-                return false; // the FunctionName property now points at a different function
-            }
-            return lambdaService.listVersionsByFunction(region, functionName).stream()
-                    .anyMatch(v -> version.equals(v.getVersion()));
-        } catch (AwsException e) {
-            return false;
-        }
-    }
-
     private void provisionAlias(StackResource r, JsonNode props, ProvisionContext ctx) {
         String functionName = ctx.resolveOptional(props, "FunctionName");
         String aliasName = ctx.resolveOptional(props, "Name");

src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java

@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
 import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
 import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
 import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.core.common.docker.DockerClientProducer;
 import io.github.hectorvent.floci.services.codebuild.BuildspecParser.ParsedArtifacts;
 import io.github.hectorvent.floci.services.codebuild.BuildspecParser.ParsedBuildspec;
 import io.github.hectorvent.floci.services.codebuild.model.Build;
@@ -38,6 +39,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -195,9 +197,10 @@ public class CodeBuildRunner implements ContainerTeardown {
             String logGroup = "/aws/codebuild/" + project.getName();
             String logStream = logStreamer.generateLogStreamName(buildId.replace(":", "/"));
 
-            String image = build.getEnvironment() != null && build.getEnvironment().getImage() != null
+            String requestedImage = build.getEnvironment() != null && build.getEnvironment().getImage() != null
                     ? build.getEnvironment().getImage()
                     : project.getEnvironment().getImage();
+            String image = resolveBuildImage(requestedImage);
 
             boolean privileged = (project.getEnvironment() != null
                     && Boolean.TRUE.equals(project.getEnvironment().getPrivilegedMode()))
@@ -214,8 +217,11 @@ public class CodeBuildRunner implements ContainerTeardown {
 
             // Keep the container alive so each phase can be run with docker exec.
             // No bind mount needed — source and artifacts are transferred with docker cp.
-            ContainerSpec spec = containerBuilder.newContainer(image)
-                    .withCmd(List.of("sh", "-c", "tail -f /dev/null"))
+            String startupCommand = "aws/codebuild/standard:7.0".equals(requestedImage)
+                    ? "apt-get update && apt-get install -y --no-install-recommends zip && tail -f /dev/null"
+                    : "tail -f /dev/null";
+            ContainerBuilder.Builder container = containerBuilder.newContainer(image)
+                    .withCmd(List.of("sh", "-c", startupCommand))
                     .withEnv(envList)
                     .withDockerNetwork(config.services().codebuild().dockerNetwork())
                     .withEmbeddedDns()
@@ -223,8 +229,9 @@ public class CodeBuildRunner implements ContainerTeardown {
                     .withPrivileged(privileged)
                     .withLogRotation()
                     .withLabels(ContainerStorageHelper.resourceIdentityLabels(
-                            "codebuild", buildId, regionResolver.getAccountId(), region))
-                    .build();
+                            "codebuild", buildId, regionResolver.getAccountId(), region));
+            exposeDockerSocketIfPrivileged(container, privileged, config.docker());
+            ContainerSpec spec = container.build();
 
             ContainerLifecycleManager.ContainerInfo info = lifecycleManager.createAndStart(spec);
             containerId = info.containerId();
@@ -261,7 +268,8 @@ public class CodeBuildRunner implements ContainerTeardown {
             if (installResult.stopped()) { finishStopped(build); return; }
             if (installResult.failed()) {
                 completePhaseWithError(build, "INSTALL", "FAILED", installResult.errorMessage());
-                buildFailed = true;
+                finishFailed(build);
+                return;
             } else {
                 completePhase(build, "INSTALL", "SUCCEEDED");
             }
@@ -276,7 +284,8 @@ public class CodeBuildRunner implements ContainerTeardown {
                 if (preBuildResult.stopped()) { finishStopped(build); return; }
                 if (preBuildResult.failed()) {
                     completePhaseWithError(build, "PRE_BUILD", "FAILED", preBuildResult.errorMessage());
-                    buildFailed = true;
+                    finishFailed(build);
+                    return;
                 } else {
                     completePhase(build, "PRE_BUILD", "SUCCEEDED");
                 }
@@ -294,7 +303,8 @@ public class CodeBuildRunner implements ContainerTeardown {
                 if (buildResult.stopped()) { finishStopped(build); return; }
                 if (buildResult.failed()) {
                     completePhaseWithError(build, "BUILD", "FAILED", buildResult.errorMessage());
-                    buildFailed = true;
+                    finishFailed(build);
+                    return;
                 } else {
                     completePhase(build, "BUILD", "SUCCEEDED");
                 }
@@ -311,7 +321,8 @@ public class CodeBuildRunner implements ContainerTeardown {
             if (postBuildResult.stopped()) { finishStopped(build); return; }
             if (postBuildResult.failed()) {
                 completePhaseWithError(build, "POST_BUILD", "FAILED", postBuildResult.errorMessage());
-                buildFailed = true;
+                finishFailed(build);
+                return;
             } else {
                 completePhase(build, "POST_BUILD", "SUCCEEDED");
             }
@@ -384,6 +395,27 @@ public class CodeBuildRunner implements ContainerTeardown {
         }
     }
 
+    private String resolveBuildImage(String image) {
+        if ("aws/codebuild/standard:7.0".equals(image)) {
+            return "golang:1.23-bookworm";
+        }
+        return image;
+    }
+
+    static void exposeDockerSocketIfPrivileged(ContainerBuilder.Builder container, boolean privileged,
+                                               EmulatorConfig.DockerConfig dockerConfig) {
+        if (!privileged) {
+            return;
+        }
+        String dockerHost = DockerClientProducer.resolveDockerConnection(dockerConfig, System::getenv).host();
+        URI dockerUri = URI.create(dockerHost);
+        if (!"unix".equals(dockerUri.getScheme()) || dockerUri.getPath() == null || dockerUri.getPath().isBlank()) {
+            LOG.debugv("CodeBuild privileged mode cannot mount non-Unix Docker endpoint {0}", dockerHost);
+            return;
+        }
+        container.withBind(dockerUri.getPath(), "/var/run/docker.sock");
+    }
+
     private String resolveAndAcquireSource(String region, Build build, Project project,
                                            String buildspecOverride, Path workspace) throws IOException {
         String sourceType = project.getSource() != null ? project.getSource().getType() : "NO_SOURCE";
@@ -660,25 +692,20 @@ public class CodeBuildRunner implements ContainerTeardown {
     private void uploadArtifacts(String region, Build build, Project project,
                                  ParsedArtifacts artifacts, Path workspace) throws IOException {
         String type = artifacts.type();
-        if ("NO_ARTIFACTS".equals(type) && project.getArtifacts() != null) {
-            type = project.getArtifacts().getType();
+        if ("NO_ARTIFACTS".equals(type) && build.getArtifacts() != null) {
+            type = build.getArtifacts().getType();
         }
-        if (type == null || "NO_ARTIFACTS".equals(type) || "CODEPIPELINE".equals(type)) {
+        if (type == null || "NO_ARTIFACTS".equals(type)) {
             return;
         }
-        if (!"S3".equals(type)) {
+        if (!"S3".equals(type) && !"CODEPIPELINE".equals(type)) {
             return;
         }
 
         String packaging = artifacts.packaging();
-        if ("ZIP".equals(packaging) && project.getArtifacts() != null
-                && project.getArtifacts().getPackaging() != null) {
-            packaging = project.getArtifacts().getPackaging();
-        }
-
-        String location = project.getArtifacts() != null ? project.getArtifacts().getLocation() : null;
-        if (location == null || location.isBlank()) {
-            return;
+        if ("ZIP".equals(packaging) && build.getArtifacts() != null
+                && build.getArtifacts().getPackaging() != null) {
+            packaging = build.getArtifacts().getPackaging();
         }
 
         List<String> filePatterns = artifacts.files();
@@ -698,6 +725,16 @@ public class CodeBuildRunner implements ContainerTeardown {
             return;
         }
 
+        if ("CODEPIPELINE".equals(type)) {
+            build.setOutputArtifact(zipFiles(baseDir, matchedFiles, artifacts.discardPaths()));
+            return;
+        }
+
+        String location = build.getArtifacts() != null ? build.getArtifacts().getLocation() : null;
+        if (location == null || location.isBlank()) {
+            return;
+        }
+
         int slash = location.indexOf('/');
         String bucket = slash > 0 ? location.substring(0, slash) : location;
         String prefix = slash > 0 ? location.substring(slash + 1) : "";
@@ -841,6 +878,13 @@ public class CodeBuildRunner implements ContainerTeardown {
         build.setBuildComplete(true);
         build.setBuildStatus("FAILED");
         build.setCurrentPhase("COMPLETED");
+        BuildPhase completed = new BuildPhase();
+        completed.setPhaseType("COMPLETED");
+        completed.setPhaseStatus("FAILED");
+        completed.setStartTime(System.currentTimeMillis() / 1000.0);
+        completed.setEndTime(System.currentTimeMillis() / 1000.0);
+        completed.setDurationInSeconds(0L);
+        build.getPhases().add(completed);
     }
 
     private Optional<BuildPhase> findPhase(Build build, String phaseType) {

src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java

@@ -489,6 +489,14 @@ public class CodeBuildService {
         return build;
     }
 
+    /**
+     * Returns the CODEPIPELINE output produced by one completed build. This is runtime state,
+     * like the build itself, and must not be inferred from a project-wide artifact location.
+     */
+    public byte[] getBuildOutputArtifact(String region, String account, String buildId) {
+        return getBuild(region, account, buildId).getOutputArtifact();
+    }
+
     public List<Build> batchGetBuilds(String region, String account, List<String> buildIds) {
         Map<String, Build> store = buildsFor(account, region);
         return buildIds.stream()

src/main/java/io/github/hectorvent/floci/services/codebuild/model/Build.java

@@ -1,5 +1,6 @@
 package io.github.hectorvent.floci.services.codebuild.model;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 
@@ -29,6 +30,7 @@ public class Build {
     private Integer timeoutInMinutes;
     private Integer queuedTimeoutInMinutes;
     private String encryptionKey;
+    private byte[] outputArtifact;
 
     public String getId() { return id; }
     public void setId(String id) { this.id = id; }
@@ -83,4 +85,11 @@ public class Build {
 
     public String getEncryptionKey() { return encryptionKey; }
     public void setEncryptionKey(String encryptionKey) { this.encryptionKey = encryptionKey; }
+
+    @JsonIgnore
+    public byte[] getOutputArtifact() { return outputArtifact != null ? outputArtifact.clone() : null; }
+
+    public void setOutputArtifact(byte[] outputArtifact) {
+        this.outputArtifact = outputArtifact != null ? outputArtifact.clone() : null;
+    }
 }

src/main/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployService.java

@@ -17,10 +17,16 @@ import io.github.hectorvent.floci.services.codedeploy.model.OnPremisesInstance;
 import io.github.hectorvent.floci.services.ec2.Ec2Service;
 import io.github.hectorvent.floci.services.ecs.EcsService;
 import io.github.hectorvent.floci.services.ecs.model.CreateTaskSetRequest;
+import io.github.hectorvent.floci.services.ecs.model.EcsLoadBalancer;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
 import io.github.hectorvent.floci.services.ecs.model.TaskSet;
 import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
 import io.github.hectorvent.floci.services.elbv2.model.TargetGroup;
+import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;
 import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.model.LambdaAlias;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import io.github.hectorvent.floci.services.s3.S3Service;
 import io.github.hectorvent.floci.services.ssm.SsmCommandService;
 import io.github.hectorvent.floci.services.lambda.model.InvocationType;
 import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
@@ -29,6 +35,8 @@ import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 import org.jboss.logging.Logger;
 
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
@@ -36,6 +44,8 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -54,7 +64,13 @@ public class CodeDeployService {
     private static final Duration DEFAULT_HOOK_CALLBACK_TIMEOUT = Duration.ofHours(1);
     private static final Duration STOP_POLL_INTERVAL = Duration.ofMillis(200);
 
+    @FunctionalInterface
+    interface Sleeper {
+        void sleep(long millis) throws InterruptedException;
+    }
+
     private final LambdaService lambdaService;
+    private final S3Service s3Service;
     private final EcsService ecsService;
     private final ElbV2Service elbV2Service;
     private final SsmCommandService ssmCommandService;
@@ -65,21 +81,48 @@ public class CodeDeployService {
     private final StorageFactory storageFactory;
     private final Duration hookCallbackTimeout;
     private final Clock clock;
+    private final Sleeper sleeper;
 
     @Inject
     public CodeDeployService(LambdaService lambdaService, EcsService ecsService,
                              ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
                              Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
-                             StorageFactory storageFactory) {
+                              StorageFactory storageFactory, S3Service s3Service) {
         this(lambdaService, ecsService, elbV2Service, ssmCommandService, ec2Service, mapper,
-                regionResolver, storageFactory, DEFAULT_HOOK_CALLBACK_TIMEOUT, Clock.systemUTC());
+                regionResolver, storageFactory, DEFAULT_HOOK_CALLBACK_TIMEOUT, Clock.systemUTC(), s3Service, Thread::sleep);
     }
 
     CodeDeployService(LambdaService lambdaService, EcsService ecsService,
                       ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
                       Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
-                      StorageFactory storageFactory, Duration hookCallbackTimeout, Clock clock) {
+                      StorageFactory storageFactory) {
+        this(lambdaService, ecsService, elbV2Service, ssmCommandService, ec2Service, mapper,
+                regionResolver, storageFactory, DEFAULT_HOOK_CALLBACK_TIMEOUT, Clock.systemUTC(), null, Thread::sleep);
+    }
+
+    CodeDeployService(LambdaService lambdaService, EcsService ecsService,
+                      ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
+                       Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
+                       StorageFactory storageFactory, Duration hookCallbackTimeout, Clock clock) {
+        this(lambdaService, ecsService, elbV2Service, ssmCommandService, ec2Service, mapper,
+                regionResolver, storageFactory, hookCallbackTimeout, clock, null, Thread::sleep);
+    }
+
+    CodeDeployService(LambdaService lambdaService, EcsService ecsService,
+                      ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
+                      Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
+                      StorageFactory storageFactory, Duration hookCallbackTimeout, Clock clock, S3Service s3Service) {
+        this(lambdaService, ecsService, elbV2Service, ssmCommandService, ec2Service, mapper,
+                regionResolver, storageFactory, hookCallbackTimeout, clock, s3Service, Thread::sleep);
+    }
+
+    CodeDeployService(LambdaService lambdaService, EcsService ecsService,
+                      ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
+                      Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
+                      StorageFactory storageFactory, Duration hookCallbackTimeout, Clock clock, S3Service s3Service,
+                      Sleeper sleeper) {
         this.lambdaService = lambdaService;
+        this.s3Service = s3Service;
         this.ecsService = ecsService;
         this.elbV2Service = elbV2Service;
         this.ssmCommandService = ssmCommandService;
@@ -90,6 +133,7 @@ public class CodeDeployService {
         this.storageFactory = storageFactory;
         this.hookCallbackTimeout = hookCallbackTimeout;
         this.clock = clock;
+        this.sleeper = sleeper;
     }
 
     // ---- Durable (persisted) ----
@@ -109,8 +153,8 @@ public class CodeDeployService {
     private final ConcurrentHashMap<String, ConcurrentHashMap<String, Deployment>> deployments = new ConcurrentHashMap<>();
     // key: region -> deploymentId -> targetId -> DeploymentTarget wrapper
     private final ConcurrentHashMap<String, ConcurrentHashMap<String, ConcurrentHashMap<String, Map<String, Object>>>> deploymentTargets = new ConcurrentHashMap<>();
-    // key: lifecycleEventHookExecutionId -> CompletableFuture<status>
-    private final ConcurrentHashMap<String, CompletableFuture<String>> hookFutures = new ConcurrentHashMap<>();
+    // key: lifecycleEventHookExecutionId -> the deployment event that owns the callback
+    private final ConcurrentHashMap<String, HookExecution> hookExecutions = new ConcurrentHashMap<>();
     // key: deploymentId -> stop flag
     private final ConcurrentHashMap<String, AtomicBoolean> stopFlags = new ConcurrentHashMap<>();
 
@@ -203,6 +247,28 @@ public class CodeDeployService {
         String afterAllowTraffic;
     }
 
+    private static final class HookExecution {
+        final String deploymentId;
+        final String lifecycleEventName;
+        final CompletableFuture<String> future;
+
+        HookExecution(String deploymentId, String lifecycleEventName, CompletableFuture<String> future) {
+            this.deploymentId = deploymentId;
+            this.lifecycleEventName = lifecycleEventName;
+            this.future = future;
+        }
+    }
+
+    private static final class AliasRoutingSnapshot {
+        final String functionVersion;
+        final Map<String, Double> routingConfig;
+
+        AliasRoutingSnapshot(LambdaAlias alias) {
+            this.functionVersion = alias.getFunctionVersion();
+            this.routingConfig = alias.getRoutingConfig() == null ? Map.of() : Map.copyOf(alias.getRoutingConfig());
+        }
+    }
+
     private static final class ServerAppSpecInfo {
         String os;
         Map<String, List<Map<String, Object>>> hooks; // hookName → [{location, timeout, runas}]
@@ -578,8 +644,10 @@ public class CodeDeployService {
             return createServerDeployment(region, appName, groupName, group, configName, revision, description);
         }
 
-        AppSpecInfo appSpec = parseAppSpec(revision);
         String effectiveConfig = configName != null ? configName : group.getDeploymentConfigName();
+        validateLambdaDeployment(region, app, group, effectiveConfig);
+        AppSpecInfo appSpec = parseAppSpec(revision);
+        validateLambdaTarget(region, appSpec);
 
         String deploymentId = generateDeploymentId();
         double now = Instant.now().toEpochMilli() / 1000.0;
@@ -695,10 +763,16 @@ public class CodeDeployService {
     }
 
     public String putLifecycleEventHookExecutionStatus(String deploymentId, String executionId, String status) {
-        CompletableFuture<String> future = hookFutures.get(executionId);
-        if (future != null && !future.isDone()) {
-            future.complete(status);
+        if (!"Succeeded".equals(status) && !"Failed".equals(status)) {
+            throw new AwsException("InvalidLifecycleEventHookExecutionStatusException",
+                    "Lifecycle event hook execution status must be Succeeded or Failed", 400);
         }
+        HookExecution execution = hookExecutions.get(executionId);
+        if (execution == null || !execution.deploymentId.equals(deploymentId) || execution.future.isDone()) {
+            throw new AwsException("InvalidLifecycleEventHookExecutionIdException",
+                    "Lifecycle event hook execution ID is not valid for this deployment", 400);
+        }
+        execution.future.complete(status);
         return executionId;
     }
 
@@ -1063,12 +1137,17 @@ public class CodeDeployService {
             for (Map<String, String> filter : ec2TagFilters) {
                 String key = filter.get("Key");
                 String value = filter.get("Value");
-                if (key != null && value != null) {
+                String type = filter.get("Type");
+                if ("KEY_ONLY".equals(type) && key != null) {
+                    filters.computeIfAbsent("tag-key", k -> new ArrayList<>()).add(key);
+                } else if ("VALUE_ONLY".equals(type) && value != null) {
+                    filters.computeIfAbsent("tag-value", k -> new ArrayList<>()).add(value);
+                } else if (key != null && value != null) {
                     filters.computeIfAbsent("tag:" + key, k -> new ArrayList<>()).add(value);
                 }
             }
             try {
-                ec2Service.describeInstances(region, null, filters).stream()
+                ec2Service.describeInstances(region, List.of(), filters).stream()
                         .flatMap(r -> r.getInstances().stream())
                         .map(inst -> inst.getInstanceId())
                         .filter(id -> id != null)
@@ -1118,38 +1197,53 @@ public class CodeDeployService {
         if (revision == null) {
             throw new AwsException("InvalidRevisionException", "Revision is required", 400);
         }
-        Object appSpecContent = revision.get("appSpecContent");
-        if (!(appSpecContent instanceof Map)) {
-            throw new AwsException("InvalidRevisionException", "Missing appSpecContent in revision", 400);
-        }
-        String content = (String) ((Map<String, Object>) appSpecContent).get("content");
+        String content = appSpecContent(revision);
         if (content == null || content.isBlank()) {
             throw new AwsException("InvalidRevisionException", "Missing content in appSpecContent", 400);
         }
 
         AppSpecInfo info = new AppSpecInfo();
         try {
-            JsonNode root = mapper.readTree(content);
-            JsonNode resources = root.get("Resources");
-            if (resources != null && resources.isArray() && !resources.isEmpty()) {
-                JsonNode firstResource = resources.get(0);
-                if (firstResource.isObject()) {
-                    JsonNode resourceNode = firstResource.fields().next().getValue();
-                    JsonNode props = resourceNode.path("Properties");
-                    info.functionName = props.path("Name").asText(null);
-                    info.aliasName = props.path("Alias").asText(null);
-                    info.currentVersion = props.path("CurrentVersion").asText(null);
-                    info.targetVersion = props.path("TargetVersion").asText(null);
-                }
+            JsonNode root = yamlMapper.readTree(content);
+            if (!root.isObject() || !"0.0".equals(root.path("version").asText())) {
+                throw new IllegalArgumentException("AppSpec version must be 0.0");
+            }
+            JsonNode resources = root.path("Resources");
+            if (!resources.isArray() || resources.size() != 1 || !resources.get(0).isObject()
+                    || resources.get(0).size() != 1) {
+                throw new IllegalArgumentException("Lambda AppSpec must contain exactly one resource");
+            }
+            JsonNode resourceNode = resources.get(0).elements().next();
+            if (!"AWS::Lambda::Function".equals(resourceNode.path("Type").asText())) {
+                throw new IllegalArgumentException("Lambda AppSpec resource Type must be AWS::Lambda::Function");
+            }
+            JsonNode props = resourceNode.path("Properties");
+            info.functionName = requiredAppSpecText(props, "Name");
+            info.aliasName = requiredAppSpecText(props, "Alias");
+            info.currentVersion = requiredAppSpecText(props, "CurrentVersion");
+            info.targetVersion = requiredAppSpecText(props, "TargetVersion");
+            if (info.currentVersion.equals(info.targetVersion)) {
+                throw new IllegalArgumentException("CurrentVersion and TargetVersion must differ");
             }
             JsonNode hooks = root.get("Hooks");
-            if (hooks != null && hooks.isArray()) {
+            if (hooks != null) {
+                if (!hooks.isArray()) {
+                    throw new IllegalArgumentException("Hooks must be an array");
+                }
                 for (JsonNode hook : hooks) {
-                    if (hook.has("BeforeAllowTraffic")) {
-                        info.beforeAllowTraffic = hook.get("BeforeAllowTraffic").asText(null);
+                    if (!hook.isObject() || hook.size() != 1) {
+                        throw new IllegalArgumentException("Each Hook must contain exactly one lifecycle event");
                     }
-                    if (hook.has("AfterAllowTraffic")) {
-                        info.afterAllowTraffic = hook.get("AfterAllowTraffic").asText(null);
+                    Map.Entry<String, JsonNode> entry = hook.fields().next();
+                    String hookFunction = requiredAppSpecText(hook, entry.getKey());
+                    if ("BeforeAllowTraffic".equals(entry.getKey())) {
+                        if (info.beforeAllowTraffic != null) { throw new IllegalArgumentException("Duplicate BeforeAllowTraffic hook"); }
+                        info.beforeAllowTraffic = hookFunction;
+                    } else if ("AfterAllowTraffic".equals(entry.getKey())) {
+                        if (info.afterAllowTraffic != null) { throw new IllegalArgumentException("Duplicate AfterAllowTraffic hook"); }
+                        info.afterAllowTraffic = hookFunction;
+                    } else {
+                        throw new IllegalArgumentException("Unsupported Lambda lifecycle event: " + entry.getKey());
                     }
                 }
             }
@@ -1157,13 +1251,73 @@ public class CodeDeployService {
             throw new AwsException("InvalidRevisionException", "Failed to parse AppSpec content: " + e.getMessage(), 400);
         }
 
-        if (info.functionName == null || info.aliasName == null || info.targetVersion == null) {
-            throw new AwsException("InvalidRevisionException",
-                    "AppSpec must specify Name, Alias, and TargetVersion", 400);
-        }
         return info;
     }
 
+    @SuppressWarnings("unchecked")
+    private String appSpecContent(Map<String, Object> revision) {
+        Object appSpecContent = revision.get("appSpecContent");
+        if (appSpecContent instanceof Map<?, ?> content) {
+            Object value = ((Map<String, Object>) content).get("content");
+            if (value instanceof String text && !text.isBlank()) {
+                return text;
+            }
+        }
+        Object s3Location = revision.get("s3Location");
+        if (!(s3Location instanceof Map<?, ?> location) || s3Service == null) {
+            throw new AwsException("InvalidRevisionException", "Missing AppSpec content in revision", 400);
+        }
+        String bucket = String.valueOf(location.get("bucket"));
+        String key = String.valueOf(location.get("key"));
+        byte[] artifact = s3Service.getObject(bucket, key).getData();
+        InputStream input = artifact == null ? InputStream.nullInputStream() : new ByteArrayInputStream(artifact);
+        try (ZipInputStream zip = new ZipInputStream(input)) {
+            ZipEntry entry;
+            while ((entry = zip.getNextEntry()) != null) {
+                String name = entry.getName();
+                if (!entry.isDirectory() && ("appspec.yml".equalsIgnoreCase(name) || "appspec.yaml".equalsIgnoreCase(name))) {
+                    return new String(zip.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8);
+                }
+            }
+        } catch (Exception e) {
+            throw new AwsException("InvalidRevisionException", "Could not read AppSpec from S3 revision: " + e.getMessage(), 400);
+        }
+        throw new AwsException("InvalidRevisionException", "S3 revision does not contain appspec.yml or appspec.yaml", 400);
+    }
+
+    private String requiredAppSpecText(JsonNode node, String field) {
+        JsonNode value = node.get(field);
+        if (value == null || !value.isTextual() || value.asText().isBlank()) {
+            throw new IllegalArgumentException("AppSpec must specify a non-empty " + field);
+        }
+        return value.asText();
+    }
+
+    private void validateLambdaDeployment(String region, Application app, DeploymentGroup group, String configName) {
+        if (!"Lambda".equals(app.getComputePlatform())) {
+            throw new AwsException("InvalidDeploymentConfigException", "Application compute platform must be Lambda", 400);
+        }
+        if (group.getComputePlatform() != null && !"Lambda".equals(group.getComputePlatform())) {
+            throw new AwsException("InvalidDeploymentConfigException", "Deployment group compute platform must be Lambda", 400);
+        }
+        DeploymentConfig config = getDeploymentConfig(region, configName);
+        if (!"Lambda".equals(config.getComputePlatform())) {
+            throw new AwsException("InvalidDeploymentConfigException", "Deployment configuration compute platform must be Lambda", 400);
+        }
+    }
+
+    private void validateLambdaTarget(String region, AppSpecInfo appSpec) {
+        LambdaAlias alias = lambdaService.getAlias(region, appSpec.functionName, appSpec.aliasName);
+        if (!appSpec.currentVersion.equals(alias.getFunctionVersion())) {
+            throw new AwsException("InvalidRevisionException", "AppSpec CurrentVersion does not match the alias version", 400);
+        }
+        List<String> versions = lambdaService.listVersionsByFunction(region, appSpec.functionName).stream()
+                .map(LambdaFunction::getVersion).toList();
+        if (!versions.contains(appSpec.currentVersion) || !versions.contains(appSpec.targetVersion)) {
+            throw new AwsException("InvalidRevisionException", "AppSpec version does not exist", 400);
+        }
+    }
+
     @SuppressWarnings("unchecked")
     private EcsAppSpecInfo parseEcsAppSpec(Map<String, Object> revision) {
         if (revision == null) {
@@ -1180,7 +1334,13 @@ public class CodeDeployService {
 
         EcsAppSpecInfo info = new EcsAppSpecInfo();
         try {
-            JsonNode root = mapper.readTree(content);
+            JsonNode root;
+            try {
+                root = mapper.readTree(content);
+            } catch (Exception ignored) {
+                // CodePipeline's AppSpecTemplatePath conventionally points to YAML.
+                root = yamlMapper.readTree(content);
+            }
             JsonNode resources = root.get("Resources");
             if (resources != null && resources.isArray() && !resources.isEmpty()) {
                 JsonNode firstResource = resources.get(0);
@@ -1367,6 +1527,21 @@ public class CodeDeployService {
                 // its startedBy, which is how a client tells a CodeDeploy set from an external one.
                 createTaskSet.setExternalId(deploymentId);
                 createTaskSet.setStartedBy("CODE_DEPLOY");
+                EcsServiceModel targetService = ecsService.describeServices(clusterName, List.of(serviceName), region)
+                        .getFirst();
+                if (targetService.getCapacityProviderStrategy() != null
+                        && !targetService.getCapacityProviderStrategy().isEmpty()) {
+                    createTaskSet.setCapacityProviderStrategy(targetService.getCapacityProviderStrategy());
+                } else {
+                    createTaskSet.setLaunchType(targetService.getLaunchType());
+                }
+                if (greenTgArn != null) {
+                    EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+                    loadBalancer.setTargetGroupArn(greenTgArn);
+                    loadBalancer.setContainerName(appSpec.containerName);
+                    loadBalancer.setContainerPort(appSpec.containerPort);
+                    createTaskSet.setLoadBalancers(List.of(loadBalancer));
+                }
                 greenTaskSet = ecsService.createTaskSet(createTaskSet, region);
                 appendTaskSetInfo(ecsTargetMap, greenTaskSet, greenTgArn, 0.0);
                 finishLifecycleEvent(installEvent, "Succeeded");
@@ -1406,6 +1581,8 @@ public class CodeDeployService {
 
             // AllowTraffic: shift ELB traffic blue → green
             if (!listenerArns.isEmpty() && blueTgArn != null && greenTgArn != null) {
+                waitForGreenTargets(region, greenTgArn, stopFlag);
+                if (stopFlag.get()) { finishEcsStopped(deployment, ecsTargetMap); return; }
                 executeEcsAllowTraffic(region, deployment, configName, ecsTargetMap,
                         listenerArns, blueTgArn, greenTgArn, stopFlag);
             } else {
@@ -1460,10 +1637,24 @@ public class CodeDeployService {
         }
     }
 
-    private void executeEcsAllowTraffic(String region, Deployment deployment, String configName,
-                                        Map<String, Object> ecsTargetMap, List<String> listenerArns,
-                                        String blueTgArn, String greenTgArn,
-                                        AtomicBoolean stopFlag) throws InterruptedException {
+    /** A real target group is switched only once every registered green task is routable. */
+    private void waitForGreenTargets(String region, String greenTgArn, AtomicBoolean stopFlag)
+            throws InterruptedException {
+        while (!stopFlag.get()) {
+            List<TargetHealth> targets = elbV2Service.describeTargetHealth(region, greenTgArn, null);
+            // Mock ECS does not start containers or register target addresses. Preserve that control-plane
+            // behavior while Docker-backed deployments wait for the ELB health checker.
+            if (targets.isEmpty() || targets.stream().allMatch(target -> "healthy".equals(target.getState()))) {
+                return;
+            }
+            sleeper.sleep(STOP_POLL_INTERVAL.toMillis());
+        }
+    }
+
+    void executeEcsAllowTraffic(String region, Deployment deployment, String configName,
+                                Map<String, Object> ecsTargetMap, List<String> listenerArns,
+                                String blueTgArn, String greenTgArn,
+                                AtomicBoolean stopFlag) throws InterruptedException {
         TrafficRoutingInfo trc = getTrafficRoutingInfo(region, configName);
         Map<String, Object> event = addLifecycleEvent(ecsTargetMap, "AllowTraffic");
         try {
@@ -1473,9 +1664,9 @@ public class CodeDeployService {
                         elbV2Service.shiftListenerForward(region, listenerArn,
                                 blueTgArn, greenTgArn, trc.percentage);
                     }
-                    long waitMs = Math.min(trc.intervalSeconds * 1000L, 5000L);
+                    long waitMs = trc.intervalSeconds * 1000L;
                     if (waitMs > 0 && !stopFlag.get()) {
-                        Thread.sleep(waitMs);
+                        sleeper.sleep(waitMs);
                     }
                     if (!stopFlag.get()) {
                         for (String listenerArn : listenerArns) {
@@ -1493,9 +1684,9 @@ public class CodeDeployService {
                                     blueTgArn, greenTgArn, pct);
                         }
                         if (pct < 100) {
-                            long waitMs = Math.min(trc.intervalSeconds * 1000L, 2000L);
+                            long waitMs = trc.intervalSeconds * 1000L;
                             if (waitMs > 0) {
-                                Thread.sleep(waitMs);
+                                sleeper.sleep(waitMs);
                             }
                         }
                     }
@@ -1588,6 +1779,9 @@ public class CodeDeployService {
     private void runStateMachine(String region, Deployment deployment, AppSpecInfo appSpec,
                                   Map<String, Object> lambdaTargetMap, AtomicBoolean stopFlag, String configName) {
         String deploymentId = deployment.getDeploymentId();
+        AliasRoutingSnapshot aliasSnapshot = new AliasRoutingSnapshot(
+                lambdaService.getAlias(region, appSpec.functionName, appSpec.aliasName));
+        boolean trafficShifted = false;
         try {
             deployment.setStatus("InProgress");
             deployment.setStartTime(Instant.now().toEpochMilli() / 1000.0);
@@ -1605,16 +1799,30 @@ public class CodeDeployService {
 
             if (stopFlag.get()) { finishStopped(deployment, lambdaTargetMap); return; }
 
+            // Once AllowTraffic starts, a failed partial shift must be restored from the captured snapshot.
+            trafficShifted = true;
             executeAllowTraffic(region, deployment, appSpec, configName, lambdaTargetMap, stopFlag);
 
-            if (stopFlag.get()) { finishStopped(deployment, lambdaTargetMap); return; }
+            if (stopFlag.get()) {
+                rollbackAliasIfEnabled(region, deployment, appSpec, aliasSnapshot, "DEPLOYMENT_STOP_ON_REQUEST");
+                finishStopped(deployment, lambdaTargetMap);
+                return;
+            }
 
             if (appSpec.afterAllowTraffic != null) {
                 boolean ok = invokeHook(region, deployment, appSpec.afterAllowTraffic,
                         "AfterAllowTraffic", lambdaTargetMap, stopFlag);
-                if (stopFlag.get()) { finishStopped(deployment, lambdaTargetMap); return; }
-                if (!ok) { finishFailed(deployment, lambdaTargetMap, "HOOK_EXECUTION_FAILURE",
-                        "Hook function reported failure: AfterAllowTraffic"); return; }
+                if (stopFlag.get()) {
+                    rollbackAliasIfEnabled(region, deployment, appSpec, aliasSnapshot, "DEPLOYMENT_STOP_ON_REQUEST");
+                    finishStopped(deployment, lambdaTargetMap);
+                    return;
+                }
+                if (!ok) {
+                    rollbackAliasIfEnabled(region, deployment, appSpec, aliasSnapshot, "DEPLOYMENT_FAILURE");
+                    finishFailed(deployment, lambdaTargetMap, "HOOK_EXECUTION_FAILURE",
+                            "Hook function reported failure: AfterAllowTraffic");
+                    return;
+                }
             }
 
             updateTargetStatus(lambdaTargetMap, "Succeeded");
@@ -1623,18 +1831,24 @@ public class CodeDeployService {
 
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
+            if (trafficShifted) {
+                rollbackAliasIfEnabled(region, deployment, appSpec, aliasSnapshot, "DEPLOYMENT_STOP_ON_REQUEST");
+            }
             finishStopped(deployment, lambdaTargetMap);
         } catch (Exception e) {
             LOG.warnv("Deployment {0} failed: {1}", deploymentId, e.getMessage());
+            if (trafficShifted) {
+                rollbackAliasIfEnabled(region, deployment, appSpec, aliasSnapshot, "DEPLOYMENT_FAILURE");
+            }
             finishFailed(deployment, lambdaTargetMap, "DeploymentFailed", e.getMessage());
         } finally {
             stopFlags.remove(deploymentId);
         }
     }
 
-    private void executeAllowTraffic(String region, Deployment deployment, AppSpecInfo appSpec,
-                                      String configName, Map<String, Object> lambdaTargetMap,
-                                      AtomicBoolean stopFlag) throws InterruptedException {
+    private boolean executeAllowTraffic(String region, Deployment deployment, AppSpecInfo appSpec,
+                                       String configName, Map<String, Object> lambdaTargetMap,
+                                       AtomicBoolean stopFlag) throws InterruptedException {
         TrafficRoutingInfo trc = getTrafficRoutingInfo(region, configName);
         Map<String, Object> event = addLifecycleEvent(lambdaTargetMap, "AllowTraffic");
 
@@ -1647,12 +1861,11 @@ public class CodeDeployService {
                     lambdaService.updateAlias(region, appSpec.functionName, appSpec.aliasName,
                             appSpec.currentVersion, null, routing);
 
-                    // Wait the canary interval (capped at 5s for emulator speed)
-                    long waitMs = Math.min(trc.intervalSeconds * 1000L, 5000L);
+                    long waitMs = trc.intervalSeconds * 1000L;
                     if (waitMs > 0 && !stopFlag.get()) {
-                        Thread.sleep(waitMs);
+                        sleeper.sleep(waitMs);
                     }
-                    if (stopFlag.get()) { return; }
+                    if (stopFlag.get()) { return true; }
 
                     // Step 2: flip 100% to targetVersion
                     lambdaService.updateAlias(region, appSpec.functionName, appSpec.aliasName,
@@ -1669,9 +1882,9 @@ public class CodeDeployService {
                             double weight = pct / 100.0;
                             lambdaService.updateAlias(region, appSpec.functionName, appSpec.aliasName,
                                     appSpec.currentVersion, null, Map.of(appSpec.targetVersion, weight));
-                            long waitMs = Math.min(trc.intervalSeconds * 1000L, 2000L);
+                            long waitMs = trc.intervalSeconds * 1000L;
                             if (waitMs > 0) {
-                                Thread.sleep(waitMs);
+                                sleeper.sleep(waitMs);
                             }
                         }
                     }
@@ -1683,18 +1896,35 @@ public class CodeDeployService {
                 }
             }
             finishLifecycleEvent(event, "Succeeded");
+            return true;
         } catch (Exception e) {
             finishLifecycleEvent(event, "Failed");
             throw e;
         }
     }
 
+    @SuppressWarnings("unchecked")
+    private void rollbackAliasIfEnabled(String region, Deployment deployment, AppSpecInfo appSpec,
+                                        AliasRoutingSnapshot snapshot, String event) {
+        DeploymentGroup group = getDeploymentGroup(region, deployment.getApplicationName(), deployment.getDeploymentGroupName());
+        Map<String, Object> config = group.getAutoRollbackConfiguration();
+        if (config == null || !Boolean.TRUE.equals(config.get("enabled"))) {
+            return;
+        }
+        Object events = config.get("events");
+        if (!(events instanceof List<?> list) || !list.contains(event)) {
+            return;
+        }
+        lambdaService.updateAlias(region, appSpec.functionName, appSpec.aliasName,
+                snapshot.functionVersion, null, snapshot.routingConfig);
+    }
+
     private boolean invokeHook(String region, Deployment deployment, String hookFunctionName,
                                 String lifecycleEventName, Map<String, Object> lambdaTargetMap,
                                 AtomicBoolean stopFlag) throws InterruptedException {
         String executionId = UUID.randomUUID().toString();
         CompletableFuture<String> future = new CompletableFuture<>();
-        hookFutures.put(executionId, future);
+        hookExecutions.put(executionId, new HookExecution(deployment.getDeploymentId(), lifecycleEventName, future));
 
         Map<String, Object> event = addLifecycleEvent(lambdaTargetMap, lifecycleEventName);
 
@@ -1757,7 +1987,7 @@ public class CodeDeployService {
             }
             return "Succeeded".equals(status);
         } finally {
-            hookFutures.remove(executionId);
+            hookExecutions.remove(executionId);
         }
     }
 

src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java

@@ -12,6 +12,9 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageFactory;
 import io.github.hectorvent.floci.services.codebuild.CodeBuildService;
 import io.github.hectorvent.floci.services.codebuild.model.Build;
+import io.github.hectorvent.floci.services.codebuild.model.ProjectArtifacts;
+import io.github.hectorvent.floci.services.cloudformation.CloudFormationService;
+import io.github.hectorvent.floci.services.cloudformation.model.Stack;
 import io.github.hectorvent.floci.services.codedeploy.CodeDeployService;
 import io.github.hectorvent.floci.services.codedeploy.model.Deployment;
 import io.github.hectorvent.floci.services.codepipeline.model.CodePipelineExecution;
@@ -19,6 +22,9 @@ import io.github.hectorvent.floci.services.codepipeline.model.CodePipelineExecut
 import io.github.hectorvent.floci.services.codepipeline.model.CodePipelinePipeline;
 import io.github.hectorvent.floci.services.codepipeline.model.CodePipelinePipeline.TransitionState;
 import io.github.hectorvent.floci.services.codepipeline.model.CodePipelineStoredItem;
+import io.github.hectorvent.floci.services.ecs.EcsService;
+import io.github.hectorvent.floci.services.ecs.model.RegisterTaskDefinitionRequest;
+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
 import io.github.hectorvent.floci.services.lambda.LambdaService;
 import io.github.hectorvent.floci.services.lambda.model.InvocationType;
 import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
@@ -30,6 +36,7 @@ import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 import org.jboss.logging.Logger;
 
+import java.io.ByteArrayInputStream;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -42,12 +49,15 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 
 @ApplicationScoped
 public class CodePipelineService {
@@ -59,6 +69,8 @@ public class CodePipelineService {
     private static final long SOURCE_POLL_INTERVAL_MS = 500L;
     private static final String SOURCE_POLL_TYPE = "source-poll";
     private static final String MISSING_SOURCE_REVISION = "missing";
+    private static final int MAX_ARTIFACT_ENTRY_SIZE = 256 * 1024 * 1024;
+    private static final int MAX_EXTRACTED_ARTIFACT_SIZE = 256 * 1024 * 1024;
 
     private final AccountAwareStorageBackend<CodePipelinePipeline> pipelineStore;
     private final AccountAwareStorageBackend<CodePipelineExecution> executionStore;
@@ -66,6 +78,8 @@ public class CodePipelineService {
     private final ObjectMapper mapper;
     private final CodeBuildService codeBuildService;
     private final CodeDeployService codeDeployService;
+    private final EcsService ecsService;
+    private final CloudFormationService cloudFormationService;
     private final LambdaService lambdaService;
     private final S3Service s3Service;
     private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();
@@ -76,13 +90,13 @@ public class CodePipelineService {
     // monitor for its whole run, so counting under that monitor would block StartPipelineExecution
     // until the running execution finished.
     private final KeyedLockPool startLocks = new KeyedLockPool();
-    private final Map<String, byte[]> runtimeArtifacts = new ConcurrentHashMap<>();
 
     @Inject
     @SuppressWarnings("unchecked")
     public CodePipelineService(StorageFactory storageFactory, ObjectMapper mapper,
-                               CodeBuildService codeBuildService, CodeDeployService codeDeployService,
-                               LambdaService lambdaService, S3Service s3Service) {
+                                 CodeBuildService codeBuildService, CodeDeployService codeDeployService,
+                                 LambdaService lambdaService, S3Service s3Service,
+                                 CloudFormationService cloudFormationService, EcsService ecsService) {
         this.pipelineStore = storageFactory.create(
                 "codepipeline", "codepipeline-pipelines.json", new TypeReference<Map<String, CodePipelinePipeline>>() {});
         this.executionStore = storageFactory.create(
@@ -92,10 +106,20 @@ public class CodePipelineService {
         this.mapper = mapper;
         this.codeBuildService = codeBuildService;
         this.codeDeployService = codeDeployService;
+        this.ecsService = ecsService;
+        this.cloudFormationService = cloudFormationService;
         this.lambdaService = lambdaService;
         this.s3Service = s3Service;
     }
 
+    CodePipelineService(StorageFactory storageFactory, ObjectMapper mapper,
+                        CodeBuildService codeBuildService, CodeDeployService codeDeployService,
+                        LambdaService lambdaService, S3Service s3Service,
+                        CloudFormationService cloudFormationService) {
+        this(storageFactory, mapper, codeBuildService, codeDeployService, lambdaService, s3Service,
+                cloudFormationService, null);
+    }
+
     public JsonNode handle(String action, JsonNode request, String region, String account) {
         return switch (action) {
             case "CreatePipeline" -> createPipeline(request, region, account);
@@ -745,22 +769,82 @@ public class CodePipelineService {
         String pipelineName = text(request, "pipelineName");
         CodePipelineExecution source = requireExecution(
                 account, region, pipelineName, text(request, "pipelineExecutionId"));
-        ObjectNode start = mapper.createObjectNode();
-        start.put("name", pipelineName);
-        start.set("sourceRevisions", mapper.valueToTree(source.getSourceRevisions()));
-        ArrayNode vars = start.putArray("variables");
-        source.getVariables().forEach(v -> vars.addObject()
-                .put("name", v.get("name"))
-                .put("value", v.get("resolvedValue")));
-        return startPipelineExecution(start, region, account);
+        CodePipelinePipeline pipeline = requirePipeline(account, region, pipelineName);
+        String stageName = text(request, "stageName");
+        String retryMode = text(request, "retryMode");
+        requireStage(pipeline, stageName);
+        if (!List.of("FAILED_ACTIONS", "ALL_ACTIONS").contains(retryMode)) {
+            throw new AwsException("ValidationException", "retryMode must be FAILED_ACTIONS or ALL_ACTIONS", 400);
+        }
+        if (!"Failed".equals(stageExecutionStatus(source, stageName))) {
+            throw new AwsException("PipelineExecutionNotRetryableException",
+                    "Stage execution is not failed: " + stageName, 400);
+        }
+
+        List<String> actionNames = new ArrayList<>();
+        for (JsonNode stage : pipeline.getDeclaration().path("stages")) {
+            if (!stageName.equals(stage.path("name").asText())) {
+                continue;
+            }
+            for (JsonNode action : stage.path("actions")) {
+                String actionName = action.path("name").asText();
+                if ("ALL_ACTIONS".equals(retryMode) || source.getActionExecutions().stream()
+                        .anyMatch(a -> stageName.equals(a.getStageName()) && actionName.equals(a.getActionName())
+                                && "Failed".equals(a.getStatus()))) {
+                    actionNames.add(actionName);
+                }
+            }
+        }
+        if (actionNames.isEmpty()) {
+            throw new AwsException("PipelineExecutionNotRetryableException",
+                    "Stage execution has no failed actions: " + stageName, 400);
+        }
+
+        CodePipelineExecution retry = new CodePipelineExecution();
+        retry.setAccountId(account);
+        retry.setRegion(region);
+        retry.setPipelineExecutionId(UUID.randomUUID().toString());
+        retry.setPipelineName(pipelineName);
+        retry.setPipelineVersion(pipeline.getVersion());
+        retry.setExecutionMode(pipeline.getDeclaration().path("executionMode").asText(DEFAULT_EXECUTION_MODE));
+        retry.setExecutionType("STANDARD");
+        retry.setStatus("InProgress");
+        retry.setStatusSummary("Stage execution retry started.");
+        retry.setStartTime(now());
+        retry.setLastUpdateTime(retry.getStartTime());
+        retry.setSourceRevisions(new ArrayList<>(source.getSourceRevisions()));
+        retry.setVariables(new ArrayList<>(source.getVariables()));
+        retry.setArtifacts(new LinkedHashMap<>(source.getArtifacts()));
+        retry.setRetryStageName(stageName);
+        retry.setRetryActionNames(actionNames);
+        retry.setTrigger(Map.of("triggerType", "RetryStageExecution", "triggerDetail", stageName));
+        if (!persistExecutionIfSlotAvailable(retry)) {
+            throw new AwsException("ConcurrentPipelineExecutionsLimitExceededException",
+                    "The pipeline has reached the limit for concurrent pipeline executions", 400);
+        }
+        applyExecutionMode(retry);
+        try {
+            executor.submit(() -> runExecution(pipeline, retry));
+        } catch (RejectedExecutionException exception) {
+            retry.setStatus("Failed");
+            retry.setStatusSummary("Pipeline execution could not be scheduled.");
+            retry.setLastUpdateTime(now());
+            putExecution(retry);
+            throw new AwsException("ConflictException", "Pipeline execution could not be scheduled.", 400);
+        }
+        return mapper.createObjectNode().put("pipelineExecutionId", retry.getPipelineExecutionId());
     }
 
     private ObjectNode rollbackStage(JsonNode request, String region, String account) {
         CodePipelineExecution target = requireExecution(
                 account, region, text(request, "pipelineName"), text(request, "targetPipelineExecutionId"));
-        ObjectNode started = retryStageExecution(mapper.createObjectNode()
-                .put("pipelineName", target.getPipelineName())
-                .put("pipelineExecutionId", target.getPipelineExecutionId()), region, account);
+        ObjectNode start = mapper.createObjectNode().put("name", target.getPipelineName());
+        start.set("sourceRevisions", mapper.valueToTree(target.getSourceRevisions()));
+        ArrayNode vars = start.putArray("variables");
+        target.getVariables().forEach(v -> vars.addObject()
+                .put("name", v.get("name"))
+                .put("value", v.get("resolvedValue")));
+        ObjectNode started = startPipelineExecution(start, region, account);
         CodePipelineExecution rollback = requireExecution(
                 account, region, target.getPipelineName(), started.path("pipelineExecutionId").asText());
         rollback.setExecutionType("ROLLBACK");
@@ -977,6 +1061,10 @@ public class CodePipelineService {
         Runnable work = () -> {
             try {
                 for (JsonNode stage : pipeline.getDeclaration().path("stages")) {
+                    if (execution.getRetryStageName() != null
+                            && !execution.getRetryStageName().equals(stage.path("name").asText())) {
+                        continue;
+                    }
                     waitForTransition(pipeline, execution, stage.path("name").asText());
                     if (finishIfStopped(execution)) {
                         return;
@@ -986,7 +1074,7 @@ public class CodePipelineService {
                     execution.getStageExecutionStatuses().put(stageName, "InProgress");
                     execution.setLastUpdateTime(now());
                     putExecution(execution);
-                    runStage(pipeline, execution, stage);
+                    runStage(pipeline, execution, stage, execution.getRetryActionNames());
                     if ("Failed".equals(execution.getStatus())) {
                         execution.getStageExecutionStatuses().put(stageName, "Failed");
                         return;
@@ -1012,7 +1100,6 @@ public class CodePipelineService {
                 execution.setCurrentStage(null);
                 execution.setLastUpdateTime(now());
                 putExecution(execution);
-                clearRuntimeArtifacts(execution);
             }
         };
         if ("QUEUED".equals(execution.getExecutionMode())) {
@@ -1022,9 +1109,14 @@ public class CodePipelineService {
         }
     }
 
-    private void runStage(CodePipelinePipeline pipeline, CodePipelineExecution execution, JsonNode stage) {
+    private void runStage(CodePipelinePipeline pipeline, CodePipelineExecution execution, JsonNode stage,
+                          List<String> retryActionNames) {
         Map<Integer, List<JsonNode>> groups = new LinkedHashMap<>();
         for (JsonNode action : stage.path("actions")) {
+            if (execution.getRetryStageName() != null
+                    && !retryActionNames.contains(action.path("name").asText())) {
+                continue;
+            }
             groups.computeIfAbsent(action.path("runOrder").asInt(1), ignored -> new ArrayList<>()).add(action);
         }
         groups.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> {
@@ -1098,7 +1190,9 @@ public class CodePipelineService {
         switch (provider) {
             case "S3" -> executeS3(execution, action, state);
             case "CodeBuild" -> executeCodeBuild(execution, action, state);
-            case "CodeDeploy" -> executeCodeDeploy(execution, action, state);
+            case "CodeDeploy" -> executeCodeDeploy(pipeline, execution, action, state);
+            case "CodeDeployToECS" -> executeCodeDeployToEcs(execution, action, state);
+            case "CloudFormation" -> executeCloudFormation(pipeline, execution, action, state);
             case "Lambda" -> executeLambda(execution, action, state);
             case "CodePipeline" -> executeNestedPipeline(execution, action, state);
             default -> throw new AwsException("InvalidActionDeclarationException",
@@ -1113,7 +1207,7 @@ public class CodePipelineService {
             String key = config.path("S3ObjectKey").asText(config.path("ObjectKey").asText(null));
             S3Object object = s3Service.getObject(bucket, key);
             for (JsonNode artifact : action.path("outputArtifacts")) {
-                runtimeArtifacts.put(artifactKey(execution, artifact.path("name").asText()), object.getData());
+                execution.getArtifacts().put(artifact.path("name").asText(), object.getData());
             }
             Map<String, Object> revision = new LinkedHashMap<>();
             revision.put("name", state.getActionName());
@@ -1129,7 +1223,7 @@ public class CodePipelineService {
         String bucket = config.path("BucketName").asText(config.path("S3Bucket").asText(null));
         String objectKey = config.path("ObjectKey").asText(state.getActionName() + ".zip");
         JsonNode input = action.path("inputArtifacts").path(0);
-        byte[] data = runtimeArtifacts.get(artifactKey(execution, input.path("name").asText()));
+        byte[] data = execution.getArtifacts().get(input.path("name").asText());
         if (data == null) {
             throw new AwsException("InvalidJobStateException", "Input artifact is not available", 400);
         }
@@ -1139,9 +1233,14 @@ public class CodePipelineService {
 
     private void executeCodeBuild(CodePipelineExecution execution, JsonNode action,
                                   ActionExecution state) throws InterruptedException {
+        for (JsonNode artifact : action.path("outputArtifacts")) {
+            execution.getArtifacts().remove(artifact.path("name").asText());
+        }
         String projectName = action.path("configuration").path("ProjectName").asText(null);
+        ProjectArtifacts artifactsOverride = new ProjectArtifacts();
+        artifactsOverride.setType("CODEPIPELINE");
         Build build = codeBuildService.startBuild(execution.getRegion(), execution.getAccountId(), projectName,
-                null, null, null, null, null, null, null);
+                null, null, artifactsOverride, null, null, null, null);
         state.setExternalExecutionId(build.getId());
         while (!Boolean.TRUE.equals(build.getBuildComplete())) {
             if (abandonExternalActionIfRequested(execution, state)) {
@@ -1160,16 +1259,38 @@ public class CodePipelineService {
             }
             throw new AwsException("ActionExecutionFailed", message, 400);
         }
+        if (!action.path("outputArtifacts").isEmpty()) {
+            byte[] output = codeBuildService.getBuildOutputArtifact(
+                    execution.getRegion(), execution.getAccountId(), build.getId());
+            if (output == null) {
+                throw new AwsException("ActionExecutionFailed",
+                        "CodeBuild build " + build.getId() + " did not produce an output artifact", 400);
+            }
+            for (JsonNode artifact : action.path("outputArtifacts")) {
+                execution.getArtifacts().put(artifact.path("name").asText(), output);
+            }
+        }
     }
 
-    private void executeCodeDeploy(CodePipelineExecution execution, JsonNode action,
+    private void executeCodeDeploy(CodePipelinePipeline pipeline, CodePipelineExecution execution, JsonNode action,
                                    ActionExecution state) throws InterruptedException {
         JsonNode config = action.path("configuration");
         Map<String, Object> revision = null;
         JsonNode input = action.path("inputArtifacts").path(0);
         if (!input.isMissingNode()) {
-            revision = Map.of("revisionType", "S3",
-                    "s3Location", Map.of("bucket", artifactBucket(action), "key", input.path("name").asText()));
+            String artifactName = input.path("name").asText();
+            byte[] artifact = execution.getArtifacts().get(artifactName);
+            if (artifact == null) {
+                throw new AwsException("InvalidJobStateException", "Input artifact is not available", 400);
+            }
+            String bucket = pipeline.getDeclaration().path("artifactStore").path("location").asText(null);
+            if (bucket == null || bucket.isBlank()) {
+                throw new AwsException("InvalidActionDeclarationException", "Pipeline artifact store is required", 400);
+            }
+            String key = execution.getPipelineExecutionId() + "/" + state.getStageName() + "/"
+                    + state.getActionName() + "/" + artifactName + ".zip";
+            s3Service.putObject(bucket, key, artifact, "application/zip", Map.of());
+            revision = Map.of("revisionType", "S3", "s3Location", Map.of("bucket", bucket, "key", key));
         }
         String deploymentId = codeDeployService.createDeployment(
                 execution.getRegion(), config.path("ApplicationName").asText(),
@@ -1195,6 +1316,370 @@ public class CodePipelineService {
         }
     }
 
+    private void executeCodeDeployToEcs(CodePipelineExecution execution, JsonNode action,
+                                        ActionExecution state) throws InterruptedException {
+        if (ecsService == null) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CodeDeployToECS is not available", 400);
+        }
+        JsonNode config = action.path("configuration");
+        String applicationName = requiredCodeDeployToEcsConfiguration(config, "ApplicationName");
+        String deploymentGroupName = requiredCodeDeployToEcsConfiguration(config, "DeploymentGroupName");
+        String taskDefinitionPath = requiredCodeDeployToEcsConfiguration(config, "TaskDefinitionTemplatePath");
+        String appSpecPath = requiredCodeDeployToEcsConfiguration(config, "AppSpecTemplatePath");
+        String taskDefinitionTemplate = readCodeDeployToEcsTemplate(execution, action, taskDefinitionPath);
+        String appSpecTemplate = readCodeDeployToEcsTemplate(execution, action, appSpecPath);
+
+        RegisterTaskDefinitionRequest taskDefinitionRequest;
+        try {
+            taskDefinitionRequest = mapper.readValue(taskDefinitionTemplate, RegisterTaskDefinitionRequest.class);
+        } catch (java.io.IOException exception) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "TaskDefinitionTemplatePath must reference a valid ECS task definition JSON template", 400);
+        }
+        if (taskDefinitionRequest.getFamily() == null || taskDefinitionRequest.getFamily().isBlank()) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "Task definition template must specify family", 400);
+        }
+        TaskDefinition taskDefinition = ecsService.registerTaskDefinition(taskDefinitionRequest, execution.getRegion());
+        String appSpec = appSpecTemplate.replace("<TASK_DEFINITION>", taskDefinition.getTaskDefinitionArn());
+        Map<String, Object> revision = Map.of("revisionType", "AppSpecContent",
+                "appSpecContent", Map.of("content", appSpec));
+        String deploymentId = codeDeployService.createDeployment(execution.getRegion(), applicationName,
+                deploymentGroupName, null, revision, "CodePipeline execution");
+        state.setExternalExecutionId(deploymentId);
+        waitForCodeDeployDeployment(execution, state, deploymentId);
+    }
+
+    private String readCodeDeployToEcsTemplate(CodePipelineExecution execution, JsonNode action, String reference) {
+        int separator = reference.indexOf("::");
+        if (separator <= 0 || separator == reference.length() - 2 || reference.indexOf("::", separator + 2) >= 0) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CodeDeployToECS template paths must use ArtifactName::TemplateFilePath", 400);
+        }
+        String artifactName = reference.substring(0, separator);
+        boolean declaredInput = false;
+        for (JsonNode input : action.path("inputArtifacts")) {
+            if (artifactName.equals(input.path("name").asText())) {
+                declaredInput = true;
+                break;
+            }
+        }
+        if (!declaredInput) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CodeDeployToECS template references an unavailable input artifact: " + artifactName, 400);
+        }
+        byte[] artifact = execution.getArtifacts().get(artifactName);
+        if (artifact == null) {
+            throw new AwsException("InvalidJobStateException", "Input artifact is not available: " + artifactName, 400);
+        }
+        return readTemplateFromArtifact(artifact, reference.substring(separator + 2));
+    }
+
+    private static String requiredCodeDeployToEcsConfiguration(JsonNode config, String field) {
+        String value = config.path(field).asText(null);
+        if (value == null || value.isBlank()) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CodeDeployToECS " + field + " is required", 400);
+        }
+        return value;
+    }
+
+    private void waitForCodeDeployDeployment(CodePipelineExecution execution, ActionExecution state, String deploymentId)
+            throws InterruptedException {
+        Deployment deployment = codeDeployService.getDeployment(execution.getRegion(), deploymentId);
+        while (!List.of("Succeeded", "Failed", "Stopped").contains(deployment.getStatus())) {
+            if (abandonExternalActionIfRequested(execution, state)) {
+                return;
+            }
+            TimeUnit.MILLISECONDS.sleep(POLL_INTERVAL_MS);
+            deployment = codeDeployService.getDeployment(execution.getRegion(), deploymentId);
+        }
+        if (abandonExternalActionIfRequested(execution, state)) {
+            return;
+        }
+        if (!"Succeeded".equals(deployment.getStatus())) {
+            String message = "CodeDeploy deployment finished with " + deployment.getStatus();
+            if (recordExternalActionFailureWhileStopping(execution, state, message)) {
+                return;
+            }
+            throw new AwsException("ActionExecutionFailed", message, 400);
+        }
+    }
+
+    private void executeCloudFormation(CodePipelinePipeline pipeline, CodePipelineExecution execution,
+                                       JsonNode action, ActionExecution state) throws InterruptedException {
+        JsonNode config = action.path("configuration");
+        String stackName = requiredActionConfiguration(config, "StackName");
+        String actionMode = requiredActionConfiguration(config, "ActionMode");
+        if (!"CREATE_UPDATE".equals(actionMode)) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CloudFormation ActionMode must be CREATE_UPDATE", 400);
+        }
+        String templatePath = requiredActionConfiguration(config, "TemplatePath");
+        int separator = templatePath.indexOf("::");
+        if (separator <= 0 || separator == templatePath.length() - 2
+                || templatePath.indexOf("::", separator + 2) >= 0) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CloudFormation TemplatePath must use ArtifactName::TemplateFilePath", 400);
+        }
+        String artifactName = templatePath.substring(0, separator);
+        byte[] artifact = execution.getArtifacts().get(artifactName);
+        if (artifact == null) {
+            throw new AwsException("InvalidJobStateException", "Input artifact is not available: " + artifactName, 400);
+        }
+        String bucket = pipeline.getDeclaration().path("artifactStore").path("location").asText(null);
+        if (bucket == null || bucket.isBlank()) {
+            throw new AwsException("InvalidActionDeclarationException", "Pipeline artifact store is required", 400);
+        }
+        Map<String, ArtifactLocation> artifactLocations = materializeCloudFormationArtifacts(
+                execution, action, state, bucket);
+        String templateBody = readTemplateFromArtifact(artifact, templatePath.substring(separator + 2));
+
+        boolean exists;
+        try {
+            cloudFormationService.describeStacks(stackName, execution.getRegion(), execution.getAccountId());
+            exists = true;
+        } catch (AwsException exception) {
+            if (!"ValidationError".equals(exception.getErrorCode())) {
+                throw exception;
+            }
+            exists = false;
+        }
+        String changeSetName = "codepipeline-" + UUID.randomUUID().toString().substring(0, 8);
+        String changeSetType = exists ? "UPDATE" : "CREATE";
+        cloudFormationService.createChangeSet(stackName, changeSetName, changeSetType, templateBody, null,
+                resolveParameterOverrides(config, execution, action, artifactLocations), List.of(), Map.of(),
+                execution.getRegion(), execution.getAccountId());
+        Future<?> deployment = cloudFormationService.executeChangeSet(
+                stackName, changeSetName, execution.getRegion(), execution.getAccountId());
+        try {
+            deployment.get();
+        } catch (ExecutionException exception) {
+            Throwable cause = exception.getCause();
+            if (cause instanceof AwsException awsException) {
+                throw awsException;
+            }
+            throw new AwsException("ActionExecutionFailed", "CloudFormation deployment failed: "
+                    + Objects.toString(cause.getMessage(), cause.getClass().getSimpleName()), 400);
+        }
+        Stack stack = cloudFormationService.describeStacks(stackName, execution.getRegion(), execution.getAccountId()).get(0);
+        if (!List.of("CREATE_COMPLETE", "UPDATE_COMPLETE").contains(stack.getStatus())) {
+            throw new AwsException("ActionExecutionFailed", "CloudFormation stack " + stackName
+                    + " finished with " + stack.getStatus(), 400);
+        }
+        state.setExternalExecutionId(stack.getStackId());
+    }
+
+    private static String requiredActionConfiguration(JsonNode config, String field) {
+        String value = config.path(field).asText(null);
+        if (value == null || value.isBlank()) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CloudFormation " + field + " is required", 400);
+        }
+        return value;
+    }
+
+    private Map<String, ArtifactLocation> materializeCloudFormationArtifacts(CodePipelineExecution execution,
+                                                                               JsonNode action,
+                                                                               ActionExecution state,
+                                                                               String bucket) {
+        Map<String, ArtifactLocation> locations = new LinkedHashMap<>();
+        for (JsonNode input : action.path("inputArtifacts")) {
+            String artifactName = input.path("name").asText();
+            byte[] artifact = execution.getArtifacts().get(artifactName);
+            if (artifact == null) {
+                throw new AwsException("InvalidJobStateException",
+                        "Input artifact is not available: " + artifactName, 400);
+            }
+            String prefix = execution.getPipelineExecutionId() + "/" + state.getStageName() + "/"
+                    + state.getActionName() + "/" + artifactName;
+            String archiveKey = prefix + ".zip";
+            s3Service.putObject(bucket, archiveKey, artifact, "application/zip", Map.of());
+            List<ArtifactLocation> nestedArtifacts = extractArtifactEntries(bucket, prefix, artifact);
+            ArtifactLocation location = nestedArtifacts.size() == 1 && nestedArtifacts.getFirst().key().endsWith(".zip")
+                    ? nestedArtifacts.getFirst() : new ArtifactLocation(bucket, archiveKey);
+            locations.put(artifactName, location);
+        }
+        return locations;
+    }
+
+    private List<ArtifactLocation> extractArtifactEntries(String bucket, String prefix, byte[] artifact) {
+        List<ArtifactLocation> nestedArtifacts = new ArrayList<>();
+        int extractedSize = 0;
+        try (ZipInputStream zip = new ZipInputStream(new ByteArrayInputStream(artifact))) {
+            ZipEntry entry;
+            while ((entry = zip.getNextEntry()) != null) {
+                if (entry.isDirectory()) {
+                    continue;
+                }
+                String entryName = entry.getName();
+                validateArtifactEntryName(entryName);
+                byte[] data = zip.readNBytes(MAX_ARTIFACT_ENTRY_SIZE + 1);
+                if (data.length > MAX_ARTIFACT_ENTRY_SIZE) {
+                    throw new AwsException("InvalidActionDeclarationException",
+                            "Artifact entry exceeds 256 MB: " + entryName, 400);
+                }
+                if (data.length > MAX_EXTRACTED_ARTIFACT_SIZE - extractedSize) {
+                    throw new AwsException("InvalidActionDeclarationException",
+                            "CloudFormation input artifact expands beyond 256 MB", 400);
+                }
+                extractedSize += data.length;
+                String key = prefix + "/" + entryName;
+                s3Service.putObject(bucket, key, data, "application/octet-stream", Map.of());
+                nestedArtifacts.add(new ArtifactLocation(bucket, key));
+            }
+        } catch (java.io.IOException exception) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "Could not read CloudFormation input artifact", 400);
+        }
+        return nestedArtifacts;
+    }
+
+    private static void validateArtifactEntryName(String entryName) {
+        if (entryName.startsWith("/") || entryName.startsWith("\\") || entryName.contains("\\")) {
+            throw new AwsException("InvalidActionDeclarationException", "Artifact entry path is not safe", 400);
+        }
+        for (String segment : entryName.split("/")) {
+            if (segment.isBlank() || ".".equals(segment) || "..".equals(segment)) {
+                throw new AwsException("InvalidActionDeclarationException", "Artifact entry path is not safe", 400);
+            }
+        }
+    }
+
+    private Map<String, String> resolveParameterOverrides(JsonNode config, CodePipelineExecution execution,
+                                                           JsonNode action,
+                                                           Map<String, ArtifactLocation> artifactLocations) {
+        String overrides = config.path("ParameterOverrides").asText(null);
+        if (overrides == null || overrides.isBlank()) {
+            return Map.of();
+        }
+        try {
+            JsonNode values = mapper.readTree(overrides);
+            if (!values.isObject()) {
+                throw new AwsException("InvalidActionDeclarationException",
+                        "CloudFormation ParameterOverrides must be a JSON object", 400);
+            }
+            Map<String, String> resolved = new LinkedHashMap<>();
+            values.fields().forEachRemaining(entry -> resolved.put(entry.getKey(),
+                    resolveParameterOverride(entry.getValue(), execution, action, artifactLocations)));
+            return resolved;
+        } catch (java.io.IOException exception) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CloudFormation ParameterOverrides must be valid JSON", 400);
+        }
+    }
+
+    private String resolveParameterOverride(JsonNode value, CodePipelineExecution execution, JsonNode action,
+                                            Map<String, ArtifactLocation> artifactLocations) {
+        if (value.isTextual() || value.isNumber() || value.isBoolean()) {
+            return value.asText();
+        }
+        if (!value.isObject()) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "CloudFormation parameter override must be a scalar or supported function", 400);
+        }
+        JsonNode artifactAtt = value.path("Fn::GetArtifactAtt");
+        if (!artifactAtt.isMissingNode()) {
+            List<String> arguments = functionArguments(artifactAtt, 2, "Fn::GetArtifactAtt");
+            ArtifactLocation location = requireArtifactLocation(arguments.getFirst(), action, artifactLocations);
+            return switch (arguments.get(1)) {
+                case "BucketName" -> location.bucket();
+                case "ObjectKey" -> location.key();
+                case "URL" -> "https://s3." + execution.getRegion() + ".amazonaws.com/"
+                        + location.bucket() + "/" + location.key();
+                default -> throw new AwsException("InvalidActionDeclarationException",
+                        "Fn::GetArtifactAtt attribute is not supported: " + arguments.get(1), 400);
+            };
+        }
+        JsonNode getParam = value.path("Fn::GetParam");
+        if (!getParam.isMissingNode()) {
+            List<String> arguments = functionArguments(getParam, 3, "Fn::GetParam");
+            requireArtifactLocation(arguments.getFirst(), action, artifactLocations);
+            byte[] artifact = execution.getArtifacts().get(arguments.getFirst());
+            String contents = readTemplateFromArtifact(artifact, arguments.get(1));
+            try {
+                JsonNode parameters = mapper.readTree(contents);
+                JsonNode parameter = parameters.path(arguments.get(2));
+                if (!parameters.isObject() || parameter.isMissingNode() || !parameter.isValueNode()) {
+                    throw new AwsException("InvalidActionDeclarationException",
+                            "Fn::GetParam could not resolve " + arguments.get(2), 400);
+                }
+                return parameter.asText();
+            } catch (java.io.IOException exception) {
+                throw new AwsException("InvalidActionDeclarationException",
+                        "Fn::GetParam file must contain a JSON object", 400);
+            }
+        }
+        throw new AwsException("InvalidActionDeclarationException",
+                "CloudFormation parameter override uses an unsupported function", 400);
+    }
+
+    private static List<String> functionArguments(JsonNode arguments, int expectedCount, String functionName) {
+        if (!arguments.isArray() || arguments.size() != expectedCount) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    functionName + " requires " + expectedCount + " string arguments", 400);
+        }
+        List<String> values = new ArrayList<>();
+        for (JsonNode argument : arguments) {
+            if (!argument.isTextual() || argument.asText().isBlank()) {
+                throw new AwsException("InvalidActionDeclarationException",
+                        functionName + " requires " + expectedCount + " string arguments", 400);
+            }
+            values.add(argument.asText());
+        }
+        return values;
+    }
+
+    private static ArtifactLocation requireArtifactLocation(String artifactName, JsonNode action,
+                                                            Map<String, ArtifactLocation> artifactLocations) {
+        boolean isInputArtifact = false;
+        for (JsonNode input : action.path("inputArtifacts")) {
+            if (artifactName.equals(input.path("name").asText())) {
+                isInputArtifact = true;
+                break;
+            }
+        }
+        ArtifactLocation location = artifactLocations.get(artifactName);
+        if (!isInputArtifact || location == null) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "Parameter override references an unavailable input artifact: " + artifactName, 400);
+        }
+        return location;
+    }
+
+    private static String readTemplateFromArtifact(byte[] artifact, String templatePath) {
+        if (templatePath.startsWith("/") || templatePath.startsWith("\\") || templatePath.contains("\\")) {
+            throw new AwsException("InvalidActionDeclarationException", "TemplatePath is not safe", 400);
+        }
+        for (String segment : templatePath.split("/")) {
+            if (segment.isBlank() || ".".equals(segment) || "..".equals(segment)) {
+                throw new AwsException("InvalidActionDeclarationException", "TemplatePath is not safe", 400);
+            }
+        }
+        try (ZipInputStream zip = new ZipInputStream(new ByteArrayInputStream(artifact))) {
+            ZipEntry entry;
+            while ((entry = zip.getNextEntry()) != null) {
+                if (!entry.isDirectory() && templatePath.equals(entry.getName())) {
+                    byte[] template = zip.readNBytes(51_201);
+                    if (template.length > 51_200) {
+                        throw new AwsException("InvalidActionDeclarationException",
+                                "CloudFormation template exceeds 51200 bytes", 400);
+                    }
+                    return new String(template, StandardCharsets.UTF_8);
+                }
+            }
+        } catch (java.io.IOException exception) {
+            throw new AwsException("InvalidActionDeclarationException",
+                    "Could not read CloudFormation template from input artifact", 400);
+        }
+        throw new AwsException("InvalidActionDeclarationException",
+                "TemplatePath does not exist in the input artifact", 400);
+    }
+
+    private record ArtifactLocation(String bucket, String key) {
+    }
+
     private boolean abandonExternalActionIfRequested(CodePipelineExecution execution, ActionExecution state) {
         if (!execution.isStopRequested() || !execution.isAbandon()) {
             return false;
@@ -1518,9 +2003,9 @@ public class CodePipelineService {
 
     private ObjectNode executionNode(CodePipelineExecution execution) {
         ObjectNode node = mapper.valueToTree(execution);
-        node.remove(List.of("accountId", "region", "startTime", "lastUpdateTime",
+        node.remove(List.of("accountId", "region", "startTime", "lastUpdateTime", "artifacts",
                 "sourceRevisions", "actionExecutions", "currentStage", "stopRequested", "abandon",
-                "rollbackTargetPipelineExecutionId"));
+                "rollbackTargetPipelineExecutionId", "retryStageName", "retryActionNames"));
         if (execution.getRollbackTargetPipelineExecutionId() != null) {
             node.putObject("rollbackMetadata").put(
                     "rollbackTargetPipelineExecutionId", execution.getRollbackTargetPipelineExecutionId());
@@ -1728,19 +2213,10 @@ public class CodePipelineService {
         return account + ":" + region + ":" + pipelineName;
     }
 
-    private static String artifactKey(CodePipelineExecution execution, String artifactName) {
-        return execution.getAccountId() + ":" + execution.getPipelineExecutionId() + ":" + artifactName;
-    }
-
     private static String artifactBucket(JsonNode action) {
         return action.path("configuration").path("BucketName").asText("codepipeline-artifacts");
     }
 
-    private void clearRuntimeArtifacts(CodePipelineExecution execution) {
-        String prefix = execution.getAccountId() + ":" + execution.getPipelineExecutionId() + ":";
-        runtimeArtifacts.keySet().removeIf(key -> key.startsWith(prefix));
-    }
-
     private static boolean isTerminal(String status) {
         return List.of("Succeeded", "Failed", "Stopped", "Superseded", "Cancelled").contains(status);
     }

src/main/java/io/github/hectorvent/floci/services/codepipeline/model/CodePipelineExecution.java

@@ -26,6 +26,7 @@ public class CodePipelineExecution {
     private Double startTime;
     private Double lastUpdateTime;
     private List<Map<String, Object>> artifactRevisions = new ArrayList<>();
+    private Map<String, byte[]> artifacts = new LinkedHashMap<>();
     private List<Map<String, Object>> sourceRevisions = new ArrayList<>();
     private List<Map<String, String>> variables = new ArrayList<>();
     private Map<String, String> trigger = new LinkedHashMap<>();
@@ -35,6 +36,8 @@ public class CodePipelineExecution {
     private volatile boolean stopRequested;
     private volatile boolean abandon;
     private String rollbackTargetPipelineExecutionId;
+    private String retryStageName;
+    private List<String> retryActionNames = new ArrayList<>();
 
     public String getAccountId() {
         return accountId;
@@ -132,6 +135,14 @@ public class CodePipelineExecution {
         this.artifactRevisions = artifactRevisions;
     }
 
+    public Map<String, byte[]> getArtifacts() {
+        return artifacts;
+    }
+
+    public void setArtifacts(Map<String, byte[]> artifacts) {
+        this.artifacts = artifacts == null ? new LinkedHashMap<>() : artifacts;
+    }
+
     public List<Map<String, Object>> getSourceRevisions() {
         return sourceRevisions;
     }
@@ -205,6 +216,22 @@ public class CodePipelineExecution {
         this.rollbackTargetPipelineExecutionId = rollbackTargetPipelineExecutionId;
     }
 
+    public String getRetryStageName() {
+        return retryStageName;
+    }
+
+    public void setRetryStageName(String retryStageName) {
+        this.retryStageName = retryStageName;
+    }
+
+    public List<String> getRetryActionNames() {
+        return retryActionNames;
+    }
+
+    public void setRetryActionNames(List<String> retryActionNames) {
+        this.retryActionNames = retryActionNames == null ? new ArrayList<>() : retryActionNames;
+    }
+
     @RegisterForReflection
     @JsonInclude(JsonInclude.Include.NON_NULL)
     public static class ActionExecution {

src/main/java/io/github/hectorvent/floci/services/ecs/EcsLoadBalancerRegistrar.java

@@ -61,6 +61,11 @@ public class EcsLoadBalancerRegistrar {
                 throw new AwsException("TargetGroupNotFound",
                         "Target group " + loadBalancer.getTargetGroupArn() + " was not found.", 400);
             }
+            if (taskDefinition.isFargateCompatible() && !"ip".equals(targetGroup.getTargetType())) {
+                throw new AwsException("InvalidParameterException",
+                        "The target group " + loadBalancer.getTargetGroupArn()
+                                + " must have target type ip for a Fargate service.", 400);
+            }
             boolean declared = taskDefinition.getContainerDefinitions().stream()
                     .filter(container -> loadBalancer.getContainerName().equals(container.getName()))
                     .flatMap(container -> container.getPortMappings() == null
@@ -78,7 +83,16 @@ public class EcsLoadBalancerRegistrar {
 
     /** Registers the task's load-balanced containers as ELBv2 targets. */
     public void registerTask(EcsTask task, EcsServiceModel svc, String region) {
-        forEachTarget(task, svc, (tgArn, td) -> {
+        register(task, svc.getLoadBalancers(), region);
+    }
+
+    /** Registers a task-set task against the load balancers declared by that task set. */
+    public void registerTaskSetTask(EcsTask task, List<EcsLoadBalancer> loadBalancers, String region) {
+        register(task, loadBalancers, region);
+    }
+
+    private void register(EcsTask task, List<EcsLoadBalancer> loadBalancers, String region) {
+        forEachTarget(task, loadBalancers, (tgArn, td) -> {
             try {
                 elbV2Service.registerTargets(region, tgArn, List.of(td));
                 LOG.infov("Registered ECS task target {0}:{1} into target group {2}",
@@ -91,7 +105,16 @@ public class EcsLoadBalancerRegistrar {
 
     /** Deregisters the task's load-balanced containers from their ELBv2 target groups. */
     public void deregisterTask(EcsTask task, EcsServiceModel svc, String region) {
-        forEachTarget(task, svc, (tgArn, td) -> {
+        deregister(task, svc.getLoadBalancers(), region);
+    }
+
+    /** Deregisters a task-set task from the load balancers declared by that task set. */
+    public void deregisterTaskSetTask(EcsTask task, List<EcsLoadBalancer> loadBalancers, String region) {
+        deregister(task, loadBalancers, region);
+    }
+
+    private void deregister(EcsTask task, List<EcsLoadBalancer> loadBalancers, String region) {
+        forEachTarget(task, loadBalancers, (tgArn, td) -> {
             try {
                 elbV2Service.deregisterTargets(region, tgArn, List.of(td));
                 LOG.infov("Deregistered ECS task target {0}:{1} from target group {2}",
@@ -102,15 +125,15 @@ public class EcsLoadBalancerRegistrar {
         });
     }
 
-    private void forEachTarget(EcsTask task, EcsServiceModel svc,
-                               BiConsumer<String, TargetDescription> action) {
-        if (svc == null || svc.getLoadBalancers() == null || svc.getLoadBalancers().isEmpty()) {
+    private void forEachTarget(EcsTask task, List<EcsLoadBalancer> loadBalancers,
+                                BiConsumer<String, TargetDescription> action) {
+        if (loadBalancers == null || loadBalancers.isEmpty()) {
             return;
         }
         if (task.getContainers() == null || task.getContainers().isEmpty()) {
             return;
         }
-        for (EcsLoadBalancer lb : svc.getLoadBalancers()) {
+        for (EcsLoadBalancer lb : loadBalancers) {
             if (lb.getTargetGroupArn() == null || lb.getTargetGroupArn().isBlank()) {
                 continue;
             }

src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java

@@ -1166,8 +1166,8 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         }
         EcsCluster cluster = resolveClusterOrDefault(request.getCluster(), region);
         TaskDefinition taskDef = resolveLaunchableTaskDefinition(request.getTaskDefinition(), region);
-        validateEc2BridgeLaunch(cluster, taskDef, request);
-        return launchTasks(cluster, taskDef, request, null, null, region);
+        validateRuntimeLaunch(cluster, taskDef, request);
+        return launchTasks(cluster, taskDef, request, null, null, null, region);
     }
 
     public List<EcsTask> startTask(String clusterRef, List<String> containerInstanceRefs,
@@ -1204,7 +1204,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         for (String instanceRef : instanceRefs) {
             ContainerInstance instance = resolveContainerInstanceOrThrow(cluster.getClusterArn(), instanceRef);
             result.addAll(launchTasks(cluster, taskDef, request,
-                    instance.getContainerInstanceArn(), null, region));
+                    instance.getContainerInstanceArn(), null, null, region));
         }
         return result;
     }
@@ -1218,7 +1218,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
      */
     private List<EcsTask> launchTasks(EcsCluster cluster, TaskDefinition taskDef, RunTaskRequest request,
                                        String containerInstanceArn,
-                                       String owningServiceArn, String region) {
+                                       String owningServiceArn, String taskSetArn, String region) {
         // Fail loudly instead of silently launching zero containers and leaving
         // a task that looks RUNNING with nothing behind it.
         if (taskDef.getContainerDefinitions() == null || taskDef.getContainerDefinitions().isEmpty()) {
@@ -1248,6 +1248,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
             task.setGroup(request.getGroup() != null ? request.getGroup() : "family:" + taskDef.getFamily());
             task.setStartedBy(request.getStartedBy());
             task.setOwningServiceArn(owningServiceArn);
+            task.setTaskSetArn(taskSetArn);
             task.setLastStatus(TaskStatus.PENDING.name());
             task.setDesiredStatus(TaskStatus.RUNNING.name());
             task.setCpu(effectiveCpu(taskDef, request));
@@ -1399,7 +1400,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
             request.setPropagateTags(PROPAGATE_TAGS_TASK_DEFINITION);
         }
         EcsTask task = launchTasks(cluster, taskDef, request, containerInstanceArn,
-                svc.getServiceArn(), region).getFirst();
+                svc.getServiceArn(), null, region).getFirst();
         task.setDeploymentId(deploymentId(svc));
         return task;
     }
@@ -1553,26 +1554,34 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         return new Placement(launchType, capacityProvider);
     }
 
-    /** Floci's Docker-backed service implementation supports EC2 bridge tasks only. */
-    private void validateEc2BridgeLaunch(EcsCluster cluster, TaskDefinition taskDef, RunTaskRequest request) {
-        if (taskDef.getNetworkMode() != NetworkMode.bridge) {
+    /**
+     * Keeps the established EC2 bridge restriction while allowing Docker-backed Fargate tasks to
+     * use their awsvpc task ENI. A capacity-provider strategy determines the effective launch
+     * type, so validate every placement rather than only the explicit launchType member.
+     */
+    private void validateRuntimeLaunch(EcsCluster cluster, TaskDefinition taskDef, RunTaskRequest request) {
+        List<Placement> placements = resolvePlacements(cluster, request, 1);
+        boolean fargate = placements.stream().anyMatch(placement -> placement.launchType() == LaunchType.FARGATE);
+        boolean ec2 = placements.stream().anyMatch(placement -> placement.launchType() == LaunchType.EC2);
+        if (fargate && ec2) {
+            throw new AwsException("ClientException",
+                    "A task definition cannot be placed on both EC2 and FARGATE capacity providers.", 400);
+        }
+        if (fargate) {
+            if (taskDef.getNetworkMode() != NetworkMode.awsvpc) {
+                throw new AwsException("ClientException",
+                        "Fargate tasks require the awsvpc network mode.", 400);
+            }
+            if (!taskDef.isFargateCompatible()) {
+                throw new AwsException("ClientException",
+                        "The task definition is not compatible with the FARGATE launch type.", 400);
+            }
+            return;
+        }
+        if (ec2 && taskDef.getNetworkMode() != NetworkMode.bridge) {
             throw new AwsException("ClientException",
                     "Only bridge network mode is supported for ECS EC2 tasks.", 400);
         }
-        LaunchType launchType = request.getLaunchType();
-        if (launchType != null && launchType != LaunchType.EC2) {
-            throw new AwsException("ClientException",
-                    "Only the EC2 launch type is supported; FARGATE is not available.", 400);
-        }
-        List<CapacityProviderStrategyItem> strategy = request.getCapacityProviderStrategy();
-        if (strategy == null || strategy.isEmpty()) {
-            strategy = clusterDefaultStrategy(cluster);
-        }
-        if (strategy != null && strategy.stream()
-                .anyMatch(item -> FARGATE_CAPACITY_PROVIDERS.contains(item.capacityProvider()))) {
-            throw new AwsException("ClientException",
-                    "FARGATE capacity providers are not supported; use an EC2 capacity provider.", 400);
-        }
     }
 
     /** Accepts the two built-in Fargate providers as well as anything CreateCapacityProvider made. */
@@ -1730,6 +1739,13 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
 
     /** Registers a freshly-started task's containers as ELBv2 targets if its service is load-balanced. */
     private void registerTaskWithLoadBalancers(EcsTask task, EcsCluster cluster, String region) {
+        if (task.getTaskSetArn() != null) {
+            TaskSet taskSet = taskSets.get(task.getTaskSetArn());
+            if (taskSet != null) {
+                lbRegistrar.registerTaskSetTask(task, taskSet.getLoadBalancers(), region);
+            }
+            return;
+        }
         EcsServiceModel svc = owningService(task, cluster);
         if (svc != null && !svc.getLoadBalancers().isEmpty()) {
             lbRegistrar.registerTask(task, svc, region);
@@ -1764,7 +1780,12 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
             return;
         }
         EcsServiceModel svc = owningService(task, cluster);
-        if (svc != null && !svc.getLoadBalancers().isEmpty()) {
+        if (task.getTaskSetArn() != null) {
+            TaskSet taskSet = taskSets.get(task.getTaskSetArn());
+            if (taskSet != null) {
+                lbRegistrar.deregisterTaskSetTask(task, taskSet.getLoadBalancers(), region);
+            }
+        } else if (svc != null && !svc.getLoadBalancers().isEmpty()) {
             lbRegistrar.deregisterTask(task, svc, region);
         }
     }
@@ -2052,7 +2073,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         RunTaskRequest launchValidation = new RunTaskRequest();
         launchValidation.setLaunchType(request.getLaunchType());
         launchValidation.setCapacityProviderStrategy(request.getCapacityProviderStrategy());
-        validateEc2BridgeLaunch(cluster, taskDef, launchValidation);
+        validateRuntimeLaunch(cluster, taskDef, launchValidation);
         lbRegistrar.validateServiceLoadBalancers(request.getLoadBalancers(), taskDef, region);
         String taskDefinition = taskDef.getTaskDefinitionArn();
         String serviceName = request.getServiceName();
@@ -2132,6 +2153,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         cluster.setActiveServicesCount(cluster.getActiveServicesCount() + 1);
         persistCluster(region, cluster);
         recordServiceDeployment(svc, taskDefinition, region);
+        if (svc.getLaunchType() == LaunchType.FARGATE && "CODE_DEPLOY".equals(svc.getDeploymentController())) {
+            createInitialCodeDeployTaskSet(cluster, svc, region);
+        }
         if (eventPublisher != null) {
             eventPublisher.emitDeploymentStateChange(svc, "SERVICE_DEPLOYMENT_STARTED",
                     "ECS deployment " + svc.getDeploymentId() + " started.", region);
@@ -2140,6 +2164,31 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         return svc;
     }
 
+    /**
+     * A Fargate CodeDeploy service starts with its blue deployment already placed. EC2 services
+     * retain their existing controller-owned task-set lifecycle.
+     */
+    private void createInitialCodeDeployTaskSet(EcsCluster cluster, EcsServiceModel svc, String region) {
+        CreateTaskSetRequest request = new CreateTaskSetRequest();
+        request.setCluster(cluster.getClusterName());
+        request.setService(svc.getServiceName());
+        request.setTaskDefinition(svc.getTaskDefinition());
+        if (svc.getCapacityProviderStrategy() != null && !svc.getCapacityProviderStrategy().isEmpty()) {
+            request.setCapacityProviderStrategy(svc.getCapacityProviderStrategy());
+        } else {
+            request.setLaunchType(LaunchType.FARGATE);
+        }
+        request.setPlatformVersion(svc.getPlatformVersion());
+        request.setScaleValue(100.0);
+        request.setNetworkConfiguration(svc.getNetworkConfiguration());
+        request.setLoadBalancers(svc.getLoadBalancers());
+        request.setServiceRegistries(svc.getServiceRegistries());
+
+        TaskSet taskSet = createTaskSet(request, region);
+        taskSet.setStatus("PRIMARY");
+        taskSet.setUpdatedAt(Instant.now());
+    }
+
     public EcsServiceModel updateService(String clusterRef, String serviceName, String taskDefinition,
                                           Integer desiredCount, NetworkConfiguration networkConfiguration,
                                           String region) {
@@ -2436,6 +2485,20 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
                                 t.getTaskArn(), e.getMessage());
                     }
                 });
+        // Service ARNs are reused when a same-named service is recreated. Retaining its task
+        // sets would make the new service reconcile and relaunch the deleted deployment.
+        taskSets.entrySet().removeIf(entry -> svc.getServiceArn().equals(entry.getValue().getServiceArn()));
+        // Inactive services are not reconciled, so retain the final observed task state rather
+        // than leaving the count that was recorded before the service drain.
+        svc.setRunningCount((int) tasks.values().stream()
+                .filter(t -> ownedBy(t, svc, cluster))
+                .filter(t -> TaskStatus.RUNNING.name().equals(t.getLastStatus()))
+                .count());
+        svc.setPendingCount((int) tasks.values().stream()
+                .filter(t -> ownedBy(t, svc, cluster))
+                .filter(t -> TaskStatus.PENDING.name().equals(t.getLastStatus()))
+                .count());
+        services.put(key, svc);
         return svc;
     }
 
@@ -3285,7 +3348,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
     private static final Set<String> TASK_SET_CONTROLLERS = Set.of("EXTERNAL", "CODE_DEPLOY");
     /** The only unit {@code Scale} accepts, so the computed count is always a percentage. */
     private static final String SCALE_UNIT_PERCENT = "PERCENT";
-    /** The stability a task set reports while Floci places no tasks of its own for one. */
+    /** Task sets report steady state when their running task count reaches the computed desired count. */
     private static final String STABILITY_STATUS_STEADY_STATE = "STEADY_STATE";
 
     public TaskSet createTaskSet(String clusterRef, String serviceRef, String taskDefinitionRef,
@@ -3363,7 +3426,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         ts.setUpdatedAt(Instant.now());
         applyTaskSetScale(ts, svc);
 
+        lbRegistrar.validateServiceLoadBalancers(ts.getLoadBalancers(), taskDef, region);
         taskSets.put(taskSetArn, ts);
+        reconcileTaskSet(cluster, svc, ts, region);
         return ts;
     }
 
@@ -3391,15 +3456,54 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
      * Stamps {@code computedDesiredCount}, which AWS derives from the service's desired count and
      * the task set's scale percentage, always rounding up: a computed 1.2 becomes 2 tasks.
      *
-     * <p>Floci places no tasks for a task set, so its running and pending counts stay at zero. A
-     * stability status derived from them would leave every set on a service with a desired count
-     * above zero in {@code STABILIZING} for ever, hanging anything that waits for the set to
-     * stabilise, so the set is reported {@code STEADY_STATE} until Floci tracks tasks for it.
      */
     private void applyTaskSetScale(TaskSet ts, EcsServiceModel svc) {
         ts.setComputedDesiredCount(
                 (int) Math.ceil(svc.getDesiredCount() * ts.getScaleValue() / 100.0));
-        ts.setStabilityStatus(STABILITY_STATUS_STEADY_STATE);
+    }
+
+    /** Places and drains task-set tasks with the same runtime launcher used by the service scheduler. */
+    private void reconcileTaskSet(EcsCluster cluster, EcsServiceModel svc, TaskSet ts, String region) {
+        List<EcsTask> running = tasks.values().stream()
+                .filter(task -> ts.getTaskSetArn().equals(task.getTaskSetArn()))
+                .filter(task -> TaskStatus.RUNNING.name().equals(task.getLastStatus()))
+                .toList();
+        int toStart = ts.getComputedDesiredCount() - running.size();
+        if (toStart > 0) {
+            TaskDefinition taskDef = resolveTaskDefinitionOrThrow(ts.getTaskDefinition(), region);
+            for (int i = 0; i < toStart; i++) {
+                RunTaskRequest request = new RunTaskRequest();
+                request.setCount(1);
+                request.setGroup("service:" + svc.getServiceName());
+                request.setStartedBy(ts.getStartedBy());
+                request.setNetworkConfiguration(ts.getNetworkConfiguration());
+                request.setCapacityProviderStrategy(ts.getCapacityProviderStrategy());
+                if (ts.getCapacityProviderStrategy() == null || ts.getCapacityProviderStrategy().isEmpty()) {
+                    request.setLaunchType(ts.getLaunchType());
+                }
+                validateRuntimeLaunch(cluster, taskDef, request);
+                launchTasks(cluster, taskDef, request, null, svc.getServiceArn(), ts.getTaskSetArn(), region);
+            }
+        } else if (toStart < 0) {
+            running.stream().limit(-toStart).forEach(task -> stopTask(cluster.getClusterName(), task.getTaskArn(),
+                    "Task set scale-in", STOP_CODE_SERVICE_SCHEDULER_INITIATED, region));
+        }
+        updateTaskSetRuntime(ts);
+    }
+
+    private void updateTaskSetRuntime(TaskSet ts) {
+        int running = (int) tasks.values().stream()
+                .filter(task -> ts.getTaskSetArn().equals(task.getTaskSetArn()))
+                .filter(task -> TaskStatus.RUNNING.name().equals(task.getLastStatus()))
+                .count();
+        int pending = (int) tasks.values().stream()
+                .filter(task -> ts.getTaskSetArn().equals(task.getTaskSetArn()))
+                .filter(task -> TaskStatus.PENDING.name().equals(task.getLastStatus()))
+                .count();
+        ts.setRunningCount(running);
+        ts.setPendingCount(pending);
+        ts.setStabilityStatus(running == ts.getComputedDesiredCount() && pending == 0
+                ? STABILITY_STATUS_STEADY_STATE : "STABILIZING");
         ts.setStabilityStatusAt(Instant.now());
     }
 
@@ -3416,6 +3520,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         ts.setScaleUnit(SCALE_UNIT_PERCENT);
         ts.setUpdatedAt(Instant.now());
         applyTaskSetScale(ts, svc);
+        reconcileTaskSet(cluster, svc, ts, region);
         return ts;
     }
 
@@ -3436,6 +3541,11 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         }
         ts.setStatus("DRAINING");
         ts.setUpdatedAt(Instant.now());
+        tasks.values().stream()
+                .filter(task -> ts.getTaskSetArn().equals(task.getTaskSetArn()))
+                .filter(task -> !TaskStatus.STOPPED.name().equals(task.getLastStatus()))
+                .forEach(task -> stopTask(cluster.getClusterName(), task.getTaskArn(),
+                        "Task set deleted", STOP_CODE_SERVICE_SCHEDULER_INITIATED, region));
         taskSets.remove(ts.getTaskSetArn());
         return ts;
     }
@@ -4128,9 +4238,20 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         if (!"ACTIVE".equals(svc.getStatus())) {
             return;
         }
-
+        String deploymentController = svc.getDeploymentController() != null
+                ? svc.getDeploymentController() : DEFAULT_DEPLOYMENT_CONTROLLER;
         String region = extractRegionFromServiceKey(key);
         String clusterName = extractClusterNameFromServiceKey(key);
+        if (!DEFAULT_DEPLOYMENT_CONTROLLER.equals(deploymentController)) {
+            EcsCluster cluster = resolveClusterOrDefault(clusterName, region);
+            for (TaskSet taskSet : taskSetsFor(svc)) {
+                reconcileTaskSet(cluster, svc, taskSet, region);
+            }
+            // Task sets are scheduled by their CodeDeploy or external controller, not by the
+            // normal service reconciler. Otherwise green tasks would be supplemented by a second
+            // desiredCount-worth of regular service tasks on every tick.
+            return;
+        }
 
         if (SCHEDULING_DAEMON.equals(svc.getSchedulingStrategy())) {
             reconcileDaemonService(key, svc, clusterName, region);
@@ -4140,6 +4261,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
         EcsCluster cluster = resolveClusterOrDefault(clusterName, region);
         List<EcsTask> runningTasks = tasks.values().stream()
                 .filter(t -> ownedBy(t, svc, cluster))
+                // Task sets own their desired count. Including them here lets the rolling-service
+                // reconciler replace a CodeDeploy green task before traffic is switched.
+                .filter(t -> t.getTaskSetArn() == null)
                 .filter(t -> TaskStatus.RUNNING.name().equals(t.getLastStatus()))
                 .toList();
         long running = runningTasks.size();

src/main/java/io/github/hectorvent/floci/services/ecs/model/EcsTask.java

@@ -21,6 +21,8 @@ public class EcsTask {
      * caller-supplied label and therefore cannot be trusted to establish ownership.
      */
     private String owningServiceArn;
+    /** Task set that launched this service-owned task, or null for a rolling-service task. */
+    private String taskSetArn;
     private LaunchType launchType;
     private String lastStatus;
     private String desiredStatus;
@@ -81,6 +83,7 @@ public class EcsTask {
         this.taskDefinitionArn = other.taskDefinitionArn;
         this.group = other.group;
         this.owningServiceArn = other.owningServiceArn;
+        this.taskSetArn = other.taskSetArn;
         this.launchType = other.launchType;
         this.lastStatus = other.lastStatus;
         this.desiredStatus = other.desiredStatus;
@@ -138,6 +141,9 @@ public class EcsTask {
     public String getOwningServiceArn() { return owningServiceArn; }
     public void setOwningServiceArn(String owningServiceArn) { this.owningServiceArn = owningServiceArn; }
 
+    public String getTaskSetArn() { return taskSetArn; }
+    public void setTaskSetArn(String taskSetArn) { this.taskSetArn = taskSetArn; }
+
     public LaunchType getLaunchType() { return launchType; }
     public void setLaunchType(LaunchType launchType) { this.launchType = launchType; }
 

src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java

@@ -764,6 +764,9 @@ public class RdsService implements Resettable, ResourceProvider {
                 } catch (RuntimeException rollbackFailure) {
                     e.addSuppressed(rollbackFailure);
                 }
+                if (!persistedResourceUsesProxyPort(proxyPort)) {
+                    releaseProxyPort(proxyPort);
+                }
                 throw e;
             }
         }

src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationLambdaAddressingIntegrationTest.java

@@ -3,8 +3,16 @@ package io.github.hectorvent.floci.services.cloudformation;
 import io.quarkus.test.junit.QuarkusTest;
 import org.junit.jupiter.api.Test;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
 import static io.restassured.RestAssured.given;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 
@@ -259,6 +267,92 @@ class CloudFormationLambdaAddressingIntegrationTest {
         awaitStackGone(stackName);
     }
 
+    @Test
+    void updatePublishesVersionAfterDependentFunctionS3CodeChanges() throws IOException {
+        String suffix = Long.toString(System.nanoTime(), 36);
+        String bucket = "cfn-version-code-" + suffix;
+        String stackName = "cfn-version-code-" + suffix;
+        String fnName = "version-code-fn-" + suffix;
+
+        given().when().put("/" + bucket).then().statusCode(200);
+        given().contentType("application/zip").body(handlerZip("v1"))
+                .when().put("/" + bucket + "/v1.zip").then().statusCode(200);
+
+        createStack(stackName, versionCodeTemplate(fnName, bucket, "v1.zip"));
+
+        String versionOneSha = given()
+                .queryParam("Qualifier", "1")
+                .when().get("/2015-03-31/functions/" + fnName + "/configuration")
+                .then().statusCode(200)
+                .extract().path("CodeSha256");
+
+        given().contentType("application/zip").body(handlerZip("v2"))
+                .when().put("/" + bucket + "/v2.zip").then().statusCode(200);
+
+        updateStack(stackName, versionCodeTemplate(fnName, bucket, "v2.zip"));
+
+        String latestSha = given()
+                .when().get("/2015-03-31/functions/" + fnName + "/configuration")
+                .then().statusCode(200)
+                .extract().path("CodeSha256");
+
+        given()
+                .queryParam("Qualifier", "2")
+                .when().get("/2015-03-31/functions/" + fnName + "/configuration")
+                .then().statusCode(200)
+                .body("CodeSha256", equalTo(latestSha))
+                .body("CodeSha256", not(equalTo(versionOneSha)));
+    }
+
+    private static void createStack(String stackName, String template) {
+        given()
+                .contentType("application/x-www-form-urlencoded")
+                .header("Authorization", CFN_AUTH)
+                .formParam("Action", "CreateStack")
+                .formParam("StackName", stackName)
+                .formParam("TemplateBody", template)
+        .when()
+                .post("/")
+        .then()
+                .statusCode(200);
+    }
+
+    private static void updateStack(String stackName, String template) {
+        given()
+                .contentType("application/x-www-form-urlencoded")
+                .header("Authorization", CFN_AUTH)
+                .formParam("Action", "UpdateStack")
+                .formParam("StackName", stackName)
+                .formParam("TemplateBody", template)
+        .when()
+                .post("/")
+        .then()
+                .statusCode(200);
+    }
+
+    private static String versionCodeTemplate(String fnName, String bucket, String key) {
+        return """
+                {"Resources": {
+                  "Ver": {"Type": "AWS::Lambda::Version", "DependsOn": "Fn",
+                    "Properties": {"FunctionName": {"Ref": "Fn"}}},
+                  "Fn": {"Type": "AWS::Lambda::Function", "Properties": {
+                    "FunctionName": "%s", "Runtime": "nodejs20.x", "Handler": "index.handler",
+                    "Role": "arn:aws:iam::000000000000:role/r",
+                    "Code": {"S3Bucket": "%s", "S3Key": "%s"}}}
+                }}
+                """.formatted(fnName, bucket, key);
+    }
+
+    private static byte[] handlerZip(String response) throws IOException {
+        ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+        try (ZipOutputStream zip = new ZipOutputStream(bytes)) {
+            zip.putNextEntry(new ZipEntry("index.js"));
+            zip.write(("exports.handler = async () => '" + response + "';").getBytes(StandardCharsets.UTF_8));
+            zip.closeEntry();
+        }
+        return bytes.toByteArray();
+    }
+
     private static void awaitStackGone(String stackName) throws InterruptedException {
         for (int i = 0; i < 200; i++) {
             var response = given()

src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/LambdaVersionAliasCfnProvisionerTest.java

@@ -90,6 +90,21 @@ class LambdaVersionAliasCfnProvisionerTest {
                 r.getAttributes().get("FunctionArn"));
     }
 
+    @Test
+    void updatePublishesThroughLambdaInsteadOfRetainingThePriorVersion() {
+        when(lambda.publishVersion(eq("us-east-1"), eq("my-fn"), isNull())).thenReturn(publishedVersion());
+
+        ObjectNode props = mapper.createObjectNode();
+        props.put("FunctionName", "my-fn");
+        StackResource r = resource("AWS::Lambda::Version", "MyFuncVersion");
+        r.setPhysicalId("arn:aws:lambda:us-east-1:000000000000:function:my-fn:2");
+
+        provisioner.provision(r, props, ctx());
+
+        verify(lambda).publishVersion("us-east-1", "my-fn", null);
+        assertEquals("arn:aws:lambda:us-east-1:000000000000:function:my-fn:3", r.getPhysicalId());
+    }
+
     @Test
     void versionWithoutFunctionNameFails() {
         StackResource r = resource("AWS::Lambda::Version", "MyFuncVersion");

src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildDockerIntegrationTest.java

@@ -0,0 +1,104 @@
+package io.github.hectorvent.floci.services.codebuild;
+
+import com.github.dockerjava.api.DockerClient;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@QuarkusTest
+class CodeBuildDockerIntegrationTest {
+
+    private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+
+    @Inject
+    DockerClient dockerClient;
+
+    @BeforeAll
+    static void configureRestAssured() {
+        RestAssuredJsonUtils.configureAwsContentTypes();
+    }
+
+    @BeforeEach
+    void requireDocker() {
+        try {
+            dockerClient.pingCmd().exec();
+        } catch (Exception e) {
+            Assumptions.assumeTrue(false,
+                    "Docker daemon must be available for CodeBuild Docker integration tests");
+        }
+    }
+
+    @Test
+    void privilegedBuildCanUseMountedDockerSocket() throws InterruptedException {
+        String projectName = "codebuild-docker-" + UUID.randomUUID().toString().substring(0, 8);
+        given()
+                .header("X-Amz-Target", "CodeBuild_20161006.CreateProject")
+                .contentType(CONTENT_TYPE)
+                .body("""
+                        {
+                          "name": "%s",
+                          "source": {"type": "NO_SOURCE"},
+                          "artifacts": {"type": "NO_ARTIFACTS"},
+                          "environment": {
+                            "type": "LINUX_CONTAINER",
+                            "image": "docker:27-cli",
+                            "computeType": "BUILD_GENERAL1_SMALL",
+                            "privilegedMode": true
+                          },
+                          "serviceRole": "arn:aws:iam::000000000000:role/codebuild-role"
+                        }
+                        """.formatted(projectName))
+                .when()
+                .post("/")
+                .then()
+                .statusCode(200);
+
+        String buildId = given()
+                .header("X-Amz-Target", "CodeBuild_20161006.StartBuild")
+                .contentType(CONTENT_TYPE)
+                .body("""
+                        {
+                          "projectName": "%s",
+                          "buildspecOverride": "version: 0.2\\nphases:\\n  build:\\n    commands:\\n      - docker version\\n      - printf 'FROM scratch\\\\n' > Dockerfile\\n      - docker build -t floci-codebuild-socket-test .\\n      - docker image rm floci-codebuild-socket-test\\n"
+                        }
+                        """.formatted(projectName))
+                .when()
+                .post("/")
+                .then()
+                .statusCode(200)
+                .extract()
+                .path("build.id");
+
+        String status = waitForBuild(buildId);
+        assertEquals("SUCCEEDED", status);
+    }
+
+    private String waitForBuild(String buildId) throws InterruptedException {
+        for (int attempts = 0; attempts < 120; attempts++) {
+            String status = given()
+                    .header("X-Amz-Target", "CodeBuild_20161006.BatchGetBuilds")
+                    .contentType(CONTENT_TYPE)
+                    .body("{\"ids\":[\"" + buildId + "\"]}")
+                    .when()
+                    .post("/")
+                    .then()
+                    .statusCode(200)
+                    .extract()
+                    .path("builds[0].buildStatus");
+            if (!"IN_PROGRESS".equals(status)) {
+                return status;
+            }
+            Thread.sleep(500);
+        }
+        return "TIMED_OUT";
+    }
+}

src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunnerTest.java

@@ -0,0 +1,52 @@
+package io.github.hectorvent.floci.services.codebuild;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+class CodeBuildRunnerTest {
+
+    @Test
+    void privilegedBuildMountsConfiguredDockerSocket() {
+        ContainerBuilder.Builder container = mock(ContainerBuilder.Builder.class);
+
+        CodeBuildRunner.exposeDockerSocketIfPrivileged(container, true,
+                dockerConfig("unix:///run/user/1000/docker.sock"));
+
+        verify(container).withBind("/run/user/1000/docker.sock", "/var/run/docker.sock");
+    }
+
+    @Test
+    void unprivilegedBuildCannotAccessDockerSocket() {
+        ContainerBuilder.Builder container = mock(ContainerBuilder.Builder.class);
+
+        CodeBuildRunner.exposeDockerSocketIfPrivileged(container, false,
+                dockerConfig("unix:///run/user/1000/docker.sock"));
+
+        verifyNoInteractions(container);
+    }
+
+    @Test
+    void privilegedBuildDoesNotInventSocketForTcpDockerHost() {
+        ContainerBuilder.Builder container = mock(ContainerBuilder.Builder.class);
+
+        CodeBuildRunner.exposeDockerSocketIfPrivileged(container, true,
+                dockerConfig("tcp://docker.example.test:2376"));
+
+        verifyNoInteractions(container);
+    }
+
+    private EmulatorConfig.DockerConfig dockerConfig(String dockerHost) {
+        EmulatorConfig.DockerConfig dockerConfig = mock(EmulatorConfig.DockerConfig.class);
+        when(dockerConfig.dockerHost()).thenReturn(dockerHost);
+        when(dockerConfig.dockerConfigPath()).thenReturn(Optional.empty());
+        return dockerConfig;
+    }
+}

src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServiceTest.java

@@ -5,21 +5,28 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import io.github.hectorvent.floci.core.common.RegionResolver;
 import io.github.hectorvent.floci.services.codedeploy.model.Deployment;
 import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.services.ec2.model.Instance;
+import io.github.hectorvent.floci.services.ec2.model.Reservation;
 import io.github.hectorvent.floci.services.ecs.EcsService;
 import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
 import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.model.LambdaAlias;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
 import io.github.hectorvent.floci.services.lambda.model.InvocationType;
 import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
 import io.github.hectorvent.floci.services.ssm.SsmCommandService;
 import io.github.hectorvent.floci.testing.MutableClock;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
 
 import java.time.Clock;
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.awaitility.Awaitility.await;
@@ -31,6 +38,7 @@ import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -48,6 +56,14 @@ class CodeDeployServiceTest {
     @BeforeEach
     void setUp() {
         lambdaService = mock(LambdaService.class);
+        LambdaAlias alias = new LambdaAlias();
+        alias.setFunctionVersion("1");
+        when(lambdaService.getAlias(eq(REGION), anyString(), anyString())).thenReturn(alias);
+        LambdaFunction versionOne = new LambdaFunction();
+        versionOne.setVersion("1");
+        LambdaFunction versionTwo = new LambdaFunction();
+        versionTwo.setVersion("2");
+        when(lambdaService.listVersionsByFunction(eq(REGION), anyString())).thenReturn(List.of(versionOne, versionTwo));
         ssmCommandService = mock(SsmCommandService.class);
         service = newService(Duration.ofHours(1));
     }
@@ -57,9 +73,13 @@ class CodeDeployServiceTest {
     }
 
     private CodeDeployService newService(Duration hookCallbackTimeout, Clock clock) {
+        return newService(hookCallbackTimeout, clock, Thread::sleep);
+    }
+
+    private CodeDeployService newService(Duration hookCallbackTimeout, Clock clock, CodeDeployService.Sleeper sleeper) {
         CodeDeployService svc = new CodeDeployService(lambdaService, mock(EcsService.class),
                 mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
-                new RegionResolver(REGION, ACCOUNT_ID), null, hookCallbackTimeout, clock);
+                new RegionResolver(REGION, ACCOUNT_ID), null, hookCallbackTimeout, clock, null, sleeper);
         svc.initializeStorage();
         return svc;
     }
@@ -155,6 +175,43 @@ class CodeDeployServiceTest {
         assertEquals("Succeeded", deployment.getStatus());
     }
 
+    @Test
+    void ec2TagFilterTargetsRegisteredInstanceAndRunsHook() {
+        Ec2Service ec2Service = mock(Ec2Service.class);
+        CodeDeployService ec2TargetService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                mock(ElbV2Service.class), ssmCommandService, ec2Service, new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        ec2TargetService.initializeStorage();
+
+        Instance instance = new Instance();
+        instance.setInstanceId("i-tagged");
+        Reservation reservation = new Reservation();
+        reservation.setInstances(List.of(instance));
+        when(ec2Service.describeInstances(eq(REGION), eq(List.of()), any())).thenReturn(List.of(reservation));
+        when(ssmCommandService.isInstanceRegistered("i-tagged", REGION)).thenReturn(true);
+        when(ssmCommandService.sendCommandToInstance(eq("i-tagged"), eq("AWS-RunShellScript"), any(), eq(30), eq(REGION)))
+                .thenReturn("cmd-tagged");
+        when(ssmCommandService.getCommandInvocationStatus("cmd-tagged", "i-tagged", REGION)).thenReturn("Success");
+
+        ec2TargetService.createApplication(REGION, "ec2-tag-app", "Server", null);
+        ec2TargetService.createDeploymentGroup(REGION, "ec2-tag-app", "ec2-tag-group",
+                "CodeDeployDefault.AllAtOnce", ROLE_ARN,
+                Map.of("ec2TagFilters", List.of(Map.of("Key", "Environment", "Value", "test", "Type", "KEY_AND_VALUE"))));
+
+        String deploymentId = ec2TargetService.createDeployment(REGION, "ec2-tag-app", "ec2-tag-group", null,
+                serverRevision(30), "EC2 tag target");
+        Deployment deployment = awaitTerminal(ec2TargetService, deploymentId, Duration.ofSeconds(5));
+
+        assertEquals("Succeeded", deployment.getStatus());
+        assertEquals("i-tagged", ec2TargetService.listDeploymentTargets(REGION, deploymentId).getFirst());
+        ArgumentCaptor<Map<String, List<String>>> filters = ArgumentCaptor.forClass(Map.class);
+        verify(ec2Service).describeInstances(eq(REGION), eq(List.of()), filters.capture());
+        assertEquals(Map.of("tag:Environment", List.of("test")), filters.getValue());
+        verify(ssmCommandService).sendCommandToInstance(eq("i-tagged"), eq("AWS-RunShellScript"),
+                eq(Map.of("commands", List.of("sh scripts/start_server.sh"))), eq(30), eq(REGION));
+    }
+
     // ---- Lambda platform hooks ----------------------------------------------------------
 
     @Test
@@ -205,7 +262,7 @@ class CodeDeployServiceTest {
         when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
                 .thenAnswer(invocation -> {
                     byte[] payload = invocation.getArgument(2);
-                    service.putLifecycleEventHookExecutionStatus("ignored", executionIdFrom(payload), "Succeeded");
+                    service.putLifecycleEventHookExecutionStatus(deploymentIdFrom(payload), executionIdFrom(payload), "Succeeded");
                     return cleanInvoke;
                 });
 
@@ -270,7 +327,7 @@ class CodeDeployServiceTest {
         when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
                 .thenAnswer(invocation -> {
                     byte[] payload = invocation.getArgument(2);
-                    service.putLifecycleEventHookExecutionStatus("ignored", executionIdFrom(payload), "Failed");
+                    service.putLifecycleEventHookExecutionStatus(deploymentIdFrom(payload), executionIdFrom(payload), "Failed");
                     return cleanInvoke;
                 });
 
@@ -304,7 +361,7 @@ class CodeDeployServiceTest {
                     byte[] payload = invocation.getArgument(2);
                     String executionId = executionIdFrom(payload);
                     capturedExecutionId.set(executionId);
-                    service.putLifecycleEventHookExecutionStatus("ignored", executionId, "Succeeded");
+                    service.putLifecycleEventHookExecutionStatus(deploymentIdFrom(payload), executionId, "Succeeded");
                     return cleanInvoke;
                 });
 
@@ -312,11 +369,131 @@ class CodeDeployServiceTest {
         Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));
         assertEquals("Succeeded", deployment.getStatus());
 
-        service.putLifecycleEventHookExecutionStatus("ignored", capturedExecutionId.get(), "Failed");
+        org.junit.jupiter.api.Assertions.assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,
+                () -> service.putLifecycleEventHookExecutionStatus(deploymentId, capturedExecutionId.get(), "Failed"));
 
         assertEquals("Succeeded", service.getDeployment(REGION, deploymentId).getStatus());
     }
 
+    @Test
+    void failedPostTrafficHookRestoresExactAliasRoutingWhenRollbackIsEnabled() {
+        service.createApplication(REGION, "app-rollback", "Lambda", null);
+        service.createDeploymentGroup(REGION, "app-rollback", "group-rollback",
+                "CodeDeployDefault.LambdaAllAtOnce", ROLE_ARN,
+                Map.of("autoRollbackConfiguration", Map.of("enabled", true,
+                        "events", List.of("DEPLOYMENT_FAILURE"))));
+        LambdaAlias alias = new LambdaAlias();
+        alias.setFunctionVersion("1");
+        alias.setRoutingConfig(Map.of("3", 0.2));
+        when(lambdaService.getAlias(eq(REGION), anyString(), anyString())).thenReturn(alias);
+        InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-rollback");
+        when(lambdaService.invoke(eq(REGION), eq("afterHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
+                .thenAnswer(invocation -> {
+                    byte[] payload = invocation.getArgument(2);
+                    service.putLifecycleEventHookExecutionStatus(deploymentIdFrom(payload), executionIdFrom(payload), "Failed");
+                    return cleanInvoke;
+                });
+
+        String deploymentId = createLambdaDeployment("app-rollback", "group-rollback", null, "afterHook");
+
+        assertEquals("Failed", awaitTerminal(deploymentId, Duration.ofSeconds(5)).getStatus());
+        ArgumentCaptor<Map<String, Double>> routing = ArgumentCaptor.forClass(Map.class);
+        ArgumentCaptor<String> version = ArgumentCaptor.forClass(String.class);
+        verify(lambdaService, times(2)).updateAlias(eq(REGION), eq("myFunc"), eq("live"), version.capture(),
+                any(), routing.capture());
+        assertEquals("2", version.getAllValues().get(0));
+        assertEquals("1", version.getAllValues().get(1));
+        assertEquals(Map.of("3", 0.2), routing.getAllValues().get(1));
+    }
+
+    @Test
+    void lambdaAppSpecRequiresCurrentVersionAndMatchingAlias() {
+        createLambdaAppAndGroup("app-invalid-appspec", "group-invalid-appspec");
+        Map<String, Object> revision = Map.of("revisionType", "AppSpecContent", "appSpecContent", Map.of("content", """
+                version: '0.0'
+                Resources:
+                  - myFunc:
+                      Type: AWS::Lambda::Function
+                      Properties:
+                        Name: myFunc
+                        Alias: live
+                        TargetVersion: '2'
+                """));
+
+        io.github.hectorvent.floci.core.common.AwsException error = org.junit.jupiter.api.Assertions.assertThrows(
+                io.github.hectorvent.floci.core.common.AwsException.class,
+                () -> service.createDeployment(REGION, "app-invalid-appspec", "group-invalid-appspec", null, revision, null));
+
+        assertEquals("InvalidRevisionException", error.getErrorCode());
+    }
+
+    @Test
+    void lambdaCanaryUsesFullNamedConfigurationInterval() {
+        AtomicLong sleptMillis = new AtomicLong();
+        CodeDeployService noWaitService = newService(Duration.ofHours(1), Clock.systemUTC(), sleptMillis::set);
+        noWaitService.createApplication(REGION, "app-canary", "Lambda", null);
+        noWaitService.createDeploymentGroup(REGION, "app-canary", "group-canary",
+                "CodeDeployDefault.LambdaCanary10Percent5Minutes", ROLE_ARN, null);
+
+        String deploymentId = noWaitService.createDeployment(REGION, "app-canary", "group-canary", null,
+                lambdaRevision(null, null), null);
+
+        assertEquals("Succeeded", awaitTerminal(noWaitService, deploymentId, Duration.ofSeconds(5)).getStatus());
+        assertEquals(Duration.ofMinutes(5).toMillis(), sleptMillis.get());
+    }
+
+    @Test
+    void lambdaLinearUsesFullNamedConfigurationInterval() {
+        AtomicLong sleptMillis = new AtomicLong();
+        CodeDeployService noWaitService = newService(Duration.ofHours(1), Clock.systemUTC(), sleptMillis::addAndGet);
+        noWaitService.createApplication(REGION, "app-linear", "Lambda", null);
+        noWaitService.createDeploymentGroup(REGION, "app-linear", "group-linear",
+                "CodeDeployDefault.LambdaLinear10PercentEvery1Minute", ROLE_ARN, null);
+
+        String deploymentId = noWaitService.createDeployment(REGION, "app-linear", "group-linear", null,
+                lambdaRevision(null, null), null);
+
+        assertEquals("Succeeded", awaitTerminal(noWaitService, deploymentId, Duration.ofSeconds(5)).getStatus());
+        assertEquals(Duration.ofMinutes(9).toMillis(), sleptMillis.get());
+    }
+
+    @Test
+    void ecsCanaryUsesFullNamedConfigurationInterval() throws InterruptedException {
+        AtomicLong sleptMillis = new AtomicLong();
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        CodeDeployService noWaitService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                sleptMillis::set);
+        noWaitService.initializeStorage();
+
+        noWaitService.executeEcsAllowTraffic(REGION, new Deployment(),
+                "CodeDeployDefault.ECSCanary10Percent5Minutes", new ConcurrentHashMap<>(), List.of("listener"),
+                "blue", "green", new java.util.concurrent.atomic.AtomicBoolean());
+
+        assertEquals(Duration.ofMinutes(5).toMillis(), sleptMillis.get());
+        verify(elbV2Service).shiftListenerForward(REGION, "listener", "blue", "green", 10);
+        verify(elbV2Service).shiftListenerForward(REGION, "listener", "blue", "green", 100);
+    }
+
+    @Test
+    void ecsLinearUsesFullNamedConfigurationInterval() throws InterruptedException {
+        AtomicLong sleptMillis = new AtomicLong();
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        CodeDeployService noWaitService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                sleptMillis::addAndGet);
+        noWaitService.initializeStorage();
+
+        noWaitService.executeEcsAllowTraffic(REGION, new Deployment(),
+                "CodeDeployDefault.ECSLinear10PercentEvery1Minutes", new ConcurrentHashMap<>(), List.of("listener"),
+                "blue", "green", new java.util.concurrent.atomic.AtomicBoolean());
+
+        assertEquals(Duration.ofMinutes(9).toMillis(), sleptMillis.get());
+        verify(elbV2Service).shiftListenerForward(REGION, "listener", "blue", "green", 100);
+    }
+
     // ---- Helpers --------------------------------------------------------------------------
 
     private String createServerDeployment(String appName, String groupName, int timeoutSeconds) {
@@ -324,19 +501,22 @@ class CodeDeployServiceTest {
     }
 
     private String createServerDeployment(CodeDeployService target, String appName, String groupName,
-                                          int timeoutSeconds) {
+                                           int timeoutSeconds) {
         target.createApplication(REGION, appName, "Server", null);
         target.createDeploymentGroup(REGION, appName, groupName, "CodeDeployDefault.AllAtOnce", ROLE_ARN, null);
+        return target.createDeployment(REGION, appName, groupName, null, serverRevision(timeoutSeconds), "server hook test");
+    }
+
+    private Map<String, Object> serverRevision(int timeoutSeconds) {
         String appSpec = """
                 os: linux
                 hooks:
                   ApplicationStart:
                     - location: scripts/start_server.sh
                       timeout: %d
-                """.formatted(timeoutSeconds);
-        Map<String, Object> revision = Map.of("revisionType", "AppSpecContent",
+        """.formatted(timeoutSeconds);
+        return Map.of("revisionType", "AppSpecContent",
                 "appSpecContent", Map.of("content", appSpec));
-        return target.createDeployment(REGION, appName, groupName, null, revision, "server hook test");
     }
 
     private void createLambdaAppAndGroup(String appName, String groupName) {
@@ -385,6 +565,14 @@ class CodeDeployServiceTest {
         }
     }
 
+    private static String deploymentIdFrom(byte[] payload) {
+        try {
+            return JSON.readTree(payload).get("DeploymentId").asText();
+        } catch (Exception e) {
+            throw new IllegalStateException("Could not parse hook payload", e);
+        }
+    }
+
     private Deployment awaitTerminal(String deploymentId, Duration atMost) {
         return awaitTerminal(service, deploymentId, atMost);
     }

src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineServiceTest.java

@@ -8,29 +8,45 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageFactory;
 import io.github.hectorvent.floci.services.codebuild.CodeBuildService;
 import io.github.hectorvent.floci.services.codebuild.model.Build;
+import io.github.hectorvent.floci.services.cloudformation.CloudFormationService;
+import io.github.hectorvent.floci.services.cloudformation.model.Stack;
 import io.github.hectorvent.floci.services.codedeploy.CodeDeployService;
 import io.github.hectorvent.floci.services.codedeploy.model.Deployment;
 import io.github.hectorvent.floci.services.codepipeline.model.CodePipelinePipeline;
+import io.github.hectorvent.floci.services.ecs.EcsService;
+import io.github.hectorvent.floci.services.ecs.model.RegisterTaskDefinitionRequest;
+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
 import io.github.hectorvent.floci.services.lambda.LambdaService;
 import io.github.hectorvent.floci.services.s3.S3Service;
 import io.github.hectorvent.floci.services.s3.model.S3Object;
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
 
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -46,7 +62,8 @@ class CodePipelineServiceTest {
             mock(CodeBuildService.class),
             mock(CodeDeployService.class),
             mock(LambdaService.class),
-            mock(S3Service.class));
+            mock(S3Service.class),
+            mock(CloudFormationService.class));
 
     @Test
     void getPipelineTreatsMissingStoredVersionAsVersionOne() throws Exception {
@@ -57,7 +74,8 @@ class CodePipelineServiceTest {
                 mock(CodeBuildService.class),
                 mock(CodeDeployService.class),
                 mock(LambdaService.class),
-                mock(S3Service.class));
+                mock(S3Service.class),
+                mock(CloudFormationService.class));
         CodePipelinePipeline pipeline = mapper.readValue("""
                 {
                     "accountId": "000000000000",
@@ -117,7 +135,8 @@ class CodePipelineServiceTest {
                 mock(CodeBuildService.class),
                 mock(CodeDeployService.class),
                 mock(LambdaService.class),
-                s3Service);
+                s3Service,
+                mock(CloudFormationService.class));
         String pipelineRequest = """
                 {
                     "pipeline": {
@@ -224,6 +243,443 @@ class CodePipelineServiceTest {
         }
     }
 
+    @Test
+    void codeBuildOutputArtifactIsMaterializedForCodeDeploy() throws Exception {
+        CodeBuildService codeBuildService = mock(CodeBuildService.class);
+        CodeDeployService codeDeployService = mock(CodeDeployService.class);
+        S3Service s3Service = mock(S3Service.class);
+        byte[] sourceBytes = "source artifact".getBytes();
+        byte[] buildOutput = "CodeBuild-produced artifact".getBytes();
+        S3Object source = new S3Object();
+        source.setData(sourceBytes);
+        source.setETag("source-etag");
+        source.setVersionId("source-version");
+        source.setLastModified(Instant.now());
+        Build build = build("SUCCEEDED", true);
+        build.setId("build-42");
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(codeBuildService.startBuild(anyString(), anyString(), anyString(),
+                any(), any(), any(), any(), any(), any(), any())).thenReturn(build);
+        when(codeBuildService.getBuildOutputArtifact(REGION, ACCOUNT, "build-42")).thenReturn(buildOutput);
+        when(codeDeployService.createDeployment(eq(REGION), eq("lambda-application"), eq("lambda-group"),
+                any(), any(), anyString())).thenReturn("deployment-42");
+        when(codeDeployService.getDeployment(REGION, "deployment-42")).thenReturn(deployment("Succeeded"));
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, codeBuildService, codeDeployService,
+                mock(LambdaService.class), s3Service, mock(CloudFormationService.class));
+
+        try {
+            handle(pipelineService, "CreatePipeline", """
+                    {
+                        "pipeline": {
+                            "name": "source-build-deploy",
+                            "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",
+                            "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},
+                            "stages": [{
+                                "name": "Source", "actions": [{
+                                    "name": "SourceAction",
+                                    "actionTypeId": {"category": "Source", "owner": "AWS", "provider": "S3", "version": "1"},
+                                    "configuration": {"S3Bucket": "source-bucket", "S3ObjectKey": "source.zip"},
+                                    "outputArtifacts": [{"name": "SourceOutput"}]
+                                }]
+                            }, {
+                                "name": "Build", "actions": [{
+                                    "name": "BuildAction",
+                                    "actionTypeId": {"category": "Build", "owner": "AWS", "provider": "CodeBuild", "version": "1"},
+                                    "configuration": {"ProjectName": "build-project"},
+                                    "inputArtifacts": [{"name": "SourceOutput"}],
+                                    "outputArtifacts": [{"name": "BuildOutput"}]
+                                }]
+                            }, {
+                                "name": "Deploy", "actions": [{
+                                    "name": "DeployAction",
+                                    "actionTypeId": {"category": "Deploy", "owner": "AWS", "provider": "CodeDeploy", "version": "1"},
+                                    "configuration": {"ApplicationName": "lambda-application", "DeploymentGroupName": "lambda-group"},
+                                    "inputArtifacts": [{"name": "BuildOutput"}]
+                                }]
+                            }]
+                        }
+                    }
+                    """);
+            String executionId = startExecution(pipelineService, "source-build-deploy");
+
+            waitForExecutionStatus(pipelineService, "source-build-deploy", executionId, "Succeeded");
+            ArgumentCaptor<byte[]> deployed = ArgumentCaptor.forClass(byte[].class);
+            verify(s3Service).putObject(eq("codepipeline-artifacts"), anyString(), deployed.capture(),
+                    eq("application/zip"), any());
+            assertArrayEquals(buildOutput, deployed.getValue());
+            ArgumentCaptor<Map<String, Object>> revision = ArgumentCaptor.forClass(Map.class);
+            verify(codeDeployService).createDeployment(eq(REGION), eq("lambda-application"), eq("lambda-group"),
+                    any(), revision.capture(), anyString());
+            assertEquals("codepipeline-artifacts", ((Map<?, ?>) revision.getValue().get("s3Location")).get("bucket"));
+            verify(codeBuildService, times(1)).getBuildOutputArtifact(REGION, ACCOUNT, "build-42");
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void cloudFormationCreateUpdateMaterializesArtifactAndWaitsForStack() throws Exception {
+        S3Service s3Service = mock(S3Service.class);
+        CloudFormationService cloudFormationService = mock(CloudFormationService.class);
+        S3Object source = new S3Object();
+        source.setData(zipArtifact("templates/application.yaml", """
+                AWSTemplateFormatVersion: '2010-09-09'
+                Resources: {}
+                """));
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        Stack stack = new Stack();
+        stack.setStackId("arn:aws:cloudformation:us-east-1:000000000000:stack/application/id");
+        stack.setStatus("CREATE_COMPLETE");
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(cloudFormationService.describeStacks("application", REGION, ACCOUNT))
+                .thenThrow(new AwsException("ValidationError", "Stack does not exist", 400))
+                .thenReturn(List.of(stack));
+        when(cloudFormationService.executeChangeSet(eq("application"), anyString(), eq(REGION), eq(ACCOUNT)))
+                .thenReturn(CompletableFuture.completedFuture(null));
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, mock(CodeBuildService.class), mock(CodeDeployService.class),
+                mock(LambdaService.class), s3Service, cloudFormationService);
+
+        try {
+            handle(pipelineService, "CreatePipeline", """
+                    {
+                        "pipeline": {
+                            "name": "cloudformation-pipeline",
+                            "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",
+                            "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},
+                            "stages": [{
+                                "name": "Source", "actions": [{
+                                    "name": "SourceAction",
+                                    "actionTypeId": {"category": "Source", "owner": "AWS", "provider": "S3", "version": "1"},
+                                    "configuration": {"S3Bucket": "source-bucket", "S3ObjectKey": "source.zip"},
+                                    "outputArtifacts": [{"name": "SourceOutput"}]
+                                }]
+                            }, {
+                                "name": "Deploy", "actions": [{
+                                    "name": "CloudFormationAction",
+                                    "actionTypeId": {"category": "Deploy", "owner": "AWS", "provider": "CloudFormation", "version": "1"},
+                                    "configuration": {"StackName": "application", "ActionMode": "CREATE_UPDATE", "TemplatePath": "SourceOutput::templates/application.yaml"},
+                                    "inputArtifacts": [{"name": "SourceOutput"}]
+                                }]
+                            }]
+                        }
+                    }
+                    """);
+            String executionId = startExecution(pipelineService, "cloudformation-pipeline");
+
+            waitForExecutionStatus(pipelineService, "cloudformation-pipeline", executionId, "Succeeded");
+            ArgumentCaptor<byte[]> artifact = ArgumentCaptor.forClass(byte[].class);
+            verify(s3Service).putObject(eq("codepipeline-artifacts"), anyString(), artifact.capture(),
+                    eq("application/zip"), any());
+            assertArrayEquals(source.getData(), artifact.getValue());
+            ArgumentCaptor<String> template = ArgumentCaptor.forClass(String.class);
+            verify(cloudFormationService).createChangeSet(eq("application"), anyString(), eq("CREATE"),
+                    template.capture(), eq(null), eq(Map.of()), eq(List.of()), eq(Map.of()),
+                    eq(REGION), eq(ACCOUNT));
+            assertEquals("AWSTemplateFormatVersion: '2010-09-09'\nResources: {}\n", template.getValue());
+            verify(cloudFormationService).executeChangeSet(eq("application"), anyString(), eq(REGION), eq(ACCOUNT));
+
+            stack.setStatus("UPDATE_COMPLETE");
+            String updateExecutionId = startExecution(pipelineService, "cloudformation-pipeline");
+            waitForExecutionStatus(pipelineService, "cloudformation-pipeline", updateExecutionId, "Succeeded");
+            verify(cloudFormationService).createChangeSet(eq("application"), anyString(), eq("UPDATE"),
+                    anyString(), eq(null), eq(Map.of()), eq(List.of()), eq(Map.of()), eq(REGION), eq(ACCOUNT));
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void codeDeployToEcsRegistersTaskDefinitionAndSubstitutesAppSpec() throws Exception {
+        S3Service s3Service = mock(S3Service.class);
+        CodeDeployService codeDeployService = mock(CodeDeployService.class);
+        EcsService ecsService = mock(EcsService.class);
+        S3Object source = new S3Object();
+        source.setData(zipArtifact(Map.of(
+                "taskdef.json", """
+                        {"family":"pipeline-task","containerDefinitions":[{"name":"app","image":"nginx"}]}
+                        """,
+                "appspec.json", """
+                        {"version":0.0,"Resources":[{"TargetService":{"Type":"AWS::ECS::Service","Properties":{"TaskDefinition":"<TASK_DEFINITION>","LoadBalancerInfo":{"ContainerName":"app","ContainerPort":80}}}}]}
+                        """)));
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        TaskDefinition taskDefinition = new TaskDefinition();
+        taskDefinition.setTaskDefinitionArn("arn:aws:ecs:us-east-1:000000000000:task-definition/pipeline-task:2");
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(ecsService.registerTaskDefinition(any(RegisterTaskDefinitionRequest.class), eq(REGION)))
+                .thenReturn(taskDefinition);
+        when(codeDeployService.createDeployment(eq(REGION), eq("ecs-app"), eq("ecs-group"),
+                any(), any(), anyString())).thenReturn("deployment-ecs");
+        when(codeDeployService.getDeployment(REGION, "deployment-ecs")).thenReturn(deployment("Succeeded"));
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, mock(CodeBuildService.class), codeDeployService,
+                mock(LambdaService.class), s3Service, mock(CloudFormationService.class), ecsService);
+
+        try {
+            createCodeDeployToEcsPipeline(pipelineService, "ecs-success-pipeline");
+            String executionId = startExecution(pipelineService, "ecs-success-pipeline");
+
+            waitForExecutionStatus(pipelineService, "ecs-success-pipeline", executionId, "Succeeded");
+            ArgumentCaptor<RegisterTaskDefinitionRequest> request = ArgumentCaptor.forClass(RegisterTaskDefinitionRequest.class);
+            verify(ecsService).registerTaskDefinition(request.capture(), eq(REGION));
+            assertEquals("pipeline-task", request.getValue().getFamily());
+            ArgumentCaptor<Map<String, Object>> revision = ArgumentCaptor.forClass(Map.class);
+            verify(codeDeployService).createDeployment(eq(REGION), eq("ecs-app"), eq("ecs-group"),
+                    eq(null), revision.capture(), anyString());
+            String appSpec = (String) ((Map<?, ?>) revision.getValue().get("appSpecContent")).get("content");
+            assertTrue(appSpec.contains(taskDefinition.getTaskDefinitionArn()));
+            assertFalse(appSpec.contains("<TASK_DEFINITION>"));
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void codeDeployToEcsPropagatesDeploymentFailure() throws Exception {
+        S3Service s3Service = mock(S3Service.class);
+        CodeDeployService codeDeployService = mock(CodeDeployService.class);
+        EcsService ecsService = mock(EcsService.class);
+        S3Object source = new S3Object();
+        source.setData(zipArtifact(Map.of(
+                "taskdef.json", "{\"family\":\"pipeline-task\",\"containerDefinitions\":[]}",
+                "appspec.json", "{\"version\":0.0,\"Resources\":[{\"TargetService\":{\"Properties\":{\"TaskDefinition\":\"<TASK_DEFINITION>\"}}}]}")));
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        TaskDefinition taskDefinition = new TaskDefinition();
+        taskDefinition.setTaskDefinitionArn("arn:aws:ecs:us-east-1:000000000000:task-definition/pipeline-task:1");
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(ecsService.registerTaskDefinition(any(RegisterTaskDefinitionRequest.class), eq(REGION)))
+                .thenReturn(taskDefinition);
+        when(codeDeployService.createDeployment(anyString(), anyString(), anyString(), any(), any(), anyString()))
+                .thenReturn("deployment-ecs");
+        when(codeDeployService.getDeployment(REGION, "deployment-ecs")).thenReturn(deployment("Failed"));
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, mock(CodeBuildService.class), codeDeployService,
+                mock(LambdaService.class), s3Service, mock(CloudFormationService.class), ecsService);
+
+        try {
+            createCodeDeployToEcsPipeline(pipelineService, "ecs-failure-pipeline");
+            String executionId = startExecution(pipelineService, "ecs-failure-pipeline");
+
+            waitForExecutionStatus(pipelineService, "ecs-failure-pipeline", executionId, "Failed");
+            JsonNode state = pipelineState(pipelineService, "ecs-failure-pipeline");
+            assertEquals("Failed", state.path("stageStates").path(1).path("actionStates").path(0)
+                    .path("latestExecution").path("status").asText());
+            assertTrue(state.path("stageStates").path(1).path("actionStates").path(0)
+                    .path("latestExecution").path("summary").asText().contains("Failed"));
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void cloudFormationExtractsCodeBuildZipAndResolvesArtifactParameterOverrides() throws Exception {
+        CodeBuildService codeBuildService = mock(CodeBuildService.class);
+        S3Service s3Service = mock(S3Service.class);
+        CloudFormationService cloudFormationService = mock(CloudFormationService.class);
+        S3Object source = new S3Object();
+        source.setData(zipArtifact("template.yaml", """
+                Parameters:
+                  CodeS3Bucket: {Type: String}
+                  CodeS3Key: {Type: String}
+                Resources: {}
+                """));
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        byte[] functionZip = "function bytes".getBytes(StandardCharsets.UTF_8);
+        Build build = build("SUCCEEDED", true);
+        build.setId("build-function");
+        Stack stack = new Stack();
+        stack.setStackId("arn:aws:cloudformation:us-east-1:000000000000:stack/application/id");
+        stack.setStatus("CREATE_COMPLETE");
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(codeBuildService.startBuild(anyString(), anyString(), anyString(),
+                any(), any(), any(), any(), any(), any(), any())).thenReturn(build);
+        when(codeBuildService.getBuildOutputArtifact(REGION, ACCOUNT, "build-function"))
+                .thenReturn(zipArtifact("function.zip", functionZip));
+        when(cloudFormationService.describeStacks("application", REGION, ACCOUNT))
+                .thenThrow(new AwsException("ValidationError", "Stack does not exist", 400))
+                .thenReturn(List.of(stack));
+        when(cloudFormationService.executeChangeSet(eq("application"), anyString(), eq(REGION), eq(ACCOUNT)))
+                .thenReturn(CompletableFuture.completedFuture(null));
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, codeBuildService, mock(CodeDeployService.class),
+                mock(LambdaService.class), s3Service, cloudFormationService);
+
+        try {
+            handle(pipelineService, "CreatePipeline", """
+                    {"pipeline":{"name":"build-cloudformation","roleArn":"arn:aws:iam::000000000000:role/codepipeline-role",
+                    "artifactStore":{"type":"S3","location":"codepipeline-artifacts"},"stages":[
+                    {"name":"Source","actions":[{"name":"SourceAction","actionTypeId":{"category":"Source","owner":"AWS","provider":"S3","version":"1"},
+                    "configuration":{"S3Bucket":"source-bucket","S3ObjectKey":"source.zip"},"outputArtifacts":[{"name":"SourceOutput"}]}]},
+                    {"name":"Build","actions":[{"name":"BuildAction","actionTypeId":{"category":"Build","owner":"AWS","provider":"CodeBuild","version":"1"},
+                    "configuration":{"ProjectName":"build-project"},"inputArtifacts":[{"name":"SourceOutput"}],"outputArtifacts":[{"name":"BuildOutput"}]}]},
+                    {"name":"Deploy","actions":[{"name":"CloudFormationAction","actionTypeId":{"category":"Deploy","owner":"AWS","provider":"CloudFormation","version":"1"},
+                    "configuration":{"StackName":"application","ActionMode":"CREATE_UPDATE","TemplatePath":"SourceOutput::template.yaml",
+                    "ParameterOverrides":"{\\"CodeS3Bucket\\":{\\"Fn::GetArtifactAtt\\":[\\"BuildOutput\\",\\"BucketName\\"]},\\"CodeS3Key\\":{\\"Fn::GetArtifactAtt\\":[\\"BuildOutput\\",\\"ObjectKey\\"]}}"},
+                    "inputArtifacts":[{"name":"SourceOutput"},{"name":"BuildOutput"}]}]}]}}
+                    """);
+            String executionId = startExecution(pipelineService, "build-cloudformation");
+
+            waitForExecutionStatus(pipelineService, "build-cloudformation", executionId, "Succeeded");
+            ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);
+            ArgumentCaptor<byte[]> contents = ArgumentCaptor.forClass(byte[].class);
+            verify(s3Service, times(4)).putObject(eq("codepipeline-artifacts"), keys.capture(), contents.capture(),
+                    anyString(), any());
+            int functionIndex = keys.getAllValues().indexOf(executionId + "/Deploy/CloudFormationAction/BuildOutput/function.zip");
+            assertTrue(functionIndex >= 0);
+            assertArrayEquals(functionZip, contents.getAllValues().get(functionIndex));
+            ArgumentCaptor<Map<String, String>> parameters = ArgumentCaptor.forClass(Map.class);
+            verify(cloudFormationService).createChangeSet(eq("application"), anyString(), eq("CREATE"),
+                    anyString(), eq(null), parameters.capture(), eq(List.of()), eq(Map.of()), eq(REGION), eq(ACCOUNT));
+            assertEquals(Map.of("CodeS3Bucket", "codepipeline-artifacts",
+                    "CodeS3Key", executionId + "/Deploy/CloudFormationAction/BuildOutput/function.zip"),
+                    parameters.getValue());
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void cloudFormationRejectsUnsafeTemplatePath() throws Exception {
+        S3Service s3Service = mock(S3Service.class);
+        S3Object source = new S3Object();
+        source.setData(zipArtifact("template.yaml", "Resources: {}"));
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, mock(CodeBuildService.class), mock(CodeDeployService.class),
+                mock(LambdaService.class), s3Service, mock(CloudFormationService.class));
+
+        try {
+            handle(pipelineService, "CreatePipeline", """
+                    {"pipeline":{"name":"unsafe-template-path","roleArn":"arn:aws:iam::000000000000:role/codepipeline-role",
+                    "artifactStore":{"type":"S3","location":"codepipeline-artifacts"},"stages":[
+                    {"name":"Source","actions":[{"name":"SourceAction","actionTypeId":{"category":"Source","owner":"AWS","provider":"S3","version":"1"},
+                    "configuration":{"S3Bucket":"source-bucket","S3ObjectKey":"source.zip"},"outputArtifacts":[{"name":"SourceOutput"}]}]},
+                    {"name":"Deploy","actions":[{"name":"CloudFormationAction","actionTypeId":{"category":"Deploy","owner":"AWS","provider":"CloudFormation","version":"1"},
+                    "configuration":{"StackName":"application","ActionMode":"CREATE_UPDATE","TemplatePath":"SourceOutput::../template.yaml"},"inputArtifacts":[{"name":"SourceOutput"}]}]}]}}
+                    """);
+            String executionId = startExecution(pipelineService, "unsafe-template-path");
+
+            JsonNode execution = waitForExecutionStatus(pipelineService, "unsafe-template-path", executionId, "Failed");
+            assertEquals("Action CloudFormationAction failed: TemplatePath is not safe",
+                    execution.path("statusSummary").asText());
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
+    @Test
+    void retryFailedActionsRerunsOnlyFailedStageActionsAndKeepsUpstreamArtifacts() throws Exception {
+        retryStageExecutionRerunsRequestedActionsOnly("FAILED_ACTIONS", 1, 2);
+    }
+
+    @Test
+    void retryAllActionsRerunsEveryRequestedStageActionAndKeepsUpstreamArtifacts() throws Exception {
+        retryStageExecutionRerunsRequestedActionsOnly("ALL_ACTIONS", 2, 2);
+    }
+
+    private void retryStageExecutionRerunsRequestedActionsOnly(String retryMode, int expectedGoodDeploys,
+                                                                int expectedFailingDeploys) throws Exception {
+        CodeBuildService codeBuildService = mock(CodeBuildService.class);
+        S3Service s3Service = mock(S3Service.class);
+        S3Object source = new S3Object();
+        source.setData("source artifact".getBytes());
+        source.setETag("source-etag");
+        source.setLastModified(Instant.now());
+        Build build = build("SUCCEEDED", true);
+        build.setId("build-retry");
+        AtomicInteger failingDeploys = new AtomicInteger();
+        when(s3Service.headObject("source-bucket", "source.zip")).thenReturn(source);
+        when(s3Service.getObject("source-bucket", "source.zip")).thenReturn(source);
+        when(codeBuildService.startBuild(anyString(), anyString(), anyString(),
+                any(), any(), any(), any(), any(), any(), any())).thenReturn(build);
+        when(codeBuildService.getBuildOutputArtifact(REGION, ACCOUNT, "build-retry"))
+                .thenReturn("build artifact".getBytes());
+        doAnswer(invocation -> {
+            if ("fail.zip".equals(invocation.getArgument(1)) && failingDeploys.incrementAndGet() == 1) {
+                throw new AwsException("ActionExecutionFailed", "first deploy fails", 400);
+            }
+            return null;
+        }).when(s3Service).putObject(anyString(), anyString(), any(), anyString(), any());
+        CodePipelineService pipelineService = new CodePipelineService(
+                new InMemoryStorageFactory(), mapper, codeBuildService, mock(CodeDeployService.class),
+                mock(LambdaService.class), s3Service, mock(CloudFormationService.class));
+
+        try {
+            handle(pipelineService, "CreatePipeline", """
+                    {
+                        "pipeline": {
+                            "name": "retry-stage-pipeline",
+                            "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",
+                            "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},
+                            "stages": [{
+                                "name": "Source", "actions": [{
+                                    "name": "SourceAction",
+                                    "actionTypeId": {"category": "Source", "owner": "AWS", "provider": "S3", "version": "1"},
+                                    "configuration": {"S3Bucket": "source-bucket", "S3ObjectKey": "source.zip"},
+                                    "outputArtifacts": [{"name": "SourceOutput"}]
+                                }]
+                            }, {
+                                "name": "Build", "actions": [{
+                                    "name": "BuildAction",
+                                    "actionTypeId": {"category": "Build", "owner": "AWS", "provider": "CodeBuild", "version": "1"},
+                                    "configuration": {"ProjectName": "build-project"},
+                                    "inputArtifacts": [{"name": "SourceOutput"}],
+                                    "outputArtifacts": [{"name": "BuildOutput"}]
+                                }]
+                            }, {
+                                "name": "Deploy", "actions": [{
+                                    "name": "GoodDeploy",
+                                    "actionTypeId": {"category": "Deploy", "owner": "AWS", "provider": "S3", "version": "1"},
+                                    "configuration": {"BucketName": "deploy-bucket", "ObjectKey": "good.zip"},
+                                    "inputArtifacts": [{"name": "BuildOutput"}]
+                                }, {
+                                    "name": "FailingDeploy",
+                                    "actionTypeId": {"category": "Deploy", "owner": "AWS", "provider": "S3", "version": "1"},
+                                    "configuration": {"BucketName": "deploy-bucket", "ObjectKey": "fail.zip"},
+                                    "inputArtifacts": [{"name": "BuildOutput"}]
+                                }]
+                            }]
+                        }
+                    }
+                    """);
+            String executionId = startExecution(pipelineService, "retry-stage-pipeline");
+            waitForExecutionStatus(pipelineService, "retry-stage-pipeline", executionId, "Failed");
+
+            String retryExecutionId = handle(pipelineService, "RetryStageExecution", """
+                    {
+                        "pipelineName": "retry-stage-pipeline",
+                        "pipelineExecutionId": "%s",
+                        "stageName": "Deploy",
+                        "retryMode": "%s"
+                    }
+                    """.formatted(executionId, retryMode)).path("pipelineExecutionId").asText();
+            waitForExecutionStatus(pipelineService, "retry-stage-pipeline", retryExecutionId, "Succeeded");
+
+            verify(s3Service, times(1)).getObject("source-bucket", "source.zip");
+            verify(codeBuildService, times(1)).startBuild(anyString(), anyString(), anyString(),
+                    any(), any(), any(), any(), any(), any(), any());
+            verify(s3Service, times(expectedGoodDeploys)).putObject(eq("deploy-bucket"), eq("good.zip"),
+                    any(), eq("application/zip"), any());
+            verify(s3Service, times(expectedFailingDeploys)).putObject(eq("deploy-bucket"), eq("fail.zip"),
+                    any(), eq("application/zip"), any());
+        } finally {
+            pipelineService.shutdown();
+        }
+    }
+
     @Test
     void stoppingCodeDeployWaitsForFailureWithoutFailingThePipeline() throws Exception {
         CodeDeployService codeDeployService = mock(CodeDeployService.class);
@@ -395,7 +851,7 @@ class CodePipelineServiceTest {
                                             CodeDeployService codeDeployService) {
         return new CodePipelineService(
                 new InMemoryStorageFactory(), mapper, codeBuildService, codeDeployService,
-                mock(LambdaService.class), mock(S3Service.class));
+                mock(LambdaService.class), mock(S3Service.class), mock(CloudFormationService.class));
     }
 
     private void createExternalActionPipeline(CodePipelineService pipelineService,
@@ -485,6 +941,72 @@ class CodePipelineServiceTest {
         return deployment;
     }
 
+    private void createCodeDeployToEcsPipeline(CodePipelineService pipelineService, String pipelineName) throws Exception {
+        handle(pipelineService, "CreatePipeline", """
+                {
+                    "pipeline": {
+                        "name": "%s",
+                        "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",
+                        "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},
+                        "stages": [{
+                            "name": "Source",
+                            "actions": [{
+                                "name": "SourceAction",
+                                "actionTypeId": {"category": "Source", "owner": "AWS", "provider": "S3", "version": "1"},
+                                "configuration": {"S3Bucket": "source-bucket", "S3ObjectKey": "source.zip"},
+                                "outputArtifacts": [{"name": "SourceOutput"}]
+                            }]
+                        }, {
+                            "name": "Deploy",
+                            "actions": [{
+                                "name": "DeployToEcs",
+                                "actionTypeId": {"category": "Deploy", "owner": "AWS", "provider": "CodeDeployToECS", "version": "1"},
+                                "configuration": {
+                                    "ApplicationName": "ecs-app",
+                                    "DeploymentGroupName": "ecs-group",
+                                    "TaskDefinitionTemplatePath": "SourceOutput::taskdef.json",
+                                    "AppSpecTemplatePath": "SourceOutput::appspec.json"
+                                },
+                                "inputArtifacts": [{"name": "SourceOutput"}]
+                            }]
+                        }]
+                    }
+                }
+                """.formatted(pipelineName));
+    }
+
+    private byte[] zipArtifact(String entryName, String contents) throws Exception {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        try (ZipOutputStream zip = new ZipOutputStream(output)) {
+            zip.putNextEntry(new ZipEntry(entryName));
+            zip.write(contents.getBytes(StandardCharsets.UTF_8));
+            zip.closeEntry();
+        }
+        return output.toByteArray();
+    }
+
+    private byte[] zipArtifact(String entryName, byte[] contents) throws Exception {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        try (ZipOutputStream zip = new ZipOutputStream(output)) {
+            zip.putNextEntry(new ZipEntry(entryName));
+            zip.write(contents);
+            zip.closeEntry();
+        }
+        return output.toByteArray();
+    }
+
+    private byte[] zipArtifact(Map<String, String> entries) throws Exception {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        try (ZipOutputStream zip = new ZipOutputStream(output)) {
+            for (Map.Entry<String, String> entry : entries.entrySet()) {
+                zip.putNextEntry(new ZipEntry(entry.getKey()));
+                zip.write(entry.getValue().getBytes(StandardCharsets.UTF_8));
+                zip.closeEntry();
+            }
+        }
+        return output.toByteArray();
+    }
+
     private static final class InMemoryStorageFactory extends StorageFactory {
         private InMemoryStorageFactory() {
             super(null, null);

src/test/java/io/github/hectorvent/floci/services/ecs/EcsEc2BridgeIntegrationTest.java

@@ -21,17 +21,17 @@ class EcsEc2BridgeIntegrationTest {
     }
 
     @Test
-    void runTaskAndCreateServiceRejectFargateAndNonBridgeTasks() {
+    void runTaskAndCreateServiceKeepEc2BridgeRestrictions() {
         call("RegisterTaskDefinition", "{\"family\":\"ec2-bridge\",\"networkMode\":\"bridge\","
                 + "\"containerDefinitions\":[{\"name\":\"app\",\"image\":\"nginx\"}]}", 200);
         call("RunTask", "{\"taskDefinition\":\"ec2-bridge\",\"launchType\":\"FARGATE\"}", 400)
-                .body("message", containsString("EC2 launch type"));
+                .body("message", containsString("awsvpc"));
         call("RunTask", "{\"taskDefinition\":\"ec2-bridge\",\"capacityProviderStrategy\":["
                 + "{\"capacityProvider\":\"FARGATE\",\"weight\":1}]}", 400)
-                .body("message", containsString("FARGATE capacity providers"));
+                .body("message", containsString("awsvpc"));
         call("CreateService", "{\"serviceName\":\"ec2-bridge\",\"taskDefinition\":\"ec2-bridge\","
                 + "\"desiredCount\":0,\"launchType\":\"FARGATE\"}", 400)
-                .body("message", containsString("EC2 launch type"));
+                .body("message", containsString("awsvpc"));
 
         call("RegisterTaskDefinition", "{\"family\":\"not-bridge\",\"networkMode\":\"awsvpc\","
                 + "\"containerDefinitions\":[{\"name\":\"app\",\"image\":\"nginx\"}]}", 200);

src/test/java/io/github/hectorvent/floci/services/ecs/EcsEc2ServiceReconciliationTest.java

@@ -0,0 +1,159 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.CreateTaskSetRequest;
+import io.github.hectorvent.floci.services.ecs.model.EcsTask;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.ecs.model.TaskSet;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class EcsEc2ServiceReconciliationTest {
+
+    private static final String REGION = "us-east-1";
+
+    @Test
+    void taskDefinitionUpdateReplacesAndDrainsEc2ServiceTasks() {
+        EcsService service = newMockModeService();
+        service.createCluster("roll-cluster", REGION);
+        TaskDefinition revisionOne = registerTaskDefinition(service, "roll", "app:1");
+        service.createService("roll-cluster", "web", revisionOne.getTaskDefinitionArn(), 2,
+                LaunchType.EC2, List.of(), null, REGION);
+        service.reconcileServices();
+
+        TaskDefinition revisionTwo = registerTaskDefinition(service, "roll", "app:2");
+        service.updateService("roll-cluster", "web", revisionTwo.getTaskDefinitionArn(), null, null, REGION);
+
+        service.reconcileServices();
+        assertEquals(4, runningTasks(service).size(), "replacements start before old tasks drain");
+        service.reconcileServices();
+
+        List<EcsTask> running = runningTasks(service);
+        assertEquals(2, running.size());
+        assertTrue(running.stream().allMatch(task -> revisionTwo.getTaskDefinitionArn()
+                .equals(task.getTaskDefinitionArn())));
+    }
+
+    @Test
+    void forceDeleteDrainsAllEc2ServiceTasks() {
+        EcsService service = newMockModeService();
+        service.createCluster("delete-cluster", REGION);
+        TaskDefinition taskDefinition = registerTaskDefinition(service, "delete", "app:1");
+        service.createService("delete-cluster", "worker", taskDefinition.getTaskDefinitionArn(), 2,
+                LaunchType.EC2, List.of(), null, REGION);
+        service.reconcileServices();
+
+        service.deleteService("delete-cluster", "worker", true, REGION);
+
+        assertEquals(0, runningTasks(service).size());
+    }
+
+    @Test
+    void codeDeployTaskSetsAreNotSupplementedByNormalEc2ServiceTasks() {
+        EcsService service = newMockModeService();
+        service.createCluster("blue-green-cluster", REGION);
+        TaskDefinition taskDefinition = registerTaskDefinition(service, "blue-green", "app:1");
+        service.createService("blue-green-cluster", "web", taskDefinition.getTaskDefinitionArn(), 2,
+                LaunchType.EC2, List.of(), null, null, null, "CODE_DEPLOY", null, REGION);
+        CreateTaskSetRequest request = new CreateTaskSetRequest();
+        request.setCluster("blue-green-cluster");
+        request.setService("web");
+        request.setTaskDefinition(taskDefinition.getTaskDefinitionArn());
+        request.setLaunchType(LaunchType.EC2);
+        TaskSet taskSet = service.createTaskSet(request, REGION);
+
+        service.reconcileServices();
+
+        assertEquals(2, taskSet.getRunningCount());
+        assertEquals(2, runningTasks(service).size());
+
+        service.deleteService("blue-green-cluster", "web", true, REGION);
+
+        assertEquals(0, runningTasks(service).size(), "force delete drains CodeDeploy task-set tasks");
+    }
+
+    @Test
+    void recreatingAnEc2CodeDeployServiceDoesNotReuseDeletedTaskSets() {
+        EcsService service = newMockModeService();
+        service.createCluster("ec2-recreate-cluster", REGION);
+        TaskDefinition taskDefinition = registerTaskDefinition(service, "ec2-recreate", "app:1");
+        service.createService("ec2-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.EC2, List.of(), null, null, null, "CODE_DEPLOY", null, REGION);
+        CreateTaskSetRequest request = new CreateTaskSetRequest();
+        request.setCluster("ec2-recreate-cluster");
+        request.setService("blue");
+        request.setTaskDefinition(taskDefinition.getTaskDefinitionArn());
+        request.setLaunchType(LaunchType.EC2);
+        service.createTaskSet(request, REGION);
+
+        service.deleteService("ec2-recreate-cluster", "blue", true, REGION);
+        service.createService("ec2-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.EC2, List.of(), null, null, null, "CODE_DEPLOY", null, REGION);
+        service.reconcileServices();
+
+        assertEquals(List.of(), service.describeTaskSets("ec2-recreate-cluster", "blue", null, REGION));
+        assertEquals(List.of(), runningTasks(service));
+    }
+
+    private static List<EcsTask> runningTasks(EcsService service) {
+        return service.describeTasks(null, service.listTasks(null, null, null, null, REGION), REGION).stream()
+                .filter(task -> "RUNNING".equals(task.getLastStatus()))
+                .toList();
+    }
+
+    private static TaskDefinition registerTaskDefinition(EcsService service, String family, String image) {
+        ContainerDefinition container = new ContainerDefinition();
+        container.setName("app");
+        container.setImage(image);
+        return service.registerTaskDefinition(family, List.of(container), null, null, null,
+                null, null, List.of(), REGION);
+    }
+
+    private static EcsService newMockModeService() {
+        EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+        when(config.services().ecs().mock()).thenReturn(true);
+        when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+        EcsService service = new EcsService(
+                new RegionResolver(REGION, "000000000000"),
+                mock(EcsContainerManager.class),
+                config,
+                mock(EcsLoadBalancerRegistrar.class),
+                new InMemoryStorageFactory(),
+                null);
+        service.initializeStorage();
+        return service;
+    }
+
+    private static final class InMemoryStorageFactory extends StorageFactory {
+        private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
+
+        private InMemoryStorageFactory() {
+            super(null, null);
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+                                                          TypeReference<Map<String, V>> typeReference) {
+            return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,
+                    ignored -> AccountAwareStorageBackend.inMemory("000000000000"));
+        }
+    }
+}

src/test/java/io/github/hectorvent/floci/services/ecs/EcsLoadBalancersIntegrationTest.java

@@ -56,6 +56,13 @@ class EcsLoadBalancersIntegrationTest {
         return targetGroup.getTargetGroupArn();
     }
 
+    private String createInstanceTargetGroup(String name) {
+        TargetGroup targetGroup = elbV2Service.createTargetGroup("us-east-1", name, "HTTP", "HTTP1",
+                8080, "vpc-default", "instance", null, null, false, null, null, null, null, null, null,
+                null, Map.of());
+        return targetGroup.getTargetGroupArn();
+    }
+
     @Test
     void createServiceParsesAndEchoesLoadBalancers() {
         seedClusterAndTaskDef("lb-cluster-1", "lb-td-1");
@@ -147,6 +154,29 @@ class EcsLoadBalancersIntegrationTest {
                 .body("message", org.hamcrest.Matchers.containsString("does not expose port 9999"));
     }
 
+    @Test
+    void fargateServiceRequiresAnIpTargetGroup() {
+        String cluster = "fargate-lb-cluster";
+        String family = "fargate-lb-task";
+        call("CreateCluster", "{\"clusterName\":\"" + cluster + "\"}");
+        call("RegisterTaskDefinition", "{\"family\":\"" + family + "\","
+                + "\"requiresCompatibilities\":[\"FARGATE\"],\"networkMode\":\"awsvpc\","
+                + "\"cpu\":\"256\",\"memory\":\"512\",\"containerDefinitions\":[{"
+                + "\"name\":\"web\",\"image\":\"nginx\",\"portMappings\":[{\"containerPort\":8080}]}]}");
+        String tgArn = createInstanceTargetGroup("fargate-instance-tg");
+
+        given().contentType(CT).header("X-Amz-Target", TARGET + "CreateService")
+                .body("{\"cluster\":\"" + cluster + "\",\"serviceName\":\"fargate-lb\","
+                        + "\"taskDefinition\":\"" + family + "\",\"desiredCount\":0,"
+                        + "\"launchType\":\"FARGATE\",\"networkConfiguration\":{\"awsvpcConfiguration\":{"
+                        + "\"subnets\":[\"subnet-default-us-east-1-a\"]}},\"loadBalancers\":[{"
+                        + "\"targetGroupArn\":\"" + tgArn + "\",\"containerName\":\"web\","
+                        + "\"containerPort\":8080}]}")
+                .when().post("/")
+                .then().statusCode(400)
+                .body("message", org.hamcrest.Matchers.containsString("target type ip"));
+    }
+
     @Test
     void serviceWithLoadBalancersIsStableInMockMode() {
         // Mock mode launches no real containers; the registrar must no-op cleanly.

src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceRolloutTest.java

@@ -7,11 +7,18 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageFactory;
 import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;
+import io.github.hectorvent.floci.services.ecs.model.AwsVpcConfiguration;
 import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.CreateTaskSetRequest;
+import io.github.hectorvent.floci.services.ecs.model.EcsLoadBalancer;
 import io.github.hectorvent.floci.services.ecs.model.EcsTask;
 import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;
+import io.github.hectorvent.floci.services.ecs.model.NetworkMode;
 import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
 import io.github.hectorvent.floci.services.ecs.model.TaskStatus;
+import io.github.hectorvent.floci.services.ecs.model.TaskSet;
 import org.junit.jupiter.api.Test;
 
 import java.util.HashMap;
@@ -23,9 +30,12 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
 import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -92,6 +102,206 @@ class EcsServiceRolloutTest {
         assertEquals(rev1.getTaskDefinitionArn(), svc.getTaskDefinition());
     }
 
+    @Test
+    void ec2TaskSetPlacesTasksRegistersGreenTargetsAndDrainsThem() {
+        EcsLoadBalancerRegistrar registrar = mock(EcsLoadBalancerRegistrar.class);
+        EcsService service = newDockerModeService(registrar);
+        service.createCluster("blue-green-cluster", REGION);
+        TaskDefinition taskDefinition = registerTaskDef(service, "blue-green-task", "app:1");
+        service.createService("blue-green-cluster", "blue-green-service", taskDefinition.getTaskDefinitionArn(),
+                2, LaunchType.EC2, List.of(), null, null, null, "CODE_DEPLOY", null, REGION);
+
+        EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+        loadBalancer.setTargetGroupArn("arn:aws:elasticloadbalancing:us-east-1:000000000000:targetgroup/green/id");
+        loadBalancer.setContainerName("app");
+        loadBalancer.setContainerPort(80);
+        CreateTaskSetRequest request = new CreateTaskSetRequest();
+        request.setCluster("blue-green-cluster");
+        request.setService("blue-green-service");
+        request.setTaskDefinition(taskDefinition.getTaskDefinitionArn());
+        request.setLaunchType(LaunchType.EC2);
+        request.setScaleValue(100.0);
+        request.setLoadBalancers(List.of(loadBalancer));
+
+        TaskSet taskSet = service.createTaskSet(request, REGION);
+
+        assertEquals(2, taskSet.getRunningCount());
+        assertEquals("STEADY_STATE", taskSet.getStabilityStatus());
+        verify(registrar, times(2)).registerTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+
+        TaskSet scaled = service.updateTaskSet("blue-green-cluster", "blue-green-service",
+                taskSet.getTaskSetArn(), 0.0, "PERCENT", REGION);
+        assertEquals(0, scaled.getRunningCount());
+        verify(registrar, times(2)).deregisterTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+    }
+
+    @Test
+    void fargateTaskSetRunsRegistersGreenTargetsAndReconcilesStoppedTasks() {
+        EcsLoadBalancerRegistrar registrar = mock(EcsLoadBalancerRegistrar.class);
+        EcsService service = newDockerModeService(registrar);
+        service.createCluster("fargate-blue-green-cluster", REGION);
+        TaskDefinition taskDefinition = registerFargateTaskDef(service, "fargate-blue-green-task", "app:1");
+        NetworkConfiguration networkConfiguration = fargateNetworkConfiguration();
+        service.createService("fargate-blue-green-cluster", "fargate-blue-green-service",
+                taskDefinition.getTaskDefinitionArn(), 2, LaunchType.FARGATE, List.of(), networkConfiguration,
+                null, null, "CODE_DEPLOY", null, REGION);
+
+        EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+        loadBalancer.setTargetGroupArn("arn:aws:elasticloadbalancing:us-east-1:000000000000:targetgroup/green/id");
+        loadBalancer.setContainerName("app");
+        loadBalancer.setContainerPort(80);
+        CreateTaskSetRequest request = new CreateTaskSetRequest();
+        request.setCluster("fargate-blue-green-cluster");
+        request.setService("fargate-blue-green-service");
+        request.setTaskDefinition(taskDefinition.getTaskDefinitionArn());
+        request.setLaunchType(LaunchType.FARGATE);
+        request.setScaleValue(100.0);
+        request.setLoadBalancers(List.of(loadBalancer));
+
+        TaskSet taskSet = service.createTaskSet(request, REGION);
+
+        assertEquals(2, taskSet.getRunningCount());
+        assertEquals("STEADY_STATE", taskSet.getStabilityStatus());
+        verify(registrar, times(4)).registerTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+
+        EcsTask firstTask = runningTasks(service).getFirst();
+        service.stopTask("fargate-blue-green-cluster", firstTask.getTaskArn(), "test stop", REGION);
+        service.reconcileServices();
+
+        assertEquals(2, taskSet.getRunningCount());
+        verify(registrar, times(5)).registerTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+        verify(registrar).deregisterTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+
+        service.deleteTaskSet("fargate-blue-green-cluster", "fargate-blue-green-service",
+                taskSet.getTaskSetArn(), true, REGION);
+        assertEquals(2, runningTasks(service).size());
+        verify(registrar, times(3)).deregisterTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+    }
+
+    @Test
+    void createFargateCodeDeployServiceBootstrapsPrimaryBlueTaskSet() {
+        EcsLoadBalancerRegistrar registrar = mock(EcsLoadBalancerRegistrar.class);
+        EcsService service = newDockerModeService(registrar);
+        service.createCluster("fargate-bootstrap-cluster", REGION);
+        TaskDefinition taskDefinition = registerFargateTaskDef(service, "fargate-bootstrap-task", "app:1");
+        EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+        loadBalancer.setTargetGroupArn("arn:aws:elasticloadbalancing:us-east-1:000000000000:targetgroup/blue/id");
+        loadBalancer.setContainerName("app");
+        loadBalancer.setContainerPort(80);
+
+        service.createService("fargate-bootstrap-cluster", "fargate-bootstrap-service",
+                taskDefinition.getTaskDefinitionArn(), 2, LaunchType.FARGATE, List.of(loadBalancer),
+                fargateNetworkConfiguration(), null, null, "CODE_DEPLOY", null, REGION);
+
+        List<TaskSet> taskSets = service.describeTaskSets("fargate-bootstrap-cluster",
+                "fargate-bootstrap-service", null, REGION);
+        assertEquals(1, taskSets.size());
+        TaskSet taskSet = taskSets.getFirst();
+        assertEquals("PRIMARY", taskSet.getStatus());
+        assertEquals(2, taskSet.getRunningCount());
+        assertEquals("STEADY_STATE", taskSet.getStabilityStatus());
+        assertEquals(List.of(loadBalancer), taskSet.getLoadBalancers());
+        verify(registrar, times(2)).registerTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+    }
+
+    @Test
+    void recreatingAFargateCodeDeployServiceDoesNotReuseDeletedTaskSets() {
+        EcsService service = newDockerModeService(mock(EcsLoadBalancerRegistrar.class));
+        service.createCluster("fargate-recreate-cluster", REGION);
+        TaskDefinition taskDefinition = registerFargateTaskDef(service, "fargate-recreate-task", "app:1");
+        NetworkConfiguration networkConfiguration = fargateNetworkConfiguration();
+        service.createService("fargate-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.FARGATE, List.of(), networkConfiguration, null, null, "CODE_DEPLOY", null, REGION);
+
+        TaskSet oldPrimary = service.describeTaskSets("fargate-recreate-cluster", "blue", null, REGION).getFirst();
+        EcsTask oldTask = runningTasks(service).getFirst();
+        service.deleteService("fargate-recreate-cluster", "blue", true, REGION);
+        service.createService("fargate-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.FARGATE, List.of(), networkConfiguration, null, null, "CODE_DEPLOY", null, REGION);
+
+        List<TaskSet> taskSets = service.describeTaskSets("fargate-recreate-cluster", "blue", null, REGION);
+        assertEquals(1, taskSets.size());
+        assertNotEquals(oldPrimary.getTaskSetArn(), taskSets.getFirst().getTaskSetArn());
+        assertEquals("PRIMARY", taskSets.getFirst().getStatus());
+        assertEquals(1, taskSets.getFirst().getRunningCount());
+        assertEquals(1, runningTasks(service).size());
+        assertEquals("STOPPED", service.describeTasks("fargate-recreate-cluster",
+                List.of(oldTask.getTaskArn()), REGION).getFirst().getLastStatus());
+    }
+
+    @Test
+    void forceDeletingCodeDeployServiceStopsAllTaskSetContainersAndDeregistersTheirTargetsBeforeRecreate() {
+        EcsLoadBalancerRegistrar registrar = mock(EcsLoadBalancerRegistrar.class);
+        EcsContainerManager containerManager = mock(EcsContainerManager.class);
+        EcsTaskHandle blueHandle = mock(EcsTaskHandle.class);
+        EcsTaskHandle greenFirstHandle = mock(EcsTaskHandle.class);
+        EcsTaskHandle greenSecondHandle = mock(EcsTaskHandle.class);
+        EcsTaskHandle otherServiceHandle = mock(EcsTaskHandle.class);
+        EcsTaskHandle recreatedHandle = mock(EcsTaskHandle.class);
+        when(containerManager.startTask(any(), any(), any(), eq(REGION))).thenReturn(
+                blueHandle, greenFirstHandle, greenSecondHandle, otherServiceHandle, recreatedHandle);
+        EcsService service = newDockerModeService(registrar, containerManager);
+        service.createCluster("delete-recreate-cluster", REGION);
+        TaskDefinition taskDefinition = registerFargateTaskDef(service, "delete-recreate-task", "app:1");
+        NetworkConfiguration networkConfiguration = fargateNetworkConfiguration();
+
+        EcsLoadBalancer blueLoadBalancer = loadBalancer("blue");
+        EcsLoadBalancer greenLoadBalancer = loadBalancer("green");
+        EcsLoadBalancer otherLoadBalancer = loadBalancer("other");
+        service.createService("delete-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.FARGATE, List.of(blueLoadBalancer), networkConfiguration, null, null,
+                "CODE_DEPLOY", null, REGION);
+        CreateTaskSetRequest greenTaskSetRequest = new CreateTaskSetRequest();
+        greenTaskSetRequest.setCluster("delete-recreate-cluster");
+        greenTaskSetRequest.setService("blue");
+        greenTaskSetRequest.setTaskDefinition(taskDefinition.getTaskDefinitionArn());
+        greenTaskSetRequest.setLaunchType(LaunchType.FARGATE);
+        greenTaskSetRequest.setNetworkConfiguration(networkConfiguration);
+        greenTaskSetRequest.setLoadBalancers(List.of(greenLoadBalancer));
+        greenTaskSetRequest.setScaleValue(200.0);
+        service.createTaskSet(greenTaskSetRequest, REGION);
+        service.createService("delete-recreate-cluster", "unaffected", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.FARGATE, List.of(otherLoadBalancer), networkConfiguration, null, null,
+                "CODE_DEPLOY", null, REGION);
+
+        service.deleteService("delete-recreate-cluster", "blue", true, REGION);
+        service.createService("delete-recreate-cluster", "blue", taskDefinition.getTaskDefinitionArn(), 1,
+                LaunchType.FARGATE, List.of(blueLoadBalancer), networkConfiguration, null, null,
+                "CODE_DEPLOY", null, REGION);
+
+        verify(containerManager).stopTaskAndCollectExitCodes(blueHandle);
+        verify(containerManager).stopTaskAndCollectExitCodes(greenFirstHandle);
+        verify(containerManager).stopTaskAndCollectExitCodes(greenSecondHandle);
+        verify(containerManager, never()).stopTaskAndCollectExitCodes(otherServiceHandle);
+        verify(containerManager, never()).stopTaskAndCollectExitCodes(recreatedHandle);
+        verify(registrar).deregisterTaskSetTask(any(EcsTask.class),
+                argThat(loadBalancers -> targets(loadBalancers, "blue")), eq(REGION));
+        verify(registrar, times(2)).deregisterTaskSetTask(any(EcsTask.class),
+                argThat(loadBalancers -> targets(loadBalancers, "green")), eq(REGION));
+        verify(registrar, never()).deregisterTask(any(EcsTask.class), any(), eq(REGION));
+        assertEquals(1, service.describeTaskSets("delete-recreate-cluster", "blue", null, REGION).size(),
+                "only the recreated service's primary task set remains");
+        assertEquals(2, runningTasks(service).size(),
+                "the recreated service and the unrelated service retain one task each");
+    }
+
+    @Test
+    void createEc2CodeDeployServiceDoesNotBootstrapATaskSet() {
+        EcsLoadBalancerRegistrar registrar = mock(EcsLoadBalancerRegistrar.class);
+        EcsService service = newDockerModeService(registrar);
+        service.createCluster("ec2-bootstrap-cluster", REGION);
+        TaskDefinition taskDefinition = registerTaskDef(service, "ec2-bootstrap-task", "app:1");
+
+        service.createService("ec2-bootstrap-cluster", "ec2-bootstrap-service",
+                taskDefinition.getTaskDefinitionArn(), 2, LaunchType.EC2, List.of(), null,
+                null, null, "CODE_DEPLOY", null, REGION);
+
+        assertEquals(List.of(), service.describeTaskSets("ec2-bootstrap-cluster",
+                "ec2-bootstrap-service", null, REGION));
+        assertEquals(List.of(), runningTasks(service));
+        verify(registrar, times(0)).registerTaskSetTask(any(EcsTask.class), anyList(), eq(REGION));
+    }
+
     @Test
     void serviceHoldingARawReferenceIsPinnedOnItsFirstTickInsteadOfRelaunchingForever() {
         EcsService service = newMockModeService();
@@ -219,6 +429,36 @@ class EcsServiceRolloutTest {
                 null, null, List.of(), REGION);
     }
 
+    private static TaskDefinition registerFargateTaskDef(EcsService service, String family, String image) {
+        ContainerDefinition cd = new ContainerDefinition();
+        cd.setName("app");
+        cd.setImage(image);
+        return service.registerTaskDefinition(family, List.of(cd), NetworkMode.awsvpc, "256", "512",
+                null, null, List.of("FARGATE"), REGION);
+    }
+
+    private static NetworkConfiguration fargateNetworkConfiguration() {
+        AwsVpcConfiguration awsvpc = new AwsVpcConfiguration();
+        awsvpc.setSubnets(List.of("subnet-12345678"));
+        NetworkConfiguration networkConfiguration = new NetworkConfiguration();
+        networkConfiguration.setAwsvpcConfiguration(awsvpc);
+        return networkConfiguration;
+    }
+
+    private static EcsLoadBalancer loadBalancer(String name) {
+        EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+        loadBalancer.setTargetGroupArn("arn:aws:elasticloadbalancing:us-east-1:000000000000:targetgroup/"
+                + name + "/id");
+        loadBalancer.setContainerName("app");
+        loadBalancer.setContainerPort(80);
+        return loadBalancer;
+    }
+
+    private static boolean targets(List<EcsLoadBalancer> loadBalancers, String name) {
+        return loadBalancers.size() == 1
+                && loadBalancers.getFirst().getTargetGroupArn().endsWith("/" + name + "/id");
+    }
+
     private static EcsService newMockModeService() {
         return newMockModeService(null);
     }
@@ -238,6 +478,29 @@ class EcsServiceRolloutTest {
         return service;
     }
 
+    private static EcsService newDockerModeService(EcsLoadBalancerRegistrar registrar) {
+        EcsContainerManager containerManager = mock(EcsContainerManager.class);
+        EcsTaskHandle taskHandle = mock(EcsTaskHandle.class);
+        when(containerManager.startTask(any(), any(), any(), eq(REGION))).thenReturn(taskHandle);
+        return newDockerModeService(registrar, containerManager);
+    }
+
+    private static EcsService newDockerModeService(EcsLoadBalancerRegistrar registrar,
+                                                    EcsContainerManager containerManager) {
+        EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+        when(config.services().ecs().mock()).thenReturn(false);
+        when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+        EcsService service = new EcsService(
+                new RegionResolver(REGION, "000000000000"),
+                containerManager,
+                config,
+                registrar,
+                new InMemoryStorageFactory(),
+                null);
+        service.initializeStorage();
+        return service;
+    }
+
     private static final class InMemoryStorageFactory extends StorageFactory {
         private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
 

src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTaskOwnershipTest.java

@@ -7,10 +7,14 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageBackend;
 import io.github.hectorvent.floci.core.storage.StorageFactory;
 import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.model.AwsVpcConfiguration;
 import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
 import io.github.hectorvent.floci.services.ecs.model.ContainerInstance;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
 import io.github.hectorvent.floci.services.ecs.model.EcsTask;
 import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import io.github.hectorvent.floci.services.ecs.model.NetworkMode;
+import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;
 import org.junit.jupiter.api.Test;
 
 import java.util.HashMap;
@@ -130,6 +134,28 @@ class EcsServiceTaskOwnershipTest {
                 "service teardown stops only the service's own tasks");
     }
 
+    @Test
+    void forceDeletingAFargateServiceClearsInactiveServiceTaskCounts() {
+        EcsService service = newMockModeService();
+        service.createCluster("fargate-delete-cluster", REGION);
+        registerFargateTaskDef(service);
+        service.createService("fargate-delete-cluster", "worker", "own-fam", 1, LaunchType.FARGATE,
+                List.of(), fargateNetworkConfiguration(), REGION);
+        service.reconcileServices();
+        service.reconcileServices();
+        EcsTask task = allTasks(service).getFirst();
+
+        service.deleteService("fargate-delete-cluster", "worker", true, REGION);
+
+        assertEquals("STOPPED", reload(service, task).getLastStatus());
+        EcsServiceModel deleted = service.describeServices("fargate-delete-cluster", List.of("worker"), REGION)
+                .getFirst();
+        assertEquals("INACTIVE", deleted.getStatus());
+        assertEquals(0, deleted.getDesiredCount());
+        assertEquals(0, deleted.getRunningCount());
+        assertEquals(0, deleted.getPendingCount());
+    }
+
     @Test
     void listTasksByServiceNameReturnsOnlyTasksTheServiceLaunched() {
         EcsService service = newMockModeService();
@@ -216,6 +242,22 @@ class EcsServiceTaskOwnershipTest {
         service.registerTaskDefinition("own-fam", List.of(cd), null, null, null, null, null, List.of(), region);
     }
 
+    private static void registerFargateTaskDef(EcsService service) {
+        ContainerDefinition cd = new ContainerDefinition();
+        cd.setName("app");
+        cd.setImage("app:1");
+        service.registerTaskDefinition("own-fam", List.of(cd), NetworkMode.awsvpc, "256", "512",
+                null, null, List.of("FARGATE"), REGION);
+    }
+
+    private static NetworkConfiguration fargateNetworkConfiguration() {
+        AwsVpcConfiguration awsvpc = new AwsVpcConfiguration();
+        awsvpc.setSubnets(List.of("subnet-12345678"));
+        NetworkConfiguration network = new NetworkConfiguration();
+        network.setAwsvpcConfiguration(awsvpc);
+        return network;
+    }
+
     private static EcsService newMockModeService() {
         EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
         when(config.services().ecs().mock()).thenReturn(true);

src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2IntegrationTest.java

@@ -23,6 +23,7 @@ class ElbV2IntegrationTest {
             "AWS4-HMAC-SHA256 Credential=test/20260427/us-east-1/ec2/aws4_request";
     private static final String ELB_V2_XMLNS =
             "https://elasticloadbalancing.amazonaws.com/doc/2015-12-01/";
+    private static final String ELB_V2_API_VERSION = "2015-12-01";
 
     private static String lbArn;
     private static String tgArn;
@@ -37,6 +38,7 @@ class ElbV2IntegrationTest {
     void createLoadBalancer() {
         lbArn = given()
                 .formParam("Action", "CreateLoadBalancer")
+                .formParam("Version", ELB_V2_API_VERSION)
                 .formParam("Name", "my-test-lb")
                 .formParam("Type", "application")
                 .formParam("Scheme", "internet-facing")
@@ -60,7 +62,7 @@ class ElbV2IntegrationTest {
                 .body("CreateLoadBalancerResponse.CreateLoadBalancerResult.LoadBalancers.member.AvailabilityZones.member.SubnetId",
                         hasItems(Ec2Service.defaultSubnetId("us-east-1", "a"), Ec2Service.defaultSubnetId("us-east-1", "b")))
                 .body("CreateLoadBalancerResponse.CreateLoadBalancerResult.LoadBalancers.member.DNSName",
-                        containsString(".elb.localhost.floci.io"))
+                        containsString(".elb.localhost"))
                 .extract()
                 .path("CreateLoadBalancerResponse.CreateLoadBalancerResult.LoadBalancers.member.LoadBalancerArn");
     }
@@ -70,6 +72,7 @@ class ElbV2IntegrationTest {
     void describeLoadBalancerByArn() {
         given()
                 .formParam("Action", "DescribeLoadBalancers")
+                .formParam("Version", ELB_V2_API_VERSION)
                 .formParam("LoadBalancerArns.member.1", lbArn)
                 .header("Authorization", AUTH)
             .when()
@@ -82,6 +85,18 @@ class ElbV2IntegrationTest {
                         equalTo("active"))
                 .body("DescribeLoadBalancersResponse.DescribeLoadBalancersResult.LoadBalancers.member.AvailabilityZones.member.SubnetId",
                         hasItems(Ec2Service.defaultSubnetId("us-east-1", "a"), Ec2Service.defaultSubnetId("us-east-1", "b")));
+
+        given()
+                .formParam("Action", "DescribeLoadBalancers")
+                .formParam("Version", ELB_V2_API_VERSION)
+                .formParam("LoadBalancerArns.member.1", lbArn)
+                .header("Authorization", AUTH)
+            .when()
+                .post("/")
+            .then()
+                .statusCode(200)
+                .body("DescribeLoadBalancersResponse.DescribeLoadBalancersResult.LoadBalancers.member.LoadBalancerArn",
+                        equalTo(lbArn));
     }
 
     @Test
@@ -89,6 +104,7 @@ class ElbV2IntegrationTest {
     void describeLoadBalancerByName() {
         given()
                 .formParam("Action", "DescribeLoadBalancers")
+                .formParam("Version", ELB_V2_API_VERSION)
                 .formParam("Names.member.1", "my-test-lb")
                 .header("Authorization", AUTH)
             .when()

src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java

@@ -1,5 +1,6 @@
 package io.github.hectorvent.floci.services.rds;
 
+import com.fasterxml.jackson.core.type.TypeReference;
 import io.github.hectorvent.floci.config.EmulatorConfig;
 import io.github.hectorvent.floci.core.common.AwsException;
 import io.github.hectorvent.floci.core.common.RegionResolver;
@@ -7,6 +8,7 @@ import io.github.hectorvent.floci.core.common.docker.CurrentContainerNetworkReso
 import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
 import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
 import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.PersistentStorage;
 import io.github.hectorvent.floci.core.storage.StorageBackend;
 import io.github.hectorvent.floci.services.ec2.Ec2Service;
 import io.github.hectorvent.floci.services.ec2.model.Subnet;
@@ -43,11 +45,13 @@ import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
 import io.github.hectorvent.floci.services.secretsmanager.model.Secret;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.CsvSource;
 import org.mockito.ArgumentCaptor;
 
 import java.lang.reflect.Field;
+import java.nio.file.Path;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Collection;
@@ -231,7 +235,7 @@ class RdsServiceTest {
 
     @Test
     void createDbInstanceRemovesVisibleRecordWhenProxyStartupFails() {
-        doThrow(new IllegalStateException("proxy down"))
+        doThrow(new IllegalStateException("proxy down")).doNothing()
                 .when(proxyManager).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),
                         any(), any(), any(), any(), any());
 
@@ -240,6 +244,58 @@ class RdsServiceTest {
                 "admin", "secret123", null, "db.t3.micro",
                 20, false, null, null, null, null, false));
         assertThrows(AwsException.class, () -> rdsService.getDbInstance("mypostgres"));
+
+        DbInstance replacement = rdsService.createDbInstance("replacement", "postgres", "13",
+                "admin", "secret123", null, "db.t3.micro",
+                20, false, null, null, null, null, false);
+        assertEquals(7000, replacement.getProxyPort());
+    }
+
+    @Test
+    void persistedInstancesPreventProxyPortReuseAfterRestart(@TempDir Path directory) {
+        when(config.services().rds().mock()).thenReturn(true);
+        Path instancesFile = directory.resolve("rds-instances.json");
+        PersistentStorage<String, DbInstance> initialInstances = new PersistentStorage<>(
+                instancesFile, new TypeReference<Map<String, DbInstance>>() {});
+        RdsService initialService = newService(containerManager, proxyManager,
+                initialInstances, new InMemoryStorage<>(), new InMemoryStorage<>(),
+                new InMemoryStorage<>(), new InMemoryStorage<>());
+        DbInstance first = initialService.createDbInstance("first", "postgres", "16.3",
+                "admin", "secret", "app", "db.t3.micro", 20, false,
+                null, null, null);
+
+        PersistentStorage<String, DbInstance> restoredInstances = new PersistentStorage<>(
+                instancesFile, new TypeReference<Map<String, DbInstance>>() {});
+        restoredInstances.load();
+        RdsService restartedService = newService(containerManager, proxyManager,
+                restoredInstances, new InMemoryStorage<>(), new InMemoryStorage<>(),
+                new InMemoryStorage<>(), new InMemoryStorage<>());
+        DbInstance second = restartedService.createDbInstance("second", "postgres", "16.3",
+                "admin", "secret", "app", "db.t3.micro", 20, false,
+                null, null, null);
+
+        assertEquals(7000, first.getProxyPort());
+        assertEquals(7001, second.getProxyPort());
+    }
+
+    @Test
+    void deletingOneInstancePreservesOtherInstances() {
+        when(config.services().rds().mock()).thenReturn(true);
+        InMemoryStorage<String, DbInstance> instances = new InMemoryStorage<>();
+        RdsService service = newService(containerManager, proxyManager, instances,
+                new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),
+                new InMemoryStorage<>());
+        service.createDbInstance("first", "postgres", "16.3", "admin", "secret", "app",
+                "db.t3.micro", 20, false, null, null, null);
+        DbInstance second = service.createDbInstance(
+                "second", "postgres", "16.3", "admin", "secret", "app",
+                "db.t3.micro", 20, false, null, null, null);
+
+        service.deleteDbInstance("first");
+
+        assertThrows(AwsException.class, () -> service.getDbInstance("first"));
+        assertEquals(second.getDbInstanceArn(), service.getDbInstance("second").getDbInstanceArn());
+        assertTrue(instances.get("us-east-1::second").isPresent());
     }
 
     @Test