bug fixes: backend services patches - Implemented and deployed systemd-backed AMIs

src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerTest.java

@@ -1,6 +1,7 @@
 package io.github.hectorvent.floci.lifecycle;
 
 import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
 import io.github.hectorvent.floci.config.FlociCertificateAuthority;
 import io.github.hectorvent.floci.core.common.ContainerTeardown;
 import io.github.hectorvent.floci.core.common.Resettable;
@@ -36,6 +37,7 @@ class EmulatorInfoControllerTest {
     @Mock private Instance<ContainerTeardown> containerTeardowns;
     @Mock private FlociCertificateAuthority certificateAuthority;
     @Mock private EmulatorConfig config;
+    @Mock private Ec2Service ec2Service;
     @Mock private ContainerTeardown sageMakerTeardown;
     @Mock private ContainerTeardown batchTeardown;
     @Mock private Resettable resettable;
@@ -45,7 +47,7 @@ class EmulatorInfoControllerTest {
     @BeforeEach
     void setUp() {
         controller = new EmulatorInfoController(serviceRegistry, initLifecycleState, storageFactory,
-                resettables, containerTeardowns, certificateAuthority, config);
+                resettables, containerTeardowns, certificateAuthority, config, ec2Service);
     }
 
     @Test

src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsMetricFilterResetTest.java

@@ -115,6 +115,6 @@ class CloudWatchLogsMetricFilterResetTest {
         when(resettables.iterator()).thenAnswer(call -> List.of(resettable).iterator());
         Instance<ContainerTeardown> teardowns = mock(Instance.class);
         when(teardowns.iterator()).thenAnswer(call -> List.<ContainerTeardown>of().iterator());
-        return new EmulatorInfoController(null, null, f.factory, resettables, teardowns, null, null);
+        return new EmulatorInfoController(null, null, f.factory, resettables, teardowns, null, null, null);
     }
 }

src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployEcsIntegrationTest.java

@@ -281,7 +281,7 @@ class CodeDeployEcsIntegrationTest {
 
     @Test
     @Order(11)
-    void deploymentCompletesSuccessfully() throws InterruptedException {
+    void deploymentFailsWithoutRealGreenTargets() throws InterruptedException {
         Assumptions.assumeTrue(deploymentId != null, "deploymentId must be set");
 
         String status = null;
@@ -302,7 +302,7 @@ class CodeDeployEcsIntegrationTest {
             TimeUnit.MILLISECONDS.sleep(500);
         }
 
-        Assertions.assertEquals("Succeeded", status, "ECS deployment should succeed");
+        Assertions.assertEquals("Failed", status, "ECS deployment without green targets must fail");
     }
 
     @Test
@@ -350,14 +350,14 @@ class CodeDeployEcsIntegrationTest {
         .then()
             .statusCode(200)
             .body("deploymentTargets[0].deploymentTargetType", equalTo("ECSTarget"))
-            .body("deploymentTargets[0].ecsTarget.status", equalTo("Succeeded"));
+            .body("deploymentTargets[0].ecsTarget.status", equalTo("Failed"));
     }
 
     @Test
     @Order(14)
-    void listenerNowPointsToGreen() {
+    void listenerRemainsOnBlueAfterFailedDeployment() {
         Assumptions.assumeTrue(listenerArn != null, "listenerArn must be set");
-        Assumptions.assumeTrue(greenTgArn != null, "greenTgArn must be set");
+        Assumptions.assumeTrue(blueTgArn != null, "blueTgArn must be set");
 
         given()
             .contentType(ELBV2_CONTENT_TYPE)
@@ -369,6 +369,6 @@ class CodeDeployEcsIntegrationTest {
         .then()
             .statusCode(200)
             .body("DescribeRulesResponse.DescribeRulesResult.Rules.member.Actions.member.TargetGroupArn",
-                    equalTo(greenTgArn));
+                    equalTo(blueTgArn));
     }
 }

src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServiceTest.java

@@ -8,7 +8,15 @@ import io.github.hectorvent.floci.services.ec2.Ec2Service;
 import io.github.hectorvent.floci.services.ec2.model.Instance;
 import io.github.hectorvent.floci.services.ec2.model.Reservation;
 import io.github.hectorvent.floci.services.ecs.EcsService;
+import io.github.hectorvent.floci.services.ecs.model.EcsLoadBalancer;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import io.github.hectorvent.floci.services.ecs.model.TaskSet;
 import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
+import io.github.hectorvent.floci.services.elbv2.model.Action;
+import io.github.hectorvent.floci.services.elbv2.model.Listener;
+import io.github.hectorvent.floci.services.elbv2.model.Rule;
+import io.github.hectorvent.floci.services.elbv2.model.TargetHealth;
 import io.github.hectorvent.floci.services.lambda.LambdaService;
 import io.github.hectorvent.floci.services.lambda.model.LambdaAlias;
 import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
@@ -32,6 +40,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import static org.awaitility.Awaitility.await;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -494,6 +503,247 @@ class CodeDeployServiceTest {
         verify(elbV2Service).shiftListenerForward(REGION, "listener", "blue", "green", 100);
     }
 
+    @Test
+    void ecsGreenTaskSetWaitsForFargateTasksToStart() throws InterruptedException {
+        EcsService ecsService = mock(EcsService.class);
+        AtomicInteger sleeps = new AtomicInteger();
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, ecsService,
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> sleeps.incrementAndGet());
+        TaskSet pending = greenTaskSet();
+        pending.setRunningCount(0);
+        TaskSet running = greenTaskSet();
+        when(ecsService.describeTaskSets("cluster", "service", List.of("green"), REGION))
+                .thenReturn(List.of(pending), List.of(running));
+
+        TaskSet actual = ecsDeploymentService.waitForGreenTaskSetRunning(REGION, "cluster", "service", pending,
+                new java.util.concurrent.atomic.AtomicBoolean());
+
+        assertEquals(running, actual);
+        assertEquals(1, sleeps.get());
+    }
+
+    @Test
+    void ecsGreenTaskSetFailsAfterFargateStartupTimeout() {
+        EcsService ecsService = mock(EcsService.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, ecsService,
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> { });
+        TaskSet pending = greenTaskSet();
+        pending.setRunningCount(0);
+        when(ecsService.describeTaskSets("cluster", "service", List.of("green"), REGION)).thenReturn(List.of(pending));
+
+        IllegalStateException exception = assertThrows(IllegalStateException.class,
+                () -> ecsDeploymentService.waitForGreenTaskSetRunning(REGION, "cluster", "service", pending,
+                        new java.util.concurrent.atomic.AtomicBoolean()));
+
+        assertEquals("ECS green task set has no running tasks", exception.getMessage());
+    }
+
+    @Test
+    void ecsGreenTaskSetMustMatchServiceLaunchTypeAndTargetGroup() {
+        EcsService ecsService = mock(EcsService.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, ecsService,
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        TaskSet green = greenTaskSet();
+        green.setLaunchType(LaunchType.EC2);
+        when(ecsService.describeTaskSets("cluster", "service", List.of("green"), REGION)).thenReturn(List.of(green));
+
+        assertThrows(IllegalStateException.class, () -> ecsDeploymentService.verifyGreenTaskSet(REGION,
+                "cluster", "service", "deployment", fargateService(), green, "green-target-group"));
+    }
+
+    @Test
+    void ecsGreenTaskSetAcceptsTheExistingEc2Path() {
+        EcsService ecsService = mock(EcsService.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, ecsService,
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        TaskSet green = greenTaskSet();
+        green.setLaunchType(LaunchType.EC2);
+        EcsServiceModel ec2Service = new EcsServiceModel();
+        ec2Service.setLaunchType(LaunchType.EC2);
+        when(ecsService.describeTaskSets("cluster", "service", List.of("green"), REGION)).thenReturn(List.of(green));
+
+        ecsDeploymentService.verifyGreenTaskSet(REGION, "cluster", "service", "deployment", ec2Service,
+                green, "green-target-group");
+    }
+
+    @Test
+    void ecsGreenTaskSetRejectsEc2WithoutRunningTasksImmediately() {
+        EcsService ecsService = mock(EcsService.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, ecsService,
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> { });
+        TaskSet green = greenTaskSet();
+        green.setLaunchType(LaunchType.EC2);
+        green.setRunningCount(0);
+        EcsServiceModel ec2Service = new EcsServiceModel();
+        ec2Service.setLaunchType(LaunchType.EC2);
+        when(ecsService.describeTaskSets("cluster", "service", List.of("green"), REGION)).thenReturn(List.of(green));
+
+        IllegalStateException exception = assertThrows(IllegalStateException.class,
+                () -> ecsDeploymentService.verifyGreenTaskSet(REGION, "cluster", "service", "deployment",
+                        ec2Service, green, "green-target-group"));
+
+        assertEquals("ECS green task set has no running tasks", exception.getMessage());
+    }
+
+    @Test
+    void ecsDeploymentVerifiesTheAuthoritativeDefaultRuleInsteadOfStaleListenerActions() {
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        Listener staleListener = new Listener();
+        staleListener.setDefaultActions(List.of(forwardAction("blue-target-group")));
+        when(elbV2Service.describeListeners(REGION, null, List.of("listener"))).thenReturn(List.of(staleListener));
+        when(elbV2Service.describeRules(REGION, "listener", null))
+                .thenReturn(List.of(defaultRule(forwardAction("green-target-group"))));
+
+        ecsDeploymentService.verifyListenersForwardToGreen(REGION, List.of("listener"), "green-target-group");
+
+        verify(elbV2Service, never()).describeListeners(anyString(), any(), any());
+    }
+
+    @Test
+    void ecsDeploymentAcceptsWeightedDefaultRuleWithAllTrafficOnGreen() {
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        Action weightedForward = new Action();
+        weightedForward.setType("forward");
+        weightedForward.setTargetGroups(List.of(targetGroup("blue-target-group", 0),
+                targetGroup("green-target-group", 100)));
+        when(elbV2Service.describeRules(REGION, "listener", null))
+                .thenReturn(List.of(defaultRule(weightedForward)));
+
+        ecsDeploymentService.verifyListenersForwardToGreen(REGION, List.of("listener"), "green-target-group");
+    }
+
+    @Test
+    void ecsDeploymentRejectsDefaultRuleWithResidualBlueTraffic() {
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                Thread::sleep);
+        Action weightedForward = new Action();
+        weightedForward.setType("forward");
+        weightedForward.setTargetGroups(List.of(targetGroup("blue-target-group", 1),
+                targetGroup("green-target-group", 99)));
+        when(elbV2Service.describeRules(REGION, "listener", null))
+                .thenReturn(List.of(defaultRule(weightedForward)));
+
+        IllegalStateException exception = assertThrows(IllegalStateException.class,
+                () -> ecsDeploymentService.verifyListenersForwardToGreen(REGION,
+                        List.of("listener"), "green-target-group"));
+
+        assertEquals("Production listener did not switch to the green target group", exception.getMessage());
+    }
+
+    @Test
+    void ecsDeploymentRejectsAnEmptyGreenTargetGroup() {
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> { });
+
+        assertThrows(IllegalStateException.class, () -> ecsDeploymentService.waitForGreenTargets(REGION,
+                "green-target-group", new java.util.concurrent.atomic.AtomicBoolean()));
+    }
+
+    @Test
+    void ecsDeploymentWaitsForHealthyGreenTargets() throws InterruptedException {
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        AtomicInteger sleeps = new AtomicInteger();
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> sleeps.incrementAndGet());
+        TargetHealth target = new TargetHealth();
+        target.setState("healthy");
+        when(elbV2Service.describeTargetHealth(REGION, "green-target-group", null))
+                .thenReturn(List.of(), List.of(target));
+
+        ecsDeploymentService.waitForGreenTargets(REGION, "green-target-group",
+                new java.util.concurrent.atomic.AtomicBoolean());
+
+        assertEquals(1, sleeps.get());
+    }
+
+    @Test
+    void ecsDeploymentWaitsForDefaultTargetGroupHealthThreshold() throws InterruptedException {
+        ElbV2Service elbV2Service = mock(ElbV2Service.class);
+        AtomicInteger polls = new AtomicInteger();
+        CodeDeployService ecsDeploymentService = new CodeDeployService(lambdaService, mock(EcsService.class),
+                elbV2Service, ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+                new RegionResolver(REGION, ACCOUNT_ID), null, Duration.ofHours(1), Clock.systemUTC(), null,
+                ignored -> { });
+        TargetHealth initial = new TargetHealth();
+        initial.setState("initial");
+        TargetHealth healthy = new TargetHealth();
+        healthy.setState("healthy");
+        when(elbV2Service.describeTargetHealth(REGION, "green-target-group", null))
+                .thenAnswer(ignored -> List.of(polls.incrementAndGet() > 150 ? healthy : initial));
+
+        ecsDeploymentService.waitForGreenTargets(REGION, "green-target-group",
+                new java.util.concurrent.atomic.AtomicBoolean());
+
+        assertEquals(151, polls.get());
+    }
+
+    private TaskSet greenTaskSet() {
+        EcsLoadBalancer loadBalancer = new EcsLoadBalancer();
+        loadBalancer.setTargetGroupArn("green-target-group");
+        TaskSet taskSet = new TaskSet();
+        taskSet.setTaskSetArn("green");
+        taskSet.setExternalId("deployment");
+        taskSet.setStartedBy("CODE_DEPLOY");
+        taskSet.setStatus("ACTIVE");
+        taskSet.setLaunchType(LaunchType.FARGATE);
+        taskSet.setComputedDesiredCount(1);
+        taskSet.setRunningCount(1);
+        taskSet.setLoadBalancers(List.of(loadBalancer));
+        return taskSet;
+    }
+
+    private EcsServiceModel fargateService() {
+        EcsServiceModel ecsService = new EcsServiceModel();
+        ecsService.setLaunchType(LaunchType.FARGATE);
+        return ecsService;
+    }
+
+    private Rule defaultRule(Action action) {
+        Rule rule = new Rule();
+        rule.setDefault(true);
+        rule.setActions(List.of(action));
+        return rule;
+    }
+
+    private Action forwardAction(String targetGroupArn) {
+        Action action = new Action();
+        action.setType("forward");
+        action.setTargetGroupArn(targetGroupArn);
+        return action;
+    }
+
+    private Action.TargetGroupTuple targetGroup(String targetGroupArn, int weight) {
+        Action.TargetGroupTuple target = new Action.TargetGroupTuple();
+        target.setTargetGroupArn(targetGroupArn);
+        target.setWeight(weight);
+        return target;
+    }
+
     // ---- Helpers --------------------------------------------------------------------------
 
     private String createServerDeployment(String appName, String groupName, int timeoutSeconds) {

src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineServiceTest.java

@@ -394,14 +394,17 @@ class CodePipelineServiceTest {
     }
 
     @Test
-    void codeDeployToEcsRegistersTaskDefinitionAndSubstitutesAppSpec() throws Exception {
+    void codeDeployToEcsSubstitutesFargateTaskDefinitionImageFromImageDetail() throws Exception {
         S3Service s3Service = mock(S3Service.class);
         CodeDeployService codeDeployService = mock(CodeDeployService.class);
         EcsService ecsService = mock(EcsService.class);
         S3Object source = new S3Object();
         source.setData(zipArtifact(Map.of(
                 "taskdef.json", """
-                        {"family":"pipeline-task","containerDefinitions":[{"name":"app","image":"nginx"}]}
+                        {"family":"pipeline-task","networkMode":"awsvpc","requiresCompatibilities":["FARGATE"],"containerDefinitions":[{"name":"app","image":"<IMAGE1_NAME>"}]}
+                        """,
+                "imageDetail.json", """
+                        {"ImageURI":"000000000000.dkr.ecr.us-east-1.amazonaws.com/pipeline-app:build-5"}
                         """,
                 "appspec.json", """
                         {"version":0.0,"Resources":[{"TargetService":{"Type":"AWS::ECS::Service","Properties":{"TaskDefinition":"<TASK_DEFINITION>","LoadBalancerInfo":{"ContainerName":"app","ContainerPort":80}}}}]}
@@ -429,6 +432,9 @@ class CodePipelineServiceTest {
             ArgumentCaptor<RegisterTaskDefinitionRequest> request = ArgumentCaptor.forClass(RegisterTaskDefinitionRequest.class);
             verify(ecsService).registerTaskDefinition(request.capture(), eq(REGION));
             assertEquals("pipeline-task", request.getValue().getFamily());
+            assertTrue(request.getValue().isFargate());
+            assertEquals("000000000000.dkr.ecr.us-east-1.amazonaws.com/pipeline-app:build-5",
+                    request.getValue().getContainerDefinitions().getFirst().getImage());
             ArgumentCaptor<Map<String, Object>> revision = ArgumentCaptor.forClass(Map.class);
             verify(codeDeployService).createDeployment(eq(REGION), eq("ecs-app"), eq("ecs-group"),
                     eq(null), revision.capture(), anyString());
@@ -447,7 +453,8 @@ class CodePipelineServiceTest {
         EcsService ecsService = mock(EcsService.class);
         S3Object source = new S3Object();
         source.setData(zipArtifact(Map.of(
-                "taskdef.json", "{\"family\":\"pipeline-task\",\"containerDefinitions\":[]}",
+                "taskdef.json", "{\"family\":\"pipeline-task\",\"containerDefinitions\":[{\"name\":\"app\",\"image\":\"<IMAGE1_NAME>\"}]}",
+                "imageDetail.json", "{\"ImageURI\":\"000000000000.dkr.ecr.us-east-1.amazonaws.com/pipeline-app:build-5\"}",
                 "appspec.json", "{\"version\":0.0,\"Resources\":[{\"TargetService\":{\"Properties\":{\"TaskDefinition\":\"<TASK_DEFINITION>\"}}}]}")));
         source.setETag("source-etag");
         source.setLastModified(Instant.now());
@@ -964,8 +971,12 @@ class CodePipelineServiceTest {
                                 "configuration": {
                                     "ApplicationName": "ecs-app",
                                     "DeploymentGroupName": "ecs-group",
-                                    "TaskDefinitionTemplatePath": "SourceOutput::taskdef.json",
-                                    "AppSpecTemplatePath": "SourceOutput::appspec.json"
+                                    "TaskDefinitionTemplateArtifact": "SourceOutput",
+                                    "TaskDefinitionTemplatePath": "taskdef.json",
+                                    "AppSpecTemplateArtifact": "SourceOutput",
+                                    "AppSpecTemplatePath": "appspec.json",
+                                    "Image1ArtifactName": "SourceOutput",
+                                    "Image1ContainerName": "app"
                                 },
                                 "inputArtifacts": [{"name": "SourceOutput"}]
                             }]

src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java

@@ -854,6 +854,14 @@ class Ec2ContainerManagerTest {
                 "dnf must be probed before yum");
     }
 
+    @Test
+    void sshdInstallProbeHandlesOpenSuse() {
+        String script = Ec2ContainerManager.sshdInstallProbeCommand()[2];
+
+        assertTrue(script.contains("command -v zypper"), script);
+        assertTrue(script.contains("zypper --non-interactive install -y openssh"), script);
+    }
+
     @Test
     void metadataProxyInstallCommandHandlesYumForAmazonLinux2() {
         // The IMDS proxy dependency install had the same gap, and unlike the sshd probe it
@@ -866,6 +874,15 @@ class Ec2ContainerManagerTest {
         assertTrue(script.contains("yum install -y iproute iptables socat curl ca-certificates"), script);
     }
 
+    @Test
+    void metadataProxyInstallCommandHandlesOpenSuse() {
+        String script = Ec2ContainerManager.metadataProxyInstallCommand()[2];
+
+        assertTrue(script.contains("command -v zypper"), script);
+        assertTrue(script.contains("zypper --non-interactive install -y iproute2 iptables socat curl ca-certificates"),
+                script);
+    }
+
     @Test
     void metadataProxyInstallCommandLetsDnfReplaceCurlMinimalWithCurl() {
         // public.ecr.aws/amazonlinux/amazonlinux:2023 -- the fallback image AmiImageResolver
@@ -1354,9 +1371,10 @@ class Ec2ContainerManagerTest {
     private static final String[] SSHD_INSTALL_PROBE_CMD = {"sh", "-c",
             "if ! command -v sshd >/dev/null 2>&1 || ! command -v scp >/dev/null 2>&1; then"
             + "  if command -v dnf >/dev/null 2>&1; then dnf install -y openssh-server openssh-clients >/dev/null 2>&1;"
-            + "  elif command -v yum >/dev/null 2>&1; then yum install -y openssh-server openssh-clients >/dev/null 2>&1;"
-            + "  elif command -v apt-get >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server openssh-client >/dev/null 2>&1;"
-            + "  elif command -v apk >/dev/null 2>&1; then apk add --no-cache openssh >/dev/null 2>&1;"
+             + "  elif command -v yum >/dev/null 2>&1; then yum install -y openssh-server openssh-clients >/dev/null 2>&1;"
+             + "  elif command -v apt-get >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server openssh-client >/dev/null 2>&1;"
+             + "  elif command -v zypper >/dev/null 2>&1; then zypper --non-interactive install -y openssh >/dev/null 2>&1;"
+             + "  elif command -v apk >/dev/null 2>&1; then apk add --no-cache openssh >/dev/null 2>&1;"
             + "  fi;"
             + "fi;"
             + "command -v sshd >/dev/null 2>&1 || exit 1;"

src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ImageCatalogTest.java

@@ -37,6 +37,7 @@ class Ec2ImageCatalogTest {
                 "ami-ubuntu2404-amd64",
                 "ami-ubuntu2404-cloud-arm64",
                 "ami-debian12",
+                "ami-debian13",
                 "ami-alpine",
                 "ami-fedora42",
                 "ami-opensuse-leap156"), imageIds);
@@ -78,9 +79,14 @@ class Ec2ImageCatalogTest {
     }
 
     @Test
-    void resolverUsesFedoraAndOpenSuseImages() {
-        assertEquals("registry.fedoraproject.org/fedora:42", amiImageResolver.resolve("ami-fedora42"));
-        assertEquals("registry.opensuse.org/opensuse/leap:15.6", amiImageResolver.resolve("ami-opensuse-leap156"));
+    void resolverUsesFedoraAndLocalSshReadyImages() {
+        assertEquals("floci/ami-ubuntu:20.04", amiImageResolver.resolve("ami-0abcdef1234567892"));
+        assertEquals("floci/ami-debian:13", amiImageResolver.resolve("ami-debian13"));
+        assertEquals("floci/ami-fedora:42", amiImageResolver.resolve("ami-fedora42"));
+        assertEquals("floci/ami-opensuse:15.6", amiImageResolver.resolve("ami-opensuse-leap156"));
+        assertTrue(amiImageResolver.resolveImage("ami-0abcdef1234567892").systemd());
+        assertTrue(amiImageResolver.resolveImage("ami-debian13").systemd());
+        assertTrue(amiImageResolver.resolveImage("ami-fedora42").systemd());
     }
 
     @Test

src/test/java/io/github/hectorvent/floci/services/ec2/Ec2IntegrationTest.java

@@ -2876,6 +2876,26 @@ class Ec2IntegrationTest {
             .extract().path("RunInstancesResponse.instancesSet.item.instanceId");
     }
 
+    @Test
+    @Order(80)
+    void runInstancesUsesSecurityGroupsFromNetworkInterface() {
+        given()
+            .formParam("Action", "RunInstances")
+            .formParam("ImageId", "ami-0abcdef1234567890")
+            .formParam("InstanceType", "t2.micro")
+            .formParam("MinCount", "1")
+            .formParam("MaxCount", "1")
+            .formParam("NetworkInterface.1.DeviceIndex", "0")
+            .formParam("NetworkInterface.1.SubnetId", subnetId)
+            .formParam("NetworkInterface.1.SecurityGroupId.1", securityGroupId)
+            .header("Authorization", AUTH_HEADER)
+        .when()
+            .post("/")
+        .then()
+            .statusCode(200)
+            .body("RunInstancesResponse.instancesSet.item.groupSet.item.groupId", equalTo(securityGroupId));
+    }
+
     @Test
     @Order(80)
     void runInstancesWithArm64ImageDescribesArm64Architecture() {

src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEcrRewriteTest.java

@@ -108,6 +108,8 @@ class EcsContainerManagerEcrRewriteTest {
         List<String> images = imageCaptor.getAllValues();
         assertEquals(REWRITTEN_IMAGE, images.get(0), "ECR-shaped image should be rewritten for the docker pull");
         assertEquals("sidecar:latest", images.get(1), "non-ECR image should pass through unchanged");
+        verify(lifecycleManager).refreshImage(REWRITTEN_IMAGE);
+        verify(lifecycleManager).refreshImage("sidecar:latest");
 
         // RunTask/DescribeTasks must keep reporting the original task-def image, not the rewrite.
         List<Container> containers = task.getContainers();

src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheIntegrationTest.java

@@ -300,6 +300,23 @@ class ElastiCacheIntegrationTest {
 
     @Test
     @Order(13)
+    void keyExpiresAtTheConfiguredTtl() throws Exception {
+        try (Socket socket = openSocket(firstProxyPort)) {
+            write(socket, respArray("AUTH", GROUP_AUTH_TOKEN));
+            assertEquals("+OK\r\n", readLine(socket));
+
+            write(socket, respArray("SET", "expires-after-one-second", "value", "EX", "1"));
+            assertEquals("+OK\r\n", readLine(socket));
+
+            Thread.sleep(1_100);
+
+            write(socket, respArray("GET", "expires-after-one-second"));
+            assertEquals("$-1\r\n", readLine(socket));
+        }
+    }
+
+    @Test
+    @Order(14)
     void deleteUserRemovesUserFromDescribeUsers() {
         given()
             .formParam("Action", "DeleteUser")
@@ -322,7 +339,7 @@ class ElastiCacheIntegrationTest {
     }
 
     @Test
-    @Order(14)
+    @Order(15)
     void deleteReplicationGroupReleasesProxyPortForReuse() {
         given()
             .formParam("Action", "DeleteReplicationGroup")

src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ServiceTest.java

@@ -225,6 +225,35 @@ class ElbV2ServiceTest {
                 .get(rule.getRuleArn()).containsKey("env"));
     }
 
+    @Test
+    void shiftListenerForwardPersistsTheGreenTargetGroup() {
+        SharedStorageFactory storageFactory = new SharedStorageFactory();
+        ElbV2Service first = serviceWithStorage(storageFactory, mock(ElbV2DataPlane.class), mock(ElbV2HealthChecker.class));
+        String lbArn = first.createLoadBalancer(
+                REGION, "traffic-lb", "internal", "application", "ipv4",
+                ALB_SUBNETS, List.of("sg-a"), Map.of()).getLoadBalancerArn();
+        String blueTargetGroupArn = first.createTargetGroup(
+                REGION, "traffic-blue", "HTTP", "HTTP1", 8080, "vpc-a", "ip",
+                "HTTP", "traffic-port", true, "/", 30, 5, 5, 2, "200", "ipv4", Map.of()).getTargetGroupArn();
+        String greenTargetGroupArn = first.createTargetGroup(
+                REGION, "traffic-green", "HTTP", "HTTP1", 8080, "vpc-a", "ip",
+                "HTTP", "traffic-port", true, "/", 30, 5, 5, 2, "200", "ipv4", Map.of()).getTargetGroupArn();
+        String listenerArn = first.createListener(
+                REGION, lbArn, "HTTP", 9080, null, List.of(),
+                List.of(forwardAction(blueTargetGroupArn)), List.of(), Map.of()).getListenerArn();
+
+        first.shiftListenerForward(REGION, listenerArn, blueTargetGroupArn, greenTargetGroupArn, 100);
+
+        ElbV2Service reloaded = serviceWithStorage(storageFactory, mock(ElbV2DataPlane.class), mock(ElbV2HealthChecker.class));
+        Rule defaultRule = reloaded.describeRules(REGION, listenerArn, null).stream()
+                .filter(Rule::isDefault)
+                .findFirst()
+                .orElseThrow();
+        assertEquals(greenTargetGroupArn, defaultRule.getActions().getFirst().getTargetGroupArn());
+        assertEquals(greenTargetGroupArn, reloaded.describeListeners(REGION, lbArn, List.of(listenerArn)).getFirst()
+                .getDefaultActions().getFirst().getTargetGroupArn());
+    }
+
     @Test
     void initializeStorageDoesNotTouchCollaborators() {
         // #1913: ElbV2DataPlane.binding() calls back into this service, and during @PostConstruct the

src/test/java/io/github/hectorvent/floci/services/kafkaconnect/KafkaConnectIntegrationTest.java

@@ -0,0 +1,60 @@
+package io.github.hectorvent.floci.services.kafkaconnect;
+
+import io.github.hectorvent.floci.services.msk.MskService;
+import io.github.hectorvent.floci.services.msk.model.CreateClusterRequest;
+import io.github.hectorvent.floci.services.msk.model.MskCluster;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+@io.quarkus.test.junit.QuarkusTest
+class KafkaConnectIntegrationTest {
+    @Inject S3Service s3Service;
+    @Inject MskService mskService;
+
+    @Test
+    void customPluginWorkerConfigurationAndConnectorPersistThroughAwsRoutes() throws Exception {
+        s3Service.createBucket("plugins", null);
+        s3Service.putObject("plugins", "sink.zip", zip(), "application/zip", Map.of());
+        CreateClusterRequest clusterRequest = new CreateClusterRequest();
+        clusterRequest.setClusterName("connect-cluster");
+        MskCluster cluster = mskService.createCluster(clusterRequest);
+
+        String pluginArn = given().contentType("application/json")
+                .body("""
+                        {"customPluginName":"s3-sink","contentType":"ZIP","contentLocation":{"s3Location":{"bucketArn":"arn:aws:s3:::plugins","fileKey":"sink.zip"}}}
+                        """)
+                .when().post("/v1/custom-plugins").then().statusCode(200)
+                .body("customPluginArn", notNullValue()).extract().path("customPluginArn");
+        String workerArn = given().contentType("application/json")
+                .body("{\"name\":\"worker\",\"propertiesFileContent\":\"offset.flush.interval.ms=1000\"}")
+                .when().post("/v1/worker-configurations").then().statusCode(200)
+                .body("revision", equalTo(1)).extract().path("workerConfigurationArn");
+
+        given().contentType("application/json")
+                .body("""
+                        {"connectorName":"sink","connectorConfiguration":{"connector.class":"example.Sink","topics":"events"},"kafkaCluster":{"apacheKafkaCluster":{"bootstrapServers":["%s"]}},"plugins":[{"customPlugin":{"customPluginArn":"%s"}}],"workerConfiguration":{"workerConfigurationArn":"%s"}}
+                        """.formatted(cluster.getBootstrapBrokers(), pluginArn, workerArn))
+                .when().post("/v1/connectors").then().statusCode(200).body("connectorState", equalTo("RUNNING"));
+        given().when().get("/v1/connectors").then().statusCode(200).body("connectors.size()", equalTo(1));
+    }
+
+    private byte[] zip() throws Exception {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        try (ZipOutputStream zip = new ZipOutputStream(output)) {
+            zip.putNextEntry(new ZipEntry("README.txt"));
+            zip.write("plugin".getBytes(java.nio.charset.StandardCharsets.UTF_8));
+            zip.closeEntry();
+        }
+        return output.toByteArray();
+    }
+}

src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisIntegrationTest.java

@@ -584,7 +584,7 @@ class KinesisIntegrationTest {
 
     @Test
     @Order(31)
-    void createStreamWithOnDemandMode() {
+    void createStreamRejectsShardCountAndOnDemandModeTogether() {
         given()
             .header("X-Amz-Target", "Kinesis_20131202.CreateStream")
             .contentType(KINESIS_CONTENT_TYPE)
@@ -594,19 +594,8 @@ class KinesisIntegrationTest {
         .when()
             .post("/")
         .then()
-            .statusCode(200);
-
-        given()
-            .header("X-Amz-Target", "Kinesis_20131202.DescribeStream")
-            .contentType(KINESIS_CONTENT_TYPE)
-            .body("""
-                {"StreamName": "on-demand-create-test"}
-                """)
-        .when()
-            .post("/")
-        .then()
-            .statusCode(200)
-            .body("StreamDescription.StreamModeDetails.StreamMode", equalTo("ON_DEMAND"));
+            .statusCode(400)
+            .body("__type", equalTo("InvalidArgumentException"));
     }
 
     @Test

src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java

@@ -339,7 +339,6 @@ class KinesisJsonHandlerTest {
     void createStreamHonorsOnDemandStreamMode() {
         ObjectNode req = MAPPER.createObjectNode();
         req.put("StreamName", "test-stream");
-        req.put("ShardCount", 1);
         req.putObject("StreamModeDetails").put("StreamMode", "ON_DEMAND");
         assertThat(handler.handle("CreateStream", req, REGION).getStatus(), is(200));
 
@@ -1249,7 +1248,6 @@ class KinesisJsonHandlerTest {
     void updateMaxRecordSizeRejectsOnDemandStreams() {
         ObjectNode create = MAPPER.createObjectNode();
         create.put("StreamName", "on-demand-stream");
-        create.put("ShardCount", 1);
         create.putObject("StreamModeDetails").put("StreamMode", "ON_DEMAND");
         assertThat(handler.handle("CreateStream", create, REGION).getStatus(), is(200));
 
@@ -1363,7 +1361,6 @@ class KinesisJsonHandlerTest {
     void updateShardCountRejectsOnDemandStream() {
         ObjectNode create = MAPPER.createObjectNode();
         create.put("StreamName", "on-demand-stream");
-        create.put("ShardCount", 2);
         create.putObject("StreamModeDetails").put("StreamMode", "ON_DEMAND");
         assertThat(handler.handle("CreateStream", create, REGION).getStatus(), is(200));
 
@@ -1584,6 +1581,76 @@ class KinesisJsonHandlerTest {
         assertEquals(Map.of("Foo", "Bar"), service.listTagsForStream("test-stream", REGION));
     }
 
+    @Test
+    void createStreamRequiresExactlyOneCapacitySetting() {
+        ObjectNode missingCapacity = MAPPER.createObjectNode();
+        missingCapacity.put("StreamName", "missing-capacity");
+        AwsException missingException = assertThrows(AwsException.class,
+                () -> handler.handle("CreateStream", missingCapacity, REGION));
+        assertEquals("InvalidArgumentException", missingException.getErrorCode());
+
+        ObjectNode bothCapacitySettings = MAPPER.createObjectNode();
+        bothCapacitySettings.put("StreamName", "both-capacity-settings");
+        bothCapacitySettings.put("ShardCount", 1);
+        bothCapacitySettings.putObject("StreamModeDetails").put("StreamMode", "ON_DEMAND");
+        AwsException bothException = assertThrows(AwsException.class,
+                () -> handler.handle("CreateStream", bothCapacitySettings, REGION));
+        assertEquals("InvalidArgumentException", bothException.getErrorCode());
+
+        ObjectNode zeroShards = MAPPER.createObjectNode();
+        zeroShards.put("StreamName", "zero-shards");
+        zeroShards.put("ShardCount", 0);
+        AwsException zeroException = assertThrows(AwsException.class,
+                () -> handler.handle("CreateStream", zeroShards, REGION));
+        assertEquals("InvalidArgumentException", zeroException.getErrorCode());
+    }
+
+    @Test
+    void getShardIteratorValidatesIteratorTypeAndSequenceNumber() {
+        createStream("test-stream");
+
+        ObjectNode invalidType = MAPPER.createObjectNode();
+        invalidType.put("StreamName", "test-stream");
+        invalidType.put("ShardId", FIRST_SHARD_ID);
+        invalidType.put("ShardIteratorType", "INVALID");
+        AwsException invalidTypeException = assertThrows(AwsException.class,
+                () -> handler.handle("GetShardIterator", invalidType, REGION));
+        assertEquals("InvalidArgumentException", invalidTypeException.getErrorCode());
+
+        ObjectNode missingSequence = MAPPER.createObjectNode();
+        missingSequence.put("StreamName", "test-stream");
+        missingSequence.put("ShardId", FIRST_SHARD_ID);
+        missingSequence.put("ShardIteratorType", "AT_SEQUENCE_NUMBER");
+        AwsException missingSequenceException = assertThrows(AwsException.class,
+                () -> handler.handle("GetShardIterator", missingSequence, REGION));
+        assertEquals("InvalidArgumentException", missingSequenceException.getErrorCode());
+    }
+
+    @Test
+    void getRecordsAcceptsKinesisMaximumLimitAndRejectsOutOfRangeValues() {
+        createStream("test-stream");
+        ObjectNode iteratorRequest = MAPPER.createObjectNode();
+        iteratorRequest.put("StreamName", "test-stream");
+        iteratorRequest.put("ShardId", FIRST_SHARD_ID);
+        iteratorRequest.put("ShardIteratorType", "TRIM_HORIZON");
+        String iterator = responseEntity(handler.handle("GetShardIterator", iteratorRequest, REGION))
+                .get("ShardIterator").asText();
+
+        ObjectNode maximumLimit = MAPPER.createObjectNode();
+        maximumLimit.put("ShardIterator", iterator);
+        maximumLimit.put("Limit", 10000);
+        assertThat(handler.handle("GetRecords", maximumLimit, REGION).getStatus(), is(200));
+
+        for (int invalidLimit : List.of(0, 10001)) {
+            ObjectNode invalidLimitRequest = MAPPER.createObjectNode();
+            invalidLimitRequest.put("ShardIterator", iterator);
+            invalidLimitRequest.put("Limit", invalidLimit);
+            AwsException exception = assertThrows(AwsException.class,
+                    () -> handler.handle("GetRecords", invalidLimitRequest, REGION));
+            assertEquals("InvalidArgumentException", exception.getErrorCode());
+        }
+    }
+
     /** Every non-string tag value shape — number, boolean, null, object, array — plus a map mixing a valid and an invalid value. */
     private static List<ObjectNode> nonStringTagValues() {
         ObjectNode number = MAPPER.createObjectNode();

src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java

@@ -284,6 +284,34 @@ class ImageCacheServiceTest {
         verify(pullImage).exec(any(PullImageResultCallback.class));
     }
 
+    @Test
+    void refreshesMutableTagWhileCachedImageStillExists() throws Exception {
+        DockerClient dockerClient = mock(DockerClient.class);
+        InspectImageCmd inspectImage = mock(InspectImageCmd.class);
+        InspectImageCmd inspectCachedImage = mock(InspectImageCmd.class);
+        PullImageCmd pullImage = mock(PullImageCmd.class);
+        PullImageResultCallback callback = mock(PullImageResultCallback.class);
+        when(dockerClient.inspectImageCmd(IMAGE)).thenReturn(inspectImage);
+        when(dockerClient.inspectImageCmd("sha256:old")).thenReturn(inspectCachedImage);
+        when(dockerClient.inspectImageCmd("sha256:new")).thenReturn(inspectCachedImage);
+        when(inspectImage.exec()).thenReturn(
+                new InspectImageResponse().withId("sha256:old").withOs("linux").withArch("amd64"),
+                new InspectImageResponse().withId("sha256:new").withOs("linux").withArch("amd64"));
+        when(inspectCachedImage.exec()).thenReturn(
+                new InspectImageResponse().withId("sha256:old").withOs("linux").withArch("amd64"),
+                new InspectImageResponse().withId("sha256:new").withOs("linux").withArch("amd64"));
+        when(dockerClient.pullImageCmd(IMAGE)).thenReturn(pullImage);
+        when(pullImage.withAuthConfig(any())).thenReturn(pullImage);
+        when(pullImage.exec(any(PullImageResultCallback.class))).thenReturn(callback);
+
+        ImageCacheService service = newService(dockerClient);
+        assertEquals("sha256:old", service.ensureImageExists(IMAGE));
+        assertEquals("sha256:new", service.refreshImage(IMAGE));
+        assertEquals("sha256:new", service.ensureImageExists(IMAGE));
+
+        verify(pullImage).exec(any(PullImageResultCallback.class));
+    }
+
     @Test
     void succeedsOnFirstAttempt() throws Exception {
         AtomicInteger calls = new AtomicInteger();

src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.java

@@ -43,7 +43,6 @@ import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;
 
 @ExtendWith(MockitoExtension.class)
@@ -177,14 +176,15 @@ class RedpandaManagerTest {
 
         int flagIndex = spec.cmd().indexOf("--advertise-kafka-addr");
         assertTrue(flagIndex >= 0, "cmd should contain --advertise-kafka-addr");
-        assertEquals("localhost:54321", spec.cmd().get(flagIndex + 1));
+        assertEquals("internal://floci-msk-abc123:9093,external://localhost:54321", spec.cmd().get(flagIndex + 1));
 
         assertEquals(Integer.valueOf(54321), spec.portBindings().get(KAFKA_PORT));
     }
 
     @Test
-    void containerModeAdvertisesContainerNameAddress() {
+    void containerModeAdvertisesInternalAndExternalAddresses() {
         when(containerDetector.isRunningInContainer()).thenReturn(true);
+        when(portAllocator.allocate(9300, 9399)).thenReturn(54321);
 
         ContainerInfo info = new ContainerInfo("container-456",
                 Map.of(KAFKA_PORT, new EndpointInfo("172.18.0.5", KAFKA_PORT)));
@@ -199,15 +199,15 @@ class RedpandaManagerTest {
 
         int flagIndex = spec.cmd().indexOf("--advertise-kafka-addr");
         assertTrue(flagIndex >= 0, "cmd should contain --advertise-kafka-addr");
-        assertEquals("floci-msk-abc123:9092", spec.cmd().get(flagIndex + 1));
+        assertEquals("internal://floci-msk-abc123:9093,external://localhost:54321", spec.cmd().get(flagIndex + 1));
 
-        assertFalse(spec.portBindings().containsKey(KAFKA_PORT), "container mode should not publish ports to host");
+        assertEquals(Integer.valueOf(54321), spec.portBindings().get(KAFKA_PORT));
         assertTrue(spec.exposedPorts().contains(KAFKA_PORT));
         assertTrue(spec.exposedPorts().contains(RedpandaManager.ADMIN_PORT));
         assertFalse(Files.exists(tempDir.resolve("msk").resolve("test-cluster")),
                 "container mode must not create host directories from inside the emulator container");
 
-        verifyNoInteractions(portAllocator);
+        verify(portAllocator).allocate(9300, 9399);
     }
 
     @Test

src/test/resources/application.yml

@@ -162,6 +162,10 @@ floci:
       enabled: true
       mock: true
       default-image: "redpandadata/redpanda:latest"
+    kafkaconnect:
+      enabled: true
+      mock: true
+      worker-image: apache/kafka:3.9.0
     amazonmq:
       enabled: true
       mock: true