feat: Lightsail + svg animation + ec2 bug fixes

This commit is contained in:
Zakaria
2026-10-11 14:19:49 -04:00
parent 019570d8a4
commit 33c480e7e1
36 changed files with 2857 additions and 153 deletions
@@ -12,8 +12,11 @@ RUN apt-get update \
curl \
ca-certificates \
&& systemctl enable ssh \
&& printf '#!/bin/sh\nmkdir -p /run/sshd\nexec "$@"\n' > /usr/local/sbin/floci-init \
&& chmod +x /usr/local/sbin/floci-init \
&& rm -rf /var/lib/apt/lists/*
ENV container=docker
STOPSIGNAL SIGRTMIN+3
ENTRYPOINT ["/usr/local/sbin/floci-init"]
CMD ["/sbin/init"]
+1
View File
@@ -16,6 +16,7 @@ The local implementation keeps Lightsail state in Floci storage and is designed
- Key pairs: create, import, get, list, delete, and download default key pair
- Discovery: regions, blueprints, bundles, active names, operations, and operations for resource
- Tags: tag and untag local Lightsail resources
- Node.js instances: the `nodejs` blueprint uses the multi-architecture `node:24-bookworm` current LTS image, supports the selected SSH key, and persists application state in a named volume at `/opt/lightsail/app`. The volume starts with `releases/<timestamp>` and `current`; the in-container supervisor runs `npm start` from `current`, captures stdout and stderr under `logs/`, restarts crashes with capped exponential backoff, and retains the three newest releases. Application deployment remains SSH/manual in this phase: no deployment API or console workflow is provided.
Read-only list operations for cloud-side Lightsail resource families that are not locally implemented yet return empty AWS-shaped lists. Cloud-only provisioning workflows such as container services, distributions, managed databases, load balancers, buckets, and snapshots are recognized from the AWS Lightsail API model and return an explicit `UnsupportedOperation` error instead of a custom endpoint or non-AWS response shape.
@@ -124,7 +124,7 @@ public class ContainerLifecycleManager {
*/
public String create(ContainerSpec spec) {
String resolvedImage = imageCacheService.ensureImageExists(spec.image());
return create(spec, resolvedImage, null);
return create(spec, resolvedImage, (String) null);
}
/**
@@ -139,6 +139,35 @@ public class ContainerLifecycleManager {
return create(spec, resolvedImage, platform);
}
public String create(ContainerSpec spec, String platform, ContainerCreationObserver observer) {
String resolvedImage = imageCacheService.ensureImageExists(spec.image(), platform,
new ImageCacheService.ImagePullObserver() {
@Override
public void started(String imageUri) {
observer.imagePullStarted(imageUri);
}
@Override
public void completed(String imageUri) {
observer.imagePullCompleted(imageUri);
}
});
observer.containerCreationStarted();
String containerId = create(spec, resolvedImage, platform);
observer.containerCreated(containerId);
return containerId;
}
public interface ContainerCreationObserver {
default void imagePullStarted(String imageUri) {}
default void imagePullCompleted(String imageUri) {}
default void containerCreationStarted() {}
default void containerCreated(String containerId) {}
}
private String create(ContainerSpec spec, String resolvedImage, String platform) {
String containerId = createWithCaBundle(spec, resolvedImage, platform);
if (spec.hasNetworkConfiguration()) {
@@ -13,6 +13,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.function.IntConsumer;
/**
* Utility for allocating free TCP ports for Docker container port bindings.
@@ -55,6 +56,10 @@ public class PortAllocator {
* @throws RuntimeException if no free port is available in the range
*/
public synchronized int allocate(int basePort, int maxPort) {
return allocate(basePort, maxPort, ignored -> {});
}
public synchronized int allocate(int basePort, int maxPort, IntConsumer occupiedPortObserver) {
// One Docker API round-trip per allocation, not one per candidate: the
// snapshot is only as fresh as the moment allocate() started either way,
// and this loop runs inside a lock every consumer of the shared
@@ -66,6 +71,7 @@ public class PortAllocator {
LOG.debugv("Allocated port {0} from range {1}-{2}", String.valueOf(port), String.valueOf(basePort), String.valueOf(maxPort));
return port;
}
occupiedPortObserver.accept(port);
}
throw new RuntimeException("No free port available in range " + basePort + "-" + maxPort);
}
@@ -7,13 +7,19 @@ import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.common.ContainerTeardowns;
import io.github.hectorvent.floci.core.common.ServiceRegistry;
import io.github.hectorvent.floci.services.ec2.Ec2Service;
import io.github.hectorvent.floci.services.lightsail.LightsailService;
import io.github.hectorvent.floci.services.lightsail.LightsailContainerManager;
import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook;
import jakarta.inject.Inject;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Consumes;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.jboss.resteasy.reactive.RestForm;
import org.jboss.resteasy.reactive.multipart.FileUpload;
import org.jboss.logging.Logger;
import io.github.hectorvent.floci.core.storage.StorageFactory;
@@ -46,6 +52,7 @@ public class EmulatorInfoController {
private final FlociCertificateAuthority certificateAuthority;
private final EmulatorConfig config;
private final Ec2Service ec2Service;
private final LightsailService lightsailService;
@Inject
public EmulatorInfoController(ServiceRegistry serviceRegistry,
@@ -55,7 +62,7 @@ public class EmulatorInfoController {
Instance<ContainerTeardown> containerTeardowns,
FlociCertificateAuthority certificateAuthority,
EmulatorConfig config,
Ec2Service ec2Service) {
Ec2Service ec2Service, LightsailService lightsailService) {
this.serviceRegistry = serviceRegistry;
this.initLifecycleState = initLifecycleState;
this.storageFactory = storageFactory;
@@ -64,6 +71,7 @@ public class EmulatorInfoController {
this.certificateAuthority = certificateAuthority;
this.config = config;
this.ec2Service = ec2Service;
this.lightsailService = lightsailService;
this.version = resolveVersion();
}
@@ -111,6 +119,40 @@ public class EmulatorInfoController {
return Response.ok(Map.of("ec2", Map.of("instances", ec2Service.diagnosticInstances()))).build();
}
@GET
@Path("/diagnostics/lightsail/instances/{instanceName}/host-ports")
public Response lightsailHostPorts(@PathParam("instanceName") String instanceName) {
return Response.ok(Map.of("mappings", lightsailService.hostPortMappings(config.defaultRegion(), instanceName))).build();
}
@POST
@Path("/diagnostics/lightsail/instances/{instanceName}/static-files")
@Consumes(MediaType.MULTIPART_FORM_DATA)
public Response deployLightsailStaticFiles(@PathParam("instanceName") String instanceName,
@RestForm("files") List<FileUpload> files) {
List<LightsailContainerManager.StaticFile> staticFiles = files == null ? List.of() : files.stream()
.map(file -> new LightsailContainerManager.StaticFile(file.fileName(), file.uploadedFile()))
.toList();
int deployed = lightsailService.deployStaticFiles(config.defaultRegion(), instanceName, staticFiles);
return Response.ok(Map.of("status", "deployed", "files", deployed)).build();
}
@POST
@Path("/diagnostics/lightsail/instances/{instanceName}/node-application")
@Consumes(MediaType.MULTIPART_FORM_DATA)
public Response deployLightsailNodeApplication(@PathParam("instanceName") String instanceName,
@RestForm("files") List<FileUpload> files,
@RestForm("startCommand") String startCommand,
@RestForm("port") Integer port) {
List<LightsailContainerManager.StaticFile> applicationFiles = files == null ? List.of() : files.stream()
.map(file -> new LightsailContainerManager.StaticFile(file.fileName(), file.uploadedFile()))
.toList();
LightsailContainerManager.NodeDeployment deployment = lightsailService.deployNodeApplication(
config.defaultRegion(), instanceName, applicationFiles, startCommand, port == null ? 3000 : port);
return Response.ok(Map.of("status", "deployed", "files", deployment.files(), "port", deployment.port(),
"releaseId", deployment.releaseId())).build();
}
@GET
@Path("/config")
public Response config() {
@@ -53,11 +53,16 @@ public class AmiImageResolver {
}
return new ResolvedAmiImage(
image.dockerImage,
image.guestRuntime == null || image.guestRuntime.isBlank()
image.initType == null || image.initType.isBlank()
? ResolvedAmiImage.DEFAULT_RUNTIME
: image.guestRuntime,
: image.initType,
Boolean.TRUE.equals(image.cloudInit),
dockerPlatform(image.architecture));
dockerPlatform(image.architecture),
image.fallbackDockerImage,
image.fallbackDockerImage == null ? "public" : "enhanced",
image.sshUser == null || image.sshUser.isBlank() ? "root" : image.sshUser,
Boolean.TRUE.equals(image.sshPreinstalled),
image.sshServiceName == null || image.sshServiceName.isBlank() ? "sshd" : image.sshServiceName);
}
private static String dockerPlatform(String architecture) {
@@ -12,6 +12,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
import io.github.hectorvent.floci.core.common.docker.PortAllocator;
import io.github.hectorvent.floci.services.ec2.model.Instance;
import io.github.hectorvent.floci.services.ec2.model.InstanceLaunchEvent;
import io.github.hectorvent.floci.services.ec2.model.InstanceNetworkInterface;
import io.github.hectorvent.floci.services.ec2.model.InstanceState;
import io.github.hectorvent.floci.services.ec2.model.GroupIdentifier;
@@ -38,8 +39,13 @@ import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashSet;
@@ -60,6 +66,7 @@ import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
import java.util.function.Consumer;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -161,6 +168,8 @@ public class Ec2ContainerManager {
static int containerBridgeIpAttempts = 30;
static long containerBridgeIpPollMillis = 500;
static int sshBannerAttempts = 30;
static long sshBannerPollMillis = 1_000;
private final ContainerBuilder containerBuilder;
private final ContainerLifecycleManager lifecycleManager;
@@ -180,9 +189,14 @@ public class Ec2ContainerManager {
private final ExecutorService executor;
private final Duration userDataExecutionTimeout;
private final Set<ResultCallback<Frame>> activeUserDataCallbacks = ConcurrentHashMap.newKeySet();
private volatile Consumer<Instance> instancePersister = ignored -> {};
private volatile boolean dockerUnavailableLogged;
public void setInstancePersister(Consumer<Instance> instancePersister) {
this.instancePersister = instancePersister == null ? ignored -> {} : instancePersister;
}
public void refreshSecurityGroups(String region, Map<String, SecurityGroup> groups,
Map<String, List<String>> prefixLists) {
if (firewallManager != null && firewallManager.enabled()) {
@@ -240,7 +254,7 @@ public class Ec2ContainerManager {
InstanceNetworkInterface eni = instance.getNetworkInterfaces().getFirst();
String logical = instance.getLogicalPrivateIpAddress() != null
? instance.getLogicalPrivateIpAddress() : eni.getPrivateIpAddress();
firewallManager.register(new SecurityGroupNftCompiler.Endpoint(regionResolver.getAccountId(), region,
firewallManager.register(new SecurityGroupNftCompiler.Endpoint(accountId(instance), region,
instance.getVpcId(), eni.getNetworkInterfaceId(), logical, address,
instance.getSecurityGroups().stream().map(GroupIdentifier::getGroupId)
.collect(Collectors.toSet()), groups), helperId, prefixLists);
@@ -375,6 +389,9 @@ public class Ec2ContainerManager {
Set<Integer> appPorts, List<SecurityGroup> groups,
Map<String, List<String>> prefixLists) {
instance.setState(InstanceState.pending());
instance.setLaunchStatus("in-progress");
instance.setSshStatus("initializing");
persist(instance);
// An instance record is metadata: id, addresses, tags and lifecycle state are all
// served without a container runtime. Only the guest itself (SSH, UserData, SSM
@@ -398,11 +415,13 @@ public class Ec2ContainerManager {
try {
executor.execute(() -> {
try {
recordLaunchEvent(instance, "selecting-image", "started", "Selecting the launch image");
ResolvedAmiImage launchImage = selectLaunchImage(instance, image);
String instanceId = instance.getInstanceId();
// IMDS endpoint that this container should use
String flociHost = dockerHostResolver.resolve();
int imdsPort = config.services().ec2().imdsPort();
StartedContainer started = createAndStartContainer(instance, image, region, flociHost, imdsPort,
StartedContainer started = createAndStartContainer(instance, launchImage, region, flociHost, imdsPort,
leasedPrivateIp, groups, prefixLists);
if (started == null) {
return;
@@ -485,6 +504,7 @@ public class Ec2ContainerManager {
failLaunch(instance, leasedPrivateIp);
return;
}
persist(instance);
// Set public-facing addresses only for instances whose subnet
// opts in via MapPublicIpOnLaunch (#1984). Private-subnet
@@ -508,14 +528,48 @@ public class Ec2ContainerManager {
// Inject SSH public key, if one was provided
if (publicKey != null && !publicKey.isBlank()) {
injectSshKey(containerId, publicKey);
recordLaunchEvent(instance, "injecting-key", "started", "Injecting the selected public key");
OperationResult injection = injectSshKey(containerId, publicKey);
instance.setKeyInjectionStatus(injection.success() ? "succeeded" : "failed");
recordLaunchEvent(instance, "injecting-key", injection.success() ? "completed" : "failed",
injection.message());
} else {
instance.setKeyInjectionStatus("skipped");
recordLaunchEvent(instance, "injecting-key", "skipped", "No key pair was selected");
}
// Systemd-backed AMIs start their enabled SSH unit during boot. Starting a second
// daemon here races it for port 22; lightweight guests still need this bootstrap.
if (!image.systemd()) {
startSshd(containerId, instanceId);
if (!launchImage.systemd()) {
recordLaunchEvent(instance, "installing-ssh", "started", "Checking and installing SSH packages");
SshSetupResult setup = startSshd(containerId, instanceId);
String installationStatus = setup.installationAttempted()
? setup.installationSucceeded() ? "completed" : "failed"
: "skipped";
recordLaunchEvent(instance, "installing-ssh", installationStatus,
setup.installationMessage());
recordLaunchEvent(instance, "starting-ssh", setup.success() ? "completed" : "failed",
setup.message());
} else {
recordLaunchEvent(instance, "installing-ssh", "skipped",
"SSH is preinstalled in the selected image");
recordLaunchEvent(instance, "starting-ssh", "skipped",
"SSH startup is managed by " + launchImage.sshServiceName());
}
recordLaunchEvent(instance, "ssh-ready", "started", "Waiting for an SSH protocol banner");
OperationResult readiness = waitForSshBanner(instance.getContainerBridgeIp(), 22);
String readinessMessage = setupFailureMessage(instance, readiness.message());
instance.setSshStatus(readiness.success() ? "ok" : "impaired");
instance.setSshStatusMessage(readinessMessage);
instance.setLaunchStatus(readiness.success() ? "ready" : "failed");
if (!readiness.success()) {
instance.setStateReasonCode("Client.SSHUnavailable");
instance.setStateReasonMessage(readinessMessage);
instance.setStateTransitionReason(readinessMessage);
}
recordLaunchEvent(instance, "ssh-ready", readiness.success() ? "completed" : "failed",
readinessMessage);
// Execute UserData
String userData = instance.getUserData();
if (userData != null && !userData.isBlank()) {
@@ -524,14 +578,14 @@ public class Ec2ContainerManager {
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
failLaunch(instance, leasedPrivateIp);
failLaunch(instance, leasedPrivateIp, "Launch interrupted");
} catch (Exception e) {
LOG.warnv("Failed to launch EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());
// The daemon can disappear between the probe above and any of the calls in
// this block. Losing Docker is not the instance's fault, so degrade to a
// metadata-only instance; a genuine container failure still fails the launch.
if ((firewallManager != null && firewallManager.enabled()) || isDockerAvailable()) {
failLaunch(instance, leasedPrivateIp);
failLaunch(instance, leasedPrivateIp, errorMessage(e));
} else {
markContainerlessRunning(instance);
}
@@ -540,10 +594,51 @@ public class Ec2ContainerManager {
} catch (RejectedExecutionException e) {
LOG.warnv("Could not schedule EC2 instance {0} launch because the launch executor is saturated or stopping",
instance.getInstanceId());
failLaunch(instance, leasedPrivateIp);
failLaunch(instance, leasedPrivateIp,
"Launch executor is saturated or stopping: " + errorMessage(e));
}
}
ResolvedAmiImage selectLaunchImage(ResolvedAmiImage image) {
return selectLaunchImage(null, image);
}
private ResolvedAmiImage selectLaunchImage(Instance instance, ResolvedAmiImage image) {
if (image.fallbackDockerImage() == null || image.fallbackDockerImage().isBlank()) {
recordSelectedImage(instance, image, "Selected " + image.imageType() + " image");
return image;
}
String preferred = containerBuilder.resolveImage(image.dockerImage());
try {
dockerClient.inspectImageCmd(preferred).exec();
recordSelectedImage(instance, image, "Enhanced AMI image is available locally");
return image;
} catch (NotFoundException expected) {
LOG.warnv("EC2 AMI image {0} is not present locally; using pullable fallback {1}",
image.dockerImage(), image.fallbackDockerImage());
ResolvedAmiImage fallback = new ResolvedAmiImage(image.fallbackDockerImage(),
ResolvedAmiImage.DEFAULT_RUNTIME, false, image.dockerPlatform(), null,
"public-fallback", image.sshUser(), false, image.sshServiceName());
recordSelectedImage(instance, fallback,
"Enhanced image " + image.dockerImage() + " was not present locally; selected public fallback");
return fallback;
}
}
private void recordSelectedImage(Instance instance, ResolvedAmiImage image, String reason) {
if (instance == null) {
return;
}
instance.setSelectedImageType(image.imageType());
instance.setSelectedImageReference(image.dockerImage());
instance.setSshUser(image.sshUser());
instance.setSshPreinstalled(image.sshPreinstalled());
instance.setInitType(image.initType());
instance.setSshServiceName(image.sshServiceName());
recordLaunchEvent(instance, "selecting-image", "completed",
reason + ": " + image.dockerImage());
}
private StartedContainer createAndStartContainer(Instance instance, ResolvedAmiImage image, String region,
String flociHost, int imdsPort, String leasedPrivateIp,
List<SecurityGroup> groups,
@@ -557,9 +652,20 @@ public class Ec2ContainerManager {
if (isLaunchCancelled(instance)) {
return null;
}
int sshHostPort = portAllocator.allocate(
config.services().ec2().sshPortRangeStart(),
config.services().ec2().sshPortRangeEnd());
recordLaunchEvent(instance, "allocating-ssh-port", "started",
"Allocating a host port for guest port 22");
List<Integer> occupiedPorts = new ArrayList<>();
int sshHostPort;
try {
sshHostPort = portAllocator.allocate(
config.services().ec2().sshPortRangeStart(),
config.services().ec2().sshPortRangeEnd(),
occupiedPorts::add);
} catch (RuntimeException e) {
recordOccupiedPorts(instance, occupiedPorts);
throw e;
}
recordOccupiedPorts(instance, occupiedPorts);
SecurityGroupFirewallManager.Namespace namespace = null;
String eniId = null;
String containerId = null;
@@ -573,8 +679,8 @@ public class Ec2ContainerManager {
eniId = eni.getNetworkInterfaceId();
instance.setLogicalPrivateIpAddress(eni.getPrivateIpAddress());
namespace = firewallManager.createNamespace("ec2", instanceId,
regionResolver.getAccountId(), region, Optional.empty(), Map.of(22, sshHostPort));
firewallManager.register(new SecurityGroupNftCompiler.Endpoint(regionResolver.getAccountId(),
accountId(instance), region, Optional.empty(), Map.of(22, sshHostPort));
firewallManager.register(new SecurityGroupNftCompiler.Endpoint(accountId(instance),
region, instance.getVpcId(), eniId, eni.getPrivateIpAddress(),
namespace.transportAddress(),
instance.getSecurityGroups().stream().map(g -> g.getGroupId())
@@ -582,10 +688,34 @@ public class Ec2ContainerManager {
namespace.helperId(), prefixLists);
}
ContainerSpec spec = buildContainerSpec(containerName, image, region, serviceEndpoint, imdsEndpoint,
instanceId, sshHostPort, namespace, instance.getIamInstanceProfileArn() != null);
containerId = image.dockerPlatform() == null
? lifecycleManager.create(spec)
: lifecycleManager.create(spec, image.dockerPlatform());
instanceId, accountId(instance), sshHostPort, namespace,
instance.getIamInstanceProfileArn() != null);
recordLaunchEvent(instance, "allocating-ssh-port", "completed",
"Allocated SSH host port " + sshHostPort);
containerId = lifecycleManager.create(spec, image.dockerPlatform(),
new ContainerLifecycleManager.ContainerCreationObserver() {
@Override
public void imagePullStarted(String imageUri) {
recordLaunchEvent(instance, "pulling-image", "started", "Pulling " + imageUri);
}
@Override
public void imagePullCompleted(String imageUri) {
recordLaunchEvent(instance, "pulling-image", "completed", "Pulled " + imageUri);
}
@Override
public void containerCreationStarted() {
recordLaunchEvent(instance, "creating-container", "started",
"Creating the instance container");
}
@Override
public void containerCreated(String createdContainerId) {
recordLaunchEvent(instance, "creating-container", "completed",
"Created container " + createdContainerId);
}
});
if (!recordCreatedContainer(instance, containerId, sshHostPort)) {
lifecycleManager.removeIfExists(containerId);
if (namespace != null) {
@@ -595,15 +725,20 @@ public class Ec2ContainerManager {
portAllocator.release(sshHostPort);
return null;
}
persist(instance);
recorded = true;
// Join the VPC's Docker network at the address the subnet allocated, before the
// container starts, so the guest comes up already holding its private IP. The
// default bridge attachment stays: it is what carries the published SSH host
// port, which a network mode set at creation time would suppress.
recordLaunchEvent(instance, "configuring-network", "started",
"Attaching the instance container to its VPC network");
String vpcAddress = namespace == null
? vpcNetworkManager.attach(region, instance.getVpcId(), instance.getSubnetId(),
containerId, leasedPrivateIp).map(network -> leasedPrivateIp).orElse(null)
: null;
recordLaunchEvent(instance, "configuring-network", "completed",
vpcAddress == null ? "Using the container network namespace" : "Attached at " + vpcAddress);
lifecycleManager.startCreated(containerId, spec);
return new StartedContainer(containerId, sshHostPort, vpcAddress, namespace, eniId);
} catch (Exception e) {
@@ -626,6 +761,8 @@ public class Ec2ContainerManager {
portAllocator.markReserved(sshHostPort);
LOG.warnv("EC2 instance {0} could not use SSH host port {1}; trying another port",
instanceId, String.valueOf(sshHostPort));
recordLaunchEvent(instance, "allocating-ssh-port", "info",
"SSH host port " + sshHostPort + " was occupied; selecting another port");
continue;
}
portAllocator.release(sshHostPort);
@@ -636,6 +773,7 @@ public class Ec2ContainerManager {
private ContainerSpec buildContainerSpec(String containerName, ResolvedAmiImage image, String region,
String serviceEndpoint, String imdsEndpoint, String instanceId,
String accountId,
int sshHostPort, SecurityGroupFirewallManager.Namespace namespace,
boolean hasInstanceProfile) {
// Minimal images keep the historic tail command, while cloud-image AMI guests can boot their init.
@@ -649,7 +787,7 @@ public class Ec2ContainerManager {
.withHostDockerInternalOnLinux()
.withLogRotation()
.withLabels(ContainerStorageHelper.resourceIdentityLabels(
"ec2", instanceId, regionResolver.getAccountId(), region))
"ec2", instanceId, accountId, region))
// Which Floci owns this container, so the startup reconciler cannot reap a
// sibling emulator's live instances off a shared daemon. See LABEL_OWNER_PORT.
.withLabels(Map.of(LABEL_OWNER_PORT, ownerIdentity()))
@@ -674,7 +812,7 @@ public class Ec2ContainerManager {
}
private void failLaunch(Instance instance) {
failLaunch(instance, instance.getPrivateIpAddress());
failLaunch(instance, instance.getPrivateIpAddress(), "EC2 container launch failed");
}
/**
@@ -687,6 +825,10 @@ public class Ec2ContainerManager {
* reserved while the endpoint stands, and would refuse the next launch handed the same one.
*/
private void failLaunch(Instance instance, String leasedPrivateIp) {
failLaunch(instance, leasedPrivateIp, "EC2 container launch failed");
}
private void failLaunch(Instance instance, String leasedPrivateIp, String reason) {
String containerId;
int sshHostPort;
String containerIp;
@@ -697,6 +839,13 @@ public class Ec2ContainerManager {
&& instance.getSshHostPort() <= 0
&& (instance.getContainerBridgeIp() == null || instance.getContainerBridgeIp().isBlank());
instance.setState(InstanceState.terminated());
instance.setStateReasonCode("Server.InternalError");
instance.setStateReasonMessage(reason);
instance.setStateTransitionReason(reason);
instance.setTerminatedAt(System.currentTimeMillis());
instance.setLaunchStatus("failed");
instance.setSshStatus("impaired");
instance.setSshStatusMessage(reason);
containerId = instance.getDockerContainerId();
sshHostPort = instance.getSshHostPort();
containerIp = instance.getContainerBridgeIp();
@@ -704,6 +853,7 @@ public class Ec2ContainerManager {
instance.setSshHostPort(0);
instance.setContainerBridgeIp(null);
}
recordLaunchEvent(instance, currentLaunchStage(instance), "failed", reason);
if (alreadyCleaned) {
return;
}
@@ -742,6 +892,7 @@ public class Ec2ContainerManager {
LOG.warnv("Error unregistering failed EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());
}
}
persist(instance);
}
/**
@@ -856,7 +1007,27 @@ public class Ec2ContainerManager {
private void markContainerlessRunning(Instance instance) {
LOG.infov("EC2 instance {0} is running without a backing container (no Docker daemon reachable)",
instance.getInstanceId());
String reason = "Docker daemon is unavailable; the instance has no backing container";
instance.setState(InstanceState.running());
instance.setLaunchStatus("failed");
instance.setSshStatus("impaired");
instance.setSshStatusMessage(reason);
instance.setStateReasonCode("Server.InternalError");
instance.setStateReasonMessage(reason);
instance.setStateTransitionReason(reason);
recordLaunchEvent(instance, "creating-container", "failed", reason);
}
private String accountId(Instance instance) {
return instance.getAccountId() == null || instance.getAccountId().isBlank()
? regionResolver.getAccountId() : instance.getAccountId();
}
private void recordOccupiedPorts(Instance instance, List<Integer> occupiedPorts) {
for (Integer occupiedPort : occupiedPorts) {
recordLaunchEvent(instance, "allocating-ssh-port", "retrying",
"SSH host port " + occupiedPort + " is occupied; trying the next port");
}
}
/**
@@ -1296,6 +1467,13 @@ public class Ec2ContainerManager {
executor.submit(() -> {
try {
dockerClient.restartContainerCmd(containerId).exec();
InspectContainerResponse container = dockerClient.inspectContainerCmd(containerId).exec();
if (isLegacyTailGuest(container.getConfig().getCmd())) {
// Instances launched before systemd-backed AMIs used tail as PID 1. Their image
// catalog entry now identifies a systemd runtime, but Docker restart retains
// the original command and therefore does not start the SSH service.
startSshd(containerId, instance.getInstanceId());
}
LOG.infov("Rebooted EC2 container {0}", containerId);
} catch (Exception e) {
LOG.warnv("Error rebooting EC2 container {0}: {1}", containerId, e.getMessage());
@@ -1303,6 +1481,11 @@ public class Ec2ContainerManager {
});
}
static boolean isLegacyTailGuest(String[] command) {
return command != null && command.length == 3
&& "tail".equals(command[0]) && "-f".equals(command[1]) && "/dev/null".equals(command[2]);
}
public boolean isContainerRunning(Instance instance) {
String containerId = instance.getDockerContainerId();
return containerId != null && lifecycleManager.isContainerRunning(containerId);
@@ -1378,7 +1561,7 @@ public class Ec2ContainerManager {
}
}
private void injectSshKey(String containerId, String publicKey) {
private OperationResult injectSshKey(String containerId, String publicKey) {
try {
// Ensure .ssh directory exists with correct permissions
execInContainer(containerId, new String[]{"sh", "-c",
@@ -1394,32 +1577,39 @@ public class Ec2ContainerManager {
execInContainer(containerId, new String[]{"chmod", "600", "/root/.ssh/authorized_keys"}, 5);
LOG.infov("Injected SSH public key into container {0}", containerId);
return new OperationResult(true, "Public key injected into /root/.ssh/authorized_keys");
} catch (Exception e) {
LOG.warnv("Could not inject SSH key into container {0}: {1}", containerId, e.getMessage());
return new OperationResult(false, errorMessage(e));
}
}
private void startSshd(String containerId, String instanceId) {
private SshSetupResult startSshd(String containerId, String instanceId) {
try {
// Exit 1 means sshd is absent and there is nothing to start; exit 2 means sshd is
// there but the client package did not land. Only the first is fatal - see
// sshdInstallProbeCommand() for why the probe distinguishes them.
ContainerExecResult install = execInContainerForResult(containerId, sshdInstallProbeCommand(), 120);
boolean installationAttempted = install.output().contains("FLOCI_SSH_INSTALL=1");
if (install.exitCode() == SSH_CLIENT_MISSING_EXIT_CODE) {
LOG.warnv("sshd is available on EC2 instance {0} but the OpenSSH client package is not:"
+ " scp is missing, so provisioners that upload files over scp will fail: {1}",
instanceId, install.summary());
} else if (install.exitCode() != 0) {
String message = "Could not install openssh-server: " + install.summary();
LOG.warnv("Could not install openssh-server for EC2 instance {0}: {1}",
instanceId, install.summary());
return;
return new SshSetupResult(false, installationAttempted, false,
installationAttempted ? "SSH package installation failed" : "SSH package installation skipped",
message);
}
// Generate host keys
ContainerExecResult keygen = execInContainerForResult(containerId, new String[]{"ssh-keygen", "-A"}, 10);
if (keygen.exitCode() != 0) {
LOG.warnv("Could not generate SSH host keys for EC2 instance {0}: {1}",
instanceId, keygen.summary());
return;
return new SshSetupResult(false, installationAttempted, true, "SSH packages are available",
"Could not generate SSH host keys: " + keygen.summary());
}
// Modern OpenSSH refuses to start without its privilege-separation directory, and /run
// is a fresh tmpfs in most container images, so /run/sshd genuinely isn't there yet.
@@ -1428,7 +1618,8 @@ public class Ec2ContainerManager {
if (mkdir.exitCode() != 0) {
LOG.warnv("Could not create /run/sshd for EC2 instance {0}: {1}",
instanceId, mkdir.summary());
return;
return new SshSetupResult(false, installationAttempted, true, "SSH packages are available",
"Could not create /run/sshd: " + mkdir.summary());
}
// Start sshd without -D so it daemonizes itself and survives this exec session. Since sshd
// requires execution with an absolute path, several paths are tried until it starts
@@ -1439,13 +1630,108 @@ public class Ec2ContainerManager {
continue;
}
LOG.infov("Started sshd in EC2 instance {0}", instanceId);
return;
return new SshSetupResult(true, installationAttempted, true,
installationAttempted ? "SSH packages installed" : "SSH was already installed",
"Started sshd using " + sshdPath);
}
return new SshSetupResult(false, installationAttempted, true, "SSH packages are available",
"No supported sshd executable started successfully");
} catch (Exception e) {
LOG.warnv("Could not start sshd in EC2 instance {0}: {1}", instanceId, e.getMessage());
return new SshSetupResult(false, false, false, "SSH package state is unknown", errorMessage(e));
}
}
private OperationResult waitForSshBanner(String host, int port) throws InterruptedException {
if (host == null || host.isBlank()) {
return new OperationResult(false, "No container address is available for the SSH readiness check");
}
String lastError = "SSH did not return a protocol banner";
for (int attempt = 0; attempt < sshBannerAttempts; attempt++) {
OperationResult probe = probeSshBanner(host, port);
if (probe.success()) {
return probe;
}
lastError = probe.message();
if (attempt + 1 < sshBannerAttempts) {
Thread.sleep(sshBannerPollMillis);
}
}
return new OperationResult(false, lastError);
}
private OperationResult probeSshBanner(String host, int port) {
try (Socket socket = new Socket()) {
socket.connect(new InetSocketAddress(host, port), 1_000);
socket.setSoTimeout(1_000);
BufferedReader reader = new BufferedReader(
new InputStreamReader(socket.getInputStream(), StandardCharsets.US_ASCII));
String banner = reader.readLine();
if (banner != null && banner.startsWith("SSH-")) {
return new OperationResult(true, "SSH banner received: " + banner);
}
return new OperationResult(false, banner == null
? "SSH connection closed without a protocol banner"
: "Endpoint returned a non-SSH banner: " + banner);
} catch (IOException e) {
return new OperationResult(false, errorMessage(e));
}
}
public OperationResult checkSshReadiness(Instance instance) {
return probeSshBanner(instance.getContainerBridgeIp(), 22);
}
private void recordLaunchEvent(Instance instance, String stage, String status, String message) {
if (instance == null) {
return;
}
synchronized (instance) {
instance.getLaunchEvents().add(new InstanceLaunchEvent(stage, status, Instant.now(), message));
}
persist(instance);
}
private String setupFailureMessage(Instance instance, String readinessMessage) {
synchronized (instance) {
List<InstanceLaunchEvent> events = instance.getLaunchEvents();
for (int i = events.size() - 1; i >= 0; i--) {
InstanceLaunchEvent event = events.get(i);
if ("starting-ssh".equals(event.getStage()) && "failed".equals(event.getStatus())) {
return event.getMessage() + "; readiness check: " + readinessMessage;
}
}
}
return readinessMessage;
}
private String currentLaunchStage(Instance instance) {
synchronized (instance) {
List<InstanceLaunchEvent> events = instance.getLaunchEvents();
if (!events.isEmpty()) {
return events.getLast().getStage();
}
}
return "launch";
}
private void persist(Instance instance) {
instancePersister.accept(instance);
}
private static String errorMessage(Throwable error) {
if (error == null) {
return "Unknown launch failure";
}
String message = error.getMessage();
return message == null || message.isBlank() ? error.getClass().getSimpleName() : message;
}
public record OperationResult(boolean success, String message) {}
private record SshSetupResult(boolean success, boolean installationAttempted, boolean installationSucceeded,
String installationMessage, String message) {}
private void executeUserData(String containerId, String instanceId, String userData, String region) {
try {
String logGroup = "/aws/ec2/" + instanceId;
@@ -1782,7 +2068,9 @@ public class Ec2ContainerManager {
*/
static String[] sshdInstallProbeCommand() {
return new String[]{"sh", "-c",
"if ! command -v sshd >/dev/null 2>&1 || ! command -v scp >/dev/null 2>&1; then" +
"installed=0;" +
"if ! command -v sshd >/dev/null 2>&1 || ! command -v scp >/dev/null 2>&1; then" +
" installed=1;" +
" if command -v dnf >/dev/null 2>&1; then dnf install -y openssh-server openssh-clients >/dev/null 2>&1;" +
// yum is probed after dnf, and the order is load-bearing: Amazon Linux
// 2023 and modern Fedora/RHEL ship both, and there dnf is the supported
@@ -1796,6 +2084,7 @@ public class Ec2ContainerManager {
" elif command -v apk >/dev/null 2>&1; then apk add --no-cache openssh >/dev/null 2>&1;" +
" fi;" +
"fi;" +
"printf 'FLOCI_SSH_INSTALL=%s\\n' \"$installed\";" +
"command -v sshd >/dev/null 2>&1 || exit 1;" +
"command -v scp >/dev/null 2>&1 || exit 2"};
}
@@ -168,6 +168,7 @@ public class Ec2ImageCatalog {
public List<String> aliases = List.of();
public List<String> publicParameterNames = List.of();
public String dockerImage;
public String fallbackDockerImage;
public String name;
public String description;
public String state;
@@ -182,8 +183,11 @@ public class Ec2ImageCatalog {
public String imageOwnerAlias;
public String creationDate;
public Boolean advertised;
public String guestRuntime;
public String initType;
public Boolean cloudInit;
public String sshUser;
public Boolean sshPreinstalled;
public String sshServiceName;
public String provenance;
public boolean advertised() {
@@ -1294,12 +1294,20 @@ public class Ec2QueryHandler {
private Response handleDescribeInstanceStatus(MultivaluedMap<String, String> p, String region) {
List<String> ids = getList(p, "InstanceId");
List<Instance> runningInstances = service.describeInstanceStatus(region, ids);
boolean includeAllInstances = Boolean.TRUE.equals(
parseOptionalBoolean(p.getFirst("IncludeAllInstances"), "IncludeAllInstances"));
List<Instance> instances = service.describeInstanceStatus(region, ids, includeAllInstances);
XmlBuilder xml = new XmlBuilder()
.start("DescribeInstanceStatusResponse", AwsNamespaces.EC2)
.elem("requestId", UUID.randomUUID().toString())
.start("instanceStatusSet");
for (Instance inst : runningInstances) {
for (Instance inst : instances) {
String reachability = inst.getSshStatus() == null ? "initializing" : inst.getSshStatus();
String detail = switch (reachability) {
case "ok" -> "passed";
case "impaired" -> "failed";
default -> "initializing";
};
xml.start("item")
.elem("instanceId", inst.getInstanceId())
.elem("availabilityZone", inst.getPlacement() != null ? inst.getPlacement().getAvailabilityZone() : "")
@@ -1314,9 +1322,9 @@ public class Ec2QueryHandler {
.end("item").end("details")
.end("systemStatus")
.start("instanceStatus")
.elem("status", "ok")
.elem("status", reachability)
.start("details").start("item")
.elem("name", "reachability").elem("status", "passed")
.elem("name", "reachability").elem("status", detail)
.end("item").end("details")
.end("instanceStatus")
.end("item");
@@ -61,6 +61,7 @@ import io.github.hectorvent.floci.services.ec2.model.Image;
import io.github.hectorvent.floci.services.ec2.model.Instance;
import io.github.hectorvent.floci.services.ec2.model.InstanceCreditSpecification;
import io.github.hectorvent.floci.services.ec2.model.InstanceCreditSpecificationListResult;
import io.github.hectorvent.floci.services.ec2.model.InstanceLaunchEvent;
import io.github.hectorvent.floci.services.ec2.model.InstanceNetworkInterface;
import io.github.hectorvent.floci.services.ec2.model.InstanceState;
import io.github.hectorvent.floci.services.ec2.model.NetworkInterface;
@@ -507,13 +508,11 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
@PostConstruct
void restoreMetadataRegistrations() {
containerManager.setInstancePersister(this::persistInstance);
if (portForwardManager != null) {
portForwardManager.setPersister(inst -> {
if (inst != null && inst.getRegion() != null && inst.getInstanceId() != null) {
instances.put(key(inst.getRegion(), inst.getInstanceId()), inst);
}
});
portForwardManager.setPersister(this::persistInstance);
}
clearPersistedPrivateKeyMaterial();
if (config.services().ec2().mock()) {
return;
}
@@ -522,8 +521,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
recoverInterruptedTerminations();
int restored = 0;
for (String key : instances.keys()) {
Instance instance = instances.get(key).orElse(null);
for (AccountAwareStorageBackend.AccountEntry<Instance> entry : allInstanceEntries()) {
Instance instance = entry.value();
if (!needsMetadataRegistration(instance)) {
continue;
}
@@ -533,11 +532,26 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
LOG.warnv("Could not restore EC2 firewall for {0}: {1}", instance.getInstanceId(), e.getMessage());
containerManager.stopForShutdown(instance);
instance.setState(InstanceState.stopped());
instances.put(key, instance);
persistOwnedInstance(entry, instance);
continue;
}
if (containerManager.restoreMetadataRegistration(instance)) {
instances.put(key, instance);
if ("pending".equals(instance.getState().getName())
|| "in-progress".equals(instance.getLaunchStatus())) {
instance.setState(InstanceState.running());
instance.setLaunchStatus("unknown");
instance.getLaunchEvents().add(new InstanceLaunchEvent("startup-reconciliation", "unknown",
Instant.now(), "Core restarted before launch completion was recorded"));
Ec2ContainerManager.OperationResult readiness = containerManager.checkSshReadiness(instance);
if (readiness == null) {
instance.setSshStatus("unknown");
instance.setSshStatusMessage("SSH readiness could not be reconstructed after restart");
} else {
instance.setSshStatus(readiness.success() ? "ok" : "impaired");
instance.setSshStatusMessage(readiness.message());
}
}
persistOwnedInstance(entry, instance);
restored++;
// Container is running: re-reserve host ports and recreate any missing socat sidecars.
if (portForwardManager != null) {
@@ -545,6 +559,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
}
}
}
failUnrecoverablePendingLaunches();
if (restored > 0) {
LOG.infov("Restored IMDS metadata registration for {0} EC2 container(s)", restored);
}
@@ -554,6 +569,68 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
containerManager.reconcileOrphanedContainers(this::instanceContainerStillWanted);
}
private void failUnrecoverablePendingLaunches() {
for (AccountAwareStorageBackend.AccountEntry<Instance> entry : allInstanceEntries()) {
Instance instance = entry.value();
if (instance == null || instance.getState() == null
|| (!"pending".equals(instance.getState().getName())
&& !"in-progress".equals(instance.getLaunchStatus()))) {
continue;
}
String reason = "Launch state is unknown because core restarted before a backing container was recorded";
instance.setState(InstanceState.terminated());
instance.setLaunchStatus("failed");
instance.setSshStatus("impaired");
instance.setSshStatusMessage(reason);
instance.setStateReasonCode("Server.InternalError");
instance.setStateReasonMessage(reason);
instance.setStateTransitionReason(reason);
instance.setTerminatedAt(System.currentTimeMillis());
instance.getLaunchEvents().add(new InstanceLaunchEvent("startup-reconciliation", "failed",
Instant.now(), reason));
persistOwnedInstance(entry, instance);
}
}
private List<AccountAwareStorageBackend.AccountEntry<Instance>> allInstanceEntries() {
if (instances instanceof AccountAwareStorageBackend<?> rawAccountAware) {
@SuppressWarnings("unchecked")
AccountAwareStorageBackend<Instance> accountAware =
(AccountAwareStorageBackend<Instance>) rawAccountAware;
return accountAware.scanAllAccountEntries(key -> true);
}
return instances.keys().stream()
.map(storageKey -> instances.get(storageKey)
.map(instance -> new AccountAwareStorageBackend.AccountEntry<Instance>(
null, storageKey, instance)))
.flatMap(Optional::stream)
.toList();
}
private void persistOwnedInstance(AccountAwareStorageBackend.AccountEntry<Instance> entry, Instance instance) {
if (entry.accountId() != null && instances instanceof AccountAwareStorageBackend<?> rawAccountAware) {
@SuppressWarnings("unchecked")
AccountAwareStorageBackend<Instance> accountAware =
(AccountAwareStorageBackend<Instance>) rawAccountAware;
accountAware.putForAccount(entry.accountId(), entry.key(), instance);
return;
}
instances.put(entry.key(), instance);
}
private void persistInstance(Instance instance) {
if (instance == null || instance.getRegion() == null || instance.getInstanceId() == null) {
return;
}
String storageKey = key(instance.getRegion(), instance.getInstanceId());
if (instances instanceof AccountAwareStorageBackend<Instance> accountAware
&& instance.getAccountId() != null && !instance.getAccountId().isBlank()) {
accountAware.putForAccount(instance.getAccountId(), storageKey, instance);
return;
}
instances.put(storageKey, instance);
}
/**
* Whether a container labelled for (region, instanceId) still backs a live instance record.
* False means the record is gone or already terminated, so the container is an orphan.
@@ -568,13 +645,13 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
}
private void recoverInterruptedTerminations() {
for (String storageKey : instances.keys()) {
Instance instance = instances.get(storageKey).orElse(null);
for (AccountAwareStorageBackend.AccountEntry<Instance> entry : allInstanceEntries()) {
Instance instance = entry.value();
if (instance == null || instance.getState() == null
|| !"shutting-down".equals(instance.getState().getName())) {
continue;
}
containerManager.terminate(instance, () -> instances.put(storageKey, instance));
containerManager.terminate(instance, () -> persistOwnedInstance(entry, instance));
}
}
@@ -617,8 +694,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
private void reserveRestoredPrivateIps() {
int reserved = 0;
int skipped = 0;
for (String storageKey : instances.keys()) {
Instance instance = instances.get(storageKey).orElse(null);
for (AccountAwareStorageBackend.AccountEntry<Instance> entry : allInstanceEntries()) {
Instance instance = entry.value();
if (instance == null || instance.getSubnetId() == null
|| instance.getPrivateIpAddress() == null
|| instance.getState() == null
@@ -2645,6 +2722,10 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
LaunchTemplateData.MetadataOptions launchMetadataOptions = LaunchTemplateData.MetadataOptions.merge(
LaunchTemplateData.MetadataOptions.launchDefaults(), metadataOptions);
ensureDefaultResources(region);
if (keyName != null && findKeyPair(region, keyName) == null) {
throw new AwsException("InvalidKeyPair.NotFound",
"The key pair '" + keyName + "' does not exist", 400);
}
// floci-kt9: a caller-supplied primary ENI (override-default-eni) governs its own
// subnet/VPC/security-groups and can only ever back a single instance.
@@ -2739,8 +2820,9 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
dockerImage = amiImageResolver.resolveImage(resolveLaunchableImageId(region, imageId));
String captured = capturedImageFor(region, imageId);
if (captured != null) {
dockerImage = new ResolvedAmiImage(captured, dockerImage.guestRuntime(),
dockerImage.cloudInit(), dockerImage.dockerPlatform());
dockerImage = new ResolvedAmiImage(captured, dockerImage.initType(),
dockerImage.cloudInit(), dockerImage.dockerPlatform(), null, "captured",
dockerImage.sshUser(), dockerImage.sshPreinstalled(), dockerImage.sshServiceName());
}
}
if (dryRun) {
@@ -2775,6 +2857,11 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
inst.setAmiLaunchIndex(i);
inst.setClientToken(clientToken);
inst.setRegion(region);
inst.setAccountId(callerAccountId());
inst.setLaunchStatus(config.services().ec2().mock() ? "ready" : "in-progress");
inst.setSshStatus(config.services().ec2().mock() ? "ok" : "initializing");
inst.getLaunchEvents().add(new InstanceLaunchEvent("launch-requested", "completed",
Instant.now(), "RunInstances accepted the launch"));
inst.setUserData(userData);
inst.setEncodedUserData(encodedUserData);
inst.setIamInstanceProfileArn(iamInstanceProfileArn);
@@ -2854,13 +2941,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
// Outside the lock: the containers are what the lock protects a reference to, not part of
// the registry, and a launch is slow.
if (!config.services().ec2().mock()) {
String publicKey = null;
if (keyName != null) {
KeyPair kp = findKeyPair(region, keyName);
if (kp != null) {
publicKey = kp.getPublicKey();
}
}
String publicKey = keyName == null ? null : findKeyPair(region, keyName).getPublicKey();
for (Instance inst : launched) {
List<SecurityGroup> policyGroups = inst.getSecurityGroups().stream()
.map(group -> securityGroups.get(key(region, group.getGroupId()))
@@ -3334,7 +3415,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
}
}
public List<Instance> describeInstanceStatus(String region, List<String> instanceIds) {
public List<Instance> describeInstanceStatus(String region, List<String> instanceIds,
boolean includeAllInstances) {
ensureDefaultResources(region);
if (config.services().ec2().mock()) {
instances.scan(k -> true).stream()
@@ -3345,11 +3427,41 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
instances.put(key(i.getRegion(), i.getInstanceId()), i);
});
}
return instances.scan(k -> true).stream()
List<Instance> matching = instances.scan(k -> true).stream()
.filter(i -> i.getRegion().equals(region))
.filter(i -> instanceIds.isEmpty() || instanceIds.contains(i.getInstanceId()))
.filter(i -> "running".equals(i.getState().getName()))
.filter(i -> includeAllInstances || "running".equals(i.getState().getName()))
.collect(Collectors.toList());
if (!config.services().ec2().mock()) {
for (Instance instance : matching) {
if (!"running".equals(instance.getState().getName())
|| "in-progress".equals(instance.getLaunchStatus())) {
continue;
}
Ec2ContainerManager.OperationResult result = containerManager.checkSshReadiness(instance);
String status = result.success() ? "ok" : "impaired";
String message = !result.success()
&& "Client.SSHUnavailable".equals(instance.getStateReasonCode())
&& instance.getStateReasonMessage() != null
? instance.getStateReasonMessage() : result.message();
if (!Objects.equals(status, instance.getSshStatus())
|| !Objects.equals(message, instance.getSshStatusMessage())) {
instance.setSshStatus(status);
instance.setSshStatusMessage(message);
if (result.success() && "failed".equals(instance.getLaunchStatus())
&& "Client.SSHUnavailable".equals(instance.getStateReasonCode())) {
instance.setLaunchStatus("ready");
instance.setStateReasonCode(null);
instance.setStateReasonMessage(null);
instance.setStateTransitionReason(null);
}
instance.getLaunchEvents().add(new InstanceLaunchEvent("ssh-status-check",
result.success() ? "completed" : "failed", Instant.now(), message));
persistInstance(instance);
}
}
}
return matching;
}
/**
@@ -4889,6 +5001,38 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
// ─── Key Pairs ─────────────────────────────────────────────────────────────
private void clearPersistedPrivateKeyMaterial() {
int cleared = 0;
if (keyPairs instanceof AccountAwareStorageBackend<?> rawAccountAware) {
@SuppressWarnings("unchecked")
AccountAwareStorageBackend<KeyPair> accountAware =
(AccountAwareStorageBackend<KeyPair>) rawAccountAware;
for (AccountAwareStorageBackend.AccountEntry<KeyPair> entry
: accountAware.scanAllAccountEntries(key -> true)) {
KeyPair keyPair = entry.value();
if (keyPair.getKeyMaterial() == null) {
continue;
}
keyPair.setKeyMaterial(null);
accountAware.putForAccount(entry.accountId(), entry.key(), keyPair);
cleared++;
}
} else {
for (String storageKey : keyPairs.keys()) {
KeyPair keyPair = keyPairs.get(storageKey).orElse(null);
if (keyPair == null || keyPair.getKeyMaterial() == null) {
continue;
}
keyPair.setKeyMaterial(null);
keyPairs.put(storageKey, keyPair);
cleared++;
}
}
if (cleared > 0) {
LOG.infov("Cleared private material from {0} persisted EC2 key pair record(s)", cleared);
}
}
public KeyPair createKeyPair(String region, String keyName) {
ensureDefaultResources(region);
requireKeyName(keyName);
@@ -4904,7 +5048,13 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
// Without storing it, a key pair created here could never authenticate anything.
kp.setPublicKey(generated.openSshPublicKey());
kp.setRegion(region);
keyPairs.put(key(region, keyPairId), kp);
KeyPair stored = new KeyPair();
stored.setKeyPairId(keyPairId);
stored.setKeyName(keyName);
stored.setKeyFingerprint(generated.fingerprint());
stored.setPublicKey(generated.openSshPublicKey());
stored.setRegion(region);
keyPairs.put(key(region, keyPairId), stored);
return kp;
}
@@ -5016,6 +5166,23 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
diagnostic.put("sshPort", instance.getSshHostPort());
diagnostic.put("containerId", instance.getDockerContainerId());
diagnostic.put("containerStatus", containerStatus);
diagnostic.put("state", instance.getState() == null ? null : instance.getState().getName());
diagnostic.put("launchStatus", instance.getLaunchStatus());
synchronized (instance) {
diagnostic.put("launchEvents", List.copyOf(instance.getLaunchEvents()));
}
diagnostic.put("selectedImageType", instance.getSelectedImageType());
diagnostic.put("selectedImageReference", instance.getSelectedImageReference());
diagnostic.put("sshUser", instance.getSshUser());
diagnostic.put("sshPreinstalled", instance.isSshPreinstalled());
diagnostic.put("initType", instance.getInitType());
diagnostic.put("sshServiceName", instance.getSshServiceName());
diagnostic.put("sshStatus", instance.getSshStatus());
diagnostic.put("sshStatusMessage", instance.getSshStatusMessage());
diagnostic.put("keyName", instance.getKeyName());
diagnostic.put("keyInjectionStatus", instance.getKeyInjectionStatus());
diagnostic.put("publishedPorts", instance.getPublishedPorts());
diagnostic.put("failure", instance.getStateReasonMessage());
return diagnostic;
}).toList();
}
@@ -1,15 +1,17 @@
package io.github.hectorvent.floci.services.ec2;
public record ResolvedAmiImage(String dockerImage, String guestRuntime, boolean cloudInit,
String dockerPlatform) {
public record ResolvedAmiImage(String dockerImage, String initType, boolean cloudInit,
String dockerPlatform, String fallbackDockerImage, String imageType,
String sshUser, boolean sshPreinstalled, String sshServiceName) {
public static final String DEFAULT_RUNTIME = "minimal";
public static final String SYSTEMD_RUNTIME = "systemd";
public static ResolvedAmiImage minimal(String dockerImage) {
return new ResolvedAmiImage(dockerImage, DEFAULT_RUNTIME, false, null);
return new ResolvedAmiImage(dockerImage, DEFAULT_RUNTIME, false, null, null,
"public", "root", false, "sshd");
}
public boolean systemd() {
return SYSTEMD_RUNTIME.equals(guestRuntime);
return SYSTEMD_RUNTIME.equals(initType);
}
}
@@ -48,8 +48,21 @@ public class Instance {
private String stateReasonCode;
private String stateReasonMessage;
private String region;
private String accountId;
private List<Tag> tags = new ArrayList<>();
private String launchStatus;
private List<InstanceLaunchEvent> launchEvents = new ArrayList<>();
private String selectedImageType;
private String selectedImageReference;
private String sshUser;
private boolean sshPreinstalled;
private String initType;
private String sshServiceName;
private String sshStatus;
private String sshStatusMessage;
private String keyInjectionStatus;
private String rootVolumeId;
// NOTE: disableApiStop and disableApiTermination are stored but ModifyInstanceAttribute
@@ -192,9 +205,50 @@ public class Instance {
public String getRegion() { return region; }
public void setRegion(String region) { this.region = region; }
public String getAccountId() { return accountId; }
public void setAccountId(String accountId) { this.accountId = accountId; }
public List<Tag> getTags() { return tags; }
public void setTags(List<Tag> tags) { this.tags = tags; }
public String getLaunchStatus() { return launchStatus; }
public void setLaunchStatus(String launchStatus) { this.launchStatus = launchStatus; }
public List<InstanceLaunchEvent> getLaunchEvents() {
if (launchEvents == null) {
launchEvents = new ArrayList<>();
}
return launchEvents;
}
public void setLaunchEvents(List<InstanceLaunchEvent> launchEvents) { this.launchEvents = launchEvents; }
public String getSelectedImageType() { return selectedImageType; }
public void setSelectedImageType(String selectedImageType) { this.selectedImageType = selectedImageType; }
public String getSelectedImageReference() { return selectedImageReference; }
public void setSelectedImageReference(String selectedImageReference) { this.selectedImageReference = selectedImageReference; }
public String getSshUser() { return sshUser; }
public void setSshUser(String sshUser) { this.sshUser = sshUser; }
public boolean isSshPreinstalled() { return sshPreinstalled; }
public void setSshPreinstalled(boolean sshPreinstalled) { this.sshPreinstalled = sshPreinstalled; }
public String getInitType() { return initType; }
public void setInitType(String initType) { this.initType = initType; }
public String getSshServiceName() { return sshServiceName; }
public void setSshServiceName(String sshServiceName) { this.sshServiceName = sshServiceName; }
public String getSshStatus() { return sshStatus; }
public void setSshStatus(String sshStatus) { this.sshStatus = sshStatus; }
public String getSshStatusMessage() { return sshStatusMessage; }
public void setSshStatusMessage(String sshStatusMessage) { this.sshStatusMessage = sshStatusMessage; }
public String getKeyInjectionStatus() { return keyInjectionStatus; }
public void setKeyInjectionStatus(String keyInjectionStatus) { this.keyInjectionStatus = keyInjectionStatus; }
public String getDockerContainerId() { return dockerContainerId; }
public void setDockerContainerId(String dockerContainerId) { this.dockerContainerId = dockerContainerId; }
@@ -0,0 +1,37 @@
package io.github.hectorvent.floci.services.ec2.model;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.time.Instant;
@RegisterForReflection
@JsonIgnoreProperties(ignoreUnknown = true)
public class InstanceLaunchEvent {
private String stage;
private String status;
private Instant timestamp;
private String message;
public InstanceLaunchEvent() {}
public InstanceLaunchEvent(String stage, String status, Instant timestamp, String message) {
this.stage = stage;
this.status = status;
this.timestamp = timestamp;
this.message = message;
}
public String getStage() { return stage; }
public void setStage(String stage) { this.stage = stage; }
public String getStatus() { return status; }
public void setStatus(String status) { this.status = status; }
public Instant getTimestamp() { return timestamp; }
public void setTimestamp(Instant timestamp) { this.timestamp = timestamp; }
public String getMessage() { return message; }
public void setMessage(String message) { this.message = message; }
}
@@ -148,10 +148,13 @@ public class ElbV2Service implements ResourceProvider {
}
}
for (Map.Entry<String, Map<String, TargetGroup>> regionEntry : targetGroups.entrySet()) {
Set<String> existingLoadBalancers = loadBalancers.getOrDefault(regionEntry.getKey(), Map.of()).keySet();
for (TargetGroup targetGroup : regionEntry.getValue().values()) {
targetGroup.getLoadBalancerArns().removeIf(lbArn -> !existingLoadBalancers.contains(lbArn));
tgToLbs.computeIfAbsent(targetGroup.getTargetGroupArn(), k -> ConcurrentHashMap.newKeySet())
.addAll(targetGroup.getLoadBalancerArns());
}
targetGroups.put(regionEntry.getKey(), regionEntry.getValue());
}
}
@@ -287,8 +290,10 @@ public class ElbV2Service implements ResourceProvider {
listeners.put(region, regionListeners);
rules.put(region, regionRules);
}
// remove from TG index
tgToLbs.values().forEach(lbSet -> lbSet.remove(arn));
// A deleted load balancer cannot keep a target group in use. Persist the removal too,
// otherwise restart rebuilds this index from stale LoadBalancerArns on the target group.
unlinkUnreferencedTargetGroups(region, arn);
persistRegion(targetGroups, region);
tags.remove(arn);
loadBalancers.put(region, regionLbs);
}
@@ -48,6 +48,10 @@ public class ImageCacheService {
}
public String ensureImageExists(String imageUri, String platform) {
return ensureImageExists(imageUri, platform, ImagePullObserver.NOOP);
}
public String ensureImageExists(String imageUri, String platform, ImagePullObserver observer) {
boolean explicitPlatform = platform != null && !platform.isBlank();
String requestedPlatform = explicitPlatform ? platform.trim() : daemonPlatform();
ImageKey imageKey = new ImageKey(imageUri, requestedPlatform);
@@ -68,7 +72,7 @@ public class ImageCacheService {
LOG.infov("Image already present locally, skipping pull: {0}", imageUri);
return resolvedImage;
}
return pullImage(imageUri, requestedPlatform, explicitPlatform, imageKey);
return pullImage(imageUri, requestedPlatform, explicitPlatform, imageKey, observer);
}
}
@@ -82,11 +86,13 @@ public class ImageCacheService {
ImageKey imageKey = new ImageKey(imageUri, requestedPlatform);
Object lock = locks.computeIfAbsent(imageUri, k -> new Object());
synchronized (lock) {
return pullImage(imageUri, requestedPlatform, explicitPlatform, imageKey);
return pullImage(imageUri, requestedPlatform, explicitPlatform, imageKey, ImagePullObserver.NOOP);
}
}
private String pullImage(String imageUri, String requestedPlatform, boolean explicitPlatform, ImageKey imageKey) {
private String pullImage(String imageUri, String requestedPlatform, boolean explicitPlatform, ImageKey imageKey,
ImagePullObserver observer) {
observer.started(imageUri);
LOG.infov("Pulling image: {0}", imageUri);
try {
runWithRetry(imageUri, MAX_PULL_ATTEMPTS, INITIAL_BACKOFF_MS, () -> {
@@ -101,6 +107,7 @@ public class ImageCacheService {
String resolvedImage = resolvedImageReference(imageUri, inspectLocalImage(imageUri));
resolvedImages.put(imageKey, resolvedImage);
LOG.infov("Image pulled successfully: {0}", imageUri);
observer.completed(imageUri);
return resolvedImage;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
@@ -108,6 +115,14 @@ public class ImageCacheService {
}
}
public interface ImagePullObserver {
ImagePullObserver NOOP = new ImagePullObserver() {};
default void started(String imageUri) {}
default void completed(String imageUri) {}
}
/**
* Runs the given pull attempt, retrying on transient registry failures with exponential
* backoff. A failure is considered transient when either:
@@ -0,0 +1,703 @@
package io.github.hectorvent.floci.services.lightsail;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.command.InspectExecResponse;
import com.github.dockerjava.api.model.Frame;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;
import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.core.common.docker.PortAllocator;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.jboss.logging.Logger;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Base64;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@ApplicationScoped
public class LightsailContainerManager {
private static final Logger LOG = Logger.getLogger(LightsailContainerManager.class);
private static final String WORDPRESS_IMAGE = "wordpress:6.7.2-apache";
private static final String MARIADB_IMAGE = "mariadb:11.7";
private static final String WORDPRESS_DATABASE = "wordpress";
private static final String WORDPRESS_DATABASE_USER = "wordpress";
private static final String WORDPRESS_DATABASE_PASSWORD = "wordpress";
private static final String NODE_APPLICATION_PATH = "/opt/lightsail/app";
private static final String NODE_APPLICATION_USER = "node";
private final ContainerBuilder containerBuilder;
private final ContainerLifecycleManager lifecycleManager;
private final EmulatorConfig config;
private final RegionResolver regionResolver;
private final PortAllocator portAllocator;
private final ObjectMapper mapper;
@Inject
public LightsailContainerManager(ContainerBuilder containerBuilder, ContainerLifecycleManager lifecycleManager,
EmulatorConfig config, RegionResolver regionResolver, PortAllocator portAllocator) {
this(containerBuilder, lifecycleManager, config, regionResolver, portAllocator, new ObjectMapper());
}
LightsailContainerManager(ContainerBuilder containerBuilder, ContainerLifecycleManager lifecycleManager,
EmulatorConfig config, RegionResolver regionResolver, PortAllocator portAllocator,
ObjectMapper mapper) {
this.containerBuilder = containerBuilder;
this.lifecycleManager = lifecycleManager;
this.config = config;
this.regionResolver = regionResolver;
this.portAllocator = portAllocator;
this.mapper = mapper;
}
public Launch launch(String instanceName, String region, String blueprintId, String bundleId, String image) {
return launch(instanceName, region, blueprintId, bundleId, image, null);
}
public Launch launch(String instanceName, String region, String blueprintId, String bundleId, String image, String publicKey) {
if (!isDockerAvailable()) {
return null;
}
Runtime runtime = runtimeFor(blueprintId);
if (runtime == null && (image == null || image.isBlank())) {
throw new IllegalArgumentException("Blueprint " + blueprintId + " has no local runtime mapping");
}
String containerName = ContainerStorageHelper.dockerName(config, "floci-lightsail-" + instanceName);
lifecycleManager.removeIfExists(containerName);
if ("wordpress".equals(blueprintId) && image == null) {
return launchWordPress(instanceName, region, bundleId, containerName);
}
if ("nodejs".equals(blueprintId) && image == null) {
return launchNode(instanceName, region, bundleId, containerName, publicKey);
}
boolean supportsSsh = image == null && runtime.sshUser() != null;
int sshHostPort = supportsSsh ? portAllocator.allocate(config.services().ec2().sshPortRangeStart(),
config.services().ec2().sshPortRangeEnd()) : 0;
ContainerBuilder.Builder builder = containerBuilder.newContainer(image == null ? runtime.image() : image)
.withName(containerName)
.withLogRotation()
.withLabels(ContainerStorageHelper.resourceIdentityLabels(
"lightsail", instanceName, regionResolver.getAccountId(), region))
.withMemoryMb(memoryMb(bundleId))
.withCpuUnits(cpuUnits(bundleId));
if (supportsSsh) {
builder.withPortBinding(22, sshHostPort);
}
if (image == null && runtime.command() != null) {
builder.withCmd(runtime.command());
}
ContainerSpec spec = builder.build();
try {
ContainerInfo container = lifecycleManager.createAndStart(spec);
if (supportsSsh && !bootstrapSsh(container.containerId(), runtime.sshUser(), publicKey, sshHostPort)) {
lifecycleManager.stopAndRemove(container.containerId(), null);
portAllocator.release(sshHostPort);
return null;
}
return new Launch(container.containerId(), null, null, supportsSsh ? sshHostPort : null);
} catch (RuntimeException e) {
if (sshHostPort > 0) {
portAllocator.release(sshHostPort);
}
LOG.warnv("Lightsail instance {0} could not start a backing container: {1}", instanceName, e.getMessage());
return null;
}
}
public static Runtime runtimeFor(String blueprintId) {
return switch (blueprintId) {
case "ubuntu_22_04" -> new Runtime("ubuntu:22.04", List.of("tail", "-f", "/dev/null"), "ubuntu", null);
case "ubuntu_24_04" -> new Runtime("ubuntu:24.04", List.of("tail", "-f", "/dev/null"), "ubuntu", null);
case "amazon_linux_2023" -> new Runtime("public.ecr.aws/amazonlinux/amazonlinux:2023",
List.of("tail", "-f", "/dev/null"), "ec2-user", null);
case "nginx" -> new Runtime("nginx:1.27-alpine", null, null, 80);
case "wordpress" -> new Runtime(WORDPRESS_IMAGE, null, null, 80);
case "nodejs" -> new Runtime("node:24-bookworm", null, NODE_APPLICATION_USER, 3000);
default -> null;
};
}
private Launch launchNode(String instanceName, String region, String bundleId, String containerName, String publicKey) {
int sshHostPort = portAllocator.allocate(config.services().ec2().sshPortRangeStart(),
config.services().ec2().sshPortRangeEnd());
String applicationVolume = containerName + "-app";
String releaseId = Long.toString(System.currentTimeMillis());
lifecycleManager.ensureVolume(applicationVolume);
ContainerSpec spec = containerBuilder.newContainer(runtimeFor("nodejs").image())
.withName(containerName)
.withLogRotation()
.withEnv("FLOCI_LIGHTSAIL_RELEASE_ID", releaseId)
.withNamedVolume(applicationVolume, NODE_APPLICATION_PATH)
.withPortBinding(22, sshHostPort)
.withLabels(ContainerStorageHelper.resourceIdentityLabels(
"lightsail", instanceName, regionResolver.getAccountId(), region))
.withMemoryMb(memoryMb(bundleId))
.withCpuUnits(cpuUnits(bundleId))
.withCmd(nodeSupervisorCommand())
.build();
try {
ContainerInfo container = lifecycleManager.createAndStart(spec);
if (!bootstrapSsh(container.containerId(), NODE_APPLICATION_USER, publicKey, sshHostPort)) {
lifecycleManager.stopAndRemove(container.containerId(), null);
lifecycleManager.removeVolume(applicationVolume);
portAllocator.release(sshHostPort);
return null;
}
return new Launch(container.containerId(), null, null, null, sshHostPort, applicationVolume, releaseId);
} catch (RuntimeException e) {
lifecycleManager.removeVolume(applicationVolume);
portAllocator.release(sshHostPort);
LOG.warnv("Lightsail Node.js instance {0} could not start: {1}", instanceName, e.getMessage());
return null;
}
}
static List<String> nodeSupervisorCommand() {
String script = "app=" + NODE_APPLICATION_PATH + ";"
+ "mkdir -p \"$app/releases\" \"$app/logs\";"
+ "if [ ! -L \"$app/current\" ]; then mkdir -p \"$app/releases/$FLOCI_LIGHTSAIL_RELEASE_ID\";"
+ "ln -s \"releases/$FLOCI_LIGHTSAIL_RELEASE_ID\" \"$app/current\"; fi;"
+ "chown -R " + NODE_APPLICATION_USER + ":" + NODE_APPLICATION_USER + " \"$app\";"
+ "trap 'exit 0' INT TERM;delay=1;"
+ "while true; do "
+ "ls -1dt \"$app\"/releases/* 2>/dev/null | tail -n +4 | xargs -r rm -rf;"
+ "if [ -f \"$app/current/package.json\" ]; then "
+ "su -s /bin/sh " + NODE_APPLICATION_USER
+ " -c 'cd /opt/lightsail/app/current && PORT=$(cat .floci-port 2>/dev/null || printf 3000) HOST=0.0.0.0;"
+ "if [ -f .floci-start ]; then exec sh -c \"$(base64 -d .floci-start)\"; else exec npm start; fi' >>\"$app/logs/app.stdout.log\" "
+ "2>>\"$app/logs/app.stderr.log\";status=$?;"
+ "printf '%s app exited with status %s\\n' \"$(date -u +%FT%TZ)\" \"$status\" >>\"$app/logs/supervisor.log\";"
+ "sleep \"$delay\";[ \"$delay\" -lt 30 ] && delay=$((delay * 2));"
+ "else delay=1;sleep 2;fi;"
+ "done";
return List.of("sh", "-c", script);
}
private Launch launchWordPress(String instanceName, String region, String bundleId, String containerName) {
String databaseContainerName = containerName + "-mariadb";
String databaseVolume = databaseContainerName + "-data";
String wordpressContentVolume = containerName + "-wp-content";
lifecycleManager.removeIfExists(databaseContainerName);
lifecycleManager.ensureVolume(databaseVolume);
lifecycleManager.ensureVolume(wordpressContentVolume);
ContainerSpec databaseSpec = containerBuilder.newContainer(MARIADB_IMAGE)
.withName(databaseContainerName)
.withLogRotation()
.withEnv("MARIADB_DATABASE", WORDPRESS_DATABASE)
.withEnv("MARIADB_USER", WORDPRESS_DATABASE_USER)
.withEnv("MARIADB_PASSWORD", WORDPRESS_DATABASE_PASSWORD)
.withEnv("MARIADB_ROOT_PASSWORD", WORDPRESS_DATABASE_PASSWORD)
.withNamedVolume(databaseVolume, "/var/lib/mysql")
.withLabels(ContainerStorageHelper.resourceIdentityLabels(
"lightsail", instanceName, regionResolver.getAccountId(), region))
.withMemoryMb(Math.max(256, memoryMb(bundleId) / 2))
.withCpuUnits(cpuUnits(bundleId))
.build();
String databaseContainerId = null;
try {
databaseContainerId = lifecycleManager.createAndStart(databaseSpec).containerId();
ContainerSpec wordpressSpec = containerBuilder.newContainer(WORDPRESS_IMAGE)
.withName(containerName)
.withLogRotation()
.withEnv("WORDPRESS_DB_HOST", "127.0.0.1:3306")
.withEnv("WORDPRESS_DB_NAME", WORDPRESS_DATABASE)
.withEnv("WORDPRESS_DB_USER", WORDPRESS_DATABASE_USER)
.withEnv("WORDPRESS_DB_PASSWORD", WORDPRESS_DATABASE_PASSWORD)
.withNetworkMode("container:" + databaseContainerId)
.withNamedVolume(wordpressContentVolume, "/var/www/html/wp-content")
.withLabels(ContainerStorageHelper.resourceIdentityLabels(
"lightsail", instanceName, regionResolver.getAccountId(), region))
.withMemoryMb(memoryMb(bundleId))
.withCpuUnits(cpuUnits(bundleId))
.build();
ContainerInfo wordpress = lifecycleManager.createAndStart(wordpressSpec);
return new Launch(wordpress.containerId(), databaseContainerId, databaseVolume, wordpressContentVolume, null);
} catch (RuntimeException e) {
if (databaseContainerId != null) {
lifecycleManager.stopAndRemove(databaseContainerId, null);
}
lifecycleManager.removeVolume(databaseVolume);
lifecycleManager.removeVolume(wordpressContentVolume);
LOG.warnv("Lightsail WordPress instance {0} could not start: {1}", instanceName, e.getMessage());
return null;
}
}
private static int memoryMb(String bundleId) {
return switch (bundleId) {
case "nano_3_0" -> 512;
case "micro_3_0" -> 1024;
case "small_3_0" -> 2048;
case "medium_3_0" -> 4096;
default -> 1024;
};
}
private static int cpuUnits(String bundleId) {
return "nano_3_0".equals(bundleId) ? 1024 : 2048;
}
public record Runtime(String image, List<String> command, String sshUser, Integer applicationPort) {
}
public record Launch(String containerId, String databaseContainerId, String databaseVolume,
String wordpressContentVolume, Integer sshHostPort, String applicationVolume,
String applicationRelease) {
public Launch(String containerId, String databaseContainerId, String databaseVolume,
String wordpressContentVolume, Integer sshHostPort) {
this(containerId, databaseContainerId, databaseVolume, wordpressContentVolume, sshHostPort, null, null);
}
public Launch(String containerId, String databaseContainerId, String databaseVolume) {
this(containerId, databaseContainerId, databaseVolume, null, null, null, null);
}
public Launch(String containerId, String databaseContainerId, String databaseVolume, Integer sshHostPort) {
this(containerId, databaseContainerId, databaseVolume, null, sshHostPort, null, null);
}
}
public void stop(String containerId, String databaseContainerId) {
if (containerId != null && !containerId.isBlank()) {
lifecycleManager.getDockerClient().stopContainerCmd(containerId).exec();
}
if (databaseContainerId != null && !databaseContainerId.isBlank()) {
lifecycleManager.getDockerClient().stopContainerCmd(databaseContainerId).exec();
}
}
public void start(String containerId, String databaseContainerId) {
start(containerId, databaseContainerId, false);
}
public void start(String containerId, String databaseContainerId, boolean sshEnabled) {
if (databaseContainerId != null && !databaseContainerId.isBlank()) {
lifecycleManager.getDockerClient().startContainerCmd(databaseContainerId).exec();
}
if (containerId != null && !containerId.isBlank()) {
lifecycleManager.getDockerClient().startContainerCmd(containerId).exec();
if (sshEnabled) {
startSshd(containerId);
}
}
}
public void reboot(String containerId, String databaseContainerId) {
reboot(containerId, databaseContainerId, false);
}
public void reboot(String containerId, String databaseContainerId, boolean sshEnabled) {
if (containerId != null && !containerId.isBlank()) {
lifecycleManager.getDockerClient().restartContainerCmd(containerId).exec();
if (sshEnabled) {
startSshd(containerId);
}
}
if (databaseContainerId != null && !databaseContainerId.isBlank()) {
lifecycleManager.getDockerClient().restartContainerCmd(databaseContainerId).exec();
}
}
public void terminate(String containerId, String databaseContainerId, String databaseVolume,
String wordpressContentVolume) {
terminate(containerId, databaseContainerId, databaseVolume, wordpressContentVolume, null);
}
public void terminate(String containerId, String databaseContainerId, String databaseVolume,
String wordpressContentVolume, String applicationVolume) {
if (containerId != null && !containerId.isBlank()) {
lifecycleManager.stopAndRemoveStrict(containerId, null);
}
if (databaseContainerId != null && !databaseContainerId.isBlank()) {
lifecycleManager.stopAndRemoveStrict(databaseContainerId, null);
}
if (databaseVolume != null && !databaseVolume.isBlank()) {
lifecycleManager.removeVolumeStrict(databaseVolume);
}
if (wordpressContentVolume != null && !wordpressContentVolume.isBlank()) {
lifecycleManager.removeVolumeStrict(wordpressContentVolume);
}
if (applicationVolume != null && !applicationVolume.isBlank()) {
lifecycleManager.removeVolumeStrict(applicationVolume);
}
}
public String capture(String containerId, String snapshotName) {
if (containerId == null || containerId.isBlank()) {
return null;
}
String image = ContainerStorageHelper.dockerName(config, "floci-lightsail-snapshot-" + snapshotName) + ":capture";
try {
lifecycleManager.getDockerClient().commitCmd(containerId)
.withRepository(image.substring(0, image.indexOf(':')))
.withTag("capture")
.exec();
return image;
} catch (RuntimeException e) {
throw new IllegalStateException("Could not capture Lightsail instance container " + containerId, e);
}
}
public void removeCapture(String image) {
if (image == null || image.isBlank()) {
return;
}
try {
lifecycleManager.getDockerClient().removeImageCmd(image).withForce(true).exec();
} catch (RuntimeException e) {
LOG.warnv("Could not remove Lightsail snapshot image {0}: {1}", image, e.getMessage());
}
}
public void deployStaticFiles(String containerId, List<StaticFile> files) {
if (containerId == null || containerId.isBlank()) {
throw new IllegalArgumentException("The Lightsail instance has no running container.");
}
if (files == null || files.isEmpty()) {
throw new IllegalArgumentException("At least one static file is required.");
}
for (StaticFile file : files) {
validateDeploymentPath(file.path());
}
try {
ByteArrayOutputStream archive = new ByteArrayOutputStream();
try (TarArchiveOutputStream tar = new TarArchiveOutputStream(archive)) {
tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
for (StaticFile file : files) {
TarArchiveEntry entry = new TarArchiveEntry(file.path());
entry.setMode(0644);
entry.setSize(Files.size(file.source()));
tar.putArchiveEntry(entry);
Files.copy(file.source(), tar);
tar.closeArchiveEntry();
}
tar.finish();
}
lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)
.withTarInputStream(new ByteArrayInputStream(archive.toByteArray()))
.withRemotePath("/usr/share/nginx/html")
.exec();
} catch (IOException | RuntimeException e) {
throw new IllegalStateException("Could not deploy static files to the Lightsail container", e);
}
}
public NodeDeployment deployNodeApplication(String containerId, List<StaticFile> uploads, String explicitStart,
int port) {
if (containerId == null || containerId.isBlank()) {
throw new IllegalArgumentException("The Lightsail instance has no running container.");
}
if (port < 1 || port > 65535) {
throw new IllegalArgumentException("PORT must be between 1 and 65535.");
}
Path staging = null;
try {
staging = Files.createTempDirectory("floci-node-deploy-");
int files = unpackNodeUploads(uploads, staging);
if (files == 0) {
throw new IllegalArgumentException("At least one application file is required.");
}
Path packageJson = staging.resolve("package.json");
if (!Files.isRegularFile(packageJson)) {
throw new IllegalArgumentException("package.json is required before deployment files are written.");
}
JsonNode packageDefinition = mapper.readTree(packageJson.toFile());
String start = explicitStart == null || explicitStart.isBlank()
? packageDefinition.path("scripts").path("start").asText(null) : explicitStart;
if (start == null || start.isBlank()) {
throw new IllegalArgumentException("Specify a start command or add scripts.start to package.json.");
}
String releaseId = UUID.randomUUID().toString();
Files.writeString(staging.resolve(".floci-start"), Base64.getEncoder().encodeToString(start.getBytes(StandardCharsets.UTF_8)));
Files.writeString(staging.resolve(".floci-port"), Integer.toString(port));
CommandResult releaseDirectory = execOutput(containerId, new String[]{"sh", "-c",
"mkdir -p " + NODE_APPLICATION_PATH + "/releases/" + releaseId}, 15);
if (!releaseDirectory.success()) {
throw new IllegalStateException("Could not prepare the application release directory. " + releaseDirectory.output());
}
copyNodeRelease(containerId, staging, releaseId);
CommandResult current = execOutput(containerId, new String[]{"sh", "-c", "readlink " + NODE_APPLICATION_PATH + "/current || true"}, 15);
String install = "set -eu; release=" + NODE_APPLICATION_PATH + "/releases/" + releaseId + ";"
+ "chown -R " + NODE_APPLICATION_USER + ":" + NODE_APPLICATION_USER + " \"$release\";"
+ "su -s /bin/sh " + NODE_APPLICATION_USER + " -c 'cd \"$0\" && "
+ (Files.exists(staging.resolve("package-lock.json")) ? "npm ci" : "npm install") + "' \"$release\";"
+ "ln -s \"releases/" + releaseId + "\" " + NODE_APPLICATION_PATH + "/current.next;"
+ "mv -Tf " + NODE_APPLICATION_PATH + "/current.next " + NODE_APPLICATION_PATH + "/current";
CommandResult result = execOutput(containerId, new String[]{"sh", "-c", install}, 300);
if (!result.success()) {
throw new IllegalStateException("Dependency installation failed. " + result.output());
}
lifecycleManager.getDockerClient().restartContainerCmd(containerId).exec();
CommandResult health = execOutput(containerId, new String[]{"sh", "-c",
"for n in $(seq 1 30); do node -e 'fetch(\"http://127.0.0.1:" + port
+ "/\").then(() => process.exit(0)).catch(() => process.exit(1))' && exit 0; sleep 1; done; exit 1"}, 35);
if (!health.success()) {
if (current.success() && !current.output().isBlank()) {
String previous = current.output().trim();
execOutput(containerId, new String[]{"sh", "-c", "ln -s \"" + previous + "\" "
+ NODE_APPLICATION_PATH + "/current.rollback && mv -Tf " + NODE_APPLICATION_PATH
+ "/current.rollback " + NODE_APPLICATION_PATH + "/current"}, 15);
lifecycleManager.getDockerClient().restartContainerCmd(containerId).exec();
}
throw new IllegalStateException("Application health check on 127.0.0.1:" + port
+ " failed. Ensure the start command listens on PORT and binds 0.0.0.0, not localhost. " + health.output());
}
return new NodeDeployment(releaseId, files, port, start);
} catch (IOException e) {
throw new IllegalArgumentException("Could not read the Node.js deployment: " + e.getMessage(), e);
} finally {
if (staging != null) {
deleteTree(staging);
}
}
}
private int unpackNodeUploads(List<StaticFile> uploads, Path staging) throws IOException {
if (uploads == null || uploads.isEmpty() || uploads.size() > 1_000) {
throw new IllegalArgumentException("Provide between 1 and 1000 deployment files.");
}
long[] size = {0};
int[] files = {0};
for (StaticFile upload : uploads) {
if (upload.path().toLowerCase().endsWith(".zip")) {
try (ZipArchiveInputStream zip = new ZipArchiveInputStream(Files.newInputStream(upload.source()))) {
ZipArchiveEntry entry;
while ((entry = zip.getNextZipEntry()) != null) {
if (entry.isUnixSymlink()) {
throw new IllegalArgumentException("Deployment archives must not contain symbolic links.");
}
if (!entry.isDirectory()) {
writeNodeFile(staging, entry.getName(), zip, size, files);
} else {
validateNodePath(entry.getName());
}
}
}
} else {
try (java.io.InputStream input = Files.newInputStream(upload.source())) {
writeNodeFile(staging, upload.path(), input, size, files);
}
}
}
return files[0];
}
private void writeNodeFile(Path staging, String name, java.io.InputStream input, long[] size, int[] files) throws IOException {
validateNodePath(name);
if (name.equals("node_modules") || name.startsWith("node_modules/")) {
return;
}
if (++files[0] > 1_000) {
throw new IllegalArgumentException("Deployment archive contains more than 1000 files.");
}
Path target = staging.resolve(name).normalize();
if (!target.startsWith(staging)) {
throw new IllegalArgumentException("Deployment archive contains a traversal path.");
}
Files.createDirectories(target.getParent());
try (OutputStream output = Files.newOutputStream(target)) {
byte[] buffer = new byte[8192];
int read;
while ((read = input.read(buffer)) >= 0) {
size[0] += read;
if (size[0] > 50L * 1024 * 1024) {
throw new IllegalArgumentException("Deployment archive exceeds the 50 MiB uncompressed limit.");
}
output.write(buffer, 0, read);
}
}
}
private void copyNodeRelease(String containerId, Path staging, String releaseId) throws IOException {
ByteArrayOutputStream archive = new ByteArrayOutputStream();
try (TarArchiveOutputStream tar = new TarArchiveOutputStream(archive)) {
tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
try (java.util.stream.Stream<Path> paths = Files.walk(staging)) {
for (Path path : paths.filter(Files::isRegularFile).toList()) {
String name = staging.relativize(path).toString().replace('\\', '/');
TarArchiveEntry entry = new TarArchiveEntry(name);
entry.setMode(0644);
entry.setSize(Files.size(path));
tar.putArchiveEntry(entry);
Files.copy(path, tar);
tar.closeArchiveEntry();
}
}
tar.finish();
}
lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)
.withTarInputStream(new ByteArrayInputStream(archive.toByteArray()))
.withRemotePath(NODE_APPLICATION_PATH + "/releases/" + releaseId).exec();
}
static void validateNodePath(String path) {
validateDeploymentPath(path);
if (path.startsWith(".floci-")) {
throw new IllegalArgumentException("Deployment contains a reserved path.");
}
}
private CommandResult execOutput(String containerId, String[] command, int timeoutSeconds) {
ByteArrayOutputStream output = new ByteArrayOutputStream();
try {
String execId = lifecycleManager.getDockerClient().execCreateCmd(containerId).withCmd(command)
.withAttachStdout(true).withAttachStderr(true).exec().getId();
CountDownLatch complete = new CountDownLatch(1);
lifecycleManager.getDockerClient().execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>() {
@Override public void onNext(Frame frame) { output.writeBytes(frame.getPayload()); }
@Override public void onComplete() { complete.countDown(); }
@Override public void onError(Throwable error) { complete.countDown(); }
});
if (!complete.await(timeoutSeconds, TimeUnit.SECONDS)) {
return new CommandResult(false, "Command timed out.");
}
InspectExecResponse result = lifecycleManager.getDockerClient().inspectExecCmd(execId).exec();
return new CommandResult(result.getExitCodeLong() != null && result.getExitCodeLong() == 0,
output.toString(StandardCharsets.UTF_8).substring(0, Math.min(output.size(), 8_000)));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return new CommandResult(false, "Command interrupted.");
} catch (RuntimeException e) {
return new CommandResult(false, e.getMessage());
}
}
private static void deleteTree(Path root) {
try (java.util.stream.Stream<Path> paths = Files.walk(root)) {
paths.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (IOException ignored) {
// The staging directory is outside persisted state and the OS will eventually clean it.
}
});
} catch (IOException ignored) {
// The staging directory is outside persisted state and the OS will eventually clean it.
}
}
static void validateDeploymentPath(String path) {
if (path == null || path.isBlank() || path.startsWith("/") || path.startsWith("\\")
|| path.contains("\\") || path.indexOf('\u0000') >= 0) {
throw new IllegalArgumentException("Static file paths must be non-empty relative paths using forward slashes.");
}
for (String segment : path.split("/")) {
if (segment.isEmpty() || ".".equals(segment) || "..".equals(segment)) {
throw new IllegalArgumentException("Static file paths must not contain traversal segments.");
}
}
}
public record StaticFile(String path, Path source) {
}
public record NodeDeployment(String releaseId, int files, int port, String startCommand) {
}
private record CommandResult(boolean success, String output) {
}
private boolean bootstrapSsh(String containerId, String user, String publicKey, int hostPort) {
if (publicKey == null || publicKey.isBlank()) {
return false;
}
String encodedKey = Base64.getEncoder().encodeToString(publicKey.trim().getBytes(StandardCharsets.UTF_8));
String setupUser = "id -u " + user + " >/dev/null 2>&1 || useradd -m -s /bin/bash " + user + ";"
+ "mkdir -p /home/" + user + "/.ssh;chmod 700 /home/" + user + "/.ssh;"
+ "printf %s '" + encodedKey + "' | base64 -d > /home/" + user + "/.ssh/authorized_keys;"
+ "chown -R " + user + ":" + user + " /home/" + user + "/.ssh;chmod 600 /home/" + user + "/.ssh/authorized_keys";
return exec(containerId, new String[]{"sh", "-c", setupUser}, 15)
&& startSshd(containerId)
&& sshBanner(hostPort);
}
private boolean startSshd(String containerId) {
String install = "if command -v dnf >/dev/null 2>&1; then dnf install -y openssh-server openssh-clients; "
+ "elif command -v apt-get >/dev/null 2>&1; then apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server openssh-client; "
+ "else exit 1; fi";
return exec(containerId, new String[]{"sh", "-c", "command -v sshd >/dev/null 2>&1 || (" + install + ")"}, 120)
&& exec(containerId, new String[]{"sh", "-c", "ssh-keygen -A && mkdir -p /run/sshd && /usr/sbin/sshd"}, 15);
}
private boolean exec(String containerId, String[] command, int timeoutSeconds) {
try {
String execId = lifecycleManager.getDockerClient().execCreateCmd(containerId).withCmd(command)
.withAttachStdout(true).withAttachStderr(true).exec().getId();
CountDownLatch complete = new CountDownLatch(1);
lifecycleManager.getDockerClient().execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>() {
@Override public void onComplete() { complete.countDown(); }
@Override public void onError(Throwable error) { complete.countDown(); }
});
if (!complete.await(timeoutSeconds, TimeUnit.SECONDS)) {
return false;
}
InspectExecResponse result = lifecycleManager.getDockerClient().inspectExecCmd(execId).exec();
return result.getExitCodeLong() != null && result.getExitCodeLong() == 0;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
} catch (RuntimeException e) {
LOG.warnv("Could not execute SSH bootstrap in container {0}: {1}", containerId, e.getMessage());
return false;
}
}
private boolean sshBanner(int hostPort) {
List<String> hosts = List.of("127.0.0.1", "host.docker.internal");
for (int attempt = 0; attempt < 20; attempt++) {
for (String host : hosts) {
try (Socket socket = new Socket()) {
socket.connect(new InetSocketAddress(host, hostPort), 1_000);
socket.setSoTimeout(1_000);
BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.US_ASCII));
String banner = reader.readLine();
if (banner != null && banner.startsWith("SSH-")) {
return true;
}
} catch (IOException ignored) {
// Docker Desktop publishes container ports on the host gateway, unavailable during early startup.
}
}
try {
Thread.sleep(250);
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
return false;
}
}
return false;
}
private boolean isDockerAvailable() {
try {
lifecycleManager.getDockerClient().pingCmd().exec();
return true;
} catch (RuntimeException e) {
LOG.debugv("Docker is unavailable for Lightsail containers: {0}", e.getMessage());
return false;
}
}
}
@@ -81,7 +81,6 @@ public class LightsailJsonHandler {
Map.entry("GetDomains", "domains"),
Map.entry("GetExportSnapshotRecords", "exportSnapshotRecords"),
Map.entry("GetInstanceMetricData", "metricData"),
Map.entry("GetInstanceSnapshots", "instanceSnapshots"),
Map.entry("GetLoadBalancerMetricData", "metricData"),
Map.entry("GetLoadBalancerTlsCertificates", "tlsCertificates"),
Map.entry("GetLoadBalancerTlsPolicies", "tlsPolicies"),
@@ -115,6 +114,11 @@ public class LightsailJsonHandler {
case "StopInstance" -> ok(service.stopInstance(region, required(request, "instanceName")));
case "RebootInstance" -> ok(service.rebootInstance(region, required(request, "instanceName")));
case "GetInstanceState" -> ok(service.getInstanceState(region, required(request, "instanceName")));
case "CreateInstanceSnapshot" -> ok(service.createInstanceSnapshot(region, request));
case "CreateInstancesFromSnapshot" -> ok(service.createInstancesFromSnapshot(region, request));
case "GetInstanceSnapshots" -> ok(service.getInstanceSnapshots(region));
case "GetInstanceSnapshot" -> ok(service.getInstanceSnapshot(region, required(request, "instanceSnapshotName")));
case "DeleteInstanceSnapshot" -> ok(service.deleteInstanceSnapshot(region, required(request, "instanceSnapshotName")));
case "GetInstancePortStates" -> ok(service.getInstancePortStates(region, required(request, "instanceName")));
case "OpenInstancePublicPorts" -> ok(service.putInstancePorts(region, requiredInstanceName(request), request, true));
case "PutInstancePublicPorts" -> ok(service.putInstancePorts(region, requiredInstanceName(request), request, false));
@@ -0,0 +1,197 @@
package io.github.hectorvent.floci.services.lightsail;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.model.ContainerNetwork;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
import io.github.hectorvent.floci.core.common.docker.PortAllocator;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ApplicationScoped
public class LightsailPortForwardManager {
static final String PORT_FORWARDS_FIELD = "_portForwards";
private static final Logger LOG = Logger.getLogger(LightsailPortForwardManager.class);
private final DockerClient dockerClient;
private final ContainerBuilder containerBuilder;
private final ContainerLifecycleManager lifecycleManager;
private final PortAllocator portAllocator;
private final EmulatorConfig config;
@Inject
public LightsailPortForwardManager(DockerClient dockerClient, ContainerBuilder containerBuilder,
ContainerLifecycleManager lifecycleManager, PortAllocator portAllocator,
EmulatorConfig config) {
this.dockerClient = dockerClient;
this.containerBuilder = containerBuilder;
this.lifecycleManager = lifecycleManager;
this.portAllocator = portAllocator;
this.config = config;
}
public void reconcile(ObjectNode instance) {
if (instance == null || !"running".equals(instance.path("state").path("name").asText())
|| !instance.hasNonNull("containerId")) {
return;
}
synchronized (instance) {
Set<Integer> desiredPorts = extractPublishablePorts(instance.path("networking").path("ports"),
config.services().ec2().maxPublishedPortsPerInstance());
// OS runtimes publish SSH directly on their container, not through a sidecar.
desiredPorts.remove(22);
ObjectNode forwards = forwards(instance);
for (Integer appPort : desiredPorts) {
if (!forwards.has(String.valueOf(appPort))) {
publish(instance, appPort, forwards);
}
}
for (String appPort : new ArrayList<>(forwards.propertyStream().map(Map.Entry::getKey).toList())) {
if (!desiredPorts.contains(Integer.parseInt(appPort))) {
unpublish(instance, appPort, forwards);
}
}
}
}
public void unpublishAll(ObjectNode instance) {
if (instance == null) {
return;
}
synchronized (instance) {
ObjectNode forwards = forwards(instance);
for (String appPort : new ArrayList<>(forwards.propertyStream().map(Map.Entry::getKey).toList())) {
unpublish(instance, appPort, forwards);
}
}
}
public void releaseSshPort(ObjectNode instance) {
if (instance != null && instance.has("_sshHostPort")) {
portAllocator.release(instance.path("_sshHostPort").asInt());
instance.remove("_sshHostPort");
}
}
static Set<Integer> extractPublishablePorts(JsonNode portRules, int maxPerInstance) {
Set<Integer> ports = new LinkedHashSet<>();
if (!portRules.isArray()) {
return ports;
}
for (JsonNode rule : portRules) {
if (!"tcp".equalsIgnoreCase(rule.path("protocol").asText()) || !hasCidrSource(rule)) {
continue;
}
int from = rule.path("fromPort").asInt(-1);
int to = rule.path("toPort").asInt(-1);
if (from < 0 || to < 0) {
continue;
}
int low = Math.min(from, to);
int high = Math.max(from, to);
if ((long) high - low + 1 > maxPerInstance) {
LOG.warnv("Skipping wide Lightsail public-port range {0}-{1} (span exceeds max {2})",
low, high, maxPerInstance);
continue;
}
for (int port = low; port <= high && ports.size() < maxPerInstance; port++) {
ports.add(port);
}
}
return ports;
}
static String forwardContainerName(String instanceName, int appPort) {
return "floci-lightsail-fwd-" + instanceName + "-" + appPort;
}
private void publish(ObjectNode instance, int appPort, ObjectNode forwards) {
int hostPort = portAllocator.allocate(config.services().ec2().appPortRangeStart(),
config.services().ec2().appPortRangeEnd());
try {
NetworkTarget target = resolveTarget(instance.path("containerId").asText());
if (target == null) {
portAllocator.release(hostPort);
return;
}
String name = forwardContainerName(instance.path("name").asText(), appPort);
lifecycleManager.removeIfExists(name);
ContainerBuilder.Builder builder = containerBuilder.newContainer(config.services().ec2().socatImage())
.withName(name)
.withEntrypoint(List.of("socat"))
.withCmd(List.of("TCP-LISTEN:" + appPort + ",fork,reuseaddr", "TCP:" + target.ip() + ":" + appPort))
.withPortBinding(appPort, hostPort)
.withLogRotation();
if (target.network() != null) {
builder.withNetworkMode(target.network());
}
ContainerSpec spec = builder.build();
String containerId = lifecycleManager.create(spec);
lifecycleManager.startCreated(containerId, spec);
forwards.put(String.valueOf(appPort), hostPort);
LOG.infov("Published Lightsail instance {0} app port {1} on host port {2}",
instance.path("name").asText(), appPort, hostPort);
} catch (Exception e) {
portAllocator.release(hostPort);
LOG.warnv("Failed to publish Lightsail instance {0} app port {1}: {2}",
instance.path("name").asText(), appPort, e.getMessage());
}
}
private void unpublish(ObjectNode instance, String appPort, ObjectNode forwards) {
lifecycleManager.removeIfExists(forwardContainerName(instance.path("name").asText(), Integer.parseInt(appPort)));
int hostPort = forwards.path(appPort).asInt();
forwards.remove(appPort);
portAllocator.release(hostPort);
}
private ObjectNode forwards(ObjectNode instance) {
JsonNode node = instance.get(PORT_FORWARDS_FIELD);
if (node instanceof ObjectNode objectNode) {
return objectNode;
}
return instance.putObject(PORT_FORWARDS_FIELD);
}
private NetworkTarget resolveTarget(String containerId) {
try {
InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();
Map<String, ContainerNetwork> networks = inspect.getNetworkSettings().getNetworks();
String networkMode = inspect.getHostConfig().getNetworkMode();
if (networks.isEmpty() && networkMode != null && networkMode.startsWith("container:")) {
inspect = dockerClient.inspectContainerCmd(networkMode.substring("container:".length())).exec();
networks = inspect.getNetworkSettings().getNetworks();
}
for (Map.Entry<String, ContainerNetwork> entry : networks.entrySet()) {
String ip = entry.getValue().getIpAddress();
if (ip != null && !ip.isBlank()) {
return new NetworkTarget("bridge".equals(entry.getKey()) ? null : entry.getKey(), ip);
}
}
} catch (Exception e) {
LOG.warnv("Could not inspect Lightsail instance container {0}: {1}", containerId, e.getMessage());
}
return null;
}
private static boolean hasCidrSource(JsonNode rule) {
return rule.path("cidrs").isArray() && !rule.path("cidrs").isEmpty()
|| rule.path("ipv6Cidrs").isArray() && !rule.path("ipv6Cidrs").isEmpty();
}
private record NetworkTarget(String network, String ip) {
}
}
@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.core.resource.ResourceProvider;
import io.github.hectorvent.floci.core.resource.SupportedResourceType;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.ec2.Ec2KeyMaterial;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -35,20 +36,30 @@ public class LightsailService implements ResourceProvider {
private static final String RESOURCE_DISK = "Disk";
private static final String RESOURCE_STATIC_IP = "StaticIp";
private static final String RESOURCE_KEY_PAIR = "KeyPair";
private static final String RESOURCE_INSTANCE_SNAPSHOT = "InstanceSnapshot";
private final StorageBackend<String, ObjectNode> resourceStore;
private final StorageBackend<String, ObjectNode> operationStore;
private final ObjectMapper mapper;
private final RegionResolver regionResolver;
private final LightsailContainerManager containerManager;
private final LightsailPortForwardManager portForwardManager;
@Inject
public LightsailService(StorageFactory storageFactory, ObjectMapper mapper, RegionResolver regionResolver) {
public LightsailService(StorageFactory storageFactory, ObjectMapper mapper, RegionResolver regionResolver,
LightsailContainerManager containerManager, LightsailPortForwardManager portForwardManager) {
this.resourceStore = storageFactory.create(SERVICE, "lightsail-resources.json",
new TypeReference<Map<String, ObjectNode>>() {});
this.operationStore = storageFactory.create(SERVICE, "lightsail-operations.json",
new TypeReference<Map<String, ObjectNode>>() {});
this.mapper = mapper;
this.regionResolver = regionResolver;
this.containerManager = containerManager;
this.portForwardManager = portForwardManager;
}
LightsailService(StorageFactory storageFactory, ObjectMapper mapper, RegionResolver regionResolver) {
this(storageFactory, mapper, regionResolver, null, null);
}
public ObjectNode createInstances(String region, JsonNode request) {
@@ -56,6 +67,11 @@ public class LightsailService implements ResourceProvider {
requireText(request, "availabilityZone");
String blueprintId = requireText(request, "blueprintId");
String bundleId = requireText(request, "bundleId");
LightsailContainerManager.Runtime runtime = LightsailContainerManager.runtimeFor(blueprintId);
if (runtime == null && textOrNull(request, "_containerImage") == null) {
throw new AwsException("InvalidInputException", "Blueprint " + blueprintId
+ " is not available locally because it has no runtime mapping.", 400);
}
String availabilityZone = request.path("availabilityZone").asText();
ArrayNode operations = mapper.createArrayNode();
@@ -75,12 +91,12 @@ public class LightsailService implements ResourceProvider {
instance.set("ipv6Addresses", mapper.createArrayNode());
instance.put("ipAddressType", request.path("ipAddressType").asText("ipv4"));
instance.set("hardware", hardware(bundleId));
instance.set("networking", defaultNetworking());
instance.set("networking", defaultNetworking(runtime == null ? null : runtime.applicationPort()));
instance.set("state", instanceState("running"));
instance.put("username", defaultUsername(blueprintId));
if (request.hasNonNull("keyPairName")) {
instance.put("sshKeyName", request.path("keyPairName").asText());
}
String keyPairName = request.path("keyPairName").asText("LightsailDefaultKeyPair");
ObjectNode keyPair = selectedKeyPair(region, keyPairName);
instance.put("sshKeyName", keyPairName);
instance.set("tags", tags(request.path("tags")));
instance.set("metadataOptions", mapper.createObjectNode()
.put("state", "applied")
@@ -88,7 +104,49 @@ public class LightsailService implements ResourceProvider {
.put("httpEndpoint", "enabled")
.put("httpPutResponseHopLimit", 1));
String snapshotImage = textOrNull(request, "_containerImage");
LightsailContainerManager.Launch launch = containerManager == null ? null
: containerManager.launch(name, region, blueprintId, bundleId, snapshotImage,
decodePublicKey(keyPair.path("publicKeyBase64").asText()));
String containerId = launch == null ? null : launch.containerId();
instance.put("runtimeStatus", containerId == null ? "container-unavailable" : "running");
if (runtime != null) {
instance.put("runtimeImage", runtime.image());
if (runtime.applicationPort() != null) {
instance.put("applicationPort", runtime.applicationPort());
}
}
if (containerId != null) {
instance.put("containerId", containerId);
}
if (launch != null && launch.sshHostPort() != null) {
instance.put("_sshHostPort", launch.sshHostPort());
}
if (launch != null && launch.databaseContainerId() != null) {
instance.put("databaseContainerId", launch.databaseContainerId());
instance.put("databaseVolume", launch.databaseVolume());
}
if (launch != null && launch.wordpressContentVolume() != null) {
instance.put("wordpressContentVolume", launch.wordpressContentVolume());
}
if (launch != null && launch.applicationVolume() != null) {
instance.put("applicationVolume", launch.applicationVolume());
instance.put("applicationRelease", launch.applicationRelease());
instance.set("runtimeMetadata", mapper.createObjectNode()
.put("launchedAt", epochSeconds(Instant.now()))
.put("applicationPath", "/opt/lightsail/app")
.put("currentRelease", launch.applicationRelease())
.put("releaseRetention", 3)
.put("stdoutLog", "/opt/lightsail/app/logs/app.stdout.log")
.put("stderrLog", "/opt/lightsail/app/logs/app.stderr.log")
.put("supervisorLog", "/opt/lightsail/app/logs/supervisor.log"));
}
resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
if (portForwardManager != null) {
portForwardManager.reconcile(instance);
resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
}
operations.add(operation(region, RESOURCE_INSTANCE, name, "CreateInstance"));
}
@@ -98,16 +156,33 @@ public class LightsailService implements ResourceProvider {
public ObjectNode getInstances(String region) {
ObjectNode response = mapper.createObjectNode();
ArrayNode instances = response.putArray("instances");
listResources(region, RESOURCE_INSTANCE).forEach(instances::add);
for (ObjectNode instance : listResources(region, RESOURCE_INSTANCE)) {
reconcileForwards(region, instance);
instances.add(instanceResponse(instance));
}
return response;
}
public ObjectNode getInstance(String region, String name) {
return mapper.createObjectNode().set("instance", requireResource(region, RESOURCE_INSTANCE, name));
return mapper.createObjectNode().set("instance", instanceResponse(requireResource(region, RESOURCE_INSTANCE, name)));
}
public ObjectNode deleteInstance(String region, String name) {
requireResource(region, RESOURCE_INSTANCE, name);
ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
if (portForwardManager != null) {
portForwardManager.unpublishAll(instance);
}
try {
if (containerManager != null) {
containerManager.terminate(instance.path("containerId").asText(null),
instance.path("databaseContainerId").asText(null), instance.path("databaseVolume").asText(null),
instance.path("wordpressContentVolume").asText(null), instance.path("applicationVolume").asText(null));
}
} finally {
if (portForwardManager != null) {
portForwardManager.releaseSshPort(instance);
}
}
resourceStore.delete(resourceKey(region, RESOURCE_INSTANCE, name));
detachStaticIpsFrom(name);
detachDisksFrom(name);
@@ -116,21 +191,42 @@ public class LightsailService implements ResourceProvider {
public ObjectNode startInstance(String region, String name) {
ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
if (containerManager != null) {
containerManager.start(instance.path("containerId").asText(null),
instance.path("databaseContainerId").asText(null), instance.has("_sshHostPort"));
}
instance.set("state", instanceState("running"));
instance.put("runtimeStatus", instance.hasNonNull("containerId") ? "running" : "container-unavailable");
if (portForwardManager != null) {
portForwardManager.reconcile(instance);
}
resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
return operations(region, RESOURCE_INSTANCE, name, "StartInstance");
}
public ObjectNode stopInstance(String region, String name) {
ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
if (portForwardManager != null) {
portForwardManager.unpublishAll(instance);
}
if (containerManager != null) {
containerManager.stop(instance.path("containerId").asText(null),
instance.path("databaseContainerId").asText(null));
}
instance.set("state", instanceState("stopped"));
instance.put("runtimeStatus", instance.hasNonNull("containerId") ? "stopped" : "container-unavailable");
resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
return operations(region, RESOURCE_INSTANCE, name, "StopInstance");
}
public ObjectNode rebootInstance(String region, String name) {
ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
if (containerManager != null) {
containerManager.reboot(instance.path("containerId").asText(null),
instance.path("databaseContainerId").asText(null), instance.has("_sshHostPort"));
}
instance.set("state", instanceState("running"));
instance.put("runtimeStatus", instance.hasNonNull("containerId") ? "running" : "container-unavailable");
resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
return operations(region, RESOURCE_INSTANCE, name, "RebootInstance");
}
@@ -140,6 +236,135 @@ public class LightsailService implements ResourceProvider {
return mapper.createObjectNode().set("state", instance.path("state").deepCopy());
}
public List<Map<String, Object>> hostPortMappings(String region, String name) {
ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
List<Map<String, Object>> mappings = new ArrayList<>();
instance.path(LightsailPortForwardManager.PORT_FORWARDS_FIELD).properties().forEach(entry -> {
Map<String, Object> mapping = new LinkedHashMap<>();
mapping.put("instancePort", Integer.parseInt(entry.getKey()));
mapping.put("host", "localhost");
mapping.put("hostPort", entry.getValue().asInt());
mapping.put("protocol", "tcp");
mappings.add(mapping);
});
if (instance.has("_sshHostPort")) {
Map<String, Object> mapping = new LinkedHashMap<>();
mapping.put("instancePort", 22);
mapping.put("host", "localhost");
mapping.put("hostPort", instance.path("_sshHostPort").asInt());
mapping.put("protocol", "tcp");
mappings.add(mapping);
}
return mappings;
}
public int deployStaticFiles(String region, String name, List<LightsailContainerManager.StaticFile> files) {
ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
if (!"nginx".equals(instance.path("blueprintId").asText())) {
throw new AwsException("InvalidInputException", "Static file deployment is available only for Nginx Lightsail instances.", 400);
}
if (containerManager == null) {
throw new AwsException("InternalError", "Lightsail container support is unavailable.", 503);
}
try {
containerManager.deployStaticFiles(instance.path("containerId").asText(null), files);
return files.size();
} catch (IllegalArgumentException e) {
throw new AwsException("InvalidInputException", e.getMessage(), 400);
} catch (IllegalStateException e) {
throw new AwsException("InternalError", e.getMessage(), 502);
}
}
public LightsailContainerManager.NodeDeployment deployNodeApplication(String region, String name,
List<LightsailContainerManager.StaticFile> files,
String startCommand, int port) {
ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
if (!"nodejs".equals(instance.path("blueprintId").asText())) {
throw new AwsException("InvalidInputException", "Node.js deployment is available only for Node.js Lightsail instances.", 400);
}
if (containerManager == null) {
throw new AwsException("InternalError", "Lightsail container support is unavailable.", 503);
}
try {
LightsailContainerManager.NodeDeployment deployment = containerManager.deployNodeApplication(
instance.path("containerId").asText(null), files, startCommand, port);
instance.put("applicationPort", port);
instance.put("applicationRelease", deployment.releaseId());
((ObjectNode) instance.path("runtimeMetadata")).put("currentRelease", deployment.releaseId());
resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
return deployment;
} catch (IllegalArgumentException e) {
throw new AwsException("InvalidInputException", e.getMessage(), 400);
} catch (IllegalStateException e) {
throw new AwsException("InternalError", e.getMessage(), 502);
}
}
public ObjectNode createInstanceSnapshot(String region, JsonNode request) {
String instanceName = requireText(request, "instanceName");
String snapshotName = requireText(request, "instanceSnapshotName");
ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, instanceName);
ensureNotExists(region, RESOURCE_INSTANCE_SNAPSHOT, snapshotName);
ObjectNode snapshot = baseResource(region, instance.path("location").path("availabilityZone").asText(),
RESOURCE_INSTANCE_SNAPSHOT, snapshotName);
snapshot.put("fromInstanceName", instanceName);
snapshot.put("blueprintId", instance.path("blueprintId").asText());
snapshot.put("bundleId", instance.path("bundleId").asText());
snapshot.put("state", "available");
snapshot.set("tags", tags(request.path("tags")));
try {
String capturedImage = containerManager == null ? null
: containerManager.capture(instance.path("containerId").asText(null), snapshotName);
if (capturedImage != null) {
snapshot.put("containerImage", capturedImage);
}
} catch (IllegalStateException e) {
throw new AwsException("InternalError", e.getMessage(), 500);
}
resourceStore.put(resourceKey(region, RESOURCE_INSTANCE_SNAPSHOT, snapshotName), snapshot);
return operations(region, RESOURCE_INSTANCE_SNAPSHOT, snapshotName, "CreateInstanceSnapshot");
}
public ObjectNode getInstanceSnapshots(String region) {
ObjectNode response = mapper.createObjectNode();
ArrayNode snapshots = response.putArray("instanceSnapshots");
listResources(region, RESOURCE_INSTANCE_SNAPSHOT).stream()
.map(this::snapshotResponse)
.forEach(snapshots::add);
return response;
}
public ObjectNode getInstanceSnapshot(String region, String name) {
return mapper.createObjectNode().set("instanceSnapshot", snapshotResponse(
requireResource(region, RESOURCE_INSTANCE_SNAPSHOT, name)));
}
public ObjectNode deleteInstanceSnapshot(String region, String name) {
ObjectNode snapshot = requireResource(region, RESOURCE_INSTANCE_SNAPSHOT, name);
if (containerManager != null) {
containerManager.removeCapture(snapshot.path("containerImage").asText(null));
}
resourceStore.delete(resourceKey(region, RESOURCE_INSTANCE_SNAPSHOT, name));
return operations(region, RESOURCE_INSTANCE_SNAPSHOT, name, "DeleteInstanceSnapshot");
}
public ObjectNode createInstancesFromSnapshot(String region, JsonNode request) {
String snapshotName = requireText(request, "instanceSnapshotName");
ObjectNode snapshot = requireResource(region, RESOURCE_INSTANCE_SNAPSHOT, snapshotName);
ObjectNode createRequest = request.deepCopy();
createRequest.put("blueprintId", snapshot.path("blueprintId").asText());
createRequest.put("bundleId", snapshot.path("bundleId").asText());
if (snapshot.hasNonNull("containerImage")) {
createRequest.put("_containerImage", snapshot.path("containerImage").asText());
}
if (!createRequest.hasNonNull("availabilityZone")) {
createRequest.put("availabilityZone", snapshot.path("location").path("availabilityZone").asText());
}
return createInstances(region, createRequest);
}
public ObjectNode getInstancePortStates(String region, String name) {
ObjectNode instance = requireResource(region, RESOURCE_INSTANCE, name);
ObjectNode response = mapper.createObjectNode();
@@ -169,6 +394,9 @@ public class LightsailService implements ResourceProvider {
ports.add(portInfo.deepCopy());
}
((ObjectNode) instance.path("networking")).set("ports", ports);
if (portForwardManager != null) {
portForwardManager.reconcile(instance);
}
resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
return mapper.createObjectNode().set("operation", operation(region, RESOURCE_INSTANCE, name,
merge ? "OpenInstancePublicPorts" : "PutInstancePublicPorts"));
@@ -187,6 +415,9 @@ public class LightsailService implements ResourceProvider {
}
}
((ObjectNode) instance.path("networking")).set("ports", remaining);
if (portForwardManager != null) {
portForwardManager.reconcile(instance);
}
resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, name), instance);
return mapper.createObjectNode().set("operation", operation(region, RESOURCE_INSTANCE, name,
"CloseInstancePublicPorts"));
@@ -349,9 +580,11 @@ public class LightsailService implements ResourceProvider {
resourceStore.put(resourceKey(region, RESOURCE_KEY_PAIR, name), keyPair);
ObjectNode response = mapper.createObjectNode();
response.set("keyPair", keyPair.deepCopy());
ObjectNode keyPairResponse = keyPair.deepCopy();
keyPairResponse.remove("privateKeyBase64");
response.set("keyPair", keyPairResponse);
response.put("publicKeyBase64", keyPair.path("publicKeyBase64").asText());
response.put("privateKeyBase64", privateKeyBase64(name));
response.put("privateKeyBase64", keyPair.path("privateKeyBase64").asText());
response.set("operation", operation(region, RESOURCE_KEY_PAIR, name, "CreateKeyPair"));
return response;
}
@@ -374,8 +607,9 @@ public class LightsailService implements ResourceProvider {
return created;
});
ObjectNode response = mapper.createObjectNode();
response.put("publicKeyBase64", publicKeyBase64(name));
response.put("privateKeyBase64", privateKeyBase64(name));
ObjectNode keyPair = requireResource(region, RESOURCE_KEY_PAIR, name);
response.put("publicKeyBase64", keyPair.path("publicKeyBase64").asText());
response.put("privateKeyBase64", keyPair.path("privateKeyBase64").asText());
response.put("createdAt", epochSeconds(Instant.now()));
return response;
}
@@ -420,7 +654,9 @@ public class LightsailService implements ResourceProvider {
blueprints.add(blueprint("ubuntu_22_04", "Ubuntu", "linux", "Ubuntu 22.04 LTS"));
blueprints.add(blueprint("ubuntu_24_04", "Ubuntu", "linux", "Ubuntu 24.04 LTS"));
blueprints.add(blueprint("amazon_linux_2023", "Amazon Linux", "linux", "Amazon Linux 2023"));
blueprints.add(blueprint("nginx", "Nginx", "nginx", "Nginx web server"));
blueprints.add(blueprint("wordpress", "WordPress", "wordpress", "WordPress"));
blueprints.add(blueprint("nodejs", "Node.js", "nodejs", "Node.js current LTS application server"));
return response;
}
@@ -530,6 +766,7 @@ public class LightsailService implements ResourceProvider {
new SupportedResourceType(SERVICE + ":" + RESOURCE_INSTANCE, SERVICE, true),
new SupportedResourceType(SERVICE + ":" + RESOURCE_DISK, SERVICE, true),
new SupportedResourceType(SERVICE + ":" + RESOURCE_KEY_PAIR, SERVICE, true),
new SupportedResourceType(SERVICE + ":" + RESOURCE_INSTANCE_SNAPSHOT, SERVICE, true),
// Lightsail static IPs are not taggable in AWS.
new SupportedResourceType(SERVICE + ":" + RESOURCE_STATIC_IP, SERVICE, false));
}
@@ -562,6 +799,12 @@ public class LightsailService implements ResourceProvider {
return resource;
}
private ObjectNode snapshotResponse(ObjectNode snapshot) {
ObjectNode response = snapshot.deepCopy();
response.remove("containerImage");
return response;
}
private ObjectNode operation(String region, String resourceType, String resourceName, String operationType) {
Instant now = Instant.now();
String id = UUID.randomUUID().toString();
@@ -627,6 +870,21 @@ public class LightsailService implements ResourceProvider {
.toList();
}
private ObjectNode instanceResponse(ObjectNode instance) {
ObjectNode response = instance.deepCopy();
response.remove(List.of("containerId", "databaseContainerId", "databaseVolume", "wordpressContentVolume", "applicationVolume", "applicationRelease", "runtimeMetadata", "runtimeStatus", "runtimeImage", "applicationPort", "_sshHostPort",
LightsailPortForwardManager.PORT_FORWARDS_FIELD));
return response;
}
private void reconcileForwards(String region, ObjectNode instance) {
if (portForwardManager == null) {
return;
}
portForwardManager.reconcile(instance);
resourceStore.put(resourceKey(region, RESOURCE_INSTANCE, instance.path("name").asText()), instance);
}
private void ensureNotExists(String region, String resourceType, String name) {
if (resourceStore.get(resourceKey(region, resourceType, name)).isPresent()) {
throw new AwsException("InvalidInputException", resourceType + " " + name + " already exists", 400);
@@ -666,11 +924,44 @@ public class LightsailService implements ResourceProvider {
private ObjectNode keyPair(String region, String name, String publicKeyBase64, ArrayNode tags) {
ObjectNode keyPair = baseResource(region, region + "a", RESOURCE_KEY_PAIR, name);
keyPair.set("tags", tags);
keyPair.put("fingerprint", Base64.getEncoder().encodeToString(("fingerprint:" + name).getBytes(StandardCharsets.UTF_8)));
keyPair.put("publicKeyBase64", publicKeyBase64 == null ? publicKeyBase64(name) : publicKeyBase64);
if (publicKeyBase64 == null || publicKeyBase64.isBlank()) {
Ec2KeyMaterial.Generated generated = Ec2KeyMaterial.generateRsa();
keyPair.put("fingerprint", generated.fingerprint());
keyPair.put("publicKeyBase64", Base64.getEncoder().encodeToString(
generated.openSshPublicKey().getBytes(StandardCharsets.UTF_8)));
keyPair.put("privateKeyBase64", Base64.getEncoder().encodeToString(
generated.privateKeyPem().getBytes(StandardCharsets.UTF_8)));
} else {
String publicKey = decodePublicKey(publicKeyBase64);
String fingerprint = Ec2KeyMaterial.fingerprintOf(publicKey);
if (fingerprint == null) {
throw new AwsException("InvalidInputException", "publicKeyBase64 must contain a valid OpenSSH public key", 400);
}
keyPair.put("fingerprint", fingerprint);
keyPair.put("publicKeyBase64", publicKeyBase64);
}
return keyPair;
}
private ObjectNode selectedKeyPair(String region, String name) {
if ("LightsailDefaultKeyPair".equals(name)) {
return resourceStore.get(resourceKey(region, RESOURCE_KEY_PAIR, name)).orElseGet(() -> {
ObjectNode created = keyPair(region, name, null, mapper.createArrayNode());
resourceStore.put(resourceKey(region, RESOURCE_KEY_PAIR, name), created);
return created;
});
}
return requireResource(region, RESOURCE_KEY_PAIR, name);
}
private static String decodePublicKey(String publicKeyBase64) {
try {
return new String(Base64.getDecoder().decode(publicKeyBase64), StandardCharsets.UTF_8);
} catch (IllegalArgumentException e) {
throw new AwsException("InvalidInputException", "publicKeyBase64 must be base64 encoded", 400);
}
}
private ObjectNode regionNode(String name, String displayName, String continentCode, boolean includeAzs) {
ObjectNode region = mapper.createObjectNode();
region.put("continentCode", continentCode);
@@ -746,7 +1037,7 @@ public class LightsailService implements ResourceProvider {
return hardware;
}
private ObjectNode defaultNetworking() {
private ObjectNode defaultNetworking(Integer applicationPort) {
ObjectNode networking = mapper.createObjectNode();
ArrayNode ports = networking.putArray("ports");
ObjectNode ssh = mapper.createObjectNode();
@@ -757,6 +1048,16 @@ public class LightsailService implements ResourceProvider {
ssh.putArray("ipv6Cidrs");
ssh.putArray("cidrListAliases");
ports.add(ssh);
if (applicationPort != null) {
ObjectNode application = mapper.createObjectNode();
application.put("fromPort", applicationPort);
application.put("toPort", applicationPort);
application.put("protocol", "tcp");
application.putArray("cidrs").add("0.0.0.0/0");
application.putArray("ipv6Cidrs");
application.putArray("cidrListAliases");
ports.add(application);
}
networking.put("monthlyTransfer", mapper.createObjectNode()
.put("gbPerMonthAllocated", 1024));
return networking;
@@ -892,21 +1193,19 @@ public class LightsailService implements ResourceProvider {
case "ubuntu_24_04" -> "Ubuntu 24.04 LTS";
case "ubuntu_22_04" -> "Ubuntu 22.04 LTS";
case "amazon_linux_2023" -> "Amazon Linux 2023";
case "nginx" -> "Nginx";
case "wordpress" -> "WordPress";
case "nodejs" -> "Node.js";
default -> blueprintId;
};
}
private static String defaultUsername(String blueprintId) {
return blueprintId != null && blueprintId.startsWith("amazon_linux") ? "ec2-user" : "ubuntu";
return switch (blueprintId) {
case "amazon_linux_2023" -> "ec2-user";
case "nodejs" -> "node";
default -> "ubuntu";
};
}
private static String publicKeyBase64(String name) {
return Base64.getEncoder().encodeToString(("ssh-rsa FLOCI-" + name).getBytes(StandardCharsets.UTF_8));
}
private static String privateKeyBase64(String name) {
return Base64.getEncoder().encodeToString(("-----BEGIN PRIVATE KEY-----\nfloci-" + name
+ "\n-----END PRIVATE KEY-----\n").getBytes(StandardCharsets.UTF_8));
}
}
@@ -134,8 +134,11 @@ public final class AmiImageTool {
staged.creationDate = image.aws.creationDate;
staged.rootDeviceType = image.aws.rootDeviceType;
staged.virtualizationType = image.aws.virtualizationType;
staged.guestRuntime = image.guest.runtime;
staged.initType = image.guest.runtime;
staged.cloudInit = image.guest.cloudInit;
staged.sshUser = "root";
staged.sshPreinstalled = true;
staged.sshServiceName = "ssh";
staged.provenance = contextDir(outputRoot, image).resolve("provenance.yaml").toString();
if (write) {
YAML.writerWithDefaultPrettyPrinter().writeValue(catalogPath.toFile(), catalog);
@@ -442,8 +445,11 @@ public final class AmiImageTool {
public String rootDeviceType;
public String virtualizationType;
public String creationDate;
public String guestRuntime;
public String initType;
public Boolean cloudInit;
public String sshUser;
public Boolean sshPreinstalled;
public String sshServiceName;
public String provenance;
}
+49 -12
View File
@@ -9,6 +9,10 @@ images:
- /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs
- /aws/service/ami-amazon-linux-latest/amzn2-ami-kernel-5.10-hvm-x86_64-gp2
dockerImage: public.ecr.aws/amazonlinux/amazonlinux:2
sshUser: root
sshPreinstalled: false
initType: minimal
sshServiceName: sshd
name: amzn2-ami-hvm-2.0.20230404.0-x86_64-gp2
description: Amazon Linux 2 AMI
ownerId: "137112412989"
@@ -24,28 +28,40 @@ images:
- /aws/service/ami-amazon-linux-latest/al2023-ami-minimal-kernel-default-x86_64
- /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-x86_64
dockerImage: floci/ami-amazonlinux:2023
fallbackDockerImage: public.ecr.aws/amazonlinux/amazonlinux:2023
name: al2023-ami-2023.0.20230315.0-kernel-6.1-x86_64
description: Amazon Linux 2023 AMI
ownerId: "137112412989"
imageOwnerAlias: amazon
architecture: x86_64
creationDate: "2023-03-15T00:00:00.000Z"
guestRuntime: systemd
sshUser: root
sshPreinstalled: true
initType: systemd
sshServiceName: sshd
- imageId: ami-0abcdef1234567892
aliases:
- ami-ubuntu2004
dockerImage: floci/ami-ubuntu:20.04
fallbackDockerImage: public.ecr.aws/docker/library/ubuntu:20.04
name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230324
description: Canonical, Ubuntu, 20.04 LTS
ownerId: "099720109477"
imageOwnerAlias: canonical
architecture: x86_64
creationDate: "2023-03-24T00:00:00.000Z"
guestRuntime: systemd
sshUser: root
sshPreinstalled: true
initType: systemd
sshServiceName: ssh
- imageId: ami-ubuntu2204
dockerImage: public.ecr.aws/docker/library/ubuntu:22.04
sshUser: root
sshPreinstalled: false
initType: minimal
sshServiceName: ssh
name: ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20230420
description: Canonical, Ubuntu, 22.04 LTS
ownerId: "099720109477"
@@ -57,6 +73,10 @@ images:
aliases:
- ami-ubuntu2404
dockerImage: public.ecr.aws/docker/library/ubuntu:24.04
sshUser: root
sshPreinstalled: false
initType: minimal
sshServiceName: ssh
name: ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-server-20260515
description: Canonical, Ubuntu, 24.04 LTS
ownerId: "099720109477"
@@ -66,6 +86,10 @@ images:
- imageId: ami-ubuntu2404-amd64
dockerImage: public.ecr.aws/docker/library/ubuntu:24.04
sshUser: root
sshPreinstalled: false
initType: minimal
sshServiceName: ssh
name: ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-20260515
description: Canonical, Ubuntu, 24.04 LTS
ownerId: "099720109477"
@@ -77,6 +101,7 @@ images:
aliases:
- ami-ubuntu2404-cloud
dockerImage: floci/ami-ubuntu:24.04-arm64
fallbackDockerImage: public.ecr.aws/docker/library/ubuntu:24.04
name: ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-server-20260515
description: Canonical, Ubuntu, 24.04 LTS cloud image
ownerId: "099720109477"
@@ -85,27 +110,30 @@ images:
rootDeviceType: ebs
virtualizationType: hvm
creationDate: "2026-05-15T00:00:00.000Z"
guestRuntime: systemd
sshUser: root
sshPreinstalled: true
initType: systemd
sshServiceName: ssh
cloudInit: true
provenance: target/ami-images/ubuntu-24.04-arm64/provenance.yaml
- imageId: ami-debian12
dockerImage: public.ecr.aws/docker/library/debian:12
sshUser: root
sshPreinstalled: false
initType: minimal
sshServiceName: ssh
name: debian-12-amd64-20230411
description: Debian 12 AMI
architecture: x86_64
creationDate: "2023-04-11T00:00:00.000Z"
- imageId: ami-debian13
dockerImage: floci/ami-debian:13
name: debian-13-amd64
description: Debian 13 AMI with SSH
architecture: x86_64
creationDate: "2026-10-06T00:00:00.000Z"
guestRuntime: systemd
- imageId: ami-alpine
dockerImage: public.ecr.aws/docker/library/alpine:latest
sshUser: root
sshPreinstalled: false
initType: minimal
sshServiceName: sshd
name: alpine-linux-latest-x86_64
description: Alpine Linux AMI
architecture: x86_64
@@ -113,14 +141,23 @@ images:
- imageId: ami-fedora42
dockerImage: floci/ami-fedora:42
fallbackDockerImage: registry.fedoraproject.org/fedora:42
name: Fedora-Cloud-Base-42-x86_64
description: Fedora Linux 42 AMI
architecture: x86_64
creationDate: "2025-04-15T00:00:00.000Z"
guestRuntime: systemd
sshUser: root
sshPreinstalled: true
initType: systemd
sshServiceName: sshd
- imageId: ami-opensuse-leap156
dockerImage: floci/ami-opensuse:15.6
fallbackDockerImage: registry.opensuse.org/opensuse/leap:15.6
sshUser: root
sshPreinstalled: true
initType: systemd
sshServiceName: sshd
name: openSUSE-Leap-15.6-x86_64
description: openSUSE Leap 15.6 AMI
architecture: x86_64
@@ -34,6 +34,20 @@ class PortAllocatorTest {
allocator.release(second);
}
@Test
void reportsOccupiedPortsSkippedDuringAllocation() {
PortAllocator allocator = new PortAllocator();
int first = allocator.allocate(19900, 19900);
List<Integer> occupied = new ArrayList<>();
int second = allocator.allocate(19900, 19901, occupied::add);
assertEquals(19901, second);
assertEquals(List.of(19900), occupied);
allocator.release(first);
allocator.release(second);
}
@Test
void releasedPortBecomesAvailableAgain() {
PortAllocator allocator = new PortAllocator();
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.lifecycle;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.services.ec2.Ec2Service;
import io.github.hectorvent.floci.services.lightsail.LightsailService;
import io.github.hectorvent.floci.config.FlociCertificateAuthority;
import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.common.Resettable;
@@ -38,6 +39,7 @@ class EmulatorInfoControllerTest {
@Mock private FlociCertificateAuthority certificateAuthority;
@Mock private EmulatorConfig config;
@Mock private Ec2Service ec2Service;
@Mock private LightsailService lightsailService;
@Mock private ContainerTeardown sageMakerTeardown;
@Mock private ContainerTeardown batchTeardown;
@Mock private Resettable resettable;
@@ -47,7 +49,7 @@ class EmulatorInfoControllerTest {
@BeforeEach
void setUp() {
controller = new EmulatorInfoController(serviceRegistry, initLifecycleState, storageFactory,
resettables, containerTeardowns, certificateAuthority, config, ec2Service);
resettables, containerTeardowns, certificateAuthority, config, ec2Service, lightsailService);
}
@Test
@@ -115,6 +115,6 @@ class CloudWatchLogsMetricFilterResetTest {
when(resettables.iterator()).thenAnswer(call -> List.of(resettable).iterator());
Instance<ContainerTeardown> teardowns = mock(Instance.class);
when(teardowns.iterator()).thenAnswer(call -> List.<ContainerTeardown>of().iterator());
return new EmulatorInfoController(null, null, f.factory, resettables, teardowns, null, null, null);
return new EmulatorInfoController(null, null, f.factory, resettables, teardowns, null, null, null, null);
}
}
@@ -10,9 +10,12 @@ import com.github.dockerjava.api.command.InspectContainerCmd;
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.command.InspectExecCmd;
import com.github.dockerjava.api.command.InspectExecResponse;
import com.github.dockerjava.api.command.InspectImageCmd;
import com.github.dockerjava.api.command.InspectImageResponse;
import com.github.dockerjava.api.command.ListContainersCmd;
import com.github.dockerjava.api.command.StartContainerCmd;
import com.github.dockerjava.api.exception.DockerException;
import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.Container;
import com.github.dockerjava.api.model.ContainerConfig;
import com.github.dockerjava.api.model.ContainerNetwork;
@@ -56,8 +59,10 @@ import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BooleanSupplier;
import java.util.function.IntConsumer;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -69,6 +74,7 @@ import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.anySet;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Answers.RETURNS_SELF;
import static org.mockito.Answers.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.after;
@@ -97,6 +103,8 @@ class Ec2ContainerManagerTest {
void resetBridgeIpPolling() {
Ec2ContainerManager.containerBridgeIpAttempts = 30;
Ec2ContainerManager.containerBridgeIpPollMillis = 500;
Ec2ContainerManager.sshBannerAttempts = 30;
Ec2ContainerManager.sshBannerPollMillis = 1_000;
}
@Test
@@ -731,6 +739,8 @@ class Ec2ContainerManagerTest {
void launchReturnsTheLeaseWhenTheLaunchThrows() throws Exception {
LaunchHarness harness = launchHarness();
when(harness.builder.build()).thenThrow(new IllegalStateException("daemon went away"));
AtomicInteger persistenceCalls = new AtomicInteger();
harness.manager.setInstancePersister(ignored -> persistenceCalls.incrementAndGet());
Instance instance = leasedInstance("i-throws");
@@ -742,6 +752,13 @@ class Ec2ContainerManagerTest {
verify(harness.vpcNetworkManager, timeout(2000)).detach("us-west-2", "vpc-lease", null);
verify(harness.vpcNetworkManager, timeout(2000))
.releasePrivateIp("us-west-2", "subnet-lease", "10.0.1.10");
assertEquals("failed", instance.getLaunchStatus());
assertEquals("impaired", instance.getSshStatus());
assertEquals("daemon went away", instance.getStateReasonMessage());
assertTrue(instance.getLaunchEvents().stream()
.anyMatch(event -> "failed".equals(event.getStatus())
&& "daemon went away".equals(event.getMessage())));
assertTrue(persistenceCalls.get() >= instance.getLaunchEvents().size());
}
private static Instance leasedInstance(String instanceId) {
@@ -789,13 +806,14 @@ class Ec2ContainerManagerTest {
when(inspect.exec()).thenReturn(response);
Instance instance = instance("i-labeltest");
instance.setAccountId("123456789012");
harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");
verify(harness.builder, timeout(2000)).withLabels(Map.of(
"io.floci", "aws",
"io.floci.service", "ec2",
"io.floci.resource-id", "i-labeltest",
"io.floci.account", "000000000000",
"io.floci.account", "123456789012",
"io.floci.region", "us-west-2"));
}
@@ -968,7 +986,8 @@ class Ec2ContainerManagerTest {
Ec2ContainerManager.containerBridgeIpAttempts = 1;
Ec2ContainerManager.containerBridgeIpPollMillis = 1;
LaunchHarness harness = launchHarness();
when(harness.lifecycleManager.create(any(ContainerSpec.class), eq("linux/arm64")))
when(harness.lifecycleManager.create(any(ContainerSpec.class), eq("linux/arm64"),
any(ContainerLifecycleManager.ContainerCreationObserver.class)))
.thenReturn(TEST_CONTAINER_ID);
InspectContainerCmd inspect = mock(InspectContainerCmd.class);
InspectContainerResponse withIp = inspectResponse("172.18.0.11");
@@ -979,7 +998,7 @@ class Ec2ContainerManagerTest {
harness.manager.launch(instance,
new ResolvedAmiImage("floci/ami-ubuntu:24.04-arm64", ResolvedAmiImage.SYSTEMD_RUNTIME, true,
"linux/arm64"),
"linux/arm64", null, "enhanced", "root", true, "ssh"),
null,
"us-west-2");
@@ -987,7 +1006,49 @@ class Ec2ContainerManagerTest {
verify(harness.builder).withCmd(List.of("/sbin/init"));
verify(harness.builder).withCgroupnsMode("host");
verify(harness.builder).withBind("/sys/fs/cgroup", "/sys/fs/cgroup");
verify(harness.lifecycleManager).create(any(ContainerSpec.class), eq("linux/arm64"));
verify(harness.lifecycleManager).create(any(ContainerSpec.class), eq("linux/arm64"),
any(ContainerLifecycleManager.ContainerCreationObserver.class));
}
@Test
void legacyTailGuestIsRecognizedForSshBootstrapAfterReboot() {
assertTrue(Ec2ContainerManager.isLegacyTailGuest(new String[]{"tail", "-f", "/dev/null"}));
assertFalse(Ec2ContainerManager.isLegacyTailGuest(new String[]{"/sbin/init"}));
}
@Test
void missingLocalAmiUsesPullableMinimalFallback() {
LaunchHarness harness = launchHarness();
InspectImageCmd inspect = mock(InspectImageCmd.class);
when(harness.containerBuilder.resolveImage("floci/ami-ubuntu:20.04"))
.thenReturn("floci/ami-ubuntu:20.04");
when(harness.dockerClient.inspectImageCmd("floci/ami-ubuntu:20.04")).thenReturn(inspect);
when(inspect.exec()).thenThrow(new NotFoundException("missing"));
ResolvedAmiImage selected = harness.manager.selectLaunchImage(new ResolvedAmiImage(
"floci/ami-ubuntu:20.04", ResolvedAmiImage.SYSTEMD_RUNTIME, false,
"linux/amd64", "public.ecr.aws/docker/library/ubuntu:20.04",
"enhanced", "root", true, "ssh"));
assertEquals("public.ecr.aws/docker/library/ubuntu:20.04", selected.dockerImage());
assertFalse(selected.systemd());
assertEquals("linux/amd64", selected.dockerPlatform());
}
@Test
void availableLocalAmiKeepsItsSystemdRuntime() {
LaunchHarness harness = launchHarness();
InspectImageCmd inspect = mock(InspectImageCmd.class);
when(harness.containerBuilder.resolveImage("floci/ami-ubuntu:20.04"))
.thenReturn("floci/ami-ubuntu:20.04");
when(harness.dockerClient.inspectImageCmd("floci/ami-ubuntu:20.04")).thenReturn(inspect);
when(inspect.exec()).thenReturn(mock(InspectImageResponse.class));
ResolvedAmiImage image = new ResolvedAmiImage(
"floci/ami-ubuntu:20.04", ResolvedAmiImage.SYSTEMD_RUNTIME, false,
"linux/amd64", "public.ecr.aws/docker/library/ubuntu:20.04",
"enhanced", "root", true, "ssh");
assertEquals(image, harness.manager.selectLaunchImage(image));
}
@Test
@@ -1091,8 +1152,9 @@ class Ec2ContainerManagerTest {
Ec2ContainerManager.containerBridgeIpAttempts = 1;
Ec2ContainerManager.containerBridgeIpPollMillis = 1;
LaunchHarness harness = launchHarness();
when(harness.portAllocator.allocate(anyInt(), anyInt())).thenReturn(2201, 2202);
when(harness.lifecycleManager.create(any(ContainerSpec.class)))
when(harness.portAllocator.allocate(anyInt(), anyInt(), any(IntConsumer.class))).thenReturn(2201, 2202);
when(harness.lifecycleManager.create(any(ContainerSpec.class), nullable(String.class),
any(ContainerLifecycleManager.ContainerCreationObserver.class)))
.thenReturn("container-conflict", TEST_CONTAINER_ID);
when(harness.lifecycleManager.startCreated(anyString(), any(ContainerSpec.class)))
.thenThrow(new RuntimeException("Bind for 0.0.0.0:2201 failed: port is already allocated"))
@@ -1115,6 +1177,28 @@ class Ec2ContainerManagerTest {
verify(harness.builder).withPortBinding(22, 2202);
}
@Test
void launchRecordsOccupiedCandidatesWhenTheSshPortRangeIsExhausted() throws Exception {
LaunchHarness harness = launchHarness();
when(harness.portAllocator.allocate(anyInt(), anyInt(), any(IntConsumer.class)))
.thenAnswer(invocation -> {
IntConsumer observer = invocation.getArgument(2);
observer.accept(2200);
observer.accept(2201);
throw new RuntimeException("No free port available in range 2200-2201");
});
Instance instance = instance("i-ports-exhausted");
harness.manager.launch(instance, "ubuntu:24.04", null, "us-west-2");
awaitUntil(() -> "terminated".equals(instance.getState().getName()), Duration.ofSeconds(2));
assertTrue(instance.getLaunchEvents().stream().anyMatch(event -> "retrying".equals(event.getStatus())
&& event.getMessage().contains("2200")));
assertTrue(instance.getLaunchEvents().stream().anyMatch(event -> "retrying".equals(event.getStatus())
&& event.getMessage().contains("2201")));
assertEquals("No free port available in range 2200-2201", instance.getStateReasonMessage());
}
@Test
void launchReleasesPortAndCleansUpContainerAfterNonRetryableStartFailure() throws Exception {
LaunchHarness harness = launchHarness();
@@ -1172,7 +1256,8 @@ class Ec2ContainerManagerTest {
LaunchHarness harness = launchHarness();
CountDownLatch createEntered = new CountDownLatch(1);
CountDownLatch allowCreate = new CountDownLatch(1);
when(harness.lifecycleManager.create(any(ContainerSpec.class))).thenAnswer(invocation -> {
when(harness.lifecycleManager.create(any(ContainerSpec.class), nullable(String.class),
any(ContainerLifecycleManager.ContainerCreationObserver.class))).thenAnswer(invocation -> {
createEntered.countDown();
assertTrue(allowCreate.await(2, TimeUnit.SECONDS));
return TEST_CONTAINER_ID;
@@ -1247,7 +1332,8 @@ class Ec2ContainerManagerTest {
ExecutorService callerExecutor = Executors.newSingleThreadExecutor();
CountDownLatch createEntered = new CountDownLatch(1);
CountDownLatch releaseCreate = new CountDownLatch(1);
when(harness.lifecycleManager().create(any(ContainerSpec.class))).thenAnswer(invocation -> {
when(harness.lifecycleManager().create(any(ContainerSpec.class), nullable(String.class),
any(ContainerLifecycleManager.ContainerCreationObserver.class))).thenAnswer(invocation -> {
createEntered.countDown();
releaseCreate.await(2, TimeUnit.SECONDS);
throw new RuntimeException("test launch failure");
@@ -1368,17 +1454,7 @@ class Ec2ContainerManagerTest {
verify(harness.dockerClient, never()).copyArchiveToContainerCmd(TEST_CONTAINER_ID);
}
private static final String[] SSHD_INSTALL_PROBE_CMD = {"sh", "-c",
"if ! command -v sshd >/dev/null 2>&1 || ! command -v scp >/dev/null 2>&1; then"
+ " if command -v dnf >/dev/null 2>&1; then dnf install -y openssh-server openssh-clients >/dev/null 2>&1;"
+ " elif command -v yum >/dev/null 2>&1; then yum install -y openssh-server openssh-clients >/dev/null 2>&1;"
+ " elif command -v apt-get >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server openssh-client >/dev/null 2>&1;"
+ " elif command -v zypper >/dev/null 2>&1; then zypper --non-interactive install -y openssh >/dev/null 2>&1;"
+ " elif command -v apk >/dev/null 2>&1; then apk add --no-cache openssh >/dev/null 2>&1;"
+ " fi;"
+ "fi;"
+ "command -v sshd >/dev/null 2>&1 || exit 1;"
+ "command -v scp >/dev/null 2>&1 || exit 2"};
private static final String[] SSHD_INSTALL_PROBE_CMD = Ec2ContainerManager.sshdInstallProbeCommand();
@Test
void launchWhenSshdInstallFails_doesNotAttemptKeygenOrStart() throws Exception {
@@ -1477,6 +1553,8 @@ class Ec2ContainerManagerTest {
}
private static LaunchHarness launchHarness(ExecutorService executor, Duration userDataTimeout) {
Ec2ContainerManager.sshBannerAttempts = 1;
Ec2ContainerManager.sshBannerPollMillis = 1;
ContainerBuilder containerBuilder = mock(ContainerBuilder.class);
ContainerBuilder.Builder builder = mock(ContainerBuilder.Builder.class, withSettings().defaultAnswer(RETURNS_SELF));
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
@@ -1484,6 +1562,13 @@ class Ec2ContainerManagerTest {
ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.create(any(ContainerSpec.class))).thenReturn(TEST_CONTAINER_ID);
when(lifecycleManager.create(any(ContainerSpec.class), nullable(String.class),
any(ContainerLifecycleManager.ContainerCreationObserver.class))).thenAnswer(invocation -> {
ContainerLifecycleManager.ContainerCreationObserver observer = invocation.getArgument(2);
observer.containerCreationStarted();
observer.containerCreated(TEST_CONTAINER_ID);
return TEST_CONTAINER_ID;
});
when(lifecycleManager.isContainerRunning(TEST_CONTAINER_ID)).thenReturn(true);
DockerHostResolver dockerHostResolver = mock(DockerHostResolver.class);
@@ -1491,7 +1576,7 @@ class Ec2ContainerManagerTest {
ContainerReachableEndpoint reachableEndpoint = mock(ContainerReachableEndpoint.class);
when(reachableEndpoint.baseUrl()).thenReturn("http://localhost.floci.io:4680");
PortAllocator portAllocator = mock(PortAllocator.class);
when(portAllocator.allocate(anyInt(), anyInt())).thenReturn(2201);
when(portAllocator.allocate(anyInt(), anyInt(), any(IntConsumer.class))).thenReturn(2201);
EmulatorConfig config = mock(EmulatorConfig.class);
EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);
@@ -1543,7 +1628,8 @@ class Ec2ContainerManagerTest {
reachableEndpoint,
executor,
userDataTimeout);
return new LaunchHarness(manager, lifecycleManager, dockerClient, metadataServer, logStreamer, builder,
return new LaunchHarness(manager, lifecycleManager, dockerClient, metadataServer, logStreamer,
containerBuilder, builder,
portAllocator, portForwardManager, config, vpcNetworkManager, new CopyOnWriteArrayList<>());
}
@@ -1771,12 +1857,19 @@ class Ec2ContainerManagerTest {
void launchRunsInstanceAsMetadataOnlyWhenNoDockerDaemonIsReachable() throws Exception {
LaunchHarness harness = launchHarness();
stubDockerPing(harness.dockerClient(), false);
AtomicInteger persistenceCalls = new AtomicInteger();
harness.manager.setInstancePersister(ignored -> persistenceCalls.incrementAndGet());
Instance instance = instance("i-nodocker");
harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");
assertEquals("running", instance.getState().getName());
assertEquals("failed", instance.getLaunchStatus());
assertEquals("impaired", instance.getSshStatus());
assertTrue(instance.getSshStatusMessage().contains("Docker daemon is unavailable"));
assertEquals("creating-container", instance.getLaunchEvents().getLast().getStage());
assertTrue(persistenceCalls.get() >= 2);
assertNull(instance.getDockerContainerId());
verify(harness.lifecycleManager(), never()).create(any(ContainerSpec.class));
verify(harness.metadataServer(), never()).registerContainer(anyString(), anyString(), any());
@@ -1788,7 +1881,8 @@ class Ec2ContainerManagerTest {
PingCmd ping = mock(PingCmd.class);
when(harness.dockerClient().pingCmd()).thenReturn(ping);
doNothing().doThrow(new RuntimeException("No such file or directory")).when(ping).exec();
when(harness.lifecycleManager().create(any(ContainerSpec.class)))
when(harness.lifecycleManager().create(any(ContainerSpec.class), nullable(String.class),
any(ContainerLifecycleManager.ContainerCreationObserver.class)))
.thenThrow(new RuntimeException("No such file or directory"));
Instance instance = instance("i-dockergone");
@@ -1846,9 +1940,10 @@ class Ec2ContainerManagerTest {
private record LaunchHarness(Ec2ContainerManager manager,
ContainerLifecycleManager lifecycleManager,
DockerClient dockerClient,
Ec2MetadataServer metadataServer,
ContainerLogStreamer logStreamer,
ContainerBuilder.Builder builder,
Ec2MetadataServer metadataServer,
ContainerLogStreamer logStreamer,
ContainerBuilder containerBuilder,
ContainerBuilder.Builder builder,
PortAllocator portAllocator,
Ec2PortForwardManager portForwardManager,
EmulatorConfig config,
@@ -37,7 +37,6 @@ class Ec2ImageCatalogTest {
"ami-ubuntu2404-amd64",
"ami-ubuntu2404-cloud-arm64",
"ami-debian12",
"ami-debian13",
"ami-alpine",
"ami-fedora42",
"ami-opensuse-leap156"), imageIds);
@@ -67,9 +66,21 @@ class Ec2ImageCatalogTest {
ResolvedAmiImage image = amiImageResolver.resolveImage("ami-amazonlinux2023");
assertEquals("floci/ami-amazonlinux:2023", image.dockerImage());
assertEquals(ResolvedAmiImage.SYSTEMD_RUNTIME, image.guestRuntime());
assertEquals(ResolvedAmiImage.SYSTEMD_RUNTIME, image.initType());
assertEquals("root", image.sshUser());
assertTrue(image.sshPreinstalled());
assertEquals("sshd", image.sshServiceName());
assertFalse(image.cloudInit());
assertTrue(image.systemd());
assertEquals("public.ecr.aws/amazonlinux/amazonlinux:2023", image.fallbackDockerImage());
}
@Test
void everyLocalCatalogImageHasAPullableFallback() {
imageCatalog.images().stream()
.filter(image -> image.dockerImage.startsWith("floci/"))
.forEach(image -> assertTrue(image.fallbackDockerImage != null
&& !image.fallbackDockerImage.isBlank(), image.imageId));
}
@Test
@@ -81,11 +92,9 @@ class Ec2ImageCatalogTest {
@Test
void resolverUsesFedoraAndLocalSshReadyImages() {
assertEquals("floci/ami-ubuntu:20.04", amiImageResolver.resolve("ami-0abcdef1234567892"));
assertEquals("floci/ami-debian:13", amiImageResolver.resolve("ami-debian13"));
assertEquals("floci/ami-fedora:42", amiImageResolver.resolve("ami-fedora42"));
assertEquals("floci/ami-opensuse:15.6", amiImageResolver.resolve("ami-opensuse-leap156"));
assertTrue(amiImageResolver.resolveImage("ami-0abcdef1234567892").systemd());
assertTrue(amiImageResolver.resolveImage("ami-debian13").systemd());
assertTrue(amiImageResolver.resolveImage("ami-fedora42").systemd());
}
@@ -2215,7 +2215,7 @@ class Ec2IntegrationTest {
.formParam("LaunchTemplateName", "sample-template")
.formParam("LaunchTemplateData.ImageId", "ami-0abcdef1234567890")
.formParam("LaunchTemplateData.InstanceType", "t3.micro")
.formParam("LaunchTemplateData.KeyName", "test-key")
.formParam("LaunchTemplateData.KeyName", keyPairName)
.formParam("LaunchTemplateData.IamInstanceProfile.Name", "sample-profile")
.formParam("LaunchTemplateData.SecurityGroupId.1", securityGroupId)
.formParam("LaunchTemplateData.UserData", launchTemplateUserData)
@@ -2328,7 +2328,7 @@ class Ec2IntegrationTest {
.formParam("VersionDescription", "updated by test")
.formParam("LaunchTemplateData.ImageId", "ami-0abcdef1234567890")
.formParam("LaunchTemplateData.InstanceType", "t3.small")
.formParam("LaunchTemplateData.KeyName", "test-key")
.formParam("LaunchTemplateData.KeyName", keyPairName)
.formParam("LaunchTemplateData.IamInstanceProfile.Name", "sample-profile-v2")
.formParam("LaunchTemplateData.SecurityGroupId.1", securityGroupId)
.formParam("LaunchTemplateData.UserData", launchTemplateVersionUserData)
@@ -2468,7 +2468,7 @@ class Ec2IntegrationTest {
.statusCode(200)
.body("RunInstancesResponse.instancesSet.item.imageId", equalTo("ami-sample-override"))
.body("RunInstancesResponse.instancesSet.item.instanceType", equalTo("c7g.large"))
.body("RunInstancesResponse.instancesSet.item.keyName", equalTo("test-key"))
.body("RunInstancesResponse.instancesSet.item.keyName", equalTo(keyPairName))
.body("RunInstancesResponse.instancesSet.item.iamInstanceProfile.arn",
equalTo("arn:aws:iam::000000000000:instance-profile/sample-profile-v2"))
.body("RunInstancesResponse.instancesSet.item.groupSet.item.groupId", equalTo(securityGroupId))
@@ -2861,7 +2861,7 @@ class Ec2IntegrationTest {
.formParam("InstanceType", "t2.micro")
.formParam("MinCount", "1")
.formParam("MaxCount", "1")
.formParam("KeyName", "test-key")
.formParam("KeyName", keyPairName)
.formParam("SubnetId", subnetId)
.formParam("SecurityGroupId.1", securityGroupId)
.header("Authorization", AUTH_HEADER)
@@ -2872,7 +2872,7 @@ class Ec2IntegrationTest {
.body("RunInstancesResponse.instancesSet.item.instanceId", startsWith("i-"))
.body("RunInstancesResponse.instancesSet.item.instanceState.name", equalTo("pending"))
.body("RunInstancesResponse.instancesSet.item.instanceType", equalTo("t2.micro"))
.body("RunInstancesResponse.instancesSet.item.keyName", equalTo("test-key"))
.body("RunInstancesResponse.instancesSet.item.keyName", equalTo(keyPairName))
.extract().path("RunInstancesResponse.instancesSet.item.instanceId");
}
@@ -3712,7 +3712,7 @@ class Ec2IntegrationTest {
.formParam("InstanceType", "t2.micro")
.formParam("MinCount", "5")
.formParam("MaxCount", "5")
.formParam("KeyName", "test-key")
.formParam("KeyName", keyPairName)
.formParam("SubnetId", subnetId)
.formParam("SecurityGroupId.1", securityGroupId)
.header("Authorization", AUTH_HEADER)
@@ -1510,6 +1510,31 @@ class Ec2ServiceTest {
assertNotEquals(first.getKeyMaterial(), second.getKeyMaterial());
assertNotEquals(first.getPublicKey(), second.getPublicKey());
assertNotEquals(first.getKeyFingerprint(), second.getKeyFingerprint());
KeyPair persisted = service.describeKeyPairs("us-east-1", List.of("usable-key"), List.of()).getFirst();
assertNull(persisted.getKeyMaterial());
assertEquals(first.getPublicKey(), persisted.getPublicKey());
}
@Test
void startupClearsLegacyPersistedPrivateKeyMaterialAcrossAccounts() {
AccountAwareStorageBackend<KeyPair> keyPairStore = AccountAwareStorageBackend.inMemory("000000000000");
KeyPair legacy = new KeyPair();
legacy.setKeyPairId("key-legacy");
legacy.setKeyName("legacy-key");
legacy.setKeyMaterial("-----BEGIN RSA PRIVATE KEY-----\nsecret\n-----END RSA PRIVATE KEY-----");
legacy.setPublicKey("ssh-rsa public");
legacy.setRegion("us-east-1");
keyPairStore.putForAccount("111111111111", "us-east-1::key-legacy", legacy);
Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),
mock(Ec2PortForwardManager.class),
mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class), new Ec2InstanceTypeCatalog(),
new InMemoryStorageFactory(Map.of("ec2-key-pairs.json", keyPairStore)));
service.restoreMetadataRegistrations();
KeyPair restored = keyPairStore.getForAccount("111111111111", "us-east-1::key-legacy").orElseThrow();
assertNull(restored.getKeyMaterial());
assertEquals("ssh-rsa public", restored.getPublicKey());
}
@Test
@@ -1856,7 +1881,7 @@ class Ec2ServiceTest {
AmiImageResolver resolver = mock(AmiImageResolver.class);
when(resolver.resolveImage("ami-arm-source"))
.thenReturn(new ResolvedAmiImage("arm-image", ResolvedAmiImage.DEFAULT_RUNTIME, false,
"linux/arm64"));
"linux/arm64", null, "public", "root", false, "sshd"));
Ec2Service service = liveService(capturingContainerManager(), resolver, catalog);
Reservation sourceReservation = service.runInstances("us-east-1", "ami-arm-source", "t4g.medium",
1, 1, null, List.of(), null, null, List.of(), null, null);
@@ -2065,6 +2090,135 @@ class Ec2ServiceTest {
verify(vpcNetworks, times(2)).reservePrivateIp("us-east-1", "subnet-a", "10.0.1.10");
}
@Test
void restorePromotesAPendingInstanceWhoseContainerIsAlreadyRunning() {
AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
Instance instance = persistedInstance("i-pending", "10.0.1.10", InstanceState.pending());
instance.setDockerContainerId("container-id");
instanceStore.put("us-east-1::i-pending", instance);
Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);
when(containerManager.restoreMetadataRegistration(instance)).thenReturn(true);
VpcNetworkManager vpcNetworks = mock(VpcNetworkManager.class);
when(vpcNetworks.enabled()).thenReturn(true);
Ec2Service service = new Ec2Service(mockConfig(false), containerManager,
mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
new Ec2InstanceTypeCatalog(),
new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), vpcNetworks);
service.restoreMetadataRegistrations();
assertEquals("running", instanceStore.get("us-east-1::i-pending").orElseThrow().getState().getName());
}
@Test
void restoreFailsAPendingInstanceWithoutARecordedContainer() {
AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
Instance instance = persistedInstance("i-pending", "10.0.1.10", InstanceState.pending());
instanceStore.put("us-east-1::i-pending", instance);
Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),
mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
new Ec2InstanceTypeCatalog(),
new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)),
mock(VpcNetworkManager.class));
service.restoreMetadataRegistrations();
Instance restored = instanceStore.get("us-east-1::i-pending").orElseThrow();
assertEquals("terminated", restored.getState().getName());
assertEquals("failed", restored.getLaunchStatus());
assertEquals("impaired", restored.getSshStatus());
assertEquals("startup-reconciliation", restored.getLaunchEvents().getLast().getStage());
assertEquals("failed", restored.getLaunchEvents().getLast().getStatus());
}
@Test
void restoreFailsPendingInstancesAcrossAccountPartitions() {
AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
Instance instance = persistedInstance("i-other-account", "10.0.1.10", InstanceState.pending());
instance.setAccountId("111111111111");
instanceStore.putForAccount("111111111111", "us-east-1::i-other-account", instance);
Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),
mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
new Ec2InstanceTypeCatalog(),
new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)),
mock(VpcNetworkManager.class));
service.restoreMetadataRegistrations();
Instance restored = instanceStore.getForAccount(
"111111111111", "us-east-1::i-other-account").orElseThrow();
assertEquals("terminated", restored.getState().getName());
assertEquals("failed", restored.getLaunchStatus());
assertTrue(instanceStore.getForAccount(
"000000000000", "us-east-1::i-other-account").isEmpty());
}
@Test
void restoreFailsARunningInstanceWhoseLaunchWasInterrupted() {
AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
Instance instance = persistedInstance("i-interrupted", "10.0.1.10", InstanceState.running());
instance.setLaunchStatus("in-progress");
instanceStore.put("us-east-1::i-interrupted", instance);
Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),
mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
new Ec2InstanceTypeCatalog(),
new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)),
mock(VpcNetworkManager.class));
service.restoreMetadataRegistrations();
Instance restored = instanceStore.get("us-east-1::i-interrupted").orElseThrow();
assertEquals("terminated", restored.getState().getName());
assertEquals("failed", restored.getLaunchStatus());
assertEquals("startup-reconciliation", restored.getLaunchEvents().getLast().getStage());
}
@Test
void describeInstanceStatusIncludesFailedLaunchOnlyWhenRequested() {
AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
Instance instance = persistedInstance("i-failed", "10.0.1.10", InstanceState.terminated());
instance.setLaunchStatus("failed");
instance.setSshStatus("impaired");
instance.setSshStatusMessage("No free port available in range 2200-2209");
instanceStore.put("us-east-1::i-failed", instance);
Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),
mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
new Ec2InstanceTypeCatalog(),
new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)));
assertTrue(service.describeInstanceStatus("us-east-1", List.of("i-failed"), false).isEmpty());
List<Instance> statuses = service.describeInstanceStatus("us-east-1", List.of("i-failed"), true);
assertEquals(1, statuses.size());
assertEquals("impaired", statuses.getFirst().getSshStatus());
assertEquals("No free port available in range 2200-2209", statuses.getFirst().getSshStatusMessage());
}
@Test
void describeInstanceStatusClearsRecoveredSshLaunchFailure() {
AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
Instance instance = persistedInstance("i-recovered", "10.0.1.10", InstanceState.running());
instance.setLaunchStatus("failed");
instance.setSshStatus("impaired");
instance.setStateReasonCode("Client.SSHUnavailable");
instance.setStateReasonMessage("connection refused");
instanceStore.put("us-east-1::i-recovered", instance);
Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);
when(containerManager.checkSshReadiness(instance))
.thenReturn(new Ec2ContainerManager.OperationResult(true, "SSH banner received: SSH-2.0-test"));
Ec2Service service = new Ec2Service(mockConfig(false), containerManager,
mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
new Ec2InstanceTypeCatalog(),
new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)));
Instance status = service.describeInstanceStatus("us-east-1", List.of("i-recovered"), false).getFirst();
assertEquals("ready", status.getLaunchStatus());
assertEquals("ok", status.getSshStatus());
assertNull(status.getStateReasonCode());
assertNull(status.getStateReasonMessage());
}
private static Instance persistedInstance(String instanceId, String privateIp, InstanceState state) {
Instance instance = new Instance();
instance.setInstanceId(instanceId);
@@ -0,0 +1,47 @@
package io.github.hectorvent.floci.services.lightsail;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import org.junit.jupiter.api.Test;
import org.mockito.InOrder;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
class LightsailContainerManagerTest {
@Test
void deploymentPathsAreRelativeAndTraversalFree() {
assertDoesNotThrow(() -> LightsailContainerManager.validateDeploymentPath("assets/site.css"));
assertThrows(IllegalArgumentException.class, () -> LightsailContainerManager.validateDeploymentPath("../index.html"));
assertThrows(IllegalArgumentException.class, () -> LightsailContainerManager.validateDeploymentPath("assets/../../index.html"));
assertThrows(IllegalArgumentException.class, () -> LightsailContainerManager.validateDeploymentPath("/etc/passwd"));
assertThrows(IllegalArgumentException.class, () -> LightsailContainerManager.validateDeploymentPath("assets\\site.css"));
assertThrows(IllegalArgumentException.class, () -> LightsailContainerManager.validateDeploymentPath("assets//site.css"));
}
@Test
void nodeDeploymentPathsRejectReservedFilesAndSupervisorUsesReleaseCommand() {
assertDoesNotThrow(() -> LightsailContainerManager.validateNodePath("src/server.js"));
assertThrows(IllegalArgumentException.class, () -> LightsailContainerManager.validateNodePath(".floci-start"));
assertThrows(IllegalArgumentException.class, () -> LightsailContainerManager.validateNodePath("../package.json"));
String command = String.join(" ", LightsailContainerManager.nodeSupervisorCommand());
org.junit.jupiter.api.Assertions.assertTrue(command.contains(".floci-start"));
org.junit.jupiter.api.Assertions.assertTrue(command.contains("HOST=0.0.0.0"));
}
@Test
void wordpressTerminationStrictlyRemovesContainersAndBothVolumes() {
ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
LightsailContainerManager manager = new LightsailContainerManager(null, lifecycleManager, null, null, null);
manager.terminate("wordpress", "mariadb", "mariadb-data", "wordpress-content");
InOrder order = inOrder(lifecycleManager);
order.verify(lifecycleManager).stopAndRemoveStrict("wordpress", null);
order.verify(lifecycleManager).stopAndRemoveStrict("mariadb", null);
order.verify(lifecycleManager).removeVolumeStrict("mariadb-data");
order.verify(lifecycleManager).removeVolumeStrict("wordpress-content");
}
}
@@ -2,9 +2,13 @@ package io.github.hectorvent.floci.services.lightsail;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.response.Response;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.util.List;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
@@ -24,6 +28,29 @@ class LightsailIntegrationTest {
RestAssuredJsonUtils.configureAwsContentTypes();
}
@AfterEach
void deleteTestInstanceContainers() {
Response instances = given()
.header("X-Amz-Target", TARGET_PREFIX + "GetInstances")
.contentType(CONTENT_TYPE)
.body("{}")
.when()
.post("/");
List<String> instanceNames = instances.jsonPath().getList("instances.name", String.class);
for (String instanceName : instanceNames) {
given()
.header("X-Amz-Target", TARGET_PREFIX + "DeleteInstance")
.contentType(CONTENT_TYPE)
.body("""
{"instanceName":"%s"}
""".formatted(instanceName))
.when()
.post("/")
.then()
.statusCode(200);
}
}
@Test
void instanceDiskAndStaticIpLifecycleRoundTripsThroughJsonHandler() {
given()
@@ -34,7 +61,7 @@ class LightsailIntegrationTest {
.post("/")
.then()
.statusCode(200)
.body("blueprints", hasSize(4))
.body("blueprints", hasSize(6))
.body("blueprints[0].blueprintId", equalTo("ubuntu_22_04"));
given()
@@ -198,6 +225,44 @@ class LightsailIntegrationTest {
.body("operations[0].id", notNullValue());
}
@Test
void publicPortRulesRemainAwsVisibleWithoutInternalForwardMappings() {
given()
.header("X-Amz-Target", TARGET_PREFIX + "CreateInstances")
.contentType(CONTENT_TYPE)
.body("""
{"instanceNames":["port-web"],"availabilityZone":"us-east-1a","blueprintId":"ubuntu_22_04","bundleId":"nano_3_0"}
""")
.when()
.post("/")
.then()
.statusCode(200);
given()
.header("X-Amz-Target", TARGET_PREFIX + "OpenInstancePublicPorts")
.contentType(CONTENT_TYPE)
.body("""
{"instanceName":"port-web","portInfo":{"fromPort":8080,"toPort":8080,"protocol":"tcp","cidrs":["0.0.0.0/0"]}}
""")
.when()
.post("/")
.then()
.statusCode(200);
given()
.header("X-Amz-Target", TARGET_PREFIX + "GetInstance")
.contentType(CONTENT_TYPE)
.body("""
{"instanceName":"port-web"}
""")
.when()
.post("/")
.then()
.statusCode(200)
.body("instance.networking.ports.fromPort", hasItem(8080))
.body("instance._portForwards", equalTo(null));
}
@Test
void keyPairsTagsRegionsAndUnsupportedOperationsUseAwsJsonShape() {
given()
@@ -303,6 +368,69 @@ class LightsailIntegrationTest {
.body("message", equalTo("keyPairName is required"));
}
@Test
void instanceSnapshotsCanBeCreatedRestoredAndDeleted() {
given()
.header("X-Amz-Target", TARGET_PREFIX + "CreateInstances")
.contentType(CONTENT_TYPE)
.body("""
{"instanceNames":["snapshot-source"],"availabilityZone":"us-east-1a","blueprintId":"ubuntu_22_04","bundleId":"nano_3_0"}
""")
.when()
.post("/")
.then()
.statusCode(200);
given()
.header("X-Amz-Target", TARGET_PREFIX + "CreateInstanceSnapshot")
.contentType(CONTENT_TYPE)
.body("""
{"instanceName":"snapshot-source","instanceSnapshotName":"snapshot-one"}
""")
.when()
.post("/")
.then()
.statusCode(200)
.body("operations[0].operationType", equalTo("CreateInstanceSnapshot"));
given()
.header("X-Amz-Target", TARGET_PREFIX + "GetInstanceSnapshot")
.contentType(CONTENT_TYPE)
.body("""
{"instanceSnapshotName":"snapshot-one"}
""")
.when()
.post("/")
.then()
.statusCode(200)
.body("instanceSnapshot.fromInstanceName", equalTo("snapshot-source"))
.body("instanceSnapshot.state", equalTo("available"));
given()
.header("X-Amz-Target", TARGET_PREFIX + "CreateInstancesFromSnapshot")
.contentType(CONTENT_TYPE)
.body("""
{"instanceSnapshotName":"snapshot-one","instanceNames":["snapshot-restored"]}
""")
.when()
.post("/")
.then()
.statusCode(200)
.body("operations[0].operationType", equalTo("CreateInstance"));
given()
.header("X-Amz-Target", TARGET_PREFIX + "DeleteInstanceSnapshot")
.contentType(CONTENT_TYPE)
.body("""
{"instanceSnapshotName":"snapshot-one"}
""")
.when()
.post("/")
.then()
.statusCode(200)
.body("operations[0].operationType", equalTo("DeleteInstanceSnapshot"));
}
@Test
void diskStaticIpAndHardwareEdgeCasesMatchLightsailBehavior() {
given()
@@ -0,0 +1,92 @@
package io.github.hectorvent.floci.services.lightsail;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class LightsailNodeJsTest {
private static final String REGION = "us-east-1";
private final ObjectMapper mapper = new ObjectMapper();
private LightsailService service;
private LightsailContainerManager containerManager;
@BeforeEach
void setUp() {
StorageFactory storageFactory = new StorageFactory(null, null) {
@Override
public synchronized <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
TypeReference<Map<String, V>> typeReference) {
return AccountAwareStorageBackend.inMemory("000000000000");
}
};
containerManager = mock(LightsailContainerManager.class);
service = new LightsailService(storageFactory, mapper, new RegionResolver(REGION, "000000000000"),
containerManager, null);
}
@Test
void nodeBlueprintUsesCurrentLtsRuntimeAndDoesNotExposeRuntimeMetadata() {
LightsailContainerManager.Runtime runtime = LightsailContainerManager.runtimeFor("nodejs");
assertEquals("node:24-bookworm", runtime.image());
assertEquals("node", runtime.sshUser());
assertEquals(3000, runtime.applicationPort());
assertEquals("nodejs", service.getBlueprints().path("blueprints").get(5).path("blueprintId").asText());
when(containerManager.launch(eq("node-app"), eq(REGION), eq("nodejs"), eq("nano_3_0"), isNull(), anyString()))
.thenReturn(new LightsailContainerManager.Launch("node-container", null, null, null, 22025,
"node-app-volume", "1760000000000"));
ObjectNode request = mapper.createObjectNode()
.put("availabilityZone", REGION + "a")
.put("blueprintId", "nodejs")
.put("bundleId", "nano_3_0");
request.set("instanceNames", mapper.createArrayNode().add("node-app"));
service.createInstances(REGION, request);
ObjectNode instance = (ObjectNode) service.getInstance(REGION, "node-app").path("instance");
assertEquals("nodejs", instance.path("blueprintId").asText());
assertEquals("node", instance.path("username").asText());
assertEquals(3000, instance.path("networking").path("ports").get(1).path("fromPort").asInt());
assertFalse(instance.has("applicationVolume"));
assertFalse(instance.has("applicationRelease"));
assertFalse(instance.has("runtimeMetadata"));
service.deleteInstance(REGION, "node-app");
verify(containerManager).terminate("node-container", null, null, null, "node-app-volume");
}
@Test
void nodeSupervisorInitializesPersistentReleaseLayoutAndRestartsApplications() {
List<String> command = LightsailContainerManager.nodeSupervisorCommand();
assertEquals(List.of("sh", "-c"), command.subList(0, 2));
String script = command.get(2);
assertTrue(script.contains("$app/releases/$FLOCI_LIGHTSAIL_RELEASE_ID"));
assertTrue(script.contains("ln -s \"releases/$FLOCI_LIGHTSAIL_RELEASE_ID\" \"$app/current\""));
assertTrue(script.contains("tail -n +4"));
assertTrue(script.contains("npm start"));
assertTrue(script.contains("app.stdout.log"));
assertTrue(script.contains("app.stderr.log"));
assertTrue(script.contains("app/logs/supervisor.log"));
assertTrue(script.contains("delay=$((delay * 2))"));
}
}
@@ -0,0 +1,56 @@
package io.github.hectorvent.floci.services.lightsail;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Test;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
class LightsailPortForwardManagerTest {
private final ObjectMapper mapper = new ObjectMapper();
@Test
void extractsOnlyTcpRulesWithCidrSources() {
ArrayNode rules = mapper.createArrayNode()
.add(rule("tcp", 8080, 8081, true))
.add(rule("udp", 53, 53, true))
.add(rule("tcp", 9090, 9090, false));
assertEquals(Set.of(8080, 8081), LightsailPortForwardManager.extractPublishablePorts(rules, 20));
}
@Test
void includesSshForOsContainerDirectBinding() {
ArrayNode rules = mapper.createArrayNode().add(rule("tcp", 22, 22, true));
assertEquals(Set.of(22), LightsailPortForwardManager.extractPublishablePorts(rules, 20));
}
@Test
void skipsRulesWiderThanThePerInstanceLimit() {
ArrayNode rules = mapper.createArrayNode().add(rule("tcp", 1, 100, true));
assertTrue(LightsailPortForwardManager.extractPublishablePorts(rules, 20).isEmpty());
}
@Test
void forwardContainerNameIsDeterministic() {
assertEquals("floci-lightsail-fwd-web-8080", LightsailPortForwardManager.forwardContainerName("web", 8080));
}
private ObjectNode rule(String protocol, int fromPort, int toPort, boolean withCidr) {
ObjectNode rule = mapper.createObjectNode()
.put("protocol", protocol)
.put("fromPort", fromPort)
.put("toPort", toPort);
if (withCidr) {
rule.putArray("cidrs").add("0.0.0.0/0");
}
return rule;
}
}
@@ -100,10 +100,10 @@ class LightsailResourceProviderTest {
class GetSupportedResourceTypes {
@Test
void advertisesFourLightsailTypesAllUnderLightsailService() {
void advertisesLightsailResourceTypesAllUnderLightsailService() {
Set<SupportedResourceType> types = service.getSupportedResourceTypes();
assertEquals(Set.of("lightsail:Instance", "lightsail:Disk",
"lightsail:StaticIp", "lightsail:KeyPair"),
"lightsail:StaticIp", "lightsail:KeyPair", "lightsail:InstanceSnapshot"),
types.stream().map(SupportedResourceType::resourceType).collect(Collectors.toSet()));
assertTrue(types.stream().allMatch(t -> t.service().equals("lightsail")));
}
@@ -0,0 +1,119 @@
package io.github.hectorvent.floci.services.lightsail;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.ec2.Ec2KeyMaterial;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class LightsailSshKeyTest {
private static final String REGION = "us-east-1";
private final ObjectMapper mapper = new ObjectMapper();
private LightsailService service;
private LightsailContainerManager containerManager;
@BeforeEach
void setUp() {
StorageFactory storageFactory = new StorageFactory(null, null) {
@Override
public synchronized <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
TypeReference<Map<String, V>> typeReference) {
return AccountAwareStorageBackend.inMemory("000000000000");
}
};
containerManager = mock(LightsailContainerManager.class);
service = new LightsailService(storageFactory, mapper, new RegionResolver(REGION, "000000000000"),
containerManager, null);
}
@Test
void generatedKeyPairReturnsUsablePrivatePemAndLaunchesWithItsPublicKey() {
ObjectNode createKey = mapper.createObjectNode().put("keyPairName", "deployer");
ObjectNode keyResponse = service.createKeyPair(REGION, createKey);
String privateKey = new String(Base64.getDecoder().decode(keyResponse.path("privateKeyBase64").asText()),
StandardCharsets.UTF_8);
String publicKey = new String(Base64.getDecoder().decode(keyResponse.path("publicKeyBase64").asText()),
StandardCharsets.UTF_8);
assertTrue(privateKey.startsWith("-----BEGIN RSA PRIVATE KEY-----"));
assertTrue(Ec2KeyMaterial.fingerprintOf(publicKey) != null);
when(containerManager.launch(anyString(), anyString(), anyString(), anyString(), isNull(), anyString()))
.thenReturn(new LightsailContainerManager.Launch("container", null, null, 22022));
ObjectNode createInstance = mapper.createObjectNode()
.put("availabilityZone", REGION + "a")
.put("blueprintId", "ubuntu_22_04")
.put("bundleId", "nano_3_0")
.put("keyPairName", "deployer");
createInstance.set("instanceNames", mapper.createArrayNode().add("ssh-instance"));
service.createInstances(REGION, createInstance);
ArgumentCaptor<String> keyCaptor = ArgumentCaptor.forClass(String.class);
verify(containerManager).launch(eq("ssh-instance"), eq(REGION), eq("ubuntu_22_04"),
eq("nano_3_0"), isNull(), keyCaptor.capture());
assertEquals(publicKey, keyCaptor.getValue());
assertEquals(22022, service.hostPortMappings(REGION, "ssh-instance").getFirst().get("hostPort"));
}
@Test
void defaultKeyPairIsGeneratedAndSelectedWhenNoKeyNameIsProvided() {
when(containerManager.launch(anyString(), anyString(), anyString(), anyString(), isNull(), anyString()))
.thenReturn(new LightsailContainerManager.Launch("container", null, null, 22023));
ObjectNode request = mapper.createObjectNode()
.put("availabilityZone", REGION + "a")
.put("blueprintId", "amazon_linux_2023")
.put("bundleId", "nano_3_0");
request.set("instanceNames", mapper.createArrayNode().add("default-ssh"));
service.createInstances(REGION, request);
ObjectNode key = service.downloadDefaultKeyPair(REGION);
String privateKey = new String(Base64.getDecoder().decode(key.path("privateKeyBase64").asText()), StandardCharsets.UTF_8);
assertTrue(privateKey.startsWith("-----BEGIN RSA PRIVATE KEY-----"));
assertEquals("LightsailDefaultKeyPair", service.getInstance(REGION, "default-ssh")
.path("instance").path("sshKeyName").asText());
}
@Test
void importedKeyPairIsDecodedAndSelectedForTheInstance() {
Ec2KeyMaterial.Generated generated = Ec2KeyMaterial.generateRsa();
ObjectNode imported = mapper.createObjectNode().put("keyPairName", "imported").put("publicKeyBase64",
Base64.getEncoder().encodeToString(generated.openSshPublicKey().getBytes(StandardCharsets.UTF_8)));
service.importKeyPair(REGION, imported);
when(containerManager.launch(anyString(), anyString(), anyString(), anyString(), isNull(), anyString()))
.thenReturn(new LightsailContainerManager.Launch("container", null, null, 22024));
ObjectNode request = mapper.createObjectNode()
.put("availabilityZone", REGION + "a")
.put("blueprintId", "ubuntu_24_04")
.put("bundleId", "nano_3_0")
.put("keyPairName", "imported");
request.set("instanceNames", mapper.createArrayNode().add("imported-ssh"));
service.createInstances(REGION, request);
ArgumentCaptor<String> keyCaptor = ArgumentCaptor.forClass(String.class);
verify(containerManager).launch(eq("imported-ssh"), eq(REGION), eq("ubuntu_24_04"),
eq("nano_3_0"), isNull(), keyCaptor.capture());
assertEquals(generated.openSshPublicKey(), keyCaptor.getValue());
}
}
@@ -0,0 +1,75 @@
package io.github.hectorvent.floci.services.lightsail;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class LightsailWordPressTest {
private static final String REGION = "us-east-1";
private final ObjectMapper mapper = new ObjectMapper();
private LightsailService service;
private LightsailContainerManager containerManager;
@BeforeEach
void setUp() {
StorageFactory storageFactory = new StorageFactory(null, null) {
@Override
public synchronized <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
TypeReference<Map<String, V>> typeReference) {
return AccountAwareStorageBackend.inMemory("000000000000");
}
};
containerManager = mock(LightsailContainerManager.class);
service = new LightsailService(storageFactory, mapper, new RegionResolver(REGION, "000000000000"),
containerManager, null);
}
@Test
void wordpressLaunchesAndManagesMariaDbWithoutLeakingRuntimeFields() {
when(containerManager.launch(eq("blog"), eq(REGION), eq("wordpress"), eq("nano_3_0"), isNull(), anyString()))
.thenReturn(new LightsailContainerManager.Launch("wordpress-id", "mariadb-id", "mariadb-data",
"wordpress-content", null));
ObjectNode request = mapper.createObjectNode()
.put("availabilityZone", REGION + "a")
.put("blueprintId", "wordpress")
.put("bundleId", "nano_3_0");
request.set("instanceNames", mapper.createArrayNode().add("blog"));
service.createInstances(REGION, request);
ObjectNode instance = (ObjectNode) service.getInstance(REGION, "blog").path("instance");
assertEquals("wordpress", instance.path("blueprintId").asText());
assertEquals(80, instance.path("networking").path("ports").get(1).path("fromPort").asInt());
assertFalse(instance.has("containerId"));
assertFalse(instance.has("databaseContainerId"));
assertFalse(instance.has("databaseVolume"));
assertFalse(instance.has("wordpressContentVolume"));
service.stopInstance(REGION, "blog");
service.startInstance(REGION, "blog");
service.rebootInstance(REGION, "blog");
service.deleteInstance(REGION, "blog");
verify(containerManager).stop("wordpress-id", "mariadb-id");
verify(containerManager).start("wordpress-id", "mariadb-id", false);
verify(containerManager).reboot("wordpress-id", "mariadb-id", false);
verify(containerManager).terminate("wordpress-id", "mariadb-id", "mariadb-data", "wordpress-content", null);
}
}
@@ -83,7 +83,7 @@ class AmiImageToolTest {
String catalogText = Files.readString(catalog);
assertTrue(catalogText.contains("ami-existing"));
assertTrue(catalogText.contains("ami-ubuntu2404-cloud-arm64"));
assertTrue(catalogText.contains("guestRuntime: \"systemd\""));
assertTrue(catalogText.contains("initType: \"systemd\""));
assertTrue(catalogText.contains("cloudInit: true"));
assertTrue(catalogText.contains("custom-output/ubuntu-24.04-arm64/provenance.yaml"));
}