From bc3d16bfb69c7e7f986b514202f81a50318ea599 Mon Sep 17 00:00:00 2001 From: Zakaria Date: Sun, 25 Oct 2026 21:44:24 -0400 Subject: [PATCH] feat(efs): one shared store per file system, served to EC2 over NFS An EFS file system is now one Docker volume, floci-efs-, that every mount of it shares, as on AWS: - An access point or ECS rootDirectory mounts its folder of that volume (Docker volume subpath), so a mount of the file system root sees every access point's files, and Lambda and ECS see each other's. Before, each access point (Lambda) or root directory (ECS) got its own volume. - An access point's root directory is created on first use from its CreationInfo, every missing directory on the path with its owner and permissions; without CreationInfo a missing root directory is refused. - A Lambda function runs as the access point's POSIX user and secondary groups. ECS tasks keep their own user, as on AWS. - CreateAccessPoint validates RootDirectory.Path, CreationInfo and PosixUser the way EFS does. - DeleteFileSystem removes the file system's volume. Each mount target is now a real EC2 network interface in its subnet, so its address comes from the subnet's lease table and no instance is handed it, and a privileged helper container serves the volume over NFSv4.1 at that address on the VPC's Docker network (kernel nfsd, image built locally from docker/efs-nfs-server.Dockerfile). A mount target is creating until its server answers. The embedded DNS server resolves [zone.]fs-id.efs.region.amazonaws.com to a mount target, so an EC2 instance mounts the file system with AWS's own commands (NFS client or amazon-efs-utils without tls). Servers come back after a restart, their addresses re-reserved first. floci.services.efs.nfs-servers turns it off. Two EC2 fixes this needed: - DeleteNetworkInterface returns the interface's addresses to the subnet; they stayed leased until restart, also for ECS awsvpc tasks. - A VPC whose CIDR is substituted keeps each subnet at its declared offset inside the substitute range, instead of the next free slice in declaration order, so the plan and persisted addresses survive a restart that loads subnets in another order. --- .../floci/config/EmulatorConfig.java | 12 + .../core/common/dns/EmbeddedDnsServer.java | 29 ++- .../core/common/docker/ContainerBuilder.java | 14 ++ .../docker/ContainerLifecycleManager.java | 83 ++++++- .../common/docker/SubpathVolumeOptions.java | 26 ++ .../floci/services/ec2/Ec2Service.java | 6 + .../services/ec2/net/VpcNetworkManager.java | 25 +- .../ecs/container/EcsContainerManager.java | 78 ++---- .../floci/services/efs/EfsException.java | 4 + .../floci/services/efs/EfsMounts.java | 144 +++++++++++ .../floci/services/efs/EfsNfsServers.java | 229 ++++++++++++++++++ .../floci/services/efs/EfsService.java | 189 +++++++++++++-- .../lambda/launcher/ContainerLauncher.java | 67 +++-- src/main/resources/application.yml | 2 + .../docker/efs-nfs-server.Dockerfile | 5 + .../common/dns/EmbeddedDnsServerTest.java | 12 + .../ec2/net/VpcNetworkManagerTest.java | 17 ++ ...agerEfsIsolationDockerIntegrationTest.java | 169 ------------- ...anagerEfsSharingDockerIntegrationTest.java | 194 +++++++++++++++ ...csContainerManagerEfsVolumeNamingTest.java | 87 ------- .../EcsContainerManagerSecurityGroupTest.java | 2 +- .../EcsContainerManagerVolumesTest.java | 14 +- .../efs/EfsAccessPointValidationTest.java | 81 +++++++ .../services/efs/EfsIntegrationTest.java | 6 +- .../efs/EfsMountTargetSubnetTest.java | 82 ++++++- .../floci/services/efs/EfsMountsTest.java | 170 +++++++++++++ .../lambda/LambdaImageConfigTest.java | 3 +- .../ContainerLauncherCodeVolumeReuseTest.java | 2 +- .../launcher/ContainerLauncherTest.java | 60 ++++- .../ContainerLauncherVolumeNamingTest.java | 27 --- src/test/resources/application.yml | 1 + 31 files changed, 1391 insertions(+), 449 deletions(-) create mode 100644 src/main/java/io/github/hectorvent/floci/core/common/docker/SubpathVolumeOptions.java create mode 100644 src/main/java/io/github/hectorvent/floci/services/efs/EfsMounts.java create mode 100644 src/main/java/io/github/hectorvent/floci/services/efs/EfsNfsServers.java create mode 100644 src/main/resources/docker/efs-nfs-server.Dockerfile delete mode 100644 src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsIsolationDockerIntegrationTest.java create mode 100644 src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsSharingDockerIntegrationTest.java delete mode 100644 src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsVolumeNamingTest.java create mode 100644 src/test/java/io/github/hectorvent/floci/services/efs/EfsAccessPointValidationTest.java create mode 100644 src/test/java/io/github/hectorvent/floci/services/efs/EfsMountsTest.java diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java index b5e75f1..2a40852 100644 --- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java +++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java @@ -1040,6 +1040,18 @@ public interface EmulatorConfig { interface EfsServiceConfig { @WithDefault("true") boolean enabled(); + + /** + * Serve every mount target over NFSv4.1 at its own address on the VPC's Docker network, + * from a privileged helper container running the kernel NFS server, so an EC2 instance + * can mount the file system as on AWS. Needs VPC networks ({@code floci.services.ec2.vpc-networks}). + */ + @WithDefault("true") + boolean nfsServers(); + + /** Image of the NFS server; the default is built locally from Floci's own recipe on first use. */ + @WithDefault("floci/efs-nfs-server:local") + String nfsServerImage(); } interface LightsailServiceConfig { diff --git a/src/main/java/io/github/hectorvent/floci/core/common/dns/EmbeddedDnsServer.java b/src/main/java/io/github/hectorvent/floci/core/common/dns/EmbeddedDnsServer.java index 881d652..c8295e7 100644 --- a/src/main/java/io/github/hectorvent/floci/core/common/dns/EmbeddedDnsServer.java +++ b/src/main/java/io/github/hectorvent/floci/core/common/dns/EmbeddedDnsServer.java @@ -68,6 +68,14 @@ public class EmbeddedDnsServer { // localhost.floci.io → localhost.floci.io, *.localhost.floci.io public static final List BUILTIN_SUFFIXES = List.of(DEFAULT_SUFFIX, LOCALSTACK_SUFFIX); + /** Answers A queries for names a service owns, such as an EFS file system's DNS name. */ + @FunctionalInterface + public interface NameResolver { + Optional resolve(String name); + } + + private final List resolvers = new java.util.concurrent.CopyOnWriteArrayList<>(); + private volatile String serverIp; private final SequencedSet suffixes = new LinkedHashSet<>(); private volatile List upstreamDnsServers = List.of(); @@ -163,11 +171,30 @@ public class EmbeddedDnsServer { return false; } + /** Adds names a service answers for; a name no resolver knows is forwarded upstream as before. */ + public void addResolver(NameResolver resolver) { + resolvers.add(resolver); + } + Optional resolveARecord(String name, String myIp) { if (matchesSuffix(name)) { return Optional.of(myIp); } - return resolveEc2PrivateDnsName(name); + Optional ec2Name = resolveEc2PrivateDnsName(name); + if (ec2Name.isPresent()) { + return ec2Name; + } + for (NameResolver resolver : resolvers) { + try { + Optional address = resolver.resolve(name); + if (address.isPresent()) { + return address; + } + } catch (RuntimeException e) { + LOG.debugv("DNS resolver failed for {0}: {1}", name, e.getMessage()); + } + } + return Optional.empty(); } Optional resolveEc2PrivateDnsName(String name) { diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java index d8e4427..1888d77 100644 --- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java +++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java @@ -362,6 +362,20 @@ public class ContainerBuilder { return this; } + /** + * Adds a named volume mount of one directory inside the volume, given relative to its root. + * The directory must already exist; Docker refuses to start the container otherwise. + */ + public Builder withNamedVolumeSubpath(String volumeName, String subpath, String containerPath, boolean readOnly) { + this.mounts.add(new Mount() + .withType(MountType.VOLUME) + .withSource(volumeName) + .withTarget(containerPath) + .withReadOnly(readOnly) + .withVolumeOptions(new SubpathVolumeOptions(subpath))); + return this; + } + /** * Inherits every volume declared by another container. */ diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java index 15a9b6c..0baa62d 100644 --- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java +++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java @@ -34,6 +34,7 @@ import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -588,6 +589,77 @@ public class ContainerLifecycleManager { rootPermissions.ifPresent(p -> script.append("chmod ").append(p).append(' ').append(mount).append(" && ")); script.append("true"); + Integer status = runVolumeHelper(volumeName, mount, initImage, List.of("sh", "-c", script.toString())); + if (status == null || status != 0) { + // Throw so the caller leaves the volume unmemoised and retries on the next launch, + // rather than leaving it root:root 0755 with no further attempt. + throw new IllegalStateException("shared-volume init for " + volumeName + + " exited with status " + status + " (cmd: " + script + ")"); + } + LOG.infov("Initialised shared volume {0} root (cmd: {1})", volumeName, script); + } + + /** Owner and permissions for a directory created inside a shared volume (EFS {@code CreationInfo}). */ + public record DirectoryCreation(long ownerUid, long ownerGid, String permissions) { + public DirectoryCreation { + if (permissions == null || !permissions.matches("^[0-7]{3,4}$")) { + throw new IllegalArgumentException("permissions must be 3-4 octal digits: " + permissions); + } + } + } + + // Walks the path one directory at a time so that every directory it creates, not only the + // last, gets the owner and permissions, as EFS does for an access point's root directory. + private static final String ENSURE_DIRECTORY_SCRIPT = """ + set -f + cd /floci-shared-volume || exit 2 + [ -d "$1" ] && exit 0 + [ "$#" -eq 4 ] || exit 3 + p= + IFS=/ + for part in $1; do + [ -n "$part" ] || continue + p="${p:+$p/}$part" + if [ ! -d "$p" ]; then + mkdir "$p" && chown "$2:$3" "$p" && chmod "$4" "$p" || exit 1 + fi + done + """; + + /** + * Ensures {@code relativePath} exists inside the named volume, emulating how Amazon EFS creates + * an access point's root directory on first use: each missing directory on the path is created + * with {@code creation}'s owner and permissions. When the directory is missing and + * {@code creation} is empty nothing is created and this returns false, since EFS then refuses + * the mount. The volume itself must already exist. + */ + public boolean ensureSharedVolumeDirectory(String volumeName, String relativePath, + Optional creation, String initImage) { + if (relativePath == null || relativePath.isBlank() || relativePath.startsWith("/") + || List.of(relativePath.split("/")).contains("..")) { + throw new IllegalArgumentException("directory must be relative to the volume root: " + relativePath); + } + String memoKey = volumeName + "/" + relativePath; + if (initializedSharedVolumes.containsKey(memoKey)) { + return true; + } + List cmd = new ArrayList<>(List.of("sh", "-c", ENSURE_DIRECTORY_SCRIPT, "sh", relativePath)); + creation.ifPresent(c -> cmd.addAll(List.of( + Long.toString(c.ownerUid()), Long.toString(c.ownerGid()), c.permissions()))); + Integer status = runVolumeHelper(volumeName, "/floci-shared-volume", initImage, cmd); + if (status != null && status == 3) { + return false; + } + if (status == null || status != 0) { + throw new IllegalStateException("creating " + relativePath + " in volume " + volumeName + + " exited with status " + status); + } + initializedSharedVolumes.put(memoKey, Boolean.TRUE); + return true; + } + + /** Runs a one-off helper container with the volume mounted at {@code mount}; returns its exit status. */ + private Integer runVolumeHelper(String volumeName, String mount, String initImage, List cmd) { String image = (initImage != null && !initImage.isBlank()) ? initImage : "busybox:stable"; imageCacheService.ensureImageExists(image); @@ -595,22 +667,15 @@ public class ContainerLifecycleManager { new Mount().withType(MountType.VOLUME).withSource(volumeName).withTarget(mount))); CreateContainerResponse created = dockerClient.createContainerCmd(image) .withHostConfig(hostConfig) - .withCmd("sh", "-c", script.toString()) + .withCmd(cmd.toArray(String[]::new)) .withLabels(ContainerStorageHelper.defaultLabels(config)) .exec(); String helperId = created.getId(); try { startContainer(helperId); - Integer status = dockerClient.waitContainerCmd(helperId) + return dockerClient.waitContainerCmd(helperId) .exec(new WaitContainerResultCallback()) .awaitStatusCode(60, TimeUnit.SECONDS); - if (status == null || status != 0) { - // Throw so the caller leaves the volume unmemoised and retries on the next launch, - // rather than leaving it root:root 0755 with no further attempt. - throw new IllegalStateException("shared-volume init for " + volumeName - + " exited with status " + status + " (cmd: " + script + ")"); - } - LOG.infov("Initialised shared volume {0} root (cmd: {1})", volumeName, script); } finally { try { dockerClient.removeContainerCmd(helperId).withForce(true).exec(); diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/SubpathVolumeOptions.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/SubpathVolumeOptions.java new file mode 100644 index 0000000..cf886dc --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/SubpathVolumeOptions.java @@ -0,0 +1,26 @@ +package io.github.hectorvent.floci.core.common.docker; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.github.dockerjava.api.model.VolumeOptions; +import io.quarkus.runtime.annotations.RegisterForReflection; + +/** + * {@link VolumeOptions} with the {@code Subpath} field (Docker Engine API 1.45+), which mounts one + * directory of a named volume instead of its root. docker-java 3.7 does not model the field yet. + */ +@RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SubpathVolumeOptions extends VolumeOptions { + + @JsonProperty("Subpath") + private final String subpath; + + public SubpathVolumeOptions(String subpath) { + this.subpath = subpath; + } + + public String getSubpath() { + return subpath; + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java index 5835d21..c80a375 100644 --- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java +++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java @@ -8554,6 +8554,12 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider { "Network interface '" + networkInterfaceId + "' is currently in use", 400); } networkInterfaces.delete(key(region, networkInterfaceId)); + // Its addresses go back to the subnet, or every task or mount target that ever held one + // would leave it leased until the emulator restarts. + if (vpcNetworkManager != null && ni.getPrivateIpAddresses() != null) { + ni.getPrivateIpAddresses().forEach(address -> + vpcNetworkManager.releasePrivateIp(region, ni.getSubnetId(), address.getPrivateIpAddress())); + } } /** diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/net/VpcNetworkManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/net/VpcNetworkManager.java index a7312d7..ab53c8e 100644 --- a/src/main/java/io/github/hectorvent/floci/services/ec2/net/VpcNetworkManager.java +++ b/src/main/java/io/github/hectorvent/floci/services/ec2/net/VpcNetworkManager.java @@ -285,7 +285,10 @@ public class VpcNetworkManager { } int desiredPrefix = declared.map(Cidr4::prefix).orElse(24); - Cidr4 slice = allocateSubBlock(vpc.effective, desiredPrefix, siblings); + Cidr4 slice = translatedIntoSubstitute(vpc, declared, siblings); + if (slice == null) { + slice = allocateSubBlock(vpc.effective, desiredPrefix, siblings); + } if (slice == null) { LOG.warnv("Subnet {0} in VPC {1}: no free /{2} slice remains inside {3}; instances in this " + "subnet fall back to synthesised private addresses.", @@ -301,6 +304,26 @@ public class VpcNetworkManager { return bindSubnet(vpc, subnetId, declared.orElse(null), slice, true); } + /** + * The declared subnet moved to the same place inside its VPC's substitute range, or null when + * that does not apply. Keeping each subnet's offset, rather than handing out the next free + * slice, is what makes the plan come out the same in every run whatever order the subnets are + * declared in, so the addresses persisted instances and mount targets hold stay valid. + */ + private static Cidr4 translatedIntoSubstitute(VpcBinding vpc, Optional declared, List siblings) { + if (!vpc.substituted || vpc.declared == null || declared.isEmpty() + || vpc.declared.prefix() != vpc.effective.prefix() || !vpc.declared.contains(declared.get())) { + return null; + } + long offset = declared.get().network() - vpc.declared.network(); + Cidr4 translated = Cidr4.parse(Cidr4.format(vpc.effective.network() + offset) + "/" + declared.get().prefix()) + .orElse(null); + if (translated == null || siblings.stream().anyMatch(sibling -> sibling.overlaps(translated))) { + return null; + } + return translated; + } + /** * Binds a subnet to a range it can actually allocate out of. * diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java index bdee7c4..196d010 100644 --- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java +++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java @@ -14,6 +14,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper; import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv; import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager; import io.github.hectorvent.floci.services.ec2.Ec2Service; +import io.github.hectorvent.floci.services.efs.EfsMounts; import io.github.hectorvent.floci.services.ec2.SecurityGroupFirewallManager; import io.github.hectorvent.floci.services.ec2.SecurityGroupNftCompiler; import io.github.hectorvent.floci.services.ec2.model.NetworkInterface; @@ -63,14 +64,11 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.time.Instant; import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.HashSet; -import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -124,6 +122,7 @@ public class EcsContainerManager { private final HostVolumePolicy hostVolumePolicy; private Ec2Service ec2Service; private SecurityGroupFirewallManager firewallManager; + private EfsMounts efsMounts; @Inject public EcsContainerManager(ContainerBuilder containerBuilder, @@ -139,11 +138,13 @@ public class EcsContainerManager { EcrRegistryManager ecrRegistryManager, HostVolumePolicy hostVolumePolicy, Ec2Service ec2Service, - SecurityGroupFirewallManager firewallManager) { + SecurityGroupFirewallManager firewallManager, + EfsMounts efsMounts) { this(containerBuilder, lifecycleManager, logStreamer, containerDetector, config, regionResolver, awsEnv, ssmService, secretsManagerService, s3Service, ecrRegistryManager, hostVolumePolicy); this.ec2Service = ec2Service; this.firewallManager = firewallManager; + this.efsMounts = efsMounts; } public EcsContainerManager(ContainerBuilder containerBuilder, @@ -170,6 +171,8 @@ public class EcsContainerManager { this.s3Service = s3Service; this.hostVolumePolicy = hostVolumePolicy; this.ecrRegistryManager = ecrRegistryManager; + // Without the EFS service only mounts that name no access point can be resolved. + this.efsMounts = new EfsMounts(null, lifecycleManager, config); } /** @@ -391,7 +394,7 @@ public class EcsContainerManager { specBuilder.withBind(sourcePath, mp.containerPath()); } } else if (efs != null) { - mountEfsVolume(specBuilder, efs, mp); + mountEfsVolume(specBuilder, efs, mp, region); } else { LOG.warnv("Skipping mountPoint with unresolved volume {0} on container {1}", mp.sourceVolume(), def.getName()); @@ -1769,20 +1772,15 @@ public class EcsContainerManager { } /** - * Materialises an EFS-configured task volume as a shared local Docker named volume scoped to - * both the file system and the mount's effective root (see {@link #efsVolumeName}), then - * initialises the volume root's POSIX ownership to emulate the EFS access point's - * RootDirectory.CreationInfo (no-op unless {@code floci.storage.efs} configures owner/permissions) - * and applies the configured PosixUser emulation (uid[:gid] and/or supplementary group) so a - * non-root task image can read/write the shared volume. + * Mounts an EFS-configured task volume: the file system's shared Docker volume, or the + * access point's or {@code rootDirectory}'s directory inside it (see {@link EfsMounts}). The + * container keeps its own user, as on AWS, unless {@code floci.storage.efs.mount-user} sets one. */ - private void mountEfsVolume(ContainerBuilder.Builder specBuilder, EfsVolumeConfiguration efs, MountPoint mp) { - String efsVolumeName = efsVolumeName(efs.fileSystemId(), efs.accessPointId(), efs.rootDirectory()); + private void mountEfsVolume(ContainerBuilder.Builder specBuilder, EfsVolumeConfiguration efs, MountPoint mp, + String region) { + efsMounts.prepare(region, efs.fileSystemId(), efs.accessPointId(), efs.rootDirectory()) + .applyTo(specBuilder, mp.containerPath(), mp.readOnly()); var efsCfg = config.storage().efs(); - lifecycleManager.ensureSharedVolume(efsVolumeName, - efsCfg.ownerUid(), efsCfg.ownerGid(), efsCfg.rootPermissions(), - efsCfg.initImage()); - specBuilder.withNamedVolume(efsVolumeName, mp.containerPath(), mp.readOnly()); efsCfg.mountUser().ifPresent(u -> { if (!u.matches("^\\d+(:\\d+)?$")) { throw new IllegalArgumentException( @@ -1793,52 +1791,6 @@ public class EcsContainerManager { efsCfg.mountGroupAdd().ifPresent(gid -> specBuilder.withGroupAdd(String.valueOf(gid))); } - /** - * Name of the local Docker named volume backing an EFS-configured task volume, scoped to - * both the file system and the mount's effective root so two mounts of the same file system - * with a different {@code rootDirectory}/{@code accessPointId} land on isolated volumes - * while identical configurations keep sharing one, matching how AWS scopes an EFS mount to - * a subpath. When {@code accessPointId} is set, it determines the effective root: on real - * AWS an access point's own root directory takes precedence over any {@code rootDirectory} - * on the volume. - */ - static String efsVolumeName(String fileSystemId, String accessPointId, String rootDirectory) { - if (accessPointId != null && !accessPointId.isBlank()) { - return "floci-efs-" + fileSystemId + "-" + sha256Hex("accessPoint:" + accessPointId); - } - String normalizedRoot = normalizeRootDirectory(rootDirectory); - if ("/".equals(normalizedRoot)) { - return "floci-efs-" + fileSystemId; - } - return "floci-efs-" + fileSystemId + "-" + sha256Hex(normalizedRoot); - } - - /** - * Canonicalises a {@code rootDirectory} path so syntactically different but equivalent - * paths (missing/blank, a missing leading slash, repeated slashes, a trailing slash) resolve - * to the same effective root instead of silently splitting shared storage across local - * volumes that AWS would treat as identical. - */ - private static String normalizeRootDirectory(String rootDirectory) { - if (rootDirectory == null || rootDirectory.isBlank()) { - return "/"; - } - String collapsed = rootDirectory.trim().replaceAll("/+", "/"); - if (collapsed.length() > 1 && collapsed.endsWith("/")) { - collapsed = collapsed.substring(0, collapsed.length() - 1); - } - return collapsed.startsWith("/") ? collapsed : "/" + collapsed; - } - - private static String sha256Hex(String value) { - try { - byte[] digest = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); - return HexFormat.of().formatHex(digest); - } catch (NoSuchAlgorithmException e) { - throw new IllegalStateException("SHA-256 is required but not available", e); - } - } - // Inner enum to avoid import cycle — mirrors model.TaskStatus for readability private enum TaskStatus {RUNNING} } diff --git a/src/main/java/io/github/hectorvent/floci/services/efs/EfsException.java b/src/main/java/io/github/hectorvent/floci/services/efs/EfsException.java index b64e10a..1542a8b 100644 --- a/src/main/java/io/github/hectorvent/floci/services/efs/EfsException.java +++ b/src/main/java/io/github/hectorvent/floci/services/efs/EfsException.java @@ -33,6 +33,10 @@ public class EfsException extends AwsException { return new EfsException(Status.BAD_REQUEST, "SubnetNotFound", "The subnet ID '" + subnetId + "' does not exist"); } + public static EfsException securityGroupNotFound(String message) { + return new EfsException(Status.BAD_REQUEST, "SecurityGroupNotFound", message); + } + public static EfsException mountTargetConflict(String message) { return new EfsException(Status.CONFLICT, "MountTargetConflict", message); } diff --git a/src/main/java/io/github/hectorvent/floci/services/efs/EfsMounts.java b/src/main/java/io/github/hectorvent/floci/services/efs/EfsMounts.java new file mode 100644 index 0000000..8e3a43b --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/efs/EfsMounts.java @@ -0,0 +1,144 @@ +package io.github.hectorvent.floci.services.efs; + +import io.github.hectorvent.floci.config.EmulatorConfig; +import io.github.hectorvent.floci.core.common.docker.ContainerBuilder; +import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager; +import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.DirectoryCreation; +import io.github.hectorvent.floci.services.efs.model.AccessPointDescription; +import io.github.hectorvent.floci.services.efs.model.CreationInfo; +import io.github.hectorvent.floci.services.efs.model.PosixUser; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +import java.util.List; +import java.util.Optional; + +/** + * Turns an EFS mount (a file system, optionally through an access point) into a Docker volume + * mount. Each file system is one named volume, so every mount of it, from Lambda or ECS, through + * any access point, shares the same data. An access point mounts its root directory, a subfolder + * of that volume, which is created on first use from the access point's {@code CreationInfo}. + */ +@ApplicationScoped +public class EfsMounts { + + /** A file system directory ready to mount, and the identity its access point enforces, if any. */ + public record Mount(String volumeName, String subpath, Optional accessPointUser) { + + public void applyTo(ContainerBuilder.Builder spec, String containerPath, boolean readOnly) { + if (subpath.isEmpty()) { + spec.withNamedVolume(volumeName, containerPath, readOnly); + } else { + spec.withNamedVolumeSubpath(volumeName, subpath, containerPath, readOnly); + } + } + } + + // A plain rootDirectory (an ECS volume without an access point) has no CreationInfo; create it + // the way a Docker named volume's own root is created. + private static final DirectoryCreation PLAIN_ROOT_DIRECTORY = new DirectoryCreation(0, 0, "755"); + + private final EfsService efsService; + private final ContainerLifecycleManager lifecycleManager; + private final EmulatorConfig config; + + @Inject + public EfsMounts(EfsService efsService, ContainerLifecycleManager lifecycleManager, EmulatorConfig config) { + this.efsService = efsService; + this.lifecycleManager = lifecycleManager; + this.config = config; + } + + public static String volumeName(String fileSystemId) { + return "floci-efs-" + fileSystemId; + } + + /** Prepares a Lambda {@code FileSystemConfig}, which names an access point by ARN. */ + public Mount prepareAccessPoint(String accessPointArn) { + String[] parts = accessPointArn == null ? new String[0] : accessPointArn.split(":", 6); + if (parts.length != 6 || !parts[5].startsWith("access-point/")) { + throw new IllegalArgumentException("Not an EFS access point ARN: " + accessPointArn); + } + return prepare(parts[3], null, parts[5].substring("access-point/".length()), null); + } + + /** + * Prepares a mount of {@code fileSystemId}. With an access point its own root directory and + * owner apply and {@code rootDirectory} is ignored, as on AWS; {@code fileSystemId} may then + * be null, and otherwise must be the access point's file system. + */ + public Mount prepare(String region, String fileSystemId, String accessPointId, String rootDirectory) { + String root; + Optional creation; + Optional user = Optional.empty(); + if (accessPointId != null && !accessPointId.isBlank()) { + if (efsService == null) { + throw new IllegalStateException("EFS access points are unavailable: the EFS service is not running"); + } + AccessPointDescription accessPoint = efsService.describeAccessPoints(region, null, accessPointId) + .stream().findFirst() + .orElseThrow(() -> EfsException.accessPointNotFound(accessPointId)); + if (fileSystemId != null && !fileSystemId.equals(accessPoint.getFileSystemId())) { + throw new IllegalArgumentException("Access point " + accessPointId + " belongs to file system " + + accessPoint.getFileSystemId() + ", not " + fileSystemId); + } + fileSystemId = accessPoint.getFileSystemId(); + var directory = accessPoint.getRootDirectory(); + root = normalizeRootDirectory(directory == null ? null : directory.getPath()); + creation = Optional.ofNullable(directory == null ? null : directory.getCreationInfo()) + .flatMap(EfsMounts::directoryCreation); + user = Optional.ofNullable(accessPoint.getPosixUser()); + } else { + root = normalizeRootDirectory(rootDirectory); + creation = Optional.of(PLAIN_ROOT_DIRECTORY); + } + + String volumeName = volumeName(fileSystemId); + var efsCfg = config.storage().efs(); + lifecycleManager.ensureSharedVolume(volumeName, + efsCfg.ownerUid(), efsCfg.ownerGid(), efsCfg.rootPermissions(), efsCfg.initImage()); + String subpath = root.substring(1); + if (!subpath.isEmpty() + && !lifecycleManager.ensureSharedVolumeDirectory(volumeName, subpath, creation, efsCfg.initImage())) { + throw new IllegalStateException("The root directory " + root + " of access point " + accessPointId + + " does not exist on file system " + fileSystemId + + ", and the access point has no CreationInfo to create it"); + } + return new Mount(volumeName, subpath, user); + } + + /** {@code "uid:gid"} for a container that runs as the access point's POSIX user. */ + public static String containerUser(PosixUser user) { + return user.getUid() + ":" + user.getGid(); + } + + public static List secondaryGids(PosixUser user) { + return user.getSecondaryGids() == null ? List.of() : user.getSecondaryGids(); + } + + private static Optional directoryCreation(CreationInfo info) { + if (info.getOwnerUid() == null || info.getOwnerGid() == null || info.getPermissions() == null) { + return Optional.empty(); + } + return Optional.of(new DirectoryCreation(info.getOwnerUid(), info.getOwnerGid(), info.getPermissions())); + } + + /** + * Canonicalises a root directory so equivalent spellings (missing or blank, no leading slash, + * repeated or trailing slashes) name the same directory. A {@code ..} component is refused. + */ + static String normalizeRootDirectory(String rootDirectory) { + if (rootDirectory == null || rootDirectory.isBlank()) { + return "/"; + } + String collapsed = rootDirectory.trim().replaceAll("/+", "/"); + if (collapsed.length() > 1 && collapsed.endsWith("/")) { + collapsed = collapsed.substring(0, collapsed.length() - 1); + } + String normalized = collapsed.startsWith("/") ? collapsed : "/" + collapsed; + if (List.of(normalized.split("/")).contains("..")) { + throw new IllegalArgumentException("A root directory cannot contain '..': " + rootDirectory); + } + return normalized; + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/efs/EfsNfsServers.java b/src/main/java/io/github/hectorvent/floci/services/efs/EfsNfsServers.java new file mode 100644 index 0000000..bc56834 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/efs/EfsNfsServers.java @@ -0,0 +1,229 @@ +package io.github.hectorvent.floci.services.efs; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.async.ResultCallback; +import com.github.dockerjava.api.command.BuildImageResultCallback; +import com.github.dockerjava.api.exception.NotFoundException; +import com.github.dockerjava.api.model.Frame; +import io.github.hectorvent.floci.config.EmulatorConfig; +import io.github.hectorvent.floci.core.common.docker.ContainerBuilder; +import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager; +import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper; +import io.github.hectorvent.floci.services.ec2.net.VpcNetworkManager; +import io.github.hectorvent.floci.services.efs.model.MountTarget; +import jakarta.annotation.PreDestroy; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.jboss.logging.Logger; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Serves each EFS mount target over NFSv4.1, as on AWS: a privileged helper container running the + * kernel NFS server holds the mount target's address on its VPC's Docker network and exports the + * file system's volume, the same one Lambda and ECS mount directly. An EC2 instance in the VPC can + * then mount the file system with the commands AWS documents, by address or by its DNS name. + */ +@ApplicationScoped +public class EfsNfsServers { + + private static final Logger LOG = Logger.getLogger(EfsNfsServers.class); + + static final String LABEL_COMPONENT = "floci_component"; + static final String COMPONENT_VALUE = "efs-nfs-server"; + static final String LABEL_OWNER = "floci_efs_owner"; + static final String DEFAULT_IMAGE = "floci/efs-nfs-server:local"; + private static final int READY_ATTEMPTS = 60; + + private final EmulatorConfig config; + private final DockerClient dockerClient; + private final ContainerBuilder containerBuilder; + private final ContainerLifecycleManager lifecycleManager; + private final VpcNetworkManager vpcNetworkManager; + private final ExecutorService starter = Executors.newSingleThreadExecutor(r -> { + Thread thread = new Thread(r, "efs-nfs-servers"); + thread.setDaemon(true); + return thread; + }); + + @Inject + public EfsNfsServers(EmulatorConfig config, DockerClient dockerClient, ContainerBuilder containerBuilder, + ContainerLifecycleManager lifecycleManager, VpcNetworkManager vpcNetworkManager) { + this.config = config; + this.dockerClient = dockerClient; + this.containerBuilder = containerBuilder; + this.lifecycleManager = lifecycleManager; + this.vpcNetworkManager = vpcNetworkManager; + } + + @PreDestroy + void shutdown() { + starter.shutdownNow(); + } + + /** Whether mount targets get an NFS server: configured on, and VPCs have Docker networks to hold their address. */ + public boolean enabled() { + return config.services().efs().nfsServers() && vpcNetworkManager.enabled(); + } + + static String containerName(String mountTargetId) { + return "floci-efs-nfs-" + mountTargetId; + } + + /** + * Starts the mount target's NFS server in the background and then runs {@code onSettled} + * with whether it is serving. A server left from an earlier run is replaced. + */ + public void serve(String region, String accountId, MountTarget mountTarget, Consumer onSettled) { + starter.execute(() -> { + boolean serving = false; + try { + serving = start(region, accountId, mountTarget); + } catch (RuntimeException e) { + LOG.warnv("Mount target {0} of {1} has no NFS server: {2}", + mountTarget.getMountTargetId(), mountTarget.getFileSystemId(), e.getMessage()); + } + onSettled.accept(serving); + }); + } + + /** + * Re-claims a persisted mount target's address after a restart. The VPC address leases live + * only in memory, so without this the next instance launched in the subnet could be handed it. + */ + public void reserveAddress(String region, MountTarget mountTarget) { + vpcNetworkManager.reservePrivateIp(region, mountTarget.getSubnetId(), mountTarget.getIpAddress()); + } + + /** Stops and removes the mount target's NFS server, if it has one. */ + public void stop(String mountTargetId) { + lifecycleManager.removeIfExists(containerName(mountTargetId)); + } + + /** Removes every NFS server this emulator started, such as those a previous run left behind. */ + public void removeAll() { + if (!enabled()) { + return; + } + dockerClient.listContainersCmd().withShowAll(true) + .withLabelFilter(Map.of(LABEL_COMPONENT, COMPONENT_VALUE, LABEL_OWNER, owner())) + .exec() + .forEach(container -> lifecycleManager.removeIfExists(container.getId())); + } + + private boolean start(String region, String accountId, MountTarget mountTarget) { + String name = containerName(mountTarget.getMountTargetId()); + String image = ensureImage(); + String volumeName = EfsMounts.volumeName(mountTarget.getFileSystemId()); + var efsCfg = config.storage().efs(); + lifecycleManager.ensureSharedVolume(volumeName, + efsCfg.ownerUid(), efsCfg.ownerGid(), efsCfg.rootPermissions(), efsCfg.initImage()); + lifecycleManager.removeIfExists(name); + + var spec = containerBuilder.newContainer(image) + .withName(name) + .withPrivileged(true) + .withNamedVolume(volumeName, "/export", false) + .withLogRotation() + .withLabels(ContainerStorageHelper.resourceIdentityLabels( + "efs", mountTarget.getMountTargetId(), accountId, region)) + .withLabels(Map.of(LABEL_COMPONENT, COMPONENT_VALUE, LABEL_OWNER, owner())) + .build(); + String containerId = lifecycleManager.createAndStart(spec).containerId(); + if (vpcNetworkManager.attach(region, mountTarget.getVpcId(), mountTarget.getSubnetId(), + containerId, mountTarget.getIpAddress()).isEmpty()) { + lifecycleManager.removeIfExists(name); + LOG.warnv("Mount target {0} could not take its address {1} on the VPC network; it has no NFS server", + mountTarget.getMountTargetId(), mountTarget.getIpAddress()); + return false; + } + if (!awaitReady(containerId)) { + lifecycleManager.removeIfExists(name); + LOG.warnv("The NFS server of mount target {0} did not start", mountTarget.getMountTargetId()); + return false; + } + LOG.infov("Mount target {0} serves file system {1} over NFS at {2}", + mountTarget.getMountTargetId(), mountTarget.getFileSystemId(), mountTarget.getIpAddress()); + return true; + } + + /** Ready once the kernel NFS server has threads and mountd answers export lookups. */ + private boolean awaitReady(String containerId) { + for (int attempt = 0; attempt < READY_ATTEMPTS; attempt++) { + try { + String execId = dockerClient.execCreateCmd(containerId) + .withCmd("sh", "-c", "[ \"$(cat /proc/fs/nfsd/threads)\" -gt 0 ] && pgrep rpc.mountd") + .withAttachStdout(true).withAttachStderr(true) + .exec().getId(); + dockerClient.execStartCmd(execId).exec(new ResultCallback.Adapter()).awaitCompletion(10, TimeUnit.SECONDS); + Long exitCode = dockerClient.inspectExecCmd(execId).exec().getExitCodeLong(); + if (exitCode != null && exitCode == 0) { + return true; + } + TimeUnit.MILLISECONDS.sleep(500); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (RuntimeException e) { + LOG.debugv("NFS server {0} not ready yet: {1}", containerId, e.getMessage()); + } + } + return false; + } + + private synchronized String ensureImage() { + String configured = config.services().efs().nfsServerImage(); + String image = containerBuilder.resolveImage(configured); + if (!DEFAULT_IMAGE.equals(configured)) { + return image; + } + try { + dockerClient.inspectImageCmd(image).exec(); + return image; + } catch (NotFoundException missing) { + // Build Floci's own recipe in the daemon that runs the workloads. + } + try (InputStream dockerfile = getClass().getResourceAsStream("/docker/efs-nfs-server.Dockerfile")) { + if (dockerfile == null) { + throw new IllegalStateException("The EFS NFS server Dockerfile is missing"); + } + byte[] content = dockerfile.readAllBytes(); + ByteArrayOutputStream archive = new ByteArrayOutputStream(content.length + 1024); + try (TarArchiveOutputStream tar = new TarArchiveOutputStream(archive)) { + TarArchiveEntry entry = new TarArchiveEntry("Dockerfile"); + entry.setSize(content.length); + tar.putArchiveEntry(entry); + tar.write(content); + tar.closeArchiveEntry(); + } + try (BuildImageResultCallback callback = new BuildImageResultCallback()) { + String built = dockerClient.buildImageCmd(new ByteArrayInputStream(archive.toByteArray())) + .withTags(Set.of(image)).exec(callback).awaitImageId(); + if (built == null || built.isBlank()) { + throw new IllegalStateException("Docker did not build the EFS NFS server image"); + } + } + return image; + } catch (IOException e) { + throw new IllegalStateException("Cannot build the EFS NFS server image", e); + } + } + + /** The emulator that owns a server, so one emulator never removes another's on a shared daemon. */ + private String owner() { + String namespace = config.docker() == null || config.docker().resourceNamespace() == null + ? "" : config.docker().resourceNamespace().orElse(""); + return namespace.isBlank() ? String.valueOf(config.port()) : namespace + "/" + config.port(); + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/efs/EfsService.java b/src/main/java/io/github/hectorvent/floci/services/efs/EfsService.java index f2bf95c..4da1c1d 100644 --- a/src/main/java/io/github/hectorvent/floci/services/efs/EfsService.java +++ b/src/main/java/io/github/hectorvent/floci/services/efs/EfsService.java @@ -4,8 +4,13 @@ import io.github.hectorvent.floci.core.storage.StorageBackend; import io.github.hectorvent.floci.core.storage.StorageFactory; import io.github.hectorvent.floci.core.common.AwsException; import io.github.hectorvent.floci.core.common.Resettable; +import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager; +import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer; import io.github.hectorvent.floci.services.ec2.Ec2Service; +import io.github.hectorvent.floci.services.ec2.model.NetworkInterface; import io.github.hectorvent.floci.services.ec2.model.Subnet; +import io.quarkus.runtime.StartupEvent; +import jakarta.enterprise.event.Observes; import io.github.hectorvent.floci.services.efs.model.*; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -17,6 +22,9 @@ import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.UUID; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.concurrent.ConcurrentHashMap; @@ -35,6 +43,18 @@ public class EfsService implements Resettable { private final StorageBackend> lifecycleConfigurationStore; private final ConcurrentHashMap syncLocks = new ConcurrentHashMap<>(); private final RegionResolver regionResolver; + private ContainerLifecycleManager lifecycleManager; + private EfsNfsServers nfsServers; + private EmbeddedDnsServer dnsServer; + + // A file system's DNS name, optionally for one Availability Zone's mount target: + // [az.]fs-id.efs.region.amazonaws.com + private static final Pattern FILE_SYSTEM_DNS_NAME = Pattern.compile( + "^(?:([a-z0-9-]+)\\.)?(fs-[0-9a-f]+)\\.efs\\.([a-z0-9-]+)\\.amazonaws\\.com\\.?$"); + + // RootDirectory.Path as the EFS API documents it: "/" or up to 100 "/name" segments, where a + // name does not start with "." and holds none of $#<>;`|&?{}^* or a newline. + private static final Pattern ROOT_DIRECTORY_PATH = Pattern.compile("^(/|(/(?!\\.)[^$#<>;`|&?{}^*/\\n]+){1,100})$"); private final Ec2Service ec2Service; @@ -60,6 +80,82 @@ public class EfsService implements Resettable { new com.fasterxml.jackson.core.type.TypeReference>>() {}); } + /** Lets a deleted file system take its stored data with it; absent in tests without Docker. */ + @Inject + void setLifecycleManager(ContainerLifecycleManager lifecycleManager) { + this.lifecycleManager = lifecycleManager; + } + + /** Serves mount targets over NFS; absent in tests that build the service by hand. */ + @Inject + void setNfsServers(EfsNfsServers nfsServers, EmbeddedDnsServer dnsServer) { + this.nfsServers = nfsServers; + this.dnsServer = dnsServer; + } + + /** + * The address a file system's DNS name resolves to: the mount target in the named Availability + * Zone, or, for the plain name, the first mount target that is serving. AWS answers the plain + * name with the mount target in the client's own zone; every mount target here is reachable + * from the whole VPC, so any serving one will do. + */ + Optional resolveDnsName(String name) { + Matcher matcher = FILE_SYSTEM_DNS_NAME.matcher(name.toLowerCase()); + if (!matcher.matches()) { + return Optional.empty(); + } + String zone = matcher.group(1); + String fileSystemId = matcher.group(2); + String region = matcher.group(3); + return mountTargetStore.scan(k -> k.startsWith(region + "::")).stream() + .filter(mt -> fileSystemId.equals(mt.getFileSystemId())) + .filter(mt -> mt.getLifeCycleState() == LifeCycleState.available) + .filter(mt -> zone == null || zone.equals(mt.getAvailabilityZoneName())) + .sorted(Comparator.comparing(MountTarget::getAvailabilityZoneName)) + .map(MountTarget::getIpAddress) + .findFirst(); + } + + /** + * Brings back the NFS servers of persisted mount targets after a restart: each one re-claims + * its address on the VPC network, so no instance is handed it, and serves again. + */ + void onStart(@Observes StartupEvent ignored) { + if (nfsServers == null || !nfsServers.enabled()) { + return; + } + dnsServer.addResolver(this::resolveDnsName); + nfsServers.removeAll(); + for (String key : mountTargetStore.keys()) { + String region = key.substring(0, key.indexOf("::")); + mountTargetStore.get(key).ifPresent(mt -> { + // Looking the subnet up first also brings up EC2, which is what rebuilds the VPC + // networks the address reservation and the server's attachment both need. + try { + requireSubnet(region, mt.getSubnetId()); + } catch (EfsException e) { + LOG.warnv("Mount target {0} is in subnet {1}, which no longer exists; it has no NFS server", + mt.getMountTargetId(), mt.getSubnetId()); + return; + } + nfsServers.reserveAddress(region, mt); + serveOverNfs(region, mt); + }); + } + } + + private void serveOverNfs(String region, MountTarget mt) { + nfsServers.serve(region, regionResolver.getAccountId(), mt, serving -> { + String key = regionKey(region, mt.getMountTargetId()); + synchronized (lockFor(key)) { + mountTargetStore.get(key).ifPresent(current -> { + current.setLifeCycleState(LifeCycleState.available); + mountTargetStore.put(key, current); + }); + } + }); + } + @Override public void clear() { fileSystemStore.clear(); @@ -273,6 +369,10 @@ public class EfsService implements Resettable { fileSystemStore.delete(key); } + if (lifecycleManager != null && !lifecycleManager.removeVolume(EfsMounts.volumeName(fileSystemId))) { + LOG.warnv("The data of deleted file system {0} is still in Docker volume {1}, which is in use", + fileSystemId, EfsMounts.volumeName(fileSystemId)); + } } // --- Tags --- @@ -384,30 +484,43 @@ public class EfsService implements Resettable { } } } - long inSubnet = regionMountTargets.stream().filter(existing -> request.getSubnetId().equals(existing.getSubnetId())).count(); - String mtId = "fsmt-" + UUID.randomUUID().toString().replace("-", "").substring(0, 17); - + // As on AWS, the mount target is a network interface in the subnet, which is what + // gives it an address no instance can be handed as well. + NetworkInterface eni; + try { + eni = ec2Service.createNetworkInterface(region, request.getSubnetId(), + "EFS mount target for " + fs.getFileSystemId() + " (" + mtId + ")", + request.getIpAddress(), null, request.getSecurityGroups(), null); + } catch (AwsException e) { + if ("InvalidGroup.NotFound".equals(e.getErrorCode())) { + throw EfsException.securityGroupNotFound(e.getMessage()); + } + throw e; + } + ec2Service.holdNetworkInterfaceForService(region, eni.getNetworkInterfaceId()); + MountTarget mt = new MountTarget(); mt.setMountTargetId(mtId); mt.setFileSystemId(request.getFileSystemId()); mt.setSubnetId(request.getSubnetId()); - mt.setIpAddress(request.getIpAddress() != null ? request.getIpAddress() : hostAddress(subnet.getCidrBlock(), inSubnet)); - mt.setLifeCycleState(LifeCycleState.available); + mt.setIpAddress(eni.getPrivateIpAddress()); mt.setVpcId(subnet.getVpcId()); mt.setAvailabilityZoneName(subnet.getAvailabilityZone()); mt.setAvailabilityZoneId(generateAzId(subnet.getAvailabilityZone())); - mt.setNetworkInterfaceId("eni-" + UUID.randomUUID().toString().replace("-", "").substring(0, 17)); - if (request.getSecurityGroups() != null) { - mt.setSecurityGroups(new ArrayList<>(request.getSecurityGroups())); - } else { - mt.setSecurityGroups(new ArrayList<>()); - } + mt.setNetworkInterfaceId(eni.getNetworkInterfaceId()); + mt.setSecurityGroups(eni.getGroups().stream().map(group -> group.getGroupId()) + .collect(Collectors.toCollection(ArrayList::new))); + boolean servedOverNfs = nfsServers != null && nfsServers.enabled(); + mt.setLifeCycleState(servedOverNfs ? LifeCycleState.creating : LifeCycleState.available); fs.setNumberOfMountTargets(fs.getNumberOfMountTargets() + 1); fileSystemStore.put(regionKey(region, fs.getFileSystemId()), fs); mountTargetStore.put(regionKey(region, mtId), mt); + if (servedOverNfs) { + serveOverNfs(region, mt); + } return mt; } } @@ -469,7 +582,11 @@ public class EfsService implements Resettable { if (mt == null) { throw EfsException.mountTargetNotFound(mountTargetId); } - + if (nfsServers != null) { + nfsServers.stop(mountTargetId); + } + releaseNetworkInterface(region, mt); + try { synchronized (lockFor(regionKey(region, mt.getFileSystemId()))) { FileSystem fs = getFileSystem(region, mt.getFileSystemId()); @@ -515,6 +632,7 @@ public class EfsService implements Resettable { synchronized (lockFor(regionKey(region, request.getFileSystemId()))) { // Validate file system exists getFileSystem(region, request.getFileSystemId()); + validateAccessPoint(request); for (AccessPointDescription existing : accessPointStore.scan(k -> k.startsWith(region + "::"))) { if (token.equals(existing.getClientToken())) { @@ -546,6 +664,31 @@ public class EfsService implements Resettable { } } + private static void validateAccessPoint(CreateAccessPointRequest request) { + PosixUser user = request.getPosixUser(); + if (user != null && (user.getUid() == null || user.getGid() == null)) { + throw EfsException.badRequest("PosixUser requires both Uid and Gid"); + } + RootDirectory root = request.getRootDirectory(); + if (root == null) { + return; + } + if (root.getPath() != null && (root.getPath().length() > 100 + || !ROOT_DIRECTORY_PATH.matcher(root.getPath()).matches())) { + throw EfsException.badRequest("RootDirectory.Path is not a valid directory path: " + root.getPath()); + } + CreationInfo info = root.getCreationInfo(); + if (info != null) { + if (info.getOwnerUid() == null || info.getOwnerGid() == null || info.getPermissions() == null) { + throw EfsException.badRequest("CreationInfo requires OwnerUid, OwnerGid and Permissions"); + } + if (!info.getPermissions().matches("^[0-7]{3,4}$")) { + throw EfsException.badRequest("CreationInfo.Permissions must be 3 or 4 octal digits: " + + info.getPermissions()); + } + } + } + public List describeAccessPoints(String region, String fileSystemId, String accessPointId) { return accessPointStore.scan(k -> k.startsWith(region + "::")).stream() .filter(ap -> fileSystemId == null || ap.getFileSystemId().equals(fileSystemId)) @@ -640,19 +783,17 @@ public class EfsService implements Resettable { } } - /** - * An address inside the subnet for its n-th mount target, skipping the first four - * addresses AWS reserves in every subnet. - */ - static String hostAddress(String cidrBlock, long index) { - String[] parts = cidrBlock.split("/"); - String[] octets = parts[0].split("\\."); - long base = 0; - for (String octet : octets) { - base = (base << 8) | Long.parseLong(octet); + private void releaseNetworkInterface(String region, MountTarget mt) { + if (mt.getNetworkInterfaceId() == null) { + return; + } + try { + ec2Service.releaseNetworkInterfaceFromService(region, mt.getNetworkInterfaceId()); + ec2Service.deleteNetworkInterface(region, mt.getNetworkInterfaceId()); + } catch (AwsException e) { + LOG.debugv("Network interface {0} of mount target {1} is already gone: {2}", + mt.getNetworkInterfaceId(), mt.getMountTargetId(), e.getMessage()); } - long address = base + 4 + index; - return ((address >> 24) & 0xff) + "." + ((address >> 16) & 0xff) + "." + ((address >> 8) & 0xff) + "." + (address & 0xff); } private String generateAzId(String azName) { diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java index 87d230e..0fa919e 100644 --- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java +++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java @@ -10,6 +10,8 @@ import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper; import io.github.hectorvent.floci.core.common.docker.DockerHostResolver; import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv; import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager; +import io.github.hectorvent.floci.services.efs.EfsMounts; +import io.github.hectorvent.floci.services.efs.model.PosixUser; import io.github.hectorvent.floci.services.iam.model.SessionCreds; import io.github.hectorvent.floci.services.lambda.LambdaLayerService; import io.github.hectorvent.floci.services.lambda.model.ContainerState; @@ -36,17 +38,13 @@ import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.HexFormat; import java.util.List; import java.util.Optional; import java.util.concurrent.CancellationException; @@ -110,6 +108,7 @@ public class ContainerLauncher implements LambdaRuntimeLauncher { private final LambdaLayerService layerService; private final LaunchedContainerAwsEnv awsEnv; private final LambdaExecutionRoleCredentials executionRoleCredentials; + private final EfsMounts efsMounts; @Inject public ContainerLauncher(ContainerBuilder containerBuilder, @@ -122,7 +121,8 @@ public class ContainerLauncher implements LambdaRuntimeLauncher { EcrRegistryManager ecrRegistryManager, LambdaLayerService layerService, LaunchedContainerAwsEnv awsEnv, - LambdaExecutionRoleCredentials executionRoleCredentials) { + LambdaExecutionRoleCredentials executionRoleCredentials, + EfsMounts efsMounts) { this.containerBuilder = containerBuilder; this.lifecycleManager = lifecycleManager; this.logStreamer = logStreamer; @@ -134,6 +134,7 @@ public class ContainerLauncher implements LambdaRuntimeLauncher { this.layerService = layerService; this.awsEnv = awsEnv; this.executionRoleCredentials = executionRoleCredentials; + this.efsMounts = efsMounts; this.populateSemaphore = new java.util.concurrent.Semaphore(resolvePopulateConcurrency(config)); } @@ -310,21 +311,28 @@ public class ContainerLauncher implements LambdaRuntimeLauncher { if (fn.getFileSystemConfigs() != null && !fn.getFileSystemConfigs().isEmpty()) { var efsCfg = config.storage().efs(); - fn.getFileSystemConfigs().forEach(fileSystem -> { - String volumeName = efsVolumeName(fileSystem.getArn()); - lifecycleManager.ensureSharedVolume(volumeName, - efsCfg.ownerUid(), efsCfg.ownerGid(), efsCfg.rootPermissions(), - efsCfg.initImage()); - specBuilder.withNamedVolume(volumeName, fileSystem.getLocalMountPath(), false); - }); - efsCfg.mountUser().ifPresent(user -> { - if (!user.matches("^\\d+(:\\d+)?$")) { - throw new IllegalArgumentException( - "floci.storage.efs.mount-user must be \"uid\" or \"uid:gid\": " + user); - } - specBuilder.withUser(user); - }); - efsCfg.mountGroupAdd().ifPresent(gid -> specBuilder.withGroupAdd(String.valueOf(gid))); + // Lambda allows one file system per function. Running as the access point's POSIX user + // is how its identity reaches every file the function creates, as EFS enforces it. + Optional accessPointUser = Optional.empty(); + for (var fileSystem : fn.getFileSystemConfigs()) { + EfsMounts.Mount mount = efsMounts.prepareAccessPoint(fileSystem.getArn()); + mount.applyTo(specBuilder, fileSystem.getLocalMountPath(), false); + accessPointUser = accessPointUser.or(mount::accessPointUser); + } + if (accessPointUser.isPresent()) { + specBuilder.withUser(EfsMounts.containerUser(accessPointUser.get())); + EfsMounts.secondaryGids(accessPointUser.get()) + .forEach(gid -> specBuilder.withGroupAdd(String.valueOf(gid))); + } else { + efsCfg.mountUser().ifPresent(user -> { + if (!user.matches("^\\d+(:\\d+)?$")) { + throw new IllegalArgumentException( + "floci.storage.efs.mount-user must be \"uid\" or \"uid:gid\": " + user); + } + specBuilder.withUser(user); + }); + efsCfg.mountGroupAdd().ifPresent(gid -> specBuilder.withGroupAdd(String.valueOf(gid))); + } } // For Image package type use ImageConfig.Command/EntryPoint/WorkingDirectory if set, otherwise fall back to Handler (Zip-style) @@ -1133,25 +1141,6 @@ public class ContainerLauncher implements LambdaRuntimeLauncher { return namePrefix + "-code-" + fname + "-" + h; } - static String efsVolumeName(String accessPointArn) { - int separator = Math.max(accessPointArn.lastIndexOf('/'), accessPointArn.lastIndexOf(':')); - String resourceId = separator >= 0 ? accessPointArn.substring(separator + 1) : accessPointArn; - if (resourceId.isBlank()) { - throw new IllegalArgumentException("File system access point ARN must include a resource id"); - } - return "floci-efs-" + resourceId + "-" + sha256Hex(accessPointArn); - } - - private static String sha256Hex(String value) { - try { - byte[] digest = MessageDigest.getInstance("SHA-256") - .digest(value.getBytes(StandardCharsets.UTF_8)); - return HexFormat.of().formatHex(digest); - } catch (NoSuchAlgorithmException e) { - throw new IllegalStateException("SHA-256 is required but not available", e); - } - } - /** * Buffer for the tar-streaming pipe. The default {@link java.io.PipedInputStream} buffer is * only 1KB, which forces a writer/reader thread hand-off (wait/notify) every 1KB. Streaming a diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index cb6b997..1ba4d18 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -747,6 +747,8 @@ floci: enabled: true efs: enabled: true + nfs-servers: true # serve each mount target over NFSv4.1 for EC2 instances + nfs-server-image: floci/efs-nfs-server:local # built locally on first use organizations: enabled: true # management-account-email: "" # When set, overrides the management-account email reported by DescribeOrganization and account listings. diff --git a/src/main/resources/docker/efs-nfs-server.Dockerfile b/src/main/resources/docker/efs-nfs-server.Dockerfile new file mode 100644 index 0000000..b85f3b5 --- /dev/null +++ b/src/main/resources/docker/efs-nfs-server.Dockerfile @@ -0,0 +1,5 @@ +FROM alpine:3.21 +RUN apk add --no-cache nfs-utils +# Serves /export, the EFS file system's volume, over NFSv4.1 from the kernel NFS server, as an EFS +# mount target does. fsid=0 makes it the NFSv4 root, so clients mount "
:/" like on AWS. +CMD ["sh", "-c", "set -e; mount -t nfsd nfsd /proc/fs/nfsd; echo '/export *(rw,fsid=0,no_subtree_check,no_root_squash,insecure,sync)' > /etc/exports; exportfs -ra; rpc.nfsd -N 3 -V 4 -V 4.1 -V 4.2 8; trap 'rpc.nfsd 0; exportfs -ua; exit 0' TERM INT; rpc.mountd -N 2 -N 3 -F & wait"] diff --git a/src/test/java/io/github/hectorvent/floci/core/common/dns/EmbeddedDnsServerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/dns/EmbeddedDnsServerTest.java index a4595c3..44954f2 100644 --- a/src/test/java/io/github/hectorvent/floci/core/common/dns/EmbeddedDnsServerTest.java +++ b/src/test/java/io/github/hectorvent/floci/core/common/dns/EmbeddedDnsServerTest.java @@ -103,6 +103,18 @@ class EmbeddedDnsServerTest { dns.resolveARecord("ip-172-16-128-9.ec2.internal", "172.16.128.5").orElseThrow()); } + @Test + void resolveARecord_asksAddedResolversForOtherNames() { + dns.addResolver(name -> name.equals("fs-1.efs.us-east-1.amazonaws.com") + ? java.util.Optional.of("172.31.0.10") : java.util.Optional.empty()); + dns.addResolver(name -> { throw new IllegalStateException("broken resolver"); }); + + assertEquals("172.31.0.10", + dns.resolveARecord("fs-1.efs.us-east-1.amazonaws.com", "172.16.128.5").orElseThrow()); + assertTrue(dns.resolveARecord("example.com", "172.16.128.5").isEmpty(), + "a name no resolver knows is forwarded upstream, even when a resolver fails"); + } + // ── matchesSuffix — built-in emulator domains ───────────────────────────── @Test diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/net/VpcNetworkManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/net/VpcNetworkManagerTest.java index c2aecab..f8aa345 100644 --- a/src/test/java/io/github/hectorvent/floci/services/ec2/net/VpcNetworkManagerTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/ec2/net/VpcNetworkManagerTest.java @@ -284,6 +284,23 @@ class VpcNetworkManagerTest { assertTrue(second.containsAddress(Cidr4.parse(b + "/32").orElseThrow().network())); } + @Test + void aSubstitutedVpcKeepsEachSubnetAtItsOffsetWhateverTheDeclarationOrder() { + existingNetwork("someone-else", "172.31.0.0/16", Map.of()); + + manager.declareVpc(REGION, "vpc-default", "172.31.0.0/16"); + manager.declareSubnet(REGION, "vpc-default", "subnet-c", "172.31.32.0/20"); + manager.declareSubnet(REGION, "vpc-default", "subnet-a", "172.31.0.0/20"); + manager.declareSubnet(REGION, "vpc-default", "subnet-b", "172.31.16.0/20"); + + assertTrue(manager.isSubstituted(REGION, "vpc-default")); + assertEquals("10.240.0.0/16", manager.effectiveVpcCidr(REGION, "vpc-default").orElseThrow()); + assertEquals("10.240.0.0/20", manager.effectiveSubnetCidr(REGION, "subnet-a").orElseThrow()); + assertEquals("10.240.16.0/20", manager.effectiveSubnetCidr(REGION, "subnet-b").orElseThrow()); + assertEquals("10.240.32.0/20", manager.effectiveSubnetCidr(REGION, "subnet-c").orElseThrow(), + "a restart that declares subnets in another order must reach the same plan"); + } + @Test void aSubnetOutsideItsVpcRangeIsRemappedInsideIt() { manager.declareVpc(REGION, "vpc-1", "10.0.0.0/16"); diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsIsolationDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsIsolationDockerIntegrationTest.java deleted file mode 100644 index 50f6711..0000000 --- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsIsolationDockerIntegrationTest.java +++ /dev/null @@ -1,169 +0,0 @@ -package io.github.hectorvent.floci.services.ecs.container; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.core.command.WaitContainerResultCallback; -import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager; -import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition; -import io.github.hectorvent.floci.services.ecs.model.EcsTask; -import io.github.hectorvent.floci.services.ecs.model.EfsVolumeConfiguration; -import io.github.hectorvent.floci.services.ecs.model.MountPoint; -import io.github.hectorvent.floci.services.ecs.model.TaskDefinition; -import io.github.hectorvent.floci.services.ecs.model.Volume; -import io.quarkus.test.junit.QuarkusTest; -import jakarta.inject.Inject; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * Docker-backed proof that an EFS-configured task volume's local mount is scoped to - * {@code (fileSystemId, rootDirectory, accessPointId)}, not just {@code fileSystemId} (#2563). - * Each scenario runs a real busybox container against the same {@code fileSystemId}, writing - * and checking marker files under {@code /mnt/efs} with a shell one-liner whose exit code - * proves whether the previous task's data was visible. - */ -@QuarkusTest -class EcsContainerManagerEfsIsolationDockerIntegrationTest { - - private static final String IMAGE = "public.ecr.aws/docker/library/busybox:latest"; - - @Inject - EcsContainerManager containerManager; - - @Inject - ContainerLifecycleManager lifecycleManager; - - @Inject - DockerClient dockerClient; - - private final List volumesToCleanUp = new ArrayList<>(); - - @BeforeEach - void requireDocker() { - Assumptions.assumeTrue(isDockerAvailable(), - "Docker daemon must be available for ECS EFS isolation integration tests"); - } - - @AfterEach - void cleanUpVolumes() { - volumesToCleanUp.forEach(lifecycleManager::removeVolume); - volumesToCleanUp.clear(); - } - - /** - * Task A writes a marker under {@code rootDirectory} {@code /team-a}; task B, on the same - * file system but a different root, must not see it (if the bug were still present, one - * shared volume keyed only by {@code fileSystemId} would leak team-a's marker into team-b's - * task before it ever reaches its own write); a third task back on {@code /team-a} must - * likewise not see what team-b wrote. - */ - @Test - void differentRootDirectoriesOnSameFileSystemAreIsolated() { - String fileSystemId = "fs-" + UUID.randomUUID().toString().substring(0, 8); - - assertEquals(0, runEfsCommand(fileSystemId, "/team-a", null, - "echo team-a > /mnt/efs/only-a.txt"), - "task A should write its marker cleanly"); - - assertEquals(0, runEfsCommand(fileSystemId, "/team-b", null, - "test ! -e /mnt/efs/only-a.txt && echo team-b > /mnt/efs/only-b.txt"), - "task B must not see task A's rootDirectory-scoped data"); - - assertEquals(0, runEfsCommand(fileSystemId, "/team-a", null, - "test ! -e /mnt/efs/only-b.txt"), - "task on /team-a must not see /team-b's data"); - } - - /** - * A second task using the identical {@code rootDirectory} must see the first task's data: - * sharing an identical configuration is existing, desired behaviour. - */ - @Test - void sameRootDirectoryOnSameFileSystemStillShares() { - String fileSystemId = "fs-" + UUID.randomUUID().toString().substring(0, 8); - - assertEquals(0, runEfsCommand(fileSystemId, "/shared", null, - "echo hello > /mnt/efs/shared.txt"), - "first task should write its marker cleanly"); - - assertEquals(0, runEfsCommand(fileSystemId, "/shared", null, - "grep -qx hello /mnt/efs/shared.txt"), - "a second task with the same rootDirectory must share the first task's data"); - } - - /** - * A plain {@code rootDirectory} mount (no {@code accessPointId}) on the same file system - * must not see access-point-scoped data and vice versa, while a second task through the - * same access point must still share its data. - */ - @Test - void accessPointIdIsolatesFromRootDirectoryOnSameFileSystem() { - String fileSystemId = "fs-" + UUID.randomUUID().toString().substring(0, 8); - String accessPointId = "fsap-" + UUID.randomUUID().toString().substring(0, 8); - - assertEquals(0, runEfsCommand(fileSystemId, null, accessPointId, - "echo via-ap > /mnt/efs/via-ap.txt"), - "task mounted through the access point should write its marker cleanly"); - - assertEquals(0, runEfsCommand(fileSystemId, "/", null, - "test ! -e /mnt/efs/via-ap.txt"), - "a plain rootDirectory mount must not see access-point-scoped data"); - - assertEquals(0, runEfsCommand(fileSystemId, null, accessPointId, - "grep -qx via-ap /mnt/efs/via-ap.txt"), - "a second task through the same access point must share its data"); - } - - /** - * Runs a single-container ECS task that mounts {@code fileSystemId} (scoped by - * {@code rootDirectory}/{@code accessPointId}) at {@code /mnt/efs} and executes - * {@code shellCommand}, then blocks until the container exits and returns its exit code. - */ - private int runEfsCommand(String fileSystemId, String rootDirectory, String accessPointId, String shellCommand) { - String suffix = UUID.randomUUID().toString().substring(0, 8); - - ContainerDefinition app = new ContainerDefinition(); - app.setName("app"); - app.setImage(IMAGE); - app.setCommand(List.of("sh", "-c", shellCommand)); - app.setMountPoints(List.of(new MountPoint("efs-data", "/mnt/efs", false))); - - TaskDefinition taskDef = new TaskDefinition(); - taskDef.setFamily("efs-isolation-" + suffix); - taskDef.setContainerDefinitions(List.of(app)); - taskDef.setVolumes(List.of(new Volume("efs-data", null, - new EfsVolumeConfiguration(fileSystemId, rootDirectory, null, null, accessPointId, null)))); - - EcsTask task = new EcsTask(); - task.setTaskArn("arn:aws:ecs:us-east-1:000000000000:task/efs-isolation/" + suffix); - - volumesToCleanUp.add(EcsContainerManager.efsVolumeName(fileSystemId, accessPointId, rootDirectory)); - - EcsTaskHandle handle = containerManager.startTask(task, taskDef, List.of(), "us-east-1"); - String dockerId = handle.getContainerIds().get("app"); - try { - return dockerClient.waitContainerCmd(dockerId) - .exec(new WaitContainerResultCallback()) - .awaitStatusCode(60, TimeUnit.SECONDS); - } finally { - containerManager.stopTask(handle); - } - } - - private boolean isDockerAvailable() { - try { - dockerClient.pingCmd().exec(); - return true; - } catch (Exception e) { - return false; - } - } -} diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsSharingDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsSharingDockerIntegrationTest.java new file mode 100644 index 0000000..6a4856c --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsSharingDockerIntegrationTest.java @@ -0,0 +1,194 @@ +package io.github.hectorvent.floci.services.ecs.container; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.core.command.WaitContainerResultCallback; +import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager; +import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition; +import io.github.hectorvent.floci.services.ecs.model.EcsTask; +import io.github.hectorvent.floci.services.ecs.model.EfsVolumeConfiguration; +import io.github.hectorvent.floci.services.ecs.model.MountPoint; +import io.github.hectorvent.floci.services.ecs.model.TaskDefinition; +import io.github.hectorvent.floci.services.ecs.model.Volume; +import io.github.hectorvent.floci.services.efs.EfsMounts; +import io.github.hectorvent.floci.services.efs.EfsService; +import io.github.hectorvent.floci.services.efs.model.CreateAccessPointRequest; +import io.github.hectorvent.floci.services.efs.model.CreateFileSystemRequest; +import io.github.hectorvent.floci.services.efs.model.CreationInfo; +import io.github.hectorvent.floci.services.efs.model.PosixUser; +import io.github.hectorvent.floci.services.efs.model.RootDirectory; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Docker-backed proof that an EFS file system is one shared store, as on AWS: every task volume + * of it mounts the same Docker volume, a {@code rootDirectory} or access point mounts its own + * folder of it, and a mount of the file system root sees every folder. Each scenario runs a real + * busybox container whose exit code proves what it could see under {@code /mnt/efs}. + */ +@QuarkusTest +class EcsContainerManagerEfsSharingDockerIntegrationTest { + + private static final String IMAGE = "public.ecr.aws/docker/library/busybox:latest"; + private static final String REGION = "us-east-1"; + + @Inject + EcsContainerManager containerManager; + + @Inject + ContainerLifecycleManager lifecycleManager; + + @Inject + EfsService efsService; + + @Inject + DockerClient dockerClient; + + private final List volumesToCleanUp = new ArrayList<>(); + + @BeforeEach + void requireDocker() { + Assumptions.assumeTrue(isDockerAvailable(), + "Docker daemon must be available for ECS EFS integration tests"); + } + + @AfterEach + void cleanUpVolumes() { + volumesToCleanUp.forEach(lifecycleManager::removeVolume); + volumesToCleanUp.clear(); + } + + /** Two root directories are separate folders, and the file system root sees both. */ + @Test + void rootDirectoriesAreFoldersOfOneFileSystem() { + String fileSystemId = "fs-" + UUID.randomUUID().toString().substring(0, 8); + + assertEquals(0, runEfsCommand(fileSystemId, "/team-a", null, + "echo team-a > /mnt/efs/only-a.txt")); + assertEquals(0, runEfsCommand(fileSystemId, "/team-b", null, + "test ! -e /mnt/efs/only-a.txt && echo team-b > /mnt/efs/only-b.txt"), + "a task on /team-b must not see /team-a's files"); + assertEquals(0, runEfsCommand(fileSystemId, "/team-a", null, + "grep -qx team-a /mnt/efs/only-a.txt && test ! -e /mnt/efs/only-b.txt"), + "a second task on /team-a shares its files and still does not see /team-b's"); + assertEquals(0, runEfsCommand(fileSystemId, "/", null, + "grep -qx team-a /mnt/efs/team-a/only-a.txt && grep -qx team-b /mnt/efs/team-b/only-b.txt"), + "a task on the file system root sees every root directory's files"); + } + + /** + * An access point mounts its root directory, created on first use with its CreationInfo, and + * shares it with every other mount of the file system. + */ + @Test + void anAccessPointMountsItsFolderOfTheFileSystem() { + String fileSystemId = createFileSystem(); + String accessPointId = createAccessPoint(fileSystemId, "/apps/orders", 1001L, 1002L, "750"); + + assertEquals(0, runEfsCommand(fileSystemId, null, accessPointId, + "echo order-1 > /mnt/efs/orders.txt")); + assertEquals(0, runEfsCommand(fileSystemId, "/", null, + "grep -qx order-1 /mnt/efs/apps/orders/orders.txt" + + " && test \"$(stat -c '%u:%g %a' /mnt/efs/apps/orders)\" = '1001:1002 750'" + + " && test \"$(stat -c '%u:%g %a' /mnt/efs/apps)\" = '1001:1002 750'"), + "the root mount sees the access point's files, in folders created from its CreationInfo"); + assertEquals(0, runEfsCommand(fileSystemId, "/apps", null, + "grep -qx order-1 /mnt/efs/orders/orders.txt"), + "a rootDirectory above the access point's folder sees it too"); + } + + /** Without CreationInfo, an access point whose root directory does not exist cannot be mounted. */ + @Test + void anAccessPointWithAMissingRootDirectoryAndNoCreationInfoIsRefused() { + String fileSystemId = createFileSystem(); + String accessPointId = createAccessPoint(fileSystemId, "/absent", null, null, null); + + assertThrows(IllegalStateException.class, + () -> runEfsCommand(fileSystemId, null, accessPointId, "true")); + } + + private String createFileSystem() { + CreateFileSystemRequest request = new CreateFileSystemRequest(); + request.setCreationToken("efs-sharing-" + UUID.randomUUID()); + return efsService.createFileSystem(request, REGION).getFileSystemId(); + } + + private String createAccessPoint(String fileSystemId, String path, Long ownerUid, Long ownerGid, + String permissions) { + RootDirectory root = new RootDirectory(); + root.setPath(path); + if (permissions != null) { + CreationInfo info = new CreationInfo(); + info.setOwnerUid(ownerUid); + info.setOwnerGid(ownerGid); + info.setPermissions(permissions); + root.setCreationInfo(info); + } + PosixUser user = new PosixUser(); + user.setUid(1001L); + user.setGid(1002L); + CreateAccessPointRequest request = new CreateAccessPointRequest(); + request.setFileSystemId(fileSystemId); + request.setRootDirectory(root); + request.setPosixUser(user); + return efsService.createAccessPoint(REGION, request).getAccessPointId(); + } + + /** + * Runs a single-container ECS task that mounts {@code fileSystemId} (through + * {@code rootDirectory} or {@code accessPointId}) at {@code /mnt/efs} and executes + * {@code shellCommand}, then blocks until the container exits and returns its exit code. + */ + private int runEfsCommand(String fileSystemId, String rootDirectory, String accessPointId, String shellCommand) { + String suffix = UUID.randomUUID().toString().substring(0, 8); + + ContainerDefinition app = new ContainerDefinition(); + app.setName("app"); + app.setImage(IMAGE); + app.setCommand(List.of("sh", "-c", shellCommand)); + app.setMountPoints(List.of(new MountPoint("efs-data", "/mnt/efs", false))); + + TaskDefinition taskDef = new TaskDefinition(); + taskDef.setFamily("efs-sharing-" + suffix); + taskDef.setContainerDefinitions(List.of(app)); + taskDef.setVolumes(List.of(new Volume("efs-data", null, + new EfsVolumeConfiguration(fileSystemId, rootDirectory, null, null, accessPointId, null)))); + + EcsTask task = new EcsTask(); + task.setTaskArn("arn:aws:ecs:us-east-1:000000000000:task/efs-sharing/" + suffix); + + if (!volumesToCleanUp.contains(EfsMounts.volumeName(fileSystemId))) { + volumesToCleanUp.add(EfsMounts.volumeName(fileSystemId)); + } + + EcsTaskHandle handle = containerManager.startTask(task, taskDef, List.of(), REGION); + String dockerId = handle.getContainerIds().get("app"); + try { + return dockerClient.waitContainerCmd(dockerId) + .exec(new WaitContainerResultCallback()) + .awaitStatusCode(60, TimeUnit.SECONDS); + } finally { + containerManager.stopTask(handle); + } + } + + private boolean isDockerAvailable() { + try { + dockerClient.pingCmd().exec(); + return true; + } catch (Exception e) { + return false; + } + } +} diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsVolumeNamingTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsVolumeNamingTest.java deleted file mode 100644 index 451f7b8..0000000 --- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsVolumeNamingTest.java +++ /dev/null @@ -1,87 +0,0 @@ -package io.github.hectorvent.floci.services.ecs.container; - -import org.junit.jupiter.api.Test; - -import java.util.regex.Pattern; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Unit tests for {@link EcsContainerManager#efsVolumeName}, the naming logic that scopes an - * EFS-backed task volume's shared Docker named volume to both {@code fileSystemId} and its - * effective root (#2563). The name must be stable for a given (fileSystemId, rootDirectory, - * accessPointId) combination (so identical mounts keep sharing one volume), distinct across - * different roots on the same file system (so mounts do not collide on the same tree), and - * always a legal Docker volume name. - */ -class EcsContainerManagerEfsVolumeNamingTest { - - private static final Pattern DOCKER_VOLUME_NAME = Pattern.compile("^[a-zA-Z0-9][a-zA-Z0-9_.-]*$"); - - @Test - void defaultsToPlainFileSystemNameWhenNoRootOrAccessPointIsSet() { - assertEquals("floci-efs-fs-abc", EcsContainerManager.efsVolumeName("fs-abc", null, null)); - assertEquals("floci-efs-fs-abc", EcsContainerManager.efsVolumeName("fs-abc", null, "")); - assertEquals("floci-efs-fs-abc", EcsContainerManager.efsVolumeName("fs-abc", null, "/")); - } - - @Test - void differentRootDirectoriesOnSameFileSystemYieldDifferentVolumes() { - String rootA = EcsContainerManager.efsVolumeName("fs-shared", null, "/team-a"); - String rootB = EcsContainerManager.efsVolumeName("fs-shared", null, "/team-b"); - String rootNone = EcsContainerManager.efsVolumeName("fs-shared", null, null); - - assertNotEquals(rootA, rootB, "different rootDirectory values must isolate the volume"); - assertNotEquals(rootA, rootNone, "a non-root rootDirectory must not collapse to the plain name"); - assertTrue(DOCKER_VOLUME_NAME.matcher(rootA).matches(), "must be docker-safe, was: " + rootA); - assertTrue(DOCKER_VOLUME_NAME.matcher(rootB).matches(), "must be docker-safe, was: " + rootB); - } - - @Test - void sameRootDirectoryOnSameFileSystemStillShares() { - String first = EcsContainerManager.efsVolumeName("fs-shared", null, "/team-a"); - String second = EcsContainerManager.efsVolumeName("fs-shared", null, "/team-a"); - - assertEquals(first, second, "identical (fileSystemId, rootDirectory) must keep sharing one volume"); - } - - @Test - void accessPointIdTakesPrecedenceOverRootDirectory() { - String viaAccessPoint = EcsContainerManager.efsVolumeName("fs-shared", "fsap-111", "/ignored"); - String viaRootOnly = EcsContainerManager.efsVolumeName("fs-shared", null, "/ignored"); - - assertNotEquals(viaAccessPoint, viaRootOnly, - "an access point's own root must take precedence over rootDirectory, not merge with it"); - } - - @Test - void differentAccessPointsOnSameFileSystemYieldDifferentVolumes() { - String apOne = EcsContainerManager.efsVolumeName("fs-shared", "fsap-111", null); - String apTwo = EcsContainerManager.efsVolumeName("fs-shared", "fsap-222", null); - - assertNotEquals(apOne, apTwo, "different accessPointId values must isolate the volume"); - } - - @Test - void sameAccessPointStillShares() { - String first = EcsContainerManager.efsVolumeName("fs-shared", "fsap-111", null); - String second = EcsContainerManager.efsVolumeName("fs-shared", "fsap-111", "/whatever-is-ignored"); - - assertEquals(first, second, - "the same accessPointId must keep sharing one volume regardless of rootDirectory"); - } - - @Test - void equivalentRootDirectorySpellingsShareTheSameVolume() { - String canonical = EcsContainerManager.efsVolumeName("fs-shared", null, "/team-a"); - - assertEquals(canonical, EcsContainerManager.efsVolumeName("fs-shared", null, "/team-a/"), - "a trailing slash must not create a distinct volume for the same directory"); - assertEquals(canonical, EcsContainerManager.efsVolumeName("fs-shared", null, "//team-a"), - "a repeated leading slash must not create a distinct volume for the same directory"); - assertEquals(canonical, EcsContainerManager.efsVolumeName("fs-shared", null, "team-a"), - "a missing leading slash must not create a distinct volume for the same directory"); - } -} diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java index c5a74b3..1a18d71 100644 --- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java @@ -89,7 +89,7 @@ class EcsContainerManagerSecurityGroupTest { containerDetector, config, regionResolver, awsEnv, mock(SsmService.class), mock(SecretsManagerService.class), mock(S3Service.class), ecrRegistryManager, mock(HostVolumePolicy.class), - ec2Service, firewallManager); + ec2Service, firewallManager, mock(io.github.hectorvent.floci.services.efs.EfsMounts.class)); } @Test diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java index 94b944f..8d9098d 100644 --- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java @@ -69,6 +69,7 @@ class EcsContainerManagerVolumesTest { lifecycleManager = mock(ContainerLifecycleManager.class); when(lifecycleManager.createAndStart(any())) .thenReturn(new ContainerInfo("docker-id", Map.of())); + when(lifecycleManager.ensureSharedVolumeDirectory(anyString(), anyString(), any(), any())).thenReturn(true); ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class); ContainerDetector containerDetector = mock(ContainerDetector.class); @@ -132,9 +133,8 @@ class EcsContainerManagerVolumesTest { } /** - * Each EFS volume resolves to a shared local Docker named volume keyed by file system id - * AND effective root (here: rootDirectory, since neither volume sets an accessPointId), - * read-write or read-only per the mountPoint. + * Each EFS volume mounts its file system's one Docker named volume: the whole volume for the + * file system root, or the rootDirectory's folder of it, read-write or read-only per the mountPoint. */ @Test void efsVolumeMountPointsResolveToSharedNamedVolumesByAccessMode() { @@ -159,10 +159,10 @@ class EcsContainerManagerVolumesTest { manager.startTask(task, taskDef, List.of(), "us-east-1"); + verify(builder, times(1)).withNamedVolumeSubpath( + "floci-efs-fs-0123456789abcdef0", "dps", "/mnt/efs", false); verify(builder, times(1)).withNamedVolume( - EcsContainerManager.efsVolumeName("fs-0123456789abcdef0", null, "/dps"), "/mnt/efs", false); - verify(builder, times(1)).withNamedVolume( - EcsContainerManager.efsVolumeName("fs-00000000000000001", null, null), "/mnt/shared", true); + "floci-efs-fs-00000000000000001", "/mnt/shared", true); // EFS volumes are never bind-mounted as host paths. verify(builder, never()).withBind(any(), any()); @@ -202,7 +202,7 @@ class EcsContainerManagerVolumesTest { configured.startTask(task, taskDef, List.of(), "us-east-1"); verify(lifecycleManager, times(1)).ensureSharedVolume( - EcsContainerManager.efsVolumeName("fs-abc", null, "/dps"), + "floci-efs-fs-abc", OptionalInt.of(1001), OptionalInt.of(1001), Optional.of("2775"), "busybox:stable"); } diff --git a/src/test/java/io/github/hectorvent/floci/services/efs/EfsAccessPointValidationTest.java b/src/test/java/io/github/hectorvent/floci/services/efs/EfsAccessPointValidationTest.java new file mode 100644 index 0000000..eba8c58 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/efs/EfsAccessPointValidationTest.java @@ -0,0 +1,81 @@ +package io.github.hectorvent.floci.services.efs; + +import io.github.hectorvent.floci.testing.RestAssuredJsonUtils; +import io.quarkus.test.junit.QuarkusTest; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.equalTo; + +/** CreateAccessPoint refuses a root directory or identity that EFS itself would refuse. */ +@QuarkusTest +class EfsAccessPointValidationTest { + + @BeforeAll + static void configureRestAssured() { + RestAssuredJsonUtils.configureAwsContentTypes(); + } + + private String createFileSystem(String token) { + return given() + .contentType("application/json") + .body("{\"CreationToken\": \"" + token + "\"}") + .when() + .post("/2015-02-01/file-systems") + .then() + .statusCode(201) + .extract().jsonPath().getString("FileSystemId"); + } + + private io.restassured.response.ValidatableResponse createAccessPoint(String body) { + return given() + .contentType("application/json") + .body(body) + .when() + .post("/2015-02-01/access-points") + .then(); + } + + @Test + void acceptsANestedRootDirectoryWithCreationInfo() { + String fileSystemId = createFileSystem("ap-valid"); + + createAccessPoint("{\"FileSystemId\": \"" + fileSystemId + "\"," + + " \"PosixUser\": {\"Uid\": 1001, \"Gid\": 1001}," + + " \"RootDirectory\": {\"Path\": \"/apps/my-function\"," + + " \"CreationInfo\": {\"OwnerUid\": 1001, \"OwnerGid\": 1001, \"Permissions\": \"0755\"}}}") + .statusCode(200) + .body("RootDirectory.Path", equalTo("/apps/my-function")); + } + + @Test + void refusesARootDirectoryThatClimbsOutOrHidesItself() { + String fileSystemId = createFileSystem("ap-bad-path"); + + for (String path : new String[] {"/a/../b", "/.hidden", "relative", "/a;rm"}) { + createAccessPoint("{\"FileSystemId\": \"" + fileSystemId + "\"," + + " \"RootDirectory\": {\"Path\": \"" + path + "\"}}") + .statusCode(400) + .body("__type", equalTo("BadRequest")); + } + } + + @Test + void refusesPartialCreationInfoAndPosixUser() { + String fileSystemId = createFileSystem("ap-partial"); + + createAccessPoint("{\"FileSystemId\": \"" + fileSystemId + "\"," + + " \"RootDirectory\": {\"Path\": \"/data\", \"CreationInfo\": {\"OwnerUid\": 1001}}}") + .statusCode(400) + .body("__type", equalTo("BadRequest")); + createAccessPoint("{\"FileSystemId\": \"" + fileSystemId + "\"," + + " \"RootDirectory\": {\"Path\": \"/data\"," + + " \"CreationInfo\": {\"OwnerUid\": 1001, \"OwnerGid\": 1001, \"Permissions\": \"rwx\"}}}") + .statusCode(400) + .body("__type", equalTo("BadRequest")); + createAccessPoint("{\"FileSystemId\": \"" + fileSystemId + "\", \"PosixUser\": {\"Uid\": 1001}}") + .statusCode(400) + .body("__type", equalTo("BadRequest")); + } +} diff --git a/src/test/java/io/github/hectorvent/floci/services/efs/EfsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/efs/EfsIntegrationTest.java index 2a567f3..da813be 100644 --- a/src/test/java/io/github/hectorvent/floci/services/efs/EfsIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/efs/EfsIntegrationTest.java @@ -91,7 +91,7 @@ class EfsIntegrationTest { { "FileSystemId": "%s", "SubnetId": "subnet-default-us-east-1-a", - "SecurityGroups": ["sg-11111"] + "SecurityGroups": ["sg-default-us-east-1"] } """.formatted(fileSystemId)) .when() @@ -103,7 +103,9 @@ class EfsIntegrationTest { .body("SubnetId", equalTo("subnet-default-us-east-1-a")) .body("VpcId", equalTo("vpc-default-us-east-1")) .body("AvailabilityZoneName", equalTo("us-east-1a")) - .body("IpAddress", equalTo("172.31.0.4")) + .body("IpAddress", startsWith("172.31.0.")) + .body("NetworkInterfaceId", startsWith("eni-")) + .body("LifeCycleState", equalTo("available")) .extract().jsonPath().getString("MountTargetId"); } diff --git a/src/test/java/io/github/hectorvent/floci/services/efs/EfsMountTargetSubnetTest.java b/src/test/java/io/github/hectorvent/floci/services/efs/EfsMountTargetSubnetTest.java index d7ae399..86f6ca6 100644 --- a/src/test/java/io/github/hectorvent/floci/services/efs/EfsMountTargetSubnetTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/efs/EfsMountTargetSubnetTest.java @@ -2,17 +2,27 @@ package io.github.hectorvent.floci.services.efs; import io.github.hectorvent.floci.testing.RestAssuredJsonUtils; import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import java.util.Optional; + import static io.restassured.RestAssured.given; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.startsWith; import static org.junit.jupiter.api.Assertions.assertEquals; -/** Mount targets take their VPC, zone and address from the EC2 subnet they are created in. */ +/** + * Mount targets take their VPC, zone and address from the EC2 subnet they are created in, hold an + * EC2 network interface for that address, and answer the file system's DNS name. + */ @QuarkusTest class EfsMountTargetSubnetTest { + @Inject + EfsService efsService; + @BeforeAll static void configureRestAssured() { RestAssuredJsonUtils.configureAwsContentTypes(); @@ -46,7 +56,7 @@ class EfsMountTargetSubnetTest { .statusCode(200) .body("VpcId", equalTo("vpc-default-us-east-1")) .body("AvailabilityZoneName", equalTo("us-east-1b")) - .body("IpAddress", equalTo("172.31.16.4")); + .body("IpAddress", startsWith("172.31.16.")); createMountTarget(fileSystemId, "subnet-default-us-east-1-c") .statusCode(200) .body("AvailabilityZoneName", equalTo("us-east-1c")); @@ -61,6 +71,21 @@ class EfsMountTargetSubnetTest { .body("__type", equalTo("SubnetNotFound")); } + @Test + void refusesASecurityGroupThatDoesNotExist() { + String fileSystemId = createFileSystem("subnet-missing-sg"); + + given() + .contentType("application/json") + .body("{\"FileSystemId\": \"" + fileSystemId + "\", \"SubnetId\": \"subnet-default-us-east-1-a\"," + + " \"SecurityGroups\": [\"sg-nope\"]}") + .when() + .post("/2015-02-01/mount-targets") + .then() + .statusCode(400) + .body("__type", equalTo("SecurityGroupNotFound")); + } + @Test void allowsOneMountTargetPerZone() { String fileSystemId = createFileSystem("subnet-one-per-zone"); @@ -84,8 +109,55 @@ class EfsMountTargetSubnetTest { } @Test - void picksAddressesAfterTheFourAwsReserves() { - assertEquals("10.0.1.4", EfsService.hostAddress("10.0.1.0/24", 0)); - assertEquals("10.0.2.1", EfsService.hostAddress("10.0.1.0/24", 253)); + void holdsAnEc2NetworkInterfaceWithItsAddressUntilDeleted() { + String fileSystemId = createFileSystem("subnet-eni"); + var mountTarget = createMountTarget(fileSystemId, "subnet-default-us-east-1-c") + .statusCode(200) + .extract().jsonPath(); + String eniId = mountTarget.getString("NetworkInterfaceId"); + + describeNetworkInterface(eniId) + .statusCode(200) + .body("DescribeNetworkInterfacesResponse.networkInterfaceSet.item.privateIpAddress", + equalTo(mountTarget.getString("IpAddress"))) + .body("DescribeNetworkInterfacesResponse.networkInterfaceSet.item.status", equalTo("in-use")) + .body("DescribeNetworkInterfacesResponse.networkInterfaceSet.item.subnetId", + equalTo("subnet-default-us-east-1-c")); + + given() + .when() + .delete("/2015-02-01/mount-targets/" + mountTarget.getString("MountTargetId")) + .then() + .statusCode(204); + describeNetworkInterface(eniId).statusCode(400); + } + + @Test + void resolvesTheFileSystemDnsNameToAMountTarget() { + String fileSystemId = createFileSystem("subnet-dns"); + String inZoneB = createMountTarget(fileSystemId, "subnet-default-us-east-1-b") + .statusCode(200).extract().jsonPath().getString("IpAddress"); + String inZoneA = createMountTarget(fileSystemId, "subnet-default-us-east-1-a") + .statusCode(200).extract().jsonPath().getString("IpAddress"); + + assertEquals(Optional.of(inZoneA), + efsService.resolveDnsName(fileSystemId + ".efs.us-east-1.amazonaws.com")); + assertEquals(Optional.of(inZoneB), + efsService.resolveDnsName("us-east-1b." + fileSystemId + ".efs.us-east-1.amazonaws.com.")); + assertEquals(Optional.empty(), + efsService.resolveDnsName("us-east-1c." + fileSystemId + ".efs.us-east-1.amazonaws.com")); + assertEquals(Optional.empty(), + efsService.resolveDnsName(fileSystemId + ".efs.eu-west-1.amazonaws.com")); + assertEquals(Optional.empty(), efsService.resolveDnsName("example.com")); + } + + private io.restassured.response.ValidatableResponse describeNetworkInterface(String eniId) { + return given() + .formParam("Action", "DescribeNetworkInterfaces") + .formParam("NetworkInterfaceId.1", eniId) + .header("Authorization", "AWS4-HMAC-SHA256 Credential=test/20260101/us-east-1/ec2/aws4_request") + .when() + .post("/") + .then(); } } diff --git a/src/test/java/io/github/hectorvent/floci/services/efs/EfsMountsTest.java b/src/test/java/io/github/hectorvent/floci/services/efs/EfsMountsTest.java new file mode 100644 index 0000000..28d282e --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/efs/EfsMountsTest.java @@ -0,0 +1,170 @@ +package io.github.hectorvent.floci.services.efs; + +import io.github.hectorvent.floci.config.EmulatorConfig; +import io.github.hectorvent.floci.core.common.docker.ContainerBuilder; +import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager; +import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.DirectoryCreation; +import io.github.hectorvent.floci.services.efs.model.AccessPointDescription; +import io.github.hectorvent.floci.services.efs.model.CreationInfo; +import io.github.hectorvent.floci.services.efs.model.PosixUser; +import io.github.hectorvent.floci.services.efs.model.RootDirectory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.RETURNS_SELF; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Every mount of a file system shares its one volume; access points mount their root directory in it. */ +class EfsMountsTest { + + private EfsService efsService; + private ContainerLifecycleManager lifecycleManager; + private EfsMounts mounts; + + @BeforeEach + void setUp() { + efsService = mock(EfsService.class); + lifecycleManager = mock(ContainerLifecycleManager.class); + when(lifecycleManager.ensureSharedVolumeDirectory(anyString(), anyString(), any(), any())).thenReturn(true); + EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS); + when(config.storage().efs().ownerUid()).thenReturn(OptionalInt.empty()); + when(config.storage().efs().ownerGid()).thenReturn(OptionalInt.empty()); + when(config.storage().efs().rootPermissions()).thenReturn(Optional.empty()); + when(config.storage().efs().initImage()).thenReturn("busybox:stable"); + mounts = new EfsMounts(efsService, lifecycleManager, config); + } + + @Test + void theFileSystemRootMountsTheWholeVolume() { + EfsMounts.Mount mount = mounts.prepare("us-east-1", "fs-abc", null, null); + + assertEquals("floci-efs-fs-abc", mount.volumeName()); + assertEquals("", mount.subpath()); + assertTrue(mount.accessPointUser().isEmpty()); + verify(lifecycleManager).ensureSharedVolume("floci-efs-fs-abc", + OptionalInt.empty(), OptionalInt.empty(), Optional.empty(), "busybox:stable"); + verify(lifecycleManager, never()).ensureSharedVolumeDirectory(anyString(), anyString(), any(), any()); + } + + @Test + void aRootDirectoryMountsItsFolderOfTheSameVolume() { + EfsMounts.Mount mount = mounts.prepare("us-east-1", "fs-abc", null, "/team-a/"); + + assertEquals("floci-efs-fs-abc", mount.volumeName()); + assertEquals("team-a", mount.subpath()); + verify(lifecycleManager).ensureSharedVolumeDirectory("floci-efs-fs-abc", "team-a", + Optional.of(new DirectoryCreation(0, 0, "755")), "busybox:stable"); + } + + @Test + void anAccessPointMountsItsRootDirectoryAndCarriesItsUser() { + when(efsService.describeAccessPoints("us-east-1", null, "fsap-1")) + .thenReturn(List.of(accessPoint("fs-abc", "/my-function", creationInfo(1001L, 1001L, "755")))); + + EfsMounts.Mount mount = mounts.prepare("us-east-1", null, "fsap-1", "/ignored"); + + assertEquals("floci-efs-fs-abc", mount.volumeName()); + assertEquals("my-function", mount.subpath()); + assertEquals("1001:1001", EfsMounts.containerUser(mount.accessPointUser().orElseThrow())); + assertEquals(List.of(2000L), EfsMounts.secondaryGids(mount.accessPointUser().orElseThrow())); + verify(lifecycleManager).ensureSharedVolumeDirectory("floci-efs-fs-abc", "my-function", + Optional.of(new DirectoryCreation(1001, 1001, "755")), "busybox:stable"); + } + + @Test + void aLambdaAccessPointArnIsLookedUpInItsRegion() { + when(efsService.describeAccessPoints("eu-west-1", null, "fsap-1")) + .thenReturn(List.of(accessPoint("fs-abc", "/", null))); + + EfsMounts.Mount mount = mounts.prepareAccessPoint( + "arn:aws:elasticfilesystem:eu-west-1:000000000000:access-point/fsap-1"); + + assertEquals("floci-efs-fs-abc", mount.volumeName()); + assertEquals("", mount.subpath()); + } + + @Test + void aMissingRootDirectoryWithoutCreationInfoCannotBeMounted() { + when(efsService.describeAccessPoints("us-east-1", null, "fsap-1")) + .thenReturn(List.of(accessPoint("fs-abc", "/absent", null))); + when(lifecycleManager.ensureSharedVolumeDirectory("floci-efs-fs-abc", "absent", Optional.empty(), + "busybox:stable")).thenReturn(false); + + IllegalStateException error = assertThrows(IllegalStateException.class, + () -> mounts.prepare("us-east-1", "fs-abc", "fsap-1", null)); + + assertTrue(error.getMessage().contains("/absent"), error.getMessage()); + } + + @Test + void anAccessPointOfAnotherFileSystemIsRefused() { + when(efsService.describeAccessPoints("us-east-1", null, "fsap-1")) + .thenReturn(List.of(accessPoint("fs-other", "/", null))); + + assertThrows(IllegalArgumentException.class, () -> mounts.prepare("us-east-1", "fs-abc", "fsap-1", null)); + } + + @Test + void anUnknownAccessPointIsRefused() { + when(efsService.describeAccessPoints("us-east-1", null, "fsap-404")).thenReturn(List.of()); + + assertThrows(EfsException.class, () -> mounts.prepare("us-east-1", "fs-abc", "fsap-404", null)); + } + + @Test + void equivalentRootDirectorySpellingsNameOneFolder() { + assertEquals("/", EfsMounts.normalizeRootDirectory(null)); + assertEquals("/", EfsMounts.normalizeRootDirectory(" ")); + assertEquals("/team-a", EfsMounts.normalizeRootDirectory("team-a")); + assertEquals("/team-a", EfsMounts.normalizeRootDirectory("//team-a/")); + assertThrows(IllegalArgumentException.class, () -> EfsMounts.normalizeRootDirectory("/a/../b")); + } + + @Test + void aFolderMountUsesTheVolumeSubpath() { + ContainerBuilder.Builder spec = mock(ContainerBuilder.Builder.class, RETURNS_SELF); + + new EfsMounts.Mount("floci-efs-fs-abc", "team-a", Optional.empty()).applyTo(spec, "/mnt/efs", true); + new EfsMounts.Mount("floci-efs-fs-abc", "", Optional.empty()).applyTo(spec, "/mnt/all", false); + + verify(spec).withNamedVolumeSubpath("floci-efs-fs-abc", "team-a", "/mnt/efs", true); + verify(spec).withNamedVolume("floci-efs-fs-abc", "/mnt/all", false); + } + + private static AccessPointDescription accessPoint(String fileSystemId, String path, CreationInfo creationInfo) { + RootDirectory root = new RootDirectory(); + root.setPath(path); + root.setCreationInfo(creationInfo); + PosixUser user = new PosixUser(); + user.setUid(1001L); + user.setGid(1001L); + user.setSecondaryGids(List.of(2000L)); + AccessPointDescription accessPoint = new AccessPointDescription(); + accessPoint.setAccessPointId("fsap-1"); + accessPoint.setFileSystemId(fileSystemId); + accessPoint.setRootDirectory(root); + accessPoint.setPosixUser(user); + return accessPoint; + } + + private static CreationInfo creationInfo(Long uid, Long gid, String permissions) { + CreationInfo info = new CreationInfo(); + info.setOwnerUid(uid); + info.setOwnerGid(gid); + info.setPermissions(permissions); + return info; + } +} diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaImageConfigTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaImageConfigTest.java index 9d6eb13..64be928 100644 --- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaImageConfigTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaImageConfigTest.java @@ -250,7 +250,8 @@ class LambdaImageConfigTest { launcher = new ContainerLauncher(containerBuilder, lifecycleManager, logStreamer, imageResolver, runtimeApiServerFactory, dockerHostResolver, config, ecrRegistryManager, mock(io.github.hectorvent.floci.services.lambda.LambdaLayerService.class), awsEnv, - mock(io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials.class)); + mock(io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials.class), + mock(io.github.hectorvent.floci.services.efs.EfsMounts.class)); when(runtimeApiServerFactory.create()).thenReturn(runtimeApiServer); when(runtimeApiServer.getPort()).thenReturn(9000); diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherCodeVolumeReuseTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherCodeVolumeReuseTest.java index 7e6b391..7ad862a 100644 --- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherCodeVolumeReuseTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherCodeVolumeReuseTest.java @@ -59,7 +59,7 @@ class ContainerLauncherCodeVolumeReuseTest { mock(ImageResolver.class), mock(RuntimeApiServerFactory.class), mock(DockerHostResolver.class), config, mock(EcrRegistryManager.class), mock(LambdaLayerService.class), mock(LaunchedContainerAwsEnv.class), - mock(LambdaExecutionRoleCredentials.class)); + mock(LambdaExecutionRoleCredentials.class), mock(io.github.hectorvent.floci.services.efs.EfsMounts.class)); } @Override diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java index 0a892bb..9bea284 100644 --- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java @@ -12,6 +12,9 @@ import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv; import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager; import io.github.hectorvent.floci.services.iam.model.SessionCreds; import io.github.hectorvent.floci.services.lambda.model.LambdaFileSystemConfig; +import io.github.hectorvent.floci.services.efs.EfsMounts; +import io.github.hectorvent.floci.services.efs.model.PosixUser; +import io.github.hectorvent.floci.core.common.docker.SubpathVolumeOptions; import io.github.hectorvent.floci.services.lambda.model.LambdaFunction; import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer; import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory; @@ -87,6 +90,7 @@ class ContainerLauncherTest { Path tempDir; ContainerLauncher launcher; + EfsMounts efsMounts = mock(EfsMounts.class); /** Collects remote paths passed to withRemotePath across all copy mocks. */ final java.util.List capturedRemotePaths = new java.util.ArrayList<>(); @@ -134,7 +138,7 @@ class ContainerLauncherTest { launcher = new ContainerLauncher(containerBuilder, lifecycleManager, logStreamer, imageResolver, runtimeApiServerFactory, dockerHostResolver, config, ecrRegistryManager, mock(io.github.hectorvent.floci.services.lambda.LambdaLayerService.class), awsEnv, - executionRoleCredentials); + executionRoleCredentials, efsMounts); when(runtimeApiServerFactory.create()).thenReturn(runtimeApiServer); when(runtimeApiServer.getPort()).thenReturn(9000); @@ -469,8 +473,15 @@ class ContainerLauncherTest { } @Test - void launchFunction_mountsConfiguredFileSystemVolume() throws Exception { + void launchFunction_mountsTheAccessPointFolderAsItsUser() throws Exception { Path codePath = Files.createDirectory(tempDir.resolve("efs-code")); + String accessPointArn = "arn:aws:elasticfilesystem:us-east-1:000000000000:access-point/fsap-0123456789abcdef0"; + PosixUser user = new PosixUser(); + user.setUid(1001L); + user.setGid(1002L); + user.setSecondaryGids(List.of(2000L)); + when(efsMounts.prepareAccessPoint(accessPointArn)) + .thenReturn(new EfsMounts.Mount("floci-efs-fs-abc", "my-function", Optional.of(user))); LambdaFunction fn = new LambdaFunction(); fn.setFunctionName("efs-fn"); @@ -478,23 +489,48 @@ class ContainerLauncherTest { fn.setRuntime("nodejs20.x"); fn.setHandler("index.handler"); fn.setCodeLocalPath(codePath.toString()); - fn.setFileSystemConfigs(List.of(new LambdaFileSystemConfig( - "arn:aws:elasticfilesystem:us-east-1:000000000000:access-point/fsap-0123456789abcdef0", - "/mnt/shared"))); + fn.setFileSystemConfigs(List.of(new LambdaFileSystemConfig(accessPointArn, "/mnt/shared"))); launcher.launch(fn); - String expectedVolumeName = "floci-efs-fsap-0123456789abcdef0-" - + "9d6eafd2aec94d4518a004f005725b4b3c673c1506436bb7368cfd5450fc0810"; - verify(lifecycleManager).ensureSharedVolume(expectedVolumeName, - OptionalInt.empty(), OptionalInt.empty(), Optional.empty(), "busybox:stable"); - Mount mount = captureRealContainerSpec().mounts().stream() + ContainerSpec spec = captureRealContainerSpec(); + Mount mount = spec.mounts().stream() .filter(candidate -> "/mnt/shared".equals(candidate.getTarget())) .findFirst() .orElseThrow(); assertEquals(MountType.VOLUME, mount.getType()); - assertEquals(expectedVolumeName, mount.getSource()); + assertEquals("floci-efs-fs-abc", mount.getSource()); + assertEquals("my-function", ((SubpathVolumeOptions) mount.getVolumeOptions()).getSubpath()); assertTrue(!Boolean.TRUE.equals(mount.getReadOnly())); + assertEquals("1001:1002", spec.user()); + assertEquals(List.of("2000"), spec.groupAdd()); + } + + @Test + void launchFunction_withoutAnAccessPointUserKeepsTheImageUser() throws Exception { + Path codePath = Files.createDirectory(tempDir.resolve("efs-root-code")); + String accessPointArn = "arn:aws:elasticfilesystem:us-east-1:000000000000:access-point/fsap-1"; + when(efsMounts.prepareAccessPoint(accessPointArn)) + .thenReturn(new EfsMounts.Mount("floci-efs-fs-abc", "", Optional.empty())); + + LambdaFunction fn = new LambdaFunction(); + fn.setFunctionName("efs-root-fn"); + fn.setFunctionArn("arn:aws:lambda:us-east-1:000000000000:function:efs-root-fn"); + fn.setRuntime("nodejs20.x"); + fn.setHandler("index.handler"); + fn.setCodeLocalPath(codePath.toString()); + fn.setFileSystemConfigs(List.of(new LambdaFileSystemConfig(accessPointArn, "/mnt/efs"))); + + launcher.launch(fn); + + ContainerSpec spec = captureRealContainerSpec(); + Mount mount = spec.mounts().stream() + .filter(candidate -> "/mnt/efs".equals(candidate.getTarget())) + .findFirst() + .orElseThrow(); + assertEquals("floci-efs-fs-abc", mount.getSource()); + assertEquals(null, mount.getVolumeOptions()); + assertEquals(null, spec.user()); } @Test @@ -1631,7 +1667,7 @@ class ContainerLauncherTest { ecrRegistryManager, mock(io.github.hectorvent.floci.services.lambda.LambdaLayerService.class), new LaunchedContainerAwsEnv(reachableEndpoint), - executionRoleCredentials); + executionRoleCredentials, efsMounts); stubExtensionDiscovery("otel-collector"); // Feed a real stdout frame through whatever callback the launcher hands to execStartCmd. diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherVolumeNamingTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherVolumeNamingTest.java index 377b45c..bb0c323 100644 --- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherVolumeNamingTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherVolumeNamingTest.java @@ -100,33 +100,6 @@ class ContainerLauncherVolumeNamingTest { "a later lastModified must produce a different volume name"); } - @Test - void efsVolumeNameIncludesStableHashOfCompleteArn() { - String arn = "arn:aws:elasticfilesystem:us-east-1:000000000000:" - + "access-point/fsap-0123456789abcdef0"; - - String volumeName = ContainerLauncher.efsVolumeName(arn); - - assertEquals("floci-efs-fsap-0123456789abcdef0-" - + "9d6eafd2aec94d4518a004f005725b4b3c673c1506436bb7368cfd5450fc0810", - volumeName); - assertEquals(volumeName, ContainerLauncher.efsVolumeName(arn)); - assertTrue(DOCKER_VOLUME_NAME.matcher(volumeName).matches(), - "EFS volume name must be Docker-safe, was: " + volumeName); - } - - @Test - void efsVolumeNameDistinguishesSameResourceIdAcrossArns() { - String eastArn = "arn:aws:elasticfilesystem:us-east-1:000000000000:" - + "access-point/fsap-0123456789abcdef0"; - String westArn = "arn:aws:elasticfilesystem:us-west-2:111111111111:" - + "access-point/fsap-0123456789abcdef0"; - - assertNotEquals(ContainerLauncher.efsVolumeName(eastArn), - ContainerLauncher.efsVolumeName(westArn), - "the complete ARN must participate in volume identity"); - } - @TempDir Path tempDir; diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml index e34e5f7..85f24dd 100644 --- a/src/test/resources/application.yml +++ b/src/test/resources/application.yml @@ -434,6 +434,7 @@ floci: enabled: true efs: enabled: true + nfs-servers: false organizations: enabled: true rum: