feat(efs): one shared store per file system, served to EC2 over NFS #17

Merged
Zakaria merged 1 commits from fix/efs-shared-file-system into main 2026-10-26 02:47:20 +00:00
31 changed files with 1391 additions and 449 deletions
@@ -1040,6 +1040,18 @@ public interface EmulatorConfig {
interface EfsServiceConfig { interface EfsServiceConfig {
@WithDefault("true") @WithDefault("true")
boolean enabled(); boolean enabled();
/**
* Serve every mount target over NFSv4.1 at its own address on the VPC's Docker network,
* from a privileged helper container running the kernel NFS server, so an EC2 instance
* can mount the file system as on AWS. Needs VPC networks ({@code floci.services.ec2.vpc-networks}).
*/
@WithDefault("true")
boolean nfsServers();
/** Image of the NFS server; the default is built locally from Floci's own recipe on first use. */
@WithDefault("floci/efs-nfs-server:local")
String nfsServerImage();
} }
interface LightsailServiceConfig { interface LightsailServiceConfig {
@@ -68,6 +68,14 @@ public class EmbeddedDnsServer {
// localhost.floci.io → localhost.floci.io, *.localhost.floci.io // localhost.floci.io → localhost.floci.io, *.localhost.floci.io
public static final List<String> BUILTIN_SUFFIXES = List.of(DEFAULT_SUFFIX, LOCALSTACK_SUFFIX); public static final List<String> BUILTIN_SUFFIXES = List.of(DEFAULT_SUFFIX, LOCALSTACK_SUFFIX);
/** Answers A queries for names a service owns, such as an EFS file system's DNS name. */
@FunctionalInterface
public interface NameResolver {
Optional<String> resolve(String name);
}
private final List<NameResolver> resolvers = new java.util.concurrent.CopyOnWriteArrayList<>();
private volatile String serverIp; private volatile String serverIp;
private final SequencedSet<String> suffixes = new LinkedHashSet<>(); private final SequencedSet<String> suffixes = new LinkedHashSet<>();
private volatile List<String> upstreamDnsServers = List.of(); private volatile List<String> upstreamDnsServers = List.of();
@@ -163,11 +171,30 @@ public class EmbeddedDnsServer {
return false; return false;
} }
/** Adds names a service answers for; a name no resolver knows is forwarded upstream as before. */
public void addResolver(NameResolver resolver) {
resolvers.add(resolver);
}
Optional<String> resolveARecord(String name, String myIp) { Optional<String> resolveARecord(String name, String myIp) {
if (matchesSuffix(name)) { if (matchesSuffix(name)) {
return Optional.of(myIp); return Optional.of(myIp);
} }
return resolveEc2PrivateDnsName(name); Optional<String> ec2Name = resolveEc2PrivateDnsName(name);
if (ec2Name.isPresent()) {
return ec2Name;
}
for (NameResolver resolver : resolvers) {
try {
Optional<String> address = resolver.resolve(name);
if (address.isPresent()) {
return address;
}
} catch (RuntimeException e) {
LOG.debugv("DNS resolver failed for {0}: {1}", name, e.getMessage());
}
}
return Optional.empty();
} }
Optional<String> resolveEc2PrivateDnsName(String name) { Optional<String> resolveEc2PrivateDnsName(String name) {
@@ -362,6 +362,20 @@ public class ContainerBuilder {
return this; return this;
} }
/**
* Adds a named volume mount of one directory inside the volume, given relative to its root.
* The directory must already exist; Docker refuses to start the container otherwise.
*/
public Builder withNamedVolumeSubpath(String volumeName, String subpath, String containerPath, boolean readOnly) {
this.mounts.add(new Mount()
.withType(MountType.VOLUME)
.withSource(volumeName)
.withTarget(containerPath)
.withReadOnly(readOnly)
.withVolumeOptions(new SubpathVolumeOptions(subpath)));
return this;
}
/** /**
* Inherits every volume declared by another container. * Inherits every volume declared by another container.
*/ */
@@ -34,6 +34,7 @@ import java.io.ByteArrayOutputStream;
import java.io.Closeable; import java.io.Closeable;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -588,6 +589,77 @@ public class ContainerLifecycleManager {
rootPermissions.ifPresent(p -> script.append("chmod ").append(p).append(' ').append(mount).append(" && ")); rootPermissions.ifPresent(p -> script.append("chmod ").append(p).append(' ').append(mount).append(" && "));
script.append("true"); script.append("true");
Integer status = runVolumeHelper(volumeName, mount, initImage, List.of("sh", "-c", script.toString()));
if (status == null || status != 0) {
// Throw so the caller leaves the volume unmemoised and retries on the next launch,
// rather than leaving it root:root 0755 with no further attempt.
throw new IllegalStateException("shared-volume init for " + volumeName
+ " exited with status " + status + " (cmd: " + script + ")");
}
LOG.infov("Initialised shared volume {0} root (cmd: {1})", volumeName, script);
}
/** Owner and permissions for a directory created inside a shared volume (EFS {@code CreationInfo}). */
public record DirectoryCreation(long ownerUid, long ownerGid, String permissions) {
public DirectoryCreation {
if (permissions == null || !permissions.matches("^[0-7]{3,4}$")) {
throw new IllegalArgumentException("permissions must be 3-4 octal digits: " + permissions);
}
}
}
// Walks the path one directory at a time so that every directory it creates, not only the
// last, gets the owner and permissions, as EFS does for an access point's root directory.
private static final String ENSURE_DIRECTORY_SCRIPT = """
set -f
cd /floci-shared-volume || exit 2
[ -d "$1" ] && exit 0
[ "$#" -eq 4 ] || exit 3
p=
IFS=/
for part in $1; do
[ -n "$part" ] || continue
p="${p:+$p/}$part"
if [ ! -d "$p" ]; then
mkdir "$p" && chown "$2:$3" "$p" && chmod "$4" "$p" || exit 1
fi
done
""";
/**
* Ensures {@code relativePath} exists inside the named volume, emulating how Amazon EFS creates
* an access point's root directory on first use: each missing directory on the path is created
* with {@code creation}'s owner and permissions. When the directory is missing and
* {@code creation} is empty nothing is created and this returns false, since EFS then refuses
* the mount. The volume itself must already exist.
*/
public boolean ensureSharedVolumeDirectory(String volumeName, String relativePath,
Optional<DirectoryCreation> creation, String initImage) {
if (relativePath == null || relativePath.isBlank() || relativePath.startsWith("/")
|| List.of(relativePath.split("/")).contains("..")) {
throw new IllegalArgumentException("directory must be relative to the volume root: " + relativePath);
}
String memoKey = volumeName + "/" + relativePath;
if (initializedSharedVolumes.containsKey(memoKey)) {
return true;
}
List<String> cmd = new ArrayList<>(List.of("sh", "-c", ENSURE_DIRECTORY_SCRIPT, "sh", relativePath));
creation.ifPresent(c -> cmd.addAll(List.of(
Long.toString(c.ownerUid()), Long.toString(c.ownerGid()), c.permissions())));
Integer status = runVolumeHelper(volumeName, "/floci-shared-volume", initImage, cmd);
if (status != null && status == 3) {
return false;
}
if (status == null || status != 0) {
throw new IllegalStateException("creating " + relativePath + " in volume " + volumeName
+ " exited with status " + status);
}
initializedSharedVolumes.put(memoKey, Boolean.TRUE);
return true;
}
/** Runs a one-off helper container with the volume mounted at {@code mount}; returns its exit status. */
private Integer runVolumeHelper(String volumeName, String mount, String initImage, List<String> cmd) {
String image = (initImage != null && !initImage.isBlank()) ? initImage : "busybox:stable"; String image = (initImage != null && !initImage.isBlank()) ? initImage : "busybox:stable";
imageCacheService.ensureImageExists(image); imageCacheService.ensureImageExists(image);
@@ -595,22 +667,15 @@ public class ContainerLifecycleManager {
new Mount().withType(MountType.VOLUME).withSource(volumeName).withTarget(mount))); new Mount().withType(MountType.VOLUME).withSource(volumeName).withTarget(mount)));
CreateContainerResponse created = dockerClient.createContainerCmd(image) CreateContainerResponse created = dockerClient.createContainerCmd(image)
.withHostConfig(hostConfig) .withHostConfig(hostConfig)
.withCmd("sh", "-c", script.toString()) .withCmd(cmd.toArray(String[]::new))
.withLabels(ContainerStorageHelper.defaultLabels(config)) .withLabels(ContainerStorageHelper.defaultLabels(config))
.exec(); .exec();
String helperId = created.getId(); String helperId = created.getId();
try { try {
startContainer(helperId); startContainer(helperId);
Integer status = dockerClient.waitContainerCmd(helperId) return dockerClient.waitContainerCmd(helperId)
.exec(new WaitContainerResultCallback()) .exec(new WaitContainerResultCallback())
.awaitStatusCode(60, TimeUnit.SECONDS); .awaitStatusCode(60, TimeUnit.SECONDS);
if (status == null || status != 0) {
// Throw so the caller leaves the volume unmemoised and retries on the next launch,
// rather than leaving it root:root 0755 with no further attempt.
throw new IllegalStateException("shared-volume init for " + volumeName
+ " exited with status " + status + " (cmd: " + script + ")");
}
LOG.infov("Initialised shared volume {0} root (cmd: {1})", volumeName, script);
} finally { } finally {
try { try {
dockerClient.removeContainerCmd(helperId).withForce(true).exec(); dockerClient.removeContainerCmd(helperId).withForce(true).exec();
@@ -0,0 +1,26 @@
package io.github.hectorvent.floci.core.common.docker;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.github.dockerjava.api.model.VolumeOptions;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* {@link VolumeOptions} with the {@code Subpath} field (Docker Engine API 1.45+), which mounts one
* directory of a named volume instead of its root. docker-java 3.7 does not model the field yet.
*/
@RegisterForReflection
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SubpathVolumeOptions extends VolumeOptions {
@JsonProperty("Subpath")
private final String subpath;
public SubpathVolumeOptions(String subpath) {
this.subpath = subpath;
}
public String getSubpath() {
return subpath;
}
}
@@ -8554,6 +8554,12 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
"Network interface '" + networkInterfaceId + "' is currently in use", 400); "Network interface '" + networkInterfaceId + "' is currently in use", 400);
} }
networkInterfaces.delete(key(region, networkInterfaceId)); networkInterfaces.delete(key(region, networkInterfaceId));
// Its addresses go back to the subnet, or every task or mount target that ever held one
// would leave it leased until the emulator restarts.
if (vpcNetworkManager != null && ni.getPrivateIpAddresses() != null) {
ni.getPrivateIpAddresses().forEach(address ->
vpcNetworkManager.releasePrivateIp(region, ni.getSubnetId(), address.getPrivateIpAddress()));
}
} }
/** /**
@@ -285,7 +285,10 @@ public class VpcNetworkManager {
} }
int desiredPrefix = declared.map(Cidr4::prefix).orElse(24); int desiredPrefix = declared.map(Cidr4::prefix).orElse(24);
Cidr4 slice = allocateSubBlock(vpc.effective, desiredPrefix, siblings); Cidr4 slice = translatedIntoSubstitute(vpc, declared, siblings);
if (slice == null) {
slice = allocateSubBlock(vpc.effective, desiredPrefix, siblings);
}
if (slice == null) { if (slice == null) {
LOG.warnv("Subnet {0} in VPC {1}: no free /{2} slice remains inside {3}; instances in this " LOG.warnv("Subnet {0} in VPC {1}: no free /{2} slice remains inside {3}; instances in this "
+ "subnet fall back to synthesised private addresses.", + "subnet fall back to synthesised private addresses.",
@@ -301,6 +304,26 @@ public class VpcNetworkManager {
return bindSubnet(vpc, subnetId, declared.orElse(null), slice, true); return bindSubnet(vpc, subnetId, declared.orElse(null), slice, true);
} }
/**
* The declared subnet moved to the same place inside its VPC's substitute range, or null when
* that does not apply. Keeping each subnet's offset, rather than handing out the next free
* slice, is what makes the plan come out the same in every run whatever order the subnets are
* declared in, so the addresses persisted instances and mount targets hold stay valid.
*/
private static Cidr4 translatedIntoSubstitute(VpcBinding vpc, Optional<Cidr4> declared, List<Cidr4> siblings) {
if (!vpc.substituted || vpc.declared == null || declared.isEmpty()
|| vpc.declared.prefix() != vpc.effective.prefix() || !vpc.declared.contains(declared.get())) {
return null;
}
long offset = declared.get().network() - vpc.declared.network();
Cidr4 translated = Cidr4.parse(Cidr4.format(vpc.effective.network() + offset) + "/" + declared.get().prefix())
.orElse(null);
if (translated == null || siblings.stream().anyMatch(sibling -> sibling.overlaps(translated))) {
return null;
}
return translated;
}
/** /**
* Binds a subnet to a range it can actually allocate out of. * Binds a subnet to a range it can actually allocate out of.
* *
@@ -14,6 +14,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv; import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager; import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
import io.github.hectorvent.floci.services.ec2.Ec2Service; import io.github.hectorvent.floci.services.ec2.Ec2Service;
import io.github.hectorvent.floci.services.efs.EfsMounts;
import io.github.hectorvent.floci.services.ec2.SecurityGroupFirewallManager; import io.github.hectorvent.floci.services.ec2.SecurityGroupFirewallManager;
import io.github.hectorvent.floci.services.ec2.SecurityGroupNftCompiler; import io.github.hectorvent.floci.services.ec2.SecurityGroupNftCompiler;
import io.github.hectorvent.floci.services.ec2.model.NetworkInterface; import io.github.hectorvent.floci.services.ec2.model.NetworkInterface;
@@ -63,14 +64,11 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import java.time.format.DateTimeParseException; import java.time.format.DateTimeParseException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
import java.util.HexFormat;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -124,6 +122,7 @@ public class EcsContainerManager {
private final HostVolumePolicy hostVolumePolicy; private final HostVolumePolicy hostVolumePolicy;
private Ec2Service ec2Service; private Ec2Service ec2Service;
private SecurityGroupFirewallManager firewallManager; private SecurityGroupFirewallManager firewallManager;
private EfsMounts efsMounts;
@Inject @Inject
public EcsContainerManager(ContainerBuilder containerBuilder, public EcsContainerManager(ContainerBuilder containerBuilder,
@@ -139,11 +138,13 @@ public class EcsContainerManager {
EcrRegistryManager ecrRegistryManager, EcrRegistryManager ecrRegistryManager,
HostVolumePolicy hostVolumePolicy, HostVolumePolicy hostVolumePolicy,
Ec2Service ec2Service, Ec2Service ec2Service,
SecurityGroupFirewallManager firewallManager) { SecurityGroupFirewallManager firewallManager,
EfsMounts efsMounts) {
this(containerBuilder, lifecycleManager, logStreamer, containerDetector, config, regionResolver, this(containerBuilder, lifecycleManager, logStreamer, containerDetector, config, regionResolver,
awsEnv, ssmService, secretsManagerService, s3Service, ecrRegistryManager, hostVolumePolicy); awsEnv, ssmService, secretsManagerService, s3Service, ecrRegistryManager, hostVolumePolicy);
this.ec2Service = ec2Service; this.ec2Service = ec2Service;
this.firewallManager = firewallManager; this.firewallManager = firewallManager;
this.efsMounts = efsMounts;
} }
public EcsContainerManager(ContainerBuilder containerBuilder, public EcsContainerManager(ContainerBuilder containerBuilder,
@@ -170,6 +171,8 @@ public class EcsContainerManager {
this.s3Service = s3Service; this.s3Service = s3Service;
this.hostVolumePolicy = hostVolumePolicy; this.hostVolumePolicy = hostVolumePolicy;
this.ecrRegistryManager = ecrRegistryManager; this.ecrRegistryManager = ecrRegistryManager;
// Without the EFS service only mounts that name no access point can be resolved.
this.efsMounts = new EfsMounts(null, lifecycleManager, config);
} }
/** /**
@@ -391,7 +394,7 @@ public class EcsContainerManager {
specBuilder.withBind(sourcePath, mp.containerPath()); specBuilder.withBind(sourcePath, mp.containerPath());
} }
} else if (efs != null) { } else if (efs != null) {
mountEfsVolume(specBuilder, efs, mp); mountEfsVolume(specBuilder, efs, mp, region);
} else { } else {
LOG.warnv("Skipping mountPoint with unresolved volume {0} on container {1}", LOG.warnv("Skipping mountPoint with unresolved volume {0} on container {1}",
mp.sourceVolume(), def.getName()); mp.sourceVolume(), def.getName());
@@ -1769,20 +1772,15 @@ public class EcsContainerManager {
} }
/** /**
* Materialises an EFS-configured task volume as a shared local Docker named volume scoped to * Mounts an EFS-configured task volume: the file system's shared Docker volume, or the
* both the file system and the mount's effective root (see {@link #efsVolumeName}), then * access point's or {@code rootDirectory}'s directory inside it (see {@link EfsMounts}). The
* initialises the volume root's POSIX ownership to emulate the EFS access point's * container keeps its own user, as on AWS, unless {@code floci.storage.efs.mount-user} sets one.
* RootDirectory.CreationInfo (no-op unless {@code floci.storage.efs} configures owner/permissions)
* and applies the configured PosixUser emulation (uid[:gid] and/or supplementary group) so a
* non-root task image can read/write the shared volume.
*/ */
private void mountEfsVolume(ContainerBuilder.Builder specBuilder, EfsVolumeConfiguration efs, MountPoint mp) { private void mountEfsVolume(ContainerBuilder.Builder specBuilder, EfsVolumeConfiguration efs, MountPoint mp,
String efsVolumeName = efsVolumeName(efs.fileSystemId(), efs.accessPointId(), efs.rootDirectory()); String region) {
efsMounts.prepare(region, efs.fileSystemId(), efs.accessPointId(), efs.rootDirectory())
.applyTo(specBuilder, mp.containerPath(), mp.readOnly());
var efsCfg = config.storage().efs(); var efsCfg = config.storage().efs();
lifecycleManager.ensureSharedVolume(efsVolumeName,
efsCfg.ownerUid(), efsCfg.ownerGid(), efsCfg.rootPermissions(),
efsCfg.initImage());
specBuilder.withNamedVolume(efsVolumeName, mp.containerPath(), mp.readOnly());
efsCfg.mountUser().ifPresent(u -> { efsCfg.mountUser().ifPresent(u -> {
if (!u.matches("^\\d+(:\\d+)?$")) { if (!u.matches("^\\d+(:\\d+)?$")) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
@@ -1793,52 +1791,6 @@ public class EcsContainerManager {
efsCfg.mountGroupAdd().ifPresent(gid -> specBuilder.withGroupAdd(String.valueOf(gid))); efsCfg.mountGroupAdd().ifPresent(gid -> specBuilder.withGroupAdd(String.valueOf(gid)));
} }
/**
* Name of the local Docker named volume backing an EFS-configured task volume, scoped to
* both the file system and the mount's effective root so two mounts of the same file system
* with a different {@code rootDirectory}/{@code accessPointId} land on isolated volumes
* while identical configurations keep sharing one, matching how AWS scopes an EFS mount to
* a subpath. When {@code accessPointId} is set, it determines the effective root: on real
* AWS an access point's own root directory takes precedence over any {@code rootDirectory}
* on the volume.
*/
static String efsVolumeName(String fileSystemId, String accessPointId, String rootDirectory) {
if (accessPointId != null && !accessPointId.isBlank()) {
return "floci-efs-" + fileSystemId + "-" + sha256Hex("accessPoint:" + accessPointId);
}
String normalizedRoot = normalizeRootDirectory(rootDirectory);
if ("/".equals(normalizedRoot)) {
return "floci-efs-" + fileSystemId;
}
return "floci-efs-" + fileSystemId + "-" + sha256Hex(normalizedRoot);
}
/**
* Canonicalises a {@code rootDirectory} path so syntactically different but equivalent
* paths (missing/blank, a missing leading slash, repeated slashes, a trailing slash) resolve
* to the same effective root instead of silently splitting shared storage across local
* volumes that AWS would treat as identical.
*/
private static String normalizeRootDirectory(String rootDirectory) {
if (rootDirectory == null || rootDirectory.isBlank()) {
return "/";
}
String collapsed = rootDirectory.trim().replaceAll("/+", "/");
if (collapsed.length() > 1 && collapsed.endsWith("/")) {
collapsed = collapsed.substring(0, collapsed.length() - 1);
}
return collapsed.startsWith("/") ? collapsed : "/" + collapsed;
}
private static String sha256Hex(String value) {
try {
byte[] digest = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8));
return HexFormat.of().formatHex(digest);
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("SHA-256 is required but not available", e);
}
}
// Inner enum to avoid import cycle — mirrors model.TaskStatus for readability // Inner enum to avoid import cycle — mirrors model.TaskStatus for readability
private enum TaskStatus {RUNNING} private enum TaskStatus {RUNNING}
} }
@@ -33,6 +33,10 @@ public class EfsException extends AwsException {
return new EfsException(Status.BAD_REQUEST, "SubnetNotFound", "The subnet ID '" + subnetId + "' does not exist"); return new EfsException(Status.BAD_REQUEST, "SubnetNotFound", "The subnet ID '" + subnetId + "' does not exist");
} }
public static EfsException securityGroupNotFound(String message) {
return new EfsException(Status.BAD_REQUEST, "SecurityGroupNotFound", message);
}
public static EfsException mountTargetConflict(String message) { public static EfsException mountTargetConflict(String message) {
return new EfsException(Status.CONFLICT, "MountTargetConflict", message); return new EfsException(Status.CONFLICT, "MountTargetConflict", message);
} }
@@ -0,0 +1,144 @@
package io.github.hectorvent.floci.services.efs;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.DirectoryCreation;
import io.github.hectorvent.floci.services.efs.model.AccessPointDescription;
import io.github.hectorvent.floci.services.efs.model.CreationInfo;
import io.github.hectorvent.floci.services.efs.model.PosixUser;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import java.util.List;
import java.util.Optional;
/**
* Turns an EFS mount (a file system, optionally through an access point) into a Docker volume
* mount. Each file system is one named volume, so every mount of it, from Lambda or ECS, through
* any access point, shares the same data. An access point mounts its root directory, a subfolder
* of that volume, which is created on first use from the access point's {@code CreationInfo}.
*/
@ApplicationScoped
public class EfsMounts {
/** A file system directory ready to mount, and the identity its access point enforces, if any. */
public record Mount(String volumeName, String subpath, Optional<PosixUser> accessPointUser) {
public void applyTo(ContainerBuilder.Builder spec, String containerPath, boolean readOnly) {
if (subpath.isEmpty()) {
spec.withNamedVolume(volumeName, containerPath, readOnly);
} else {
spec.withNamedVolumeSubpath(volumeName, subpath, containerPath, readOnly);
}
}
}
// A plain rootDirectory (an ECS volume without an access point) has no CreationInfo; create it
// the way a Docker named volume's own root is created.
private static final DirectoryCreation PLAIN_ROOT_DIRECTORY = new DirectoryCreation(0, 0, "755");
private final EfsService efsService;
private final ContainerLifecycleManager lifecycleManager;
private final EmulatorConfig config;
@Inject
public EfsMounts(EfsService efsService, ContainerLifecycleManager lifecycleManager, EmulatorConfig config) {
this.efsService = efsService;
this.lifecycleManager = lifecycleManager;
this.config = config;
}
public static String volumeName(String fileSystemId) {
return "floci-efs-" + fileSystemId;
}
/** Prepares a Lambda {@code FileSystemConfig}, which names an access point by ARN. */
public Mount prepareAccessPoint(String accessPointArn) {
String[] parts = accessPointArn == null ? new String[0] : accessPointArn.split(":", 6);
if (parts.length != 6 || !parts[5].startsWith("access-point/")) {
throw new IllegalArgumentException("Not an EFS access point ARN: " + accessPointArn);
}
return prepare(parts[3], null, parts[5].substring("access-point/".length()), null);
}
/**
* Prepares a mount of {@code fileSystemId}. With an access point its own root directory and
* owner apply and {@code rootDirectory} is ignored, as on AWS; {@code fileSystemId} may then
* be null, and otherwise must be the access point's file system.
*/
public Mount prepare(String region, String fileSystemId, String accessPointId, String rootDirectory) {
String root;
Optional<DirectoryCreation> creation;
Optional<PosixUser> user = Optional.empty();
if (accessPointId != null && !accessPointId.isBlank()) {
if (efsService == null) {
throw new IllegalStateException("EFS access points are unavailable: the EFS service is not running");
}
AccessPointDescription accessPoint = efsService.describeAccessPoints(region, null, accessPointId)
.stream().findFirst()
.orElseThrow(() -> EfsException.accessPointNotFound(accessPointId));
if (fileSystemId != null && !fileSystemId.equals(accessPoint.getFileSystemId())) {
throw new IllegalArgumentException("Access point " + accessPointId + " belongs to file system "
+ accessPoint.getFileSystemId() + ", not " + fileSystemId);
}
fileSystemId = accessPoint.getFileSystemId();
var directory = accessPoint.getRootDirectory();
root = normalizeRootDirectory(directory == null ? null : directory.getPath());
creation = Optional.ofNullable(directory == null ? null : directory.getCreationInfo())
.flatMap(EfsMounts::directoryCreation);
user = Optional.ofNullable(accessPoint.getPosixUser());
} else {
root = normalizeRootDirectory(rootDirectory);
creation = Optional.of(PLAIN_ROOT_DIRECTORY);
}
String volumeName = volumeName(fileSystemId);
var efsCfg = config.storage().efs();
lifecycleManager.ensureSharedVolume(volumeName,
efsCfg.ownerUid(), efsCfg.ownerGid(), efsCfg.rootPermissions(), efsCfg.initImage());
String subpath = root.substring(1);
if (!subpath.isEmpty()
&& !lifecycleManager.ensureSharedVolumeDirectory(volumeName, subpath, creation, efsCfg.initImage())) {
throw new IllegalStateException("The root directory " + root + " of access point " + accessPointId
+ " does not exist on file system " + fileSystemId
+ ", and the access point has no CreationInfo to create it");
}
return new Mount(volumeName, subpath, user);
}
/** {@code "uid:gid"} for a container that runs as the access point's POSIX user. */
public static String containerUser(PosixUser user) {
return user.getUid() + ":" + user.getGid();
}
public static List<Long> secondaryGids(PosixUser user) {
return user.getSecondaryGids() == null ? List.of() : user.getSecondaryGids();
}
private static Optional<DirectoryCreation> directoryCreation(CreationInfo info) {
if (info.getOwnerUid() == null || info.getOwnerGid() == null || info.getPermissions() == null) {
return Optional.empty();
}
return Optional.of(new DirectoryCreation(info.getOwnerUid(), info.getOwnerGid(), info.getPermissions()));
}
/**
* Canonicalises a root directory so equivalent spellings (missing or blank, no leading slash,
* repeated or trailing slashes) name the same directory. A {@code ..} component is refused.
*/
static String normalizeRootDirectory(String rootDirectory) {
if (rootDirectory == null || rootDirectory.isBlank()) {
return "/";
}
String collapsed = rootDirectory.trim().replaceAll("/+", "/");
if (collapsed.length() > 1 && collapsed.endsWith("/")) {
collapsed = collapsed.substring(0, collapsed.length() - 1);
}
String normalized = collapsed.startsWith("/") ? collapsed : "/" + collapsed;
if (List.of(normalized.split("/")).contains("..")) {
throw new IllegalArgumentException("A root directory cannot contain '..': " + rootDirectory);
}
return normalized;
}
}
@@ -0,0 +1,229 @@
package io.github.hectorvent.floci.services.efs;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.command.BuildImageResultCallback;
import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.Frame;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.services.ec2.net.VpcNetworkManager;
import io.github.hectorvent.floci.services.efs.model.MountTarget;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.jboss.logging.Logger;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
/**
* Serves each EFS mount target over NFSv4.1, as on AWS: a privileged helper container running the
* kernel NFS server holds the mount target's address on its VPC's Docker network and exports the
* file system's volume, the same one Lambda and ECS mount directly. An EC2 instance in the VPC can
* then mount the file system with the commands AWS documents, by address or by its DNS name.
*/
@ApplicationScoped
public class EfsNfsServers {
private static final Logger LOG = Logger.getLogger(EfsNfsServers.class);
static final String LABEL_COMPONENT = "floci_component";
static final String COMPONENT_VALUE = "efs-nfs-server";
static final String LABEL_OWNER = "floci_efs_owner";
static final String DEFAULT_IMAGE = "floci/efs-nfs-server:local";
private static final int READY_ATTEMPTS = 60;
private final EmulatorConfig config;
private final DockerClient dockerClient;
private final ContainerBuilder containerBuilder;
private final ContainerLifecycleManager lifecycleManager;
private final VpcNetworkManager vpcNetworkManager;
private final ExecutorService starter = Executors.newSingleThreadExecutor(r -> {
Thread thread = new Thread(r, "efs-nfs-servers");
thread.setDaemon(true);
return thread;
});
@Inject
public EfsNfsServers(EmulatorConfig config, DockerClient dockerClient, ContainerBuilder containerBuilder,
ContainerLifecycleManager lifecycleManager, VpcNetworkManager vpcNetworkManager) {
this.config = config;
this.dockerClient = dockerClient;
this.containerBuilder = containerBuilder;
this.lifecycleManager = lifecycleManager;
this.vpcNetworkManager = vpcNetworkManager;
}
@PreDestroy
void shutdown() {
starter.shutdownNow();
}
/** Whether mount targets get an NFS server: configured on, and VPCs have Docker networks to hold their address. */
public boolean enabled() {
return config.services().efs().nfsServers() && vpcNetworkManager.enabled();
}
static String containerName(String mountTargetId) {
return "floci-efs-nfs-" + mountTargetId;
}
/**
* Starts the mount target's NFS server in the background and then runs {@code onSettled}
* with whether it is serving. A server left from an earlier run is replaced.
*/
public void serve(String region, String accountId, MountTarget mountTarget, Consumer<Boolean> onSettled) {
starter.execute(() -> {
boolean serving = false;
try {
serving = start(region, accountId, mountTarget);
} catch (RuntimeException e) {
LOG.warnv("Mount target {0} of {1} has no NFS server: {2}",
mountTarget.getMountTargetId(), mountTarget.getFileSystemId(), e.getMessage());
}
onSettled.accept(serving);
});
}
/**
* Re-claims a persisted mount target's address after a restart. The VPC address leases live
* only in memory, so without this the next instance launched in the subnet could be handed it.
*/
public void reserveAddress(String region, MountTarget mountTarget) {
vpcNetworkManager.reservePrivateIp(region, mountTarget.getSubnetId(), mountTarget.getIpAddress());
}
/** Stops and removes the mount target's NFS server, if it has one. */
public void stop(String mountTargetId) {
lifecycleManager.removeIfExists(containerName(mountTargetId));
}
/** Removes every NFS server this emulator started, such as those a previous run left behind. */
public void removeAll() {
if (!enabled()) {
return;
}
dockerClient.listContainersCmd().withShowAll(true)
.withLabelFilter(Map.of(LABEL_COMPONENT, COMPONENT_VALUE, LABEL_OWNER, owner()))
.exec()
.forEach(container -> lifecycleManager.removeIfExists(container.getId()));
}
private boolean start(String region, String accountId, MountTarget mountTarget) {
String name = containerName(mountTarget.getMountTargetId());
String image = ensureImage();
String volumeName = EfsMounts.volumeName(mountTarget.getFileSystemId());
var efsCfg = config.storage().efs();
lifecycleManager.ensureSharedVolume(volumeName,
efsCfg.ownerUid(), efsCfg.ownerGid(), efsCfg.rootPermissions(), efsCfg.initImage());
lifecycleManager.removeIfExists(name);
var spec = containerBuilder.newContainer(image)
.withName(name)
.withPrivileged(true)
.withNamedVolume(volumeName, "/export", false)
.withLogRotation()
.withLabels(ContainerStorageHelper.resourceIdentityLabels(
"efs", mountTarget.getMountTargetId(), accountId, region))
.withLabels(Map.of(LABEL_COMPONENT, COMPONENT_VALUE, LABEL_OWNER, owner()))
.build();
String containerId = lifecycleManager.createAndStart(spec).containerId();
if (vpcNetworkManager.attach(region, mountTarget.getVpcId(), mountTarget.getSubnetId(),
containerId, mountTarget.getIpAddress()).isEmpty()) {
lifecycleManager.removeIfExists(name);
LOG.warnv("Mount target {0} could not take its address {1} on the VPC network; it has no NFS server",
mountTarget.getMountTargetId(), mountTarget.getIpAddress());
return false;
}
if (!awaitReady(containerId)) {
lifecycleManager.removeIfExists(name);
LOG.warnv("The NFS server of mount target {0} did not start", mountTarget.getMountTargetId());
return false;
}
LOG.infov("Mount target {0} serves file system {1} over NFS at {2}",
mountTarget.getMountTargetId(), mountTarget.getFileSystemId(), mountTarget.getIpAddress());
return true;
}
/** Ready once the kernel NFS server has threads and mountd answers export lookups. */
private boolean awaitReady(String containerId) {
for (int attempt = 0; attempt < READY_ATTEMPTS; attempt++) {
try {
String execId = dockerClient.execCreateCmd(containerId)
.withCmd("sh", "-c", "[ \"$(cat /proc/fs/nfsd/threads)\" -gt 0 ] && pgrep rpc.mountd")
.withAttachStdout(true).withAttachStderr(true)
.exec().getId();
dockerClient.execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>()).awaitCompletion(10, TimeUnit.SECONDS);
Long exitCode = dockerClient.inspectExecCmd(execId).exec().getExitCodeLong();
if (exitCode != null && exitCode == 0) {
return true;
}
TimeUnit.MILLISECONDS.sleep(500);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
} catch (RuntimeException e) {
LOG.debugv("NFS server {0} not ready yet: {1}", containerId, e.getMessage());
}
}
return false;
}
private synchronized String ensureImage() {
String configured = config.services().efs().nfsServerImage();
String image = containerBuilder.resolveImage(configured);
if (!DEFAULT_IMAGE.equals(configured)) {
return image;
}
try {
dockerClient.inspectImageCmd(image).exec();
return image;
} catch (NotFoundException missing) {
// Build Floci's own recipe in the daemon that runs the workloads.
}
try (InputStream dockerfile = getClass().getResourceAsStream("/docker/efs-nfs-server.Dockerfile")) {
if (dockerfile == null) {
throw new IllegalStateException("The EFS NFS server Dockerfile is missing");
}
byte[] content = dockerfile.readAllBytes();
ByteArrayOutputStream archive = new ByteArrayOutputStream(content.length + 1024);
try (TarArchiveOutputStream tar = new TarArchiveOutputStream(archive)) {
TarArchiveEntry entry = new TarArchiveEntry("Dockerfile");
entry.setSize(content.length);
tar.putArchiveEntry(entry);
tar.write(content);
tar.closeArchiveEntry();
}
try (BuildImageResultCallback callback = new BuildImageResultCallback()) {
String built = dockerClient.buildImageCmd(new ByteArrayInputStream(archive.toByteArray()))
.withTags(Set.of(image)).exec(callback).awaitImageId();
if (built == null || built.isBlank()) {
throw new IllegalStateException("Docker did not build the EFS NFS server image");
}
}
return image;
} catch (IOException e) {
throw new IllegalStateException("Cannot build the EFS NFS server image", e);
}
}
/** The emulator that owns a server, so one emulator never removes another's on a shared daemon. */
private String owner() {
String namespace = config.docker() == null || config.docker().resourceNamespace() == null
? "" : config.docker().resourceNamespace().orElse("");
return namespace.isBlank() ? String.valueOf(config.port()) : namespace + "/" + config.port();
}
}
@@ -4,8 +4,13 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory; import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.core.common.AwsException; import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.Resettable; import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
import io.github.hectorvent.floci.services.ec2.Ec2Service; import io.github.hectorvent.floci.services.ec2.Ec2Service;
import io.github.hectorvent.floci.services.ec2.model.NetworkInterface;
import io.github.hectorvent.floci.services.ec2.model.Subnet; import io.github.hectorvent.floci.services.ec2.model.Subnet;
import io.quarkus.runtime.StartupEvent;
import jakarta.enterprise.event.Observes;
import io.github.hectorvent.floci.services.efs.model.*; import io.github.hectorvent.floci.services.efs.model.*;
import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject; import jakarta.inject.Inject;
@@ -17,6 +22,9 @@ import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
@@ -35,6 +43,18 @@ public class EfsService implements Resettable {
private final StorageBackend<String, List<LifecyclePolicy>> lifecycleConfigurationStore; private final StorageBackend<String, List<LifecyclePolicy>> lifecycleConfigurationStore;
private final ConcurrentHashMap<String, Object> syncLocks = new ConcurrentHashMap<>(); private final ConcurrentHashMap<String, Object> syncLocks = new ConcurrentHashMap<>();
private final RegionResolver regionResolver; private final RegionResolver regionResolver;
private ContainerLifecycleManager lifecycleManager;
private EfsNfsServers nfsServers;
private EmbeddedDnsServer dnsServer;
// A file system's DNS name, optionally for one Availability Zone's mount target:
// [az.]fs-id.efs.region.amazonaws.com
private static final Pattern FILE_SYSTEM_DNS_NAME = Pattern.compile(
"^(?:([a-z0-9-]+)\\.)?(fs-[0-9a-f]+)\\.efs\\.([a-z0-9-]+)\\.amazonaws\\.com\\.?$");
// RootDirectory.Path as the EFS API documents it: "/" or up to 100 "/name" segments, where a
// name does not start with "." and holds none of $#<>;`|&?{}^* or a newline.
private static final Pattern ROOT_DIRECTORY_PATH = Pattern.compile("^(/|(/(?!\\.)[^$#<>;`|&?{}^*/\\n]+){1,100})$");
private final Ec2Service ec2Service; private final Ec2Service ec2Service;
@@ -60,6 +80,82 @@ public class EfsService implements Resettable {
new com.fasterxml.jackson.core.type.TypeReference<java.util.Map<String, List<LifecyclePolicy>>>() {}); new com.fasterxml.jackson.core.type.TypeReference<java.util.Map<String, List<LifecyclePolicy>>>() {});
} }
/** Lets a deleted file system take its stored data with it; absent in tests without Docker. */
@Inject
void setLifecycleManager(ContainerLifecycleManager lifecycleManager) {
this.lifecycleManager = lifecycleManager;
}
/** Serves mount targets over NFS; absent in tests that build the service by hand. */
@Inject
void setNfsServers(EfsNfsServers nfsServers, EmbeddedDnsServer dnsServer) {
this.nfsServers = nfsServers;
this.dnsServer = dnsServer;
}
/**
* The address a file system's DNS name resolves to: the mount target in the named Availability
* Zone, or, for the plain name, the first mount target that is serving. AWS answers the plain
* name with the mount target in the client's own zone; every mount target here is reachable
* from the whole VPC, so any serving one will do.
*/
Optional<String> resolveDnsName(String name) {
Matcher matcher = FILE_SYSTEM_DNS_NAME.matcher(name.toLowerCase());
if (!matcher.matches()) {
return Optional.empty();
}
String zone = matcher.group(1);
String fileSystemId = matcher.group(2);
String region = matcher.group(3);
return mountTargetStore.scan(k -> k.startsWith(region + "::")).stream()
.filter(mt -> fileSystemId.equals(mt.getFileSystemId()))
.filter(mt -> mt.getLifeCycleState() == LifeCycleState.available)
.filter(mt -> zone == null || zone.equals(mt.getAvailabilityZoneName()))
.sorted(Comparator.comparing(MountTarget::getAvailabilityZoneName))
.map(MountTarget::getIpAddress)
.findFirst();
}
/**
* Brings back the NFS servers of persisted mount targets after a restart: each one re-claims
* its address on the VPC network, so no instance is handed it, and serves again.
*/
void onStart(@Observes StartupEvent ignored) {
if (nfsServers == null || !nfsServers.enabled()) {
return;
}
dnsServer.addResolver(this::resolveDnsName);
nfsServers.removeAll();
for (String key : mountTargetStore.keys()) {
String region = key.substring(0, key.indexOf("::"));
mountTargetStore.get(key).ifPresent(mt -> {
// Looking the subnet up first also brings up EC2, which is what rebuilds the VPC
// networks the address reservation and the server's attachment both need.
try {
requireSubnet(region, mt.getSubnetId());
} catch (EfsException e) {
LOG.warnv("Mount target {0} is in subnet {1}, which no longer exists; it has no NFS server",
mt.getMountTargetId(), mt.getSubnetId());
return;
}
nfsServers.reserveAddress(region, mt);
serveOverNfs(region, mt);
});
}
}
private void serveOverNfs(String region, MountTarget mt) {
nfsServers.serve(region, regionResolver.getAccountId(), mt, serving -> {
String key = regionKey(region, mt.getMountTargetId());
synchronized (lockFor(key)) {
mountTargetStore.get(key).ifPresent(current -> {
current.setLifeCycleState(LifeCycleState.available);
mountTargetStore.put(key, current);
});
}
});
}
@Override @Override
public void clear() { public void clear() {
fileSystemStore.clear(); fileSystemStore.clear();
@@ -273,6 +369,10 @@ public class EfsService implements Resettable {
fileSystemStore.delete(key); fileSystemStore.delete(key);
} }
if (lifecycleManager != null && !lifecycleManager.removeVolume(EfsMounts.volumeName(fileSystemId))) {
LOG.warnv("The data of deleted file system {0} is still in Docker volume {1}, which is in use",
fileSystemId, EfsMounts.volumeName(fileSystemId));
}
} }
// --- Tags --- // --- Tags ---
@@ -384,30 +484,43 @@ public class EfsService implements Resettable {
} }
} }
} }
long inSubnet = regionMountTargets.stream().filter(existing -> request.getSubnetId().equals(existing.getSubnetId())).count();
String mtId = "fsmt-" + UUID.randomUUID().toString().replace("-", "").substring(0, 17); String mtId = "fsmt-" + UUID.randomUUID().toString().replace("-", "").substring(0, 17);
// As on AWS, the mount target is a network interface in the subnet, which is what
// gives it an address no instance can be handed as well.
NetworkInterface eni;
try {
eni = ec2Service.createNetworkInterface(region, request.getSubnetId(),
"EFS mount target for " + fs.getFileSystemId() + " (" + mtId + ")",
request.getIpAddress(), null, request.getSecurityGroups(), null);
} catch (AwsException e) {
if ("InvalidGroup.NotFound".equals(e.getErrorCode())) {
throw EfsException.securityGroupNotFound(e.getMessage());
}
throw e;
}
ec2Service.holdNetworkInterfaceForService(region, eni.getNetworkInterfaceId());
MountTarget mt = new MountTarget(); MountTarget mt = new MountTarget();
mt.setMountTargetId(mtId); mt.setMountTargetId(mtId);
mt.setFileSystemId(request.getFileSystemId()); mt.setFileSystemId(request.getFileSystemId());
mt.setSubnetId(request.getSubnetId()); mt.setSubnetId(request.getSubnetId());
mt.setIpAddress(request.getIpAddress() != null ? request.getIpAddress() : hostAddress(subnet.getCidrBlock(), inSubnet)); mt.setIpAddress(eni.getPrivateIpAddress());
mt.setLifeCycleState(LifeCycleState.available);
mt.setVpcId(subnet.getVpcId()); mt.setVpcId(subnet.getVpcId());
mt.setAvailabilityZoneName(subnet.getAvailabilityZone()); mt.setAvailabilityZoneName(subnet.getAvailabilityZone());
mt.setAvailabilityZoneId(generateAzId(subnet.getAvailabilityZone())); mt.setAvailabilityZoneId(generateAzId(subnet.getAvailabilityZone()));
mt.setNetworkInterfaceId("eni-" + UUID.randomUUID().toString().replace("-", "").substring(0, 17)); mt.setNetworkInterfaceId(eni.getNetworkInterfaceId());
if (request.getSecurityGroups() != null) { mt.setSecurityGroups(eni.getGroups().stream().map(group -> group.getGroupId())
mt.setSecurityGroups(new ArrayList<>(request.getSecurityGroups())); .collect(Collectors.toCollection(ArrayList::new)));
} else { boolean servedOverNfs = nfsServers != null && nfsServers.enabled();
mt.setSecurityGroups(new ArrayList<>()); mt.setLifeCycleState(servedOverNfs ? LifeCycleState.creating : LifeCycleState.available);
}
fs.setNumberOfMountTargets(fs.getNumberOfMountTargets() + 1); fs.setNumberOfMountTargets(fs.getNumberOfMountTargets() + 1);
fileSystemStore.put(regionKey(region, fs.getFileSystemId()), fs); fileSystemStore.put(regionKey(region, fs.getFileSystemId()), fs);
mountTargetStore.put(regionKey(region, mtId), mt); mountTargetStore.put(regionKey(region, mtId), mt);
if (servedOverNfs) {
serveOverNfs(region, mt);
}
return mt; return mt;
} }
} }
@@ -469,7 +582,11 @@ public class EfsService implements Resettable {
if (mt == null) { if (mt == null) {
throw EfsException.mountTargetNotFound(mountTargetId); throw EfsException.mountTargetNotFound(mountTargetId);
} }
if (nfsServers != null) {
nfsServers.stop(mountTargetId);
}
releaseNetworkInterface(region, mt);
try { try {
synchronized (lockFor(regionKey(region, mt.getFileSystemId()))) { synchronized (lockFor(regionKey(region, mt.getFileSystemId()))) {
FileSystem fs = getFileSystem(region, mt.getFileSystemId()); FileSystem fs = getFileSystem(region, mt.getFileSystemId());
@@ -515,6 +632,7 @@ public class EfsService implements Resettable {
synchronized (lockFor(regionKey(region, request.getFileSystemId()))) { synchronized (lockFor(regionKey(region, request.getFileSystemId()))) {
// Validate file system exists // Validate file system exists
getFileSystem(region, request.getFileSystemId()); getFileSystem(region, request.getFileSystemId());
validateAccessPoint(request);
for (AccessPointDescription existing : accessPointStore.scan(k -> k.startsWith(region + "::"))) { for (AccessPointDescription existing : accessPointStore.scan(k -> k.startsWith(region + "::"))) {
if (token.equals(existing.getClientToken())) { if (token.equals(existing.getClientToken())) {
@@ -546,6 +664,31 @@ public class EfsService implements Resettable {
} }
} }
private static void validateAccessPoint(CreateAccessPointRequest request) {
PosixUser user = request.getPosixUser();
if (user != null && (user.getUid() == null || user.getGid() == null)) {
throw EfsException.badRequest("PosixUser requires both Uid and Gid");
}
RootDirectory root = request.getRootDirectory();
if (root == null) {
return;
}
if (root.getPath() != null && (root.getPath().length() > 100
|| !ROOT_DIRECTORY_PATH.matcher(root.getPath()).matches())) {
throw EfsException.badRequest("RootDirectory.Path is not a valid directory path: " + root.getPath());
}
CreationInfo info = root.getCreationInfo();
if (info != null) {
if (info.getOwnerUid() == null || info.getOwnerGid() == null || info.getPermissions() == null) {
throw EfsException.badRequest("CreationInfo requires OwnerUid, OwnerGid and Permissions");
}
if (!info.getPermissions().matches("^[0-7]{3,4}$")) {
throw EfsException.badRequest("CreationInfo.Permissions must be 3 or 4 octal digits: "
+ info.getPermissions());
}
}
}
public List<AccessPointDescription> describeAccessPoints(String region, String fileSystemId, String accessPointId) { public List<AccessPointDescription> describeAccessPoints(String region, String fileSystemId, String accessPointId) {
return accessPointStore.scan(k -> k.startsWith(region + "::")).stream() return accessPointStore.scan(k -> k.startsWith(region + "::")).stream()
.filter(ap -> fileSystemId == null || ap.getFileSystemId().equals(fileSystemId)) .filter(ap -> fileSystemId == null || ap.getFileSystemId().equals(fileSystemId))
@@ -640,19 +783,17 @@ public class EfsService implements Resettable {
} }
} }
/** private void releaseNetworkInterface(String region, MountTarget mt) {
* An address inside the subnet for its n-th mount target, skipping the first four if (mt.getNetworkInterfaceId() == null) {
* addresses AWS reserves in every subnet. return;
*/ }
static String hostAddress(String cidrBlock, long index) { try {
String[] parts = cidrBlock.split("/"); ec2Service.releaseNetworkInterfaceFromService(region, mt.getNetworkInterfaceId());
String[] octets = parts[0].split("\\."); ec2Service.deleteNetworkInterface(region, mt.getNetworkInterfaceId());
long base = 0; } catch (AwsException e) {
for (String octet : octets) { LOG.debugv("Network interface {0} of mount target {1} is already gone: {2}",
base = (base << 8) | Long.parseLong(octet); mt.getNetworkInterfaceId(), mt.getMountTargetId(), e.getMessage());
} }
long address = base + 4 + index;
return ((address >> 24) & 0xff) + "." + ((address >> 16) & 0xff) + "." + ((address >> 8) & 0xff) + "." + (address & 0xff);
} }
private String generateAzId(String azName) { private String generateAzId(String azName) {
@@ -10,6 +10,8 @@ import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.core.common.docker.DockerHostResolver; import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv; import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager; import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
import io.github.hectorvent.floci.services.efs.EfsMounts;
import io.github.hectorvent.floci.services.efs.model.PosixUser;
import io.github.hectorvent.floci.services.iam.model.SessionCreds; import io.github.hectorvent.floci.services.iam.model.SessionCreds;
import io.github.hectorvent.floci.services.lambda.LambdaLayerService; import io.github.hectorvent.floci.services.lambda.LambdaLayerService;
import io.github.hectorvent.floci.services.lambda.model.ContainerState; import io.github.hectorvent.floci.services.lambda.model.ContainerState;
@@ -36,17 +38,13 @@ import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HexFormat;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.CancellationException; import java.util.concurrent.CancellationException;
@@ -110,6 +108,7 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {
private final LambdaLayerService layerService; private final LambdaLayerService layerService;
private final LaunchedContainerAwsEnv awsEnv; private final LaunchedContainerAwsEnv awsEnv;
private final LambdaExecutionRoleCredentials executionRoleCredentials; private final LambdaExecutionRoleCredentials executionRoleCredentials;
private final EfsMounts efsMounts;
@Inject @Inject
public ContainerLauncher(ContainerBuilder containerBuilder, public ContainerLauncher(ContainerBuilder containerBuilder,
@@ -122,7 +121,8 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {
EcrRegistryManager ecrRegistryManager, EcrRegistryManager ecrRegistryManager,
LambdaLayerService layerService, LambdaLayerService layerService,
LaunchedContainerAwsEnv awsEnv, LaunchedContainerAwsEnv awsEnv,
LambdaExecutionRoleCredentials executionRoleCredentials) { LambdaExecutionRoleCredentials executionRoleCredentials,
EfsMounts efsMounts) {
this.containerBuilder = containerBuilder; this.containerBuilder = containerBuilder;
this.lifecycleManager = lifecycleManager; this.lifecycleManager = lifecycleManager;
this.logStreamer = logStreamer; this.logStreamer = logStreamer;
@@ -134,6 +134,7 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {
this.layerService = layerService; this.layerService = layerService;
this.awsEnv = awsEnv; this.awsEnv = awsEnv;
this.executionRoleCredentials = executionRoleCredentials; this.executionRoleCredentials = executionRoleCredentials;
this.efsMounts = efsMounts;
this.populateSemaphore = new java.util.concurrent.Semaphore(resolvePopulateConcurrency(config)); this.populateSemaphore = new java.util.concurrent.Semaphore(resolvePopulateConcurrency(config));
} }
@@ -310,21 +311,28 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {
if (fn.getFileSystemConfigs() != null && !fn.getFileSystemConfigs().isEmpty()) { if (fn.getFileSystemConfigs() != null && !fn.getFileSystemConfigs().isEmpty()) {
var efsCfg = config.storage().efs(); var efsCfg = config.storage().efs();
fn.getFileSystemConfigs().forEach(fileSystem -> { // Lambda allows one file system per function. Running as the access point's POSIX user
String volumeName = efsVolumeName(fileSystem.getArn()); // is how its identity reaches every file the function creates, as EFS enforces it.
lifecycleManager.ensureSharedVolume(volumeName, Optional<PosixUser> accessPointUser = Optional.empty();
efsCfg.ownerUid(), efsCfg.ownerGid(), efsCfg.rootPermissions(), for (var fileSystem : fn.getFileSystemConfigs()) {
efsCfg.initImage()); EfsMounts.Mount mount = efsMounts.prepareAccessPoint(fileSystem.getArn());
specBuilder.withNamedVolume(volumeName, fileSystem.getLocalMountPath(), false); mount.applyTo(specBuilder, fileSystem.getLocalMountPath(), false);
}); accessPointUser = accessPointUser.or(mount::accessPointUser);
efsCfg.mountUser().ifPresent(user -> { }
if (!user.matches("^\\d+(:\\d+)?$")) { if (accessPointUser.isPresent()) {
throw new IllegalArgumentException( specBuilder.withUser(EfsMounts.containerUser(accessPointUser.get()));
"floci.storage.efs.mount-user must be \"uid\" or \"uid:gid\": " + user); EfsMounts.secondaryGids(accessPointUser.get())
} .forEach(gid -> specBuilder.withGroupAdd(String.valueOf(gid)));
specBuilder.withUser(user); } else {
}); efsCfg.mountUser().ifPresent(user -> {
efsCfg.mountGroupAdd().ifPresent(gid -> specBuilder.withGroupAdd(String.valueOf(gid))); if (!user.matches("^\\d+(:\\d+)?$")) {
throw new IllegalArgumentException(
"floci.storage.efs.mount-user must be \"uid\" or \"uid:gid\": " + user);
}
specBuilder.withUser(user);
});
efsCfg.mountGroupAdd().ifPresent(gid -> specBuilder.withGroupAdd(String.valueOf(gid)));
}
} }
// For Image package type use ImageConfig.Command/EntryPoint/WorkingDirectory if set, otherwise fall back to Handler (Zip-style) // For Image package type use ImageConfig.Command/EntryPoint/WorkingDirectory if set, otherwise fall back to Handler (Zip-style)
@@ -1133,25 +1141,6 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {
return namePrefix + "-code-" + fname + "-" + h; return namePrefix + "-code-" + fname + "-" + h;
} }
static String efsVolumeName(String accessPointArn) {
int separator = Math.max(accessPointArn.lastIndexOf('/'), accessPointArn.lastIndexOf(':'));
String resourceId = separator >= 0 ? accessPointArn.substring(separator + 1) : accessPointArn;
if (resourceId.isBlank()) {
throw new IllegalArgumentException("File system access point ARN must include a resource id");
}
return "floci-efs-" + resourceId + "-" + sha256Hex(accessPointArn);
}
private static String sha256Hex(String value) {
try {
byte[] digest = MessageDigest.getInstance("SHA-256")
.digest(value.getBytes(StandardCharsets.UTF_8));
return HexFormat.of().formatHex(digest);
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("SHA-256 is required but not available", e);
}
}
/** /**
* Buffer for the tar-streaming pipe. The default {@link java.io.PipedInputStream} buffer is * Buffer for the tar-streaming pipe. The default {@link java.io.PipedInputStream} buffer is
* only 1KB, which forces a writer/reader thread hand-off (wait/notify) every 1KB. Streaming a * only 1KB, which forces a writer/reader thread hand-off (wait/notify) every 1KB. Streaming a
+2
View File
@@ -747,6 +747,8 @@ floci:
enabled: true enabled: true
efs: efs:
enabled: true enabled: true
nfs-servers: true # serve each mount target over NFSv4.1 for EC2 instances
nfs-server-image: floci/efs-nfs-server:local # built locally on first use
organizations: organizations:
enabled: true enabled: true
# management-account-email: "" # When set, overrides the management-account email reported by DescribeOrganization and account listings. # management-account-email: "" # When set, overrides the management-account email reported by DescribeOrganization and account listings.
@@ -0,0 +1,5 @@
FROM alpine:3.21
RUN apk add --no-cache nfs-utils
# Serves /export, the EFS file system's volume, over NFSv4.1 from the kernel NFS server, as an EFS
# mount target does. fsid=0 makes it the NFSv4 root, so clients mount "<address>:/" like on AWS.
CMD ["sh", "-c", "set -e; mount -t nfsd nfsd /proc/fs/nfsd; echo '/export *(rw,fsid=0,no_subtree_check,no_root_squash,insecure,sync)' > /etc/exports; exportfs -ra; rpc.nfsd -N 3 -V 4 -V 4.1 -V 4.2 8; trap 'rpc.nfsd 0; exportfs -ua; exit 0' TERM INT; rpc.mountd -N 2 -N 3 -F & wait"]
@@ -103,6 +103,18 @@ class EmbeddedDnsServerTest {
dns.resolveARecord("ip-172-16-128-9.ec2.internal", "172.16.128.5").orElseThrow()); dns.resolveARecord("ip-172-16-128-9.ec2.internal", "172.16.128.5").orElseThrow());
} }
@Test
void resolveARecord_asksAddedResolversForOtherNames() {
dns.addResolver(name -> name.equals("fs-1.efs.us-east-1.amazonaws.com")
? java.util.Optional.of("172.31.0.10") : java.util.Optional.empty());
dns.addResolver(name -> { throw new IllegalStateException("broken resolver"); });
assertEquals("172.31.0.10",
dns.resolveARecord("fs-1.efs.us-east-1.amazonaws.com", "172.16.128.5").orElseThrow());
assertTrue(dns.resolveARecord("example.com", "172.16.128.5").isEmpty(),
"a name no resolver knows is forwarded upstream, even when a resolver fails");
}
// ── matchesSuffix — built-in emulator domains ───────────────────────────── // ── matchesSuffix — built-in emulator domains ─────────────────────────────
@Test @Test
@@ -284,6 +284,23 @@ class VpcNetworkManagerTest {
assertTrue(second.containsAddress(Cidr4.parse(b + "/32").orElseThrow().network())); assertTrue(second.containsAddress(Cidr4.parse(b + "/32").orElseThrow().network()));
} }
@Test
void aSubstitutedVpcKeepsEachSubnetAtItsOffsetWhateverTheDeclarationOrder() {
existingNetwork("someone-else", "172.31.0.0/16", Map.of());
manager.declareVpc(REGION, "vpc-default", "172.31.0.0/16");
manager.declareSubnet(REGION, "vpc-default", "subnet-c", "172.31.32.0/20");
manager.declareSubnet(REGION, "vpc-default", "subnet-a", "172.31.0.0/20");
manager.declareSubnet(REGION, "vpc-default", "subnet-b", "172.31.16.0/20");
assertTrue(manager.isSubstituted(REGION, "vpc-default"));
assertEquals("10.240.0.0/16", manager.effectiveVpcCidr(REGION, "vpc-default").orElseThrow());
assertEquals("10.240.0.0/20", manager.effectiveSubnetCidr(REGION, "subnet-a").orElseThrow());
assertEquals("10.240.16.0/20", manager.effectiveSubnetCidr(REGION, "subnet-b").orElseThrow());
assertEquals("10.240.32.0/20", manager.effectiveSubnetCidr(REGION, "subnet-c").orElseThrow(),
"a restart that declares subnets in another order must reach the same plan");
}
@Test @Test
void aSubnetOutsideItsVpcRangeIsRemappedInsideIt() { void aSubnetOutsideItsVpcRangeIsRemappedInsideIt() {
manager.declareVpc(REGION, "vpc-1", "10.0.0.0/16"); manager.declareVpc(REGION, "vpc-1", "10.0.0.0/16");
@@ -1,169 +0,0 @@
package io.github.hectorvent.floci.services.ecs.container;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.core.command.WaitContainerResultCallback;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
import io.github.hectorvent.floci.services.ecs.model.EcsTask;
import io.github.hectorvent.floci.services.ecs.model.EfsVolumeConfiguration;
import io.github.hectorvent.floci.services.ecs.model.MountPoint;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
import io.github.hectorvent.floci.services.ecs.model.Volume;
import io.quarkus.test.junit.QuarkusTest;
import jakarta.inject.Inject;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* Docker-backed proof that an EFS-configured task volume's local mount is scoped to
* {@code (fileSystemId, rootDirectory, accessPointId)}, not just {@code fileSystemId} (#2563).
* Each scenario runs a real busybox container against the same {@code fileSystemId}, writing
* and checking marker files under {@code /mnt/efs} with a shell one-liner whose exit code
* proves whether the previous task's data was visible.
*/
@QuarkusTest
class EcsContainerManagerEfsIsolationDockerIntegrationTest {
private static final String IMAGE = "public.ecr.aws/docker/library/busybox:latest";
@Inject
EcsContainerManager containerManager;
@Inject
ContainerLifecycleManager lifecycleManager;
@Inject
DockerClient dockerClient;
private final List<String> volumesToCleanUp = new ArrayList<>();
@BeforeEach
void requireDocker() {
Assumptions.assumeTrue(isDockerAvailable(),
"Docker daemon must be available for ECS EFS isolation integration tests");
}
@AfterEach
void cleanUpVolumes() {
volumesToCleanUp.forEach(lifecycleManager::removeVolume);
volumesToCleanUp.clear();
}
/**
* Task A writes a marker under {@code rootDirectory} {@code /team-a}; task B, on the same
* file system but a different root, must not see it (if the bug were still present, one
* shared volume keyed only by {@code fileSystemId} would leak team-a's marker into team-b's
* task before it ever reaches its own write); a third task back on {@code /team-a} must
* likewise not see what team-b wrote.
*/
@Test
void differentRootDirectoriesOnSameFileSystemAreIsolated() {
String fileSystemId = "fs-" + UUID.randomUUID().toString().substring(0, 8);
assertEquals(0, runEfsCommand(fileSystemId, "/team-a", null,
"echo team-a > /mnt/efs/only-a.txt"),
"task A should write its marker cleanly");
assertEquals(0, runEfsCommand(fileSystemId, "/team-b", null,
"test ! -e /mnt/efs/only-a.txt && echo team-b > /mnt/efs/only-b.txt"),
"task B must not see task A's rootDirectory-scoped data");
assertEquals(0, runEfsCommand(fileSystemId, "/team-a", null,
"test ! -e /mnt/efs/only-b.txt"),
"task on /team-a must not see /team-b's data");
}
/**
* A second task using the identical {@code rootDirectory} must see the first task's data:
* sharing an identical configuration is existing, desired behaviour.
*/
@Test
void sameRootDirectoryOnSameFileSystemStillShares() {
String fileSystemId = "fs-" + UUID.randomUUID().toString().substring(0, 8);
assertEquals(0, runEfsCommand(fileSystemId, "/shared", null,
"echo hello > /mnt/efs/shared.txt"),
"first task should write its marker cleanly");
assertEquals(0, runEfsCommand(fileSystemId, "/shared", null,
"grep -qx hello /mnt/efs/shared.txt"),
"a second task with the same rootDirectory must share the first task's data");
}
/**
* A plain {@code rootDirectory} mount (no {@code accessPointId}) on the same file system
* must not see access-point-scoped data and vice versa, while a second task through the
* same access point must still share its data.
*/
@Test
void accessPointIdIsolatesFromRootDirectoryOnSameFileSystem() {
String fileSystemId = "fs-" + UUID.randomUUID().toString().substring(0, 8);
String accessPointId = "fsap-" + UUID.randomUUID().toString().substring(0, 8);
assertEquals(0, runEfsCommand(fileSystemId, null, accessPointId,
"echo via-ap > /mnt/efs/via-ap.txt"),
"task mounted through the access point should write its marker cleanly");
assertEquals(0, runEfsCommand(fileSystemId, "/", null,
"test ! -e /mnt/efs/via-ap.txt"),
"a plain rootDirectory mount must not see access-point-scoped data");
assertEquals(0, runEfsCommand(fileSystemId, null, accessPointId,
"grep -qx via-ap /mnt/efs/via-ap.txt"),
"a second task through the same access point must share its data");
}
/**
* Runs a single-container ECS task that mounts {@code fileSystemId} (scoped by
* {@code rootDirectory}/{@code accessPointId}) at {@code /mnt/efs} and executes
* {@code shellCommand}, then blocks until the container exits and returns its exit code.
*/
private int runEfsCommand(String fileSystemId, String rootDirectory, String accessPointId, String shellCommand) {
String suffix = UUID.randomUUID().toString().substring(0, 8);
ContainerDefinition app = new ContainerDefinition();
app.setName("app");
app.setImage(IMAGE);
app.setCommand(List.of("sh", "-c", shellCommand));
app.setMountPoints(List.of(new MountPoint("efs-data", "/mnt/efs", false)));
TaskDefinition taskDef = new TaskDefinition();
taskDef.setFamily("efs-isolation-" + suffix);
taskDef.setContainerDefinitions(List.of(app));
taskDef.setVolumes(List.of(new Volume("efs-data", null,
new EfsVolumeConfiguration(fileSystemId, rootDirectory, null, null, accessPointId, null))));
EcsTask task = new EcsTask();
task.setTaskArn("arn:aws:ecs:us-east-1:000000000000:task/efs-isolation/" + suffix);
volumesToCleanUp.add(EcsContainerManager.efsVolumeName(fileSystemId, accessPointId, rootDirectory));
EcsTaskHandle handle = containerManager.startTask(task, taskDef, List.of(), "us-east-1");
String dockerId = handle.getContainerIds().get("app");
try {
return dockerClient.waitContainerCmd(dockerId)
.exec(new WaitContainerResultCallback())
.awaitStatusCode(60, TimeUnit.SECONDS);
} finally {
containerManager.stopTask(handle);
}
}
private boolean isDockerAvailable() {
try {
dockerClient.pingCmd().exec();
return true;
} catch (Exception e) {
return false;
}
}
}
@@ -0,0 +1,194 @@
package io.github.hectorvent.floci.services.ecs.container;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.core.command.WaitContainerResultCallback;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
import io.github.hectorvent.floci.services.ecs.model.EcsTask;
import io.github.hectorvent.floci.services.ecs.model.EfsVolumeConfiguration;
import io.github.hectorvent.floci.services.ecs.model.MountPoint;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
import io.github.hectorvent.floci.services.ecs.model.Volume;
import io.github.hectorvent.floci.services.efs.EfsMounts;
import io.github.hectorvent.floci.services.efs.EfsService;
import io.github.hectorvent.floci.services.efs.model.CreateAccessPointRequest;
import io.github.hectorvent.floci.services.efs.model.CreateFileSystemRequest;
import io.github.hectorvent.floci.services.efs.model.CreationInfo;
import io.github.hectorvent.floci.services.efs.model.PosixUser;
import io.github.hectorvent.floci.services.efs.model.RootDirectory;
import io.quarkus.test.junit.QuarkusTest;
import jakarta.inject.Inject;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
/**
* Docker-backed proof that an EFS file system is one shared store, as on AWS: every task volume
* of it mounts the same Docker volume, a {@code rootDirectory} or access point mounts its own
* folder of it, and a mount of the file system root sees every folder. Each scenario runs a real
* busybox container whose exit code proves what it could see under {@code /mnt/efs}.
*/
@QuarkusTest
class EcsContainerManagerEfsSharingDockerIntegrationTest {
private static final String IMAGE = "public.ecr.aws/docker/library/busybox:latest";
private static final String REGION = "us-east-1";
@Inject
EcsContainerManager containerManager;
@Inject
ContainerLifecycleManager lifecycleManager;
@Inject
EfsService efsService;
@Inject
DockerClient dockerClient;
private final List<String> volumesToCleanUp = new ArrayList<>();
@BeforeEach
void requireDocker() {
Assumptions.assumeTrue(isDockerAvailable(),
"Docker daemon must be available for ECS EFS integration tests");
}
@AfterEach
void cleanUpVolumes() {
volumesToCleanUp.forEach(lifecycleManager::removeVolume);
volumesToCleanUp.clear();
}
/** Two root directories are separate folders, and the file system root sees both. */
@Test
void rootDirectoriesAreFoldersOfOneFileSystem() {
String fileSystemId = "fs-" + UUID.randomUUID().toString().substring(0, 8);
assertEquals(0, runEfsCommand(fileSystemId, "/team-a", null,
"echo team-a > /mnt/efs/only-a.txt"));
assertEquals(0, runEfsCommand(fileSystemId, "/team-b", null,
"test ! -e /mnt/efs/only-a.txt && echo team-b > /mnt/efs/only-b.txt"),
"a task on /team-b must not see /team-a's files");
assertEquals(0, runEfsCommand(fileSystemId, "/team-a", null,
"grep -qx team-a /mnt/efs/only-a.txt && test ! -e /mnt/efs/only-b.txt"),
"a second task on /team-a shares its files and still does not see /team-b's");
assertEquals(0, runEfsCommand(fileSystemId, "/", null,
"grep -qx team-a /mnt/efs/team-a/only-a.txt && grep -qx team-b /mnt/efs/team-b/only-b.txt"),
"a task on the file system root sees every root directory's files");
}
/**
* An access point mounts its root directory, created on first use with its CreationInfo, and
* shares it with every other mount of the file system.
*/
@Test
void anAccessPointMountsItsFolderOfTheFileSystem() {
String fileSystemId = createFileSystem();
String accessPointId = createAccessPoint(fileSystemId, "/apps/orders", 1001L, 1002L, "750");
assertEquals(0, runEfsCommand(fileSystemId, null, accessPointId,
"echo order-1 > /mnt/efs/orders.txt"));
assertEquals(0, runEfsCommand(fileSystemId, "/", null,
"grep -qx order-1 /mnt/efs/apps/orders/orders.txt"
+ " && test \"$(stat -c '%u:%g %a' /mnt/efs/apps/orders)\" = '1001:1002 750'"
+ " && test \"$(stat -c '%u:%g %a' /mnt/efs/apps)\" = '1001:1002 750'"),
"the root mount sees the access point's files, in folders created from its CreationInfo");
assertEquals(0, runEfsCommand(fileSystemId, "/apps", null,
"grep -qx order-1 /mnt/efs/orders/orders.txt"),
"a rootDirectory above the access point's folder sees it too");
}
/** Without CreationInfo, an access point whose root directory does not exist cannot be mounted. */
@Test
void anAccessPointWithAMissingRootDirectoryAndNoCreationInfoIsRefused() {
String fileSystemId = createFileSystem();
String accessPointId = createAccessPoint(fileSystemId, "/absent", null, null, null);
assertThrows(IllegalStateException.class,
() -> runEfsCommand(fileSystemId, null, accessPointId, "true"));
}
private String createFileSystem() {
CreateFileSystemRequest request = new CreateFileSystemRequest();
request.setCreationToken("efs-sharing-" + UUID.randomUUID());
return efsService.createFileSystem(request, REGION).getFileSystemId();
}
private String createAccessPoint(String fileSystemId, String path, Long ownerUid, Long ownerGid,
String permissions) {
RootDirectory root = new RootDirectory();
root.setPath(path);
if (permissions != null) {
CreationInfo info = new CreationInfo();
info.setOwnerUid(ownerUid);
info.setOwnerGid(ownerGid);
info.setPermissions(permissions);
root.setCreationInfo(info);
}
PosixUser user = new PosixUser();
user.setUid(1001L);
user.setGid(1002L);
CreateAccessPointRequest request = new CreateAccessPointRequest();
request.setFileSystemId(fileSystemId);
request.setRootDirectory(root);
request.setPosixUser(user);
return efsService.createAccessPoint(REGION, request).getAccessPointId();
}
/**
* Runs a single-container ECS task that mounts {@code fileSystemId} (through
* {@code rootDirectory} or {@code accessPointId}) at {@code /mnt/efs} and executes
* {@code shellCommand}, then blocks until the container exits and returns its exit code.
*/
private int runEfsCommand(String fileSystemId, String rootDirectory, String accessPointId, String shellCommand) {
String suffix = UUID.randomUUID().toString().substring(0, 8);
ContainerDefinition app = new ContainerDefinition();
app.setName("app");
app.setImage(IMAGE);
app.setCommand(List.of("sh", "-c", shellCommand));
app.setMountPoints(List.of(new MountPoint("efs-data", "/mnt/efs", false)));
TaskDefinition taskDef = new TaskDefinition();
taskDef.setFamily("efs-sharing-" + suffix);
taskDef.setContainerDefinitions(List.of(app));
taskDef.setVolumes(List.of(new Volume("efs-data", null,
new EfsVolumeConfiguration(fileSystemId, rootDirectory, null, null, accessPointId, null))));
EcsTask task = new EcsTask();
task.setTaskArn("arn:aws:ecs:us-east-1:000000000000:task/efs-sharing/" + suffix);
if (!volumesToCleanUp.contains(EfsMounts.volumeName(fileSystemId))) {
volumesToCleanUp.add(EfsMounts.volumeName(fileSystemId));
}
EcsTaskHandle handle = containerManager.startTask(task, taskDef, List.of(), REGION);
String dockerId = handle.getContainerIds().get("app");
try {
return dockerClient.waitContainerCmd(dockerId)
.exec(new WaitContainerResultCallback())
.awaitStatusCode(60, TimeUnit.SECONDS);
} finally {
containerManager.stopTask(handle);
}
}
private boolean isDockerAvailable() {
try {
dockerClient.pingCmd().exec();
return true;
} catch (Exception e) {
return false;
}
}
}
@@ -1,87 +0,0 @@
package io.github.hectorvent.floci.services.ecs.container;
import org.junit.jupiter.api.Test;
import java.util.regex.Pattern;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Unit tests for {@link EcsContainerManager#efsVolumeName}, the naming logic that scopes an
* EFS-backed task volume's shared Docker named volume to both {@code fileSystemId} and its
* effective root (#2563). The name must be stable for a given (fileSystemId, rootDirectory,
* accessPointId) combination (so identical mounts keep sharing one volume), distinct across
* different roots on the same file system (so mounts do not collide on the same tree), and
* always a legal Docker volume name.
*/
class EcsContainerManagerEfsVolumeNamingTest {
private static final Pattern DOCKER_VOLUME_NAME = Pattern.compile("^[a-zA-Z0-9][a-zA-Z0-9_.-]*$");
@Test
void defaultsToPlainFileSystemNameWhenNoRootOrAccessPointIsSet() {
assertEquals("floci-efs-fs-abc", EcsContainerManager.efsVolumeName("fs-abc", null, null));
assertEquals("floci-efs-fs-abc", EcsContainerManager.efsVolumeName("fs-abc", null, ""));
assertEquals("floci-efs-fs-abc", EcsContainerManager.efsVolumeName("fs-abc", null, "/"));
}
@Test
void differentRootDirectoriesOnSameFileSystemYieldDifferentVolumes() {
String rootA = EcsContainerManager.efsVolumeName("fs-shared", null, "/team-a");
String rootB = EcsContainerManager.efsVolumeName("fs-shared", null, "/team-b");
String rootNone = EcsContainerManager.efsVolumeName("fs-shared", null, null);
assertNotEquals(rootA, rootB, "different rootDirectory values must isolate the volume");
assertNotEquals(rootA, rootNone, "a non-root rootDirectory must not collapse to the plain name");
assertTrue(DOCKER_VOLUME_NAME.matcher(rootA).matches(), "must be docker-safe, was: " + rootA);
assertTrue(DOCKER_VOLUME_NAME.matcher(rootB).matches(), "must be docker-safe, was: " + rootB);
}
@Test
void sameRootDirectoryOnSameFileSystemStillShares() {
String first = EcsContainerManager.efsVolumeName("fs-shared", null, "/team-a");
String second = EcsContainerManager.efsVolumeName("fs-shared", null, "/team-a");
assertEquals(first, second, "identical (fileSystemId, rootDirectory) must keep sharing one volume");
}
@Test
void accessPointIdTakesPrecedenceOverRootDirectory() {
String viaAccessPoint = EcsContainerManager.efsVolumeName("fs-shared", "fsap-111", "/ignored");
String viaRootOnly = EcsContainerManager.efsVolumeName("fs-shared", null, "/ignored");
assertNotEquals(viaAccessPoint, viaRootOnly,
"an access point's own root must take precedence over rootDirectory, not merge with it");
}
@Test
void differentAccessPointsOnSameFileSystemYieldDifferentVolumes() {
String apOne = EcsContainerManager.efsVolumeName("fs-shared", "fsap-111", null);
String apTwo = EcsContainerManager.efsVolumeName("fs-shared", "fsap-222", null);
assertNotEquals(apOne, apTwo, "different accessPointId values must isolate the volume");
}
@Test
void sameAccessPointStillShares() {
String first = EcsContainerManager.efsVolumeName("fs-shared", "fsap-111", null);
String second = EcsContainerManager.efsVolumeName("fs-shared", "fsap-111", "/whatever-is-ignored");
assertEquals(first, second,
"the same accessPointId must keep sharing one volume regardless of rootDirectory");
}
@Test
void equivalentRootDirectorySpellingsShareTheSameVolume() {
String canonical = EcsContainerManager.efsVolumeName("fs-shared", null, "/team-a");
assertEquals(canonical, EcsContainerManager.efsVolumeName("fs-shared", null, "/team-a/"),
"a trailing slash must not create a distinct volume for the same directory");
assertEquals(canonical, EcsContainerManager.efsVolumeName("fs-shared", null, "//team-a"),
"a repeated leading slash must not create a distinct volume for the same directory");
assertEquals(canonical, EcsContainerManager.efsVolumeName("fs-shared", null, "team-a"),
"a missing leading slash must not create a distinct volume for the same directory");
}
}
@@ -89,7 +89,7 @@ class EcsContainerManagerSecurityGroupTest {
containerDetector, config, regionResolver, awsEnv, containerDetector, config, regionResolver, awsEnv,
mock(SsmService.class), mock(SecretsManagerService.class), mock(S3Service.class), mock(SsmService.class), mock(SecretsManagerService.class), mock(S3Service.class),
ecrRegistryManager, mock(HostVolumePolicy.class), ecrRegistryManager, mock(HostVolumePolicy.class),
ec2Service, firewallManager); ec2Service, firewallManager, mock(io.github.hectorvent.floci.services.efs.EfsMounts.class));
} }
@Test @Test
@@ -69,6 +69,7 @@ class EcsContainerManagerVolumesTest {
lifecycleManager = mock(ContainerLifecycleManager.class); lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.createAndStart(any())) when(lifecycleManager.createAndStart(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of())); .thenReturn(new ContainerInfo("docker-id", Map.of()));
when(lifecycleManager.ensureSharedVolumeDirectory(anyString(), anyString(), any(), any())).thenReturn(true);
ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class); ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);
ContainerDetector containerDetector = mock(ContainerDetector.class); ContainerDetector containerDetector = mock(ContainerDetector.class);
@@ -132,9 +133,8 @@ class EcsContainerManagerVolumesTest {
} }
/** /**
* Each EFS volume resolves to a shared local Docker named volume keyed by file system id * Each EFS volume mounts its file system's one Docker named volume: the whole volume for the
* AND effective root (here: rootDirectory, since neither volume sets an accessPointId), * file system root, or the rootDirectory's folder of it, read-write or read-only per the mountPoint.
* read-write or read-only per the mountPoint.
*/ */
@Test @Test
void efsVolumeMountPointsResolveToSharedNamedVolumesByAccessMode() { void efsVolumeMountPointsResolveToSharedNamedVolumesByAccessMode() {
@@ -159,10 +159,10 @@ class EcsContainerManagerVolumesTest {
manager.startTask(task, taskDef, List.of(), "us-east-1"); manager.startTask(task, taskDef, List.of(), "us-east-1");
verify(builder, times(1)).withNamedVolumeSubpath(
"floci-efs-fs-0123456789abcdef0", "dps", "/mnt/efs", false);
verify(builder, times(1)).withNamedVolume( verify(builder, times(1)).withNamedVolume(
EcsContainerManager.efsVolumeName("fs-0123456789abcdef0", null, "/dps"), "/mnt/efs", false); "floci-efs-fs-00000000000000001", "/mnt/shared", true);
verify(builder, times(1)).withNamedVolume(
EcsContainerManager.efsVolumeName("fs-00000000000000001", null, null), "/mnt/shared", true);
// EFS volumes are never bind-mounted as host paths. // EFS volumes are never bind-mounted as host paths.
verify(builder, never()).withBind(any(), any()); verify(builder, never()).withBind(any(), any());
@@ -202,7 +202,7 @@ class EcsContainerManagerVolumesTest {
configured.startTask(task, taskDef, List.of(), "us-east-1"); configured.startTask(task, taskDef, List.of(), "us-east-1");
verify(lifecycleManager, times(1)).ensureSharedVolume( verify(lifecycleManager, times(1)).ensureSharedVolume(
EcsContainerManager.efsVolumeName("fs-abc", null, "/dps"), "floci-efs-fs-abc",
OptionalInt.of(1001), OptionalInt.of(1001), Optional.of("2775"), "busybox:stable"); OptionalInt.of(1001), OptionalInt.of(1001), Optional.of("2775"), "busybox:stable");
} }
@@ -0,0 +1,81 @@
package io.github.hectorvent.floci.services.efs;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.equalTo;
/** CreateAccessPoint refuses a root directory or identity that EFS itself would refuse. */
@QuarkusTest
class EfsAccessPointValidationTest {
@BeforeAll
static void configureRestAssured() {
RestAssuredJsonUtils.configureAwsContentTypes();
}
private String createFileSystem(String token) {
return given()
.contentType("application/json")
.body("{\"CreationToken\": \"" + token + "\"}")
.when()
.post("/2015-02-01/file-systems")
.then()
.statusCode(201)
.extract().jsonPath().getString("FileSystemId");
}
private io.restassured.response.ValidatableResponse createAccessPoint(String body) {
return given()
.contentType("application/json")
.body(body)
.when()
.post("/2015-02-01/access-points")
.then();
}
@Test
void acceptsANestedRootDirectoryWithCreationInfo() {
String fileSystemId = createFileSystem("ap-valid");
createAccessPoint("{\"FileSystemId\": \"" + fileSystemId + "\","
+ " \"PosixUser\": {\"Uid\": 1001, \"Gid\": 1001},"
+ " \"RootDirectory\": {\"Path\": \"/apps/my-function\","
+ " \"CreationInfo\": {\"OwnerUid\": 1001, \"OwnerGid\": 1001, \"Permissions\": \"0755\"}}}")
.statusCode(200)
.body("RootDirectory.Path", equalTo("/apps/my-function"));
}
@Test
void refusesARootDirectoryThatClimbsOutOrHidesItself() {
String fileSystemId = createFileSystem("ap-bad-path");
for (String path : new String[] {"/a/../b", "/.hidden", "relative", "/a;rm"}) {
createAccessPoint("{\"FileSystemId\": \"" + fileSystemId + "\","
+ " \"RootDirectory\": {\"Path\": \"" + path + "\"}}")
.statusCode(400)
.body("__type", equalTo("BadRequest"));
}
}
@Test
void refusesPartialCreationInfoAndPosixUser() {
String fileSystemId = createFileSystem("ap-partial");
createAccessPoint("{\"FileSystemId\": \"" + fileSystemId + "\","
+ " \"RootDirectory\": {\"Path\": \"/data\", \"CreationInfo\": {\"OwnerUid\": 1001}}}")
.statusCode(400)
.body("__type", equalTo("BadRequest"));
createAccessPoint("{\"FileSystemId\": \"" + fileSystemId + "\","
+ " \"RootDirectory\": {\"Path\": \"/data\","
+ " \"CreationInfo\": {\"OwnerUid\": 1001, \"OwnerGid\": 1001, \"Permissions\": \"rwx\"}}}")
.statusCode(400)
.body("__type", equalTo("BadRequest"));
createAccessPoint("{\"FileSystemId\": \"" + fileSystemId + "\", \"PosixUser\": {\"Uid\": 1001}}")
.statusCode(400)
.body("__type", equalTo("BadRequest"));
}
}
@@ -91,7 +91,7 @@ class EfsIntegrationTest {
{ {
"FileSystemId": "%s", "FileSystemId": "%s",
"SubnetId": "subnet-default-us-east-1-a", "SubnetId": "subnet-default-us-east-1-a",
"SecurityGroups": ["sg-11111"] "SecurityGroups": ["sg-default-us-east-1"]
} }
""".formatted(fileSystemId)) """.formatted(fileSystemId))
.when() .when()
@@ -103,7 +103,9 @@ class EfsIntegrationTest {
.body("SubnetId", equalTo("subnet-default-us-east-1-a")) .body("SubnetId", equalTo("subnet-default-us-east-1-a"))
.body("VpcId", equalTo("vpc-default-us-east-1")) .body("VpcId", equalTo("vpc-default-us-east-1"))
.body("AvailabilityZoneName", equalTo("us-east-1a")) .body("AvailabilityZoneName", equalTo("us-east-1a"))
.body("IpAddress", equalTo("172.31.0.4")) .body("IpAddress", startsWith("172.31.0."))
.body("NetworkInterfaceId", startsWith("eni-"))
.body("LifeCycleState", equalTo("available"))
.extract().jsonPath().getString("MountTargetId"); .extract().jsonPath().getString("MountTargetId");
} }
@@ -2,17 +2,27 @@ package io.github.hectorvent.floci.services.efs;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils; import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.QuarkusTest;
import jakarta.inject.Inject;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.util.Optional;
import static io.restassured.RestAssured.given; import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.startsWith;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
/** Mount targets take their VPC, zone and address from the EC2 subnet they are created in. */ /**
* Mount targets take their VPC, zone and address from the EC2 subnet they are created in, hold an
* EC2 network interface for that address, and answer the file system's DNS name.
*/
@QuarkusTest @QuarkusTest
class EfsMountTargetSubnetTest { class EfsMountTargetSubnetTest {
@Inject
EfsService efsService;
@BeforeAll @BeforeAll
static void configureRestAssured() { static void configureRestAssured() {
RestAssuredJsonUtils.configureAwsContentTypes(); RestAssuredJsonUtils.configureAwsContentTypes();
@@ -46,7 +56,7 @@ class EfsMountTargetSubnetTest {
.statusCode(200) .statusCode(200)
.body("VpcId", equalTo("vpc-default-us-east-1")) .body("VpcId", equalTo("vpc-default-us-east-1"))
.body("AvailabilityZoneName", equalTo("us-east-1b")) .body("AvailabilityZoneName", equalTo("us-east-1b"))
.body("IpAddress", equalTo("172.31.16.4")); .body("IpAddress", startsWith("172.31.16."));
createMountTarget(fileSystemId, "subnet-default-us-east-1-c") createMountTarget(fileSystemId, "subnet-default-us-east-1-c")
.statusCode(200) .statusCode(200)
.body("AvailabilityZoneName", equalTo("us-east-1c")); .body("AvailabilityZoneName", equalTo("us-east-1c"));
@@ -61,6 +71,21 @@ class EfsMountTargetSubnetTest {
.body("__type", equalTo("SubnetNotFound")); .body("__type", equalTo("SubnetNotFound"));
} }
@Test
void refusesASecurityGroupThatDoesNotExist() {
String fileSystemId = createFileSystem("subnet-missing-sg");
given()
.contentType("application/json")
.body("{\"FileSystemId\": \"" + fileSystemId + "\", \"SubnetId\": \"subnet-default-us-east-1-a\","
+ " \"SecurityGroups\": [\"sg-nope\"]}")
.when()
.post("/2015-02-01/mount-targets")
.then()
.statusCode(400)
.body("__type", equalTo("SecurityGroupNotFound"));
}
@Test @Test
void allowsOneMountTargetPerZone() { void allowsOneMountTargetPerZone() {
String fileSystemId = createFileSystem("subnet-one-per-zone"); String fileSystemId = createFileSystem("subnet-one-per-zone");
@@ -84,8 +109,55 @@ class EfsMountTargetSubnetTest {
} }
@Test @Test
void picksAddressesAfterTheFourAwsReserves() { void holdsAnEc2NetworkInterfaceWithItsAddressUntilDeleted() {
assertEquals("10.0.1.4", EfsService.hostAddress("10.0.1.0/24", 0)); String fileSystemId = createFileSystem("subnet-eni");
assertEquals("10.0.2.1", EfsService.hostAddress("10.0.1.0/24", 253)); var mountTarget = createMountTarget(fileSystemId, "subnet-default-us-east-1-c")
.statusCode(200)
.extract().jsonPath();
String eniId = mountTarget.getString("NetworkInterfaceId");
describeNetworkInterface(eniId)
.statusCode(200)
.body("DescribeNetworkInterfacesResponse.networkInterfaceSet.item.privateIpAddress",
equalTo(mountTarget.getString("IpAddress")))
.body("DescribeNetworkInterfacesResponse.networkInterfaceSet.item.status", equalTo("in-use"))
.body("DescribeNetworkInterfacesResponse.networkInterfaceSet.item.subnetId",
equalTo("subnet-default-us-east-1-c"));
given()
.when()
.delete("/2015-02-01/mount-targets/" + mountTarget.getString("MountTargetId"))
.then()
.statusCode(204);
describeNetworkInterface(eniId).statusCode(400);
}
@Test
void resolvesTheFileSystemDnsNameToAMountTarget() {
String fileSystemId = createFileSystem("subnet-dns");
String inZoneB = createMountTarget(fileSystemId, "subnet-default-us-east-1-b")
.statusCode(200).extract().jsonPath().getString("IpAddress");
String inZoneA = createMountTarget(fileSystemId, "subnet-default-us-east-1-a")
.statusCode(200).extract().jsonPath().getString("IpAddress");
assertEquals(Optional.of(inZoneA),
efsService.resolveDnsName(fileSystemId + ".efs.us-east-1.amazonaws.com"));
assertEquals(Optional.of(inZoneB),
efsService.resolveDnsName("us-east-1b." + fileSystemId + ".efs.us-east-1.amazonaws.com."));
assertEquals(Optional.empty(),
efsService.resolveDnsName("us-east-1c." + fileSystemId + ".efs.us-east-1.amazonaws.com"));
assertEquals(Optional.empty(),
efsService.resolveDnsName(fileSystemId + ".efs.eu-west-1.amazonaws.com"));
assertEquals(Optional.empty(), efsService.resolveDnsName("example.com"));
}
private io.restassured.response.ValidatableResponse describeNetworkInterface(String eniId) {
return given()
.formParam("Action", "DescribeNetworkInterfaces")
.formParam("NetworkInterfaceId.1", eniId)
.header("Authorization", "AWS4-HMAC-SHA256 Credential=test/20260101/us-east-1/ec2/aws4_request")
.when()
.post("/")
.then();
} }
} }
@@ -0,0 +1,170 @@
package io.github.hectorvent.floci.services.efs;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.DirectoryCreation;
import io.github.hectorvent.floci.services.efs.model.AccessPointDescription;
import io.github.hectorvent.floci.services.efs.model.CreationInfo;
import io.github.hectorvent.floci.services.efs.model.PosixUser;
import io.github.hectorvent.floci.services.efs.model.RootDirectory;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Optional;
import java.util.OptionalInt;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.RETURNS_SELF;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/** Every mount of a file system shares its one volume; access points mount their root directory in it. */
class EfsMountsTest {
private EfsService efsService;
private ContainerLifecycleManager lifecycleManager;
private EfsMounts mounts;
@BeforeEach
void setUp() {
efsService = mock(EfsService.class);
lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.ensureSharedVolumeDirectory(anyString(), anyString(), any(), any())).thenReturn(true);
EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
when(config.storage().efs().ownerUid()).thenReturn(OptionalInt.empty());
when(config.storage().efs().ownerGid()).thenReturn(OptionalInt.empty());
when(config.storage().efs().rootPermissions()).thenReturn(Optional.empty());
when(config.storage().efs().initImage()).thenReturn("busybox:stable");
mounts = new EfsMounts(efsService, lifecycleManager, config);
}
@Test
void theFileSystemRootMountsTheWholeVolume() {
EfsMounts.Mount mount = mounts.prepare("us-east-1", "fs-abc", null, null);
assertEquals("floci-efs-fs-abc", mount.volumeName());
assertEquals("", mount.subpath());
assertTrue(mount.accessPointUser().isEmpty());
verify(lifecycleManager).ensureSharedVolume("floci-efs-fs-abc",
OptionalInt.empty(), OptionalInt.empty(), Optional.empty(), "busybox:stable");
verify(lifecycleManager, never()).ensureSharedVolumeDirectory(anyString(), anyString(), any(), any());
}
@Test
void aRootDirectoryMountsItsFolderOfTheSameVolume() {
EfsMounts.Mount mount = mounts.prepare("us-east-1", "fs-abc", null, "/team-a/");
assertEquals("floci-efs-fs-abc", mount.volumeName());
assertEquals("team-a", mount.subpath());
verify(lifecycleManager).ensureSharedVolumeDirectory("floci-efs-fs-abc", "team-a",
Optional.of(new DirectoryCreation(0, 0, "755")), "busybox:stable");
}
@Test
void anAccessPointMountsItsRootDirectoryAndCarriesItsUser() {
when(efsService.describeAccessPoints("us-east-1", null, "fsap-1"))
.thenReturn(List.of(accessPoint("fs-abc", "/my-function", creationInfo(1001L, 1001L, "755"))));
EfsMounts.Mount mount = mounts.prepare("us-east-1", null, "fsap-1", "/ignored");
assertEquals("floci-efs-fs-abc", mount.volumeName());
assertEquals("my-function", mount.subpath());
assertEquals("1001:1001", EfsMounts.containerUser(mount.accessPointUser().orElseThrow()));
assertEquals(List.of(2000L), EfsMounts.secondaryGids(mount.accessPointUser().orElseThrow()));
verify(lifecycleManager).ensureSharedVolumeDirectory("floci-efs-fs-abc", "my-function",
Optional.of(new DirectoryCreation(1001, 1001, "755")), "busybox:stable");
}
@Test
void aLambdaAccessPointArnIsLookedUpInItsRegion() {
when(efsService.describeAccessPoints("eu-west-1", null, "fsap-1"))
.thenReturn(List.of(accessPoint("fs-abc", "/", null)));
EfsMounts.Mount mount = mounts.prepareAccessPoint(
"arn:aws:elasticfilesystem:eu-west-1:000000000000:access-point/fsap-1");
assertEquals("floci-efs-fs-abc", mount.volumeName());
assertEquals("", mount.subpath());
}
@Test
void aMissingRootDirectoryWithoutCreationInfoCannotBeMounted() {
when(efsService.describeAccessPoints("us-east-1", null, "fsap-1"))
.thenReturn(List.of(accessPoint("fs-abc", "/absent", null)));
when(lifecycleManager.ensureSharedVolumeDirectory("floci-efs-fs-abc", "absent", Optional.empty(),
"busybox:stable")).thenReturn(false);
IllegalStateException error = assertThrows(IllegalStateException.class,
() -> mounts.prepare("us-east-1", "fs-abc", "fsap-1", null));
assertTrue(error.getMessage().contains("/absent"), error.getMessage());
}
@Test
void anAccessPointOfAnotherFileSystemIsRefused() {
when(efsService.describeAccessPoints("us-east-1", null, "fsap-1"))
.thenReturn(List.of(accessPoint("fs-other", "/", null)));
assertThrows(IllegalArgumentException.class, () -> mounts.prepare("us-east-1", "fs-abc", "fsap-1", null));
}
@Test
void anUnknownAccessPointIsRefused() {
when(efsService.describeAccessPoints("us-east-1", null, "fsap-404")).thenReturn(List.of());
assertThrows(EfsException.class, () -> mounts.prepare("us-east-1", "fs-abc", "fsap-404", null));
}
@Test
void equivalentRootDirectorySpellingsNameOneFolder() {
assertEquals("/", EfsMounts.normalizeRootDirectory(null));
assertEquals("/", EfsMounts.normalizeRootDirectory(" "));
assertEquals("/team-a", EfsMounts.normalizeRootDirectory("team-a"));
assertEquals("/team-a", EfsMounts.normalizeRootDirectory("//team-a/"));
assertThrows(IllegalArgumentException.class, () -> EfsMounts.normalizeRootDirectory("/a/../b"));
}
@Test
void aFolderMountUsesTheVolumeSubpath() {
ContainerBuilder.Builder spec = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
new EfsMounts.Mount("floci-efs-fs-abc", "team-a", Optional.empty()).applyTo(spec, "/mnt/efs", true);
new EfsMounts.Mount("floci-efs-fs-abc", "", Optional.empty()).applyTo(spec, "/mnt/all", false);
verify(spec).withNamedVolumeSubpath("floci-efs-fs-abc", "team-a", "/mnt/efs", true);
verify(spec).withNamedVolume("floci-efs-fs-abc", "/mnt/all", false);
}
private static AccessPointDescription accessPoint(String fileSystemId, String path, CreationInfo creationInfo) {
RootDirectory root = new RootDirectory();
root.setPath(path);
root.setCreationInfo(creationInfo);
PosixUser user = new PosixUser();
user.setUid(1001L);
user.setGid(1001L);
user.setSecondaryGids(List.of(2000L));
AccessPointDescription accessPoint = new AccessPointDescription();
accessPoint.setAccessPointId("fsap-1");
accessPoint.setFileSystemId(fileSystemId);
accessPoint.setRootDirectory(root);
accessPoint.setPosixUser(user);
return accessPoint;
}
private static CreationInfo creationInfo(Long uid, Long gid, String permissions) {
CreationInfo info = new CreationInfo();
info.setOwnerUid(uid);
info.setOwnerGid(gid);
info.setPermissions(permissions);
return info;
}
}
@@ -250,7 +250,8 @@ class LambdaImageConfigTest {
launcher = new ContainerLauncher(containerBuilder, lifecycleManager, logStreamer, imageResolver, launcher = new ContainerLauncher(containerBuilder, lifecycleManager, logStreamer, imageResolver,
runtimeApiServerFactory, dockerHostResolver, config, ecrRegistryManager, runtimeApiServerFactory, dockerHostResolver, config, ecrRegistryManager,
mock(io.github.hectorvent.floci.services.lambda.LambdaLayerService.class), awsEnv, mock(io.github.hectorvent.floci.services.lambda.LambdaLayerService.class), awsEnv,
mock(io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials.class)); mock(io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials.class),
mock(io.github.hectorvent.floci.services.efs.EfsMounts.class));
when(runtimeApiServerFactory.create()).thenReturn(runtimeApiServer); when(runtimeApiServerFactory.create()).thenReturn(runtimeApiServer);
when(runtimeApiServer.getPort()).thenReturn(9000); when(runtimeApiServer.getPort()).thenReturn(9000);
@@ -59,7 +59,7 @@ class ContainerLauncherCodeVolumeReuseTest {
mock(ImageResolver.class), mock(RuntimeApiServerFactory.class), mock(ImageResolver.class), mock(RuntimeApiServerFactory.class),
mock(DockerHostResolver.class), config, mock(EcrRegistryManager.class), mock(DockerHostResolver.class), config, mock(EcrRegistryManager.class),
mock(LambdaLayerService.class), mock(LaunchedContainerAwsEnv.class), mock(LambdaLayerService.class), mock(LaunchedContainerAwsEnv.class),
mock(LambdaExecutionRoleCredentials.class)); mock(LambdaExecutionRoleCredentials.class), mock(io.github.hectorvent.floci.services.efs.EfsMounts.class));
} }
@Override @Override
@@ -12,6 +12,9 @@ import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager; import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
import io.github.hectorvent.floci.services.iam.model.SessionCreds; import io.github.hectorvent.floci.services.iam.model.SessionCreds;
import io.github.hectorvent.floci.services.lambda.model.LambdaFileSystemConfig; import io.github.hectorvent.floci.services.lambda.model.LambdaFileSystemConfig;
import io.github.hectorvent.floci.services.efs.EfsMounts;
import io.github.hectorvent.floci.services.efs.model.PosixUser;
import io.github.hectorvent.floci.core.common.docker.SubpathVolumeOptions;
import io.github.hectorvent.floci.services.lambda.model.LambdaFunction; import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer; import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;
import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory; import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;
@@ -87,6 +90,7 @@ class ContainerLauncherTest {
Path tempDir; Path tempDir;
ContainerLauncher launcher; ContainerLauncher launcher;
EfsMounts efsMounts = mock(EfsMounts.class);
/** Collects remote paths passed to withRemotePath across all copy mocks. */ /** Collects remote paths passed to withRemotePath across all copy mocks. */
final java.util.List<String> capturedRemotePaths = new java.util.ArrayList<>(); final java.util.List<String> capturedRemotePaths = new java.util.ArrayList<>();
@@ -134,7 +138,7 @@ class ContainerLauncherTest {
launcher = new ContainerLauncher(containerBuilder, lifecycleManager, logStreamer, imageResolver, launcher = new ContainerLauncher(containerBuilder, lifecycleManager, logStreamer, imageResolver,
runtimeApiServerFactory, dockerHostResolver, config, ecrRegistryManager, runtimeApiServerFactory, dockerHostResolver, config, ecrRegistryManager,
mock(io.github.hectorvent.floci.services.lambda.LambdaLayerService.class), awsEnv, mock(io.github.hectorvent.floci.services.lambda.LambdaLayerService.class), awsEnv,
executionRoleCredentials); executionRoleCredentials, efsMounts);
when(runtimeApiServerFactory.create()).thenReturn(runtimeApiServer); when(runtimeApiServerFactory.create()).thenReturn(runtimeApiServer);
when(runtimeApiServer.getPort()).thenReturn(9000); when(runtimeApiServer.getPort()).thenReturn(9000);
@@ -469,8 +473,15 @@ class ContainerLauncherTest {
} }
@Test @Test
void launchFunction_mountsConfiguredFileSystemVolume() throws Exception { void launchFunction_mountsTheAccessPointFolderAsItsUser() throws Exception {
Path codePath = Files.createDirectory(tempDir.resolve("efs-code")); Path codePath = Files.createDirectory(tempDir.resolve("efs-code"));
String accessPointArn = "arn:aws:elasticfilesystem:us-east-1:000000000000:access-point/fsap-0123456789abcdef0";
PosixUser user = new PosixUser();
user.setUid(1001L);
user.setGid(1002L);
user.setSecondaryGids(List.of(2000L));
when(efsMounts.prepareAccessPoint(accessPointArn))
.thenReturn(new EfsMounts.Mount("floci-efs-fs-abc", "my-function", Optional.of(user)));
LambdaFunction fn = new LambdaFunction(); LambdaFunction fn = new LambdaFunction();
fn.setFunctionName("efs-fn"); fn.setFunctionName("efs-fn");
@@ -478,23 +489,48 @@ class ContainerLauncherTest {
fn.setRuntime("nodejs20.x"); fn.setRuntime("nodejs20.x");
fn.setHandler("index.handler"); fn.setHandler("index.handler");
fn.setCodeLocalPath(codePath.toString()); fn.setCodeLocalPath(codePath.toString());
fn.setFileSystemConfigs(List.of(new LambdaFileSystemConfig( fn.setFileSystemConfigs(List.of(new LambdaFileSystemConfig(accessPointArn, "/mnt/shared")));
"arn:aws:elasticfilesystem:us-east-1:000000000000:access-point/fsap-0123456789abcdef0",
"/mnt/shared")));
launcher.launch(fn); launcher.launch(fn);
String expectedVolumeName = "floci-efs-fsap-0123456789abcdef0-" ContainerSpec spec = captureRealContainerSpec();
+ "9d6eafd2aec94d4518a004f005725b4b3c673c1506436bb7368cfd5450fc0810"; Mount mount = spec.mounts().stream()
verify(lifecycleManager).ensureSharedVolume(expectedVolumeName,
OptionalInt.empty(), OptionalInt.empty(), Optional.empty(), "busybox:stable");
Mount mount = captureRealContainerSpec().mounts().stream()
.filter(candidate -> "/mnt/shared".equals(candidate.getTarget())) .filter(candidate -> "/mnt/shared".equals(candidate.getTarget()))
.findFirst() .findFirst()
.orElseThrow(); .orElseThrow();
assertEquals(MountType.VOLUME, mount.getType()); assertEquals(MountType.VOLUME, mount.getType());
assertEquals(expectedVolumeName, mount.getSource()); assertEquals("floci-efs-fs-abc", mount.getSource());
assertEquals("my-function", ((SubpathVolumeOptions) mount.getVolumeOptions()).getSubpath());
assertTrue(!Boolean.TRUE.equals(mount.getReadOnly())); assertTrue(!Boolean.TRUE.equals(mount.getReadOnly()));
assertEquals("1001:1002", spec.user());
assertEquals(List.of("2000"), spec.groupAdd());
}
@Test
void launchFunction_withoutAnAccessPointUserKeepsTheImageUser() throws Exception {
Path codePath = Files.createDirectory(tempDir.resolve("efs-root-code"));
String accessPointArn = "arn:aws:elasticfilesystem:us-east-1:000000000000:access-point/fsap-1";
when(efsMounts.prepareAccessPoint(accessPointArn))
.thenReturn(new EfsMounts.Mount("floci-efs-fs-abc", "", Optional.empty()));
LambdaFunction fn = new LambdaFunction();
fn.setFunctionName("efs-root-fn");
fn.setFunctionArn("arn:aws:lambda:us-east-1:000000000000:function:efs-root-fn");
fn.setRuntime("nodejs20.x");
fn.setHandler("index.handler");
fn.setCodeLocalPath(codePath.toString());
fn.setFileSystemConfigs(List.of(new LambdaFileSystemConfig(accessPointArn, "/mnt/efs")));
launcher.launch(fn);
ContainerSpec spec = captureRealContainerSpec();
Mount mount = spec.mounts().stream()
.filter(candidate -> "/mnt/efs".equals(candidate.getTarget()))
.findFirst()
.orElseThrow();
assertEquals("floci-efs-fs-abc", mount.getSource());
assertEquals(null, mount.getVolumeOptions());
assertEquals(null, spec.user());
} }
@Test @Test
@@ -1631,7 +1667,7 @@ class ContainerLauncherTest {
ecrRegistryManager, ecrRegistryManager,
mock(io.github.hectorvent.floci.services.lambda.LambdaLayerService.class), mock(io.github.hectorvent.floci.services.lambda.LambdaLayerService.class),
new LaunchedContainerAwsEnv(reachableEndpoint), new LaunchedContainerAwsEnv(reachableEndpoint),
executionRoleCredentials); executionRoleCredentials, efsMounts);
stubExtensionDiscovery("otel-collector"); stubExtensionDiscovery("otel-collector");
// Feed a real stdout frame through whatever callback the launcher hands to execStartCmd. // Feed a real stdout frame through whatever callback the launcher hands to execStartCmd.
@@ -100,33 +100,6 @@ class ContainerLauncherVolumeNamingTest {
"a later lastModified must produce a different volume name"); "a later lastModified must produce a different volume name");
} }
@Test
void efsVolumeNameIncludesStableHashOfCompleteArn() {
String arn = "arn:aws:elasticfilesystem:us-east-1:000000000000:"
+ "access-point/fsap-0123456789abcdef0";
String volumeName = ContainerLauncher.efsVolumeName(arn);
assertEquals("floci-efs-fsap-0123456789abcdef0-"
+ "9d6eafd2aec94d4518a004f005725b4b3c673c1506436bb7368cfd5450fc0810",
volumeName);
assertEquals(volumeName, ContainerLauncher.efsVolumeName(arn));
assertTrue(DOCKER_VOLUME_NAME.matcher(volumeName).matches(),
"EFS volume name must be Docker-safe, was: " + volumeName);
}
@Test
void efsVolumeNameDistinguishesSameResourceIdAcrossArns() {
String eastArn = "arn:aws:elasticfilesystem:us-east-1:000000000000:"
+ "access-point/fsap-0123456789abcdef0";
String westArn = "arn:aws:elasticfilesystem:us-west-2:111111111111:"
+ "access-point/fsap-0123456789abcdef0";
assertNotEquals(ContainerLauncher.efsVolumeName(eastArn),
ContainerLauncher.efsVolumeName(westArn),
"the complete ARN must participate in volume identity");
}
@TempDir @TempDir
Path tempDir; Path tempDir;
+1
View File
@@ -434,6 +434,7 @@ floci:
enabled: true enabled: true
efs: efs:
enabled: true enabled: true
nfs-servers: false
organizations: organizations:
enabled: true enabled: true
rum: rum: