fix(cloudfront): stream origin responses and viewer request bodies (#5278)

A distribution read every origin response whole before answering the
viewer: an S3 origin's object through getObject, a custom origin's body
through the HTTP client into a byte array. A viewer's POST, PUT, PATCH
or DELETE body was likewise read whole before it was forwarded. So an
object or upload of about 2 GiB or more failed, and a smaller one could
run the heap out of memory.

S3 origins now open the object as a stream, the snapshot GetObject
serves, and custom origins return their response before its body is
read; either body is written to the viewer as it arrives, with its
Content-Length, or chunked as the origin sent it. A viewer's body is
streamed to the custom origin with its Content-Length, or chunked when
the viewer sent it chunked or, as HTTP/2 allows, with no length.

An origin response closed before its end, as when the viewer goes away
or a custom error page replaces it, cancels its request instead of
reading the rest of the body to reuse the connection. Since a streamed
response holds its connection for the whole transfer, one origin may
now use the whole connection pool, as CloudFront sets no per-origin
limit.

(cherry picked from commit 1f1d4f94dd8a179972bda124bc3aa47b9704f69b)

src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.java

@@ -16,22 +16,29 @@ import io.github.hectorvent.floci.services.cloudfront.model.ResponseHeadersPolic
 import io.quarkus.test.junit.QuarkusTest;
 import io.quarkus.test.junit.QuarkusTestProfile;
 import io.quarkus.test.junit.TestProfile;
+import io.restassured.RestAssured;
 import io.restassured.response.Response;
 import jakarta.inject.Inject;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
 import java.net.InetSocketAddress;
+import java.net.Socket;
 import java.nio.charset.StandardCharsets;
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
 import java.security.Signature;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.Base64;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -43,6 +50,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 @QuarkusTest
@@ -316,6 +324,106 @@ class CloudFrontCustomOriginServingTest {
         assertEquals(5, hits.get());
     }
 
+    @Test
+    void streamsALargeOriginResponseWithItsLength() throws Exception {
+        byte[] video = randomBytes(3 * 1024 * 1024 + 1);
+        startServingOrigin(video, true);
+        Distribution created = cloudFrontService.createDistribution(
+                customOriginDistribution(allMethodsBehavior()), Map.of());
+
+        Response response = given().header("Host", created.getDomainName()).when().get("/video.bin");
+
+        assertEquals(200, response.statusCode());
+        assertEquals(Integer.toString(video.length), response.header("Content-Length"));
+        assertArrayEquals(video, response.asByteArray());
+    }
+
+    @Test
+    void passesAChunkedOriginResponseThroughChunked() throws Exception {
+        byte[] video = randomBytes(3 * 1024 * 1024 + 1);
+        startServingOrigin(video, false);
+        Distribution created = cloudFrontService.createDistribution(
+                customOriginDistribution(allMethodsBehavior()), Map.of());
+
+        Response response = given().header("Host", created.getDomainName()).when().get("/video.bin");
+
+        assertEquals(200, response.statusCode());
+        assertNull(response.header("Content-Length"), "an origin that sent no length was given one");
+        assertArrayEquals(video, response.asByteArray());
+    }
+
+    @Test
+    void streamsALargeViewerBodyToTheOriginWithItsLength() throws Exception {
+        AtomicReference<ReceivedRequest> received = new AtomicReference<>();
+        startRecordingOrigin(received, new AtomicInteger());
+        Distribution created = cloudFrontService.createDistribution(
+                customOriginDistribution(allMethodsBehavior()), Map.of());
+        byte[] upload = randomBytes(3 * 1024 * 1024 + 1);
+
+        given()
+            .header("Host", created.getDomainName())
+            .contentType("application/octet-stream")
+            .body(upload)
+        .when()
+            .put("/uploads/video.bin")
+        .then()
+            .statusCode(200);
+
+        assertEquals(Integer.toString(upload.length), received.get().header("Content-Length"));
+        assertArrayEquals(upload, received.get().body());
+    }
+
+    @Test
+    void forwardsAChunkedViewerBodyChunked() throws Exception {
+        AtomicReference<ReceivedRequest> received = new AtomicReference<>();
+        startRecordingOrigin(received, new AtomicInteger());
+        Distribution created = cloudFrontService.createDistribution(
+                customOriginDistribution(allMethodsBehavior()), Map.of());
+        byte[] upload = randomBytes(200_001);
+
+        int status = assertTimeoutPreemptively(Duration.ofSeconds(60),
+                () -> postChunked(created.getDomainName(), "/uploads", upload));
+
+        assertEquals(201, status);
+        assertEquals("chunked", received.get().header("Transfer-Encoding"));
+        assertNull(received.get().header("Content-Length"));
+        assertArrayEquals(upload, received.get().body());
+    }
+
+    @Test
+    void releasesTheOriginConnectionOfEveryResponseItDoesNotSend() throws Exception {
+        // The origin client keeps a fixed number of connections. An error response replaced by a custom
+        // error page, or a HEAD's, that kept its connection would leave a request past that number
+        // waiting for one.
+        originServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        originServer.createContext("/", exchange -> {
+            boolean errorPage = "/errors/404.html".equals(exchange.getRequestURI().getRawPath());
+            byte[] body = (errorPage ? "error-page" : "x".repeat(100_000)).getBytes(StandardCharsets.UTF_8);
+            exchange.getResponseHeaders().add("Content-Type", "text/plain");
+            boolean head = "HEAD".equals(exchange.getRequestMethod());
+            exchange.sendResponseHeaders(errorPage ? 200 : 404, head ? -1 : body.length);
+            if (!head) {
+                exchange.getResponseBody().write(body);
+            }
+            exchange.close();
+        });
+        originServer.start();
+        Map<String, Object> errorPage = new LinkedHashMap<>(Map.of(
+                "ErrorCode", "404", "ResponseCode", "200", "ResponsePagePath", "/errors/404.html"));
+        Distribution distribution = customOriginDistribution(allMethodsBehavior());
+        distribution.getConfig().setCustomErrorResponses(List.of(errorPage));
+        Distribution created = cloudFrontService.createDistribution(distribution, Map.of());
+
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> {
+            for (int i = 0; i < CloudFrontOriginHttpClient.MAX_CONNECTIONS + 10; i++) {
+                given().header("Host", created.getDomainName()).when().get("/missing/" + i)
+                        .then().statusCode(200).body(equalTo("error-page"));
+                given().header("Host", created.getDomainName()).when().head("/missing/" + i)
+                        .then().statusCode(200);
+            }
+        });
+    }
+
     @Test
     void forwardsTheViewerAuthorizationOnWriteMethodsOnly() throws Exception {
         AtomicReference<ReceivedRequest> received = new AtomicReference<>();
@@ -705,6 +813,51 @@ class CloudFrontCustomOriginServingTest {
         originServer.start();
     }
 
+    /** An origin that answers every GET with {@code body}, with its length or chunked. */
+    private void startServingOrigin(byte[] body, boolean withLength) throws IOException {
+        originServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        originServer.createContext("/", exchange -> {
+            exchange.getRequestBody().readAllBytes();
+            exchange.getResponseHeaders().add("Content-Type", "application/octet-stream");
+            exchange.sendResponseHeaders(200, withLength ? body.length : 0);
+            exchange.getResponseBody().write(body);
+            exchange.close();
+        });
+        originServer.start();
+    }
+
+    /**
+     * POSTs {@code body} with {@code Transfer-Encoding: chunked} over a raw connection, since an HTTP
+     * client library buffers a stream it is given and sends its length instead.
+     */
+    private static int postChunked(String host, String path, byte[] body) throws IOException {
+        try (Socket socket = new Socket("localhost", RestAssured.port)) {
+            // A response that never comes fails the read instead of leaving it waiting.
+            socket.setSoTimeout(30_000);
+            OutputStream out = socket.getOutputStream();
+            out.write(("POST " + path + " HTTP/1.1\r\nHost: " + host
+                    + "\r\nContent-Type: application/octet-stream\r\nTransfer-Encoding: chunked"
+                    + "\r\nConnection: close\r\n\r\n").getBytes(StandardCharsets.US_ASCII));
+            for (int offset = 0; offset < body.length; offset += 8192) {
+                int size = Math.min(8192, body.length - offset);
+                out.write((Integer.toHexString(size) + "\r\n").getBytes(StandardCharsets.US_ASCII));
+                out.write(body, offset, size);
+                out.write("\r\n".getBytes(StandardCharsets.US_ASCII));
+            }
+            out.write("0\r\n\r\n".getBytes(StandardCharsets.US_ASCII));
+            out.flush();
+            String statusLine = new BufferedReader(
+                    new InputStreamReader(socket.getInputStream(), StandardCharsets.US_ASCII)).readLine();
+            return Integer.parseInt(statusLine.split(" ")[1]);
+        }
+    }
+
+    private static byte[] randomBytes(int size) {
+        byte[] bytes = new byte[size];
+        new Random(size).nextBytes(bytes);
+        return bytes;
+    }
+
     private Distribution customOriginDistribution(DefaultCacheBehavior behavior) {
         return customOriginDistribution(behavior, null);
     }

src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontDistributionServingTest.java

@@ -273,6 +273,7 @@ class CloudFrontDistributionServingTest {
                 .header("x-amz-meta-source", equalTo("origin"))
                 .header("x-amz-server-side-encryption", nullValue())
                 .header("x-amz-checksum-sha256", nullValue())
+                .header("Content-Length", equalTo(Integer.toString(body.getBytes(StandardCharsets.UTF_8).length)))
                 .body(equalTo(body));
 
         given().header("Host", dist.getDomainName()).when().head("/asset.txt")

src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.java

@@ -16,12 +16,15 @@ import javax.net.ssl.SNIHostName;
 import javax.net.ssl.SNIServerName;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManagerFactory;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.URI;
 import java.net.UnknownHostException;
 import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
 import java.nio.charset.StandardCharsets;
 import java.security.KeyStore;
 import java.security.Security;
@@ -29,12 +32,18 @@ import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 class CloudFrontOriginHttpClientTest {
 
@@ -46,8 +55,7 @@ class CloudFrontOriginHttpClientTest {
                 resolver(InetAddress.getByName("127.0.0.1")), List.of())) {
             HttpRequest request = request("http://blocked.invalid:" + server.getAddress().getPort() + "/");
 
-            assertThrows(UnknownHostException.class,
-                    () -> client.send(request, HttpResponse.BodyHandlers.ofByteArray()));
+            assertThrows(UnknownHostException.class, () -> get(client, request));
             assertEquals(0, hits.get());
         } finally {
             server.stop(0);
@@ -60,9 +68,7 @@ class CloudFrontOriginHttpClientTest {
                 InetAddress.getByName("8.8.8.8"),
                 InetAddress.getByName("127.0.0.1"));
         try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(resolver, List.of())) {
-            assertThrows(UnknownHostException.class, () -> client.send(
-                    request("http://mixed.invalid:8080/"),
-                    HttpResponse.BodyHandlers.ofByteArray()));
+            assertThrows(UnknownHostException.class, () -> get(client, request("http://mixed.invalid:8080/")));
         }
     }
 
@@ -99,12 +105,11 @@ class CloudFrontOriginHttpClientTest {
 
         try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
                 resolver, List.of("rebind.invalid"))) {
-            HttpResponse<byte[]> response = client.send(
-                    request("http://rebind.invalid:" + server.getAddress().getPort() + "/"),
-                    HttpResponse.BodyHandlers.ofByteArray());
+            CloudFrontOriginHttpClient.OpenResponse response = get(client,
+                    request("http://rebind.invalid:" + server.getAddress().getPort() + "/"));
 
             assertEquals(200, response.statusCode());
-            assertArrayEquals("origin-ok".getBytes(StandardCharsets.UTF_8), response.body());
+            assertArrayEquals("origin-ok".getBytes(StandardCharsets.UTF_8), readAll(response));
             assertEquals(1, resolutions.get());
             assertEquals(1, hits.get());
             assertEquals("rebind.invalid:" + server.getAddress().getPort(), hostHeader.get());
@@ -131,9 +136,9 @@ class CloudFrontOriginHttpClientTest {
 
         try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
                 resolver(InetAddress.getByName("127.0.0.1")), List.of("redirect.invalid"))) {
-            HttpResponse<byte[]> response = client.send(
-                    request("http://redirect.invalid:" + server.getAddress().getPort() + "/start"),
-                    HttpResponse.BodyHandlers.ofByteArray());
+            CloudFrontOriginHttpClient.OpenResponse response = get(client,
+                    request("http://redirect.invalid:" + server.getAddress().getPort() + "/start"));
+            readAll(response);
 
             assertEquals(302, response.statusCode());
             assertEquals(0, redirectedHits.get());
@@ -162,10 +167,7 @@ class CloudFrontOriginHttpClientTest {
                     .GET()
                     .build();
 
-            client.send(
-                    request,
-                    Map.of("X-Origin-Verify", "configured-value"),
-                    HttpResponse.BodyHandlers.ofByteArray());
+            readAll(client.open(request, List.of(), Map.of("X-Origin-Verify", "configured-value"), null, -1));
 
             assertEquals(List.of("configured-value"), receivedValues.get());
         } finally {
@@ -200,8 +202,9 @@ class CloudFrontOriginHttpClientTest {
                     .method("PUT", HttpRequest.BodyPublishers.noBody())
                     .build();
 
-            HttpResponse<byte[]> response = client.send(
-                    request, List.of(), Map.of(), payload, HttpResponse.BodyHandlers.ofByteArray());
+            CloudFrontOriginHttpClient.OpenResponse response = client.open(
+                    request, List.of(), Map.of(), new ByteArrayInputStream(payload), payload.length);
+            readAll(response);
 
             assertEquals(204, response.statusCode());
             assertEquals("PUT", method.get());
@@ -249,21 +252,125 @@ class CloudFrontOriginHttpClientTest {
         try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
                 resolver, List.of("origin.invalid", "wrong.invalid"), clientContext)) {
             int port = server.getAddress().getPort();
-            HttpResponse<byte[]> response = client.send(
-                    request("https://origin.invalid:" + port + "/"),
-                    HttpResponse.BodyHandlers.ofByteArray());
+            CloudFrontOriginHttpClient.OpenResponse response = get(client,
+                    request("https://origin.invalid:" + port + "/"));
+            readAll(response);
 
             assertEquals(200, response.statusCode());
             assertEquals("origin.invalid:" + port, hostHeader.get());
             assertEquals("origin.invalid", requestedSni.get());
-            assertThrows(IOException.class, () -> client.send(
-                    request("https://wrong.invalid:" + port + "/"),
-                    HttpResponse.BodyHandlers.ofByteArray()));
+            assertThrows(IOException.class, () -> get(client, request("https://wrong.invalid:" + port + "/")));
         } finally {
             server.stop(0);
         }
     }
 
+    @Test
+    void streamsARequestBodyChunkedWhenItsLengthIsUnknown() throws Exception {
+        AtomicReference<String> transferEncoding = new AtomicReference<>();
+        AtomicReference<byte[]> body = new AtomicReference<>();
+        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        server.createContext("/", exchange -> {
+            transferEncoding.set(exchange.getRequestHeaders().getFirst("Transfer-Encoding"));
+            body.set(exchange.getRequestBody().readAllBytes());
+            exchange.sendResponseHeaders(204, -1);
+            exchange.close();
+        });
+        server.start();
+
+        byte[] payload = new byte[100_000];
+        new Random(1).nextBytes(payload);
+        try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
+                resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {
+            HttpRequest request = HttpRequest.newBuilder(URI.create(
+                            "http://origin.invalid:" + server.getAddress().getPort() + "/items"))
+                    .timeout(Duration.ofSeconds(5))
+                    .method("POST", HttpRequest.BodyPublishers.noBody())
+                    .build();
+
+            readAll(client.open(request, List.of(), Map.of(), new ByteArrayInputStream(payload), -1));
+
+            assertEquals("chunked", transferEncoding.get());
+            assertArrayEquals(payload, body.get());
+        } finally {
+            server.stop(0);
+        }
+    }
+
+    @Test
+    void closingAResponseBeforeItsEndDoesNotReadTheRestOfIt() throws Exception {
+        long size = 512L * 1024 * 1024;
+        AtomicLong sent = new AtomicLong();
+        CountDownLatch done = new CountDownLatch(1);
+        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        server.createContext("/", exchange -> {
+            byte[] chunk = new byte[64 * 1024];
+            exchange.sendResponseHeaders(200, size);
+            try (OutputStream out = exchange.getResponseBody()) {
+                for (long written = 0; written < size; written += chunk.length) {
+                    out.write(chunk);
+                    sent.addAndGet(chunk.length);
+                }
+            } catch (IOException expected) {
+                // The client went away mid-body, which is what the test does.
+            } finally {
+                done.countDown();
+            }
+        });
+        server.start();
+
+        try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
+                resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {
+            CloudFrontOriginHttpClient.OpenResponse response = get(client,
+                    request("http://origin.invalid:" + server.getAddress().getPort() + "/big"));
+            assertEquals(size, response.contentLength());
+            response.body().readNBytes(1024);
+            response.body().close();
+
+            assertTrue(done.await(30, TimeUnit.SECONDS), "the origin is still sending");
+            assertTrue(sent.get() < size / 2, "closing read " + (sent.get() >> 20) + " MiB of " + (size >> 20));
+        } finally {
+            server.stop(0);
+        }
+    }
+
+    @Test
+    void aResponseReadToItsEndLeavesItsConnectionForTheNextRequest() throws Exception {
+        List<InetSocketAddress> peers = new CopyOnWriteArrayList<>();
+        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        server.createContext("/", exchange -> {
+            peers.add(exchange.getRemoteAddress());
+            byte[] body = "origin-ok".getBytes(StandardCharsets.UTF_8);
+            exchange.sendResponseHeaders(200, body.length);
+            exchange.getResponseBody().write(body);
+            exchange.close();
+        });
+        server.start();
+
+        try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
+                resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {
+            String uri = "http://origin.invalid:" + server.getAddress().getPort() + "/";
+            readAll(get(client, request(uri)));
+            readAll(get(client, request(uri)));
+
+            assertEquals(2, peers.size());
+            assertEquals(peers.get(0), peers.get(1), "the second request opened a new connection");
+        } finally {
+            server.stop(0);
+        }
+    }
+
+    private static CloudFrontOriginHttpClient.OpenResponse get(CloudFrontOriginHttpClient client,
+                                                               HttpRequest request) throws Exception {
+        return client.open(request, List.of(), Map.of(), null, -1);
+    }
+
+    private static byte[] readAll(CloudFrontOriginHttpClient.OpenResponse response) throws IOException {
+        try (InputStream body = response.body()) {
+            return body.readAllBytes();
+        }
+    }
+
     private static HttpServer server(AtomicInteger hits) throws Exception {
         HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
         server.createContext("/", exchange -> {

src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontS3DiskOriginServingTest.java

@@ -0,0 +1,108 @@
+package io.github.hectorvent.floci.services.cloudfront;
+
+import io.github.hectorvent.floci.services.cloudfront.model.DefaultCacheBehavior;
+import io.github.hectorvent.floci.services.cloudfront.model.Distribution;
+import io.github.hectorvent.floci.services.cloudfront.model.DistributionConfig;
+import io.github.hectorvent.floci.services.cloudfront.model.Origin;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import io.restassured.response.Response;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * An S3 origin whose objects live on disk, where CloudFront streams the object file to the viewer
+ * instead of reading it into memory first.
+ */
+@QuarkusTest
+@TestProfile(CloudFrontS3DiskOriginServingTest.S3OnDiskProfile.class)
+class CloudFrontS3DiskOriginServingTest {
+
+    private static final String REGION = "us-east-1";
+
+    @Inject
+    S3Service s3Service;
+
+    @Inject
+    CloudFrontService cloudFrontService;
+
+    @Test
+    void streamsAnObjectFileWithItsLength() {
+        String suffix = Long.toString(System.nanoTime(), 36);
+        String bucket = "cf-disk-origin-" + suffix;
+        // Larger than any one buffer on the way, so the body arrives over many writes.
+        byte[] body = new byte[3 * 1024 * 1024 + 1];
+        new Random(5080).nextBytes(body);
+        s3Service.createBucket(bucket, REGION);
+        try {
+            s3Service.putObject(bucket, "video.bin", body, "application/octet-stream", Map.of());
+            s3Service.putObject(bucket, "index.html", ("INDEX-ON-DISK-" + suffix).getBytes(StandardCharsets.UTF_8),
+                    "text/html", Map.of());
+            Distribution dist = distributionFor(bucket);
+
+            Response get = given().header("Host", dist.getDomainName()).when().get("/video.bin");
+
+            assertEquals(200, get.statusCode());
+            assertEquals(Integer.toString(body.length), get.header("Content-Length"));
+            assertArrayEquals(body, get.asByteArray());
+            given().header("Host", dist.getDomainName()).when().head("/video.bin")
+                    .then().statusCode(200)
+                    .header("Content-Length", equalTo(Integer.toString(body.length)));
+            // The single-page-app fallback serves its error page through the same stream.
+            given().header("Host", dist.getDomainName()).when().get("/missing/route")
+                    .then().statusCode(200)
+                    .header("Content-Length", equalTo(Integer.toString(("INDEX-ON-DISK-" + suffix).length())))
+                    .body(containsString("INDEX-ON-DISK-" + suffix));
+        } finally {
+            // The bucket is on disk and outlives the test, so it is removed rather than left for the next run.
+            s3Service.deleteObject(bucket, "video.bin");
+            s3Service.deleteObject(bucket, "index.html");
+            s3Service.deleteBucket(bucket);
+        }
+    }
+
+    private Distribution distributionFor(String bucket) {
+        Origin origin = new Origin();
+        origin.setId("disk-origin");
+        origin.setDomainName(bucket + ".s3." + REGION + ".amazonaws.com");
+        origin.setS3OriginConfig(new LinkedHashMap<>(Map.of("OriginAccessIdentity", "")));
+        DefaultCacheBehavior behavior = new DefaultCacheBehavior();
+        behavior.setTargetOriginId(origin.getId());
+        behavior.setViewerProtocolPolicy("allow-all");
+        Map<String, Object> spaFallback = new LinkedHashMap<>();
+        spaFallback.put("ErrorCode", "404");
+        spaFallback.put("ResponseCode", "200");
+        spaFallback.put("ResponsePagePath", "/index.html");
+        DistributionConfig config = new DistributionConfig();
+        config.setEnabled(true);
+        config.setOrigins(List.of(origin));
+        config.setDefaultCacheBehavior(behavior);
+        config.setCustomErrorResponses(List.of(spaFallback));
+        Distribution distribution = new Distribution();
+        distribution.setConfig(config);
+        return cloudFrontService.createDistribution(distribution, Map.of());
+    }
+
+    public static final class S3OnDiskProfile implements QuarkusTestProfile {
+        @Override
+        public Map<String, String> getConfigOverrides() {
+            return Map.of(
+                    "floci.storage.services.s3.mode", "persistent",
+                    "floci.storage.persistent-path", "target/cloudfront-s3-disk-origin-it");
+        }
+    }
+}

src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontViewerBodyTest.java

@@ -0,0 +1,62 @@
+package io.github.hectorvent.floci.services.cloudfront;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/** How the length of a viewer's request body is taken before it is forwarded to a custom origin. */
+class CloudFrontViewerBodyTest {
+
+    private static final byte[] DATA = "a viewer upload".getBytes(StandardCharsets.UTF_8);
+
+    @Test
+    void aChunkedBodyIsForwardedChunkedWhateverItsContentLength() throws IOException {
+        CloudFrontServingController.ViewerBody body =
+                CloudFrontServingController.viewerBody(new ByteArrayInputStream(DATA), "chunked", "3");
+
+        assertEquals(-1, body.length());
+        assertArrayEquals(DATA, readAll(body));
+    }
+
+    @Test
+    void aBodyWithAContentLengthKeepsIt() throws IOException {
+        CloudFrontServingController.ViewerBody body = CloudFrontServingController.viewerBody(
+                new ByteArrayInputStream(DATA), null, Integer.toString(DATA.length));
+
+        assertEquals(DATA.length, body.length());
+        assertArrayEquals(DATA, readAll(body));
+    }
+
+    @Test
+    void anInvalidContentLengthIsForwardedChunked() {
+        assertEquals(-1, CloudFrontServingController.viewerBody(
+                new ByteArrayInputStream(DATA), null, "not-a-length").length());
+    }
+
+    @Test
+    void aRequestWithNeitherHeaderAndNoBodyIsEmpty() {
+        assertEquals(0, CloudFrontServingController.viewerBody(InputStream.nullInputStream(), null, null).length());
+        assertEquals(0, CloudFrontServingController.viewerBody(null, null, null).length());
+    }
+
+    @Test
+    void aBodyWithNeitherHeaderAsHttp2AllowsIsForwardedChunkedAndWhole() throws IOException {
+        CloudFrontServingController.ViewerBody body =
+                CloudFrontServingController.viewerBody(new ByteArrayInputStream(DATA), null, null);
+
+        assertEquals(-1, body.length());
+        assertArrayEquals(DATA, readAll(body), "the byte read to look at the body was lost");
+    }
+
+    private static byte[] readAll(CloudFrontServingController.ViewerBody body) throws IOException {
+        try (InputStream stream = body.stream()) {
+            return stream.readAllBytes();
+        }
+    }
+}