fix(glue): write timestamps the AWS SDK for JavaScript can read

Glue times went out as nanosecond epoch numbers (1792964720.708701378). The
AWS SDK for JavaScript v3 keeps any number whose text is not its shortest form
as a BigDecimal and then refuses it as a timestamp, so GetTables still failed.
They are now written as AWS writes them: epoch seconds to the millisecond, in
the shortest form. CreateDatabase also records the database's CreateTime.

src/main/java/io/github/hectorvent/floci/services/acm/AcmJsonHandler.java

@@ -563,8 +563,10 @@ public class AcmJsonHandler {
 
     private CertificateOptions parseOptions(JsonNode optionsNode) {
         if (optionsNode.isMissingNode()) return null;
-        String ctPref = optionsNode.path("CertificateTransparencyLoggingPreference").asText("ENABLED");
+        String ctPref = parseOptionValue(optionsNode, "CertificateTransparencyLoggingPreference");
-        return new CertificateOptions(ctPref, "DISABLED");
+        String export = parseOptionValue(optionsNode, "Export");
+        // An exportable public certificate is requested with Export ENABLED, and keeps it for life.
+        return new CertificateOptions(ctPref != null ? ctPref : "ENABLED", export != null ? export : "DISABLED");
     }
 
     private List<CertificateStatus> parseCertificateStatuses(JsonNode node) {

src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java

@@ -243,6 +243,9 @@ public class GlueService {
         }
         database.setName(databaseName);
         database.setCatalogId(regionResolver.getAccountId());
+        if (database.getCreateTime() == null) {
+            database.setCreateTime(Instant.now());
+        }
         databaseStore.put(databaseName, database);
         if (tags != null && !tags.isEmpty()) {
             resourceGroupsTaggingService.tagResources(List.of(databaseArn(region, databaseName)), tags, region);

src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java

@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model;
 import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 
 import java.time.Instant;
@@ -47,10 +48,12 @@ public class Connection {
 
     @JsonProperty("CreationTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant creationTime;
 
     @JsonProperty("LastUpdatedTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastUpdatedTime;
 
     @JsonProperty("LastUpdatedBy")
@@ -64,6 +67,7 @@ public class Connection {
 
     @JsonProperty("LastConnectionValidationTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastConnectionValidationTime;
 
     @JsonProperty("AuthenticationConfiguration")

src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java

@@ -1,12 +1,15 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 import java.time.Instant;
 import java.util.List;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class Crawler {
     @JsonProperty("Classifiers")
     private List<String> classifiers;
@@ -22,6 +25,7 @@ public class Crawler {
 
     @JsonProperty("CreationTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant creationTime;
 
     @JsonProperty("DatabaseName")
@@ -38,6 +42,7 @@ public class Crawler {
 
     @JsonProperty("LastUpdated")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastUpdated;
 
     @JsonProperty("LineageConfiguration")

src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java

@@ -1,12 +1,15 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 import java.time.Instant;
 import java.util.Map;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class Database {
     @JsonProperty("Name")
     private String name;
@@ -18,6 +21,7 @@ public class Database {
     private Map<String, String> parameters;
     @JsonProperty("CreateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant createTime;
     @JsonProperty("CatalogId")
     private String catalogId;

src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java

@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model;
 import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 
 import java.time.Instant;
@@ -22,10 +23,12 @@ public class GluePolicy {
 
     @JsonProperty("CreateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant createTime;
 
     @JsonProperty("UpdateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant updateTime;
 
     public String getPolicyInJson() { return policyInJson; }

src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java

@@ -0,0 +1,29 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+
+/**
+ * Writes a Glue timestamp as AWS does: epoch seconds to the millisecond, in the shortest
+ * form ({@code 1792964720.27}, not {@code 1792964720.270000000}). The AWS SDK for
+ * JavaScript v3 keeps any number whose text differs from its shortest form as a
+ * BigDecimal, and then refuses it as a timestamp.
+ */
+@RegisterForReflection
+public class GlueTimestampSerializer extends JsonSerializer<Instant> {
+
+    @Override
+    public void serialize(Instant value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
+        gen.writeNumber(epochSeconds(value));
+    }
+
+    static String epochSeconds(Instant value) {
+        return BigDecimal.valueOf(value.toEpochMilli(), 3).stripTrailingZeros().toPlainString();
+    }
+}

src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java

@@ -1,7 +1,9 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import com.fasterxml.jackson.databind.JsonNode;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 
@@ -9,6 +11,7 @@ import java.time.Instant;
 import java.util.Map;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class Job {
     @JsonProperty("AllocatedCapacity")
     private Integer allocatedCapacity;
@@ -24,6 +27,7 @@ public class Job {
 
     @JsonProperty("CreatedOn")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant createdOn;
 
     @JsonProperty("DefaultArguments")
@@ -49,6 +53,7 @@ public class Job {
 
     @JsonProperty("LastModifiedOn")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastModifiedOn;
 
     @JsonProperty("LogUri")

src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java

@@ -1,11 +1,14 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 import java.time.Instant;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class LastCrawlInfo {
     @JsonProperty("ErrorMessage")
     private String errorMessage;
@@ -21,6 +24,7 @@ public class LastCrawlInfo {
 
     @JsonProperty("StartTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant startTime;
 
     @JsonProperty("Status")

src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java

@@ -1,13 +1,16 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class Partition {
     @JsonProperty("Values")
     private List<String> values;
@@ -17,9 +20,11 @@ public class Partition {
     private String tableName;
     @JsonProperty("CreationTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant creationTime;
     @JsonProperty("LastAccessTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastAccessTime;
     @JsonProperty("StorageDescriptor")
     private StorageDescriptor storageDescriptor;

src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java

@@ -1,13 +1,16 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class Table {
     @JsonProperty("Name")
     private String name;
@@ -19,12 +22,15 @@ public class Table {
     private String owner;
     @JsonProperty("CreateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant createTime;
     @JsonProperty("UpdateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant updateTime;
     @JsonProperty("LastAccessTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastAccessTime;
     @JsonProperty("PartitionKeys")
     private List<Column> partitionKeys;

src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java

@@ -1,13 +1,16 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 
 import java.time.Instant;
 import java.util.List;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class UserDefinedFunction {
     @JsonProperty("FunctionName")
     private String functionName;
@@ -23,6 +26,7 @@ public class UserDefinedFunction {
     private String ownerType;
     @JsonProperty("CreateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant createTime;
     @JsonProperty("ResourceUris")
     private List<ResourceUri> resourceUris;

src/test/java/io/github/hectorvent/floci/services/acm/AcmExportableRequestTest.java

@@ -0,0 +1,70 @@
+package io.github.hectorvent.floci.services.acm;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+@QuarkusTest
+class AcmExportableRequestTest {
+
+    private static final String ACM_CONTENT_TYPE = "application/x-amz-json-1.1";
+
+    @BeforeAll
+    static void configureRestAssured() {
+        RestAssuredJsonUtils.configureAwsContentTypes();
+    }
+
+    @Test
+    void requestCertificateKeepsTheExportOption() {
+        String exportable = request("{\"DomainName\":\"exportable.example.com\",\"Options\":{\"Export\":\"ENABLED\"}}");
+
+        call("DescribeCertificate", "{\"CertificateArn\":\"" + exportable + "\"}")
+            .statusCode(200)
+            .body("Certificate.Options.Export", equalTo("ENABLED"))
+            .body("Certificate.Options.CertificateTransparencyLoggingPreference", equalTo("ENABLED"));
+
+        call("ExportCertificate", "{\"CertificateArn\":\"" + exportable + "\",\"Passphrase\":\"c2VjcmV0\"}")
+            .statusCode(200)
+            .body("PrivateKey", notNullValue());
+    }
+
+    @Test
+    void requestCertificateWithoutExportIsNotExportable() {
+        String plain = request("{\"DomainName\":\"plain.example.com\",\"Options\":{\"CertificateTransparencyLoggingPreference\":\"DISABLED\"}}");
+
+        call("DescribeCertificate", "{\"CertificateArn\":\"" + plain + "\"}")
+            .statusCode(200)
+            .body("Certificate.Options.Export", equalTo("DISABLED"))
+            .body("Certificate.Options.CertificateTransparencyLoggingPreference", equalTo("DISABLED"));
+
+        call("ExportCertificate", "{\"CertificateArn\":\"" + plain + "\",\"Passphrase\":\"c2VjcmV0\"}")
+            .statusCode(400)
+            .body("__type", equalTo("ValidationException"));
+    }
+
+    @Test
+    void requestCertificateRejectsAnUnknownExportValue() {
+        call("RequestCertificate", "{\"DomainName\":\"bad.example.com\",\"Options\":{\"Export\":\"MAYBE\"}}")
+            .statusCode(400)
+            .body("__type", equalTo("ValidationException"));
+    }
+
+    private String request(String body) {
+        return call("RequestCertificate", body).statusCode(200).extract().jsonPath().getString("CertificateArn");
+    }
+
+    private io.restassured.response.ValidatableResponse call(String action, String body) {
+        return given()
+            .header("X-Amz-Target", "CertificateManager." + action)
+            .contentType(ACM_CONTENT_TYPE)
+            .body(body)
+        .when()
+            .post("/")
+        .then();
+    }
+}

src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java

@@ -0,0 +1,67 @@
+package io.github.hectorvent.floci.services.glue;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * AWS leaves unset fields out of Glue responses and reports times in milliseconds.
+ * A null or nanosecond timestamp instead breaks the AWS SDK for JavaScript v3,
+ * which refuses to read either.
+ */
+@QuarkusTest
+class GlueOmitsNullFieldsIntegrationTest {
+
+    private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+    private static final String DATABASE = "nulls_" + UUID.randomUUID().toString().substring(0, 8);
+
+    @BeforeAll
+    static void configureRestAssured() {
+        RestAssuredJsonUtils.configureAwsContentTypes();
+    }
+
+    private static io.restassured.response.ValidatableResponse glue(String action, String body) {
+        return given().contentType(CONTENT_TYPE)
+                .header("X-Amz-Target", "AWSGlue." + action)
+                .body(body)
+        .when().post("/")
+        .then().statusCode(200);
+    }
+
+    @Test
+    void aNewTableHasOnlyAMillisecondCreateTime() {
+        glue("CreateDatabase", "{\"DatabaseInput\": {\"Name\": \"" + DATABASE + "\"}}");
+        glue("CreateTable", """
+                {
+                  "DatabaseName": "%s",
+                  "TableInput": {
+                    "Name": "orders",
+                    "StorageDescriptor": {"Columns": [{"Name": "id", "Type": "int"}], "Location": "s3://data/orders/"}
+                  }
+                }
+                """.formatted(DATABASE));
+
+        String tables = glue("GetTables", "{\"DatabaseName\": \"" + DATABASE + "\"}")
+                .body("TableList[0].Name", equalTo("orders"))
+                .body("TableList[0].CreateTime", notNullValue())
+                .body("TableList[0]", not(hasKey("UpdateTime")))
+                .body("TableList[0]", not(hasKey("LastAccessTime")))
+                .extract().asString();
+        // Milliseconds in their shortest form, as AWS sends them: no trailing zeros.
+        assertTrue(tables.matches("(?s).*\"CreateTime\":\\d+(\\.\\d{0,2}[1-9])?[,}].*"), tables);
+
+        glue("GetDatabase", "{\"Name\": \"" + DATABASE + "\"}")
+                .body("Database", not(hasKey("Description")))
+                .body("Database.CreateTime", notNullValue());
+    }
+}

src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java

@@ -0,0 +1,17 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class GlueTimestampSerializerTest {
+
+    @Test
+    void writesMillisecondsInTheirShortestForm() {
+        assertEquals("1792964720.273", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720, 273_456_789)));
+        assertEquals("1792964720.27", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720, 270_000_000)));
+        assertEquals("1792964720", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720)));
+    }
+}