Merge pull request 'fix(glue): leave unset fields out of catalog responses' (#15) from fix/glue-omit-null-fields into main

Reviewed-on: #15

src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java

@@ -243,6 +243,9 @@ public class GlueService {
         }
         database.setName(databaseName);
         database.setCatalogId(regionResolver.getAccountId());
+        if (database.getCreateTime() == null) {
+            database.setCreateTime(Instant.now());
+        }
         databaseStore.put(databaseName, database);
         if (tags != null && !tags.isEmpty()) {
             resourceGroupsTaggingService.tagResources(List.of(databaseArn(region, databaseName)), tags, region);

src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java

@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model;
 import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 
 import java.time.Instant;
@@ -47,10 +48,12 @@ public class Connection {
 
     @JsonProperty("CreationTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant creationTime;
 
     @JsonProperty("LastUpdatedTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastUpdatedTime;
 
     @JsonProperty("LastUpdatedBy")
@@ -64,6 +67,7 @@ public class Connection {
 
     @JsonProperty("LastConnectionValidationTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastConnectionValidationTime;
 
     @JsonProperty("AuthenticationConfiguration")

src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java

@@ -1,12 +1,15 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 import java.time.Instant;
 import java.util.List;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class Crawler {
     @JsonProperty("Classifiers")
     private List<String> classifiers;
@@ -22,6 +25,7 @@ public class Crawler {
 
     @JsonProperty("CreationTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant creationTime;
 
     @JsonProperty("DatabaseName")
@@ -38,6 +42,7 @@ public class Crawler {
 
     @JsonProperty("LastUpdated")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastUpdated;
 
     @JsonProperty("LineageConfiguration")

src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java

@@ -1,12 +1,15 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 import java.time.Instant;
 import java.util.Map;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class Database {
     @JsonProperty("Name")
     private String name;
@@ -18,6 +21,7 @@ public class Database {
     private Map<String, String> parameters;
     @JsonProperty("CreateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant createTime;
     @JsonProperty("CatalogId")
     private String catalogId;

src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java

@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model;
 import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 
 import java.time.Instant;
@@ -22,10 +23,12 @@ public class GluePolicy {
 
     @JsonProperty("CreateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant createTime;
 
     @JsonProperty("UpdateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant updateTime;
 
     public String getPolicyInJson() { return policyInJson; }

src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java

@@ -0,0 +1,29 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+
+/**
+ * Writes a Glue timestamp as AWS does: epoch seconds to the millisecond, in the shortest
+ * form ({@code 1792964720.27}, not {@code 1792964720.270000000}). The AWS SDK for
+ * JavaScript v3 keeps any number whose text differs from its shortest form as a
+ * BigDecimal, and then refuses it as a timestamp.
+ */
+@RegisterForReflection
+public class GlueTimestampSerializer extends JsonSerializer<Instant> {
+
+    @Override
+    public void serialize(Instant value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
+        gen.writeNumber(epochSeconds(value));
+    }
+
+    static String epochSeconds(Instant value) {
+        return BigDecimal.valueOf(value.toEpochMilli(), 3).stripTrailingZeros().toPlainString();
+    }
+}

src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java

@@ -1,7 +1,9 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import com.fasterxml.jackson.databind.JsonNode;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 
@@ -9,6 +11,7 @@ import java.time.Instant;
 import java.util.Map;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class Job {
     @JsonProperty("AllocatedCapacity")
     private Integer allocatedCapacity;
@@ -24,6 +27,7 @@ public class Job {
 
     @JsonProperty("CreatedOn")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant createdOn;
 
     @JsonProperty("DefaultArguments")
@@ -49,6 +53,7 @@ public class Job {
 
     @JsonProperty("LastModifiedOn")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastModifiedOn;
 
     @JsonProperty("LogUri")

src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java

@@ -1,11 +1,14 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 import java.time.Instant;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class LastCrawlInfo {
     @JsonProperty("ErrorMessage")
     private String errorMessage;
@@ -21,6 +24,7 @@ public class LastCrawlInfo {
 
     @JsonProperty("StartTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant startTime;
 
     @JsonProperty("Status")

src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java

@@ -1,13 +1,16 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class Partition {
     @JsonProperty("Values")
     private List<String> values;
@@ -17,9 +20,11 @@ public class Partition {
     private String tableName;
     @JsonProperty("CreationTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant creationTime;
     @JsonProperty("LastAccessTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastAccessTime;
     @JsonProperty("StorageDescriptor")
     private StorageDescriptor storageDescriptor;

src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java

@@ -1,13 +1,16 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class Table {
     @JsonProperty("Name")
     private String name;
@@ -19,12 +22,15 @@ public class Table {
     private String owner;
     @JsonProperty("CreateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant createTime;
     @JsonProperty("UpdateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant updateTime;
     @JsonProperty("LastAccessTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant lastAccessTime;
     @JsonProperty("PartitionKeys")
     private List<Column> partitionKeys;

src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java

@@ -1,13 +1,16 @@
 package io.github.hectorvent.floci.services.glue.model;
 
 import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.quarkus.runtime.annotations.RegisterForReflection;
 
 import java.time.Instant;
 import java.util.List;
 
 @RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public class UserDefinedFunction {
     @JsonProperty("FunctionName")
     private String functionName;
@@ -23,6 +26,7 @@ public class UserDefinedFunction {
     private String ownerType;
     @JsonProperty("CreateTime")
     @JsonFormat(shape = JsonFormat.Shape.NUMBER)
+    @JsonSerialize(using = GlueTimestampSerializer.class)
     private Instant createTime;
     @JsonProperty("ResourceUris")
     private List<ResourceUri> resourceUris;

src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java

@@ -0,0 +1,67 @@
+package io.github.hectorvent.floci.services.glue;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * AWS leaves unset fields out of Glue responses and reports times in milliseconds.
+ * A null or nanosecond timestamp instead breaks the AWS SDK for JavaScript v3,
+ * which refuses to read either.
+ */
+@QuarkusTest
+class GlueOmitsNullFieldsIntegrationTest {
+
+    private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+    private static final String DATABASE = "nulls_" + UUID.randomUUID().toString().substring(0, 8);
+
+    @BeforeAll
+    static void configureRestAssured() {
+        RestAssuredJsonUtils.configureAwsContentTypes();
+    }
+
+    private static io.restassured.response.ValidatableResponse glue(String action, String body) {
+        return given().contentType(CONTENT_TYPE)
+                .header("X-Amz-Target", "AWSGlue." + action)
+                .body(body)
+        .when().post("/")
+        .then().statusCode(200);
+    }
+
+    @Test
+    void aNewTableHasOnlyAMillisecondCreateTime() {
+        glue("CreateDatabase", "{\"DatabaseInput\": {\"Name\": \"" + DATABASE + "\"}}");
+        glue("CreateTable", """
+                {
+                  "DatabaseName": "%s",
+                  "TableInput": {
+                    "Name": "orders",
+                    "StorageDescriptor": {"Columns": [{"Name": "id", "Type": "int"}], "Location": "s3://data/orders/"}
+                  }
+                }
+                """.formatted(DATABASE));
+
+        String tables = glue("GetTables", "{\"DatabaseName\": \"" + DATABASE + "\"}")
+                .body("TableList[0].Name", equalTo("orders"))
+                .body("TableList[0].CreateTime", notNullValue())
+                .body("TableList[0]", not(hasKey("UpdateTime")))
+                .body("TableList[0]", not(hasKey("LastAccessTime")))
+                .extract().asString();
+        // Milliseconds in their shortest form, as AWS sends them: no trailing zeros.
+        assertTrue(tables.matches("(?s).*\"CreateTime\":\\d+(\\.\\d{0,2}[1-9])?[,}].*"), tables);
+
+        glue("GetDatabase", "{\"Name\": \"" + DATABASE + "\"}")
+                .body("Database", not(hasKey("Description")))
+                .body("Database.CreateTime", notNullValue());
+    }
+}

src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java

@@ -0,0 +1,17 @@
+package io.github.hectorvent.floci.services.glue.model;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class GlueTimestampSerializerTest {
+
+    @Test
+    void writesMillisecondsInTheirShortestForm() {
+        assertEquals("1792964720.273", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720, 273_456_789)));
+        assertEquals("1792964720.27", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720, 270_000_000)));
+        assertEquals("1792964720", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720)));
+    }
+}