diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java index e060b74..395f798 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java @@ -243,6 +243,9 @@ public class GlueService { } database.setName(databaseName); database.setCatalogId(regionResolver.getAccountId()); + if (database.getCreateTime() == null) { + database.setCreateTime(Instant.now()); + } databaseStore.put(databaseName, database); if (tags != null && !tags.isEmpty()) { resourceGroupsTaggingService.tagResources(List.of(databaseArn(region, databaseName)), tags, region); diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java index 95105c8..145118a 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @@ -47,10 +48,12 @@ public class Connection { @JsonProperty("CreationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant creationTime; @JsonProperty("LastUpdatedTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastUpdatedTime; @JsonProperty("LastUpdatedBy") @@ -64,6 +67,7 @@ public class Connection { @JsonProperty("LastConnectionValidationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastConnectionValidationTime; @JsonProperty("AuthenticationConfiguration") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java index 85de429..77f9dc7 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java @@ -1,12 +1,15 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.List; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class Crawler { @JsonProperty("Classifiers") private List classifiers; @@ -22,6 +25,7 @@ public class Crawler { @JsonProperty("CreationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant creationTime; @JsonProperty("DatabaseName") @@ -38,6 +42,7 @@ public class Crawler { @JsonProperty("LastUpdated") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastUpdated; @JsonProperty("LineageConfiguration") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java index 0eb7efb..c68c899 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java @@ -1,12 +1,15 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.Map; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class Database { @JsonProperty("Name") private String name; @@ -18,6 +21,7 @@ public class Database { private Map parameters; @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("CatalogId") private String catalogId; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java index c4d2a4a..8b28089 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @@ -22,10 +23,12 @@ public class GluePolicy { @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("UpdateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant updateTime; public String getPolicyInJson() { return policyInJson; } diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java new file mode 100644 index 0000000..267e938 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java @@ -0,0 +1,29 @@ +package io.github.hectorvent.floci.services.glue.model; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.quarkus.runtime.annotations.RegisterForReflection; + +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Instant; + +/** + * Writes a Glue timestamp as AWS does: epoch seconds to the millisecond, in the shortest + * form ({@code 1792964720.27}, not {@code 1792964720.270000000}). The AWS SDK for + * JavaScript v3 keeps any number whose text differs from its shortest form as a + * BigDecimal, and then refuses it as a timestamp. + */ +@RegisterForReflection +public class GlueTimestampSerializer extends JsonSerializer { + + @Override + public void serialize(Instant value, JsonGenerator gen, SerializerProvider serializers) throws IOException { + gen.writeNumber(epochSeconds(value)); + } + + static String epochSeconds(Instant value) { + return BigDecimal.valueOf(value.toEpochMilli(), 3).stripTrailingZeros().toPlainString(); + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java index 1353a4e..f012acf 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java @@ -1,7 +1,9 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.JsonNode; import io.quarkus.runtime.annotations.RegisterForReflection; @@ -9,6 +11,7 @@ import java.time.Instant; import java.util.Map; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class Job { @JsonProperty("AllocatedCapacity") private Integer allocatedCapacity; @@ -24,6 +27,7 @@ public class Job { @JsonProperty("CreatedOn") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createdOn; @JsonProperty("DefaultArguments") @@ -49,6 +53,7 @@ public class Job { @JsonProperty("LastModifiedOn") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastModifiedOn; @JsonProperty("LogUri") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java index 7a18eea..b78ac3a 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java @@ -1,11 +1,14 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class LastCrawlInfo { @JsonProperty("ErrorMessage") private String errorMessage; @@ -21,6 +24,7 @@ public class LastCrawlInfo { @JsonProperty("StartTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant startTime; @JsonProperty("Status") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java index aa52d5c..7a4489d 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java @@ -1,13 +1,16 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.List; import java.util.Map; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class Partition { @JsonProperty("Values") private List values; @@ -17,9 +20,11 @@ public class Partition { private String tableName; @JsonProperty("CreationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant creationTime; @JsonProperty("LastAccessTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastAccessTime; @JsonProperty("StorageDescriptor") private StorageDescriptor storageDescriptor; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java index 46a6698..013e512 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java @@ -1,13 +1,16 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.List; import java.util.Map; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class Table { @JsonProperty("Name") private String name; @@ -19,12 +22,15 @@ public class Table { private String owner; @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("UpdateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant updateTime; @JsonProperty("LastAccessTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastAccessTime; @JsonProperty("PartitionKeys") private List partitionKeys; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java index 9fa3f0b..379a1c9 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java @@ -1,13 +1,16 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.List; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class UserDefinedFunction { @JsonProperty("FunctionName") private String functionName; @@ -23,6 +26,7 @@ public class UserDefinedFunction { private String ownerType; @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("ResourceUris") private List resourceUris; diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java new file mode 100644 index 0000000..1e55601 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java @@ -0,0 +1,67 @@ +package io.github.hectorvent.floci.services.glue; + +import io.github.hectorvent.floci.testing.RestAssuredJsonUtils; +import io.quarkus.test.junit.QuarkusTest; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * AWS leaves unset fields out of Glue responses and reports times in milliseconds. + * A null or nanosecond timestamp instead breaks the AWS SDK for JavaScript v3, + * which refuses to read either. + */ +@QuarkusTest +class GlueOmitsNullFieldsIntegrationTest { + + private static final String CONTENT_TYPE = "application/x-amz-json-1.1"; + private static final String DATABASE = "nulls_" + UUID.randomUUID().toString().substring(0, 8); + + @BeforeAll + static void configureRestAssured() { + RestAssuredJsonUtils.configureAwsContentTypes(); + } + + private static io.restassured.response.ValidatableResponse glue(String action, String body) { + return given().contentType(CONTENT_TYPE) + .header("X-Amz-Target", "AWSGlue." + action) + .body(body) + .when().post("/") + .then().statusCode(200); + } + + @Test + void aNewTableHasOnlyAMillisecondCreateTime() { + glue("CreateDatabase", "{\"DatabaseInput\": {\"Name\": \"" + DATABASE + "\"}}"); + glue("CreateTable", """ + { + "DatabaseName": "%s", + "TableInput": { + "Name": "orders", + "StorageDescriptor": {"Columns": [{"Name": "id", "Type": "int"}], "Location": "s3://data/orders/"} + } + } + """.formatted(DATABASE)); + + String tables = glue("GetTables", "{\"DatabaseName\": \"" + DATABASE + "\"}") + .body("TableList[0].Name", equalTo("orders")) + .body("TableList[0].CreateTime", notNullValue()) + .body("TableList[0]", not(hasKey("UpdateTime"))) + .body("TableList[0]", not(hasKey("LastAccessTime"))) + .extract().asString(); + // Milliseconds in their shortest form, as AWS sends them: no trailing zeros. + assertTrue(tables.matches("(?s).*\"CreateTime\":\\d+(\\.\\d{0,2}[1-9])?[,}].*"), tables); + + glue("GetDatabase", "{\"Name\": \"" + DATABASE + "\"}") + .body("Database", not(hasKey("Description"))) + .body("Database.CreateTime", notNullValue()); + } +} diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java new file mode 100644 index 0000000..69306d6 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java @@ -0,0 +1,17 @@ +package io.github.hectorvent.floci.services.glue.model; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class GlueTimestampSerializerTest { + + @Test + void writesMillisecondsInTheirShortestForm() { + assertEquals("1792964720.273", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720, 273_456_789))); + assertEquals("1792964720.27", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720, 270_000_000))); + assertEquals("1792964720", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720))); + } +}