diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java index e060b74..395f798 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java @@ -243,6 +243,9 @@ public class GlueService { } database.setName(databaseName); database.setCatalogId(regionResolver.getAccountId()); + if (database.getCreateTime() == null) { + database.setCreateTime(Instant.now()); + } databaseStore.put(databaseName, database); if (tags != null && !tags.isEmpty()) { resourceGroupsTaggingService.tagResources(List.of(databaseArn(region, databaseName)), tags, region); diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java index 95105c8..145118a 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @@ -47,10 +48,12 @@ public class Connection { @JsonProperty("CreationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant creationTime; @JsonProperty("LastUpdatedTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastUpdatedTime; @JsonProperty("LastUpdatedBy") @@ -64,6 +67,7 @@ public class Connection { @JsonProperty("LastConnectionValidationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastConnectionValidationTime; @JsonProperty("AuthenticationConfiguration") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java index 10693da..77f9dc7 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java @@ -1,6 +1,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import io.quarkus.runtime.annotations.RegisterForReflection; @@ -24,6 +25,7 @@ public class Crawler { @JsonProperty("CreationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant creationTime; @JsonProperty("DatabaseName") @@ -40,6 +42,7 @@ public class Crawler { @JsonProperty("LastUpdated") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastUpdated; @JsonProperty("LineageConfiguration") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java index 15fb688..c68c899 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.Map; @@ -20,6 +21,7 @@ public class Database { private Map parameters; @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("CatalogId") private String catalogId; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java index c4d2a4a..8b28089 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @@ -22,10 +23,12 @@ public class GluePolicy { @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("UpdateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant updateTime; public String getPolicyInJson() { return policyInJson; } diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java new file mode 100644 index 0000000..267e938 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java @@ -0,0 +1,29 @@ +package io.github.hectorvent.floci.services.glue.model; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.quarkus.runtime.annotations.RegisterForReflection; + +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Instant; + +/** + * Writes a Glue timestamp as AWS does: epoch seconds to the millisecond, in the shortest + * form ({@code 1792964720.27}, not {@code 1792964720.270000000}). The AWS SDK for + * JavaScript v3 keeps any number whose text differs from its shortest form as a + * BigDecimal, and then refuses it as a timestamp. + */ +@RegisterForReflection +public class GlueTimestampSerializer extends JsonSerializer { + + @Override + public void serialize(Instant value, JsonGenerator gen, SerializerProvider serializers) throws IOException { + gen.writeNumber(epochSeconds(value)); + } + + static String epochSeconds(Instant value) { + return BigDecimal.valueOf(value.toEpochMilli(), 3).stripTrailingZeros().toPlainString(); + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java index e0c3d9f..f012acf 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.JsonNode; import io.quarkus.runtime.annotations.RegisterForReflection; @@ -26,6 +27,7 @@ public class Job { @JsonProperty("CreatedOn") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createdOn; @JsonProperty("DefaultArguments") @@ -51,6 +53,7 @@ public class Job { @JsonProperty("LastModifiedOn") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastModifiedOn; @JsonProperty("LogUri") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java index c234f4a..b78ac3a 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java @@ -1,6 +1,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import io.quarkus.runtime.annotations.RegisterForReflection; @@ -23,6 +24,7 @@ public class LastCrawlInfo { @JsonProperty("StartTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant startTime; @JsonProperty("Status") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java index 532b298..7a4489d 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.List; @@ -19,9 +20,11 @@ public class Partition { private String tableName; @JsonProperty("CreationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant creationTime; @JsonProperty("LastAccessTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastAccessTime; @JsonProperty("StorageDescriptor") private StorageDescriptor storageDescriptor; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java index 971b559..013e512 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.List; @@ -21,12 +22,15 @@ public class Table { private String owner; @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("UpdateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant updateTime; @JsonProperty("LastAccessTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastAccessTime; @JsonProperty("PartitionKeys") private List partitionKeys; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java index e2accdf..379a1c9 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @@ -25,6 +26,7 @@ public class UserDefinedFunction { private String ownerType; @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("ResourceUris") private List resourceUris; diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java index 9d9a3f1..1e55601 100644 --- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java @@ -12,10 +12,12 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +import static org.junit.jupiter.api.Assertions.assertTrue; /** - * AWS leaves unset fields out of Glue responses. A null timestamp instead breaks - * the AWS SDK for JavaScript v3, which rejects anything but a number there. + * AWS leaves unset fields out of Glue responses and reports times in milliseconds. + * A null or nanosecond timestamp instead breaks the AWS SDK for JavaScript v3, + * which refuses to read either. */ @QuarkusTest class GlueOmitsNullFieldsIntegrationTest { @@ -37,7 +39,7 @@ class GlueOmitsNullFieldsIntegrationTest { } @Test - void aNewTableHasNoUpdateOrLastAccessTime() { + void aNewTableHasOnlyAMillisecondCreateTime() { glue("CreateDatabase", "{\"DatabaseInput\": {\"Name\": \"" + DATABASE + "\"}}"); glue("CreateTable", """ { @@ -49,12 +51,17 @@ class GlueOmitsNullFieldsIntegrationTest { } """.formatted(DATABASE)); - glue("GetTables", "{\"DatabaseName\": \"" + DATABASE + "\"}") + String tables = glue("GetTables", "{\"DatabaseName\": \"" + DATABASE + "\"}") .body("TableList[0].Name", equalTo("orders")) .body("TableList[0].CreateTime", notNullValue()) .body("TableList[0]", not(hasKey("UpdateTime"))) - .body("TableList[0]", not(hasKey("LastAccessTime"))); + .body("TableList[0]", not(hasKey("LastAccessTime"))) + .extract().asString(); + // Milliseconds in their shortest form, as AWS sends them: no trailing zeros. + assertTrue(tables.matches("(?s).*\"CreateTime\":\\d+(\\.\\d{0,2}[1-9])?[,}].*"), tables); + glue("GetDatabase", "{\"Name\": \"" + DATABASE + "\"}") - .body("Database", not(hasKey("Description"))); + .body("Database", not(hasKey("Description"))) + .body("Database.CreateTime", notNullValue()); } } diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java new file mode 100644 index 0000000..69306d6 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java @@ -0,0 +1,17 @@ +package io.github.hectorvent.floci.services.glue.model; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class GlueTimestampSerializerTest { + + @Test + void writesMillisecondsInTheirShortestForm() { + assertEquals("1792964720.273", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720, 273_456_789))); + assertEquals("1792964720.27", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720, 270_000_000))); + assertEquals("1792964720", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720))); + } +}