From 1b00488b7cb3ebe7df75da74297ee25229b1d4af Mon Sep 17 00:00:00 2001 From: Zakaria Date: Sun, 25 Oct 2026 17:48:12 -0400 Subject: [PATCH] fix(glue): write timestamps the AWS SDK for JavaScript can read Glue times went out as nanosecond epoch numbers (1792964720.708701378). The AWS SDK for JavaScript v3 keeps any number whose text is not its shortest form as a BigDecimal and then refuses it as a timestamp, so GetTables still failed. They are now written as AWS writes them: epoch seconds to the millisecond, in the shortest form. CreateDatabase also records the database's CreateTime. --- .../floci/services/glue/GlueService.java | 3 ++ .../floci/services/glue/model/Connection.java | 4 +++ .../floci/services/glue/model/Crawler.java | 3 ++ .../floci/services/glue/model/Database.java | 2 ++ .../floci/services/glue/model/GluePolicy.java | 3 ++ .../glue/model/GlueTimestampSerializer.java | 29 +++++++++++++++++++ .../floci/services/glue/model/Job.java | 3 ++ .../services/glue/model/LastCrawlInfo.java | 2 ++ .../floci/services/glue/model/Partition.java | 3 ++ .../floci/services/glue/model/Table.java | 4 +++ .../glue/model/UserDefinedFunction.java | 2 ++ .../GlueOmitsNullFieldsIntegrationTest.java | 19 ++++++++---- .../model/GlueTimestampSerializerTest.java | 17 +++++++++++ 13 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java create mode 100644 src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java index e060b74..395f798 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java @@ -243,6 +243,9 @@ public class GlueService { } database.setName(databaseName); database.setCatalogId(regionResolver.getAccountId()); + if (database.getCreateTime() == null) { + database.setCreateTime(Instant.now()); + } databaseStore.put(databaseName, database); if (tags != null && !tags.isEmpty()) { resourceGroupsTaggingService.tagResources(List.of(databaseArn(region, databaseName)), tags, region); diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java index 95105c8..145118a 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Connection.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @@ -47,10 +48,12 @@ public class Connection { @JsonProperty("CreationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant creationTime; @JsonProperty("LastUpdatedTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastUpdatedTime; @JsonProperty("LastUpdatedBy") @@ -64,6 +67,7 @@ public class Connection { @JsonProperty("LastConnectionValidationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastConnectionValidationTime; @JsonProperty("AuthenticationConfiguration") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java index 10693da..77f9dc7 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java @@ -1,6 +1,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import io.quarkus.runtime.annotations.RegisterForReflection; @@ -24,6 +25,7 @@ public class Crawler { @JsonProperty("CreationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant creationTime; @JsonProperty("DatabaseName") @@ -40,6 +42,7 @@ public class Crawler { @JsonProperty("LastUpdated") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastUpdated; @JsonProperty("LineageConfiguration") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java index 15fb688..c68c899 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.Map; @@ -20,6 +21,7 @@ public class Database { private Map parameters; @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("CatalogId") private String catalogId; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java index c4d2a4a..8b28089 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GluePolicy.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @@ -22,10 +23,12 @@ public class GluePolicy { @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("UpdateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant updateTime; public String getPolicyInJson() { return policyInJson; } diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java new file mode 100644 index 0000000..267e938 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializer.java @@ -0,0 +1,29 @@ +package io.github.hectorvent.floci.services.glue.model; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.quarkus.runtime.annotations.RegisterForReflection; + +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Instant; + +/** + * Writes a Glue timestamp as AWS does: epoch seconds to the millisecond, in the shortest + * form ({@code 1792964720.27}, not {@code 1792964720.270000000}). The AWS SDK for + * JavaScript v3 keeps any number whose text differs from its shortest form as a + * BigDecimal, and then refuses it as a timestamp. + */ +@RegisterForReflection +public class GlueTimestampSerializer extends JsonSerializer { + + @Override + public void serialize(Instant value, JsonGenerator gen, SerializerProvider serializers) throws IOException { + gen.writeNumber(epochSeconds(value)); + } + + static String epochSeconds(Instant value) { + return BigDecimal.valueOf(value.toEpochMilli(), 3).stripTrailingZeros().toPlainString(); + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java index e0c3d9f..f012acf 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.JsonNode; import io.quarkus.runtime.annotations.RegisterForReflection; @@ -26,6 +27,7 @@ public class Job { @JsonProperty("CreatedOn") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createdOn; @JsonProperty("DefaultArguments") @@ -51,6 +53,7 @@ public class Job { @JsonProperty("LastModifiedOn") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastModifiedOn; @JsonProperty("LogUri") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java index c234f4a..b78ac3a 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java @@ -1,6 +1,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import io.quarkus.runtime.annotations.RegisterForReflection; @@ -23,6 +24,7 @@ public class LastCrawlInfo { @JsonProperty("StartTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant startTime; @JsonProperty("Status") diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java index 532b298..7a4489d 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.List; @@ -19,9 +20,11 @@ public class Partition { private String tableName; @JsonProperty("CreationTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant creationTime; @JsonProperty("LastAccessTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastAccessTime; @JsonProperty("StorageDescriptor") private StorageDescriptor storageDescriptor; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java index 971b559..013e512 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.List; @@ -21,12 +22,15 @@ public class Table { private String owner; @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("UpdateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant updateTime; @JsonProperty("LastAccessTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant lastAccessTime; @JsonProperty("PartitionKeys") private List partitionKeys; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java index e2accdf..379a1c9 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java @@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @@ -25,6 +26,7 @@ public class UserDefinedFunction { private String ownerType; @JsonProperty("CreateTime") @JsonFormat(shape = JsonFormat.Shape.NUMBER) + @JsonSerialize(using = GlueTimestampSerializer.class) private Instant createTime; @JsonProperty("ResourceUris") private List resourceUris; diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java index 9d9a3f1..1e55601 100644 --- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java @@ -12,10 +12,12 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +import static org.junit.jupiter.api.Assertions.assertTrue; /** - * AWS leaves unset fields out of Glue responses. A null timestamp instead breaks - * the AWS SDK for JavaScript v3, which rejects anything but a number there. + * AWS leaves unset fields out of Glue responses and reports times in milliseconds. + * A null or nanosecond timestamp instead breaks the AWS SDK for JavaScript v3, + * which refuses to read either. */ @QuarkusTest class GlueOmitsNullFieldsIntegrationTest { @@ -37,7 +39,7 @@ class GlueOmitsNullFieldsIntegrationTest { } @Test - void aNewTableHasNoUpdateOrLastAccessTime() { + void aNewTableHasOnlyAMillisecondCreateTime() { glue("CreateDatabase", "{\"DatabaseInput\": {\"Name\": \"" + DATABASE + "\"}}"); glue("CreateTable", """ { @@ -49,12 +51,17 @@ class GlueOmitsNullFieldsIntegrationTest { } """.formatted(DATABASE)); - glue("GetTables", "{\"DatabaseName\": \"" + DATABASE + "\"}") + String tables = glue("GetTables", "{\"DatabaseName\": \"" + DATABASE + "\"}") .body("TableList[0].Name", equalTo("orders")) .body("TableList[0].CreateTime", notNullValue()) .body("TableList[0]", not(hasKey("UpdateTime"))) - .body("TableList[0]", not(hasKey("LastAccessTime"))); + .body("TableList[0]", not(hasKey("LastAccessTime"))) + .extract().asString(); + // Milliseconds in their shortest form, as AWS sends them: no trailing zeros. + assertTrue(tables.matches("(?s).*\"CreateTime\":\\d+(\\.\\d{0,2}[1-9])?[,}].*"), tables); + glue("GetDatabase", "{\"Name\": \"" + DATABASE + "\"}") - .body("Database", not(hasKey("Description"))); + .body("Database", not(hasKey("Description"))) + .body("Database.CreateTime", notNullValue()); } } diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java new file mode 100644 index 0000000..69306d6 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/glue/model/GlueTimestampSerializerTest.java @@ -0,0 +1,17 @@ +package io.github.hectorvent.floci.services.glue.model; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class GlueTimestampSerializerTest { + + @Test + void writesMillisecondsInTheirShortestForm() { + assertEquals("1792964720.273", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720, 273_456_789))); + assertEquals("1792964720.27", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720, 270_000_000))); + assertEquals("1792964720", GlueTimestampSerializer.epochSeconds(Instant.ofEpochSecond(1792964720))); + } +}