From dfbfa4d2a7c08409a5d9b42cd833893c5f3b1e66 Mon Sep 17 00:00:00 2001 From: Zakaria Date: Sun, 25 Oct 2026 17:41:57 -0400 Subject: [PATCH] fix(glue): leave unset fields out of catalog responses Database, Table, Partition, UserDefinedFunction, Job, Crawler and LastCrawlInfo were serialized with every unset field as null. AWS omits them, and the AWS SDK for JavaScript v3 rejects a null timestamp outright, so GetTables failed for any table that had never been updated (UpdateTime and LastAccessTime: null). --- .../floci/services/glue/model/Crawler.java | 2 + .../floci/services/glue/model/Database.java | 2 + .../floci/services/glue/model/Job.java | 2 + .../services/glue/model/LastCrawlInfo.java | 2 + .../floci/services/glue/model/Partition.java | 2 + .../floci/services/glue/model/Table.java | 2 + .../glue/model/UserDefinedFunction.java | 2 + .../GlueOmitsNullFieldsIntegrationTest.java | 60 +++++++++++++++++++ 8 files changed, 74 insertions(+) create mode 100644 src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java index 85de429..10693da 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Crawler.java @@ -2,11 +2,13 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.List; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class Crawler { @JsonProperty("Classifiers") private List classifiers; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java index 0eb7efb..15fb688 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Database.java @@ -1,12 +1,14 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; import java.util.Map; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class Database { @JsonProperty("Name") private String name; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java index 1353a4e..e0c3d9f 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Job.java @@ -1,6 +1,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import io.quarkus.runtime.annotations.RegisterForReflection; @@ -9,6 +10,7 @@ import java.time.Instant; import java.util.Map; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class Job { @JsonProperty("AllocatedCapacity") private Integer allocatedCapacity; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java index 7a18eea..c234f4a 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/LastCrawlInfo.java @@ -2,10 +2,12 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class LastCrawlInfo { @JsonProperty("ErrorMessage") private String errorMessage; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java index aa52d5c..532b298 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Partition.java @@ -1,6 +1,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @@ -8,6 +9,7 @@ import java.util.List; import java.util.Map; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class Partition { @JsonProperty("Values") private List values; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java index 46a6698..971b559 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/Table.java @@ -1,6 +1,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import io.quarkus.runtime.annotations.RegisterForReflection; import java.time.Instant; @@ -8,6 +9,7 @@ import java.util.List; import java.util.Map; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class Table { @JsonProperty("Name") private String name; diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java b/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java index 9fa3f0b..e2accdf 100644 --- a/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java +++ b/src/main/java/io/github/hectorvent/floci/services/glue/model/UserDefinedFunction.java @@ -1,6 +1,7 @@ package io.github.hectorvent.floci.services.glue.model; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import io.quarkus.runtime.annotations.RegisterForReflection; @@ -8,6 +9,7 @@ import java.time.Instant; import java.util.List; @RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) public class UserDefinedFunction { @JsonProperty("FunctionName") private String functionName; diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java new file mode 100644 index 0000000..9d9a3f1 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueOmitsNullFieldsIntegrationTest.java @@ -0,0 +1,60 @@ +package io.github.hectorvent.floci.services.glue; + +import io.github.hectorvent.floci.testing.RestAssuredJsonUtils; +import io.quarkus.test.junit.QuarkusTest; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; + +/** + * AWS leaves unset fields out of Glue responses. A null timestamp instead breaks + * the AWS SDK for JavaScript v3, which rejects anything but a number there. + */ +@QuarkusTest +class GlueOmitsNullFieldsIntegrationTest { + + private static final String CONTENT_TYPE = "application/x-amz-json-1.1"; + private static final String DATABASE = "nulls_" + UUID.randomUUID().toString().substring(0, 8); + + @BeforeAll + static void configureRestAssured() { + RestAssuredJsonUtils.configureAwsContentTypes(); + } + + private static io.restassured.response.ValidatableResponse glue(String action, String body) { + return given().contentType(CONTENT_TYPE) + .header("X-Amz-Target", "AWSGlue." + action) + .body(body) + .when().post("/") + .then().statusCode(200); + } + + @Test + void aNewTableHasNoUpdateOrLastAccessTime() { + glue("CreateDatabase", "{\"DatabaseInput\": {\"Name\": \"" + DATABASE + "\"}}"); + glue("CreateTable", """ + { + "DatabaseName": "%s", + "TableInput": { + "Name": "orders", + "StorageDescriptor": {"Columns": [{"Name": "id", "Type": "int"}], "Location": "s3://data/orders/"} + } + } + """.formatted(DATABASE)); + + glue("GetTables", "{\"DatabaseName\": \"" + DATABASE + "\"}") + .body("TableList[0].Name", equalTo("orders")) + .body("TableList[0].CreateTime", notNullValue()) + .body("TableList[0]", not(hasKey("UpdateTime"))) + .body("TableList[0]", not(hasKey("LastAccessTime"))); + glue("GetDatabase", "{\"Name\": \"" + DATABASE + "\"}") + .body("Database", not(hasKey("Description"))); + } +}