fix(glue): leave unset fields out of catalog responses
Database, Table, Partition, UserDefinedFunction, Job, Crawler and LastCrawlInfo were serialized with every unset field as null. AWS omits them, and the AWS SDK for JavaScript v3 rejects a null timestamp outright, so GetTables failed for any table that had never been updated (UpdateTime and LastAccessTime: null).
This commit is contained in:
@@ -2,11 +2,13 @@ package io.github.hectorvent.floci.services.glue.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
|
||||
@RegisterForReflection
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class Crawler {
|
||||
@JsonProperty("Classifiers")
|
||||
private List<String> classifiers;
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
package io.github.hectorvent.floci.services.glue.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
|
||||
@RegisterForReflection
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class Database {
|
||||
@JsonProperty("Name")
|
||||
private String name;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package io.github.hectorvent.floci.services.glue.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||
@@ -9,6 +10,7 @@ import java.time.Instant;
|
||||
import java.util.Map;
|
||||
|
||||
@RegisterForReflection
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class Job {
|
||||
@JsonProperty("AllocatedCapacity")
|
||||
private Integer allocatedCapacity;
|
||||
|
||||
@@ -2,10 +2,12 @@ package io.github.hectorvent.floci.services.glue.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||
import java.time.Instant;
|
||||
|
||||
@RegisterForReflection
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class LastCrawlInfo {
|
||||
@JsonProperty("ErrorMessage")
|
||||
private String errorMessage;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package io.github.hectorvent.floci.services.glue.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||
import java.time.Instant;
|
||||
@@ -8,6 +9,7 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@RegisterForReflection
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class Partition {
|
||||
@JsonProperty("Values")
|
||||
private List<String> values;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package io.github.hectorvent.floci.services.glue.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||
import java.time.Instant;
|
||||
@@ -8,6 +9,7 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@RegisterForReflection
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class Table {
|
||||
@JsonProperty("Name")
|
||||
private String name;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package io.github.hectorvent.floci.services.glue.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||
|
||||
@@ -8,6 +9,7 @@ import java.time.Instant;
|
||||
import java.util.List;
|
||||
|
||||
@RegisterForReflection
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class UserDefinedFunction {
|
||||
@JsonProperty("FunctionName")
|
||||
private String functionName;
|
||||
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
package io.github.hectorvent.floci.services.glue;
|
||||
|
||||
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
|
||||
import io.quarkus.test.junit.QuarkusTest;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.UUID;
|
||||
|
||||
import static io.restassured.RestAssured.given;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.hasKey;
|
||||
import static org.hamcrest.Matchers.not;
|
||||
import static org.hamcrest.Matchers.notNullValue;
|
||||
|
||||
/**
|
||||
* AWS leaves unset fields out of Glue responses. A null timestamp instead breaks
|
||||
* the AWS SDK for JavaScript v3, which rejects anything but a number there.
|
||||
*/
|
||||
@QuarkusTest
|
||||
class GlueOmitsNullFieldsIntegrationTest {
|
||||
|
||||
private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
|
||||
private static final String DATABASE = "nulls_" + UUID.randomUUID().toString().substring(0, 8);
|
||||
|
||||
@BeforeAll
|
||||
static void configureRestAssured() {
|
||||
RestAssuredJsonUtils.configureAwsContentTypes();
|
||||
}
|
||||
|
||||
private static io.restassured.response.ValidatableResponse glue(String action, String body) {
|
||||
return given().contentType(CONTENT_TYPE)
|
||||
.header("X-Amz-Target", "AWSGlue." + action)
|
||||
.body(body)
|
||||
.when().post("/")
|
||||
.then().statusCode(200);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aNewTableHasNoUpdateOrLastAccessTime() {
|
||||
glue("CreateDatabase", "{\"DatabaseInput\": {\"Name\": \"" + DATABASE + "\"}}");
|
||||
glue("CreateTable", """
|
||||
{
|
||||
"DatabaseName": "%s",
|
||||
"TableInput": {
|
||||
"Name": "orders",
|
||||
"StorageDescriptor": {"Columns": [{"Name": "id", "Type": "int"}], "Location": "s3://data/orders/"}
|
||||
}
|
||||
}
|
||||
""".formatted(DATABASE));
|
||||
|
||||
glue("GetTables", "{\"DatabaseName\": \"" + DATABASE + "\"}")
|
||||
.body("TableList[0].Name", equalTo("orders"))
|
||||
.body("TableList[0].CreateTime", notNullValue())
|
||||
.body("TableList[0]", not(hasKey("UpdateTime")))
|
||||
.body("TableList[0]", not(hasKey("LastAccessTime")));
|
||||
glue("GetDatabase", "{\"Name\": \"" + DATABASE + "\"}")
|
||||
.body("Database", not(hasKey("Description")));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user