fix(glue): leave unset fields out of catalog responses
Database, Table, Partition, UserDefinedFunction, Job, Crawler and LastCrawlInfo were serialized with every unset field as null. AWS omits them, and the AWS SDK for JavaScript v3 rejects a null timestamp outright, so GetTables failed for any table that had never been updated (UpdateTime and LastAccessTime: null).
This commit is contained in:
@@ -2,11 +2,13 @@ package io.github.hectorvent.floci.services.glue.model;
|
|||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@RegisterForReflection
|
@RegisterForReflection
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public class Crawler {
|
public class Crawler {
|
||||||
@JsonProperty("Classifiers")
|
@JsonProperty("Classifiers")
|
||||||
private List<String> classifiers;
|
private List<String> classifiers;
|
||||||
|
|||||||
@@ -1,12 +1,14 @@
|
|||||||
package io.github.hectorvent.floci.services.glue.model;
|
package io.github.hectorvent.floci.services.glue.model;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@RegisterForReflection
|
@RegisterForReflection
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public class Database {
|
public class Database {
|
||||||
@JsonProperty("Name")
|
@JsonProperty("Name")
|
||||||
private String name;
|
private String name;
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package io.github.hectorvent.floci.services.glue.model;
|
package io.github.hectorvent.floci.services.glue.model;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.fasterxml.jackson.databind.JsonNode;
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||||
@@ -9,6 +10,7 @@ import java.time.Instant;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@RegisterForReflection
|
@RegisterForReflection
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public class Job {
|
public class Job {
|
||||||
@JsonProperty("AllocatedCapacity")
|
@JsonProperty("AllocatedCapacity")
|
||||||
private Integer allocatedCapacity;
|
private Integer allocatedCapacity;
|
||||||
|
|||||||
@@ -2,10 +2,12 @@ package io.github.hectorvent.floci.services.glue.model;
|
|||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
|
|
||||||
@RegisterForReflection
|
@RegisterForReflection
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public class LastCrawlInfo {
|
public class LastCrawlInfo {
|
||||||
@JsonProperty("ErrorMessage")
|
@JsonProperty("ErrorMessage")
|
||||||
private String errorMessage;
|
private String errorMessage;
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package io.github.hectorvent.floci.services.glue.model;
|
package io.github.hectorvent.floci.services.glue.model;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
@@ -8,6 +9,7 @@ import java.util.List;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@RegisterForReflection
|
@RegisterForReflection
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public class Partition {
|
public class Partition {
|
||||||
@JsonProperty("Values")
|
@JsonProperty("Values")
|
||||||
private List<String> values;
|
private List<String> values;
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package io.github.hectorvent.floci.services.glue.model;
|
package io.github.hectorvent.floci.services.glue.model;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
@@ -8,6 +9,7 @@ import java.util.List;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@RegisterForReflection
|
@RegisterForReflection
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public class Table {
|
public class Table {
|
||||||
@JsonProperty("Name")
|
@JsonProperty("Name")
|
||||||
private String name;
|
private String name;
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package io.github.hectorvent.floci.services.glue.model;
|
package io.github.hectorvent.floci.services.glue.model;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import io.quarkus.runtime.annotations.RegisterForReflection;
|
import io.quarkus.runtime.annotations.RegisterForReflection;
|
||||||
|
|
||||||
@@ -8,6 +9,7 @@ import java.time.Instant;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@RegisterForReflection
|
@RegisterForReflection
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public class UserDefinedFunction {
|
public class UserDefinedFunction {
|
||||||
@JsonProperty("FunctionName")
|
@JsonProperty("FunctionName")
|
||||||
private String functionName;
|
private String functionName;
|
||||||
|
|||||||
+60
@@ -0,0 +1,60 @@
|
|||||||
|
package io.github.hectorvent.floci.services.glue;
|
||||||
|
|
||||||
|
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
|
||||||
|
import io.quarkus.test.junit.QuarkusTest;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
import static io.restassured.RestAssured.given;
|
||||||
|
import static org.hamcrest.Matchers.equalTo;
|
||||||
|
import static org.hamcrest.Matchers.hasKey;
|
||||||
|
import static org.hamcrest.Matchers.not;
|
||||||
|
import static org.hamcrest.Matchers.notNullValue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AWS leaves unset fields out of Glue responses. A null timestamp instead breaks
|
||||||
|
* the AWS SDK for JavaScript v3, which rejects anything but a number there.
|
||||||
|
*/
|
||||||
|
@QuarkusTest
|
||||||
|
class GlueOmitsNullFieldsIntegrationTest {
|
||||||
|
|
||||||
|
private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
|
||||||
|
private static final String DATABASE = "nulls_" + UUID.randomUUID().toString().substring(0, 8);
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void configureRestAssured() {
|
||||||
|
RestAssuredJsonUtils.configureAwsContentTypes();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static io.restassured.response.ValidatableResponse glue(String action, String body) {
|
||||||
|
return given().contentType(CONTENT_TYPE)
|
||||||
|
.header("X-Amz-Target", "AWSGlue." + action)
|
||||||
|
.body(body)
|
||||||
|
.when().post("/")
|
||||||
|
.then().statusCode(200);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void aNewTableHasNoUpdateOrLastAccessTime() {
|
||||||
|
glue("CreateDatabase", "{\"DatabaseInput\": {\"Name\": \"" + DATABASE + "\"}}");
|
||||||
|
glue("CreateTable", """
|
||||||
|
{
|
||||||
|
"DatabaseName": "%s",
|
||||||
|
"TableInput": {
|
||||||
|
"Name": "orders",
|
||||||
|
"StorageDescriptor": {"Columns": [{"Name": "id", "Type": "int"}], "Location": "s3://data/orders/"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
""".formatted(DATABASE));
|
||||||
|
|
||||||
|
glue("GetTables", "{\"DatabaseName\": \"" + DATABASE + "\"}")
|
||||||
|
.body("TableList[0].Name", equalTo("orders"))
|
||||||
|
.body("TableList[0].CreateTime", notNullValue())
|
||||||
|
.body("TableList[0]", not(hasKey("UpdateTime")))
|
||||||
|
.body("TableList[0]", not(hasKey("LastAccessTime")));
|
||||||
|
glue("GetDatabase", "{\"Name\": \"" + DATABASE + "\"}")
|
||||||
|
.body("Database", not(hasKey("Description")));
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user