fix(cloudwatch): store datums sent in PutMetricData EntityMetricData (#6070)
The CloudWatch Agent sends PutMetricData with an empty top-level MetricData and every datum nested under EntityMetricData[].MetricData. Both the JSON 1.0 handler (also reached by Smithy RPC v2 CBOR) and the Query handler read only the top-level list, so the agent's metrics were silently dropped. Both handlers now append each EntityMetricData entry's datums to the top-level ones and store them under the request Namespace, with the same parsing as top-level datums. Entity and StrictEntityValidation are accepted and ignored. The per-request INFO logs of the raw request and parsed datums are lowered to a DEBUG count. Fixes #6066 (cherry picked from commit 03f57303969561b049c3ebba38f1fd9d2b68df4c)
This commit is contained in:
Mohith
+156
@@ -0,0 +1,156 @@
|
||||
package io.github.hectorvent.floci.services.cloudwatch.metrics;
|
||||
|
||||
import io.quarkus.test.junit.QuarkusTest;
|
||||
import io.restassured.RestAssured;
|
||||
import io.restassured.config.EncoderConfig;
|
||||
import io.restassured.http.ContentType;
|
||||
import io.restassured.parsing.Parser;
|
||||
import io.restassured.specification.RequestSpecification;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
import static io.restassured.RestAssured.config;
|
||||
import static io.restassured.RestAssured.given;
|
||||
import static org.hamcrest.Matchers.containsInAnyOrder;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.hasItem;
|
||||
|
||||
/**
|
||||
* PutMetricData datums nested under EntityMetricData, as the CloudWatch Agent sends them,
|
||||
* must be stored under the request namespace on every protocol path. floci-io/floci#6066.
|
||||
*/
|
||||
@QuarkusTest
|
||||
class CloudWatchMetricsEntityMetricDataIntegrationTest {
|
||||
|
||||
private static final String CW_AUTH =
|
||||
"AWS4-HMAC-SHA256 Credential=test/20261020/us-east-1/monitoring/aws4_request";
|
||||
private static final String JSON_1_0 = "application/x-amz-json-1.0";
|
||||
|
||||
@BeforeAll
|
||||
static void registerJsonParser() {
|
||||
RestAssured.registerParser(JSON_1_0, Parser.JSON);
|
||||
}
|
||||
|
||||
private static RequestSpecification json(String operation) {
|
||||
return given()
|
||||
.header("Authorization", CW_AUTH)
|
||||
.contentType(JSON_1_0)
|
||||
.config(config().encoderConfig(EncoderConfig.encoderConfig()
|
||||
.encodeContentTypeAs(JSON_1_0, ContentType.TEXT)))
|
||||
.header("X-Amz-Target", "GraniteServiceVersion20100801." + operation);
|
||||
}
|
||||
|
||||
private static RequestSpecification query(String action) {
|
||||
return given()
|
||||
.header("Authorization", CW_AUTH)
|
||||
.contentType("application/x-www-form-urlencoded")
|
||||
.formParam("Action", action)
|
||||
.formParam("Version", "2010-08-01");
|
||||
}
|
||||
|
||||
@Test
|
||||
void jsonEntityMetricDataOnlyIsListedAndAggregated() {
|
||||
long now = Instant.now().getEpochSecond();
|
||||
json("PutMetricData")
|
||||
.body("""
|
||||
{"Namespace":"CWAgent/Json","MetricData":[],
|
||||
"EntityMetricData":[{
|
||||
"Entity":{"KeyAttributes":{"Type":"AWS::Resource",
|
||||
"ResourceType":"AWS::EC2::Instance","Identifier":"i-0123456789abcdef0"},
|
||||
"Attributes":{}},
|
||||
"MetricData":[
|
||||
{"MetricName":"mem_used_percent","Value":6.5,"Unit":"Percent",
|
||||
"StorageResolution":60,"Timestamp":%d,
|
||||
"Dimensions":[{"Name":"host","Value":"h1"}]},
|
||||
{"MetricName":"disk_used_percent","Value":0.75,"Unit":"Percent",
|
||||
"StorageResolution":60,"Timestamp":%d,
|
||||
"Dimensions":[{"Name":"host","Value":"h1"}]}]}],
|
||||
"StrictEntityValidation":false}
|
||||
""".formatted(now, now))
|
||||
.when().post("/")
|
||||
.then().statusCode(200);
|
||||
|
||||
json("ListMetrics")
|
||||
.body("{\"Namespace\":\"CWAgent/Json\"}")
|
||||
.when().post("/")
|
||||
.then()
|
||||
.statusCode(200)
|
||||
.body("Metrics.MetricName", containsInAnyOrder("mem_used_percent", "disk_used_percent"));
|
||||
|
||||
json("GetMetricStatistics")
|
||||
.body("""
|
||||
{"Namespace":"CWAgent/Json","MetricName":"mem_used_percent",
|
||||
"Dimensions":[{"Name":"host","Value":"h1"}],
|
||||
"StartTime":%d,"EndTime":%d,"Period":60,"Statistics":["Sum"]}
|
||||
""".formatted(now - 120, now + 120))
|
||||
.when().post("/")
|
||||
.then()
|
||||
.statusCode(200)
|
||||
.body("Datapoints.Sum", hasItem(6.5f));
|
||||
}
|
||||
|
||||
@Test
|
||||
void jsonTopLevelAndEntityMetricDataAreBothStored() {
|
||||
json("PutMetricData")
|
||||
.body("""
|
||||
{"Namespace":"CWAgent/JsonMixed",
|
||||
"MetricData":[{"MetricName":"TopLevel","Value":1}],
|
||||
"EntityMetricData":[{"Entity":{"KeyAttributes":{"Type":"Service",
|
||||
"Name":"checkout","Environment":"prod"}},
|
||||
"MetricData":[{"MetricName":"FromEntity","Value":2}]}]}
|
||||
""")
|
||||
.when().post("/")
|
||||
.then().statusCode(200);
|
||||
|
||||
json("ListMetrics")
|
||||
.body("{\"Namespace\":\"CWAgent/JsonMixed\"}")
|
||||
.when().post("/")
|
||||
.then()
|
||||
.statusCode(200)
|
||||
.body("Metrics.MetricName", containsInAnyOrder("TopLevel", "FromEntity"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryTopLevelAndEntityMetricDataAreListedAndAggregated() {
|
||||
Instant now = Instant.now();
|
||||
query("PutMetricData")
|
||||
.formParam("Namespace", "CWAgent/Query")
|
||||
.formParam("MetricData.member.1.MetricName", "TopLevel")
|
||||
.formParam("MetricData.member.1.Value", "1")
|
||||
.formParam("EntityMetricData.member.1.Entity.KeyAttributes.entry.1.key", "Type")
|
||||
.formParam("EntityMetricData.member.1.Entity.KeyAttributes.entry.1.value", "AWS::Resource")
|
||||
.formParam("EntityMetricData.member.1.MetricData.member.1.MetricName", "mem_used_percent")
|
||||
.formParam("EntityMetricData.member.1.MetricData.member.1.Value", "6.5")
|
||||
.formParam("EntityMetricData.member.1.MetricData.member.1.Timestamp", now.toString())
|
||||
.formParam("EntityMetricData.member.1.MetricData.member.1.Dimensions.member.1.Name", "host")
|
||||
.formParam("EntityMetricData.member.1.MetricData.member.1.Dimensions.member.1.Value", "h1")
|
||||
.formParam("StrictEntityValidation", "false")
|
||||
.when().post("/")
|
||||
.then().statusCode(200);
|
||||
|
||||
query("ListMetrics")
|
||||
.formParam("Namespace", "CWAgent/Query")
|
||||
.when().post("/")
|
||||
.then()
|
||||
.statusCode(200)
|
||||
.body("ListMetricsResponse.ListMetricsResult.Metrics.member.MetricName",
|
||||
containsInAnyOrder("TopLevel", "mem_used_percent"));
|
||||
|
||||
query("GetMetricStatistics")
|
||||
.formParam("Namespace", "CWAgent/Query")
|
||||
.formParam("MetricName", "mem_used_percent")
|
||||
.formParam("Dimensions.member.1.Name", "host")
|
||||
.formParam("Dimensions.member.1.Value", "h1")
|
||||
.formParam("StartTime", now.minusSeconds(120).toString())
|
||||
.formParam("EndTime", now.plusSeconds(120).toString())
|
||||
.formParam("Period", "60")
|
||||
.formParam("Statistics.member.1", "Sum")
|
||||
.when().post("/")
|
||||
.then()
|
||||
.statusCode(200)
|
||||
.body("GetMetricStatisticsResponse.GetMetricStatisticsResult.Datapoints.member.Sum",
|
||||
equalTo("6.5"));
|
||||
}
|
||||
}
|
||||
+59
@@ -17,6 +17,12 @@ import java.math.BigDecimal;
|
||||
import java.time.Instant;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Handler-level tests for CloudWatchMetricsJsonHandler.
|
||||
@@ -146,6 +152,59 @@ class CloudWatchMetricsJsonHandlerTest {
|
||||
"metric stored with 24h-ago timestamp must not appear in a 20-second window around now");
|
||||
}
|
||||
|
||||
// The CloudWatch Agent sends an empty top-level MetricData and nests every datum under
|
||||
// EntityMetricData, floci-io/floci#6066.
|
||||
@Test
|
||||
void putMetricData_entityMetricDataOnly_storesNestedDatumsUnderTheRequestNamespace() {
|
||||
ObjectNode req = MAPPER.createObjectNode();
|
||||
req.put("Namespace", "CWAgent");
|
||||
req.putArray("MetricData");
|
||||
ObjectNode entityMetricData = req.putArray("EntityMetricData").addObject();
|
||||
ObjectNode entity = entityMetricData.putObject("Entity");
|
||||
entity.putObject("KeyAttributes")
|
||||
.put("Type", "AWS::Resource")
|
||||
.put("ResourceType", "AWS::EC2::Instance")
|
||||
.put("Identifier", "i-0123456789abcdef0");
|
||||
entity.putObject("Attributes");
|
||||
ObjectNode datum = entityMetricData.putArray("MetricData").addObject();
|
||||
datum.put("MetricName", "mem_used_percent");
|
||||
datum.put("Value", 6.43);
|
||||
datum.put("Unit", "Percent");
|
||||
datum.put("StorageResolution", 60);
|
||||
datum.put("Timestamp", sdkTimestamp(EPOCH_NOW));
|
||||
datum.putArray("Dimensions").addObject().put("Name", "host").put("Value", "h1");
|
||||
req.put("StrictEntityValidation", false);
|
||||
|
||||
assertEquals(200, handler.handle("PutMetricData", req, REGION).getStatus());
|
||||
|
||||
ObjectNode stats = getStats("CWAgent", "mem_used_percent", "host", "h1",
|
||||
EPOCH_NOW.minusSeconds(60), EPOCH_NOW.plusSeconds(60), 60);
|
||||
assertEquals(1, stats.get("Datapoints").size());
|
||||
assertEquals(6.43, stats.get("Datapoints").get(0).get("Sum").asDouble());
|
||||
}
|
||||
|
||||
@Test
|
||||
void putMetricData_topLevelAndEntityMetricData_storesBoth() {
|
||||
ObjectNode req = MAPPER.createObjectNode();
|
||||
req.put("Namespace", "Mixed");
|
||||
req.putArray("MetricData").addObject()
|
||||
.put("MetricName", "TopLevel").put("Value", 1.0);
|
||||
ArrayNode entityMetricData = req.putArray("EntityMetricData");
|
||||
entityMetricData.addObject().putArray("MetricData").addObject()
|
||||
.put("MetricName", "FirstEntity").put("Value", 2.0);
|
||||
entityMetricData.addObject().putArray("MetricData").addObject()
|
||||
.put("MetricName", "SecondEntity").put("Value", 3.0);
|
||||
|
||||
assertEquals(200, handler.handle("PutMetricData", req, REGION).getStatus());
|
||||
|
||||
ObjectNode listReq = MAPPER.createObjectNode();
|
||||
listReq.put("Namespace", "Mixed");
|
||||
ObjectNode listed = (ObjectNode) handler.handle("ListMetrics", listReq, REGION).getEntity();
|
||||
List<String> names = new ArrayList<>();
|
||||
listed.get("Metrics").forEach(m -> names.add(m.get("MetricName").asText()));
|
||||
assertEquals(List.of("FirstEntity", "SecondEntity", "TopLevel"), names.stream().sorted().toList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void setAlarmState_updatesFieldsCorrectly() {
|
||||
putMetric("NS", "M", "type", "current", 100.0, EPOCH_NOW);
|
||||
|
||||
+58
@@ -1,6 +1,8 @@
|
||||
package io.github.hectorvent.floci.services.cloudwatch.metrics;
|
||||
|
||||
import io.github.hectorvent.floci.services.cloudwatch.dashboards.CloudWatchDashboardsService;
|
||||
import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;
|
||||
import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricDatum;
|
||||
import io.github.hectorvent.floci.services.cloudwatch.metricstreams.CloudWatchMetricStreamsService;
|
||||
import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;
|
||||
import jakarta.ws.rs.core.MultivaluedHashMap;
|
||||
@@ -17,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -123,4 +126,59 @@ class CloudWatchMetricsQueryHandlerTest {
|
||||
anyInt(), any(), unitCaptor.capture(), any());
|
||||
assertEquals("Count", unitCaptor.getValue());
|
||||
}
|
||||
|
||||
// The Query serialization of EntityMetricData nests each datum two lists deep;
|
||||
// floci-io/floci#6066 is the JSON-side report of the same dropped datums.
|
||||
@Test
|
||||
void putMetricDataStoresTopLevelAndEntityMetricDataUnderTheRequestNamespace() {
|
||||
MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
|
||||
params.putSingle("Namespace", "CWAgent");
|
||||
params.putSingle("MetricData.member.1.MetricName", "TopLevel");
|
||||
params.putSingle("MetricData.member.1.Value", "1");
|
||||
params.putSingle("EntityMetricData.member.1.Entity.KeyAttributes.entry.1.key", "Type");
|
||||
params.putSingle("EntityMetricData.member.1.Entity.KeyAttributes.entry.1.value", "AWS::Resource");
|
||||
params.putSingle("EntityMetricData.member.1.MetricData.member.1.MetricName", "mem_used_percent");
|
||||
params.putSingle("EntityMetricData.member.1.MetricData.member.1.Value", "6.5");
|
||||
params.putSingle("EntityMetricData.member.1.MetricData.member.1.Unit", "Percent");
|
||||
params.putSingle("EntityMetricData.member.1.MetricData.member.1.Dimensions.member.1.Name", "host");
|
||||
params.putSingle("EntityMetricData.member.1.MetricData.member.1.Dimensions.member.1.Value", "h1");
|
||||
params.putSingle("EntityMetricData.member.2.MetricData.member.1.MetricName", "disk_used_percent");
|
||||
params.putSingle("EntityMetricData.member.2.MetricData.member.1.Value", "0.75");
|
||||
params.putSingle("StrictEntityValidation", "false");
|
||||
|
||||
handler.handle("PutMetricData", params, REGION);
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
ArgumentCaptor<List<MetricDatum>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(metricsService).putMetricData(eq("CWAgent"), captor.capture(), eq(REGION));
|
||||
List<MetricDatum> datums = captor.getValue();
|
||||
assertEquals(List.of("TopLevel", "mem_used_percent", "disk_used_percent"),
|
||||
datums.stream().map(MetricDatum::getMetricName).toList());
|
||||
MetricDatum entityDatum = datums.get(1);
|
||||
assertEquals(6.5, entityDatum.getValue());
|
||||
assertEquals("Percent", entityDatum.getUnit());
|
||||
assertEquals("host", entityDatum.getDimensions().get(0).name());
|
||||
assertEquals("h1", entityDatum.getDimensions().get(0).value());
|
||||
}
|
||||
|
||||
@Test
|
||||
void putMetricDataStoresEntityMetricDataAfterAnEmptyEntityEntry() {
|
||||
MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
|
||||
params.putSingle("Namespace", "CWAgent");
|
||||
params.putSingle("EntityMetricData.member.2.MetricData.member.1.MetricName", "M");
|
||||
params.putSingle("EntityMetricData.member.2.MetricData.member.1.Value", "1");
|
||||
params.putSingle("EntityMetricData.member.10.MetricData.member.1.MetricName", "N");
|
||||
params.putSingle("EntityMetricData.member.10.MetricData.member.1.Value", "2");
|
||||
params.putSingle("StrictEntityValidation", "false");
|
||||
|
||||
handler.handle("PutMetricData", params, REGION);
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
ArgumentCaptor<List<MetricDatum>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(metricsService).putMetricData(eq("CWAgent"), captor.capture(), eq(REGION));
|
||||
List<MetricDatum> datums = captor.getValue();
|
||||
assertEquals(List.of("M", "N"), datums.stream().map(MetricDatum::getMetricName).toList());
|
||||
assertEquals(1.0, datums.get(0).getValue());
|
||||
assertEquals(2.0, datums.get(1).getValue());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user