fix(cloudwatch): store datums sent in PutMetricData EntityMetricData (#6070)

The CloudWatch Agent sends PutMetricData with an empty top-level MetricData and every datum nested under EntityMetricData[].MetricData. Both the JSON 1.0 handler (also reached by Smithy RPC v2 CBOR) and the Query handler read only the top-level list, so the agent's metrics were silently dropped. Both handlers now append each EntityMetricData entry's datums to the top-level ones and store them under the request Namespace, with the same parsing as top-level datums. Entity and StrictEntityValidation are accepted and ignored. The per-request INFO logs of the raw request and parsed datums are lowered to a DEBUG count. Fixes #6066 (cherry picked from commit 03f57303969561b049c3ebba38f1fd9d2b68df4c)
2026-10-20 17:49:42 +05:30
parent 2e0a438d70
commit 21625c5a1b
6 changed files with 358 additions and 14 deletions
@@ -6,6 +6,7 @@ import software.amazon.awssdk.services.cloudwatch.model.*;

 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.Map;

 import static org.assertj.core.api.Assertions.*;

@@ -185,4 +186,51 @@ class CloudWatchTest {
        );
        assertThat(noMatch.datapoints()).isEmpty();
    }
+
+    @Test
+    @DisplayName("PutMetricData stores datums nested under EntityMetricData")
+    void testPutMetricDataWithEntityMetricData() {
+        String namespace = "JavaTestEntityMetricData";
+        Instant now = Instant.now();
+
+        cloudWatchClient.putMetricData(request -> request
+                .namespace(namespace)
+                .metricData(datum -> datum
+                        .metricName("TopLevel")
+                        .value(1.0)
+                        .timestamp(now))
+                .entityMetricData(entityMetricData -> entityMetricData
+                        .entity(entity -> entity
+                                .keyAttributes(Map.of(
+                                        "Type", "AWS::Resource",
+                                        "ResourceType", "AWS::EC2::Instance",
+                                        "Identifier", "i-0123456789abcdef0")))
+                        .metricData(MetricDatum.builder()
+                                .metricName("mem_used_percent")
+                                .value(6.5)
+                                .unit(StandardUnit.PERCENT)
+                                .storageResolution(60)
+                                .timestamp(now)
+                                .dimensions(Dimension.builder().name("host").value("h1").build())
+                                .build()))
+                .strictEntityValidation(false)
+        );
+
+        ListMetricsResponse listed = cloudWatchClient.listMetrics(request -> request.namespace(namespace));
+        assertThat(listed.metrics())
+                .extracting(Metric::metricName)
+                .containsExactlyInAnyOrder("TopLevel", "mem_used_percent");
+
+        GetMetricStatisticsResponse stats = cloudWatchClient.getMetricStatistics(request -> request
+                .namespace(namespace)
+                .metricName("mem_used_percent")
+                .dimensions(Dimension.builder().name("host").value("h1").build())
+                .startTime(now.minus(1, ChronoUnit.HOURS))
+                .endTime(now.plus(1, ChronoUnit.MINUTES))
+                .period(3600)
+                .statistics(Statistic.SUM)
+        );
+        assertThat(stats.datapoints()).hasSize(1);
+        assertThat(stats.datapoints().get(0).sum()).isEqualTo(6.5);
+    }
 }
@@ -82,10 +82,11 @@ public class CloudWatchMetricsJsonHandler {

    private Response handlePutMetricData(JsonNode request, String region) {
        String namespace = request.path("Namespace").asText();
-        LOG.infov("JSON PutMetricData raw: {0}", request);
        List<MetricDatum> datums = parseMetricDataJson(request.path("MetricData"));
-        LOG.infov("JSON PutMetricData parsed {0} datums, sums={1}", datums.size(),
-                datums.stream().map(d -> d.getMetricName() + "=" + d.getSum()).toList());
+        for (JsonNode entityMetricData : request.path("EntityMetricData")) {
+            datums.addAll(parseMetricDataJson(entityMetricData.path("MetricData")));
+        }
+        LOG.debugv("JSON PutMetricData parsed {0} datums for namespace {1}", datums.size(), namespace);
        metricsService.putMetricData(namespace, datums, region);
        return Response.ok(objectMapper.createObjectNode()).build();
    }
@@ -26,11 +26,16 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import org.jboss.logging.Logger;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;

@ApplicationScoped
 public class CloudWatchMetricsQueryHandler {

    private static final Logger LOG = Logger.getLogger(CloudWatchMetricsQueryHandler.class);
+    private static final Pattern ENTITY_METRIC_DATA_MEMBER = Pattern.compile("EntityMetricData\\.member\\.(\\d{1,9})\\.");

    private CloudWatchMetricsService metricsService;
    private final CloudWatchDashboardsService dashboardsService;
@@ -591,42 +596,59 @@ public class CloudWatchMetricsQueryHandler {
    // ──────────────────────────── Parsing Helpers ────────────────────────────

    private List<MetricDatum> parseMetricData(MultivaluedMap<String, String> params) {
-        LOG.infov("Query PutMetricData params: {0}", params);
+        List<MetricDatum> datums = parseMetricData(params, "MetricData");
+        // An empty EntityMetricData entry serializes to no keys, so its index is absent
+        // while later entries keep theirs: iterate the indexes present, not 1..first gap.
+        SortedSet<Integer> entityIndexes = new TreeSet<>();
+        for (String key : params.keySet()) {
+            Matcher matcher = ENTITY_METRIC_DATA_MEMBER.matcher(key);
+            if (matcher.lookingAt()) {
+                entityIndexes.add(Integer.parseInt(matcher.group(1)));
+            }
+        }
+        for (int index : entityIndexes) {
+            datums.addAll(parseMetricData(params, "EntityMetricData.member." + index + ".MetricData"));
+        }
+        LOG.debugv("Query PutMetricData parsed {0} datums", datums.size());
+        return datums;
+    }
+
+    private List<MetricDatum> parseMetricData(MultivaluedMap<String, String> params, String listPrefix) {
        List<MetricDatum> datums = new ArrayList<>();
        for (int i = 1; ; i++) {
-            String metricName = params.getFirst("MetricData.member." + i + ".MetricName");
+            String metricName = params.getFirst(listPrefix + ".member." + i + ".MetricName");
            if (metricName == null) {
                break;
            }
            MetricDatum datum = new MetricDatum();
            datum.setMetricName(metricName);
-            datum.setUnit(params.getFirst("MetricData.member." + i + ".Unit"));
+            datum.setUnit(params.getFirst(listPrefix + ".member." + i + ".Unit"));

-            String valueStr = params.getFirst("MetricData.member." + i + ".Value");
+            String valueStr = params.getFirst(listPrefix + ".member." + i + ".Value");
            if (valueStr != null) {
                datum.setValue(parseDouble(valueStr, 0));
            }

-            String tsStr = params.getFirst("MetricData.member." + i + ".Timestamp");
+            String tsStr = params.getFirst(listPrefix + ".member." + i + ".Timestamp");
            if (tsStr != null) {
                Instant ts = parseInstant(tsStr);
                datum.setTimestamp(ts != null ? ts.getEpochSecond() : 0);
            }

            // StatisticValues
-            String sc = params.getFirst("MetricData.member." + i + ".StatisticValues.SampleCount");
+            String sc = params.getFirst(listPrefix + ".member." + i + ".StatisticValues.SampleCount");
            if (sc != null) {
                datum.setSampleCount(parseDouble(sc, 0));
-                datum.setSum(parseDouble(params.getFirst("MetricData.member." + i + ".StatisticValues.Sum"), 0));
-                datum.setMinimum(parseDouble(params.getFirst("MetricData.member." + i + ".StatisticValues.Minimum"), 0));
-                datum.setMaximum(parseDouble(params.getFirst("MetricData.member." + i + ".StatisticValues.Maximum"), 0));
+                datum.setSum(parseDouble(params.getFirst(listPrefix + ".member." + i + ".StatisticValues.Sum"), 0));
+                datum.setMinimum(parseDouble(params.getFirst(listPrefix + ".member." + i + ".StatisticValues.Minimum"), 0));
+                datum.setMaximum(parseDouble(params.getFirst(listPrefix + ".member." + i + ".StatisticValues.Maximum"), 0));
            }

            // Dimensions
            List<Dimension> dims = new ArrayList<>();
            for (int j = 1; ; j++) {
-                String dimName = params.getFirst("MetricData.member." + i + ".Dimensions.member." + j + ".Name");
-                String dimValue = params.getFirst("MetricData.member." + i + ".Dimensions.member." + j + ".Value");
+                String dimName = params.getFirst(listPrefix + ".member." + i + ".Dimensions.member." + j + ".Name");
+                String dimValue = params.getFirst(listPrefix + ".member." + i + ".Dimensions.member." + j + ".Value");
                if (dimName == null) {
                    break;
                }
@@ -0,0 +1,156 @@
+package io.github.hectorvent.floci.services.cloudwatch.metrics;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.RestAssured;
+import io.restassured.config.EncoderConfig;
+import io.restassured.http.ContentType;
+import io.restassured.parsing.Parser;
+import io.restassured.specification.RequestSpecification;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+
+import static io.restassured.RestAssured.config;
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+
+/**
+ * PutMetricData datums nested under EntityMetricData, as the CloudWatch Agent sends them,
+ * must be stored under the request namespace on every protocol path. floci-io/floci#6066.
+ */
+@QuarkusTest
+class CloudWatchMetricsEntityMetricDataIntegrationTest {
+
+    private static final String CW_AUTH =
+            "AWS4-HMAC-SHA256 Credential=test/20261020/us-east-1/monitoring/aws4_request";
+    private static final String JSON_1_0 = "application/x-amz-json-1.0";
+
+    @BeforeAll
+    static void registerJsonParser() {
+        RestAssured.registerParser(JSON_1_0, Parser.JSON);
+    }
+
+    private static RequestSpecification json(String operation) {
+        return given()
+                .header("Authorization", CW_AUTH)
+                .contentType(JSON_1_0)
+                .config(config().encoderConfig(EncoderConfig.encoderConfig()
+                        .encodeContentTypeAs(JSON_1_0, ContentType.TEXT)))
+                .header("X-Amz-Target", "GraniteServiceVersion20100801." + operation);
+    }
+
+    private static RequestSpecification query(String action) {
+        return given()
+                .header("Authorization", CW_AUTH)
+                .contentType("application/x-www-form-urlencoded")
+                .formParam("Action", action)
+                .formParam("Version", "2010-08-01");
+    }
+
+    @Test
+    void jsonEntityMetricDataOnlyIsListedAndAggregated() {
+        long now = Instant.now().getEpochSecond();
+        json("PutMetricData")
+                .body("""
+                        {"Namespace":"CWAgent/Json","MetricData":[],
+                         "EntityMetricData":[{
+                           "Entity":{"KeyAttributes":{"Type":"AWS::Resource",
+                             "ResourceType":"AWS::EC2::Instance","Identifier":"i-0123456789abcdef0"},
+                             "Attributes":{}},
+                           "MetricData":[
+                             {"MetricName":"mem_used_percent","Value":6.5,"Unit":"Percent",
+                              "StorageResolution":60,"Timestamp":%d,
+                              "Dimensions":[{"Name":"host","Value":"h1"}]},
+                             {"MetricName":"disk_used_percent","Value":0.75,"Unit":"Percent",
+                              "StorageResolution":60,"Timestamp":%d,
+                              "Dimensions":[{"Name":"host","Value":"h1"}]}]}],
+                         "StrictEntityValidation":false}
+                        """.formatted(now, now))
+        .when().post("/")
+        .then().statusCode(200);
+
+        json("ListMetrics")
+                .body("{\"Namespace\":\"CWAgent/Json\"}")
+        .when().post("/")
+        .then()
+                .statusCode(200)
+                .body("Metrics.MetricName", containsInAnyOrder("mem_used_percent", "disk_used_percent"));
+
+        json("GetMetricStatistics")
+                .body("""
+                        {"Namespace":"CWAgent/Json","MetricName":"mem_used_percent",
+                         "Dimensions":[{"Name":"host","Value":"h1"}],
+                         "StartTime":%d,"EndTime":%d,"Period":60,"Statistics":["Sum"]}
+                        """.formatted(now - 120, now + 120))
+        .when().post("/")
+        .then()
+                .statusCode(200)
+                .body("Datapoints.Sum", hasItem(6.5f));
+    }
+
+    @Test
+    void jsonTopLevelAndEntityMetricDataAreBothStored() {
+        json("PutMetricData")
+                .body("""
+                        {"Namespace":"CWAgent/JsonMixed",
+                         "MetricData":[{"MetricName":"TopLevel","Value":1}],
+                         "EntityMetricData":[{"Entity":{"KeyAttributes":{"Type":"Service",
+                           "Name":"checkout","Environment":"prod"}},
+                           "MetricData":[{"MetricName":"FromEntity","Value":2}]}]}
+                        """)
+        .when().post("/")
+        .then().statusCode(200);
+
+        json("ListMetrics")
+                .body("{\"Namespace\":\"CWAgent/JsonMixed\"}")
+        .when().post("/")
+        .then()
+                .statusCode(200)
+                .body("Metrics.MetricName", containsInAnyOrder("TopLevel", "FromEntity"));
+    }
+
+    @Test
+    void queryTopLevelAndEntityMetricDataAreListedAndAggregated() {
+        Instant now = Instant.now();
+        query("PutMetricData")
+                .formParam("Namespace", "CWAgent/Query")
+                .formParam("MetricData.member.1.MetricName", "TopLevel")
+                .formParam("MetricData.member.1.Value", "1")
+                .formParam("EntityMetricData.member.1.Entity.KeyAttributes.entry.1.key", "Type")
+                .formParam("EntityMetricData.member.1.Entity.KeyAttributes.entry.1.value", "AWS::Resource")
+                .formParam("EntityMetricData.member.1.MetricData.member.1.MetricName", "mem_used_percent")
+                .formParam("EntityMetricData.member.1.MetricData.member.1.Value", "6.5")
+                .formParam("EntityMetricData.member.1.MetricData.member.1.Timestamp", now.toString())
+                .formParam("EntityMetricData.member.1.MetricData.member.1.Dimensions.member.1.Name", "host")
+                .formParam("EntityMetricData.member.1.MetricData.member.1.Dimensions.member.1.Value", "h1")
+                .formParam("StrictEntityValidation", "false")
+        .when().post("/")
+        .then().statusCode(200);
+
+        query("ListMetrics")
+                .formParam("Namespace", "CWAgent/Query")
+        .when().post("/")
+        .then()
+                .statusCode(200)
+                .body("ListMetricsResponse.ListMetricsResult.Metrics.member.MetricName",
+                        containsInAnyOrder("TopLevel", "mem_used_percent"));
+
+        query("GetMetricStatistics")
+                .formParam("Namespace", "CWAgent/Query")
+                .formParam("MetricName", "mem_used_percent")
+                .formParam("Dimensions.member.1.Name", "host")
+                .formParam("Dimensions.member.1.Value", "h1")
+                .formParam("StartTime", now.minusSeconds(120).toString())
+                .formParam("EndTime", now.plusSeconds(120).toString())
+                .formParam("Period", "60")
+                .formParam("Statistics.member.1", "Sum")
+        .when().post("/")
+        .then()
+                .statusCode(200)
+                .body("GetMetricStatisticsResponse.GetMetricStatisticsResult.Datapoints.member.Sum",
+                        equalTo("6.5"));
+    }
+}
@@ -17,6 +17,12 @@ import java.math.BigDecimal;
 import java.time.Instant;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;

 /**
 * Handler-level tests for CloudWatchMetricsJsonHandler.
@@ -146,6 +152,59 @@ class CloudWatchMetricsJsonHandlerTest {
                "metric stored with 24h-ago timestamp must not appear in a 20-second window around now");
    }

+    // The CloudWatch Agent sends an empty top-level MetricData and nests every datum under
+    // EntityMetricData, floci-io/floci#6066.
+    @Test
+    void putMetricData_entityMetricDataOnly_storesNestedDatumsUnderTheRequestNamespace() {
+        ObjectNode req = MAPPER.createObjectNode();
+        req.put("Namespace", "CWAgent");
+        req.putArray("MetricData");
+        ObjectNode entityMetricData = req.putArray("EntityMetricData").addObject();
+        ObjectNode entity = entityMetricData.putObject("Entity");
+        entity.putObject("KeyAttributes")
+                .put("Type", "AWS::Resource")
+                .put("ResourceType", "AWS::EC2::Instance")
+                .put("Identifier", "i-0123456789abcdef0");
+        entity.putObject("Attributes");
+        ObjectNode datum = entityMetricData.putArray("MetricData").addObject();
+        datum.put("MetricName", "mem_used_percent");
+        datum.put("Value", 6.43);
+        datum.put("Unit", "Percent");
+        datum.put("StorageResolution", 60);
+        datum.put("Timestamp", sdkTimestamp(EPOCH_NOW));
+        datum.putArray("Dimensions").addObject().put("Name", "host").put("Value", "h1");
+        req.put("StrictEntityValidation", false);
+
+        assertEquals(200, handler.handle("PutMetricData", req, REGION).getStatus());
+
+        ObjectNode stats = getStats("CWAgent", "mem_used_percent", "host", "h1",
+                EPOCH_NOW.minusSeconds(60), EPOCH_NOW.plusSeconds(60), 60);
+        assertEquals(1, stats.get("Datapoints").size());
+        assertEquals(6.43, stats.get("Datapoints").get(0).get("Sum").asDouble());
+    }
+
+    @Test
+    void putMetricData_topLevelAndEntityMetricData_storesBoth() {
+        ObjectNode req = MAPPER.createObjectNode();
+        req.put("Namespace", "Mixed");
+        req.putArray("MetricData").addObject()
+                .put("MetricName", "TopLevel").put("Value", 1.0);
+        ArrayNode entityMetricData = req.putArray("EntityMetricData");
+        entityMetricData.addObject().putArray("MetricData").addObject()
+                .put("MetricName", "FirstEntity").put("Value", 2.0);
+        entityMetricData.addObject().putArray("MetricData").addObject()
+                .put("MetricName", "SecondEntity").put("Value", 3.0);
+
+        assertEquals(200, handler.handle("PutMetricData", req, REGION).getStatus());
+
+        ObjectNode listReq = MAPPER.createObjectNode();
+        listReq.put("Namespace", "Mixed");
+        ObjectNode listed = (ObjectNode) handler.handle("ListMetrics", listReq, REGION).getEntity();
+        List<String> names = new ArrayList<>();
+        listed.get("Metrics").forEach(m -> names.add(m.get("MetricName").asText()));
+        assertEquals(List.of("FirstEntity", "SecondEntity", "TopLevel"), names.stream().sorted().toList());
+    }
+
    @Test
    void setAlarmState_updatesFieldsCorrectly() {
        putMetric("NS", "M", "type", "current", 100.0, EPOCH_NOW);
@@ -1,6 +1,8 @@
 package io.github.hectorvent.floci.services.cloudwatch.metrics;

 import io.github.hectorvent.floci.services.cloudwatch.dashboards.CloudWatchDashboardsService;
+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;
+import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricDatum;
 import io.github.hectorvent.floci.services.cloudwatch.metricstreams.CloudWatchMetricStreamsService;
 import io.github.hectorvent.floci.services.cloudwatch.metrics.model.MetricAlarm;
 import jakarta.ws.rs.core.MultivaluedHashMap;
@@ -17,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -123,4 +126,59 @@ class CloudWatchMetricsQueryHandlerTest {
                anyInt(), any(), unitCaptor.capture(), any());
        assertEquals("Count", unitCaptor.getValue());
    }
+
+    // The Query serialization of EntityMetricData nests each datum two lists deep;
+    // floci-io/floci#6066 is the JSON-side report of the same dropped datums.
+    @Test
+    void putMetricDataStoresTopLevelAndEntityMetricDataUnderTheRequestNamespace() {
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.putSingle("Namespace", "CWAgent");
+        params.putSingle("MetricData.member.1.MetricName", "TopLevel");
+        params.putSingle("MetricData.member.1.Value", "1");
+        params.putSingle("EntityMetricData.member.1.Entity.KeyAttributes.entry.1.key", "Type");
+        params.putSingle("EntityMetricData.member.1.Entity.KeyAttributes.entry.1.value", "AWS::Resource");
+        params.putSingle("EntityMetricData.member.1.MetricData.member.1.MetricName", "mem_used_percent");
+        params.putSingle("EntityMetricData.member.1.MetricData.member.1.Value", "6.5");
+        params.putSingle("EntityMetricData.member.1.MetricData.member.1.Unit", "Percent");
+        params.putSingle("EntityMetricData.member.1.MetricData.member.1.Dimensions.member.1.Name", "host");
+        params.putSingle("EntityMetricData.member.1.MetricData.member.1.Dimensions.member.1.Value", "h1");
+        params.putSingle("EntityMetricData.member.2.MetricData.member.1.MetricName", "disk_used_percent");
+        params.putSingle("EntityMetricData.member.2.MetricData.member.1.Value", "0.75");
+        params.putSingle("StrictEntityValidation", "false");
+
+        handler.handle("PutMetricData", params, REGION);
+
+        @SuppressWarnings("unchecked")
+        ArgumentCaptor<List<MetricDatum>> captor = ArgumentCaptor.forClass(List.class);
+        verify(metricsService).putMetricData(eq("CWAgent"), captor.capture(), eq(REGION));
+        List<MetricDatum> datums = captor.getValue();
+        assertEquals(List.of("TopLevel", "mem_used_percent", "disk_used_percent"),
+                datums.stream().map(MetricDatum::getMetricName).toList());
+        MetricDatum entityDatum = datums.get(1);
+        assertEquals(6.5, entityDatum.getValue());
+        assertEquals("Percent", entityDatum.getUnit());
+        assertEquals("host", entityDatum.getDimensions().get(0).name());
+        assertEquals("h1", entityDatum.getDimensions().get(0).value());
+    }
+
+    @Test
+    void putMetricDataStoresEntityMetricDataAfterAnEmptyEntityEntry() {
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.putSingle("Namespace", "CWAgent");
+        params.putSingle("EntityMetricData.member.2.MetricData.member.1.MetricName", "M");
+        params.putSingle("EntityMetricData.member.2.MetricData.member.1.Value", "1");
+        params.putSingle("EntityMetricData.member.10.MetricData.member.1.MetricName", "N");
+        params.putSingle("EntityMetricData.member.10.MetricData.member.1.Value", "2");
+        params.putSingle("StrictEntityValidation", "false");
+
+        handler.handle("PutMetricData", params, REGION);
+
+        @SuppressWarnings("unchecked")
+        ArgumentCaptor<List<MetricDatum>> captor = ArgumentCaptor.forClass(List.class);
+        verify(metricsService).putMetricData(eq("CWAgent"), captor.capture(), eq(REGION));
+        List<MetricDatum> datums = captor.getValue();
+        assertEquals(List.of("M", "N"), datums.stream().map(MetricDatum::getMetricName).toList());
+        assertEquals(1.0, datums.get(0).getValue());
+        assertEquals(2.0, datums.get(1).getValue());
+    }
 }