AWS ECS And Node.js Monitoring

https://docs.aws.amazon.com/AmazonECS/latest/developerguide/available-metrics.html
https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/using-metric-math.html
https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/

Period 是每个 datapoint 的统计窗口，不是告警持续时间。
Alarm After 约等于 Period * EvaluationPeriods；应用指标的 `for` 直接表示持续时间。
rate/histogram window 是计算窗口，不等于告警 for。

current_5m > absolute_threshold
and
current_5m > 2 * avg_over_time(metric[1h])
for 10m

[
  {
    "Id": "running",
    "MetricStat": {
      "Metric": {
        "Namespace": "ECS/ContainerInsights",
        "MetricName": "RunningTaskCount",
        "Dimensions": [
          { "Name": "ClusterName", "Value": "prod-cluster" },
          { "Name": "ServiceName", "Value": "api" }
        ]
      },
      "Period": 60,
      "Stat": "Average"
    },
    "ReturnData": false
  },
  {
    "Id": "desired",
    "MetricStat": {
      "Metric": {
        "Namespace": "ECS/ContainerInsights",
        "MetricName": "DesiredTaskCount",
        "Dimensions": [
          { "Name": "ClusterName", "Value": "prod-cluster" },
          { "Name": "ServiceName", "Value": "api" }
        ]
      },
      "Period": 60,
      "Stat": "Average"
    },
    "ReturnData": false
  },
  {
    "Id": "task_gap",
    "Expression": "desired-running",
    "Label": "desired minus running",
    "ReturnData": true
  }
]

aws cloudwatch get-metric-data --start-time 2026-06-02T00:00:00Z --end-time 2026-06-02T01:00:00Z --metric-data-queries file://ecs-running-vs-desired.json

metric math id: task_gap
comparison: GreaterThanThreshold
threshold: 0
evaluation_periods: 5
datapoints_to_alarm: 5
period: 60
severity: P1

# 用正则先找出当前 YACE 暴露出来的 ECS metric 名称。
# 这里不是告警表达式，只是落地前确认 metric name 和 label name。
{__name__=~"aws_ecs_.*(running|desired|cpu|memory).*"}

# running task 小于 desired task，表示 ECS 想要的任务数没有全部跑起来。
# 这个一般是 deployment 卡住、镜像拉取失败、健康检查失败、容量不足。
aws_ecs_running_task_count_average{cluster_name="prod-cluster", service_name="api"}
  < aws_ecs_desired_task_count_average{cluster_name="prod-cluster", service_name="api"}

# ECS service 平均 CPU 使用率超过 80%。
# 适合做 P2 容量风险，不建议单独作为 P1 可用性告警。
aws_ecs_cpu_utilization_average{cluster_name="prod-cluster", service_name="api"} >= 80

# ECS service 平均内存使用率超过 85%。
# Fargate / container 内存打满通常会导致 OOM kill，所以阈值比 CPU 更敏感。
aws_ecs_memory_utilization_average{cluster_name="prod-cluster", service_name="api"} >= 85

# 计算 5 分钟内 HTTP 5xx 错误率:
#   分子 = 5xx 请求速率
#   分母 = 全部请求速率
#   乘以 100 = 百分比
# 结果 >= 5 表示 5xx 错误率达到 5%。
100 *
sum(rate(http_requests_total{service="api",status=~"5.."}[5m]))
/
sum(rate(http_requests_total{service="api"}[5m]))
>= 5

# 计算 HTTP 请求耗时的 p95:
#   http_request_duration_seconds_bucket 是 Prometheus histogram bucket
#   rate(...[5m]) 取最近 5 分钟每个 bucket 的增长速率
#   sum by (le) 保留 bucket 边界 le，再把不同 instance 的 bucket 合并
#   histogram_quantile(0.95, ...) 根据 bucket 估算 p95
# 结果单位是 seconds，>= 1 表示 p95 请求耗时超过 1 秒。
histogram_quantile(
  0.95,
  sum by (le) (rate(http_request_duration_seconds_bucket{service="api"}[5m]))
) >= 1

# 计算 Node.js event loop lag 的 p95:
#   event loop lag 高说明 JS 主线程被 CPU、同步代码或 GC 阻塞
#   结果单位是 seconds，>= 0.2 表示 p95 lag 超过 200ms。
histogram_quantile(
  0.95,
  sum by (le) (rate(nodejs_eventloop_lag_seconds_bucket{service="api"}[5m]))
) >= 0.2

# 计算 V8 heap 使用率:
#   used bytes / total bytes = heap 使用比例
#   >= 0.85 表示 heap 已经用了 85%，需要排查内存泄漏或调大内存。
nodejs_heap_size_used_bytes{service="api"}
/
nodejs_heap_size_total_bytes{service="api"}
>= 0.85

groups:
  - name: ecs-nodejs.rules
    rules:
      - alert: ECSServiceRunningTaskTooLow
        # desired > running 时触发，表示 ECS service 没有达到期望任务数。
        expr: aws_ecs_running_task_count_average < aws_ecs_desired_task_count_average
        for: 5m
        labels:
          severity: P1
          component: ecs
        annotations:
          summary: "ECS running task count is below desired count"

      - alert: NodeJSHigh5xxRate
        # 5xx rate = 5xx 请求速率 / 总请求速率 * 100。
        # by (service) 是为了每个服务单独计算，不把多个服务混在一起。
        expr: |
          100 * sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service) >= 5
        for: 5m
        labels:
          severity: P1
          component: app
        annotations:
          summary: "Node.js HTTP 5xx rate is >= 5%"

      - alert: NodeJSHighP95Latency
        # 从 histogram bucket 计算每个 service 的 p95 延迟，单位 seconds。
        expr: |
          histogram_quantile(0.95, sum by (service, le) (rate(http_request_duration_seconds_bucket[5m]))) >= 1
        for: 10m
        labels:
          severity: P1
          component: app
        annotations:
          summary: "Node.js HTTP p95 latency is >= 1s"