TLS


https://prometheus.io/docs/alerting/latest/configuration/
https://prometheus.io/docs/prometheus/latest/configuration/https/
https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alertmanager_config

1. Important Points#

Alertmanager TLS has three common paths:
    inbound:
        Prometheus / users / amtool connect to Alertmanager over HTTPS

    outbound:
        Alertmanager sends notifications to HTTPS webhook

    SMTP:
        smtp_require_tls controls STARTTLS for email receiver

recommended:
    use --web.config.file for HTTPS server
    use tls_config in webhook http_config
    use smtp_require_tls=true for email

2. Inbound HTTPS#

web.yml#

tls_server_config:
  cert_file: /etc/alertmanager/tls/server.crt
  key_file: /etc/alertmanager/tls/server.key
  min_version: TLS12

start#

alertmanager \
  --config.file=/etc/alertmanager/alertmanager.yml \
  --web.config.file=/etc/alertmanager/web.yml

Prometheus to Alertmanager#

alerting:
  alertmanagers:
    - scheme: https
      static_configs:
        - targets:
            - alertmanager.example.com:9093
      tls_config:
        ca_file: /etc/prometheus/ca/company-ca.pem
        server_name: alertmanager.example.com

3. Outbound TLS#

webhook receiver#

receivers:
  - name: platform-webhook
    webhook_configs:
      - url: https://alerts.example.com/alertmanager
        send_resolved: true
        http_config:
          tls_config:
            ca_file: /etc/alertmanager/ca/company-ca.pem
            server_name: alerts.example.com
            min_version: TLS12

email#

global:
  smtp_smarthost: smtp.example.com:587
  smtp_from: alertmanager@example.com
  smtp_require_tls: true

4. Java#

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

public class AlertmanagerTlsExample {
  public static void main(String[] args) throws Exception {
    String body = """
      [{
        "labels": {"alertname": "TestAlert", "severity": "warning"},
        "annotations": {"summary": "TLS test"}
      }]
      """;

    HttpRequest request = HttpRequest.newBuilder()
        .uri(URI.create("https://alertmanager.example.com:9093/api/v2/alerts"))
        .header("content-type", "application/json")
        .POST(HttpRequest.BodyPublishers.ofString(body))
        .build();

    HttpResponse<String> response = HttpClient.newHttpClient()
        .send(request, HttpResponse.BodyHandlers.ofString());
    System.out.println(response.statusCode());
  }
}

5. Python#

pip install requests
import requests

payload = [{
    "labels": {"alertname": "TestAlert", "severity": "warning"},
    "annotations": {"summary": "TLS test"},
}]

resp = requests.post(
    "https://alertmanager.example.com:9093/api/v2/alerts",
    json=payload,
    verify="/etc/ssl/company-ca.pem",
    timeout=5,
)
resp.raise_for_status()

6. Go#

package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"net/http"
	"os"
)

func main() {
	ca, _ := os.ReadFile("/etc/ssl/company-ca.pem")
	pool := x509.NewCertPool()
	pool.AppendCertsFromPEM(ca)

	client := &http.Client{Transport: &http.Transport{
		TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12},
	}}

	body := []byte(`[{"labels":{"alertname":"TestAlert","severity":"warning"},"annotations":{"summary":"TLS test"}}]`)
	resp, err := client.Post("https://alertmanager.example.com:9093/api/v2/alerts", "application/json", bytes.NewReader(body))
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
}

7. Node.js#

import fs from "node:fs";
import https from "node:https";

const body = JSON.stringify([{
  labels: { alertname: "TestAlert", severity: "warning" },
  annotations: { summary: "TLS test" }
}]);

const options = {
  hostname: "alertmanager.example.com",
  port: 9093,
  path: "/api/v2/alerts",
  method: "POST",
  headers: {
    "content-type": "application/json",
    "content-length": Buffer.byteLength(body)
  },
  ca: fs.readFileSync("/etc/ssl/company-ca.pem"),
  minVersion: "TLSv1.2",
  rejectUnauthorized: true
};

const req = https.request(options, (res) => {
  console.log(res.statusCode);
});

req.on("error", (err) => {
  throw err;
});
req.write(body);
req.end();

8. Production Checklist#

inbound:
    Alertmanager HTTPS enabled
    Prometheus alertmanager tls_config configured
    cert hostname matches target

outbound:
    webhook http_config.tls_config configured
    smtp_require_tls=true for email
    insecure_skip_verify=false

operations:
    amtool / curl verification documented
    certificate expiry alert exists
    notification failures monitored