Kubernetes 多团队资源计量与成本分摊实战

shared-prod
├── payment-prod      team=payment
├── payment-dev       team=payment
├── order-prod        team=order
├── monitoring        平台公共成本
├── opencost          平台公共成本
└── kube-system       平台公共成本

apiVersion: v1
kind: Namespace
metadata:
  name: payment-prod
  labels:
    team: payment
    cost-center: payment
    environment: prod

kubectl apply -f payment-namespace.yaml
kubectl get namespaces -L team,cost-center,environment

apiVersion: v1
kind: ResourceQuota
metadata:
  name: team-budget
  namespace: payment-prod
spec:
  hard:
    requests.cpu: "8"
    requests.memory: 16Gi
    limits.cpu: "16"
    limits.memory: 32Gi
    requests.storage: 100Gi
    persistentvolumeclaims: "10"
---
apiVersion: v1
kind: LimitRange
metadata:
  name: container-defaults
  namespace: payment-prod
spec:
  limits:
    - type: Container
      defaultRequest:
        cpu: 100m
        memory: 128Mi
      default:
        cpu: "1"
        memory: 512Mi

kubectl apply -f payment-budget.yaml
kubectl -n payment-prod describe resourcequota team-budget

申请累计量 = 2 核 × 10 小时 = 20 核时

实际消耗 = 18,000 / 3,600 = 5 核时
平均使用 = 5 核时 / 10 小时 = 0.5 核
相对申请量的平均利用率 = 5 / 20 = 25%

kubectl config current-context
kubectl get nodes
helm version
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo add opencost-charts https://opencost.github.io/opencost-helm-chart
helm repo update
helm search repo prometheus-community/kube-prometheus-stack --versions
helm search repo opencost-charts/opencost --versions

# 替换成经过兼容性核对的 Chart 版本，保存到部署记录。
export MONITORING_CHART_VERSION='<监控 Chart 版本>'
export OPENCOST_CHART_VERSION='<OpenCost Chart 版本>'
helm show values prometheus-community/kube-prometheus-stack --version "$MONITORING_CHART_VERSION" > monitoring-default-values.yaml
helm show values opencost-charts/opencost --version "$OPENCOST_CHART_VERSION" > opencost-default-values.yaml

kube-state-metrics:
  metricLabelsAllowlist:
    - namespaces=[team,cost-center,environment]
    - pods=[team,app,environment]
prometheus:
  prometheusSpec:
    scrapeInterval: 30s
    evaluationInterval: 1m
    retention: 45d

helm upgrade --install monitoring prometheus-community/kube-prometheus-stack -n monitoring --create-namespace --version "$MONITORING_CHART_VERSION" -f monitoring-values.yaml --wait --timeout 10m
kubectl -n monitoring get prometheus,pods,services

opencost:
  exporter:
    defaultClusterId: shared-prod
  prometheus:
    internal:
      enabled: true
      serviceName: monitoring-kube-prometheus-prometheus
      namespaceName: monitoring
      port: 9090
    external:
      enabled: false
  metrics:
    serviceMonitor:
      enabled: true
      additionalLabels:
        release: monitoring
      scrapeInterval: 30s
  ui:
    enabled: true

helm upgrade --install opencost opencost-charts/opencost -n opencost --create-namespace --version "$OPENCOST_CHART_VERSION" -f opencost-values.yaml --wait --timeout 5m
kubectl -n opencost get deployment,pods,servicemonitor
kubectl -n opencost logs deployment/opencost --all-containers=true --tail=100
kubectl -n opencost port-forward deployment/opencost 9003:9003 9090:9090

count(container_cpu_usage_seconds_total{container!="",container!="POD",namespace!=""})
count(container_memory_working_set_bytes{container!="",container!="POD",namespace!=""})
count(kube_pod_container_resource_requests)
count(kube_namespace_labels{label_team!=""})
count(node_cpu_hourly_cost)
count(node_ram_hourly_cost)

sum by (namespace) (
  increase(container_cpu_usage_seconds_total{
    namespace!="", container!="", container!="POD",
    metrics_path="/metrics/cadvisor"
  }[24h])
) / 3600

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: namespace-cost-metering
  namespace: monitoring
  labels:
    release: monitoring
spec:
  groups:
    - name: namespace-cost-metering
      interval: 1m
      rules:
        - record: namespace:cpu_requests_running:cores
          expr: |
            sum by (namespace) (
              max by (namespace, pod, uid, container) (
                kube_pod_container_resource_requests{resource="cpu",unit="core",node!=""}
              )
              and on (namespace, pod, uid)
              (max by (namespace, pod, uid) (kube_pod_status_phase{phase="Running"}) == 1)
            )
        - record: namespace:memory_working_set:gib
          expr: |
            sum by (namespace) (
              container_memory_working_set_bytes{
                namespace!="",container!="",container!="POD",
                metrics_path="/metrics/cadvisor"
              }
            ) / 1073741824

kubectl apply -f cost-recording-rules.yaml
kubectl -n monitoring get prometheusrule namespace-cost-metering

sum_over_time(namespace:cpu_requests_running:cores[24h]) / 60

sum_over_time(namespace:memory_working_set:gib[24h]) / 60

curl --fail --silent --show-error --get http://localhost:9003/allocation --data-urlencode 'window=24h' --data-urlencode 'aggregate=namespace' --data-urlencode 'resolution=1m' --data-urlencode 'includeIdle=true' --data-urlencode 'shareIdle=false' --data-urlencode 'idleByNode=true' > allocation-preview.json
jq -e '.code == 200 and (.data | type == "array")' allocation-preview.json

export COST_WINDOW_START='2026-09-20T00:00:00Z'
export COST_WINDOW_END='2026-09-21T00:00:00Z'

curl --fail --silent --show-error --get http://localhost:9003/allocation --data-urlencode "window=${COST_WINDOW_START},${COST_WINDOW_END}" --data-urlencode 'step=24h' --data-urlencode 'resolution=1m' --data-urlencode 'aggregate=namespace' --data-urlencode 'includeIdle=true' --data-urlencode 'shareIdle=false' --data-urlencode 'idleByNode=true' > allocation-day.json

jq -e '.code == 200 and (.data | type == "array") and (.data | length > 0)' allocation-day.json
jq -r '
  ["allocation","cpuCost","ramCost","pvCost","networkCost","totalCost"],
  (.data[] | to_entries[] |
    [.key,.value.cpuCost,.value.ramCost,.value.pvCost,
     .value.networkCost,.value.totalCost]) | @csv
' allocation-day.json > allocation-day.csv

节点成本 ≈ CPU 分配成本 + 内存分配成本 + GPU 等适用成本
节点成本分配结果 = 团队份额 + 公共工作负载份额 + 闲置/系统开销 + 差异

支付团队 = 4,000 + 1,000 × 4,000 / 6,000 = ¥4,666.67
订单团队 = 2,000 + 1,000 × 2,000 / 6,000 = ¥2,333.33
平台     = ¥2,000
合计     = ¥9,000

apiVersion: apps/v1
kind: Deployment
metadata:
  name: cost-demo
  namespace: payment-prod
spec:
  replicas: 1
  selector:
    matchLabels:
      app: cost-demo
  template:
    metadata:
      labels:
        app: cost-demo
        team: payment
        environment: prod
    spec:
      containers:
        - name: sleeper
          image: busybox:1.37.0
          command: ["sh", "-c", "sleep 86400"]
          resources:
            requests:
              cpu: "2"
              memory: 256Mi
            limits:
              cpu: "2"
              memory: 512Mi

kubectl apply -f cost-demo.yaml
kubectl -n payment-prod rollout status deployment/cost-demo --timeout=120s
kubectl -n payment-prod get pods -l app=cost-demo -o wide

kubectl -n payment-prod delete deployment cost-demo