Docker Monitoring: Prometheus, Grafana and Loki (2026)

Running containers in production without monitoring is flying blind. The standard open-source observability stack for Docker is: cAdvisor for container resource metrics, Prometheus for scraping and storing time-series data, Grafana for dashboards and alerting, and Loki + Promtail for log aggregation. This phase covers deploying the full stack with Docker Compose, the essential PromQL queries for container health, structured logging best practices, and setting up alerts for high CPU, OOM kills and container restarts.

The Monitoring Stack

# Full observability stack with Docker Compose
# compose.monitoring.yml

name: monitoring

services:
  # Container metrics collector
  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.49.1
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
    ports:
      - "8080:8080"
    networks:
      - monitoring

  # Metrics time-series database
  prometheus:
    image: prom/prometheus:v2.53.0
    volumes:
      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./monitoring/alerts.yml:/etc/prometheus/alerts.yml:ro
      - prometheus_data:/prometheus
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.retention.time=30d
      - --web.enable-lifecycle        # Allow config reload via HTTP POST
    ports:
      - "9090:9090"
    networks:
      - monitoring

  # Visualization and dashboards
  grafana:
    image: grafana/grafana:11.1.0
    environment:
      GF_SECURITY_ADMIN_PASSWORD: admin123
      GF_PATHS_PROVISIONING: /etc/grafana/provisioning
    volumes:
      - grafana_data:/var/lib/grafana
      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
    ports:
      - "3000:3000"
    networks:
      - monitoring

  # Log aggregation
  loki:
    image: grafana/loki:3.1.0
    ports:
      - "3100:3100"
    volumes:
      - loki_data:/loki
    networks:
      - monitoring

  # Log shipper (runs on each host)
  promtail:
    image: grafana/promtail:3.1.0
    volumes:
      - /var/log:/var/log:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - ./monitoring/promtail.yml:/etc/promtail/config.yml:ro
    networks:
      - monitoring

  # Alerting
  alertmanager:
    image: prom/alertmanager:v0.27.0
    volumes:
      - ./monitoring/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
    ports:
      - "9093:9093"
    networks:
      - monitoring

networks:
  monitoring:
    driver: bridge

volumes:
  prometheus_data:
  grafana_data:
  loki_data:

cAdvisor: Container Metrics

# cAdvisor (Container Advisor) automatically collects:
# - CPU usage per container
# - Memory usage and limits
# - Network I/O (bytes in/out)
# - Filesystem I/O
# - Container restarts and uptime

# cAdvisor exposes metrics at http://localhost:8080/metrics
# It scrapes from the Docker daemon via mounted volumes —
# no changes needed in your app containers.

# Key metrics cAdvisor exposes (Prometheus format):
#
# container_cpu_usage_seconds_total{name="myapp-web-1"}
# container_memory_usage_bytes{name="myapp-web-1"}
# container_memory_working_set_bytes{name="myapp-web-1"}
# container_network_receive_bytes_total{name="myapp-web-1",interface="eth0"}
# container_network_transmit_bytes_total{name="myapp-web-1",interface="eth0"}
# container_fs_reads_bytes_total{name="myapp-web-1"}
# container_fs_writes_bytes_total{name="myapp-web-1"}
# container_last_seen{name="myapp-web-1"}   # Disappears when container stops
# container_oom_events_total{name="myapp-web-1"}  # OOM kill counter
# container_restarts_total{name="myapp-web-1"}

# Access cAdvisor web UI (built-in container browser):
# http://localhost:8080/containers/
# http://localhost:8080/docker/   ← Docker-specific view

Prometheus Configuration

# monitoring/prometheus.yml

global:
  scrape_interval: 15s          # How often to scrape targets
  evaluation_interval: 15s      # How often to evaluate alert rules

rule_files:
  - /etc/prometheus/alerts.yml  # Alert rule definitions

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']

scrape_configs:
  # Scrape Prometheus itself
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Scrape cAdvisor for container metrics
  - job_name: 'cadvisor'
    static_configs:
      - targets: ['cadvisor:8080']
    metric_relabel_configs:
      # Drop high-cardinality metrics we don't need
      - source_labels: [__name__]
        regex: 'container_(network_tcp_usage|tasks_state|memory_failures).*'
        action: drop

  # Scrape Docker daemon metrics (enable in daemon.json first)
  - job_name: 'docker'
    static_configs:
      - targets: ['host.docker.internal:9323']

  # Scrape your application (if it exposes /metrics)
  - job_name: 'myapp'
    static_configs:
      - targets: ['web:3000']
    metrics_path: /metrics

  # Node Exporter (host metrics: CPU, memory, disk, network)
  - job_name: 'node'
    static_configs:
      - targets: ['node-exporter:9100']

# Reload config without restart:
# curl -X POST http://localhost:9090/-/reload

Grafana Dashboards

# Grafana auto-provisioning — define datasources and dashboards as code

# monitoring/grafana/provisioning/datasources/datasources.yml
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    url: http://prometheus:9090
    isDefault: true
    editable: false

  - name: Loki
    type: loki
    url: http://loki:3100
    editable: false

# Import community dashboards by ID (Grafana.com/dashboards):
# monitoring/grafana/provisioning/dashboards/dashboards.yml
apiVersion: 1
providers:
  - name: default
    folder: Docker
    type: file
    options:
      path: /etc/grafana/provisioning/dashboards

# Download dashboard JSON files:
# 193   — Docker and system monitoring (cAdvisor + node)
# 11600 — Docker container and host metrics
# 13639 — Logs / Loki dashboard
# curl -o monitoring/grafana/provisioning/dashboards/docker.json \
#   https://grafana.com/api/dashboards/193/revisions/latest/download

# Key Grafana panels to build for Docker:
# 1. Container CPU usage (%)       — rate(container_cpu_usage_seconds_total[5m]) * 100
# 2. Container memory usage (MB)   — container_memory_working_set_bytes / 1024 / 1024
# 3. Container restart count       — increase(container_restarts_total[1h])
# 4. Network in/out (MB/s)         — rate(container_network_receive_bytes_total[5m])
# 5. OOM kills                     — increase(container_oom_events_total[1h])
# 6. Container count by status     — count(container_last_seen) by (name)

Loki and Log Aggregation

# Promtail ships Docker container logs to Loki.
# Logs are queryable by container name, label, or log content.

# monitoring/promtail.yml
server:
  http_listen_port: 9080

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
  - job_name: docker
    docker_sd_configs:
      - host: unix:///var/run/docker.sock
        refresh_interval: 5s
        filters:
          - name: status
            values: [running]
    relabel_configs:
      # Use container name as the stream label
      - source_labels: [__meta_docker_container_name]
        regex: '/(.*)'
        target_label: container
      # Add compose service name
      - source_labels: [__meta_docker_container_label_com_docker_compose_service]
        target_label: service
      # Add compose project name
      - source_labels: [__meta_docker_container_label_com_docker_compose_project]
        target_label: project

# Query logs in Grafana (LogQL):
# {container="myapp-web-1"} |= "ERROR"              # Filter by container + text
# {service="web"} | json | level="error"             # Parse JSON logs, filter by level
# {project="myapp"} | logfmt | method="POST"         # Logfmt parsing
# rate({service="web"} |= "error" [5m])              # Error rate over time

# Structured logging best practices (makes Loki queries powerful):
# Log JSON from your app:
# {"level":"error","msg":"Database connection failed","service":"api","traceId":"abc123"}
# Then in Loki: {service="api"} | json | level="error"

Docker Daemon Metrics

# Enable Prometheus metrics in the Docker daemon itself.
# Exposes engine metrics: image/container/volume counts, build cache size.

# /etc/docker/daemon.json
{
  "metrics-addr": "0.0.0.0:9323",
  "experimental": true
}

# Restart daemon: sudo systemctl restart docker

# Metrics available at http://localhost:9323/metrics
# Key metrics:
# engine_daemon_container_states_containers{state="running"}
# engine_daemon_container_states_containers{state="stopped"}
# engine_daemon_image_actions_seconds_count{action="pull"}
# engine_daemon_network_actions_seconds_count
# engine_daemon_builder_builds_failed_total
# engine_daemon_builder_builds_triggered_total

# Add to Prometheus scrape config:
# - job_name: 'docker-daemon'
#   static_configs:
#     - targets: ['host.docker.internal:9323']
#   # On Linux, use the host's IP directly:
#   # - targets: ['172.17.0.1:9323']

# Node exporter (host-level metrics — CPU, memory, disk, network):
services:
  node-exporter:
    image: prom/node-exporter:v1.8.2
    pid: host
    network_mode: host
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - --path.procfs=/host/proc
      - --path.sysfs=/host/sys
      - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run)($|/)

Alerting with Alertmanager

# monitoring/alerts.yml — Prometheus alert rules

groups:
  - name: docker
    rules:
      # Container down (disappeared from cAdvisor)
      - alert: ContainerDown
        expr: absent(container_last_seen{name=~"myapp-.*"})
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Container {{ $labels.name }} is down"

      # High CPU (>80% for 5 minutes)
      - alert: ContainerHighCPU
        expr: |
          rate(container_cpu_usage_seconds_total{name=~"myapp-.*"}[5m]) * 100 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container {{ $labels.name }} CPU > 80%"
          description: "CPU usage: {{ $value | printf \"%.1f\" }}%"

      # High memory (>85% of limit)
      - alert: ContainerHighMemory
        expr: |
          container_memory_working_set_bytes{name=~"myapp-.*"}
          / container_spec_memory_limit_bytes{name=~"myapp-.*"} * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container {{ $labels.name }} memory > 85% of limit"

      # OOM kill
      - alert: ContainerOOMKilled
        expr: increase(container_oom_events_total{name=~"myapp-.*"}[1h]) > 0
        labels:
          severity: critical
        annotations:
          summary: "Container {{ $labels.name }} was OOM killed"

      # Container restarting
      - alert: ContainerRestarting
        expr: increase(container_restarts_total{name=~"myapp-.*"}[1h]) > 3
        labels:
          severity: warning
        annotations:
          summary: "Container {{ $labels.name }} restarted {{ $value }} times in 1h"

# monitoring/alertmanager.yml — route alerts to Slack
route:
  receiver: slack
  group_by: [alertname, name]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h

receivers:
  - name: slack
    slack_configs:
      - api_url: $SLACK_WEBHOOK_URL
        channel: '#alerts'
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'

Essential PromQL Queries

# Copy these into Grafana panels or Prometheus /graph

# CPU usage % per container (5m average)
rate(container_cpu_usage_seconds_total{image!=""}[5m]) * 100

# Memory usage MB per container
container_memory_working_set_bytes{image!=""} / 1024 / 1024

# Memory usage as % of limit
container_memory_working_set_bytes / container_spec_memory_limit_bytes * 100

# Network received MB/s per container
rate(container_network_receive_bytes_total{image!=""}[5m]) / 1024 / 1024

# Network transmitted MB/s per container
rate(container_network_transmit_bytes_total{image!=""}[5m]) / 1024 / 1024

# Container restart rate (restarts per hour)
increase(container_restarts_total{image!=""}[1h])

# OOM events in last 24h
increase(container_oom_events_total[24h])

# Disk I/O read MB/s
rate(container_fs_reads_bytes_total{image!=""}[5m]) / 1024 / 1024

# Count of running containers
count(container_last_seen{image!=""})

# Top 5 containers by CPU
topk(5, rate(container_cpu_usage_seconds_total{image!=""}[5m]) * 100)

# Top 5 containers by memory
topk(5, container_memory_working_set_bytes{image!=""})

# Filter by label (useful when containers have compose labels)
rate(container_cpu_usage_seconds_total{
  container_label_com_docker_compose_project="myapp",
  container_label_com_docker_compose_service="web"
}[5m]) * 100