Prometheus Grafana

Prometheus & Grafana

Collect metrics and visualize system performance with the Prometheus-Grafana stack.

When to Use This Skill

Use this skill when:

• Setting up metrics collection infrastructure
• Creating monitoring dashboards
• Writing PromQL queries for analysis
• Configuring alerting rules
• Monitoring Kubernetes clusters

Prerequisites

• Docker or Kubernetes for deployment
• Network access to monitored targets
• Basic understanding of metrics concepts

Prometheus Setup

Docker Deployment

# docker-compose.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:v2.48.0
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./rules:/etc/prometheus/rules
      - prometheus-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=15d'

  grafana:
    image: grafana/grafana:10.2.0
    ports:
      - "3000:3000"
    volumes:
      - grafana-data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin

volumes:
  prometheus-data:
  grafana-data:

Configuration

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alertmanager:9093

rule_files:
  - /etc/prometheus/rules/*.yml

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node'
    static_configs:
      - targets:
          - 'node-exporter:9100'

  - job_name: 'applications'
    static_configs:
      - targets:
          - 'app1:8080'
          - 'app2:8080'
    metrics_path: /metrics

Kubernetes Deployment

Using Helm

# Add Prometheus community Helm repo
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts

# Install kube-prometheus-stack
helm install prometheus prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace \
  --set grafana.adminPassword=admin

ServiceMonitor

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: myapp
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: myapp
  endpoints:
    - port: metrics
      interval: 30s
      path: /metrics
  namespaceSelector:
    matchNames:
      - default

PromQL Queries

Basic Queries

# Current CPU usage
node_cpu_seconds_total{mode="idle"}

# Rate of HTTP requests per second
rate(http_requests_total[5m])

# Average response time
avg(http_request_duration_seconds_sum / http_request_duration_seconds_count)

# Memory usage percentage
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100

Aggregations

# Sum requests by status code
sum by (status_code) (rate(http_requests_total[5m]))

# Average CPU by instance
avg by (instance) (rate(node_cpu_seconds_total{mode!="idle"}[5m]))

# Top 5 endpoints by request count
topk(5, sum by (endpoint) (rate(http_requests_total[5m])))

# 95th percentile latency
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))

Time-Based Queries

# Compare to 1 hour ago
http_requests_total - http_requests_total offset 1h

# Predict disk space in 4 hours
predict_linear(node_filesystem_avail_bytes[1h], 4 * 3600)

# Changes in last 5 minutes
changes(up[5m])

# Average over 24 hours
avg_over_time(http_requests_total[24h])

Alerting Rules

# rules/alerts.yml
groups:
  - name: application
    rules:
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m]))
          / sum(rate(http_requests_total[5m])) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate detected"
          description: "Error rate is {{ $value | humanizePercentage }}"

      - alert: ServiceDown
        expr: up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Service {{ $labels.instance }} is down"

      - alert: HighMemoryUsage
        expr: |
          (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.9
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High memory usage on {{ $labels.instance }}"
          description: "Memory usage is {{ $value | humanizePercentage }}"

      - alert: DiskSpaceLow
        expr: |
          (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Disk space low on {{ $labels.instance }}"

Alertmanager

# alertmanager.yml
global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/xxx'

route:
  receiver: 'slack-notifications'
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - match:
        severity: critical
      receiver: 'pagerduty'

receivers:
  - name: 'slack-notifications'
    slack_configs:
      - channel: '#alerts'
        send_resolved: true
        title: '{{ .Status | toUpper }}: {{ .CommonAnnotations.summary }}'
        text: '{{ .CommonAnnotations.description }}'

  - name: 'pagerduty'
    pagerduty_configs:
      - service_key: 'xxx'
        severity: critical

Grafana Dashboards

Dashboard JSON Structure

{
  "dashboard": {
    "title": "Application Metrics",
    "panels": [
      {
        "title": "Request Rate",
        "type": "graph",
        "targets": [
          {
            "expr": "sum(rate(http_requests_total[5m])) by (status_code)",
            "legendFormat": "{{ status_code }}"
          }
        ],
        "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}
      },
      {
        "title": "Latency P95",
        "type": "gauge",
        "targets": [
          {
            "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))"
          }
        ],
        "gridPos": {"x": 12, "y": 0, "w": 6, "h": 8}
      }
    ]
  }
}

Provisioning Dashboards

# grafana/provisioning/dashboards/dashboards.yml
apiVersion: 1

providers:
  - name: 'default'
    orgId: 1
    folder: ''
    type: file
    disableDeletion: false
    updateIntervalSeconds: 30
    options:
      path: /var/lib/grafana/dashboards

Data Source Provisioning

# grafana/provisioning/datasources/prometheus.yml
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: false

Recording Rules

# rules/recording.yml
groups:
  - name: aggregations
    interval: 30s
    rules:
      - record: job:http_requests:rate5m
        expr: sum by (job) (rate(http_requests_total[5m]))

      - record: instance:node_cpu:avg_rate5m
        expr: |
          avg by (instance) (
            rate(node_cpu_seconds_total{mode!="idle"}[5m])
          )

      - record: job:http_latency:p95
        expr: |
          histogram_quantile(0.95,
            sum by (job, le) (rate(http_request_duration_seconds_bucket[5m]))
          )

Application Instrumentation

Go Application

import (
    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

var httpRequests = prometheus.NewCounterVec(
    prometheus.CounterOpts{
        Name: "http_requests_total",
        Help: "Total HTTP requests",
    },
    []string{"method", "endpoint", "status"},
)

func init() {
    prometheus.MustRegister(httpRequests)
}

// Expose metrics endpoint
http.Handle("/metrics", promhttp.Handler())

Node.js Application

const client = require('prom-client');

const httpRequests = new client.Counter({
  name: 'http_requests_total',
  help: 'Total HTTP requests',
  labelNames: ['method', 'endpoint', 'status']
});

// Middleware
app.use((req, res, next) => {
  res.on('finish', () => {
    httpRequests.inc({
      method: req.method,
      endpoint: req.path,
      status: res.statusCode
    });
  });
  next();
});

// Expose metrics
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', client.register.contentType);
  res.end(await client.register.metrics());
});

Common Issues

Issue: Targets Not Discovered

Problem: Prometheus not scraping targets Solution: Check network connectivity, verify target labels

Issue: High Memory Usage

Problem: Prometheus using excessive memory Solution: Reduce retention, use recording rules, limit cardinality

Issue: Slow Queries

Problem: PromQL queries timing out Solution: Use recording rules, limit time ranges, optimize queries

Issue: Missing Data Points

Problem: Gaps in metrics data Solution: Check scrape interval, verify target availability

Best Practices

• Use recording rules for frequently-used queries
• Limit label cardinality to prevent memory issues
• Set appropriate retention based on storage capacity
• Use histogram metrics for latency measurement
• Implement proper alerting thresholds
• Version control dashboards as code
• Use federation for large-scale deployments
• Regularly review and prune unused metrics

Related Skills

• alerting-oncall - Alert management
• loki-logging - Log aggregation
• kubernetes-ops - K8s monitoring

Prometheus Alerting Rules Guide

Rule Structure

groups:
- name: example
  rules:
  - alert: AlertName
    expr: <PromQL expression>
    for: <duration>
    labels:
      severity: <critical|warning|info>
      team: <team-name>
    annotations:
      summary: "Brief description"
      description: "Detailed description with {{ $labels.instance }}"
      runbook_url: "https://wiki.example.com/alerts/AlertName"

Essential Alerts

Infrastructure Alerts

groups:
- name: infrastructure
  rules:
  
  # Node down
  - alert: NodeDown
    expr: up{job="node"} == 0
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "Node {{ $labels.instance }} is down"
  
  # High CPU
  - alert: HighCPU
    expr: |
      100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: "High CPU on {{ $labels.instance }}"
      description: "CPU usage is {{ $value | printf \"%.1f\" }}%"
  
  # High Memory
  - alert: HighMemory
    expr: |
      (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High memory on {{ $labels.instance }}"
  
  # Disk Space Low
  - alert: DiskSpaceLow
    expr: |
      (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} 
       / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) * 100 < 15
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Low disk space on {{ $labels.instance }}"
      description: "{{ $labels.mountpoint }} has {{ $value | printf \"%.1f\" }}% free"
  
  # Disk Space Critical
  - alert: DiskSpaceCritical
    expr: |
      (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} 
       / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) * 100 < 5
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Critical disk space on {{ $labels.instance }}"

Application Alerts

groups:
- name: application
  rules:
  
  # High Error Rate
  - alert: HighErrorRate
    expr: |
      sum by (service) (rate(http_requests_total{status=~"5.."}[5m]))
      / sum by (service) (rate(http_requests_total[5m])) > 0.05
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "High error rate for {{ $labels.service }}"
      description: "Error rate is {{ $value | printf \"%.2f\" }}%"
  
  # High Latency
  - alert: HighLatency
    expr: |
      histogram_quantile(0.95, 
        sum by (le, service) (rate(http_request_duration_seconds_bucket[5m]))
      ) > 0.5
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High latency for {{ $labels.service }}"
      description: "P95 latency is {{ $value | printf \"%.2f\" }}s"
  
  # Service Down
  - alert: ServiceDown
    expr: up{job="app"} == 0
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Service {{ $labels.instance }} is down"

Kubernetes Alerts

groups:
- name: kubernetes
  rules:
  
  # Pod CrashLooping
  - alert: PodCrashLooping
    expr: |
      rate(kube_pod_container_status_restarts_total[15m]) * 60 * 15 > 0
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Pod {{ $labels.pod }} is crash looping"
  
  # Pod Not Ready
  - alert: PodNotReady
    expr: |
      kube_pod_status_ready{condition="true"} == 0
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: "Pod {{ $labels.pod }} is not ready"
  
  # Deployment Replicas Mismatch
  - alert: DeploymentReplicasMismatch
    expr: |
      kube_deployment_spec_replicas != kube_deployment_status_replicas_available
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: "Deployment {{ $labels.deployment }} has replica mismatch"

Best Practices

1. Use `for` duration - Avoid alert flapping 2. Include runbook URLs - Link to remediation docs 3. Use severity labels - Route alerts appropriately 4. Template annotations - Include relevant context 5. Test alerts - Use promtool check rules

PromQL Cheat Sheet

Basic Queries

Instant Vectors

# Simple metric
http_requests_total

# With label filter
http_requests_total{status="200"}

# Multiple labels
http_requests_total{status="200", method="GET"}

# Regex matching
http_requests_total{status=~"2.."}
http_requests_total{status!~"5.."}

Range Vectors

# Last 5 minutes
http_requests_total[5m]

# Last 1 hour
http_requests_total[1h]

# Time units: s, m, h, d, w, y

Functions

Rate and Increase

# Per-second rate over 5m
rate(http_requests_total[5m])

# Total increase over 1h
increase(http_requests_total[1h])

# For gauges that can decrease
irate(http_requests_total[5m])  # instant rate

Aggregations

# Sum across all instances
sum(http_requests_total)

# Sum by label
sum by (status) (http_requests_total)

# Sum excluding label
sum without (instance) (http_requests_total)

# Other aggregations
avg, min, max, count, stddev, stdvar
topk(5, http_requests_total)
bottomk(3, http_requests_total)

Histogram Quantiles

# 95th percentile
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))

# With grouping
histogram_quantile(0.95, 
  sum by (le, endpoint) (
    rate(http_request_duration_seconds_bucket[5m])
  )
)

Common Patterns

Request Rate

# Total request rate
sum(rate(http_requests_total[5m]))

# Request rate by endpoint
sum by (endpoint) (rate(http_requests_total[5m]))

Error Rate

# Error percentage
sum(rate(http_requests_total{status=~"5.."}[5m])) 
/ 
sum(rate(http_requests_total[5m])) 
* 100

Latency

# Average latency
rate(http_request_duration_seconds_sum[5m])
/
rate(http_request_duration_seconds_count[5m])

# P99 latency
histogram_quantile(0.99, 
  sum by (le) (rate(http_request_duration_seconds_bucket[5m]))
)

Resource Usage

# CPU usage percentage
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# Memory usage percentage
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100

# Disk usage percentage
(1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes)) * 100

Kubernetes

# Pod CPU usage
sum by (pod) (rate(container_cpu_usage_seconds_total{container!=""}[5m]))

# Pod memory usage
sum by (pod) (container_memory_usage_bytes{container!=""})

# Pod restart count
sum by (pod) (kube_pod_container_status_restarts_total)

Alert Examples

High Error Rate

- alert: HighErrorRate
  expr: |
    sum(rate(http_requests_total{status=~"5.."}[5m])) 
    / sum(rate(http_requests_total[5m])) > 0.05
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: "High error rate detected"

High Latency

- alert: HighLatency
  expr: |
    histogram_quantile(0.95, 
      sum by (le) (rate(http_request_duration_seconds_bucket[5m]))
    ) > 0.5
  for: 5m
  labels:
    severity: warning

Low Disk Space

- alert: LowDiskSpace
  expr: |
    (node_filesystem_avail_bytes / node_filesystem_size_bytes) < 0.1
  for: 5m
  labels:
    severity: warning