APM-Grafana-Mimir代替Prometheus实现指标长期存储-EW帮帮网

APM-Grafana-Mimir代替Prometheus实现指标长期存储

博主其他文章

基础版的APMALG(Alloy+Loki+Grafana)轻量级日志系统

进阶版的APMAPM-基于Grafana生态以及OTLP协议的Java轻量级日志监控系统(ALG系统进阶版)_otlp 存储s3-CSDN博客

Quickwit代替ALG中LokiAPM-基于Quickwit以及OTLP协议的轻量级Java日志系统

初始化

文件夹

# 配置文件
mkdir -p /data/apm/mimir-config/config

# rustfs(国产的对象存储, 类似Minio, 兼容s3协议)配置以及bugckt桶
mkdir -p /data/apm/rustfs/mimir-data

# mimir数据卷
mkdir -p /data/apm/mimir-1-data
mkdir -p /data/apm/mimir-2-data

配置

alloy配置文件

vim /data/apm/mimir-config/config/config.alloy

// 基础设施指标收集组件一般都是prometheus.exporter.*的形式, 见官方文档https://grafana.com/docs/alloy/latest/reference/components/prometheus/
// 目标组件, 这里目标组件是Linux
prometheus.exporter.unix "node" {}
 
// 抓取目标组件指标
prometheus.scrape "node" {
  // 源
  targets         = prometheus.exporter.unix.node.targets
  // 指标转发
  forward_to      = [prometheus.relabel.node.receiver]
  // 10s抓取一次指标
  scrape_interval = "10s"
}

// 筛选指标
// prometheus.relabel 组件通常用于筛选 Prometheus 指标或标准化传递给一个或多个下游接收器的标签集
prometheus.relabel "node" {
  // 将address或者instance替换成host
  rule {
    action        = "replace"
    source_labels = ["__address__", "instance"]
    separator     = "/"
    target_label  = "host"
  }
  // 处理完后转发到组件
  forward_to = [prometheus.remote_write.metrics_service.receiver]
}

prometheus.remote_write "metrics_service" {
  endpoint {
    // 转发给ng, ng转发给mimir
    url = "http://load-balancer:9009/api/v1/push"
    headers = {
      // 替换为你的实际 org id, 如果没有这个会导致401没有权限
      "X-Scope-OrgID" = "demo",  
    }
    send_native_histograms = true
    send_exemplars = true
  }
}

grafana数据源配置

vim /data/apm/mimir-config/config/grafana-provisioning-datasources.yaml

apiVersion: 1

datasources:
  - name: Mimir
    # 注意了, mimir的类型是prometheus
    type: prometheus
    access: proxy
    orgId: 1
    url: http://load-balancer:9009/prometheus
    version: 1
    editable: true
    jsonData:
      httpHeaderName1: "X-Scope-OrgID"
      alertmanagerUid: "alertmanager"
    secureJsonData:
      # 这个是租户ID, 用于过滤同一系统下不同租户的指标
      httpHeaderValue1: "demo"
    isDefault: true
  # 预警, 这里暂时用不上
  - name: Mimir Alertmanager
    uid: alertmanager
    type: alertmanager
    access: proxy
    orgId: 1
    url: http://load-balancer:9009/
    version: 1
    editable: true
    jsonData:
      httpHeaderName1: "X-Scope-OrgID"
      implementation: "cortex"
    secureJsonData:
      httpHeaderValue1: "demo"

Mimir配置

# Do not use this configuration in production.
# It is for demonstration purposes only.
# Run Mimir in single process mode, with all components running in 1 process.
# target: all,alertmanager,overrides-exporter
# 这里选择单体多实例部署, 不采用微服务模式
target: all

# Configure Mimir to use Minio as object storage backend.
# s3对象存储配置
common:
  storage:
    backend: s3
    s3:
      endpoint: obs:9000
      access_key_id: whiteBrocade
      secret_access_key: whiteBrocade
      insecure: true
      # 桶的名称
      bucket_name: mimir-data

# Blocks storage requires a prefix when using a common object storage bucket.
blocks_storage:
  # 路径前缀
  # 在mimir中存储长这样, /mimir-data/mimir/租户ID/具体数据
  # /mimir-data是桶
  # mimir是存储路径前缀
  storage_prefix: mimir
  tsdb:
    dir: /data/ingester

# Use memberlist, a gossip-based protocol, to enable the 3 Mimir replicas to communicate
memberlist:
  join_members: [mimir-1, mimir-2]

# 预警规则和预警器都关了, 不启用
# ruler:
#   rule_path: /data/ruler
#   alertmanager_url: http://127.0.0.1:8080/alertmanager
#   ring:
#     # Quickly detect unhealthy rulers to speed up the tutorial.
#     heartbeat_period: 2s
#     heartbeat_timeout: 10s

# alertmanager:
#   data_dir: /data/alertmanager
#   fallback_config_file: /etc/alertmanager-fallback-config.yaml
#   external_url: http://localhost:9009/alertmanager

server:
  log_level: warn

ngnix配置

events {
    worker_connections 1024;
}

http {
    # mimir负载均衡
    upstream backend {
        server mimir-1:8080 max_fails=1 fail_timeout=1s;
        server mimir-2:8080 max_fails=1 fail_timeout=1s backup;
    }

    server {
        listen 9009;
        access_log /dev/null;
        location / {
            proxy_pass http://backend;
        }
    }
}

docker compose文件

version: "3.4"

networks:
  mimir_network:  # 创建专用网络确保服务隔离
    driver: bridge


services:
  # obs:
  #   image: minio/minio:RELEASE.2025-05-24T17-08-30Z
  #   container_name: mimio
  #   entrypoint: [""]
  #   command: ["sh", "-c", "mkdir -p /mimir-data/mimir && minio server --console-address :9001 --quiet /data"]
  #   environment:
  #     - MINIO_ROOT_USER=whiteBrocade
  #     - MINIO_ROOT_PASSWORD=whiteBrocade
  #   volumes:
  #     -/data/apm/minio:/data
  #   ports:
  #     - 9000:9000
  #     - 9001:9001
  #   networks:
  #     - mimir_network

  # 国产开源minio代替品
  obs:
    image: rustfs/rustfs:latest
    container_name: rustfs
    ports:
      - 9000:9000
    volumes:
      - /data/apm/rustfs:/data
    environment:
      # 控制台账号/密码
      - RUSTFS_ACCESS_KEY=whiteBrocade
      - RUSTFS_SECRET_KEY=whiteBrocade
      # 开启控制台
      - RUSTFS_CONSOLE_ENABLE=true
      # 白名单过滤
      # - RUSTFS_SERVER_DOMAINS=example.com
    networks:
      - mimir_network

  grafana:
    image: grafana/grafana:latest
    pull_policy: always
    container_name: grafana
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
      - GF_USERS_DEFAULT_THEME=light
      - GF_LOG_MODE=console
      - GF_LOG_LEVEL=critical
    volumes:
      - /data/apm/mimir-config/config/grafana-provisioning-dashboards.yaml:/etc/grafana/provisioning/dashboards/provisioning-dashboards.yaml:ro
      - /data/apm/mimir-config/config/grafana-provisioning-datasources.yaml:/etc/grafana/provisioning/datasources/provisioning-datasources.yaml:ro
    ports:
      - 3000:3000
    networks:
      - mimir_network


  load-balancer:
    image: nginx:latest
    container_name: load-balancer
    volumes:
      - /data/apm/mimir-config/config/nginx.conf:/etc/nginx/nginx.conf:ro
    depends_on:
      - "mimir-1"
      - "mimir-2"
    ports:
      - 9009:9009
    networks:
      - mimir_network

  mimir-1:
    image: grafana/mimir:latest
    container_name: mimir-1
    command: ["-config.file=/etc/mimir.yaml"]
    depends_on:
      - obs
    volumes:
      - /data/apm/mimir-config/config/mimir.yaml:/etc/mimir.yaml
      # - /data/apm/mimir-config/config/alertmanager-fallback-config.yaml:/etc/alertmanager-fallback-config.yaml
      - /data/apm/mimir-1-data:/data
    networks:
      - mimir_network

  mimir-2:
    image: grafana/mimir:latest
    command: ["-config.file=/etc/mimir.yaml"]
    container_name: mimir-2
    hostname: mimir-2
    depends_on:
      - obs
    volumes:
      - /data/apm/mimir-config/config/mimir.yaml:/etc/mimir.yaml
      # 告警
      # - /data/apm/mimir-config/config/alertmanager-fallback-config.yaml:/etc/alertmanager-fallback-config.yaml
      - /data/apm/mimir-2-data:/data
    networks:
      - mimir_network

  alloy:
    image: grafana/alloy:latest
    container_name: alloy
    command: run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
    volumes:
      - /data/apm/mimir-config/config/config.alloy:/etc/alloy/config.alloy
    ports:
      - 12345:12345 
    networks:
      - mimir_network