Linux: Prometheus-exporter

https://prometheus.io/docs/instrumenting/exporters/

指标导出器一些应用现实

grafana 模板

https://grafana.com/grafana/dashboards/

数据库

mysqld-exporter

https://github.com/prometheus/mysqld_exporter

数据库授权

CREATE USER 'mysqld_exporter'@'%' IDENTIFIED BY 'hIhO6h_j(azn=F17' WITH MAX_USER_CONNECTIONS 10;
GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'mysqld_exporter'@'%';
flush privileges;

部署方法1-docker-compose

准备配置文件 

cat <<\EOF> .my.cnf 
[client]
user=xxx
password=xxxxx
[client.servers]
user=mysqld_exporter
password=hIhO6h_j(azn=F17
EOF

启动 

cat <<\EOF> docker-compose.yml 
volumes:
networks:
  vm_net:
    name: vm_net
    driver: bridge
    ipam:
      config:
        - subnet: 172.26.0.0/24
services:
  mysqld_exporter:
    image: prom/mysqld-exporter:v0.15.1
    hostname: mysqld-exporter
    container_name: mysqld_exporter
    ports:
      - 9104:9104
    command:
      - "--config.my-cnf=/opt/my.cnf"
    volumes:
      - .my.cnf:/opt/my.cnf
    #restart: always
    networks:
      - vm_net
EOF

验证： 

curl 'http://localhost:9104/probe?auth_module=client.servers&target=pc-txxx.mysql.polardb.singapore.rds.aliyuncs.com:3306' | grep mysql_version_info

pormetheus集成 

- job_name: mysql
  metrics_path: /probe
  params:
    auth_module: [client.servers]
  static_configs:
  - targets:
    - xx:3306
    labels:
      project: mysql-test
  - targets:
    - 47.xxx:3306
    labels:
      project: mysql-test2

  relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - source_labels: [__param_target]
      target_label: instance
    - target_label: __address__
      # The mysqld_exporter host:port
      replacement: 172.xxx:9104

部署方法2-k8s

安装 mysqld-export 

cat >exporter.yaml <<EOF
apiVersion: v1
data:
  my.cnf: |-
    [client.servers]
    user=mysqld_exporter
    password=hIhO6h_j(azn=F17
kind: ConfigMap
metadata:
  name: mysqld-exporter-cm
  namespace: monitoring
---
apiVersion: v1
kind: Service
metadata:
  name: mysqld-exporter
  namespace: monitoring
  #annotations:
  #  prometheus.io/scrape: "true"
  labels:
    app: mysqld-exporter
spec:
  selector:
    app: mysqld-exporter
  ports:
    - name: metrics
      protocol: TCP
      port: 9104
      targetPort: 9104
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mysqld-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: mysqld-exporter
  template:
    metadata:
      #annotations:
      #  prometheus.io/path: "/metrics"
      #  prometheus.io/port: "9104"
      #  prometheus.io/scrape: "true"
      labels:
        app: mysqld-exporter
    spec:
      #nodeSelector:
      #  type: ops-prod-prometheus
      #imagePullSecrets:
      #- name: docker-secret
      containers:
      - name: mysqld-exporter
        image: prom/mysqld-exporter:v0.20.0
        imagePullPolicy: IfNotPresent
        command:
        - /bin/mysqld_exporter
        - --config.my-cnf=/data/my.cnf
        resources:
          limits:
            cpu: 300m
            memory: 300Mi
          requests:
            cpu: 10m
            memory: 10Mi
        ports:
        - containerPort: 9104
        volumeMounts:
        - mountPath: /data
          name: conf
      volumes:
      - configMap:
          defaultMode: 420
          items:
          - key: my.cnf
            path: my.cnf
          name: mysqld-exporter-cm
        name: conf
---
#apiVersion: monitoring.coreos.com/v1
#kind: ServiceMonitor
#metadata:
#  name: mysqld-exporter
#  namespace: monitoring
#  labels:
#    app.kubernetes.io/name: mysqld-exporter
#    app.kubernetes.io/instance: mysqld-exporter
#    app.kubernetes.io/component: exporter
#    release: kps
#spec:
#  endpoints:
#    - path: /metrics
#      port: tcp-9104
#      #interval: 15s
#  namespaceSelector:
#    matchNames:
#      - monitoring
#  selector:
#    matchLabels:
#      app: mysqld-exporter
EOF

kubectl apply -f exporter.yaml

方法1-增加 prometheus 静态配置 

cat >prometheus-additional.yaml <<EOF
- job_name: mysql
  metrics_path: /probe
  params:
    auth_module: [client.servers]
  static_configs:
  - targets:
    - 1xx.xx.rds.amazonaws.com:3306
    labels:
      business: jp-test
      type: rds
      service: "my1"
  - targets:
    - xx.xx.rds.amazonaws.com:3306
    labels:
      business: jp-test
      type: rds
      service: "my2"
  relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - source_labels: [__param_target]
      target_label: instance
    - target_label: __address__
      # The mysqld_exporter host:port
      replacement: mysqld-exporter:9104
EOF
# Update
kubectl -n monitoring create secret generic additional-scrape-configs --from-file=prometheus-additional.yaml --dry-run=client -oyaml|kubectl replace -f-

方法2-ScrapeConfig 

cat >mysql-scrapeconfig.yaml << EOF
# 备选方案:Prometheus Operator >= v0.65 的 ScrapeConfig CRD
# 和原来的 static_configs 几乎 1:1,实例多的时候比 ServiceMonitor 好维护
apiVersion: monitoring.coreos.com/v1alpha1
kind: ScrapeConfig
metadata:
  name: mysql
  namespace: monitoring
  labels:
    release: prometheus      # 对应 Prometheus CR 的 scrapeConfigSelector
spec:
  metricsPath: /probe
  params:
    auth_module:
      - client.servers
  staticConfigs:
    - targets:
        - 1xx.xx.rds.amazonaws.com:3306
      labels:
        business: jp-test
        type: rds
        service: "my1"
    - targets:
        - xx.xx.rds.amazonaws.com:3306
      labels:
        business: jp-test
        type: rds
        service: "my2"
  relabelings:
    - sourceLabels: [__address__]
      targetLabel: __param_target
    - sourceLabels: [__param_target]
      targetLabel: instance
    - targetLabel: __address__
      # mysqld-exporter 的 host:port(跨 namespace 写全:mysqld-exporter.monitoring.svc:9104)
      replacement: mysqld-exporter:9104
EOF

方法3-ServiceMonitor 

cat >mysql-servicemonitor.yaml <<EOF
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: mysql
  namespace: monitoring
  labels:
    release: kps      # 让 Prometheus CR 的 serviceMonitorSelector 能选中(kube-prometheus-stack 默认)
spec:
  namespaceSelector:
    matchNames:
      - monitoring           # mysqld-exporter Service 所在的 namespace
  selector:
    matchLabels:
      app: mysqld-exporter
  endpoints:
    # ---------- my1 ----------
    - port: metrics
      path: /probe
      interval: 30s
      params:
        auth_module:
          - client.servers
        target:
          - 1xx.xx.rds.amazonaws.com:3306
      relabelings:
        - targetLabel: job
          replacement: mysql
        - targetLabel: instance
          replacement: 1xx.xx.rds.amazonaws.com:3306
        - targetLabel: business
          replacement: jp-test
        - targetLabel: type
          replacement: rds
        - targetLabel: service
          replacement: my1

    # ---------- my2 ----------
    - port: metrics
      path: /probe
      interval: 30s
      params:
        auth_module:
          - client.servers
        target:
          - xx.xx.rds.amazonaws.com:3306
      relabelings:
        - targetLabel: job
          replacement: mysql
        - targetLabel: instance
          replacement: xx.xx.rds.amazonaws.com:3306
        - targetLabel: business
          replacement: jp-test
        - targetLabel: type
          replacement: rds
        - targetLabel: service
          replacement: my2
EOF

Redis exporter

https://github.com/oliver006/redis_exporter

部署方法1-docker-compose

准备密码文件 

cat <<\EOF> redis-pwd-file 
{
  "redis://redis6:6379": "",
  "rediss://user1@replica.redixxxx.com:6379": "xxx"
}
EOF

启动redis-exporter 

cat <<\EOF> docker-compose.yml 
networks:
  vm_net:
    name: vm_net
    driver: bridge
    ipam:
      config:
        - subnet: 172.26.0.0/24
services:
  redis_exporter:
    image: oliver006/redis_exporter:v1.67.0
    container_name: redis_exporter
    ports:
      - 9121:9121
    command:
      - "--redis.password-file=/opt/sample-pwd-file.json"
      - '--debug'
    volumes:
      - ./redis-pwd-file:/opt/sample-pwd-file.json
    networks:
      - vm_net
EOF

验证： 

curl http://127.0.0.1:9121/scrape?target=redis://xxxx:6379

部署方法2-k8s

cat >exporter.yaml <<EOF
apiVersion: v1
data:
  sample-pwd-file.json: |-
    {
      "redis://redis6:6379": "",
      "rediss://user@clustercfg.xxxx.cache.amazonaws.com:6379": "passwordxx"
    }
kind: ConfigMap
metadata:
  name: redis-pwd-config
  namespace: monitoring
---
apiVersion: v1
kind: Service
metadata:
  name: redis-exporter
  namespace: monitoring
  #annotations:
  #  prometheus.io/scrape: "true"
  labels:
    app: redis-exporter
spec:
  selector:
    app: redis-exporter
  ports:
    - name: tcp-9121
      protocol: TCP
      port: 9121
      targetPort: 9121
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: redis-exporter
  template:
    metadata:
      #annotations:
      #  prometheus.io/path: "/metrics"
      #  prometheus.io/port: "9121"
      #  prometheus.io/scrape: "true"
      labels:
        app: redis-exporter
    spec:
      #nodeSelector:
      #  type: ops-prod-prometheus
      #imagePullSecrets:
      #- name: docker-secret
      containers:
      - name: redis-exporter
        image: oliver006/redis_exporter:v1.91.1
        imagePullPolicy: IfNotPresent
        env:
        - name: REDIS_PASSWORD_FILE
          value: /data/sample-pwd-file.json
        resources:
          limits:
            cpu: 300m
            memory: 300Mi
          requests:
            cpu: 10m
            memory: 10Mi
        ports:
        - containerPort: 9121
        volumeMounts:
        - mountPath: /data
          name: redis-pwd-file
      volumes:
      - configMap:
          defaultMode: 420
          items:
          - key: sample-pwd-file.json
            path: sample-pwd-file.json
          name: redis-pwd-config
        name: redis-pwd-file
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: redis-exporter
  namespace: monitoring
  labels:
    app.kubernetes.io/name: redis-exporter
    app.kubernetes.io/instance: redis-exporter
    app.kubernetes.io/version: "1.91.1"
    app.kubernetes.io/component: exporter
    release: kps
spec:
  endpoints:
    - path: /metrics
      port: tcp-9121
      #interval: 15s
  namespaceSelector:
    matchNames:
      - monitoring
  selector:
    matchLabels:
      app: redis-exporter
EOF

pormetheus集成 

- job_name: redis-exporter
  metrics_path: /scrape
  static_configs:
  - targets:
    - redis://user1@replica.redixxxx.com:6379
    labels:
      dbtype: "redis"
      project: test
  - targets:
    - redis://xxx:6379
    labels:
      dbtype: "redis"
      project: test2
  relabel_configs:
  - source_labels: [__address__]
    target_label: __param_target
  - source_labels: [__param_target]
    target_label: instance
  - target_label: __address__
    replacement: 172.xxx:9121

部署方法1-docker-compose

启动mongodb-exporter 

cat <<\EOF> docker-compose.yml 
networks:
  vm_net:
    name: vm_net
    driver: bridge
    ipam:
      config:
        - subnet: 172.26.0.0/24
services:
  mongodb-exporter:
    image: percona/mongodb_exporter:0.43
    container_name: mongodb-exporter
    #environment:
    #  MONGODB_URI: "mongodb://user:pass@host1:27017,host2:27017/admin"
    ports:
      - 9216:9216
    command:
      - --mongodb.uri=mongodb://root:xxxs.com:3717,xxx.com:3717/admin?replicaSet=mgset-310035630,mongodb://root:xxx14@dxxcom:3717,ddsxxx:3717/admin?replicaSet=mgset-310037394
      - '--collect-all'
      - '--split-cluster=true'
    networks:
      - vm_net
EOF

验证： 

curl 127.0.0.1:9216/scrape?target=mongodb://ddsxxx:3717|grep mongodb_up

pormetheus集成 

- job_name: mongodb-exporter
  metrics_path: /scrape
  static_configs:
  - targets:
    - mongodb://dxxx:3717
    labels:
      dbtype: "mongodb"
      project: "测试1"
  - targets:
    - mongodb://ddxxx:3717
    labels:
      dbtype: "mongodb"
      project: "测试2"
  relabel_configs:
  - source_labels: [__address__]
    target_label: __param_target
  - source_labels: [__param_target]
    target_label: instance
  - target_label: __address__
    replacement: 172.xxx:9216

grafana监控：https://github.com/percona/grafana-dashboards/blob/main/dashboards/MongoDB/MongoDB_Cluster_Summary.json

硬件相关

问题跟踪器和持续集成

消息系统

RocketMQ exporter

https://github.com/apache/rocketmq-exporter

启动redis-exporter 

cat <<\EOF> docker-compose.yml 
networks:
  vm_net:
    name: vm_net
    driver: bridge
    ipam:
      config:
        - subnet: 172.26.0.0/24
services:
  rocketmq-exporter:
    image: apache/rocketmq-exporter:0.0.2-SNAPSHOT
    container_name: rocketmq-exporter
    ports:
      - 5557:5557
    entrypoint:
      - java
      - -jar
      - rocketmq-exporter.jar
      - --server.port=5557
      - --rocketmq.config.namesrvAddr=127.0.0.1:9876
      - --rocketmq.config.enableACL=true
      - --rocketmq.config.accessKey=C65GKxxxx
      - --rocketmq.config.secretKey=oqHc0xxxx
    deploy:
      resources:
        limits:
          cpus: '2'
          memory: 2048M
        reservations:
          cpus: '0.25'
          memory: 256M
    networks:
      - vm_net
EOF

pormetheus集成 

- job_name: rocketmq-exporter
  static_configs:
  - targets: ['172.xxx:5557']

RabbitMQ exporter

添加监控用户 在 Amazon MQ Web 控制台创建监控用户步骤

  1. 打开你的 Amazon MQ RabbitMQ Web 控制台（截图里那个 https 地址）
  2. 登录（用初始 admin 账号）
  3. 进入 Admin > Users > Add a user
    • Username：rmq-monitor
    • Password：设置强密码
    • Tags：填 monitoring （不要写 administrator）
  4. 设置 vhost 权限（默认 vhost / ）
    • Read regex: .*
    • Write regex: 留空
    • Configure regex: 留空
#验证账号是否可用
curl -u rmq-monitor:你的密码 https://b-16993496-8824-465b-8fb7-94f1c35b3f78.mq.ap-southeast-1.on.aws/api/overview

部署方法2-k8s

cat >exporter.yaml<<\EOF
apiVersion: v1
kind: Secret
metadata:
  name: rabbitmq-exporter-secret
  namespace: monitoring
type: Opaque
stringData:
  RABBIT_PASSWORD: "xxxxx"
---
apiVersion: v1
kind: Service
metadata:
  name: rabbitmq-exporter
  namespace: monitoring
  #annotations:
  #  prometheus.io/scrape: "true"
  labels:
    app: rabbitmq-exporter
spec:
  selector:
    app: rabbitmq-exporter
  ports:
    - name: tcp-9419
      protocol: TCP
      port: 9419
      targetPort: 9419
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: rabbitmq-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rabbitmq-exporter
  template:
    metadata:
      #annotations:
      #  prometheus.io/path: "/metrics"
      #  prometheus.io/port: "9419"
      #  prometheus.io/scrape: "true"
      labels:
        app: rabbitmq-exporter
    spec:
      #nodeSelector:
      #  type: ops-prod-prometheus
      #imagePullSecrets:
      #- name: docker-secret
      containers:
      - name: rabbitmq-exporter
        image: kbudde/rabbitmq-exporter:1.0.0
        imagePullPolicy: IfNotPresent
        env:
        - name: RABBIT_URL
          value: "https://xx.mq.ap-southeast-1.on.aws"
        - name: RABBIT_USER
          value: "rmq-monitor"
        - name: RABBIT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: rabbitmq-exporter-secret
              key: RABBIT_PASSWORD
        - name: PUBLISH_PORT
          value: "9419"
        - name: RABBIT_TIMEOUT
          value: "30"
        # Amazon MQ 不支持 bert，必须显式关掉，否则 overview 采集会失败
        - name: RABBIT_CAPABILITIES
          value: "no_sort"
        # 不采集队列长列表，防止指标爆炸；生产队列数 >50 时按需调整
        - name: RABBIT_EXPORTERS
          value: "overview,queue,node,exchange"
          #value: "overview,queue"
         #不采集队列长列表，防止指标爆炸，按需开启
         #生产，队列数量 >50
         #排除所有 *_details 系列速率桶指标，保留核心指标：队列深度、unacked 数量、消费者数、内存占用。
        - name: EXCLUDE_METRICS
          value: "queue_totals.messages_ready_details,queue_totals.messages_unacknowledged_details,message_stats.deliver_details,message_stats.publish_details,message_stats.ack_details,message_stats.redeliver_details"
        # 超过阈值就不再逐队列上报，避免指标爆炸
        - name: MAX_QUEUES
          value: "1000"
        resources:
          limits:
            cpu: 300m
            memory: 300Mi
          requests:
            cpu: 10m
            memory: 10Mi
        ports:
        - containerPort: 9419
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: rabbitmq-exporter
  namespace: monitoring
  labels:
    app.kubernetes.io/name: rabbitmq-exporter
    app.kubernetes.io/instance: rabbitmq-exporter
    app.kubernetes.io/component: exporter
    release: kps
# Amazon MQ 的 management API 查询队列列表本身会消耗 broker CPU，采集间隔不要小于 30s，推荐 60s
spec:
  endpoints:
    - path: /metrics
      port: tcp-9419
      interval: 60s
  namespaceSelector:
    matchNames:
      - monitoring
  selector:
    matchLabels:
      app: rabbitmq-exporter
EOF

存储

HTTP

APIs

日志

金融运营

其他