失眠网 > prometheus+alertmanager 企业微信告警

prometheus+alertmanager 企业微信告警

时间：2020-05-07 20:30:33

一、应用安装启动

1、软件准备

alertmanager-0.24.0.linux-amd64.tar.gz
blackbox_exporter-0.22.0.linux-amd64.tar.gz
node_exporter-1.4.0.linux-amd64.tar.gz
prometheus-2.40.0-rc.0.linux-amd64.tar.gz

2、配置文件

cat /data/prometheus/prometheus.yml #根据自己情况更改

global:scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.# scrape_timeout is set to the global default (10s).alerting:alertmanagers:- static_configs:- targets:- 这里填写alertmanagers的ip:9093# - alertmanager:9093rule_files:- "rules/*.rules" #这里定义rule文件# - "second_rules.yml"scrape_configs:# The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.- job_name: "prometheus"# metrics_path defaults to '/metrics'# scheme defaults to 'http'.static_configs:- targets: ["localhost:9090"]- job_name: '定义一个名称'metrics_path: /probeparams:module: [blackbox_exporter里面的module名称]static_configs:- targets:- http://api的ip/apirelabel_configs:- source_labels: [__address__]target_label: __param_target- source_labels: [__param_target]target_label: instance- target_label: __address__replacement: 这里填写blackbox_exporter的ip:9115- job_name: 'node'static_configs:- targets: ['localhost:9100']

cat /data/prometheus/rules/node.rules #这是一个node 的rules配置，可以直接使用，无需更改

groups:- name: 主机状态-监控告警rules:- alert: 主机状态expr: up == 0for: 1mlabels:status: 非常严重severity: warningannotations:#summary: "服务器宕机"description: "服务器延时超过5分钟"- alert: CPU使用情况expr: 100-(avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by(instance)* 100) > 80for: 1mlabels:status: 一般告警severity: warningannotations:#summary: "CPU使用率过高!"description: "CPU使用大于80%(目前使用:{{$value}}%)"- alert: 内存使用expr: round(100- node_memory_MemAvailable_bytes{instance!="10.152.120.25:9100"}/node_memory_MemTotal_bytes{instance!="10.152.120.25:9100"}*100) > 80for: 1mlabels:status: 一般告警severity: warningannotations:# summary: "内存使用率过高"description: "内存使用率{{ $value }}%"- alert: 25-内存使用expr: round(100- node_memory_MemAvailable_bytes{instance=~"10.152.120.25:9100"}/node_memory_MemTotal_bytes{instance=~"10.152.120.25:9100"}*100) > 95for: 1mlabels:status: 一般告警severity: warningannotations:#summary: "内存使用率过高"description: "内存使用率{{ $value }}%"- alert: IO性能expr: 100-(avg(irate(node_disk_io_time_seconds_total[1m])) by(instance)* 100) < 60for: 1mlabels:status: 严重告警severity: warningannotations:# summary: "{{$labels.mountpoint}} 流入磁盘IO使用率过高！"description: "{{$labels.mountpoint }} 流入磁盘IO大于60%(目前使用:{{$value}})"- alert: 网络expr: ((sum(rate (node_network_receive_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance)) / 100) > 102400for: 1mlabels:status: 严重告警severity: warningannotations:# summary: "{{$labels.mountpoint}} 流入网络带宽过高！"description: "{{$labels.mountpoint }}流入网络带宽持续2分钟高于100M. RX带宽使用率{{$value}}"- alert: TCP会话expr: node_netstat_Tcp_CurrEstab > 1000for: 1mlabels:severity: warningstatus: 严重告警annotations:# summary: "{{$labels.mountpoint}} TCP_ESTABLISHED过高！"description: "{{$labels.mountpoint }} TCP_ESTABLISHED大于1000%(目前使用:{{$value}}%)"- alert: 磁盘容量expr: 100-(node_filesystem_free_bytes{fstype=~"ext4|xfs"}/node_filesystem_size_bytes {fstype=~"ext4|xfs"}*100) > 90for: 1mlabels:status: 严重告警severity: warningannotations:# summary: "{{$labels.mountpoint}} 磁盘分区使用率过高！"description: "{{$labels.mountpoint }} 磁盘分区使用大于90%(目前使用:{{$value}}%)"

cat /data/prometheus/rules/blackbox_http.rules #这是自定义的api监控rules，需要根据自己情况更改

groups:- name: 接口状态 #组的名字，在这个文件中必须要唯一rules:- alert: http-api #告警的名字，在组中需要唯一expr: probe_success{job="这里要对应job里面的名称"} == 0 #表达式, 执行结果为true: 表示需要告警for: 1s #超过多少时间才认为需要告警(即up==0需要持续的时间)labels:status: 非常严重 severity: warning #定义标签annotations:description: "Job {{ $labels.job }} 中的接口 {{ $labels.instance }} 已经down掉."summary: '接口 {{ $labels.instance }} down ! ! !'

cat /data/alertmanager/alertmanager.yml #定义告警配置

global:resolve_timeout: 5m #每5分钟检测一次是否恢复templates:- '/data/alertmanager/wechat.tmpl' # Alertmanager微信告警模板route:group_by: ['alertname']group_wait: 5s # 初次发送告警延时group_interval: 1m# 距离第一次发送告警，等待多久再次发送告警repeat_interval: 5m # 告警重发时间receiver: 'wechat'receivers:- name: 'wechat'wechat_configs:- corp_id: 'ww8f28'# 企业微信中企业IDto_party: '42' # 企业微信中创建的接收告警的告警部门ID#to_user: 'zhai' # 企业微信中创建的接收告警的单个人唯一IDagent_id: '100' # 企业微信中创建应用的AgentIdapi_secret: 'wiZIFkuo' # 企业微信中，Prometheus应用的Secretsend_resolved: trueinhibit_rules:- source_match:severity: 'critical'target_match:severity: 'warning'equal: ['alertname', 'dev', 'instance']

cat /data/blackbox_exporter/blackbox.yml

modules:http_2xx:prober: httphttp_post_2xx:prober: httphttp:method: POSTtcp_connect:prober: tcppop3s_banner:prober: tcptcp:query_response:- expect: "^+OK"tls: truetls_config:insecure_skip_verify: falsegrpc:prober: grpcgrpc:tls: truepreferred_ip_protocol: "ip4"grpc_plain:prober: grpcgrpc:tls: falseservice: "service1"ssh_banner:prober: tcptcp:query_response:- expect: "^SSH-2.0-"- send: "SSH-2.0-blackbox-ssh-check"irc_banner:prober: tcptcp:query_response:- send: "NICK prober"- send: "USER prober prober prober :prober"- expect: "PING :([^ ]+)"send: "PONG ${1}"- expect: "^:[^ ]+ 001"icmp:prober: icmpicmp_ttl5:prober: icmptimeout: 5sicmp:ttl: 5##################以上都是默认配置无需更改##############http_api: #定义模块名称，和prometheus的module一样prober: http timeout: 18shttp:method: GETheaders:token: 3579333KX4abK04i5Content-Type: application/json

3、服务启动

#prometheus/data/prometheus/prometheus --config.file=prometheus.yml --log.level=debug &#备注开启debug日志，方便差错#alertmanager/data/alertmanager/alertmanager --config.file=alertmanager.yml --log.level=debug &#blackbox_exporter/data/blackbox_exporter/blackbox_exporter --config.file=blackbox.yml &#node_exporter/data/node_exporter/node_exporter &