环境：ansible主机一台：10.1.234.11
node主机3台：node1 10.1.234.110 Prometheus+grafana+node_exporter+pushgateway+alertmanager
node2 10.1.234.111 node_exporter
node3 10.1.234.112 node_exporter
搭建全在ansible主机：其他主机不用登陆：
[root@ansible-11 promethes]# tree.├── Alertmanager.yaml├── Blackbox.yaml├── conf│ ├── alertmanager.back.yml│ ├── alertmanager.service│ ├── alertmanager.yml│ ├── blackbox_exporter.service│ ├── node_exporter.service│ ├── prometheus.service│ └── pushgateway.service├── Grafana.yaml├── hosts├── inster_promethes.sh├── node_exporter.yaml├── pkg│ ├── alertmanager-0.20.0.linux-amd64.tar.gz│ ├── blackbox_exporter-0.16.0.linux-amd64.tar.gz│ ├── grafana-6.1.3-1.x86_64.rpm│ ├── node_exporter-1.0.0-rc.1.linux-amd64.tar.gz│ ├── prometheus-2.8.1.linux-amd64.tar.gz│ ├── pushgateway-0.4.0.linux-amd64│ │ ├── LICENSE│ │ ├── NOTICE│ │ └── pushgateway│ └── pushgateway-0.4.0.linux-amd64.tar.gz├── Prometheus.yaml├── Pushgateway.yaml├── reload_Promethes│ ├── hosts│ ├── node.yml│ ├── prometheus.yml│ ├── prometheus.ymlback│ └── reload_promethes.yaml└── template_file├── blackbox-exporter_rev1.json├── MySQL_Overview-1589503416459.json├── Node_Exporter_0.16_0.17_for_Prometheus-1589503429385.json└── Redis_Dashboard_for_Prometheus_Redis_Exporter_1.x-1589503441940.json
[root@ansible-11 promethes]# cat Alertmanager.yaml ---- hosts: mastervars: remote_user: rootgather_facts: falsetasks:- name: 分发node_exporter二进制包 unarchive: src=pkg/alertmanager-0.20.0.linux-amd64.tar.gz dest=/tmp- name: 创建文件夹file: dest=/usr/local/prometheus state=directory- name: 创建数据目录file: dest=/data/prometheus/alertmanager/data state=directory- name: 创建用户user: name=prometheus state=present- name: 文件重命名shell: mv /tmp/alertmanager-0.20.0.linux-amd64 /usr/local/prometheus/alertmanager- name: 把文件划到组shell: chown -R prometheus:prometheus /usr/local/prometheus /data/prometheus- name: 拷贝alertmanager.service启动文件copy: src=conf/alertmanager.service dest=/usr/lib/systemd/system/alertmanager.service- name: 启动服务并设置开机自启systemd: name=alertmanager state=restarted enabled=yes- name: 拷贝配置文件到mastercopy: src=conf/alertmanager.yml dest=/usr/local/prometheus/alertmanager/alertmanager.yml- name: 重启服务systemd: name=alertmanager state=restarted - name: 查看状态并将结果注入到alertmanager变量shell: ss -nutlp |grep 9093 register: alertmanager- name: 将结果输出到控制台debug: var=alertmanager.stdout_lines
[root@ansible-11 promethes]# cat Blackbox.yaml ---- hosts: mastervars: remote_user: rootgather_facts: falsetasks:- name: 分发blackbox_exporter二进制包 unarchive: src=pkg/blackbox_exporter-0.16.0.linux-amd64.tar.gz dest=/tmp- name: 创建文件夹file: dest=/usr/local/prometheus state=directory- name: 创建用户user: name=prometheus state=present- name: 文件重命名shell: mv /tmp/blackbox_exporter-0.16.0.linux-amd64 /usr/local/prometheus/blackbox_exporter- name: 把文件划到组shell: chown -R prometheus:prometheus /usr/local/prometheus - name: 拷贝blackbox_exporter.service启动文件copy: src=conf/blackbox_exporter.service dest=/usr/lib/systemd/system/blackbox_exporter.service- name: 启动服务并设置开机自启systemd: name=blackbox_exporter state=restarted enabled=yes- name: 查看状态并将结果注入到blackbox_exporter变量shell: ss -nutlp |grep 9115 register: blackbox_exporter- name: 将结果输出到控制台debug: var=blackbox_exporter.stdout_lines
[root@ansible-11 promethes]# cat Grafana.yaml ---- hosts: mastervars: remote_user: rootgather_facts: falsetasks:- name: 拷贝grafana rpm文件copy: src=pkg/grafana-6.1.3-1.x86_64.rpm dest=/root- name: 安装grafanayum: name=grafana-6.1.3-1.x86_64.rpm - name: 启动服务并设置开机自启systemd: name=grafana-server state=restarted enabled=yes- name: 安装grafana所在server安装饼图插件shell: grafana-cli plugins install grafana-piechart-panel- name: 重启服务systemd: name=grafana-server state=restarted - name: 查看状态并将结果注入到grafana变量shell: ss -nutlp |grep 3000register: grafana- name: 将结果输出到控制台debug: var=grafana.stdout_lines
[root@ansible-11 promethes]# cat hosts [master]10.1.234.110[node]10.1.234.11010.1.234.11110.1.234.112
[root@ansible-11 promethes]# cat node_exporter.yaml ---- hosts: nodevars: remote_user: rootgather_facts: falsetasks:- name: 分发node_exporter二进制包 unarchive: src=pkg/node_exporter-1.0.0-rc.1.linux-amd64.tar.gz dest=/tmp- name: 创建文件夹file: dest=/usr/local/prometheus state=directory- name: 创建用户user: name=prometheus state=present- name: 文件重命名shell: mv /tmp/node_exporter-1.0.0-rc.1.linux-amd64 /usr/local/prometheus/node_exporter- name: 把文件划到组shell: chown -R prometheus:prometheus /usr/local/prometheus - name: 拷贝node_exporter.service启动文件copy: src=conf/node_exporter.service dest=/usr/lib/systemd/system/node_exporter.service- name: 启动服务并设置开机自启systemd: name=node_exporter state=restarted enabled=yes- name: 查看状态并将结果注入到node_exporter变量shell: ss -nutlp |grep 9100 register: node_exporter- name: 将结果输出到控制台debug: var=node_exporter.stdout_lines
[root@ansible-11 promethes]# cat Prometheus.yaml ---- hosts: mastervars: remote_user: rootgather_facts: falsetasks:- name: 分发prometheus二进制包 unarchive: src=pkg/prometheus-2.8.1.linux-amd64.tar.gz dest=/tmp- name: 创建文件夹file: dest=/usr/local/prometheus state=directory- name: 创建数据目录file: dest=/data/prometheus/data state=directory- name: 文件重命名shell: mv /tmp/prometheus-2.8.1.linux-amd64 /usr/local/prometheus/prometheus- name: 创建用户user: name=prometheus state=present- name: 把文件划到组shell: chown -R prometheus:prometheus /usr/local/prometheus /data/prometheus- name: 拷贝prometheus.service 启动文件copy: src=conf/prometheus.service dest=/usr/lib/systemd/system/prometheus.service - name: 启动服务并设置开机自启systemd: name=prometheus state=restarted enabled=yes- name: 查看状态并将结果注入到prometheus变量shell: ss -nutlp |grep 9090 register: prometheus- name: 将结果输出到控制台debug: var=prometheus.stdout_lines
[root@ansible-11 promethes]# cat Pushgateway.yaml ---- hosts: mastervars: remote_user: rootgather_facts: falsetasks:- name: 分发pushgateway二进制包 unarchive: src=pkg/pushgateway-0.4.0.linux-amd64.tar.gz dest=/tmp- name: 创建文件夹file: dest=/usr/local/prometheus state=directory- name: 文件重命名shell: mv /tmp/pushgateway-0.4.0.linux-amd64 /usr/local/prometheus/pushgateway- name: 创建用户user: name=prometheus state=present- name: 把文件划到组shell: chown -R prometheus:prometheus /usr/local/prometheus - name: 拷贝prometheus.service 启动文件copy: src=conf/pushgateway.service dest=/usr/lib/systemd/system/pushgateway.service- name: 启动服务并设置开机自启systemd: name=pushgateway state=restarted enabled=yes- name: 查看状态并将结果注入到pushgateway变量shell: ss -nutlp |grep 9091 register: pushgateway- name: 将结果输出到控制台debug: var=pushgateway.stdout_lines
[root@ansible-11 conf]# lsalertmanager.back.yml alertmanager.service alertmanager.yml blackbox_exporter.service node_exporter.service prometheus.service pushgateway.service
[root@ansible-11 reload_Promethes]# cat reload_promethes.yaml ---- hosts: mastergather_facts: notasks:- name: 更新配置文件copy:src: prometheus.ymldest: /usr/local/prometheus/prometheus/prometheus.yml- name: 创建规则文件file: dest=/usr/local/prometheus/prometheus/rules state=directory- name: 拷贝规则文件copy: src=node.yml dest=/usr/local/prometheus/prometheus/rules/node.yml- name: 把文件划到组shell: chown -R prometheus:prometheus /usr/local/prometheus - name: 重动服务systemd: name=prometheus state=restarted - name: 查看状态并将结果注入到prometheus变量shell: ss -nutlp |grep 9090 register: prometheus- name: 将结果输出到控制台debug: var=prometheus.stdout_lines
[root@ansible-11 reload_Promethes]# cat prometheus.yml# my global configglobal:scrape_interval:15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.# scrape_timeout is set to the global default (10s).# Alertmanager configurationalerting:alertmanagers:- static_configs:- targets:- 127.0.0.1:9093# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.rule_files:- "rules/node.yml"# - "second_rules.yml"# A scrape configuration containing exactly one endpoint to scrape:# Here it's Prometheus itself.scrape_configs:# The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.- job_name: 'prometheus'# metrics_path defaults to '/metrics'# scheme defaults to 'http'.static_configs:- targets: ['localhost:9090']- job_name: 'node-1'static_configs:- targets: ['10.1.234.110:9100']- job_name: 'node-2'static_configs:- targets: ['10.1.234.111:9100']- job_name: 'node-3'static_configs:- targets: ['10.1.234.112:9100']- job_name: 'pushgateway'static_configs:- targets: ['10.1.234.110:9091']
[root@ansible-11 reload_Promethes]# cat node.yml # groups：组告警groups:# name：组名。报警规则组名称- name: general.rules# rules：定义角色rules:# alert：告警名称。 任何实例5分钟内无法访问发出告警- alert: NodeFilesystemUsage_disk# expr：表达式。 获取磁盘使用率 大于百分之80 触发expr: 100 - (node_filesystem_free_bytes{mountpoint="/",fstype=~"ext4|xfs"} / node_filesystem_size_bytes{fstype=~"ext4|xfs"} * 100) > 80# for：持续时间。 表示持续一分钟获取不到信息，则触发报警。0表示不使用持续时间for: 1m# labels：定义当前告警规则级别labels:# severity: 指定告警级别。severity: warning# annotations: 注释 告警通知annotations:# 调用标签具体指附加通知信息summary: "Instance {{ $labels.instance }} ：{{ $labels.mountpoint }} 分区使用率过高" # 自定义摘要description: "{{ $labels.instance }} ： {{ $labels.job }} ：{{ $labels.mountpoint }} 这个分区使用大于百分之80% (当前值：{{ $value }})" # 自定义具体描述
[root@ansible-11 promethes]# cat inster_promethes.sh #!/bin/bashecho '--------我只用了3台搭建，增加node节点请自己加---------'read -p '请输入promethesmaster的ip地址:' xread -p '请输入promethes node1的ip地址:' yread -p '请输入promethes node2的ip地址:' zfor i in `grep -r 10.1.234.110 . |awk -F : '{print $1}' |uniq`dosed -i "s#10.1.234.110#$x#g" $idone for j in `grep -r 10.1.234.111 . |awk -F : '{print $1}' |uniq`dosed -i "s#10.1.234.111#$y#g" $jdone for k in `grep -r 10.1.234.112 . |awk -F : '{print $1}' |uniq`dosed -i "s#10.1.234.112#$z#g" $kdone ansible-playbook -i hosts Prometheus.yamlansible-playbook -i hosts node_exporter.yamlansible-playbook -i hosts Grafana.yamlansible-playbook -i hosts Alertmanager.yamlansible-playbook -i hosts Pushgateway.yamlecho 'promethes安装完成'cd reload_Promethesansible-playbook -i hosts reload_promethes.yamlecho '规则添加完成，请导入模板'echo '-----------修改报警的微信不要用我的--------------'
##########################################################################
直接执行 ansible-palybook -i hostinster_promethes.sh 大概4-5分钟安装完成，测试报警完成
看起来有点繁琐，后面再改进
###################################################################################
企业微信告警配置
[root@ansible-11 ~]# cat ansible/prometheus/conf/alertmanager.yml global:# 每2分钟检查一次是否恢复resolve_timeout: 2m# SMTP的相关配置smtp_smarthost: ':25'smtp_from: '18802676921@'smtp_auth_username: '18802676921@'smtp_auth_password: '123qqq...A'# 自定义 通知的模板的 目录 或者 文件.#templates:# - '/usr/local/prometheus/alertmanager/template/wechat.tmpl'# 路由树的根节点, 每个传进来的报警从这里开始.route:# 将传入的报警中有这些标签的分为一个组.# 比如, cluster=A 和 alertname=LatencyHigh 会分成一个组.group_by: ['alertname_wechat']# 指分组创建多久后才可以发送压缩的警报，也就是初次发警报的延时.# 这样会确保第一次通知的时候, 有更多的报警被压缩在一起.group_wait: 10s# 当第一个通知发送，等待多久发送压缩的警报group_interval: 10s# 默认的接收器receiver: 'wechat'# 如果报警发送成功, 等待多久重新发送一次repeat_interval: 1hreceivers:#SMTP配置- name: 'email'email_configs:- to: '582167559@'send_resolved: true- name: 'wechat'wechat_configs:- corp_id: 'wwab37c47350318435'to_party: '2'agent_id: '1000002'api_secret: 'ti3TXKv7sdZs6r7EUZdgpRoUgjR1ne97R8KSYTtPpDY'send_resolved: true
####################################################
放入周期性计划任务
[root@test_dc_rpdns_com ~]# crontab -lSHELL=/bin/shPATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin#0 */1 * * * /win/sh/rm_laji_log_data.sh*/5 * * * * /root/node_exporter_shell.sh*/5 * * * * /root/port.sh
[root@test_dc_rpdns_com ~]# cat port.sh #!/bin/bash#获取主机名，常传输到Prometheus标签以主机名instance_name=`hostname -f | cut -d'.' -f1`#判断主机名不能是localhost不然发送过的数据不知道是那个主机的 if [ $instance_name == "localhost" ];thenecho "Hostname must not localhost"exit 1fi#自定义key，在Prometheus即可使用key查询label="node_port_8500" node_port_8500=`ss -nutlp | grep 8500 |wc -l`echo "$label $node_port_8500" | curl --data-binary @- http://10.1.234.110:9091/metrics/job/pushgateway/instance/$instance_namelabel="node_port_9201" node_port_9201=`ss -nutlp | grep 9201 |wc -l`echo "$label $node_port_9201" | curl --data-binary @- http://10.1.234.110:9091/metrics/job/pushgateway/instance/$instance_namelabel="node_port_5601" node_port_5601=`ss -nutlp | grep 5601 |wc -l`echo "$label $node_port_5601" | curl --data-binary @- http://10.1.234.110:9091/metrics/job/pushgateway/instance/$instance_name
如果觉得《Prometheus+grafana+node_exporter+pushgateway+alertmanager企业微信告警》对你有帮助，请点赞、收藏，并留下你的观点哦！
