[kubernetes]-k8s安装alertmanager和prometheus-webhook-dingtalk
作者：互联网
安装alertmanager
创建存放数据及插件的文件夹
# 在指定的node上创建文件夹
mkdir -p /data/k8s/alertmanager
chown -R 65534:root alertmanager
创建alertmanager-cm.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
name: alertmanager-config
namespace: kube-ops
data:
alertmanager.yml: |-
# 全局配置项
global:
resolve_timeout: 5m # 处理超时时间，默认为5min
smtp_from: 'it@hz-health.cn'
smtp_smarthost: 'smtp.exmail.qq.com:465'
#smtp_smarthost: 'smtp.aliyun.com:465'
smtp_auth_username: 'it@hz-health.cn'
smtp_auth_password: 'yourpassword'
smtp_require_tls: false
# 定义路由树信息
route:
group_by: [alertname] # 报警分组依据
receiver: ops_notify # 设置默认接收人
group_wait: 30s # 最初即第一次等待多久时间发送一组警报的通知
group_interval: 60s # 在发送新警报前的等待时间 下一次报警开车时间
repeat_interval: 1h # 重复发送告警时间。默认1h 第一次报警时间为group_interval 重复报警的时间为group_interval+repeat_interval
routes:
- receiver: ops_notify # 基础告警通知
group_wait: 10s
match_re:
alertname: 实例存活告警|磁盘使用率告警 # 匹配告警规则中的名称发送
- receiver: info_notify # 消息告警通知
group_wait: 10s
match_re:
alertname: 内存使用率告警|CPU使用率告警
# 定义基础告警接收者
receivers:
- name: ops_notify
webhook_configs:
- url: http://prometheus-webhook-dingtalk.ihaozhuo.com/dingtalk/ops_dingding/send
send_resolved: true # 警报被解决之后是否通知
email_configs:
# - to: '423308591@qq.com‘ 如果两个邮箱中间需要空格
- to: 'it@hz-health.cn, xujiamin@hz-health.cn'
send_resolved: true
# 定义消息告警接收者
- name: info_notify
webhook_configs:
- url: http://prometheus-webhook-dingtalk.ihaozhuo.com/dingtalk/info_dingding/send
send_resolved: true
# 一个inhibition规则是在与另一组匹配器匹配的警报存在的条件下，使匹配一组匹配器的警报失效的规则。两个警报必须具有一组相同的标签。
inhibit_rules:
- source_match:
severity: 'critical'
target_match:
severity: 'warning'
equal: ['alertname', 'dev', 'instance']
创建alertmanager-deploy.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: alertmanager
namespace: kube-ops
labels:
app: alertmanager
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: alertmanager
minReadySeconds: 0
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 1
template:
metadata:
labels:
app: alertmanager
spec:
affinity: #亲和性的调度设置
nodeAffinity: #策略为节点亲和性
requiredDuringSchedulingIgnoredDuringExecution: #亲和性的硬策略
nodeSelectorTerms: #这里不再使用nodeselector，使用这个参数可以进行相对简单的逻辑运算
- matchExpressions: #匹配表达式
- key: kubernetes.io/hostname #具体匹配规则(可以通过kubectl get node --show-labels找到相应规则)
operator: In #不在，简单的来说就是不在k8s-04节点
values:
- prod-k8s-n006
containers:
- name: alertmanager
image: prom/alertmanager:v0.21.0
args:
- "--config.file=/etc/alertmanager/alertmanager.yml"
- "--storage.path=/alertmanager/data"
resources:
limits:
memory: "256Mi"
cpu: "100m"
readinessProbe:
tcpSocket:
port: 9093
initialDelaySeconds: 30
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 9093
initialDelaySeconds: 60
periodSeconds: 10
ports:
- containerPort: 9093
protocol: TCP
name: http
volumeMounts:
- name: data
mountPath: /alertmanager/data
- name: config-volume
mountPath: "/etc/alertmanager/alertmanager.yml"
subPath: alertmanager.yml
volumes:
- name: data
hostPath:
# 宿主上目录位置
path: /data/k8s/alertmanager
type: DirectoryOrCreate
- configMap:
name: alertmanager-config
name: config-volume
创建alertmanager-svc.yaml
---
apiVersion: v1
kind: Service
metadata:
name: alertmanager
namespace: kube-ops
labels:
app: alertmanager
spec:
selector:
app: alertmanager
type: NodePort
ports:
- port: 80
protocol: TCP
targetPort: 9093
创建alertmanager-ingress.yaml
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
# 通过添加下面的annotations 来开启白名单
# 关闭80强制跳转443 为ingress配置增加注解（annotations）：nginx.ingress.kubernetes.io/ssl-redirect: 'false' 就可以禁止http强制跳转至https
annotations:
#nginx.ingress.kubernetes.io/whitelist-source-range: "60.191.70.64/29, xx.xxx.0.0/16"
nginx.ingress.kubernetes.io/ssl-redirect: 'false'
name: prod-alertmanager
namespace: kube-ops
spec:
rules:
- host: alertmanager.ihaozhuo.com
http:
paths:
- path: /
backend:
serviceName: alertmanager
servicePort: 80
运行alertmanager
kubectl apply -f ./
安装prometheus-webhook-dingtalk
先看一下prometheus-webhook-dingtalk的参数 打算通过configmap的方式挂载config.yml并开启ui。
之前都是直接启动的时候通过命令/srv/alertmanager/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk --ding.profile=ops_dingding=https://oapi.dingtalk.com/robot/send?access_token=1234567890 --web.enable-ui 启动，研究这个config花了点时间
制作一个prometheus-webhook-dingtalk的镜像
创建Dockerfile
FROM centos:centos7.4.1708
MAINTAINER PDABC Enterprise Container Images <jiaminxu@hz-health.cn>
ADD prometheus-webhook-dingtalk /etc/prometheus-webhook-dingtalk
RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
WORKDIR /etc/prometheus-webhook-dingtalk
#公开端口
EXPOSE 8060
USER root
#设置启动命令
ENTRYPOINT ["sh", "-c","/etc/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk"]
# 想写死的话 也是可以的。
#ENTRYPOINT ["sh", "-c","/etc/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk --ding.profile=ops_dingding=https://oapi.dingtalk.com/robot/send?access_token=1234567890 --web.enable-ui "]
编译并上床镜像
docker build -t registry.cn-shanghai.aliyuncs.com/yjk-datag/prometheus-webhook-dingtalk:v3 .
docker push registry.cn-shanghai.aliyuncs.com/yjk-datag/prometheus-webhook-dingtalk:v3
创建prometheus-webhook-dingtalk-cm.yaml
https://github.com/timonwong/prometheus-webhook-dingtalk
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-webhook-dingtalk-config
namespace: kube-ops
data:
config.yml: |
targets:
# 这个是我用到的
ops_dingding:
url: https://oapi.dingtalk.com/robot/send?access_token=10bda98979ae2155b6822b699cde1841d4fbd8514c0441bbbb4485caddf3a388x #这是是钉钉机器人的webhook
webhook2:
url: https://oapi.dingtalk.com/robot/send?access_token=10bda98979ae2155b6822b699cde1841d4fbd8514c0441bbbb4485caddf3a388x #这是是钉钉机器人的webhook
webhook_legacy:
url: https://oapi.dingtalk.com/robot/send?access_token=10bda98979ae2155b6822b699cde1841d4fbd8514c0441bbbb4485caddf3a388x #这是是钉钉机器人的webhook
# Customize template content
message:
# Use legacy template
title: '{{ template "default.title" . }}'
text: '{{ template "default.content" . }}'
webhook_mention_all:
url: https://oapi.dingtalk.com/robot/send?access_token=10bda98979ae2155b6822b699cde1841d4fbd8514c0441bbbb4485caddf3a388x #这是是钉钉机器人的webhook
mention:
all: true
webhook_mention_users:
url: https://oapi.dingtalk.com/robot/send?access_token=10bda98979ae2155b6822b699cde1841d4fbd8514c0441bbbb4485caddf3a388x #这是是钉钉机器人的webhook
mention:
mobiles: ['15xxxxxxxx0']
创建prometheus-webhook-dingtalk-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus-webhook-dingtalk
namespace: kube-ops
labels:
app: prometheus-webhook-dingtalk
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: prometheus-webhook-dingtalk
minReadySeconds: 0
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 1
template:
metadata:
labels:
app: prometheus-webhook-dingtalk
spec:
affinity: #亲和性的调度设置
nodeAffinity: #策略为节点亲和性
requiredDuringSchedulingIgnoredDuringExecution: #亲和性的硬策略
nodeSelectorTerms: #这里不再使用nodeselector，使用这个参数可以进行相对简单的逻辑运算
- matchExpressions: #匹配表达式
- key: kubernetes.io/hostname #具体匹配规则(可以通过kubectl get node --show-labels找到相应规则)
operator: In #不在，简单的来说就是不在k8s-04节点
values:
- prod-k8s-n006
containers:
- name: prometheus-webhook-dingtalk
image: registry.cn-shanghai.aliyuncs.com/yjk-datag/prometheus-webhook-dingtalk:v3
args:
# - "--web.enable-ui" # 这个web ui貌似在这里添加不生效。我也用不上 可以在构建镜像的时候加上试试
- "----config.file=/etc/prometheus-webhook-dingtalk/config.yml"
resources:
limits:
memory: "256Mi"
cpu: "100m"
readinessProbe:
tcpSocket:
port: 8060
initialDelaySeconds: 30
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 8060
initialDelaySeconds: 60
periodSeconds: 10
ports:
- containerPort: 8060
protocol: TCP
name: http
volumeMounts:
- name: config-volume
mountPath: "/etc/prometheus-webhook-dingtalk/config.yml"
subPath: config.yml
volumes:
- configMap:
name: prometheus-webhook-dingtalk-config
name: config-volume
创建prometheus-webhook-dingtalk-svc.yaml
---
apiVersion: v1
kind: Service
metadata:
name: prometheus-webhook-dingtalk
namespace: kube-ops
labels:
app: prometheus-webhook-dingtalk
spec:
selector:
app: prometheus-webhook-dingtalk
type: NodePort
ports:
- port: 80
protocol: TCP
targetPort: 8060
创建prometheus-webhook-dingtalk-ingress.yaml
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
# 通过添加下面的annotations 来开启白名单
# 关闭80强制跳转443 为ingress配置增加注解（annotations）：nginx.ingress.kubernetes.io/ssl-redirect: 'false' 就可以禁止http强制跳转至https
annotations:
#nginx.ingress.kubernetes.io/whitelist-source-range: "60.191.70.64/29, xx.xxx.0.0/16"
nginx.ingress.kubernetes.io/ssl-redirect: 'false'
name: prod-prometheus-webhook-dingtalk
namespace: kube-ops
spec:
rules:
- host: prometheus-webhook-dingtalk.ihaozhuo.com
http:
paths:
- path: /
backend:
serviceName: prometheus-webhook-dingtalk
servicePort: 80
运行prometheus-webhook-dingtalk
kubectl apply -f ./
测试告警
修改prometheus-webhook-dingtalk的svc为headless
好处是可以不用域名访问 可以使用prometheus-webhook-dingtalk-headless.kube-ops.svc.cluster.local:8060 来请求
创建prometheus-webhook-dingtalk-headless-svc.yaml
---
apiVersion: v1
kind: Service
metadata:
name: prometheus-webhook-dingtalk-headless
namespace: kube-ops
labels:
app: prometheus-webhook-dingtalk
spec:
ports:
- port: 80
name: http
# clusterIP 设置为 None
clusterIP: None
selector:
app: prometheus-webhook-dingtalk
kubectl apply -f ./
应用之后 通过其他容器测试
标签：alertmanager,dingtalk,kubernetes,webhook,prometheus,com,name 来源： https://blog.csdn.net/xujiamin0022016/article/details/115211919