Monitoring-K8s资源观测工具Kubewatch
Monitoring-K8s资源观测工具Kubewatch
一、Kubewatch介绍
Kubewatch是一款Kubernetes 资源观测工具，实时监控 Kubernetes 集群中各种资源的新建、更新和删除，并实时通知到各种协作软件/聊天软件，目前支持的通知渠道有：
slack
hipchat
mattermost
flock
webhook
二、Helm部署kubewatch
官网github（bitnami） ：https://github.com/bitnami-labs/kubewatch
调配 chart values ：监视的ns、nodeSelector、webhook url
helm3 部署
helm install kubewatch ./kubewatch --set='rbac.create=true,resourcesToWatch.pod=true,resourcesToWatch.daemonset=true' --namespace monitoring helm3 uninstall kubewatch --namespace monitoring# kubectl get pods -n monitoringkubewatch-5994fbcb59-qt9cz 1/1 Running 0 20h
三、Webhook 配置例子
我这里使用Python Flask编写了一个 webhook proxy 将消息格式化并过滤后推送至Worktile；
代码仅作为范例参考，也可直接推送到DingDing ，参考地址 ：https://segmentfault.com/a/1190000021219010?utm_source=tag-newest
# cat hook-server/flask-message.py#!/usr/bin/python3from flask import Flaskfrom flask import request, jsonify, abortimport datetimeimport jsonimport requests# 日志import loggingfrom logging import handlersclass Logger(object):
level_relations = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'crit': logging.CRITICAL
} # 日志级别关系映射
def __init__(self, filename, level='info', when='D', backCount=3,
fmt='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'):
self.logger = logging.getLogger(filename)
format_str = logging.Formatter(fmt) # 设置日志格式
self.logger.setLevel(self.level_relations.get(level)) # 设置日志级别
sh = logging.StreamHandler() # 往屏幕上输出
sh.setFormatter(format_str) # 设置屏幕上显示的格式
th = handlers.TimedRotatingFileHandler(filename=filename, when=when, backupCount=backCount,
encoding='utf-8') # 往文件里写入#指定间隔时间自动生成文件的处理器
# midnight 每天凌晨
th.setFormatter(format_str) # 设置文件里写入的格式
self.logger.addHandler(sh) # 把对象加到logger里
self.logger.addHandler(th)# worktile webhookwt_webhook_url = 'https://hook.toptops.com/incoming/asdasdnashjdbasjdbasjdbajsd'def worktile_webhook(hookurl=wt_webhook_url, string=None):
print('send message to worktile webhook ')
headers = { "Content-Type": "application/json; charset=UTF-8", # "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36",
}
Data = { "text": string, # "safe": "0"
}
print(Data, 'worktile')
response = requests.post(hookurl, data=json.dumps(Data), headers=headers).text
print(response)# 字段格式化class Json_sik(object): @staticmethod
def js(payload):
# print(payload)
"""
format :
pod - 删除
时间 - 2021-03-17T10:18:24
空间 - default
详情 - A `pod` in namespace `` has been `updated`:`default/web-1`
"""
# 格式化字符串
kind = payload['eventmeta']['kind']
reason = payload['eventmeta']['reason'] # time = payload['time']
time = datetime.datetime.now().replace(microsecond=0)
namespace = payload['eventmeta']['name']
msg = payload['text'] if str(reason) == str("deleted") or str(reason) == str("created"): # send_text=
text = 'Action : `{}`-`{}`\nTime : {}\nMessage : {}'.format(kind, reason, time, msg) # print(text)
worktile_webhook(hookurl=wt_webhook_url, string=text) return text
app = Flask(__name__)@app.route('/kubewatch', methods=['GET', 'POST'])def webhook():
if request.method == 'GET':
logging.debug('Get Method not implemented') return jsonify({'status': 'success'}), 200
elif request.method == 'POST':
payload = request.json
sim = Json_sik().js(payload=payload) return jsonify({'status': 'success'}), 200
else:
abort(400)if __name__ == '__main__':
app.run(host='0.0.0.0', port='10010')测试，可以临时删除一个监视的ns下的pod，查看kubewatch日志，是否获取到事件
time="2021-03-18T11:27:33Z" level=info msg="Processing add to pod: prod/wt-rd-admin-prod-6d499bf6df-ml9sf" pkg=kubewatch-pod time="2021-03-18T11:27:33Z" level=info msg="Processing add to pod: prod/pc-jira-importer-sky-prod-886b6ff65-wbkh7" pkg=kubewatch-pod time="2021-03-18T11:27:33Z" level=info msg="Processing add to pod: prod/wt-sky-portal-prod-8476b457bf-lcs8h" pkg=kubewatch-pod time="2021-03-18T11:27:33Z" level=info msg="Processing add to pod: prod/wt-rd-devops-rpc-prod-5fd46465d8-9562d" pkg=kubewatch-pod
优化消息冗余过滤，仅有资源删除、创建时推送消息；
from flask import Flaskfrom flask import request, jsonify, abort# import simplejson# import subprocessimport datetimeimport jsonimport requestsimport re# 日志import loggingfrom logging import handlersclass Logger(object):
level_relations = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'crit': logging.CRITICAL
} # 日志级别关系映射
def __init__(self, filename, level='info', when='D', backCount=3,
fmt='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'):
self.logger = logging.getLogger(filename)
format_str = logging.Formatter(fmt) # 设置日志格式
self.logger.setLevel(self.level_relations.get(level)) # 设置日志级别
sh = logging.StreamHandler() # 往屏幕上输出
sh.setFormatter(format_str) # 设置屏幕上显示的格式
th = handlers.TimedRotatingFileHandler(filename=filename, when=when, backupCount=backCount,
encoding='utf-8') # 往文件里写入#指定间隔时间自动生成文件的处理器
# 实例化TimedRotatingFileHandler
# interval是时间间隔，backupCount是备份文件的个数，如果超过这个个数，就会自动删除，when是间隔的时间单位，单位有以下几种：
# S 秒
# M 分
# H 小时、
# D 天、
# W 每星期（interval==0时代表星期一）
# midnight 每天凌晨
th.setFormatter(format_str) # 设置文件里写入的格式
self.logger.addHandler(sh) # 把对象加到logger里
self.logger.addHandler(th)# worktile webhookwt_webhook_url = 'https://hook.toptops.com/incoming/asdasdnashjdbasjdbasjdbajsd'def worktile_webhook(hookurl=wt_webhook_url, string=None):
print('send message to worktile webhook ')
headers = { "Content-Type": "application/json; charset=UTF-8", # "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36",
}
Data = { "text": string, # "safe": "0"
}
print(Data, 'worktile')
response = requests.post(hookurl, data=json.dumps(Data), headers=headers).text
print(response)
create_pods = []
delete_pods = []# 字段格式化class Json_sik(object): @staticmethod
def js(payload):
# print(payload)
"""
format :
pod - 删除
时间 - 2021-03-17T10:18:24
空间 - default
详情 - A `pod` in namespace `` has been `updated`:`default/web-1`
"""
# print(payload)
global create_pods, delete_pods # 格式化字符串
kind = payload['eventmeta']['kind']
reason = payload['eventmeta']['reason'] # time = payload['time']
time = datetime.datetime.now().replace(microsecond=0)
namespace = payload['eventmeta']['name']
msg = payload['text'] # print(type(reason), reason)
if str(reason) == str("deleted") or str(reason) == str("created"): # for i in ns_and_pod_list:
# if i not in new_list:
# new_list.append(i)
# print(new_list)
# print(ns_and_pod_list, 'ns_and_pod_list 列表展示')
print(payload) if str(reason) == str("created"): try:
print('1~~~~~~~')
reason = payload['eventmeta']['reason'] # 动作
msg_str = payload['text']
ns_and_pod = msg_str[msg_str.rfind('\n'):]
ns_and_pod = re.sub(r'[:/\\?*“”<>|`/\n]', '', ns_and_pod)
new_str = '{}-{}'.format(ns_and_pod, reason) list = (new_str.split('-'))
new_str_c = '{}-{}-{}'.format(list[0], list[1], list[2])
create_pods.append(new_str_c) except Exception as e:
print('2~~~~~~~~')
reason = payload['eventmeta']['reason'] # 动作
msg_str = payload['text']
ns_and_pod = msg_str[msg_str.rfind('\n'):]
ns_and_pod = re.sub(r'[:/\\?*“”<>|`/\n]', '', ns_and_pod)
new_str_c = '{}-{}'.format(ns_and_pod, reason)
create_pods.append(new_str_c)
print(create_pods) if len(create_pods) < 1:
print(create_pods, '创建pods')
text = 'Action : `{}`-`{}`\nTime : {}\nMessage : {}'.format(kind, reason, time, msg)
worktile_webhook(hookurl=wt_webhook_url, string=text) return text elif create_pods.count(new_str_c) <= 1:
print(create_pods, '创建pods')
text = 'Action : `{}`-`{}`\nTime : {}\nMessage : {}'.format(kind, reason, time, msg)
worktile_webhook(hookurl=wt_webhook_url, string=text) return text else: try:
print('3~~~~~~~')
reason = payload['eventmeta']['reason'] # 动作
msg_str = payload['text']
ns_and_pod = msg_str[msg_str.rfind('/'):]
ns_and_pod = re.sub(r'[:/\\?*“”<>|`]', '', ns_and_pod)
new_str = '{}-{}'.format(ns_and_pod, reason) list = (new_str.split('-'))
new_str_d = '{}-{}-{}'.format(list[0], list[1], list[2])
delete_pods.append(new_str_d) except Exception as e:
print('4~~~~~~~')
reason = payload['eventmeta']['reason'] # 动作
msg_str = payload['text']
ns_and_pod = msg_str[msg_str.rfind('/'):]
ns_and_pod = re.sub(r'[:/\\?*“”<>|`]', '', ns_and_pod)
new_str_d = '{}-{}'.format(ns_and_pod, reason)
delete_pods.append(new_str_d)
print(delete_pods) if len(delete_pods) < 1:
print(delete_pods, '删除pods')
text = 'Action : `{}`-`{}`\nTime : {}\nMessage : {}'.format(kind, reason, time, msg)
worktile_webhook(hookurl=wt_webhook_url, string=text) return text elif delete_pods.count(new_str_d) <= 1:
print(delete_pods, '删除pods')
text = 'Action : `{}`-`{}`\nTime : {}\nMessage : {}'.format(kind, reason, time, msg)
worktile_webhook(hookurl=wt_webhook_url, string=text) return text# 定时清理import scheduleimport timedef job():
create_pods.clear()
delete_pods.clear()
app = Flask(__name__)@app.route('/kubewatch', methods=['GET', 'POST'])def webhook():
if request.method == 'GET':
logging.debug('Get Method not implemented') return jsonify({'status': 'success'}), 200
elif request.method == 'POST':
schedule.every().hour.do(job)
payload = request.json
sim = Json_sik().js(payload=payload) return jsonify({'status': 'success'}), 200
else:
abort(400)if __name__ == '__main__':
app.run(host='0.0.0.0', port='10010')__EOF__
本文作者：Brains
本文链接：https://www.cnblogs.com/dai-zhe/p/14826003.html