python实现采集yarn队列资源使用百分比，暴露为prometheus的exporter格式，进行yarn队列资源监控

python脚本实现，访问官方提供的地址，获取队列资源使用指标，并将指标进行json解析，之后再转换为prometheus认识的数据格式，暴露于端口下

#coding=utf-8
'''
通过访问官方提供的yarn restful api界面,获取yarn资源指标
分析json取的所需指标
以pormetheus的数据格式将指标暴露在指定端口下

入参：yarn的ip:port yarn队列数量 export暴露的机器IP(建议为本机IP) export暴露的端口号
'''

import prometheus_client
from prometheus_client import Gauge
from prometheus_client.core import CollectorRegistry
from flask import Response, Flask

import json
import urllib.request
import sys


hjson = json.loads('{}')

app = Flask(__name__)

REGISTRY = CollectorRegistry(auto_describe=False)

def getMetric(ipport,queue_num):

    html = urllib.request.urlopen(r'http://'+ipport+'/ws/v1/cluster/scheduler')

    data = html.read().decode("utf-8")

    global hjson
    hjson = json.loads(data)
    for i in range(0,queue_num):
     queuename = hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'][i]['queueName']
     usevCores = hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'][i]['usedResources']['vCores']
     usermemory = hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'][i]['usedResources']['memory']
     maxvCores = hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'][i]['maxResources']['vCores']
     maxmemory = hjson['scheduler']['schedulerInfo']['rootQueue']['childQueues']['queue'][i]['maxResources']['memory']
     usermemory_percentage = 0.0000
     usevCores_percentage = 0.0000
     if(maxvCores != 0):
      usevCores_percentage = round(usevCores/maxvCores,4)
     if(maxmemory != 0):
      usermemory_percentage = round(usermemory/maxmemory,4)
     queuename = str(queuename).replace(".","_")
     registerMetric(queuename,usermemory_percentage,usevCores_percentage)



def registerMetric(queueName,usermemory_percentage,usevCores_percentage):
    usememory_percentage_temp = Gauge(
        queueName+'_vcores_percentage',
        queueName+'_vcores_percentage',
        registry=REGISTRY)
    usevCores_percentage_temp = Gauge(
        queueName+'_memory_percentage',
        queueName+'_memory_percentage',
        registry=REGISTRY)
    usememory_percentage_temp.set(usermemory_percentage)
    usevCores_percentage_temp.set(usevCores_percentage)



@app.route('/metrics')
def r_value():
    return Response(prometheus_client.generate_latest(REGISTRY),
                    mimetype="text/plain")


@app.route('/')
def index():
    return "namenode jmx metric address: ip:port/metrics"



if __name__ == "__main__":
    yarn_ipport = str(sys.argv[1])
    queue_num = int(sys.argv[2])
    showIp = str(sys.argv[3])
    showPort = int(sys.argv[4])

    # yarn_ipport = "127.0.0.1:8001"
    # showIp = "localhost"
    # showPort = 8000
    # queue_num = 38
    getMetric(yarn_ipport,queue_num)
    app.run(host=showIp,port=showPort,debug=True)

标签：队列,hjson,yarn,queue,python,import,percentage,usevCores
来源： https://blog.csdn.net/qq_40498209/article/details/114666028