其他分享
首页 > 其他分享> > check OpenStack server

check OpenStack server

作者:互联网

#!/bin/bash   source /root/admin-openrc.sh   VIP=$(cat /etc/hosts | awk '/# For openstack management/,/# For ceph management/ { print }' | egrep -v '# For ceph management|# For openstack management' | awk '{print $1}') CONTROLER_IP_LIST=$(nova service-list | awk -F '|' '/nova-scheduler/ {print $4}'| sort -u) IP_LIST=$(cat /etc/hosts | awk '/# For management/, /# For openstack management/ { print }' | egrep -v '# For management|# For openstack management' | awk '{print $1}') SSH_USERNAME=jitstack SSH_KEY=/home/jitstack/.ssh/id_rsa SSH_PARAMETER="ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no ${SSH_USERNAME}"     SHUTDOWN_VIRTBR() {   for i in $IP_LIST   do     virbr=$(${SSH_PARAMETER}@$i sudo virsh net-list |awk '/default/ {print $1}')     if [[ ${virbr} == default ]]     then       ${SSH_PARAMETER}@$i sudo virsh net-destroy default > /dev/null 2>&1       ${SSH_PARAMETER}@$i sudo virsh net-undefine default > /dev/null 2>&1     fi   done }     CHECK_HARDWARE() {   for i in $IP_LIST   do     HARDWARE_TEST=$(${SSH_PARAMETER}@$i sudo dmesg -T|egrep -i 'error|failed' 2>/dev/null)     if [[ -n ${HARDWARE_TEST} ]]     then       HARDWARE_WARNING_LIST[a]=${i}       let a++     fi   done   if [[ -n ${HARDWARE_WARNING_LIST[*]} ]]   then     echo -e "HARDWARE \033[33mWARNING\033[0m ${HARDWARE_WARNING_LIST[*]}"   else     echo -e "HARDWARE \033[32mOK     \033[0m"   fi }     CHECK_NTP() {   for i in $IP_LIST   do     NTP_TEST=$(${SSH_PARAMETER}@$i sudo chronyc -n sources 2>/dev/null | awk 'NR>3 { print }')     if [[ -z ${NTP_TEST} ]]     then       NTP_ERROR_LIST[a]=${i}       let a++     fi   done   if [[ -n ${NTP_ERROR_LIST[*]} ]]   then     echo -e "NTP      \033[31mERROR  \033[0m ${NTP_ERROR_LIST[*]}"   else     echo -e "NTP      \033[32mOK     \033[0m"   fi }     CHECK_CPU() {   for i in $IP_LIST   do     CPU_TEST=$(${SSH_PARAMETER}@$i sudo vmstat 2>/dev/null| awk 'NR>2 {print $15}')     if [[ ${CPU_TEST} -le 30 ]]     then       CPU_WARNING_LIST[a]=${i}       let a++     fi   done   if [[ -n ${CPU_WARNING_LIST[*]} ]]   then     echo -e "CPU      \033[33mWARNING\033[0m ${CPU_WARNING_LIST[*]}"   else     echo -e "CPU      \033[32mOK     \033[0m"   fi }     CHECK_MEM() {   for i in $IP_LIST   do     MEM_TEST=$(${SSH_PARAMETER}@$i sudo free -g 2>/dev/null| awk 'NR==2 {print $3}')     if [[ ${MEM_TEST} -le 20 ]]     then       MEM_WARNING_LIST[a]=${i}       let a++     fi   done   if [[ -n ${MEM_WARNING_LIST[*]} ]]   then     echo -e "MEM      \033[33mWARNING\033[0m ${MEM_WARNING_LIST[*]}"   else     echo -e "MEM      \033[32mOK     \033[0m"   fi }     CHECK_DISK() {   for i in $IP_LIST   do     DISK_TEST=$(${SSH_PARAMETER}@$i sudo df -Th 2>/dev/null| awk 'NR>1 {print $6}'| sed 's/%//g')     for j in ${DISK_TEST}     do       if [[ ${j} -ge 80 ]]       then         DISK_WARNING_TEST_LIST[a]=${i}         let a++       fi     done     if [[ -n ${DISK_WARNING_TEST_LIST[@]} ]]     then       DISK_WARNING_LIST[b]=${i}       let b++     fi   done   if [[ -n ${DISK_WARNING_LIST[@]} ]]   then     echo -e "DISK     \033[33mWARNING\033[0m ${DISK_WARNING_LIST[*]}"   else     echo -e "DISK     \033[32mOK     \033[0m"   fi }     CHECK_NETWORK() {   ALL_VIPS=$(cat /etc/keepalived/keepalived.conf | awk '/virtual_ipaddress/, /track_script/ {print $1}'| egrep -v 'virtual_ipaddress|}|track_script|^$')   NET_ADDR=$(ip route | awk 'NR>1 {print $1}'| awk -F '0/' '{print $1}')   HOST_ADDR=$(cat /etc/hosts | awk '/# For management/, /# For openstack management/ { print }' | egrep -v '# For management|# For openstack management' | awk '{print $1}'| awk -F '.' '{print $NF}')   for i in ${NET_ADDR}   do     for j in ${HOST_ADDR}     do       ALL_IP_LIST[a]=${i}${j}       let a++     done   done   sudo fping ${ALL_IP_LIST[@]} ${ALL_VIPS[@]} 2>/dev/null > /tmp/fping.log   NETWORK_ERROR_LIST=$(cat /tmp/fping.log | grep unreachable | awk '{print $1}')   if [[ -n ${NETWORK_ERROR_LIST} ]]   then     echo -e "NETWORK  \033[31mERROR  \033[0m" ${NETWORK_ERROR_LIST[@]}   else     echo -e "NETWORK  \033[32mOK     \033[0m"   fi }     CHECK_MEMCACHE() {   for i in ${CONTROLER_IP_LIST}   do     MEMCACHE_TEST=$(${SSH_PARAMETER}@$i sudo netstat -tnlp 2>/dev/null |grep memcached)     if [[ -z ${MEMCACHE_TEST} ]]     then       MEMCACHE_ERROR_LIST[a]=${i}       let a++     fi   done   if [[ -n ${MEMCACHE_ERROR_LIST[@]} ]]   then     echo -e "MEMCACHE \033[31mERROR  \033[0m ${MEMCACHE_ERROR_LIST[@]}"   else     echo -e "MEMCACHE \033[32mOK     \033[0m"   fi }     CHECK_MYSQL_SLAVE() {   for i in ${CONTROLER_IP_LIST}   do     Slave_IO_Running=$(${SSH_PARAMETER}@$i 'sudo mysql -e "show slave status\G;"' | awk -F ': ' '/Slave_IO_Running/ {print $2}')     Slave_SQL_Running=$(${SSH_PARAMETER}@$i 'sudo mysql -e "show slave status\G;"' | awk -F ': ' '/Slave_SQL_Running/ {print $2}')     if [[ -z ${Slave_IO_Running} &&  -z ${Slave_SQL_Running} ]]     then       :     elif [[ ${Slave_IO_Running} != "Yes" || ${Slave_SQL_Running} != "Yes" ]]     then       MYSQL_ERROR_LIST[a]=$i       let a++     fi   done   if [[ -n ${MYSQL_ERROR_LIST[@]} ]]   then     echo -e "MYSQL    \033[31mERROR  \033[0m ${MYSQL_ERROR_LIST[@]}"   else     echo -e "MYSQL    \033[32mOK     \033[0m"   fi }     CHECK_RABBITMQ() {   for i in ${CONTROLER_IP_LIST}   do     status=$(${SSH_PARAMETER}@$i sudo rabbitmqctl cluster_status 2>/dev/null | grep running_nodes)     if [[ -z ${status} ]]     then       ${SSH_PARAMETER}@$i sudo systemctl restart rabbitmq-server 2>/dev/null       status_agagin=$(${SSH_PARAMETER}@$i sudo rabbitmqctl cluster_status 2>/dev/null | grep running_nodes)       if [[ -z ${status_agagin} ]]       then         RABBITMQ_ERROR_LIST[a]=${i}         let a++       fi     fi   done   if [[ -n ${RABBITMQ_ERROR_LIST[@]} ]]   then     echo -e "RABBITMQ \033[31mERROR  \033[0m ${RABBITMQ_ERROR_LIST[@]}"   else     echo -e "RABBITMQ \033[32mOK     \033[0m"   fi }     CHECK_REDIS() {   for i in ${CONTROLER_IP_LIST}   do     status=$(${SSH_PARAMETER}@$i sudo redis-cli info 2>/dev/null|grep Replication)     if [[ -z ${status} ]]     then       ${SSH_PARAMETER}@$i sudo systemctl restart redis 2>/dev/null       status_agagin=$(${SSH_PARAMETER}@$i sudo redis-cli info 2>/dev/null|grep Replication)       if [[ -z ${status_agagin} ]]       then         REDIS_ERROR_LIST[a]=${i}         let a++       fi     fi   done   if [[ -n ${REDIS_ERROR_LIST[@]} ]]   then     echo -e "REDIS    \033[31mERROR  \033[0m ${REDIS_ERROR_LIST[@]}"   else     echo -e "REDIS    \033[32mOK     \033[0m"   fi }     CHECK_KEYSTONE() {   KEYSTONE_ERROR_LIST=$(openstack user list|grep admin)   if [[ -z ${KEYSTONE_ERROR_LIST} ]]   then     echo -e "KEYSTONE \033[31mERROR  \033[0m ${VIP}"   else     echo -e "KEYSTONE \033[32mOK     \033[0m"   fi }     CHECK_GLANCE() {   GLANCE_ERROR_LIST=$(glance image-list|grep Name)   if [[ -z ${GLANCE_ERROR_LIST} ]]   then     echo -e "GLANCE   \033[31mERROR  \033[0m ${VIP}"   else     echo -e "GLANCE   \033[32mOK     \033[0m"   fi }     CHECK_NOVA() {   NOVA_ERROR_LIST=$(nova service-list| grep -v Binary| awk -F '|' '/down/ {print $4}'| sort -u)   if [[ -n ${NOVA_ERROR_LIST} ]]   then     echo -e "NOVA     \033[31mERROR  \033[0m ${NOVA_ERROR_LIST[*]}"   else     echo -e "NOVA     \033[32mOK     \033[0m"   fi }     CHECK_NEUTRON() {   NEUTRON_ERROR_LIST=$(neutron agent-list | awk -F '|' '/XXX/ {print $4}'| sort -u)   if [[ -n ${NEUTRON_ERROR_LIST} ]]   then     echo -e "NEUTRON  \033[31mERROR  \033[0m ${NEUTRON_ERROR_LIST[*]}"   else     echo -e "NEUTRON  \033[32mOK     \033[0m"   fi }     CHECK_CINDER() {   CINDER_ERROR_LIST=$(cinder service-list| awk -F '|' '/down/ {print $3}'| awk -F '@' '{print $1}'| sort -u)   if [[ -n ${CINDER_ERROR_LIST} ]]   then     echo -e "CINDER   \033[31mERROR  \033[0m ${CINDER_ERROR_LIST[*]}"   else     echo -e "CINDER   \033[32mOK     \033[0m"   fi }     CHECK_NGINX() {   for i in ${CONTROLER_IP_LIST}   do     status=$(${SSH_PARAMETER}@$i sudo curl -s --max-time 10 localhost:80 2>/dev/null |grep nginx)     if [[ -z ${status} ]]     then       ${SSH_PARAMETER}@$i sudo systemctl restart nginx 2>/dev/null       status_agagin=$(${SSH_PARAMETER}@$i sudo curl -s --max-time 10 localhost:80 2>/dev/null |grep nginx)       if [[ -z ${status_agagin} ]]       then         NGINX_ERROR_LIST[a]=${i}         let a++       fi     fi   done   if [[ -n ${NGINX_ERROR_LIST[@]} ]]   then     echo -e "NGINX    \033[31mERROR  \033[0m ${NGINX_ERROR_LIST[@]}"   else     echo -e "NGINX    \033[32mOK     \033[0m"   fi }     CHECK_TOMCAT() {   for i in ${CONTROLER_IP_LIST}   do     #status=$(${SSH_PARAMETER}@$i sudo curl -s --max-time 10 localhost:8080/jitstack/#/login 2>/dev/null|grep webConfig.js)     status=$(${SSH_PARAMETER}@$i sudo curl -I -m 10 -s --max-time 10 -o /dev/null -s -w %{http_code} localhost:8080/jitstack/)     if [[ ${status} != 200 ]]     then       ${SSH_PARAMETER}@$i sudo systemctl restart tomcat 2>/dev/null       status_agagin=$(${SSH_PARAMETER}@$i sudo curl -I -m 10 -s --max-time 10 -o /dev/null -s -w %{http_code} localhost:8080/jitstack/)       if [[ ${status_agagin} != 200 ]]       then         TOMCAT_ERROR_LIST[a]=${i}         let a++       fi     fi   done   if [[ -n ${TOMCAT_ERROR_LIST[@]} ]]   then     echo -e "TOMCAT   \033[31mERROR  \033[0m ${TOMCAT_ERROR_LIST[@]}"   else     echo -e "TOMCAT   \033[32mOK     \033[0m"   fi }     CHECK_APACHE2() {   for i in ${CONTROLER_IP_LIST}   do     status=$(${SSH_PARAMETER}@$i sudo curl -s --max-time 10 localhost:5000 2>/dev/null|grep openstack)     if [[ -z ${status} ]]     then       ${SSH_PARAMETER}@$i sudo systemctl restart apache2 2>/dev/null       status_agagin=$(${SSH_PARAMETER}@$i sudo curl -s --max-time 10 localhost:5000 2>/dev/null|grep openstack)       if [[ -z ${status_agagin} ]]       then         APACHE2_ERROR_LIST[a]=${i}         let a++       fi     fi   done   if [[ -n ${APACHE2_ERROR_LIST[@]} ]]   then     echo -e "APACHE2  \033[31mERROR  \033[0m ${APACHE2_ERROR_LIST[@]}"   else     echo -e "APACHE2  \033[32mOK     \033[0m"   fi }     CHECK_CEPH() {   size=$(ceph osd pool ls detail |awk '/vms/ {print $6}')   ceph osd stat|grep -w osds|perl -pe 's/.*osdmap.*?://'| awk '// {{total_osd=$1}; {up_osd=$3}; {in_osd=$5}} END {printf "OSDs     ";printf("Total: %s UP: %s IN: %s Down: %s\n"),total_osd,up_osd,in_osd,total_osd-up_osd}'   #rados df | perl -pe 's/total /total_/'| awk -v rs=$size '/total_/ {if($1=="total_space"){total=$2}; if($1=="total_used"){used=$2}; if($1=="total_avail"){avail=$2}} END {printf "CEPH     ";printf("Total: %sGB Used: %sGB Available: %sGB\n"),total/rs,used/rs,avail/rs}'   ceph osd df | awk -v rs=$size '/TOTAL/ {print "CEPH     Total: "$2/rs"GB","Used: "$3/rs"GB","Available: "$4/rs"GB"}' }     CHECK_VMS() {   nova list|awk '{line[NR]=$0} END {for(i=4;i<NR;i++) print line[i]}'| awk -F '|' '{print $4}'| sed 's/ //g' | awk '{a[$1]++;b++} END {printf "VMS      Total: "b" ";for(i in a) printf i ": " a[i]" " ;}'   echo ""   ALL_PH_CPUS=$(nova hypervisor-stats |grep -w vcpus| awk -F '|' '{print $3}' | sed 's/ //g')   nova flavor-list | awk '{line[NR]=$0} END {for(i=4;i<NR;i++) print line[i]}' | awk -F '|' '{print $8}'| sed 's/ //g' |awk -v all_ph_cpus=$ALL_PH_CPUS '{vcpus+=$1} END {printf("CPUS     CPUS: %s VCPUS: %s RATIO: %s\n"),all_ph_cpus,vcpus,vcpus/all_ph_cpus}'   ALL_PH_RAMS=$(nova hypervisor-stats |grep -w memory_mb| awk -F '|' '{print $3}' | sed 's/ //g')   nova flavor-list | awk '{line[NR]=$0} END {for(i=4;i<NR;i++) print line[i]}' | awk -F '|' '{print $4}'| sed 's/ //g' |awk -v all_ph_rams=$ALL_PH_RAMS '{vrams+=$1} END {printf("RAMS     RAMS: %sGB VRAMS: %sGB RATIO: %s\n"),all_ph_rams/1024,vrams/1024,vrams/all_ph_rams}'   size=$(ceph osd pool ls detail |awk '/vms/ {print $6}')   #TOTAL_SPACE=$(rados df|awk '/total_space/ {print $2}'| sed 's/G//g')   TOTAL_SPACE=$(ceph osd df| awk '/TOTAL/ {print $2}' | sed 's/G//g')   #cinder list | awk '{line[NR]=$0} END {for(i=4;i<NR;i++) print line[i]}' | awk -F '|' '{print $5}'| sed 's/ //g' |awk -v sz=$size -v total_space=$TOTAL_SPACE '{vdisks+=$1} END {printf("DISKS    DISKS: %sGB VDISKS: %sGB RATIO: %s\n"),total_space/sz,vdisks,vdisks*sz/total_space}'   cinder list | cinder list | grep -v '\-\-\-\-' | awk -F '|' 'NR==1{for(i=1;i<=NF;i++)if($i~/Size/)n=i}NR>1{print $n}' | sed 's/ //g' |awk -v sz=$size -v total_space=$TOTAL_SPACE '{vdisks+=$1} END {printf("DISKS    DISKS: %sGB VDISKS: %sGB RATIO: %s\n"),total_space/sz,vdisks,vdisks*sz/total_space}' }     SHUTDOWN_VIRTBR   echo -e "\033[35mStart Check Services ...\033[0m" CHECK_HARDWARE CHECK_NTP CHECK_CPU CHECK_MEM CHECK_DISK CHECK_NETWORK CHECK_MYSQL_SLAVE CHECK_RABBITMQ CHECK_MEMCACHE CHECK_REDIS CHECK_TOMCAT CHECK_NGINX CHECK_APACHE2 CHECK_KEYSTONE CHECK_GLANCE CHECK_NOVA CHECK_NEUTRON CHECK_CINDER   echo "" echo -e "\033[35mStart Check Resources Usage ...\033[0m" CHECK_CEPH CHECK_VMS 

标签:echo,ERROR,LIST,CHECK,OpenStack,033,server,check,0m
来源: https://www.cnblogs.com/zhjie1205/p/15954945.html