#!/bin/bash

#ding-alert-钉钉告警
function dingding_alert {

  data_TIME=`date "+%Y-%m-%d %H:%M:%S"`
  dingding_url="https://oapi.dingtalk.com/robot/send?access_token=6f43eab82ef830c309106481283dfc28f527f2ef2bf7276fafdeed3a7113813f"

  curl "$dingding_url" \
    -H "Content-Type: application/json" \
               -d "{ \
                                  \"msgtype\":\"markdown\", \
                                  \"markdown\":{ \
                          \"title\":\"jenkins\", \
                         \"text\":\"### <font color="#FF0000">k8s集群资源运行异常告警：</font><br /> \n * 告警时间: ${data_TIME} \n * k8s集群名称：${k8s_name} \n * 告警资源类型: ${k8s_resource_type} \n * 告警资源名: ${resource_name} \n  * 所属环境或namespace：${k8s_namespace} \n * 故障时间: ${fail_time} \n * 故障内容描述: ${k8s_describe}\n * 查询地址: ${grafana_url} \n\" \
                    } \
              }"

}

#wx-alert-微信告警
function wx_alert {

  #wx_url="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=a03a2e14-1c4d-43aa-a337-81509534d12a"
  wx_url="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=33ccbbae-b40a-4a81-b836-166630c9303b"
  alter_date=`date +%Y-%m-%d`


      /usr/bin/curl "$wx_url" \
  -H 'Content-Type: application/json' \
   -d '
   {
  "msgtype": "text",
  "text": {
    "content": "【k8s集群资源运行异常告警:】 \n  告警时间： '${alter_date}'\n  k8s集群名称：'${k8s_name}'\n  告警资源类型：'${k8s_resource_type}'\n  告警资源名：'${resource_name}'\n   所属环境或namespace： '${k8s_namespace}'\n   故障时间:  '${fail_time}'\n  故障内容描述:  '${k8s_describe}'\n 查询地址：'${grafana_url}'",
    "mentioned_list":["@吴未明","@all"]
  }
}'

}


k8s_name="国内计划任务k8s集群"



###############################################k8s-pod-check#############################################################

#check-pod-img-error
#kubectl get pod -A | grep ErrImagePull  

for resource_name in `kubectl get pod -A | grep 'ImagePullBackOff' | awk '{print $2}'`
do
    
  k8s_namespace=`kubectl get pod -A | grep 'ImagePullBackOff' | grep "$resource_name" | awk '{print $1}'`
  fail_time=`kubectl get pod -A | grep 'ImagePullBackOff' | grep "$resource_name" | awk '{print $NF}'`
  k8s_resource_type="pod"
  k8s_describe="k8s-pod容器运行镜像拉取失败"
  grafana_url="https://grafana.hahabianli.com"

  #微信告警
  wx_alert  
  #dingding_alert

done


for resource_name in `kubectl get pod -A | grep 'ErrImagePull' | awk '{print $2}'`
do

  k8s_namespace=`kubectl get pod -A | grep 'ErrImagePull' |  grep "$resource_name" | awk '{print $1}'`
  fail_time=`kubectl get pod -A | grep 'ErrImagePull' | grep "$resource_name" | awk '{print $NF}'`
  k8s_resource_type="pod"
  k8s_describe="k8s-pod容器运行镜像拉取失败"
  grafana_url="https://grafana.hahabianli.com"

  #微信告警
  wx_alert
  #dingding_alert

done


#pod运行error
for resource_name in `kubectl get pod -A | grep 'Error' | awk '{print $2}'`
do

  k8s_namespace=`kubectl get pod -A | grep 'Error' |  grep "$resource_name" | awk '{print $1}'`
  fail_time=`kubectl get pod -A | grep 'Error' | grep "$resource_name" | awk '{print $NF}'`
  k8s_resource_type="pod"
  k8s_describe="k8s-pod容器启动命令运行失败"
  grafana_url="https://grafana.hahabianli.com"

  #微信告警
  wx_alert
  #dingding_alert

done



#kubectl get pod -A | grep Pending

for resource_name in `kubectl get pod -A | grep 'Pending' | awk '{print $2}'`
do

  k8s_namespace=`kubectl get pod -A | grep 'Pending'| grep "$resource_name" | awk '{print $1}'`
  fail_time=`kubectl get pod -A | grep 'Pending' | grep "$resource_name" | awk '{print $NF}'`
  k8s_resource_type="pod"
  k8s_describe="k8s-pod容器启动使用持续Pending"
  grafana_url="https://grafana.hahabianli.com"

  #微信告警
  kubectl get pod -A | grep 'Pending' | grep "$resource_name" | awk '{print $NF}' | grep '2m' &&  wx_alert
  kubectl get pod -A | grep 'Pending' | grep "$resource_name" | awk '{print $NF}' | grep '3m' &&  wx_alert
  kubectl get pod -A | grep 'Pending' | grep "$resource_name" | awk '{print $NF}' | grep '4m' &&  wx_alert
  kubectl get pod -A | grep 'Pending' | grep "$resource_name" | awk '{print $NF}' | grep '5m' &&  wx_alert
  kubectl get pod -A | grep 'Pending' | grep "$resource_name" | awk '{print $NF}' | grep '6m' &&  wx_alert
  kubectl get pod -A | grep 'Pending' | grep "$resource_name" | awk '{print $NF}' | grep '8m' &&  wx_alert
  kubectl get pod -A | grep 'Pending' | grep "$resource_name" | awk '{print $NF}' | grep '10m' &&  wx_alert
  #dingding_alert

done



#kubectl get pod -A | grep CrashLoopBackOff

for resource_name in `kubectl get pod -A | grep 'CrashLoopBackOff' | awk '{print $2}'`
do

  k8s_namespace=`kubectl get pod -A | grep 'CrashLoopBackOff' | grep "$resource_name" | awk '{print $1}'`
  fail_time=`kubectl get pod -A | grep 'CrashLoopBackOff' | grep "$resource_name" | awk '{print $NF}'`
  k8s_resource_type="pod"
  k8s_describe="k8s-pod容器启动出现立即退出CrashLoopBackOff"
  grafana_url="https://grafana.hahabianli.com"

  #微信告警
  kubectl get pod -A | grep 'CrashLoopBackOff' | grep "$resource_name" | awk '{print $NF}' | grep '2m' &&  wx_alert
  kubectl get pod -A | grep 'CrashLoopBackOff' | grep "$resource_name" | awk '{print $NF}' | grep '3m' &&  wx_alert
  kubectl get pod -A | grep 'CrashLoopBackOff' | grep "$resource_name" | awk '{print $NF}' | grep '4m' &&  wx_alert
  kubectl get pod -A | grep 'CrashLoopBackOff' | grep "$resource_name" | awk '{print $NF}' | grep '5m' &&  wx_alert
  kubectl get pod -A | grep 'CrashLoopBackOff' | grep "$resource_name" | awk '{print $NF}' | grep '6m' &&  wx_alert
  kubectl get pod -A | grep 'CrashLoopBackOff' | grep "$resource_name" | awk '{print $NF}' | grep '8m' &&  wx_alert
  kubectl get pod -A | grep 'CrashLoopBackOff' | grep "$resource_name" | awk '{print $NF}' | grep '10m' &&  wx_alert
  #dingding_alert

done


#kubectl get pod -A | grep Terminating

for resource_name in `kubectl get pod -A | grep 'Terminating' | awk '{print $2}'`
do

  k8s_namespace=`kubectl get pod -A | grep 'Terminating' | grep "$resource_name" | awk '{print $1}'`
  fail_time=`kubectl get pod -A | grep 'Terminating' | grep "$resource_name" | awk '{print $NF}'`
  k8s_resource_type="pod"
  k8s_describe="k8s-pod容器退出无法删除-finalizers字段中存在未完成的逻辑清理"
  grafana_url="https://grafana.hahabianli.com"

  #微信告警
  kubectl get pod -A | grep 'Terminating' | grep "$resource_name" | awk '{print $NF}' | grep '2m' &&  wx_alert
  kubectl get pod -A | grep 'Terminating' | grep "$resource_name" | awk '{print $NF}' | grep '3m' &&  wx_alert
  kubectl get pod -A | grep 'Terminating' | grep "$resource_name" | awk '{print $NF}' | grep '4m' &&  wx_alert
  kubectl get pod -A | grep 'Terminating' | grep "$resource_name" | awk '{print $NF}' | grep '5m' &&  wx_alert
  kubectl get pod -A | grep 'Terminating' | grep "$resource_name" | awk '{print $NF}' | grep '6m' &&  wx_alert
  kubectl get pod -A | grep 'Terminating' | grep "$resource_name" | awk '{print $NF}' | grep '8m' &&  wx_alert
  kubectl get pod -A | grep 'Terminating' | grep "$resource_name" | awk '{print $NF}' | grep '10m' && wx_alert
  #dingding_alert

done

########################################################################################################################




###############################################k8s-node-check#############################################################

#检查k8s-node节点状态是否为Ready

for resource_name in `kubectl get nodes -o wide | grep -v 'STATUS' | awk '{print $1}'`
do

  k8s_node_status=`kubectl get nodes -o wide | grep "$resource_name" | awk '{print $2}' | grep -v '^Ready'` || continue

  k8s_namespace="K8s集群生产服务器节点"
  fail_time=`kubectl get nodes -o wide | grep "$k8s_node_status" | grep "$resource_name" | awk '{print $4}'`
  k8s_resource_type="Node"
  k8s_describe="k8s集群-服务器节点异常状态-"$k8s_node_status""
  grafana_url="https://grafana.hahabianli.com"

  #微信告警
  #kubectl get  nodes -o wide | grep "$k8s_node_status" | grep "$resource_name" | awk '{print $4}' | grep '2m' &&  wx_alert
  #kubectl get  nodes -o wide | grep "$k8s_node_status" | grep "$resource_name" | awk '{print $4}' | grep '3m' &&  wx_alert
  #kubectl get  nodes -o wide | grep "$k8s_node_status" | grep "$resource_name" | awk '{print $4}' | grep '4m' &&  wx_alert
  #kubectl get  nodes -o wide | grep "$k8s_node_status" | grep "$resource_name" | awk '{print $4}' | grep '5m' &&  wx_alert
  #kubectl get  nodes -o wide | grep "$k8s_node_status" | grep "$resource_name" | awk '{print $4}' | grep '6m' &&  wx_alert
  #kubectl get  nodes -o wide | grep "$k8s_node_status" | grep "$resource_name" | awk '{print $4}' | grep '7m' &&  wx_alert
  #kubectl get  nodes -o wide | grep "$k8s_node_status" | grep "$resource_name" | awk '{print $4}' | grep '8m' &&  wx_alert
  #dingding_alert
  wx_alert

done


#k8s集群节点硬件资源使用率告警

for resource_name in `kubectl top node | grep -v 'NAME' | awk '{print $1}'`
do

  k8s_node_cpu=`kubectl top node | grep "$resource_name" | awk '{print $3}' | awk -F '%' '{print $1}'`
  k8s_node_memory=`kubectl top node | grep "$resource_name" | awk '{print $5}' | awk -F '%' '{print $1}'`

  if [ "$k8s_node_cpu" -ge 70 -o "$k8s_node_memory" -ge 80 ];
 
 then

  k8s_namespace="K8s集群生产服务器节点"
  #fail_time=`kubectl get nodes -o wide | grep "$resource_name" | awk '{print $4}'`
  fail_time=`date +%Y-%m-%d-%H-%M`
  k8s_resource_type="Node"
  k8s_describe="k8s集群-服务器节点性能异常-当前CPU使用率-"$k8s_node_cpu"-内存使用率-"$k8s_node_memory""
  grafana_url="https://grafana.hahabianli.com"
  wx_alert
  fi
 
done

########################################################################################################################



###############################################k8s-api-check#############################################################

#检测k8s集群证书到期日期告警

datetime_hour=`date +%H`
datetime_min=`date +%M`

if [ "$datetime_hour" -gt 09 -a "$datetime_hour" -lt 19 ]; then

   if [ "$datetime_min" -gt 05 -a "$datetime_min" -lt 10 ]; then


k8s_year=`kubeadm certs check-expiration | grep 'apiserver' | sed -n 1p | awk '{print $4}'`
k8s_month_tmp=`kubeadm certs check-expiration | grep 'apiserver' | sed -n 1p | awk '{print $2}'`
k8s_day=`kubeadm certs check-expiration | grep 'apiserver' | sed -n 1p | awk '{print $3}' | awk -F ',' '{print $1}'`

if [ "$k8s_month_tmp" == "Jan" ]; then

   k8s_month="01"

elif [ "$k8s_month_tmp" == "Feb" ]; then
 
   k8s_month="02"

elif [ "$k8s_month_tmp" == "Mar" ]; then

   k8s_month="03"

elif [ "$k8s_month_tmp" == "Apr" ]; then

   k8s_month="04"

elif [ "$k8s_month_tmp" == "May" ]; then

   k8s_month="05"

elif [ "$k8s_month_tmp" == "Jun" ]; then

   k8s_month="06"

elif [ "$k8s_month_tmp" == "Jul" ]; then

   k8s_month="07"

elif [ "$k8s_month_tmp" == "Aug" ]; then

   k8s_month="08"

elif [ "$k8s_month_tmp" == "Sep" ]; then

   k8s_month="09"

elif [ "$k8s_month_tmp" == "Oct" ]; then

   k8s_month="10"

elif [ "$k8s_month_tmp" == "Nov" ]; then

   k8s_month="11"

elif [ "$k8s_month_tmp" == "Dec" ]; then

   k8s_month="12"

else

   echo "$k8s_month_tmp"

fi

k8s_api_datetime=""$k8s_year""$k8s_month""$k8s_day""

#datetime=`date -d '-20 days ago' '+%Y%m%d'`
datetime=`date -d '-20 days ago' '+%Y%m%d'`

if [ "$datetime" -ge "$k8s_api_datetime" ];

then

  k8s_namespace="生产k8s集群证书"
  resource_name=`hostname`
  fail_time="证书到期时间-"$k8s_year""$k8s_month""$k8s_day""
  k8s_resource_type="k8s-apiserver-certs"
  k8s_describe="k8s-apiserver证书即将到期，请及时更新证书"
  grafana_url="https://grafana.hahabianli.com"

  #微信告警
  wx_alert
  #dingding_alert

fi

fi
fi