通过监控异常的pod，及时发现k8s存在的问题。原理是使用kubectl get pod –all-namespaces，找到ready列，如果ready数量与desire数量不一致的，或者非RUNNING的状态POD,则认为这个pod异常，，进而告警。
配置agent监控项
/etc/zabbix/zabbix_agentd.d/k8s.conf
UserParameter=abnormal.pod, kubectl get pod --all-namespaces -o wide | awk 'NR>1{cmd="echo "$3" | bc";cmd|getline ret;close(cmd);if (ret != 1 || $4 != "Running"){print}}'
导入模板
<?xml version="1.0" encoding="UTF-8"?>
<zabbix_export>
<version>3.2</version>
<date>2017-06-16T03:51:42Z</date>
<groups>
<group>
<name>Templates</name>
</group>
</groups>
<templates>
<template>
<template>Template App K8s</template>
<name>Template App K8s</name>
<description/>
<groups>
<group>
<name>Templates</name>
</group>
</groups>
<applications>
<application>
<name>k8s</name>
</application>
</applications>
<items>
<item>
<name>abnormal pod</name>
<type>0</type>
<snmp_community/>
<multiplier>0</multiplier>
<snmp_oid/>
<key>abnormal.pod</key>
<delay>60</delay>
<history>90</history>
<trends>0</trends>
<status>0</status>
<value_type>4</value_type>
<allowed_hosts/>
<units/>
<delta>0</delta>
<snmpv3_contextname/>
<snmpv3_securityname/>
<snmpv3_securitylevel>0</snmpv3_securitylevel>
<snmpv3_authprotocol>0</snmpv3_authprotocol>
<snmpv3_authpassphrase/>
<snmpv3_privprotocol>0</snmpv3_privprotocol>
<snmpv3_privpassphrase/>
<formula>1</formula>
<delay_flex/>
<params/>
<ipmi_sensor/>
<data_type>0</data_type>
<authtype>0</authtype>
<username/>
<password/>
<publickey/>
<privatekey/>
<port/>
<description/>
<inventory_link>0</inventory_link>
<applications>
<application>
<name>k8s</name>
</application>
</applications>
<valuemap/>
<logtimefmt/>
</item>
</items>
<discovery_rules/>
<httptests/>
<macros/>
<templates/>
<screens/>
</template>
</templates>
<triggers>
<trigger>
<expression>{Template App K8s:abnormal.pod.strlen()}<>0</expression>
<recovery_mode>0</recovery_mode>
<recovery_expression/>
<name>abnormal pod</name>
<correlation_mode>0</correlation_mode>
<correlation_tag/>
<url/>
<status>0</status>
<priority>2</priority>
<description/>
<type>0</type>
<manual_close>0</manual_close>
<dependencies/>
<tags/>
</trigger>
</triggers>
</zabbix_export>
应用模板
找到能执行kubectl的节点，添加k8s模板。
