SRE文档1. kubectl describe
可通过该命令查看 Pod 在节点上的资源分配情况（Request、Limits），如下：
# product 表示生产环境
kubectl describe node -l env=product
# 执行结果
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits AGE
--------- ---- ------------ ---------- --------------- ------------- ---
product annoroad-clinicallims3-754dddb5cb-k5xkn 0 (0%) 0 (0%) 0 (0%) 0 (0%) 85d
product annoroad-crm-796884585d-ts2xn 0 (0%) 0 (0%) 0 (0%) 0 (0%) 85d
product annoroad-dms-6d658d564f-m6j7r 0 (0%) 0 (0%) 0 (0%) 0 (0%) 16d
product annoroad-dms-server-6557c5bc85-45sw2 0 (0%) 0 (0%) 0 (0%) 0 (0%) 15d# product 表示生产环境
kubectl describe node -l env=product
# 执行结果
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits AGE
--------- ---- ------------ ---------- --------------- ------------- ---
product annoroad-clinicallims3-754dddb5cb-k5xkn 0 (0%) 0 (0%) 0 (0%) 0 (0%) 85d
product annoroad-crm-796884585d-ts2xn 0 (0%) 0 (0%) 0 (0%) 0 (0%) 85d
product annoroad-dms-6d658d564f-m6j7r 0 (0%) 0 (0%) 0 (0%) 0 (0%) 16d
product annoroad-dms-server-6557c5bc85-45sw2 0 (0%) 0 (0%) 0 (0%) 0 (0%) 15d2. kubectl top node
可通过该命令查看节点的资源使用情况，如下：
bash
# product 表示生产环境
kubectl top node -l env=product
# 执行结果
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
cn-beijing.192.16.168.102 904m 11% 10340Mi 69%
cn-beijing.192.16.168.103 1393m 17% 12810Mi 86%
cn-beijing.192.16.168.104 1994m 24% 13281Mi 89%
cn-beijing.192.16.168.105 987m 12% 9399Mi 63%
cn-beijing.192.16.168.115 638m 15% 12017Mi 80%
cn-beijing.192.16.168.118 806m 20% 9372Mi 62%
cn-beijing.192.16.168.135 258m 6% 9696Mi 65%
cn-beijing.192.16.168.136 871m 21% 10957Mi 73%
cn-beijing.192.16.168.137 310m 7% 9322Mi 62%
cn-beijing.192.16.168.138 943m 23% 11908Mi 80%# product 表示生产环境
kubectl top node -l env=product
# 执行结果
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
cn-beijing.192.16.168.102 904m 11% 10340Mi 69%
cn-beijing.192.16.168.103 1393m 17% 12810Mi 86%
cn-beijing.192.16.168.104 1994m 24% 13281Mi 89%
cn-beijing.192.16.168.105 987m 12% 9399Mi 63%
cn-beijing.192.16.168.115 638m 15% 12017Mi 80%
cn-beijing.192.16.168.118 806m 20% 9372Mi 62%
cn-beijing.192.16.168.135 258m 6% 9696Mi 65%
cn-beijing.192.16.168.136 871m 21% 10957Mi 73%
cn-beijing.192.16.168.137 310m 7% 9322Mi 62%
cn-beijing.192.16.168.138 943m 23% 11908Mi 80%- top pod
# product 表示生产环境
kubectl top pod -n product# product 表示生产环境
kubectl top pod -n product3. 查询每个Pod
kubectl查询每个pod占用磁盘空间
bash
[root@kube-master ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready control-plane,master 121d v1.22.17
kube-node01 Ready <none> 119d v1.22.17
kube-node02 Ready <none> 120d v1.22.17
kube-node03 Ready <none> 120d v1.22.17
$ kubectl get --raw /api/v1/nodes/kube-node01/proxy/stats/summary | jq '.pods[] | "PodName: \(.podRef.name)", "usedBytes: \(.containers[].rootfs.usedBytes)", "======================================================"'
"PodName: openelb-keepalive-vip-fmqp8"
"usedBytes: 94208"
"======================================================"
"PodName: consul-0"
"usedBytes: 983040"
"======================================================"
"PodName: process-exporter-cwwr9"
"usedBytes: 0"
"======================================================"
"PodName: calico-typha-67c6dc57d6-jkxfh"
"usedBytes: 0"
"======================================================"
"PodName: minio-1-ss-0-0"
"usedBytes: 0"
"usedBytes: 0"[root@kube-master ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready control-plane,master 121d v1.22.17
kube-node01 Ready <none> 119d v1.22.17
kube-node02 Ready <none> 120d v1.22.17
kube-node03 Ready <none> 120d v1.22.17
$ kubectl get --raw /api/v1/nodes/kube-node01/proxy/stats/summary | jq '.pods[] | "PodName: \(.podRef.name)", "usedBytes: \(.containers[].rootfs.usedBytes)", "======================================================"'
"PodName: openelb-keepalive-vip-fmqp8"
"usedBytes: 94208"
"======================================================"
"PodName: consul-0"
"usedBytes: 983040"
"======================================================"
"PodName: process-exporter-cwwr9"
"usedBytes: 0"
"======================================================"
"PodName: calico-typha-67c6dc57d6-jkxfh"
"usedBytes: 0"
"======================================================"
"PodName: minio-1-ss-0-0"
"usedBytes: 0"
"usedBytes: 0"3.1 节点上容器
bash
# 节点上查看磁盘使用情况,并安装从大到小排序,可以看到各个容器的占用磁盘空间。
docker ps -a --format "table {{.Size}}\t{{.Names}}" | sort -hr
# 查看磁盘总体使用情况
docker system df -v
#清理所有未使用的资源（镜像、容器、卷和网络）
docker system prune -a
#解决方式，配置日志轮询
#docker
{
"log-driver": "json-file"
"log-opts": {
"max-size": "100m",
"max-file": "3"
}
}# 节点上查看磁盘使用情况,并安装从大到小排序,可以看到各个容器的占用磁盘空间。
docker ps -a --format "table {{.Size}}\t{{.Names}}" | sort -hr
# 查看磁盘总体使用情况
docker system df -v
#清理所有未使用的资源（镜像、容器、卷和网络）
docker system prune -a
#解决方式，配置日志轮询
#docker
{
"log-driver": "json-file"
"log-opts": {
"max-size": "100m",
"max-file": "3"
}
}