SRE文档

深浅模式

1. clean old images ​

1.1 非报警 ​

yaml
apiVersion: batch/v1
kind: CronJob
metadata:
  name: docker-image-cleanup
spec:
  schedule: "0 0 * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: docker-image-cleanup
            image: docker
            command:
            - /bin/sh
            - -c
            - >
              echo y | docker system prune --force --filter "until=24h" --volumes
            volumeMounts:
            - name: docker-socket
              mountPath: /var/run/docker.sock
          volumes:
          - name: docker-socket
            hostPath:
              path: /var/run/docker.sock
          restartPolicy: OnFailure
apiVersion: batch/v1
kind: CronJob
metadata:
  name: docker-image-cleanup
spec:
  schedule: "0 0 * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: docker-image-cleanup
            image: docker
            command:
            - /bin/sh
            - -c
            - >
              echo y | docker system prune --force --filter "until=24h" --volumes
            volumeMounts:
            - name: docker-socket
              mountPath: /var/run/docker.sock
          volumes:
          - name: docker-socket
            hostPath:
              path: /var/run/docker.sock
          restartPolicy: OnFailure
  • 执行
kubectl apply -f docker-image-cleanup.yaml
kubectl apply -f docker-image-cleanup.yaml

1.2 TG报警 ​

yaml
apiVersion: batch/v1
kind: CronJob
metadata:
  name: docker-image-cleanup
spec:
  schedule: "0 0 * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: docker-image-cleanup
            image: docker
            command:
            - /bin/sh
            - -c
            - >
              echo y | docker system prune --force --filter "until=24h" && apk add curl && curl -s -X POST https://api.telegram.org/bot<TOKEN>/sendMessage -d chat_id=<your_channel_id> -d text="☸️ k8s-services-stage: Docker cleanup complete %0AFree disk space: $(df -h | grep sda2 | awk '{print $4}' | head -c 6) %0AAvailable memory: $(free -h |grep Mem | awk '{print $7}')"
            volumeMounts:
            - name: docker-socket
              mountPath: /var/run/docker.sock
          volumes:
          - name: docker-socket
            hostPath:
              path: /var/run/docker.sock
          restartPolicy: OnFailure
apiVersion: batch/v1
kind: CronJob
metadata:
  name: docker-image-cleanup
spec:
  schedule: "0 0 * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: docker-image-cleanup
            image: docker
            command:
            - /bin/sh
            - -c
            - >
              echo y | docker system prune --force --filter "until=24h" && apk add curl && curl -s -X POST https://api.telegram.org/bot<TOKEN>/sendMessage -d chat_id=<your_channel_id> -d text="☸️ k8s-services-stage: Docker cleanup complete %0AFree disk space: $(df -h | grep sda2 | awk '{print $4}' | head -c 6) %0AAvailable memory: $(free -h |grep Mem | awk '{print $7}')"
            volumeMounts:
            - name: docker-socket
              mountPath: /var/run/docker.sock
          volumes:
          - name: docker-socket
            hostPath:
              path: /var/run/docker.sock
          restartPolicy: OnFailure

执行替换chat_id和token

  • 执行
bash
kubectl apply -f docker-image-cleanup_alert_telegram.yaml
kubectl apply -f docker-image-cleanup_alert_telegram.yaml

2. clean Evicted Pods ​

  • 清理特定namespace
bash
kubectl delete pods --field-selector 'status.phase==Failed' -n <namespace>
kubectl delete pods --field-selector 'status.phase==Failed' -n <namespace>
  • 清理所有
bash
kubectl delete pods --field-selector 'status.phase==Failed' --all-namespaces
kubectl delete pods --field-selector 'status.phase==Failed' --all-namespaces
yaml
apiVersion: batch/v1
kind: CronJob
metadata:
  name: evicted-pod-cleanup
spec:
  schedule: "0 1 * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: cleanup-container
            image: bitnami/kubectl
            command: ["sh", "-c", "kubectl delete pods --all --field-selector 'status.phase==Failed' --all-namespaces"]
          restartPolicy: OnFailure
apiVersion: batch/v1
kind: CronJob
metadata:
  name: evicted-pod-cleanup
spec:
  schedule: "0 1 * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: cleanup-container
            image: bitnami/kubectl
            command: ["sh", "-c", "kubectl delete pods --all --field-selector 'status.phase==Failed' --all-namespaces"]
          restartPolicy: OnFailure