SRE文档

深浅模式

1. 有jdk情况 ​

1.1 添加钩子函数 ​

yaml
  name: idk-mob-sdk-server
  image: harbor.example.com/base/example:v1
  imagePullPolicy: IfNotPresent
  lifecycle:
    preStop:
      exec:
        command: ["sh", "-c", "jstack -F $(jps |grep -v Jps | awk '{print $1}') | tee -a /logs/thread.dump && jmap -dump:format=b,file=/logs/$(date +'%Y-%m-%d_%H%M%S').hprof $(jps |grep -v Jps | awk '{print $1}')"]
  readinessProbe:
    failureThreshold: 2
    httpGet:
      path: /actuator/info
      port: 5003
      scheme: HTTP
    initialDelaySeconds: 30
    periodSeconds: 15
    successThreshold: 1
    timeoutSeconds: 2
  volumeMounts:
  - mountPath: /logs
    name: app-logs
volumes:
- hostPath:
    path: /logs
    type: DirectoryOrCreate
  name: app-logs
  name: idk-mob-sdk-server
  image: harbor.example.com/base/example:v1
  imagePullPolicy: IfNotPresent
  lifecycle:
    preStop:
      exec:
        command: ["sh", "-c", "jstack -F $(jps |grep -v Jps | awk '{print $1}') | tee -a /logs/thread.dump && jmap -dump:format=b,file=/logs/$(date +'%Y-%m-%d_%H%M%S').hprof $(jps |grep -v Jps | awk '{print $1}')"]
  readinessProbe:
    failureThreshold: 2
    httpGet:
      path: /actuator/info
      port: 5003
      scheme: HTTP
    initialDelaySeconds: 30
    periodSeconds: 15
    successThreshold: 1
    timeoutSeconds: 2
  volumeMounts:
  - mountPath: /logs
    name: app-logs
volumes:
- hostPath:
    path: /logs
    type: DirectoryOrCreate
  name: app-logs

在线分析,https://memory.console.heapdump.cn/

2. 没有jdk情况 ​

jattach 实现了HotSpot Attach API

2.1 下载 ​

bash
wget https://github.com/jattach/jattach/releases/download/v2.2/jattach
wget https://github.com/jattach/jattach/releases/download/v2.2/jattach
  • 重新生成dockerfile文件，吧jattach复制到镜像中去
bash
#!/bin/sh
# 导出当前内存信息
jattach 1 dumpheap /opt/dump/dumpheap_"$HOSTNAME"_`date +%Y%m%d-%H%M%S`.hprof
# 导出当前线程信息
for i in `seq 3`
do 
    jattach 1 threaddump > /opt/dump/threaddump_"$HOSTNAME"_`date +%Y%m%d-%H%M%S`.log && sleep 1
done
# 导出当前使用CPU最高的线程
top -H -p 1 -n 3 -c -b > /opt/dump/cpudump_"$HOSTNAME"_`date +%Y%m%d-%H%M%S`.log
#!/bin/sh
# 导出当前内存信息
jattach 1 dumpheap /opt/dump/dumpheap_"$HOSTNAME"_`date +%Y%m%d-%H%M%S`.hprof
# 导出当前线程信息
for i in `seq 3`
do 
    jattach 1 threaddump > /opt/dump/threaddump_"$HOSTNAME"_`date +%Y%m%d-%H%M%S`.log && sleep 1
done
# 导出当前使用CPU最高的线程
top -H -p 1 -n 3 -c -b > /opt/dump/cpudump_"$HOSTNAME"_`date +%Y%m%d-%H%M%S`.log

使用使用MAT(MemoryAnalyzer Tool)进行分析

3.arthas ​

https://github.com/yilingyi/k8s-java-thread-dumper