SRE文档

深浅模式

1. ingress-nginx-controller Pod ​

nginx-ingress-controller的配置方法

1.1 Pod 容器内核优化 ​

在 ingress-nginx-controller 容器的 deploy.spec.template.spec 添加 initContainers

yaml
      initContainers:
        - name: systctl
          image: alpine:3.10
          imagePullPolicy: IfNotPresent
          command:
            - sh
            - -c
            - |
              mount -o remount rw /proc/sys
              sysctl -w net.core.somaxconn=65535
              sysctl -w net.ipv4.tcp_tw_reuse=1
              sysctl -w net.ipv4.ip_local_port_range="1024 65535"
              sysctl -w fs.file-max=1048576
              sysctl -w fs.inotify.max_user_instances=16384
              sysctl -w fs.inotify.max_user_watches=524288
              sysctl -w fs.inotify.max_queued_events=16384
          securityContext:
            privileged: true
      initContainers:
        - name: systctl
          image: alpine:3.10
          imagePullPolicy: IfNotPresent
          command:
            - sh
            - -c
            - |
              mount -o remount rw /proc/sys
              sysctl -w net.core.somaxconn=65535
              sysctl -w net.ipv4.tcp_tw_reuse=1
              sysctl -w net.ipv4.ip_local_port_range="1024 65535"
              sysctl -w fs.file-max=1048576
              sysctl -w fs.inotify.max_user_instances=16384
              sysctl -w fs.inotify.max_user_watches=524288
              sysctl -w fs.inotify.max_queued_events=16384
          securityContext:
            privileged: true

2 configmap ​

2.1 查看nginx.conf配置 ​

bash
[root@kube-master ingress]# kubectl -n ingress-nginx exec -ti ingress-nginx-controller-gcv4v cat /etc/nginx/nginx.conf
[root@kube-master ingress]# kubectl -n ingress-nginx exec -ti ingress-nginx-controller-gcv4v cat /etc/nginx/nginx.conf

2.2 ingress-nginx 资源查看 ​

# 查看 Ingress-nginx 全局配置参数:
kubectl get cm -n ingress-nginx nginx-ingress-controller -o yaml

# 修改 Ingress-nginx 全局配置参数:
kubectl edit cm -n ingress-nginx nginx-ingress-controller
# 查看 Ingress-nginx 全局配置参数:
kubectl get cm -n ingress-nginx nginx-ingress-controller -o yaml

# 修改 Ingress-nginx 全局配置参数:
kubectl edit cm -n ingress-nginx nginx-ingress-controller

2.3 参数 ​

# 负载工作机制,轮询
load-balance: "round_robin"

# 错误日志等级设置 (debug, info, notice, warn, error, crit, alert, or emerg)
error-log-level: "notice"

# 启用Gzip资源压缩 (3k以上)
use-gzip: "true"
gzip-level: "2"
gzip-min-length: "3072"
gzip-types: "text/html text/plain text/css text/javascript application/javascript application/x-javascript application/xml application/x-httpd-php application/x-font-ttf application/json image/x-icon image/svg+xml image/avif image/webp font/ttf font/opentype"
# 不建议进行照片压缩 image/jpeg image/gif image/png 可能反而会增加其体积

# 启用Brotli资源压缩（同等条件下优于Gzip，任选一个）
enable-brotli: "true"
brotli-level: 5
brotli-min-length: '3072'
brotli-types: 'text/xml image/svg+xml application/x-font-ttf image/vnd.microsoft.icon application/x-font-opentype application/json font/eot application/vnd.ms-fontobject application/javascript font/otf application/xml application/xhtml+xml text/javascript application/x-javascript text/plain application/x-font-truetype application/xml+rss image/x-icon font/opentype text/css image/x-win-bitmap'
# 不建议进行照片压缩 image/jpeg image/gif image/png 可能反而会增加其体积

# 启用http2支持(实际上默认是开启的，如果过关闭请将其设置为true)
use-http2: "true"

# ssl 会话复用
ssl_session_cache: "shared:SSL:10m;"
ssl-session-timeout: "10m"

# worker 每个工作进程可以打开的最大文件数与同时打开最大连接数设置
worker-processes: "auto" 
max-worker-open-files: "10240"
max-worker-connections: "32767"

# 连接复用
enable-multi-accept: "true"

# keep-alive 连接超时和最大请求数调整 
keep-alive: "75"
keep-alive-requests: "10000"

# upstream-keepalive 与上游Pod连接超时与最大请求数调整
upstream-keepalive-time: "30m"
upstream-keepalive-timeout: "60"
upstream-keepalive-requests: "10000"
upstream-keepalive-connections: "512"

# proxy-connect 设置 ingress-nginx 与 pstream pod 之间连接请求超时实践。
# 设置与代理服务器建立连接的超时时间(不能超过75s)
proxy-connect-timeout: "30"
# 设置将请求传输到代理服务器的超时时间（以秒为单位）（超时仅在两个连续的写操作之间设置，而不是为整个请求的传输设置）
proxy-send-timeout: "120"
# 设置从代理服务器读取响应的超时时间（以秒为单位）
proxy-read-timeout: "120"
# 负载工作机制,轮询
load-balance: "round_robin"

# 错误日志等级设置 (debug, info, notice, warn, error, crit, alert, or emerg)
error-log-level: "notice"

# 启用Gzip资源压缩 (3k以上)
use-gzip: "true"
gzip-level: "2"
gzip-min-length: "3072"
gzip-types: "text/html text/plain text/css text/javascript application/javascript application/x-javascript application/xml application/x-httpd-php application/x-font-ttf application/json image/x-icon image/svg+xml image/avif image/webp font/ttf font/opentype"
# 不建议进行照片压缩 image/jpeg image/gif image/png 可能反而会增加其体积

# 启用Brotli资源压缩（同等条件下优于Gzip，任选一个）
enable-brotli: "true"
brotli-level: 5
brotli-min-length: '3072'
brotli-types: 'text/xml image/svg+xml application/x-font-ttf image/vnd.microsoft.icon application/x-font-opentype application/json font/eot application/vnd.ms-fontobject application/javascript font/otf application/xml application/xhtml+xml text/javascript application/x-javascript text/plain application/x-font-truetype application/xml+rss image/x-icon font/opentype text/css image/x-win-bitmap'
# 不建议进行照片压缩 image/jpeg image/gif image/png 可能反而会增加其体积

# 启用http2支持(实际上默认是开启的，如果过关闭请将其设置为true)
use-http2: "true"

# ssl 会话复用
ssl_session_cache: "shared:SSL:10m;"
ssl-session-timeout: "10m"

# worker 每个工作进程可以打开的最大文件数与同时打开最大连接数设置
worker-processes: "auto" 
max-worker-open-files: "10240"
max-worker-connections: "32767"

# 连接复用
enable-multi-accept: "true"

# keep-alive 连接超时和最大请求数调整 
keep-alive: "75"
keep-alive-requests: "10000"

# upstream-keepalive 与上游Pod连接超时与最大请求数调整
upstream-keepalive-time: "30m"
upstream-keepalive-timeout: "60"
upstream-keepalive-requests: "10000"
upstream-keepalive-connections: "512"

# proxy-connect 设置 ingress-nginx 与 pstream pod 之间连接请求超时实践。
# 设置与代理服务器建立连接的超时时间(不能超过75s)
proxy-connect-timeout: "30"
# 设置将请求传输到代理服务器的超时时间（以秒为单位）（超时仅在两个连续的写操作之间设置，而不是为整个请求的传输设置）
proxy-send-timeout: "120"
# 设置从代理服务器读取响应的超时时间（以秒为单位）
proxy-read-timeout: "120"

keep-alive ​

bash
#查看
[root@kube-master ingress]# kubectl get configmaps -n ingress-nginx
NAME                       DATA   AGE
ingress-nginx-controller   3      3d1h

#编辑
[root@kube-master ingress]# kubectl edit configmaps ingress-nginx-controller -n ingress-nginx
...
apiVersion: v1
data:
  keep-alive: "60"
  keep-alive-requests: "10000"
  upstream-keepalive-connections: "10000"
  upstream-keepalive-requests: "100"
  upstream-keepalive-timeout: "60"
  max-worker-connections: "65536"
kind: ConfigMap
...
#查看
[root@kube-master ingress]# kubectl get configmaps -n ingress-nginx
NAME                       DATA   AGE
ingress-nginx-controller   3      3d1h

#编辑
[root@kube-master ingress]# kubectl edit configmaps ingress-nginx-controller -n ingress-nginx
...
apiVersion: v1
data:
  keep-alive: "60"
  keep-alive-requests: "10000"
  upstream-keepalive-connections: "10000"
  upstream-keepalive-requests: "100"
  upstream-keepalive-timeout: "60"
  max-worker-connections: "65536"
kind: ConfigMap
...
  • 参数介绍
yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-ingress-controller
# nginx ingress 性能优化: https://www.nginx.com/blog/tuning-nginx/
data:
  # nginx 与 client 保持的一个长连接能处理的请求数量，默认100，高并发场景建议调高。  
  # 参考: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#keep-alive-requests
  keep-alive-requests: "10000"
  # nginx 与 upstream 保持长连接的最大空闲连接数 (不是最大连接数)，默认 320，在高并发下场景下调大，避免频繁建联导致 TIME_WAIT 飙升。  
  # 参考: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#upstream-keepalive-connections
  upstream-keepalive-connections: "2000"
  # 每个 worker 进程可以打开的最大连接数，默认 16384。  
  # 参考: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#max-worker-connections
  max-worker-connections: "65536"
apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-ingress-controller
# nginx ingress 性能优化: https://www.nginx.com/blog/tuning-nginx/
data:
  # nginx 与 client 保持的一个长连接能处理的请求数量，默认100，高并发场景建议调高。  
  # 参考: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#keep-alive-requests
  keep-alive-requests: "10000"
  # nginx 与 upstream 保持长连接的最大空闲连接数 (不是最大连接数)，默认 320，在高并发下场景下调大，避免频繁建联导致 TIME_WAIT 飙升。  
  # 参考: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#upstream-keepalive-connections
  upstream-keepalive-connections: "2000"
  # 每个 worker 进程可以打开的最大连接数，默认 16384。  
  # 参考: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#max-worker-connections
  max-worker-connections: "65536"
  • 参考

Tuning NGINX for Performance

ngx_http_upstream_module 官方文档

3. ingress annotions规则 ​

3.1 参数 ​

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    # 解决: 413 Request Entity Too Large
    ingress.kubernetes.io/proxy-body-size: "50m"

    # 解决：后端大文件上传问题
    nginx.ingress.kubernetes.io/client-body-buffer-size: 50m
    nginx.ingress.kubernetes.io/proxy-max-temp-file-size: 100m

    # 解决: 上传文件较慢问题
    nginx.ingress.kubernetes.io/proxy-buffer-size: 50m
    nginx.ingress.kubernetes.io/proxy-buffering: "on"
    nginx.ingress.kubernetes.io/proxy-buffers-number: "4"

    # 解决: 与后端backend超时问题
    nginx.ingress.kubernetes.io/proxy-connect-timeout: 300s
    nginx.ingress.kubernetes.io/proxy-read-timeout: 180s
    nginx.ingress.kubernetes.io/proxy-send-timeout: 180s

    # 解决: 处理Nginx代理转发与后端服务文件上传缓存区设置(原生命令)
    nginx.ingress.kubernetes.io/server-snippet: |
      location ~ fastfile {
        client_max_body_size 1024m;   # 允许客户端请求的最大单文件字节数，人话：能上传多大文件
        client_body_buffer_size 10m;  # 缓冲区代理缓冲用户端请求的最大字节数，人话：一次能接受多少文件，建议根据带宽上限设置，减少磁盘读写，加快速度
        proxy_connect_timeout 300;    # Nginx与后端代理连接超时时间
        proxy_read_timeout 300;       # 后端服务器响应时间(代理接收超时)时间
        proxy_buffer_size 1024k;      # 设置代理服务器（nginx）保存用户头信息的缓冲区大小
        proxy_buffers 6 500k;             # proxy_buffers缓冲区，网页平均在32k以下的话>，这样设置
        proxy_busy_buffers_size 1024k;    # 高负荷下缓冲大小（proxy_buffers*2）
        proxy_temp_file_write_size 1024k; # 设定缓存文件夹大小，大于这个值将从upstream服务器传输
      }
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    # 解决: 413 Request Entity Too Large
    ingress.kubernetes.io/proxy-body-size: "50m"

    # 解决：后端大文件上传问题
    nginx.ingress.kubernetes.io/client-body-buffer-size: 50m
    nginx.ingress.kubernetes.io/proxy-max-temp-file-size: 100m

    # 解决: 上传文件较慢问题
    nginx.ingress.kubernetes.io/proxy-buffer-size: 50m
    nginx.ingress.kubernetes.io/proxy-buffering: "on"
    nginx.ingress.kubernetes.io/proxy-buffers-number: "4"

    # 解决: 与后端backend超时问题
    nginx.ingress.kubernetes.io/proxy-connect-timeout: 300s
    nginx.ingress.kubernetes.io/proxy-read-timeout: 180s
    nginx.ingress.kubernetes.io/proxy-send-timeout: 180s

    # 解决: 处理Nginx代理转发与后端服务文件上传缓存区设置(原生命令)
    nginx.ingress.kubernetes.io/server-snippet: |
      location ~ fastfile {
        client_max_body_size 1024m;   # 允许客户端请求的最大单文件字节数，人话：能上传多大文件
        client_body_buffer_size 10m;  # 缓冲区代理缓冲用户端请求的最大字节数，人话：一次能接受多少文件，建议根据带宽上限设置，减少磁盘读写，加快速度
        proxy_connect_timeout 300;    # Nginx与后端代理连接超时时间
        proxy_read_timeout 300;       # 后端服务器响应时间(代理接收超时)时间
        proxy_buffer_size 1024k;      # 设置代理服务器（nginx）保存用户头信息的缓冲区大小
        proxy_buffers 6 500k;             # proxy_buffers缓冲区，网页平均在32k以下的话>，这样设置
        proxy_busy_buffers_size 1024k;    # 高负荷下缓冲大小（proxy_buffers*2）
        proxy_temp_file_write_size 1024k; # 设定缓存文件夹大小，大于这个值将从upstream服务器传输
      }