SRE文档

深浅模式

bash
Ingres官方文档:
　　　　https://kubernetes.io/docs/concepts/services-networking/ingress/
　　　　
Ingress-Nginx github 地址：https://github.com/kubernetes/ingress-nginx
Ingress-Nginx 官方网站：https://kubernetes.github.io/ingress-nginx/
Ingres官方文档:
　　　　https://kubernetes.io/docs/concepts/services-networking/ingress/
　　　　
Ingress-Nginx github 地址：https://github.com/kubernetes/ingress-nginx
Ingress-Nginx 官方网站：https://kubernetes.github.io/ingress-nginx/

1. Ingress基于URL实现路由 ​

1.1 部署demo ​

  • 创建deploy yaml文件
yaml
[root@kube-master ingress]# cat app-deploy-service.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-prod
spec:
  replicas: 3
  selector:
    matchLabels:
      app: app-prod
      role: app-prod-role
  template:
    metadata:
      labels:
        app: app-prod
        role: app-prod-role
    spec:
      containers:
      - name: app
        image: registry.cn-zhangjiakou.aliyuncs.com/hsuing/demoapp:v1
        ports:
        - containerPort: 80
#Service
---
apiVersion: v1
kind: Service
metadata:
  name: app-prod-service
spec:
  type: NodePort
  selector:
    app: app-prod
    role: app-prod-role
  ports:
  - port: 80
    targetPort: 80
[root@kube-master ingress]# cat app-deploy-service.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-prod
spec:
  replicas: 3
  selector:
    matchLabels:
      app: app-prod
      role: app-prod-role
  template:
    metadata:
      labels:
        app: app-prod
        role: app-prod-role
    spec:
      containers:
      - name: app
        image: registry.cn-zhangjiakou.aliyuncs.com/hsuing/demoapp:v1
        ports:
        - containerPort: 80
#Service
---
apiVersion: v1
kind: Service
metadata:
  name: app-prod-service
spec:
  type: NodePort
  selector:
    app: app-prod
    role: app-prod-role
  ports:
  - port: 80
    targetPort: 80
  • 执行apply
[root@kube-master ingress]# kubectl apply  -f app-deploy-service.yaml
deployment.apps/app-prod created
service/app-prod-service created
[root@kube-master ingress]# kubectl apply  -f app-deploy-service.yaml
deployment.apps/app-prod created
service/app-prod-service created
  • 查看
bash
[root@kube-master ingress]# kubectl get svc
[root@kube-master ingress]# kubectl get svc
NAME               TYPE        CLUSTER-IP        EXTERNAL-IP   PORT(S)        AGE
app-prod-service   NodePort    192.168.172.245   <none>        80:32532/TCP   3m38s
[root@kube-master ingress]# kubectl get svc
[root@kube-master ingress]# kubectl get svc
NAME               TYPE        CLUSTER-IP        EXTERNAL-IP   PORT(S)        AGE
app-prod-service   NodePort    192.168.172.245   <none>        80:32532/TCP   3m38s

1.2 部署tomcat ​

  • 创建yaml文件
yaml
[root@kube-master ingress]# cat java-deploy-service.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-prod
spec:
  replicas: 2
  selector:
    matchLabels:
      role: java
  template:
    metadata:
      labels:
        role: java
    spec:
      containers:
      - name: tomcat-prod
        image: tomcat:9.0.63
        ports:
        - containerPort: 8080
        resources:
          limits:
            cpu: 100m
            memory: 128Mi
          requests:
            cpu: 50m
            memory: 64Mi
        lifecycle:
          postStart:
            exec:
              command: ["sh", "-c", "cp -rf /usr/local/tomcat/webapps.dist/* /usr/local/tomcat/webapps/"]
---
apiVersion: v1
kind: Service
metadata:
  name: tomcat-prod-service
spec:
  type: NodePort
  selector:
    role: java
  ports:
  - port: 80
    targetPort: 8080
[root@kube-master ingress]# cat java-deploy-service.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-prod
spec:
  replicas: 2
  selector:
    matchLabels:
      role: java
  template:
    metadata:
      labels:
        role: java
    spec:
      containers:
      - name: tomcat-prod
        image: tomcat:9.0.63
        ports:
        - containerPort: 8080
        resources:
          limits:
            cpu: 100m
            memory: 128Mi
          requests:
            cpu: 50m
            memory: 64Mi
        lifecycle:
          postStart:
            exec:
              command: ["sh", "-c", "cp -rf /usr/local/tomcat/webapps.dist/* /usr/local/tomcat/webapps/"]
---
apiVersion: v1
kind: Service
metadata:
  name: tomcat-prod-service
spec:
  type: NodePort
  selector:
    role: java
  ports:
  - port: 80
    targetPort: 8080
  • 执行
bash
kubectl apply  -f java-deploy-service.yaml
kubectl apply  -f java-deploy-service.yaml
  • 查看
bash
[root@kube-master ingress]# kubectl get svc
tomcat-prod-service   NodePort    192.168.204.162   <none>        80:30763/TCP   7m11s
[root@kube-master ingress]# kubectl get svc
tomcat-prod-service   NodePort    192.168.204.162   <none>        80:30763/TCP   7m11s

1.3 配置Ingress ​

yaml
[root@kube-master ingress]# cat ingress-app-java.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: foo-ingress
  annotations:
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
    nginx.ingress.kubernetes.io/rewrite-target: /$2 # 配置Rewrite规则
spec:
  ingressClassName: nginx
  rules:
  - host: foo.host.com
    http:
      paths:
      - path: /java(/|$)(.*)
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
      - path: /app(/|$)(.*)
        pathType: Prefix
        backend:
          service:
            name: app-prod-service
            port:
              number: 80
[root@kube-master ingress]# cat ingress-app-java.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: foo-ingress
  annotations:
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
    nginx.ingress.kubernetes.io/rewrite-target: /$2 # 配置Rewrite规则
spec:
  ingressClassName: nginx
  rules:
  - host: foo.host.com
    http:
      paths:
      - path: /java(/|$)(.*)
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
      - path: /app(/|$)(.*)
        pathType: Prefix
        backend:
          service:
            name: app-prod-service
            port:
              number: 80
  • 查看ingress
shell
[root@kube-master ingress]# kubectl get ingress
NAME          CLASS   HOSTS          ADDRESS           PORTS   AGE
foo-ingress   nginx   foo.host.com   192.168.104.132   80      29m
my-ingress    nginx   my.host.com    192.168.104.132   80      17h
[root@kube-master ingress]# kubectl get ingress
NAME          CLASS   HOSTS          ADDRESS           PORTS   AGE
foo-ingress   nginx   foo.host.com   192.168.104.132   80      29m
my-ingress    nginx   my.host.com    192.168.104.132   80      17h
  • 解析域名

image-20240521113217906

  • 验证结果

image-20240521112937741

image-20240521113113860

2. Ingress基于名称虚拟主机 ​

场景：将来⾃不同的域名的请求调度到不同Service

image-20240521114742782

2.1 部署demo ​

yaml
[root@kube-master ingress]# cat app-deploy-service.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-prod
spec:
  replicas: 3
  selector:
    matchLabels:
      app: app-prod
      role: app-prod-role
  template:
    metadata:
      labels:
        app: app-prod
        role: app-prod-role
    spec:
      containers:
      - name: app
        image: registry.cn-zhangjiakou.aliyuncs.com/hsuing/demoapp:v1
        ports:
        - containerPort: 80
#Service
---
apiVersion: v1
kind: Service
metadata:
  name: app-prod-service
spec:
  selector:
    app: app-prod
    role: app-prod-role
  ports:
  - port: 80
    targetPort: 80
[root@kube-master ingress]# cat app-deploy-service.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-prod
spec:
  replicas: 3
  selector:
    matchLabels:
      app: app-prod
      role: app-prod-role
  template:
    metadata:
      labels:
        app: app-prod
        role: app-prod-role
    spec:
      containers:
      - name: app
        image: registry.cn-zhangjiakou.aliyuncs.com/hsuing/demoapp:v1
        ports:
        - containerPort: 80
#Service
---
apiVersion: v1
kind: Service
metadata:
  name: app-prod-service
spec:
  selector:
    app: app-prod
    role: app-prod-role
  ports:
  - port: 80
    targetPort: 80

2.2 部署tomcat ​

yaml
[root@kube-master ingress]# cat java-deploy-service.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-prod
spec:
  replicas: 2
  selector:
    matchLabels:
      role: java
  template:
    metadata:
      labels:
        role: java
    spec:
      containers:
      - name: tomcat-prod
        image: tomcat:9.0.63
        ports:
        - containerPort: 8080
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        lifecycle:
          postStart:
            exec:
              command: ["/bin/bash", "-c", "cp -rf /usr/local/tomcat/webapps.dist/* /usr/local/tomcat/webapps"]
---
apiVersion: v1
kind: Service
metadata:
  name: tomcat-prod-service
spec:
  type: NodePort
  selector:
    role: java
  ports:
  - port: 80
    targetPort: 8080
[root@kube-master ingress]# cat java-deploy-service.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-prod
spec:
  replicas: 2
  selector:
    matchLabels:
      role: java
  template:
    metadata:
      labels:
        role: java
    spec:
      containers:
      - name: tomcat-prod
        image: tomcat:9.0.63
        ports:
        - containerPort: 8080
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        lifecycle:
          postStart:
            exec:
              command: ["/bin/bash", "-c", "cp -rf /usr/local/tomcat/webapps.dist/* /usr/local/tomcat/webapps"]
---
apiVersion: v1
kind: Service
metadata:
  name: tomcat-prod-service
spec:
  type: NodePort
  selector:
    role: java
  ports:
  - port: 80
    targetPort: 8080

2.3 创建ingress ​

yaml
[root@kube-master ingress]# cat ingress-domain.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
spec:
  ingressClassName: nginx
  rules:
  - host: app.host.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: app-prod-service
            port:
              number: 80
  - host: java.host.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
[root@kube-master ingress]# cat ingress-domain.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
spec:
  ingressClassName: nginx
  rules:
  - host: app.host.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: app-prod-service
            port:
              number: 80
  - host: java.host.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
  • 执行apply
shell
kubectl apply  -f ingress-domain.yaml
kubectl apply  -f ingress-domain.yaml
  • 查看ingress
bash
[root@kube-master ingress]# kubectl get ingress
NAME             CLASS   HOSTS                        ADDRESS           PORTS   AGE
ingress-domain   nginx   app.host.com,java.host.com   192.168.104.132   80      2m55s
[root@kube-master ingress]# kubectl get ingress
NAME             CLASS   HOSTS                        ADDRESS           PORTS   AGE
ingress-domain   nginx   app.host.com,java.host.com   192.168.104.132   80      2m55s
  • 验证效果

image-20240521142359404

image-20240521142420260

3. Ingress实现HTTPS ​

在 Ingress 中引⽤ Secret资源，然后告诉 Ingress 控制器使⽤TLS 加密从客户端到负载均衡器的通道

默认是强制http-https

3.1 创建TLS证书 ​

bash
#创建key
openssl  genrsa -out java.key 2048

#生成crt
openssl req -new -x509 -key java.key -out java.crt -subj "/C=CN/ST=Beijing/L=Beijing/O=host/CN=java.host.com"

#时间为一年
$ openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=java.host.com"
#创建key
openssl  genrsa -out java.key 2048

#生成crt
openssl req -new -x509 -key java.key -out java.crt -subj "/C=CN/ST=Beijing/L=Beijing/O=host/CN=java.host.com"

#时间为一年
$ openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=java.host.com"

3.2 创建Secrets ​

bash
[root@kube-master ingress]# kubectl create secret tls java-host-com-tls --key=java.key --cert=java.crt
secret/java-host-com-tls created
[root@kube-master ingress]# kubectl create secret tls java-host-com-tls --key=java.key --cert=java.crt
secret/java-host-com-tls created

3.3 配置Ingress ​

yaml
[root@kube-master ingress]# cat java-ingress-tls.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
[root@kube-master ingress]# cat java-ingress-tls.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80

或者

bash
cat <<EOF | kubectl create -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
EOF
cat <<EOF | kubectl create -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
EOF
  • 执行apply
shell
kubectl apply  -f java-ingress-tls.yaml
kubectl apply  -f java-ingress-tls.yaml
  • 查看ingress
bash
[root@kube-master ingress]# kubectl get ingress
NAME             CLASS   HOSTS           ADDRESS           PORTS     AGE
ingress-domain   nginx   java.host.com   192.168.104.132   80, 443   3m5s
[root@kube-master ingress]# kubectl get ingress
NAME             CLASS   HOSTS           ADDRESS           PORTS     AGE
ingress-domain   nginx   java.host.com   192.168.104.132   80, 443   3m5s
  • 验证效果

image-20240521143740530

3.4 更新证书 ​

查看证书名字 ​

bash
[root@kube-master ingress]# kubectl get secrets
NAME                  TYPE                                  DATA   AGE
java-host-com-tls     kubernetes.io/tls                     2      20m
[root@kube-master ingress]# kubectl get secrets
NAME                  TYPE                                  DATA   AGE
java-host-com-tls     kubernetes.io/tls                     2      20m

查看过期时间 ​

bash
[root@kube-master ingress]# openssl x509 -in java.crt -noout -dates
notBefore=May 21 06:42:13 2024 GMT
notAfter=Jun 20 06:42:13 2024 GMT
[root@kube-master ingress]# openssl x509 -in java.crt -noout -dates
notBefore=May 21 06:42:13 2024 GMT
notAfter=Jun 20 06:42:13 2024 GMT

或者

shell
#检查ingress里的配置证书的过期时间
for i in $(kubectl get ingress -o yaml|grep secretName:|awk '{print $2}'|sort|uniq);do echo -n "$i - "; kubectl get secret $i -o "jsonpath={.data['tls\.crt']}" | base64 -d | openssl x509 -enddate -noout;done

#检查所有创建的tls证书的过期时间
for i in $(kubectl get secret|awk '/tls/{print $1}');do echo -n "$i - "; kubectl get secret $i -o "jsonpath={.data['tls\.crt']}" | base64 -d | openssl x509 -enddate -noout;done
#检查ingress里的配置证书的过期时间
for i in $(kubectl get ingress -o yaml|grep secretName:|awk '{print $2}'|sort|uniq);do echo -n "$i - "; kubectl get secret $i -o "jsonpath={.data['tls\.crt']}" | base64 -d | openssl x509 -enddate -noout;done

#检查所有创建的tls证书的过期时间
for i in $(kubectl get secret|awk '/tls/{print $1}');do echo -n "$i - "; kubectl get secret $i -o "jsonpath={.data['tls\.crt']}" | base64 -d | openssl x509 -enddate -noout;done

备份 ​

shell
kubectl get secret java-host-com-tls -o yaml > secret-herlly-cn-https.yaml
kubectl get secret java-host-com-tls -o yaml > secret-herlly-cn-https.yaml

更新新的证书 ​

bash
#第一种，优雅替换
[root@kube-master ingress]# kubectl apply -f  <(kubectl create secret tls java-host-com-tls  --dry-run=client --key=java.key --cert=java.crt -oyaml)
secret/java-host-com-tls configured

kubectl apply -f  <(kubectl create secret tls java-host-com-tls  --save-config --dry-run=client --key=java.key --cert=java.crt -oyaml)

#第二种
TLS_KEY=$(base64 < "./tls.key" | tr -d '\n')
TLS_CRT=$(base64 < "./tls.crt" | tr -d '\n')

kubectl get secrets tls-rancher-ingress -o json \

        | jq '.data["tls.key"] |= "$TLS_KEY"' \

        | jq '.data["tls.crt"] |= "$TLS_CRT"' \

        | kubectl apply -f -
#第一种，优雅替换
[root@kube-master ingress]# kubectl apply -f  <(kubectl create secret tls java-host-com-tls  --dry-run=client --key=java.key --cert=java.crt -oyaml)
secret/java-host-com-tls configured

kubectl apply -f  <(kubectl create secret tls java-host-com-tls  --save-config --dry-run=client --key=java.key --cert=java.crt -oyaml)

#第二种
TLS_KEY=$(base64 < "./tls.key" | tr -d '\n')
TLS_CRT=$(base64 < "./tls.crt" | tr -d '\n')

kubectl get secrets tls-rancher-ingress -o json \

        | jq '.data["tls.key"] |= "$TLS_KEY"' \

        | jq '.data["tls.crt"] |= "$TLS_CRT"' \

        | kubectl apply -f -

回滚 ​

验证有问题，先恢复旧证书，问题解决后，再重新执行替换操作，重新验证。

shell
kubectl apply -f secret-herlly-cn-https.yaml
kubectl apply -f secret-herlly-cn-https.yaml

3.5 配置ssl ​

https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#ssl-protocols

ingress-nginx 默认使用 TLSv1.2 TLSv1.3

4. Ingress Rewrite ​

官方文档nginx-config

官方案例，https://kubernetes.github.io/ingress-nginx/examples/rewrite/

rewrite 可以使用下面的 anntations 进行控制：

名称描述值
nginx.ingress.kubernetes.io/rewrite-target将匹配到的url重定向到rewrite-target 注解指定路径string
nginx.ingress.kubernetes.io/ssl-redirect表示位置部分是否可访问SSL(当lngress包含证书时默认为True)bool
nginx.ingress.kubernetes.io/force-ssl-redirect强制重定向到HTTPS，即使入口没有启用TLSbool
nginx.ingress.kubernetes.io/app-root访问主域名的时候会自动跳转到app-root 注解指定的路径string
nginx.ingress.kubernetes.io/use-regex表示Ingress上定义的路径是否使用正则表达式boolbool

4.1 匹配/java ​

类似和nginx中proxy_pass

yaml
[root@kube-master ingress]# cat ingress-app-java.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: foo-ingress
  annotations:
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
    nginx.ingress.kubernetes.io/rewrite-target: /$2 # 配置Rewrite规则
spec:
  ingressClassName: nginx
  rules:
  - host: foo.host.com
    http:
      paths:
      - path: /java(/|$)(.*)
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
[root@kube-master ingress]# cat ingress-app-java.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: foo-ingress
  annotations:
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
    nginx.ingress.kubernetes.io/rewrite-target: /$2 # 配置Rewrite规则
spec:
  ingressClassName: nginx
  rules:
  - host: foo.host.com
    http:
      paths:
      - path: /java(/|$)(.*)
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80

4.2 自定义跳转 ​

yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80

4.3 新老域名交替 ​

yaml
[root@kube-master ingress]# cat ingress-nginx-whitelist.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/permanent-redirect: "https://www.baidu.com"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
[root@kube-master ingress]# cat ingress-nginx-whitelist.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/permanent-redirect: "https://www.baidu.com"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80

💡 说明

nginx.ingress.kubernetes.io/permanent-redirect: "https://www.baidu.com"

的上协议，否则会出现重定向次数过多（因域名如果也做了跳转）

5. Ingress自定义页面 ​

5.1 默认自带 ​

创建yaml文件 ​

yaml
[root@kube-master ingress]# cat nginx-error.yaml
---
apiVersion: v1
kind: Service
metadata:
  namespace: ingress-nginx
  name: nginx-errors
  labels:
    app.kubernetes.io/name: nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
spec:
  selector:
    app.kubernetes.io/name: nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
  ports:
  - port: 80
    targetPort: 8080
    name: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: ingress-nginx
  name: nginx-errors
  labels:
    app.kubernetes.io/name: nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: nginx-errors
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: nginx-errors
        app.kubernetes.io/part-of: ingress-nginx
    spec:
      containers:
      - name: nginx-error-server
        image: registry.cn-zhangjiakou.aliyuncs.com/hsuing/nginx-error:v1
        ports:
        - containerPort: 8080
        # Setting the environment variable DEBUG we can see the headers sent
        # by the ingress controller to the backend in the client response.
        # env:
        # - name: DEBUG
        #   value: "true"
[root@kube-master ingress]# cat nginx-error.yaml
---
apiVersion: v1
kind: Service
metadata:
  namespace: ingress-nginx
  name: nginx-errors
  labels:
    app.kubernetes.io/name: nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
spec:
  selector:
    app.kubernetes.io/name: nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
  ports:
  - port: 80
    targetPort: 8080
    name: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: ingress-nginx
  name: nginx-errors
  labels:
    app.kubernetes.io/name: nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: nginx-errors
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: nginx-errors
        app.kubernetes.io/part-of: ingress-nginx
    spec:
      containers:
      - name: nginx-error-server
        image: registry.cn-zhangjiakou.aliyuncs.com/hsuing/nginx-error:v1
        ports:
        - containerPort: 8080
        # Setting the environment variable DEBUG we can see the headers sent
        # by the ingress controller to the backend in the client response.
        # env:
        # - name: DEBUG
        #   value: "true"

执行 ​

bash
 kubectl apply -f nginx-error.yaml
 kubectl apply -f nginx-error.yaml

查看 ​

shell
[root@kube-master ingress]# kubectl get pod,svc -n ingress-nginx
NAME                                       READY   STATUS      RESTARTS   AGE
pod/nginx-errors-7b878fb7fb-9kjrm          1/1     Running     0          24m

NAME                                         TYPE        CLUSTER-IP        EXTERNAL-IP   PORT(S)                      AGE
                     45h
service/nginx-errors                         ClusterIP   192.168.207.125   <none>        80/TCP                       24m
[root@kube-master ingress]# kubectl get pod,svc -n ingress-nginx
NAME                                       READY   STATUS      RESTARTS   AGE
pod/nginx-errors-7b878fb7fb-9kjrm          1/1     Running     0          24m

NAME                                         TYPE        CLUSTER-IP        EXTERNAL-IP   PORT(S)                      AGE
                     45h
service/nginx-errors                         ClusterIP   192.168.207.125   <none>        80/TCP                       24m

修改configmap ​

bash
#查看有哪些configmap
[root@kube-master ingress]# kubectl get configmaps  -n ingress-nginx
NAME                       DATA   AGE
ingress-nginx-controller   2      45h

[root@kube-master ingress]# kubectl edit configmaps ingress-nginx-controller -n ingress-nginx
apiVersion: v1
data:
  allow-snippet-annotations: "true"
  custom-http-errors: "403,404,500,503" #添加此行
#查看有哪些configmap
[root@kube-master ingress]# kubectl get configmaps  -n ingress-nginx
NAME                       DATA   AGE
ingress-nginx-controller   2      45h

[root@kube-master ingress]# kubectl edit configmaps ingress-nginx-controller -n ingress-nginx
apiVersion: v1
data:
  allow-snippet-annotations: "true"
  custom-http-errors: "403,404,500,503" #添加此行

配置启动参数 ​

bash
[root@kube-master ingress]# kubectl edit ds ingress-nginx-controller  -n ingress-nginx

...
    spec:
      containers:
      - args:
        - /nginx-ingress-controller
        - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller
        - --election-id=ingress-controller-leader
        - --controller-class=k8s.io/ingress-nginx
        - --ingress-class=nginx
        - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
        - --validating-webhook=:8443
        - --validating-webhook-certificate=/usr/local/certificates/cert
        - --validating-webhook-key=/usr/local/certificates/key
        - --default-backend-service=ingress-nginx/nginx-errors  # 添加此行
...
[root@kube-master ingress]# kubectl edit ds ingress-nginx-controller  -n ingress-nginx

...
    spec:
      containers:
      - args:
        - /nginx-ingress-controller
        - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller
        - --election-id=ingress-controller-leader
        - --controller-class=k8s.io/ingress-nginx
        - --ingress-class=nginx
        - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
        - --validating-webhook=:8443
        - --validating-webhook-certificate=/usr/local/certificates/cert
        - --validating-webhook-key=/usr/local/certificates/key
        - --default-backend-service=ingress-nginx/nginx-errors  # 添加此行
...
  • 测试效果

image-20240522141059026

5.2 定制自定义错误页面 ​

创建yaml ​

yaml
[root@kube-master ingress]# cat custom-nginx-error.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: custom-nginx-errors
  labels:
    app.kubernetes.io/name: custom-nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
  namespace: ingress-nginx
spec:
  selector:
    app.kubernetes.io/name: custom-nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
  ports:
  - port: 80
    targetPort: 80
    name: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: custom-nginx-errors
  labels:
    app.kubernetes.io/name: custom-nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
  namespace: ingress-nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: custom-nginx-errors
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: custom-nginx-errors
        app.kubernetes.io/part-of: ingress-nginx
    spec:
      containers:
      - name: custom-nginx-errors
        image: registry.cn-zhangjiakou.aliyuncs.com/hsuing/nginx-error:v2
        ports:
        - containerPort: 80
        # Setting the environment variable DEBUG we can see the headers sent
        # by the ingress controller to the backend in the client response.
        # env:
        # - name: DEBUG
        #   value: "true"
[root@kube-master ingress]# cat custom-nginx-error.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: custom-nginx-errors
  labels:
    app.kubernetes.io/name: custom-nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
  namespace: ingress-nginx
spec:
  selector:
    app.kubernetes.io/name: custom-nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
  ports:
  - port: 80
    targetPort: 80
    name: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: custom-nginx-errors
  labels:
    app.kubernetes.io/name: custom-nginx-errors
    app.kubernetes.io/part-of: ingress-nginx
  namespace: ingress-nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: custom-nginx-errors
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: custom-nginx-errors
        app.kubernetes.io/part-of: ingress-nginx
    spec:
      containers:
      - name: custom-nginx-errors
        image: registry.cn-zhangjiakou.aliyuncs.com/hsuing/nginx-error:v2
        ports:
        - containerPort: 80
        # Setting the environment variable DEBUG we can see the headers sent
        # by the ingress controller to the backend in the client response.
        # env:
        # - name: DEBUG
        #   value: "true"

执行 ​

bash
kubectl apply -f custom-nginx-error.yaml
kubectl apply -f custom-nginx-error.yaml

查看 ​

bash
[root@kube-master ingress]# kubectl get pod,svc,ds -n ingress-nginx
NAME                                       READY   STATUS      RESTARTS   AGE
pod/custom-nginx-errors-6df8ddcc64-tprjh   1/1     Running     0          18m

NAME                                         TYPE        CLUSTER-IP        EXTERNAL-IP   PORT(S)                      AGE
service/custom-nginx-errors                  ClusterIP   192.168.139.222   <none>        80/TCP                       18m

NAME                                      DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR    AGE
daemonset.apps/ingress-nginx-controller   2         2         2       2            2           kubernetes.io/os=linux,node-role=ingress   45h
[root@kube-master ingress]# kubectl get pod,svc,ds -n ingress-nginx
NAME                                       READY   STATUS      RESTARTS   AGE
pod/custom-nginx-errors-6df8ddcc64-tprjh   1/1     Running     0          18m

NAME                                         TYPE        CLUSTER-IP        EXTERNAL-IP   PORT(S)                      AGE
service/custom-nginx-errors                  ClusterIP   192.168.139.222   <none>        80/TCP                       18m

NAME                                      DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR    AGE
daemonset.apps/ingress-nginx-controller   2         2         2       2            2           kubernetes.io/os=linux,node-role=ingress   45h

修改configmap ​

参考5.1

配置启动参数 ​

bash
[root@kube-master ingress]#  kubectl edit ds ingress-nginx-controller  -n ingress-nginx
- args:
        - /nginx-ingress-controller
        - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller
        - --election-id=ingress-controller-leader
        - --controller-class=k8s.io/ingress-nginx
        - --ingress-class=nginx
        - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
        - --validating-webhook=:8443
        - --validating-webhook-certificate=/usr/local/certificates/cert
        - --validating-webhook-key=/usr/local/certificates/key
        - --default-backend-service=ingress-nginx/custom-nginx-errors #修改此处
[root@kube-master ingress]#  kubectl edit ds ingress-nginx-controller  -n ingress-nginx
- args:
        - /nginx-ingress-controller
        - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller
        - --election-id=ingress-controller-leader
        - --controller-class=k8s.io/ingress-nginx
        - --ingress-class=nginx
        - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
        - --validating-webhook=:8443
        - --validating-webhook-certificate=/usr/local/certificates/cert
        - --validating-webhook-key=/usr/local/certificates/key
        - --default-backend-service=ingress-nginx/custom-nginx-errors #修改此处
  • 测试效果

image-20240522144812526

6. Ingress自定义日志格式 ​

yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: my-ingress
  annotations:
    nginx.ingress.kubernetes.io/enable-access-log: "true"
    nginx.ingress.kubernetes.io/access-log-format: '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'
spec:
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: my-ingress
  annotations:
    nginx.ingress.kubernetes.io/enable-access-log: "true"
    nginx.ingress.kubernetes.io/access-log-format: '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'
spec:

7. Ingress客户端真实IP ​

  • 第一种
shell
kubectl edit configmaps ingress-nginx-controller  -n ingress-nginx

apiVersion: v1
data:
  use-forwarded-headers: "true"
  compute-full-forwarded-for: "true"
kubectl edit configmaps ingress-nginx-controller  -n ingress-nginx

apiVersion: v1
data:
  use-forwarded-headers: "true"
  compute-full-forwarded-for: "true"
  • 第二种
bash
apiVersion: v1
data:
  http-snippet: |
    real_ip_header X-Forwarded-For;
apiVersion: v1
data:
  http-snippet: |
    real_ip_header X-Forwarded-For;
  • 创建yaml
yaml
[root@kube-master ingress]# cat echoserver-ingress.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: ingress-nginx
  name: echoserver
  labels:
    app: echoserver
spec:
  selector:
    matchLabels:
      app: echoserver
  template:
    metadata:
      labels:
        app: echoserver
    spec:
      containers:
      - name: echoserver
        image: registry.cn-zhangjiakou.aliyuncs.com/hsuing/echoserver:latest
        ports:
        - containerPort: 8080
        env:
          - name: NODE_NAME
            valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
          - name: POD_NAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: POD_NAMESPACE
            valueFrom:
              fieldRef:
                fieldPath: metadata.namespace
          - name: POD_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP

---
apiVersion: v1
kind: Service
metadata:
  namespace: ingress-nginx
  name: echoserver
  labels:
    app: echoserver
spec:
  ports:
  - port: 80
    targetPort: 8080
    name: http
  selector:
    app: echoserver

---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  namespace: ingress-nginx
  name: echoserver
spec:
  ingressClassName: nginx
  rules:
  - host: echo.host.com
    http:
      paths:
      - path: "/"
        pathType: Prefix
        backend:
          service:
            name: echoserver
            port:
              number: 80
[root@kube-master ingress]# cat echoserver-ingress.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: ingress-nginx
  name: echoserver
  labels:
    app: echoserver
spec:
  selector:
    matchLabels:
      app: echoserver
  template:
    metadata:
      labels:
        app: echoserver
    spec:
      containers:
      - name: echoserver
        image: registry.cn-zhangjiakou.aliyuncs.com/hsuing/echoserver:latest
        ports:
        - containerPort: 8080
        env:
          - name: NODE_NAME
            valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
          - name: POD_NAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: POD_NAMESPACE
            valueFrom:
              fieldRef:
                fieldPath: metadata.namespace
          - name: POD_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP

---
apiVersion: v1
kind: Service
metadata:
  namespace: ingress-nginx
  name: echoserver
  labels:
    app: echoserver
spec:
  ports:
  - port: 80
    targetPort: 8080
    name: http
  selector:
    app: echoserver

---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  namespace: ingress-nginx
  name: echoserver
spec:
  ingressClassName: nginx
  rules:
  - host: echo.host.com
    http:
      paths:
      - path: "/"
        pathType: Prefix
        backend:
          service:
            name: echoserver
            port:
              number: 80

7.1 配置 ​

https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-real-ip-cidr

yaml
---
apiVersion: v1
data:
 #realip
  compute-full-forwarded-for: 'true'
  enable-real-ip: 'true'
  enable-underscores-in-headers: 'true'
  forwarded-for-header: X_FORWARDED_FOR
  proxy-real-ip-cidr: 192.168.10.1/24,10.41.40.21/28,172.20.0.0/16 #根据自己的网络进行修改
  proxy-set-headers: X_FORWARDED_FOR
  use-forwarded-headers: 'true'
---
apiVersion: v1
data:
 #realip
  compute-full-forwarded-for: 'true'
  enable-real-ip: 'true'
  enable-underscores-in-headers: 'true'
  forwarded-for-header: X_FORWARDED_FOR
  proxy-real-ip-cidr: 192.168.10.1/24,10.41.40.21/28,172.20.0.0/16 #根据自己的网络进行修改
  proxy-set-headers: X_FORWARDED_FOR
  use-forwarded-headers: 'true'

参数解析:

  1. enable-underscores-in-headers: 是否在标题名称中启用下划线, 缺省默认为off，表示如果请求中header name 中包含下划线，则忽略掉不会传递到后端代理或者应用程序，即获取不到该Header，所以此处为了不丢弃A10传递过来的 X_FORWARDED_FOR Header 需要将该参数设置为True。
  2. use-forwarded-headers: 如果设置为True时，则将设定的X-Forwarded-* Header传递给后端, 当Ingress在L7 代理/负载均衡器之后使用此选项。 如果设置为 false 时，则会忽略传入的X-Forwarded-*Header, 当 Ingress 直接暴露在互联网或者 L3/数据包的负载均衡器后面,并且不会更改数据包中的源 IP请使用此选项。
  3. forwarded-for-header: 设置用于标识客户端的原始 IP 地址的 Header 字段。默认值X-Forwarded-For，此处由于A10带入的是自定义记录IP的Header,所以此处填入是X_FORWARDED_FOR.
  4. compute-full-forwarded-for: 将 remote address 附加到 X-Forwarded-For Header而不是替换它。当启用此选项后端应用程序负责根据自己的受信任代理列表排除并提取客户端 IP。
  5. proxy-real-ip-cidr: 如果启用 use-forwarded-headers 或 use-proxy-protocol，则可以使用该参数其定义了外部负载衡器 、网络代理、CDN等地址，多个地址可以以逗号分隔的 CIDR 。默认值: “0.0.0.0/0”

上述参数配置后在ingress-nginx-control中的/etc/nginx/nginx.conf结果如下:

bash
# use-forwarded-headers 参数设置后
real_ip_header      X_FORWARDED_FOR;
real_ip_recursive   on;
set_real_ip_from    192.168.10.11;
set_real_ip_from    192.168.4.11;

# We can't use $proxy_add_x_forwarded_for because the realip module replaces the remote_addr too soon
# 我们不能使用 `$proxy_add_x_forwarded_for`, 因为 realip 模块过早地替换了远程 remote_addr。
map $http_x_forwarded_for $full_x_forwarded_for {
  default          "$http_x_forwarded_for, $realip_remote_addr";
  ''               "$realip_remote_addr";
}
# use-forwarded-headers 参数设置后
real_ip_header      X_FORWARDED_FOR;
real_ip_recursive   on;
set_real_ip_from    192.168.10.11;
set_real_ip_from    192.168.4.11;

# We can't use $proxy_add_x_forwarded_for because the realip module replaces the remote_addr too soon
# 我们不能使用 `$proxy_add_x_forwarded_for`, 因为 realip 模块过早地替换了远程 remote_addr。
map $http_x_forwarded_for $full_x_forwarded_for {
  default          "$http_x_forwarded_for, $realip_remote_addr";
  ''               "$realip_remote_addr";
}
  • logformat配置
bash
# /etc/nginx/nginx.conf
log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
  '$status $body_bytes_sent "$http_referer" '
  '"$http_user_agent" "$http_x_forwarded_for" - "$http_X_FORWARDED_FOR" - "$http_X_Real_IP"';
# 输出结果 "10.41.40.22" -- "10.41.40.22" --  "10.20.172.103"  

proxy_set_header X-Real-IP              $remote_addr;   # realip 模块 已经将 X_FORWARDED_FOR 字段赋值给 $remote_addr, 所以该字段也记录了真实IP.
proxy_set_header X-Forwarded-For        $full_x_forwarded_for;
# /etc/nginx/nginx.conf
log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
  '$status $body_bytes_sent "$http_referer" '
  '"$http_user_agent" "$http_x_forwarded_for" - "$http_X_FORWARDED_FOR" - "$http_X_Real_IP"';
# 输出结果 "10.41.40.22" -- "10.41.40.22" --  "10.20.172.103"  

proxy_set_header X-Real-IP              $remote_addr;   # realip 模块 已经将 X_FORWARDED_FOR 字段赋值给 $remote_addr, 所以该字段也记录了真实IP.
proxy_set_header X-Forwarded-For        $full_x_forwarded_for;
bash
tcpdump -nnX port 80 -vv -w test.pcap
tcpdump -nnX port 80 -vv -w test.pcap

8. Ingress黑白名单 ​

💡 说明

Annotations：只对指定svc的lngress生效;

ConfigMap：全局生效;

黑名单可以使用ConfigMap去配置，白名单建议使用Annotations去配置。

  • 名单是默认是拒绝所有，只允许一个地址去访问;
  • 黑名单是不允许该地址去访问所有;

若是同时配置了Annotations和configmap，一般都是annotations生效,configmap不生效，因为annotations优先级比configmap高;

官方文档，https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#whitelist-source-range

8.1 白名单 ​

yaml
[root@kube-master ingress]# cat ingress-nginx-whitelist.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
    nginx.ingress.kubernetes.io/whitelist-source-range: 10.103.236.204
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
[root@kube-master ingress]# cat ingress-nginx-whitelist.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
    nginx.ingress.kubernetes.io/whitelist-source-range: 10.103.236.204
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80

多个ip用逗号隔开

  • 测试效果

image-20240523144533008

bash
#其他机器
[root@kube-master ingress]# curl -I -k -H "Host:java.host.com" 10.103.236.202/java/index.jsp
HTTP/1.1 403 Forbidden
Date: Thu, 23 May 2024 06:46:07 GMT
Content-Type: text/html
Content-Length: 107786
Connection: keep-alive
Vary: Accept-Encoding
ETag: "5fd9999d-1a50a"
#其他机器
[root@kube-master ingress]# curl -I -k -H "Host:java.host.com" 10.103.236.202/java/index.jsp
HTTP/1.1 403 Forbidden
Date: Thu, 23 May 2024 06:46:07 GMT
Content-Type: text/html
Content-Length: 107786
Connection: keep-alive
Vary: Accept-Encoding
ETag: "5fd9999d-1a50a"

8.2 黑名单 ​

全局上面配置 ​

bash
#查看
[root@kube-master ingress]# kubectl get configmaps -n ingress-nginx
NAME                       DATA   AGE
ingress-nginx-controller   3      2d21h

#编辑
[root@kube-master ingress]# kubectl edit configmaps ingress-nginx-controller -n ingress-nginx
apiVersion: v1
data:
  block-cidrs: 192.168.1.11
#查看
[root@kube-master ingress]# kubectl get configmaps -n ingress-nginx
NAME                       DATA   AGE
ingress-nginx-controller   3      2d21h

#编辑
[root@kube-master ingress]# kubectl edit configmaps ingress-nginx-controller -n ingress-nginx
apiVersion: v1
data:
  block-cidrs: 192.168.1.11

annotations ​

  • 创建

nginx.ingress.kubernetes.io/server-snippet:

yaml
[root@kube-master ingress]# cat ingress-nginx-blackList.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
    nginx.ingress.kubernetes.io/server-snippet: |
      deny 10.103.236.203;
      allow all;
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
[root@kube-master ingress]# cat ingress-nginx-blackList.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
    nginx.ingress.kubernetes.io/server-snippet: |
      deny 10.103.236.203;
      allow all;
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80

  • 执行apply

  • 验证效果

image-20240523145537815

❌ 注意

同理annotations也能在白名单上面配置

9. Ingress基本认证 ​

9.1 创建nginx认证用户 ​

bash
## 用http的命令工具来生成
yum -y install httpd-tools

#创建用户
[root@kube-master ingress]# htpasswd -c auth admin
New password:
Re-type new password:
Adding password for user admin


# 查看密文
cat auth
## 用http的命令工具来生成
yum -y install httpd-tools

#创建用户
[root@kube-master ingress]# htpasswd -c auth admin
New password:
Re-type new password:
Adding password for user admin


# 查看密文
cat auth

新增用户 ​

bash
[root@kube-master ingress]# htpasswd -b auth username password
Adding password for user admin
[root@kube-master ingress]# htpasswd -b auth username password
Adding password for user admin

9.2 创建的密码文件用secrets资源存储 ​

需要将secret认证文件和业务pod放在同一个 名称空间下，否则出现错误

  • 创建名字nginx-basic-auth
bash
kubectl create secret generic nginx-basic-auth --from-file=auth
kubectl create secret generic nginx-basic-auth --from-file=auth
  • 查看
bash
[root@kube-master ingress]# kubectl get secrets 
NAME                                  TYPE                                  DATA   AGE
nginx-basic-auth                      Opaque                                1      39s
[root@kube-master ingress]# kubectl get secrets 
NAME                                  TYPE                                  DATA   AGE
nginx-basic-auth                      Opaque                                1      39s
  • 创建yaml配置文件
yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    # 指定认证类型
    nginx.ingress.kubernetes.io/auth-type: basic
    # 登录的提示信息
    nginx.ingress.kubernetes.io/auth-realm: Please Input Your Username and Passowrd
    # 对应认证信息，也就是我们创建的secrets资源名称，里面保存了我们创建的有效用户
    nginx.ingress.kubernetes.io/auth-secret: nginx-basic-auth

    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    # 指定认证类型
    nginx.ingress.kubernetes.io/auth-type: basic
    # 登录的提示信息
    nginx.ingress.kubernetes.io/auth-realm: Please Input Your Username and Passowrd
    # 对应认证信息，也就是我们创建的secrets资源名称，里面保存了我们创建的有效用户
    nginx.ingress.kubernetes.io/auth-secret: nginx-basic-auth

    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
  • 验证效果

image-20240523132336640

更新secret ​

bash
#新添加用户之后，更新secret
kubectl apply -f <(kubectl create secret generic nginx-basic-auth --dry-run=client --from-file=auth -oyaml)
#新添加用户之后，更新secret
kubectl apply -f <(kubectl create secret generic nginx-basic-auth --dry-run=client --from-file=auth -oyaml)

10. 跨域 ​

文档，https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#enable-cors

  • 参数
yaml
nginx.ingress.kubernetes.io/cors-allow-headers: |-
       DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization
    nginx.ingress.kubernetes.io/cors-allow-methods: 'GET, POST, OPTIONS'
    nginx.ingress.kubernetes.io/cors-allow-origin: '*'
    nginx.ingress.kubernetes.io/enable-cors: 'true'
nginx.ingress.kubernetes.io/cors-allow-headers: |-
       DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization
    nginx.ingress.kubernetes.io/cors-allow-methods: 'GET, POST, OPTIONS'
    nginx.ingress.kubernetes.io/cors-allow-origin: '*'
    nginx.ingress.kubernetes.io/enable-cors: 'true'
  • 案例
yaml
[root@kube-master ingress]# cat  ingress-nginx-pc.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
    nginx.ingress.kubernetes.io/server-snippet: |-
      set $agentflag 0;
      if ($http_user_agent ~* "(iPhone|iPod|android)" ){
         set $agentflag 1;
      }
      if ( $agentflag = 1 ) {
        return 302 https://m.baidu.com;
      }
    nginx.ingress.kubernetes.io/cors-allow-headers: |-
       DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization
    nginx.ingress.kubernetes.io/cors-allow-methods: 'GET, POST, OPTIONS'
    nginx.ingress.kubernetes.io/cors-allow-origin: '*'
    nginx.ingress.kubernetes.io/enable-cors: 'true'
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
[root@kube-master ingress]# cat  ingress-nginx-pc.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
    nginx.ingress.kubernetes.io/server-snippet: |-
      set $agentflag 0;
      if ($http_user_agent ~* "(iPhone|iPod|android)" ){
         set $agentflag 1;
      }
      if ( $agentflag = 1 ) {
        return 302 https://m.baidu.com;
      }
    nginx.ingress.kubernetes.io/cors-allow-headers: |-
       DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization
    nginx.ingress.kubernetes.io/cors-allow-methods: 'GET, POST, OPTIONS'
    nginx.ingress.kubernetes.io/cors-allow-origin: '*'
    nginx.ingress.kubernetes.io/enable-cors: 'true'
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
  • 验证效果

image-20240523162922570

11. http和https共存 ​

参数，nginx.ingress.kubernetes.io/ssl-redirect: "false"

  • 创建
yaml
[root@kube-master ingress]# cat ingress-nginx-whitelist.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
[root@kube-master ingress]# cat ingress-nginx-whitelist.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80

  • 执行apply

  • 查看

bash
[root@kube-master ingress]# kubectl get ingress
NAME             CLASS   HOSTS           ADDRESS           PORTS     AGE
ingress-domain   nginx   java.host.com   192.168.104.132   80, 443   16m


[root@kube-master ingress]# kubectl describe ingress ingress-domain
...
Annotations:     nginx.ingress.kubernetes.io/configuration-snippet:
                   rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
                   rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
                   rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
                 nginx.ingress.kubernetes.io/rewrite-target: /$2
                 nginx.ingress.kubernetes.io/ssl-redirect: false
                 nginx.ingress.kubernetes.io/use-regex: true
...
[root@kube-master ingress]# kubectl get ingress
NAME             CLASS   HOSTS           ADDRESS           PORTS     AGE
ingress-domain   nginx   java.host.com   192.168.104.132   80, 443   16m


[root@kube-master ingress]# kubectl describe ingress ingress-domain
...
Annotations:     nginx.ingress.kubernetes.io/configuration-snippet:
                   rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
                   rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
                   rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
                 nginx.ingress.kubernetes.io/rewrite-target: /$2
                 nginx.ingress.kubernetes.io/ssl-redirect: false
                 nginx.ingress.kubernetes.io/use-regex: true
...

12. ingress重定向307跳转，https无法切换回http访问 ​

K8S官网：https://kubernetes.io/

ingress官方文档：https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#hsts

出现这个的原因是hsts

  • 检查是否关闭强制SSL转跳
bash
在对应pod的ingress配置文件中检查是否关闭强制SSL跳转
nginx.ingress.kubernetes.io/ssl-redirect: “false”
在对应pod的ingress配置文件中检查是否关闭强制SSL跳转
nginx.ingress.kubernetes.io/ssl-redirect: “false”
  • 检查nginx-ingress的configmap配置文件禁用HSTS

当出现307状态码时，就是HSTS保护策略拒绝了HTTP请求，这种情况单纯清理缓存无法处理，需要使用无痕窗口测试，当第一次访问http正常，访问过https再切换http出现307时就是HSTS策略问题

yaml
# 在data里面添加hsts: "false"
apiVersion: v1
data:
  hsts: "false"
# 在data里面添加hsts: "false"
apiVersion: v1
data:
  hsts: "false"
  • 重启ingress-controller

找到对应的nginx-ingress服务pod，进行无配置文件更新

bash
#查看pod
[root@kube-master ingress]# kubectl get pod -n ingress-nginx
NAME                                   READY   STATUS      RESTARTS        AGE
ingress-nginx-admission-create-c799q   0/1     Completed   0               2d21h
ingress-nginx-admission-patch-wzm82    0/1     Completed   0               2d21h
ingress-nginx-controller-gcv4v         1/1     Running     1 (4h45m ago)   24h
ingress-nginx-controller-hs58n         1/1     Running     1 (4h45m ago)   24h


# 重启ingrss服务，分别将所有ingress的pod都执行一边即可
kubectl get pod {podname} -n {namespace} -o yaml | kubectl replace --force -f -
#查看pod
[root@kube-master ingress]# kubectl get pod -n ingress-nginx
NAME                                   READY   STATUS      RESTARTS        AGE
ingress-nginx-admission-create-c799q   0/1     Completed   0               2d21h
ingress-nginx-admission-patch-wzm82    0/1     Completed   0               2d21h
ingress-nginx-controller-gcv4v         1/1     Running     1 (4h45m ago)   24h
ingress-nginx-controller-hs58n         1/1     Running     1 (4h45m ago)   24h


# 重启ingrss服务，分别将所有ingress的pod都执行一边即可
kubectl get pod {podname} -n {namespace} -o yaml | kubectl replace --force -f -

13. ingress图片文件上传 返回413错误码 ​

K8S官网：https://kubernetes.io/

ingress官方文档：https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-body-size

  • 修改configmap配置文件
bash
[root@kube-master ingress]# kubectl get configmap -n ingress-nginx
NAME                       DATA   AGE
ingress-nginx-controller   3      2d21h

[root@kube-master ingress]# kubectl edit configmap ingress-nginx-controller -n ingress-nginx
# 在metadata:---annotations:添加
nginx.ingress.kubernetes.io/proxy-body-size: 200m

metadata:
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: 200m
[root@kube-master ingress]# kubectl get configmap -n ingress-nginx
NAME                       DATA   AGE
ingress-nginx-controller   3      2d21h

[root@kube-master ingress]# kubectl edit configmap ingress-nginx-controller -n ingress-nginx
# 在metadata:---annotations:添加
nginx.ingress.kubernetes.io/proxy-body-size: 200m

metadata:
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: 200m
  • 重启ingress-controller
bash
#查看pod
[root@kube-master ingress]# kubectl get pod -n ingress-nginx
NAME                                   READY   STATUS      RESTARTS        AGE
ingress-nginx-admission-create-c799q   0/1     Completed   0               2d21h
ingress-nginx-admission-patch-wzm82    0/1     Completed   0               2d21h
ingress-nginx-controller-gcv4v         1/1     Running     1 (4h45m ago)   24h
ingress-nginx-controller-hs58n         1/1     Running     1 (4h45m ago)   24h


# 重启ingrss服务，分别将所有ingress的pod都执行一边即可
kubectl get pod {podname} -n {namespace} -o yaml | kubectl replace --force -f -
#查看pod
[root@kube-master ingress]# kubectl get pod -n ingress-nginx
NAME                                   READY   STATUS      RESTARTS        AGE
ingress-nginx-admission-create-c799q   0/1     Completed   0               2d21h
ingress-nginx-admission-patch-wzm82    0/1     Completed   0               2d21h
ingress-nginx-controller-gcv4v         1/1     Running     1 (4h45m ago)   24h
ingress-nginx-controller-hs58n         1/1     Running     1 (4h45m ago)   24h


# 重启ingrss服务，分别将所有ingress的pod都执行一边即可
kubectl get pod {podname} -n {namespace} -o yaml | kubectl replace --force -f -

14. ingress pc端和手机端区分 ​

参数，nginx.ingress.kubernetes.io/server-snippet

yaml
[root@kube-master ingress]# cat ingress-nginx-pc.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
    nginx.ingress.kubernetes.io/server-snippet: |-
      set $agentflag 0;
      if ($http_user_agent ~* "(iPhone|iPod|android)" ){
         set $agentflag 1;
      }
      if ( $agentflag = 1 ) {
        return 302 https://m.baidu.com;
      }
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80
[root@kube-master ingress]# cat ingress-nginx-pc.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-domain
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/configuration-snippet: |
      rewrite ^/manager/(.*)$ /java/manager/$1 redirect;
      rewrite ^/host-manager/(.*)$ /java/host-manager/$1 redirect;
      rewrite ^/docs/(.*)$ /java/docs/$1 redirect;
    nginx.ingress.kubernetes.io/server-snippet: |-
      set $agentflag 0;
      if ($http_user_agent ~* "(iPhone|iPod|android)" ){
         set $agentflag 1;
      }
      if ( $agentflag = 1 ) {
        return 302 https://m.baidu.com;
      }
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - java.host.com
    secretName: java-host-com-tls # secrets资源名称
  rules:
  - host: java.host.com
    http:
      paths:
      - path: "/java(/|$)(.*)"
        pathType: Prefix
        backend:
          service:
            name: tomcat-prod-service
            port:
              number: 80

15. 开始brotli压缩 ​

yaml
---
apiVersion: v1
data:
  enable-brotli: 'true'
  brotli-level: '6'
  brotli-types: 'text/xml image/svg+xml application/x-font-ttf image/vnd.microsoft.icon application/x-font-opentype application/json font/eot application/vnd.ms-fontobject application/javascript font/otf application/xml application/xhtml+xml text/javascript application/x-javascript text/plain application/x-font-truetype application/xml+rss image/x-icon font/opentype text/css image/x-win-bitmap'
---
apiVersion: v1
data:
  enable-brotli: 'true'
  brotli-level: '6'
  brotli-types: 'text/xml image/svg+xml application/x-font-ttf image/vnd.microsoft.icon application/x-font-opentype application/json font/eot application/vnd.ms-fontobject application/javascript font/otf application/xml application/xhtml+xml text/javascript application/x-javascript text/plain application/x-font-truetype application/xml+rss image/x-icon font/opentype text/css image/x-win-bitmap'

16. 开启http2 ​

默认是开启的

17. 速率限制 ​

17.1 参数 ​

yaml
# 配置映射-ConfigMap
$ kubectl get cm -n ingress-nginx ingress-nginx-controller -o yaml
data:
  http-snippet: |
    limit_req_zone $http_authorization zone=my-zone:20m rate=5r/s;
    limit_req_zone $binary_remote_addr zone=my-zone:20m rate=10r/s;
    limit_req_zone $http_someheader zone=my-zone:20m rate=20r/s;

# 配置注释-annotations
# - 入口资源中的注释：
metadata:
  annotations:
    nginx.ingress.kubernetes.io/configuration-snippet: |
      limit_req zone=my-zone-1 burst=10 nodelay;
      limit_req_log_level notice;
      limit_req_status 429;

# - 一个入口定义的不同位置的不同节流示例：
metadata:
  annotations:
    nginx.ingress.kubernetes.io/server-snippet: |
      location content/images/ {
        limit_req zone=my-zone-2 burst=50 nodelay;
      }
      location content/texts/ {
        limit_req zone=my-zone-3 burst=50 nodelay;
      }
    nginx.ingress.kubernetes.io/configuration-snippet: |
      limit_req zone=my-zone-1 burst=10 nodelay;
      limit_req_log_level notice;
      limit_req_status 429;
# 配置映射-ConfigMap
$ kubectl get cm -n ingress-nginx ingress-nginx-controller -o yaml
data:
  http-snippet: |
    limit_req_zone $http_authorization zone=my-zone:20m rate=5r/s;
    limit_req_zone $binary_remote_addr zone=my-zone:20m rate=10r/s;
    limit_req_zone $http_someheader zone=my-zone:20m rate=20r/s;

# 配置注释-annotations
# - 入口资源中的注释：
metadata:
  annotations:
    nginx.ingress.kubernetes.io/configuration-snippet: |
      limit_req zone=my-zone-1 burst=10 nodelay;
      limit_req_log_level notice;
      limit_req_status 429;

# - 一个入口定义的不同位置的不同节流示例：
metadata:
  annotations:
    nginx.ingress.kubernetes.io/server-snippet: |
      location content/images/ {
        limit_req zone=my-zone-2 burst=50 nodelay;
      }
      location content/texts/ {
        limit_req zone=my-zone-3 burst=50 nodelay;
      }
    nginx.ingress.kubernetes.io/configuration-snippet: |
      limit_req zone=my-zone-1 burst=10 nodelay;
      limit_req_log_level notice;
      limit_req_status 429;

18. 日志格式 ​

https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/log-format/

#默认
pod name
#默认
pod name
bash
$ kubectl -n ingress-nginx edit configmap ingress-nginx-controller
# 在 data 字段添加下面内容
data:
  log-format-upstream: 
    '{"time": "$time_iso8601", "namespace": "$namespace", "service_name": "$service_name",
      "service_port": $service_port, "domain": " $host", "path": "$uri", "request_id": "$req_id",
      "remote_user": "$remote_user", "request_query": "$args", "bytes_sent": $bytes_sent, 
      "status": $status, "request_time": $request_time, "request_proto": "$server_protocol", 
      "request_length": $request_length, "duration": $request_time, "method": "$request_method", 
      "http_referrer": "$http_referer", "remote_addr":"$remote_addr", "remote_port": "$remote_port",
      "proxy_protocol_addr": "$proxy_protocol_addr", "proxy_add_x_forwarded_for": "$proxy_add_x_forwarded_for",
      "x_forwarded_for": "$http_x_forwarded_for", "http_user_agent": "$http_user_agent"
    }'
$ kubectl -n ingress-nginx edit configmap ingress-nginx-controller
# 在 data 字段添加下面内容
data:
  log-format-upstream: 
    '{"time": "$time_iso8601", "namespace": "$namespace", "service_name": "$service_name",
      "service_port": $service_port, "domain": " $host", "path": "$uri", "request_id": "$req_id",
      "remote_user": "$remote_user", "request_query": "$args", "bytes_sent": $bytes_sent, 
      "status": $status, "request_time": $request_time, "request_proto": "$server_protocol", 
      "request_length": $request_length, "duration": $request_time, "method": "$request_method", 
      "http_referrer": "$http_referer", "remote_addr":"$remote_addr", "remote_port": "$remote_port",
      "proxy_protocol_addr": "$proxy_protocol_addr", "proxy_add_x_forwarded_for": "$proxy_add_x_forwarded_for",
      "x_forwarded_for": "$http_x_forwarded_for", "http_user_agent": "$http_user_agent"
    }'

19. header头 ​

当header头中有关键字（foo 或 new）字段的时候，自动将流量转发至new-demo；

yaml
[root@kube-master cannary]# cat header-ingress.yaml
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: header-demo-ingress
  annotations:
    nginx.ingress.kubernetes.io/configuration-snippet: |
      if ($http_name ~ "^.*foo$|^.*new$") {
        proxy_pass http://new-demo.default:80;
        break;
      }
spec:
  ingressClassName: nginx
  rules:
  - host: java.host.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: old-demo    #必须是old 在前面，否则new，在前面，会覆盖old，导致不能访问到old
            port:
              number: 80
      - path: /
        pathType: Prefix
        backend:
          service:
            name: new-demo
            port:
              number: 80
[root@kube-master cannary]# cat header-ingress.yaml
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: header-demo-ingress
  annotations:
    nginx.ingress.kubernetes.io/configuration-snippet: |
      if ($http_name ~ "^.*foo$|^.*new$") {
        proxy_pass http://new-demo.default:80;
        break;
      }
spec:
  ingressClassName: nginx
  rules:
  - host: java.host.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: old-demo    #必须是old 在前面，否则new，在前面，会覆盖old，导致不能访问到old
            port:
              number: 80
      - path: /
        pathType: Prefix
        backend:
          service:
            name: new-demo
            port:
              number: 80
  • 测试效果
bash
➜ curl -H "name:new" http://java.host.com
hsuing demoapp v1.2 !! ClientIP: 10.103.236.202, PodName: new-demo-5d96dfb47d-s4v8j, PodIP: 172.30.0.180!


➜ curl -H "Host:java.host.com" http://java.host.com
hsuing demoapp v1.1 !! ClientIP: 172.30.0.128, PodName: old-demo-687c6ccd99-mf7ks, PodIP: 172.23.127.96!
➜ curl -H "name:new" http://java.host.com
hsuing demoapp v1.2 !! ClientIP: 10.103.236.202, PodName: new-demo-5d96dfb47d-s4v8j, PodIP: 172.30.0.180!


➜ curl -H "Host:java.host.com" http://java.host.com
hsuing demoapp v1.1 !! ClientIP: 172.30.0.128, PodName: old-demo-687c6ccd99-mf7ks, PodIP: 172.23.127.96!

20. 传递用户真实IP ​

文档, https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration

在data配置块下添加

yaml
$ kubectl -n ingress-nginx edit configmap ingress-nginx-controller 
# 在 data 字段添加下面几行
data:
  # 源地址附加到 X-Forwarded-For 标头，而不是替换它
  compute-full-forwarded-for: "true"
  # 标头允许下划线，默认是关闭
  enable-underscores-in-headers: "true"
  # 设置用于标识客户端的原始 IP 地址的标头字段。 默认值：X-Forwarded-For
  forwarded-for-header: X-Forwarded-For
  # 当期望将 X-Forwarded-* 的标头信息传递给后端服务时，需要设置为true
  use-forwarded-headers: "true"
  # 启用或禁用 PROXY 协议以接收通过代理服务器和负载均衡器传递的客户端连接（真实 IP 地址）信息
  use-proxy-protocol: "true"

# 重启 ingress-nginx-controller 容器
$ kubectl -n ingress-nginx delete pod -l app.kubernetes.io/component=controller
$ kubectl -n ingress-nginx edit configmap ingress-nginx-controller 
# 在 data 字段添加下面几行
data:
  # 源地址附加到 X-Forwarded-For 标头，而不是替换它
  compute-full-forwarded-for: "true"
  # 标头允许下划线，默认是关闭
  enable-underscores-in-headers: "true"
  # 设置用于标识客户端的原始 IP 地址的标头字段。 默认值：X-Forwarded-For
  forwarded-for-header: X-Forwarded-For
  # 当期望将 X-Forwarded-* 的标头信息传递给后端服务时，需要设置为true
  use-forwarded-headers: "true"
  # 启用或禁用 PROXY 协议以接收通过代理服务器和负载均衡器传递的客户端连接（真实 IP 地址）信息
  use-proxy-protocol: "true"

# 重启 ingress-nginx-controller 容器
$ kubectl -n ingress-nginx delete pod -l app.kubernetes.io/component=controller

21. ModSecurity ​

  • 启动服务
bash
kubectl create deployment modsecurity --image=httpd --port=80
kubectl expose deployment modsecurity
kubectl create ingress ingress-localhost --class=nginx --rule='modsecurity/*=modsecurity:80'
kubectl create deployment modsecurity --image=httpd --port=80
kubectl expose deployment modsecurity
kubectl create ingress ingress-localhost --class=nginx --rule='modsecurity/*=modsecurity:80'
  • 配置
kubectl edit configmaps --namespace ingress-nginx ingress-nginx-controller
kubectl edit configmaps --namespace ingress-nginx ingress-nginx-controller
data:
  allow-snippet-annotations: "true"
  enable-modsecurity: "true"
  enable-owasp-modsecurity-crs: "true"
data:
  allow-snippet-annotations: "true"
  enable-modsecurity: "true"
  enable-owasp-modsecurity-crs: "true"

1. 配置规则 ​

data:
  allow-snippet-annotations: "true"
  enable-modsecurity: "true"
  enable-owasp-modsecurity-crs: "true"
  modsecurity-snippet: |-
    SecRuleEngine On
    SecRequestBodyAccess On
data:
  allow-snippet-annotations: "true"
  enable-modsecurity: "true"
  enable-owasp-modsecurity-crs: "true"
  modsecurity-snippet: |-
    SecRuleEngine On
    SecRequestBodyAccess On
  • 验证
bash
curl 'http://modsecurity:8080/?param="><script>alert(1);</script>'
curl 'http://modsecurity:8080/?param="><script>alert(1);</script>'

22. Cache ​

1. 创建证书 ​

kubectl create secret tls tls-certificate --namespace default --key certificate-key.key --cert cetificate.crt
kubectl create secret tls tls-certificate --namespace default --key certificate-key.key --cert cetificate.crt

2. 创建cache.yaml ​

yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: ingress-nginx-controller
  namespace: default
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
data:
  proxy-connect-timeout: "10"
  proxy-read-timeout: "120"
  proxy-send-timeout: "120"
  http-snippet: "proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=static-cache:10m max_size=4g inactive=120m use_temp_path=off;"
apiVersion: v1
kind: ConfigMap
metadata:
  name: ingress-nginx-controller
  namespace: default
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
data:
  proxy-connect-timeout: "10"
  proxy-read-timeout: "120"
  proxy-send-timeout: "120"
  http-snippet: "proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=static-cache:10m max_size=4g inactive=120m use_temp_path=off;"
  • 执行
bash
kubectl apply -f 1.configmap_cache.yaml
kubectl apply -f 1.configmap_cache.yaml

3. 配置Public ip ​

yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: cdn-config
data:
  default.conf: |
    server {
      listen                  80;
      server_name             _;
      root                    /usr/share/nginx/html;
      location / {
      proxy_read_timeout 150;
     }
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cdn
spec:
  selector:
    matchLabels:
      app: cdn
  replicas: 1
  template:
    metadata:
      labels:
        app: cdn
    spec:
      containers:
      - name: nginx
        image: nginx:latest
        ports:
        - containerPort: 80
        volumeMounts:
        - name: cdn-config
          mountPath: /etc/nginx/conf.d/default.conf
          subPath: default.conf
        - name: cdn-images
          mountPath: "/usr/share/nginx/html/"
      volumes:
        - name: cdn-config
          configMap:
            name: cdn-config
        - name: cdn-images
          persistentVolumeClaim:
            claimName: cdn-images
---
apiVersion: v1
kind: Service
metadata:
  name: cdn
spec:
  selector:
    app: cdn
  ports:
    - port: 80
      targetPort: 80
  externalIPs:
  - YOUR public ip
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-cdn
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/rewrite-target: /
    nginx.ingress.kubernetes.io/proxy-body-size: 8m
    nginx.ingress.kubernetes.io/proxy-buffering: "on"
    nginx.ingress.kubernetes.io/configuration-snippet: |
      proxy_cache static-cache;
      proxy_cache_valid any 120m;
      add_header X-Cache-Status $upstream_cache_status;
spec:
  tls:
    - hosts:
      - cdn.example.com
      secretName: tls-certificate
  rules:
  - host: cdn.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: cdn
            port:
              number: 80
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: cdn-images
spec:
  storageClassName: manual
  capacity:
    storage: 4Gi
  accessModes:
  - ReadWriteOnce
  hostPath:
    path: /home/root2/cdn-images/
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: cdn-images
spec:
  storageClassName: manual
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 4Gi
apiVersion: v1
kind: ConfigMap
metadata:
  name: cdn-config
data:
  default.conf: |
    server {
      listen                  80;
      server_name             _;
      root                    /usr/share/nginx/html;
      location / {
      proxy_read_timeout 150;
     }
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cdn
spec:
  selector:
    matchLabels:
      app: cdn
  replicas: 1
  template:
    metadata:
      labels:
        app: cdn
    spec:
      containers:
      - name: nginx
        image: nginx:latest
        ports:
        - containerPort: 80
        volumeMounts:
        - name: cdn-config
          mountPath: /etc/nginx/conf.d/default.conf
          subPath: default.conf
        - name: cdn-images
          mountPath: "/usr/share/nginx/html/"
      volumes:
        - name: cdn-config
          configMap:
            name: cdn-config
        - name: cdn-images
          persistentVolumeClaim:
            claimName: cdn-images
---
apiVersion: v1
kind: Service
metadata:
  name: cdn
spec:
  selector:
    app: cdn
  ports:
    - port: 80
      targetPort: 80
  externalIPs:
  - YOUR public ip
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-cdn
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/rewrite-target: /
    nginx.ingress.kubernetes.io/proxy-body-size: 8m
    nginx.ingress.kubernetes.io/proxy-buffering: "on"
    nginx.ingress.kubernetes.io/configuration-snippet: |
      proxy_cache static-cache;
      proxy_cache_valid any 120m;
      add_header X-Cache-Status $upstream_cache_status;
spec:
  tls:
    - hosts:
      - cdn.example.com
      secretName: tls-certificate
  rules:
  - host: cdn.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: cdn
            port:
              number: 80
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: cdn-images
spec:
  storageClassName: manual
  capacity:
    storage: 4Gi
  accessModes:
  - ReadWriteOnce
  hostPath:
    path: /home/root2/cdn-images/
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: cdn-images
spec:
  storageClassName: manual
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 4Gi
  • 执行
kubectl apply -f 2.cdn.yaml
kubectl apply -f 2.cdn.yaml

方壶外史

龙潜虎跃

意拳站桩

。。

晃海