SRE文档

深浅模式

1.说明 ​

文档，https://docs.min.io/docs/deploy-minio-on-docker-compose

1.1说明 ​

在RELEASE.2022-06-30T20-58-09Z版本之前，MinIO集群最少需要四个独立的驱动器，否则启动时会直接报错，提示以下信息Please provide an even number of endpoints greater or equal to 4

从RELEASE.2022-06-30T20-58-09Z版本开始支持在只有2个或者3个驱动器情况下启动MinIO服务，详情参见版本发布信息或者#15171

在MinIO集群中奇偶校验块的数量不能大于数据块的数量，也就是说MinIO集群的磁盘利用率最小为50%，即如果集群总的驱动器容量为2T，则至少可以存储1T的用户数据。

MinIO服务器在不超过一半的驱动器停止工作时仍能够正常提供服务，保证数据不丢失，并在故障节点再次上线时自动同步数据。

注意：当奇偶校验块的数量N等于数据块的数量时，如果有一半的驱动器停止工作，则只能读不能写，只有当有N+1个驱动器正常工作时才能进行写入操作。

更多纠删码的信息参见Erasure Coding

2.2清单 ​

主机IP
minio110.0.0.1
minio210.0.0.2
minio310.0.0.3
nginx10.0.0.4
软件版本
docker20.10.15
minioRELEASE.2024-03-26T22-10-45Z
nginx1.21.6

2.安装 ​

2.1使用国内yum 源 ​

shell
# yum install -y yum-utils device-mapper-persistent-data lvm2
# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# yum install -y yum-utils device-mapper-persistent-data lvm2
# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

2.2卸载旧版本的docker ​

shell
# 如果主机上已经有docker存在且不是想要安装的版本，需要先进行卸载。
# yum remove -y docker \
              docker-client \
              docker-client-latest \
              docker-common \
              docker-latest \
              docker-latest-logrotate \
              docker-logrotate \
              docker-selinux \
              docker-engine-selinux \
              docker-engine \
              container*
# 如果主机上已经有docker存在且不是想要安装的版本，需要先进行卸载。
# yum remove -y docker \
              docker-client \
              docker-client-latest \
              docker-common \
              docker-latest \
              docker-latest-logrotate \
              docker-logrotate \
              docker-selinux \
              docker-engine-selinux \
              docker-engine \
              container*

3.部署服务 ​

三个节点统一执行

shell
mkdir /data/minio/data{1,2} -pv
mkdir /data/minio/data{1,2} -pv

3.1部署节点1 ​

shell
# cat << EOF >> install.sh

#!/bin/bash

docker run -d --net=host --name=minio \
  -p 9000:9000 \
  -p 9001:9001 \
  -v /data/minio/data1:/data1 \
  -v /data/minio/data2:/data2 \
  -e "MINIO_ACCESS_KEY=minio" \
  -e "MINIO_SECRET_KEY=minio123" \
  --add-host minio-1:10.0.0.1 \
  --add-host minio-2:10.0.0.2 \
  --add-host minio-3:10.0.0.3 \
  minio/minio:RELEASE.2024-03-26T22-10-45Z \
  server --address 10.0.0.1 \    # 填写 节点1 IP地址
  http://minio-1/data1 http://minio-1/data2 \
  http://minio-2/data1 http://minio-2/data2 \
  http://minio-3/data1 http://minio-3/data2 \
  --console-address ":9001"
EOF
# cat << EOF >> install.sh

#!/bin/bash

docker run -d --net=host --name=minio \
  -p 9000:9000 \
  -p 9001:9001 \
  -v /data/minio/data1:/data1 \
  -v /data/minio/data2:/data2 \
  -e "MINIO_ACCESS_KEY=minio" \
  -e "MINIO_SECRET_KEY=minio123" \
  --add-host minio-1:10.0.0.1 \
  --add-host minio-2:10.0.0.2 \
  --add-host minio-3:10.0.0.3 \
  minio/minio:RELEASE.2024-03-26T22-10-45Z \
  server --address 10.0.0.1 \    # 填写 节点1 IP地址
  http://minio-1/data1 http://minio-1/data2 \
  http://minio-2/data1 http://minio-2/data2 \
  http://minio-3/data1 http://minio-3/data2 \
  --console-address ":9001"
EOF

3.2部署节点2 ​

shell
# cat << EOF >> install.sh

#!/bin/bash

docker run -d --net=host --name=minio \
  -p 9000:9000 \
  -p 9001:9001 \
  -v /data/minio/data1:/data1 \
  -v /data/minio/data2:/data2 \
  -e "MINIO_ACCESS_KEY=minio" \
  -e "MINIO_SECRET_KEY=minio123" \
  --add-host minio-1:10.0.0.1 \
  --add-host minio-2:10.0.0.2 \
  --add-host minio-3:10.0.0.3 \
  minio/minio:RELEASE.2024-03-26T22-10-45Z \
  server --address 10.0.0.2 \    # 填写 节点2 IP地址
  http://minio-1/data1 http://minio-1/data2 \
  http://minio-2/data1 http://minio-2/data2 \
  http://minio-3/data1 http://minio-3/data2 \
  --console-address ":9001"
EOF
# cat << EOF >> install.sh

#!/bin/bash

docker run -d --net=host --name=minio \
  -p 9000:9000 \
  -p 9001:9001 \
  -v /data/minio/data1:/data1 \
  -v /data/minio/data2:/data2 \
  -e "MINIO_ACCESS_KEY=minio" \
  -e "MINIO_SECRET_KEY=minio123" \
  --add-host minio-1:10.0.0.1 \
  --add-host minio-2:10.0.0.2 \
  --add-host minio-3:10.0.0.3 \
  minio/minio:RELEASE.2024-03-26T22-10-45Z \
  server --address 10.0.0.2 \    # 填写 节点2 IP地址
  http://minio-1/data1 http://minio-1/data2 \
  http://minio-2/data1 http://minio-2/data2 \
  http://minio-3/data1 http://minio-3/data2 \
  --console-address ":9001"
EOF

3.3部署节点3 ​

shell
# cat << EOF >> install.sh

#!/bin/bash

docker run -d --net=host --name=minio \
  -p 9000:9000 \
  -p 9001:9001 \
  -v /data/minio/data1:/data1 \
  -v /data/minio/data2:/data2 \
  -e "MINIO_ACCESS_KEY=minio" \
  -e "MINIO_SECRET_KEY=minio123" \
  --add-host minio-1:10.0.0.1 \
  --add-host minio-2:10.0.0.2 \
  --add-host minio-3:10.0.0.3 \
  minio/minio:RELEASE.2022-02-18T01-50-10Z \
  server --address 10.0.0.3 \    # 填写 节点3 IP地址
  http://minio-1/data1 http://minio-1/data2 \
  http://minio-2/data1 http://minio-2/data2 \
  http://minio-3/data1 http://minio-3/data2 \
  --console-address ":9001"
EOF
# cat << EOF >> install.sh

#!/bin/bash

docker run -d --net=host --name=minio \
  -p 9000:9000 \
  -p 9001:9001 \
  -v /data/minio/data1:/data1 \
  -v /data/minio/data2:/data2 \
  -e "MINIO_ACCESS_KEY=minio" \
  -e "MINIO_SECRET_KEY=minio123" \
  --add-host minio-1:10.0.0.1 \
  --add-host minio-2:10.0.0.2 \
  --add-host minio-3:10.0.0.3 \
  minio/minio:RELEASE.2022-02-18T01-50-10Z \
  server --address 10.0.0.3 \    # 填写 节点3 IP地址
  http://minio-1/data1 http://minio-1/data2 \
  http://minio-2/data1 http://minio-2/data2 \
  http://minio-3/data1 http://minio-3/data2 \
  --console-address ":9001"
EOF

4.部署高可用 ​

  • 创建nginx目录
mkdir -pv /data/nginx/
mkdir -pv /data/nginx/
  • 创建nginx配置
shell
user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections  4096;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;

    # include /etc/nginx/conf.d/*.conf;
	# websocket
    map $http_upgrade $connection_upgrade {
        default upgrade;
        ''      close;
    }
	
    upstream minio {
        server minio1:9000;
        server minio2:9000;
        server minio3:9000;
    }

    upstream console {
        ip_hash;
        server minio1:9001;
        server minio2:9001;
        server minio3:9001;
    }

    server {
        listen       9000;
        listen  [::]:9000;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;

            proxy_pass http://minio;
        }
    }

    server {
        listen       9001;
        listen  [::]:9001;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-NginX-Proxy true;

            # This is necessary to pass the correct IP to be hashed
            real_ip_header X-Real-IP;

            proxy_connect_timeout 300;
            
            # To support websocket
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection $connection_upgrade;
            
            chunked_transfer_encoding off;

            proxy_pass http://console;
        }
    }
}
user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections  4096;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;

    # include /etc/nginx/conf.d/*.conf;
	# websocket
    map $http_upgrade $connection_upgrade {
        default upgrade;
        ''      close;
    }
	
    upstream minio {
        server minio1:9000;
        server minio2:9000;
        server minio3:9000;
    }

    upstream console {
        ip_hash;
        server minio1:9001;
        server minio2:9001;
        server minio3:9001;
    }

    server {
        listen       9000;
        listen  [::]:9000;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;

            proxy_pass http://minio;
        }
    }

    server {
        listen       9001;
        listen  [::]:9001;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-NginX-Proxy true;

            # This is necessary to pass the correct IP to be hashed
            real_ip_header X-Real-IP;

            proxy_connect_timeout 300;
            
            # To support websocket
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection $connection_upgrade;
            
            chunked_transfer_encoding off;

            proxy_pass http://console;
        }
    }
}
  • 启动服务
shell
# cat << EOF >> install.sh
#/bin/bash

docker run -d --name nginx_minio \
  --restart always \
  -p 9000:9000 \
  -p 9001:9001 \
  --add-host minio1:10.0.0.1 \
  --add-host minio2:10.0.0.2 \
  --add-host minio3:10.0.0.3 \
  -v /data/nginx/nginx.conf:/etc/nginx/nginx.conf \
  -v /etc/localtime:/etc/localtime \
  nginx:1.21.6
EOF
# cat << EOF >> install.sh
#/bin/bash

docker run -d --name nginx_minio \
  --restart always \
  -p 9000:9000 \
  -p 9001:9001 \
  --add-host minio1:10.0.0.1 \
  --add-host minio2:10.0.0.2 \
  --add-host minio3:10.0.0.3 \
  -v /data/nginx/nginx.conf:/etc/nginx/nginx.conf \
  -v /etc/localtime:/etc/localtime \
  nginx:1.21.6
EOF
  • 访问服务

在浏览器访问 http://10.0.0.4:9000 访问minio web 访问页面

参考：

https://www.cnblogs.com/liugp/p/16560313.html#2添加磁盘