SRE文档

深浅模式

阿里云 ​

1.server端 ​

systemctl stop openvpn-server@server.service

systemctl start openvpn-server@server.service

[root@vpn server]# cat server.conf
port 60168
proto tcp
dev tun
tun-mtu 9000
sndbuf 393216
rcvbuf 393216
ca ca.crt
cert server.crt
key server.key
dh dh.pem
cipher AES-256-CBC
auth SHA512
tls-auth ta.key 0
topology subnet

server 10.8.8.0 255.255.255.0

ifconfig-pool-persist ipp.txt

push "dhcp-option DNS 114.114.114.114"
push "dhcp-option DNS 8.8.8.8"

push "sndbuf 393216"
push "rcvbuf 393216"

push "route 172.16.195.0 255.255.255.0 vpn_gateway"

;duplicate-cn
client-to-client
verify-client-cert

keepalive 10 120
user nobody
group nobody
persist-key
persist-tun
status openvpn-status.log
log-append openvpn.log
verb 3
crl-verify crl.pem
mute 20
max-clients 100
[root@vpn server]# cat server.conf
port 60168
proto tcp
dev tun
tun-mtu 9000
sndbuf 393216
rcvbuf 393216
ca ca.crt
cert server.crt
key server.key
dh dh.pem
cipher AES-256-CBC
auth SHA512
tls-auth ta.key 0
topology subnet

server 10.8.8.0 255.255.255.0

ifconfig-pool-persist ipp.txt

push "dhcp-option DNS 114.114.114.114"
push "dhcp-option DNS 8.8.8.8"

push "sndbuf 393216"
push "rcvbuf 393216"

push "route 172.16.195.0 255.255.255.0 vpn_gateway"

;duplicate-cn
client-to-client
verify-client-cert

keepalive 10 120
user nobody
group nobody
persist-key
persist-tun
status openvpn-status.log
log-append openvpn.log
verb 3
crl-verify crl.pem
mute 20
max-clients 100
  • pam
auth [user_unknown=ignore success=ok ignore=ignore default=bad] pam_securetty.so
auth required /usr/lib64/security/pam_google_authenticator.so secret=/etc/openvpn/google-authenticator/${USER} user=gauth forward_pass
auth include system-auth
account include system-auth
password include system-auth
auth [user_unknown=ignore success=ok ignore=ignore default=bad] pam_securetty.so
auth required /usr/lib64/security/pam_google_authenticator.so secret=/etc/openvpn/google-authenticator/${USER} user=gauth forward_pass
auth include system-auth
account include system-auth
password include system-auth

2.client端 ​

client
dev tun
proto tcp
;tun-mtu 9000
tun-mtu 1500
tun-mtu-extra 32
sndbuf 393216
rcvbuf 393216

remote 123.56.187.47 60168
;route 10.8.8.0 255.255.255.0

resolv-retry infinite
nobind
persist-key
persist-tun
remote-cert-tls server
auth SHA512
cipher AES-256-CBC
setenv opt block-outside-dns
key-direction 1

verb 3
auth-nocache
route-method exe
route-delay 2
reneg-sec 36000

;无线网络有很多多余的头文件，设置忽略它
mute-replay-warnings

;route-nopull
;redirect-gateway def1 bypass-dhcp
;redirect-gateway local def1
;setenv opt block-outside-dns
;ignore-unknown-option block-outside-dns
;block-outside-dns
max-routes 1000
;route 192.168.1.0 255.255.255.0 vpn_gateway
;route 172.16.195.0 255.255.255.0 vpn_gateway

<ca>
........
</tls-auth>
client
dev tun
proto tcp
;tun-mtu 9000
tun-mtu 1500
tun-mtu-extra 32
sndbuf 393216
rcvbuf 393216

remote 123.56.187.47 60168
;route 10.8.8.0 255.255.255.0

resolv-retry infinite
nobind
persist-key
persist-tun
remote-cert-tls server
auth SHA512
cipher AES-256-CBC
setenv opt block-outside-dns
key-direction 1

verb 3
auth-nocache
route-method exe
route-delay 2
reneg-sec 36000

;无线网络有很多多余的头文件，设置忽略它
mute-replay-warnings

;route-nopull
;redirect-gateway def1 bypass-dhcp
;redirect-gateway local def1
;setenv opt block-outside-dns
;ignore-unknown-option block-outside-dns
;block-outside-dns
max-routes 1000
;route 192.168.1.0 255.255.255.0 vpn_gateway
;route 172.16.195.0 255.255.255.0 vpn_gateway

<ca>
........
</tls-auth>

3.防火墙 ​

#开启转发
[root@beta ~]# vim /etc/sysctl.conf 
net.ipv4.ip_forward = 1


firewall-cmd --add-masquerade --permanent
firewall-cmd --permanent  --add-port=60168/tcp
firewall-cmd --permanent --direct --passthrough ipv4 -t nat -A POSTROUTING -s 10.8.8.0/24 -o eth0 -j MASQUERADE
firewall-cmd --reload

firewall-cmd --zone=public --query-port=60168/tcp # 查看端口号是否开放成功，输出yes开放成功，no则失败

##########
iptables -P FORWARD ACCEPT
iptables -I INPUT -p tcp --dport 1194 -m comment --comment "openvpn" -j ACCEPT
iptables -t nat -A POSTROUTING -s 10.8.8.0/24 -j MASQUERADE
#开启转发
[root@beta ~]# vim /etc/sysctl.conf 
net.ipv4.ip_forward = 1


firewall-cmd --add-masquerade --permanent
firewall-cmd --permanent  --add-port=60168/tcp
firewall-cmd --permanent --direct --passthrough ipv4 -t nat -A POSTROUTING -s 10.8.8.0/24 -o eth0 -j MASQUERADE
firewall-cmd --reload

firewall-cmd --zone=public --query-port=60168/tcp # 查看端口号是否开放成功，输出yes开放成功，no则失败

##########
iptables -P FORWARD ACCEPT
iptables -I INPUT -p tcp --dport 1194 -m comment --comment "openvpn" -j ACCEPT
iptables -t nat -A POSTROUTING -s 10.8.8.0/24 -j MASQUERADE

4.白名单 ​

  1. 公司内部网段是 192.168.99.0/24；
  2. 所有人允许访问反向代理主机为 192.168.99.130，但不能访问其他服务器；
  3. 特定的用户允许访问 数据库服务器为 192.168.99.131，不能访问其他服务器；
  4. 管理员能访问 所有公司内网服务器

将VPN地址池的网段划分为管理员网段、客户组网段、普通网络

# 在Openvpn服务端配置文件server.conf增加：
#10.8.0.0是给所有VPN客户端的IP段；
server 10.8.0.0 255.255.255.0

#10.8.1.0是给管理员分配的IP段；
server 10.8.1.0 255.255.255.0

#10.8.2.0就是给特定用户组分配的IP段；
server 10.8.2.0 255.255.255.0

#下面是定义服务器读取特殊客户端配置文件的目录为ccd,ccd是与Openvpn服务端配置文件同级目录中的ccd目录
client-config-dir ccd
# 在Openvpn服务端配置文件server.conf增加：
#10.8.0.0是给所有VPN客户端的IP段；
server 10.8.0.0 255.255.255.0

#10.8.1.0是给管理员分配的IP段；
server 10.8.1.0 255.255.255.0

#10.8.2.0就是给特定用户组分配的IP段；
server 10.8.2.0 255.255.255.0

#下面是定义服务器读取特殊客户端配置文件的目录为ccd,ccd是与Openvpn服务端配置文件同级目录中的ccd目录
client-config-dir ccd
  • 管理员配置访问网络
cat ccd/sysadmin1
ifconfig-push 10.8.1.1 10.8.1.2
cat ccd/sysadmin1
ifconfig-push 10.8.1.1 10.8.1.2
  • 客户组网络:
cat ccd/kehugroup
ifconfig-push 10.8.2.1 10.8.2.2
cat ccd/kehugroup
ifconfig-push 10.8.2.1 10.8.2.2

这里需要注意的是, ccd目录下的文件名就是用户的Common Name，OpenVPN是根据该名称来获得指定客户端的，客户端的IP地址不是任意指定的，由于Windows的TAP驱动必须采用/30网段的IP，为兼容该协议，应从特定的IP地址中选择，而且是成组出现的。

最后在完成网络的划分之后，在OpenVPN端进行Iptables限制

iptables -A FORWARD -i tun0 -s 10.8.0.0/24 -d 192.168.99.130 -j ACCEPT
iptables -A FORWARD -i tun0 -s 10.8.1.0/24 -d 192.168.99.0/24 -j ACCEPT
iptables -A FORWARD -i tun0 -s 10.8.2.0/24 -d 192.168.99.131 -j ACCEPT
iptables -A FORWARD -i tun0 -s 10.8.0.0/24 -d 192.168.99.130 -j ACCEPT
iptables -A FORWARD -i tun0 -s 10.8.1.0/24 -d 192.168.99.0/24 -j ACCEPT
iptables -A FORWARD -i tun0 -s 10.8.2.0/24 -d 192.168.99.131 -j ACCEPT

打通OpenVPN客户端与服务端的内网 ​

#让所有客户端都增加到内网192.168.99.0/24的路由
push "route 192.168.99.0 255.255.255.0"

# 让所有的客户端都能访问仅允许服务端访问的网站(约束白名单)
push "route 39.156.69.79 255.255.255.255"

# 向客户端推送增加访问服务端子网的192.168.10.0/24的路由，注意服务端的IP是否是子网的网关，否则需要在子网网关处添加到达192.168.99.0的路由(客户端也是如此)
push "route 192.168.10.0 255.255.255.0"
#让所有客户端都增加到内网192.168.99.0/24的路由
push "route 192.168.99.0 255.255.255.0"

# 让所有的客户端都能访问仅允许服务端访问的网站(约束白名单)
push "route 39.156.69.79 255.255.255.255"

# 向客户端推送增加访问服务端子网的192.168.10.0/24的路由，注意服务端的IP是否是子网的网关，否则需要在子网网关处添加到达192.168.99.0的路由(客户端也是如此)
push "route 192.168.10.0 255.255.255.0"

dhcp ​

# openvpn服务端的配置
#定义客户端的DNS服务器地址
push "dhcp-option DNS 114.114.114.114" # 这是首选DNS
push "dhcp-option DNS 8.8.8.8" # 这是备选DNS

#定义客户端的WINS服务器地址
push "dhcp-options WINS 192.168.228.1" # 这是设置IP和主机名之间的映射与IP和域名之间的映射不同,较少使用
# openvpn服务端的配置
#定义客户端的DNS服务器地址
push "dhcp-option DNS 114.114.114.114" # 这是首选DNS
push "dhcp-option DNS 8.8.8.8" # 这是备选DNS

#定义客户端的WINS服务器地址
push "dhcp-options WINS 192.168.228.1" # 这是设置IP和主机名之间的映射与IP和域名之间的映射不同,较少使用

流量走vpn ​

#让客户端发起的所有IP请求都通过OpenVPN服务器,可用于全局代理使用，启用后会出现浏览器内打不开网站等情况
push "redirect-gateway def1 bypass-dhcp"
#让客户端发起的所有IP请求都通过OpenVPN服务器,可用于全局代理使用，启用后会出现浏览器内打不开网站等情况
push "redirect-gateway def1 bypass-dhcp"

5.dns ​

firewall-cmd --add-service=dns --permanent

firewall-cmd --add-port=53/tcp --zone=public --permanent

firewall-cmd --add-port=53/udp --zone=public --permanent

firewall-cmd --reload

systemctl restart firewalld

###
iptables -I INPUT 1 -s 10.8.8.0/24 -p udp --dport 53 -j ACCEPT
iptables -I INPUT 1 -s 10.8.8.0/24 -p tdp --dport 53 -j ACCEPT
firewall-cmd --add-service=dns --permanent

firewall-cmd --add-port=53/tcp --zone=public --permanent

firewall-cmd --add-port=53/udp --zone=public --permanent

firewall-cmd --reload

systemctl restart firewalld

###
iptables -I INPUT 1 -s 10.8.8.0/24 -p udp --dport 53 -j ACCEPT
iptables -I INPUT 1 -s 10.8.8.0/24 -p tdp --dport 53 -j ACCEPT

安装 dnsmasq ​

yum install -y dnsmasq
yum install -y dnsmasq

修改配置 ​

[root@vpn server]# grep -v '^#' /etc/dnsmasq.conf |grep -v '^$'
resolv-file=/etc/resolv.dnsmasq.conf
strict-order
server=114.114.114.114
listen-address=172.16.195.196,127.0.0.1
addn-hosts=/etc/dnsmasq.hosts
cache-size=1024  # 缓存条数
bogus-nxdomain=114.114.114.114
log-queries  # 记录查询日志
log-facility=/var/log/dnsmasq/dnsmasq.log
conf-dir=/etc/dnsmasq.d
conf-dir=/etc/dnsmasq.d,.bak
conf-dir=/etc/dnsmasq.d/,*.conf
conf-dir=/etc/dnsmasq.d,.rpmnew,.rpmsave,.rpmorig
[root@vpn server]# grep -v '^#' /etc/dnsmasq.conf |grep -v '^$'
resolv-file=/etc/resolv.dnsmasq.conf
strict-order
server=114.114.114.114
listen-address=172.16.195.196,127.0.0.1
addn-hosts=/etc/dnsmasq.hosts
cache-size=1024  # 缓存条数
bogus-nxdomain=114.114.114.114
log-queries  # 记录查询日志
log-facility=/var/log/dnsmasq/dnsmasq.log
conf-dir=/etc/dnsmasq.d
conf-dir=/etc/dnsmasq.d,.bak
conf-dir=/etc/dnsmasq.d/,*.conf
conf-dir=/etc/dnsmasq.d,.rpmnew,.rpmsave,.rpmorig

创建目录 ​

#创建日志目录，不然无法启动

mkdir /var/log/dnsmasq/
#创建日志目录，不然无法启动

mkdir /var/log/dnsmasq/

修改 resolv-file ​

[root@vpn server]# cat /etc/resolv.dnsmasq.conf
nameserver 114.114.114.114
nameserver 100.100.2.136
nameserver 100.100.2.138
nameserver 1.0.0.1
nameserver 1.1.1.1
[root@vpn server]# cat /etc/resolv.dnsmasq.conf
nameserver 114.114.114.114
nameserver 100.100.2.136
nameserver 100.100.2.138
nameserver 1.0.0.1
nameserver 1.1.1.1
  • aws
bash
nameserver 100.100.2.136
nameserver 100.100.2.138
nameserver 8.8.4.4
nameserver 172.31.33.214
nameserver 100.100.2.136
nameserver 100.100.2.138
nameserver 8.8.4.4
nameserver 172.31.33.214

添加 hosts ​

[root@vpn server]# cat /etc/dnsmasq.hosts
172.16.195.191 codefenglei.leihuofeng.net
172.16.195.194 yidujenkins.leihuofeng.net


#一行一个，添加依次重启服务
[root@vpn server]# cat /etc/dnsmasq.hosts
172.16.195.191 codefenglei.leihuofeng.net
172.16.195.194 yidujenkins.leihuofeng.net


#一行一个，添加依次重启服务

启动服务 ​

systemctl enable dnsmasq --now
systemctl enable dnsmasq --now
  • 测试
[root@vpn server]# nslookup yidujenkins.leihuofeng.net
Server:		172.16.195.196
Address:	172.16.195.196#53

Name:	yidujenkins.leihuofeng.net
Address: 172.16.195.194
[root@vpn server]# nslookup yidujenkins.leihuofeng.net
Server:		172.16.195.196
Address:	172.16.195.196#53

Name:	yidujenkins.leihuofeng.net
Address: 172.16.195.194

找一台机器将 dns 指向 dnsmasq 服务器地址

vim /etc/resolv.conf

  • 防火墙配置

vpnserver ​

[root@vpn server]# cat server.conf
port 60168
proto tcp
dev tun
tun-mtu 9000
sndbuf 393216
rcvbuf 393216
ca ca.crt
cert server.crt
key server.key
dh dh.pem
cipher AES-256-CBC
auth SHA512
tls-auth ta.key 0
topology subnet
server 10.8.8.0 255.255.255.0

ifconfig-pool-persist ipp.txt

push "sndbuf 393216"
push "rcvbuf 393216"

push "route 172.16.195.0 255.255.255.0 vpn_gateway"
push "route 123.57.146.30 255.255.255.255 vpn_gateway"

;这两条是关键
push "dhcp-option DNS 172.16.195.196"
push "dhcp-option DNS 114.114.114.114"

;push "register-dns"
;push "block-outside-dns"
;push "route-nopull"
push "block-ipv6"


;duplicate-cn
client-to-client
verify-client-cert

keepalive 10 120
user nobody
group nobody
persist-key
persist-tun
status openvpn-status.log
log-append openvpn.log
verb 3
crl-verify crl.pem
mute 20
max-clients 100

plugin /usr/lib64/openvpn/plugins/openvpn-plugin-auth-pam.so openvpn
client-cert-not-required
username-as-common-name
reneg-sec 36000
[root@vpn server]# cat server.conf
port 60168
proto tcp
dev tun
tun-mtu 9000
sndbuf 393216
rcvbuf 393216
ca ca.crt
cert server.crt
key server.key
dh dh.pem
cipher AES-256-CBC
auth SHA512
tls-auth ta.key 0
topology subnet
server 10.8.8.0 255.255.255.0

ifconfig-pool-persist ipp.txt

push "sndbuf 393216"
push "rcvbuf 393216"

push "route 172.16.195.0 255.255.255.0 vpn_gateway"
push "route 123.57.146.30 255.255.255.255 vpn_gateway"

;这两条是关键
push "dhcp-option DNS 172.16.195.196"
push "dhcp-option DNS 114.114.114.114"

;push "register-dns"
;push "block-outside-dns"
;push "route-nopull"
push "block-ipv6"


;duplicate-cn
client-to-client
verify-client-cert

keepalive 10 120
user nobody
group nobody
persist-key
persist-tun
status openvpn-status.log
log-append openvpn.log
verb 3
crl-verify crl.pem
mute 20
max-clients 100

plugin /usr/lib64/openvpn/plugins/openvpn-plugin-auth-pam.so openvpn
client-cert-not-required
username-as-common-name
reneg-sec 36000
  • 客户端测试，连接vpn之后

6.跨云主机 ​

  • 被访问端安全组

server vpn ​

push "route 8.212.15.90 255.255.255.255 vpn_gateway"
push "route 172.20.198.222 255.255.255.255 vpn_gateway"
push "route 8.212.15.90 255.255.255.255 vpn_gateway"
push "route 172.20.198.222 255.255.255.255 vpn_gateway"

dns ​

[root@vpn server]# cat /etc/dnsmasq.hosts 
8.212.15.90 nex.qechacha.com
[root@vpn server]# cat /etc/dnsmasq.hosts 
8.212.15.90 nex.qechacha.com

重启所有服务

systemctl restart openvpn-server@server.service
systemctl restart openvpn-server@server.service

aws ​

dnsmasq 推送必须写tun0的ip地址，否则写默认eth0地址，将不通

  • server.conf
port 56619
proto tcp
dev tun
tun-mtu 9000
sndbuf 393216
rcvbuf 393216
ca ca.crt
cert server.crt
key server.key
dh dh.pem
cipher AES-256-CBC
auth SHA512
tls-auth ta.key 0
topology subnet

server 10.8.8.0 255.255.255.0

ifconfig-pool-persist ipp.txt

push "sndbuf 393216"
push "rcvbuf 393216"

;jump
push "route 172.x 255.255.255.255 vpn_gateway"
;vpn
push "route 172.31.33.0 255.255.255.0 vpn_gateway"

push "route 172.x 255.255.255.255 vpn_gateway"
push "route 172.x 255.255.255.255 vpn_gateway"




push "dhcp-option DNS 172.31.33.214"
push "dhcp-option WINS 10.8.8.1"
push "dhcp-option DNS 8.8.8.8"

push "register-dns"
push "block-outside-dns"
;push "route-nopull"
;push "block-ipv6"

client-to-client
verify-client-cert

keepalive 10 120
user nobody
group nobody
persist-key
persist-tun
status openvpn-status.log
log-append openvpn.log
verb 3
crl-verify crl.pem
mute 20
max-clients 100

plugin /usr/lib64/openvpn/plugins/openvpn-plugin-auth-pam.so openvpn
client-cert-not-required
username-as-common-name
reneg-sec 0
port 56619
proto tcp
dev tun
tun-mtu 9000
sndbuf 393216
rcvbuf 393216
ca ca.crt
cert server.crt
key server.key
dh dh.pem
cipher AES-256-CBC
auth SHA512
tls-auth ta.key 0
topology subnet

server 10.8.8.0 255.255.255.0

ifconfig-pool-persist ipp.txt

push "sndbuf 393216"
push "rcvbuf 393216"

;jump
push "route 172.x 255.255.255.255 vpn_gateway"
;vpn
push "route 172.31.33.0 255.255.255.0 vpn_gateway"

push "route 172.x 255.255.255.255 vpn_gateway"
push "route 172.x 255.255.255.255 vpn_gateway"




push "dhcp-option DNS 172.31.33.214"
push "dhcp-option WINS 10.8.8.1"
push "dhcp-option DNS 8.8.8.8"

push "register-dns"
push "block-outside-dns"
;push "route-nopull"
;push "block-ipv6"

client-to-client
verify-client-cert

keepalive 10 120
user nobody
group nobody
persist-key
persist-tun
status openvpn-status.log
log-append openvpn.log
verb 3
crl-verify crl.pem
mute 20
max-clients 100

plugin /usr/lib64/openvpn/plugins/openvpn-plugin-auth-pam.so openvpn
client-cert-not-required
username-as-common-name
reneg-sec 0
  • /etc/dnsmasq.conf
no-poll
clear-on-reload
no-negcache

resolv-file=/etc/resolv.dnsmasq.conf
strict-order
listen-address=172.31.33.214,127.0.0.1,10.8.8.1
addn-hosts=/etc/dnsmasq.hosts
cache-size=0 # 缓存条数


log-queries  # 记录查询日志
log-facility=/var/log/dnsmasq/dnsmasq.log
conf-dir=/etc/dnsmasq.d
conf-dir=/etc/dnsmasq.d,.bak
conf-dir=/etc/dnsmasq.d/,*.conf
conf-dir=/etc/dnsmasq.d,.rpmnew,.rpmsave,.rpmorig
no-poll
clear-on-reload
no-negcache

resolv-file=/etc/resolv.dnsmasq.conf
strict-order
listen-address=172.31.33.214,127.0.0.1,10.8.8.1
addn-hosts=/etc/dnsmasq.hosts
cache-size=0 # 缓存条数


log-queries  # 记录查询日志
log-facility=/var/log/dnsmasq/dnsmasq.log
conf-dir=/etc/dnsmasq.d
conf-dir=/etc/dnsmasq.d,.bak
conf-dir=/etc/dnsmasq.d/,*.conf
conf-dir=/etc/dnsmasq.d,.rpmnew,.rpmsave,.rpmorig
如果设置了 block-outside-dns

这样 OpenVPN 会添加 Windows 防火墙记录，拦掉除 tap 以外的所有网络接口上的 DNS 请求。需要把这行从你配置文件中删掉
如果设置了 block-outside-dns

这样 OpenVPN 会添加 Windows 防火墙记录，拦掉除 tap 以外的所有网络接口上的 DNS 请求。需要把这行从你配置文件中删掉

FAQ ​

路由：
		服务端推送路由规则：
			push "route 10.0.0.0 255.0.0.0 vpn_gateway"  #vpn网关
			push "route 10.0.0.0 255.0.0.0 net_gateway" #本地net上网网关
		客户端：
			route 10.0.0.0 255.0.0.0 vpn_gateway  #vpn网关
			route 10.0.0.0 255.0.0.0 net_gateway  #本地net上网网关
路由：
		服务端推送路由规则：
			push "route 10.0.0.0 255.0.0.0 vpn_gateway"  #vpn网关
			push "route 10.0.0.0 255.0.0.0 net_gateway" #本地net上网网关
		客户端：
			route 10.0.0.0 255.0.0.0 vpn_gateway  #vpn网关
			route 10.0.0.0 255.0.0.0 net_gateway  #本地net上网网关

tap-win ​

OpenVPN Error: All TAP-Windows adapters on this system are currently in use

Solution 1: re-enable the adapter

Open Control Panel > Network and Internet > Network and Sharing Center > Change adapter settings. Check description of network adapters to find the Tap-Windows Adapter.
Right click on it > Disable. Right click again > Enable.

Solution 2: reinstall the Tap driver

Check if it is already installed by opening C:\Program Files\TAP-Windows Right click on Uninstall and select Run as administrator Reboot the system (not required, but recommended) Download the Tap driver from OpenVPN repository and install it (right click on it > Run as administrator)