1.15.3 HA kubernetes Cluster Playbook (load balance)

Objective

variables

  1. ## change if necessary
  2. # hostname
  3. master01Name='master01'
  4. master02Name='master02'
  5. master03Name='master03'
  6. # ipaddress
  7. master01IP='192.168.100.200'
  8. master01IP='192.168.100.201'
  9. master01IP='192.168.100.202'
  10. virtualIP='192.168.100.250'
  11. leadIP="${master01IP}"
  12. leadName="${master01Name}"
  13. k8sVer='v1.15.3'
  14. cfsslDownloadUrl='https://pkg.cfssl.org/R1.2'
  15. etcdVer='v3.3.15'
  16. etcdDownloadUrl='https://github.com/etcd-io/etcd/releases/download'
  17. etcdSSLPath='/etc/etcd/ssl'
  18. etcdInitialCluster="${master01Name}=https://${master01IP}:2380,${master02Name}=https://${master02IP}:2380,${master03Name}=https://${master03IP}:2380"
  19. keepaliveVer='2.0.18'
  20. haproxyVer='2.0.6'
  21. helmVer='v2.14.3'
  22. interface=$(netstat -nr | grep -E 'UG|UGSc' | grep -E '^0.0.0|default' | grep -E '[0-9.]{7,15}' | awk -F' ' '{print $NF}')
  23. ipAddr=$(ip a s "${interface}" | sed -rn 's|\W*inet[^6]\W*([0-9\.]{7,15}).*$|\1|p')
  24. peerName=$(hostname)

keepalived

configuration

  • with haproxy

    1. $ sudo bash -c 'cat > /etc/keepalived/keepalived.conf' << EOF
    2. ! Configuration File for keepalived
    3. global_defs {
    4. router_id LVS_DEVEL
    5. }
    6. vrrp_script check_haproxy {
    7. script "killall -0 haproxy"
    8. interval 3
    9. weight -2
    10. fall 10
    11. rise 2
    12. }
    13. vrrp_instance VI_1 {
    14. state MASTER
    15. interface ${interface}
    16. virtual_router_id 51
    17. priority 50
    18. advert_int 1
    19. authentication {
    20. auth_type PASS
    21. auth_pass 35f18af7190d51c9f7f78f37300a0cbd
    22. }
    23. virtual_ipaddress {
    24. ${virtualIP}
    25. }
    26. track_script {
    27. check_haproxy
    28. }
    29. }
    30. EOF
  • without haproxy

    • keepalived.conf

      1. $ sudo bash -c 'cat > /etc/keepalived/keepalived.conf' << EOF
      2. ! Configuration File for keepalived
      3. global_defs {
      4. router_id LVS_DEVEL
      5. }
      6. vrrp_script check_apiserver {
      7. script "/etc/keepalived/check_apiserver.sh"
      8. interval 3
      9. weight -2
      10. fall 10
      11. rise 2
      12. }
      13. vrrp_instance VI_1 {
      14. state MASTER
      15. interface ${interface}
      16. virtual_router_id 51
      17. priority 50
      18. authentication {
      19. auth_type PASS
      20. auth_pass 4be37dc3b4c90194d1600c483e10ad1d
      21. }
      22. virtual_ipaddress {
      23. ${virtualIP}
      24. }
      25. track_script {
      26. check_apiserver
      27. }
      28. }
      29. EOF
    • check_apiserver.sh

      1. $ sudo bash -c 'cat > /etc/keepalived/check_apiserver.sh' << EOF
      2. #!/bin/sh
      3. errorExit() {
      4. echo "*** \$*" 1>&2
      5. exit 1
      6. }
      7. curl --silent \
      8. --max-time 2 \
      9. --insecure https://localhost:6443/ \
      10. -o /dev/null \
      11. || errorExit 'Error GET https://localhost:6443/'
      12. if ip addr | grep -q ${virtualIP}; then
      13. curl --silent \
      14. --max-time 2 \
      15. --insecure https://${virtualIP}:6443/ \
      16. -o /dev/null \
      17. || errorExit "Error GET https://${virtualIP}:6443/"
      18. fi
      19. EOF

enable keepalived services in all masters

  • start keepalived serice and verify
    1. $ sudo systemctl enable keepalived.service
    2. $ sudo systemctl start keepalived.service

  • verify

    1. $ sudo systemctl is-enabled keepalived.service
    2. enabled
    3. $ sudo systemctl is-active keepalived.service
    4. active
    5. $ ip -4 a s ${interface}
    6. 2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    7. link/ether 00:50:50:85:96:64 brd ff:ff:ff:ff:ff:ff
    8. inet 192.168.100.202/24 brd 192.168.100.255 scope global noprefixroute eno1
    9. valid_lft forever preferred_lft forever
    10. inet 192.168.100.250/32 scope global eno1
    11. valid_lft forever preferred_lft forever
click for more details
  1. $ for i in {1..3}; do
  2. -> echo '---------'
  3. -> ssh -q devops@master0${i} "/usr/sbin/ip -4 a s $(netstat -nr | grep -E 'UG|UGSc' | grep -E '^0.0.0|default' | grep -E '[0-9.]{7,15}' | awk -F' ' '{print $NF}')"
  4. -> done
  5. ---------
  6. 2: eno1: mtu 1500 qdisc mq state UP group default qlen 1000
  7. inet 192.168.100.200/24 brd 192.168.100.255 scope global noprefixroute eno1
  8. valid_lft forever preferred_lft forever
  9. ---------
  10. 2: eno1: mtu 1500 qdisc mq state UP group default qlen 1000
  11. inet 192.168.100.201/24 brd 192.168.100.255 scope global noprefixroute eno1
  12. valid_lft forever preferred_lft forever
  13. ---------
  14. 2: eno1: mtu 1500 qdisc mq state UP group default qlen 1000
  15. inet 192.168.100.202/24 brd 192.168.100.255 scope global noprefixroute eno1
  16. valid_lft forever preferred_lft forever
  17. inet 192.168.100.250/32 scope global eno1 <<<< virtual ip in master node 03
  18. valid_lft forever preferred_lft forever

  1. $ for i in {1..3}; do
  2. -> ssh -q devops@master0${i} "sudo systemctl status keepalived"
  3. -> echo ''
  4. -> done
  5. keepalived.service - LVS and VRRP High Availability Monitor
  6. Loaded: loaded (/etc/systemd/system/keepalived.service; enabled; vendor preset: disabled)
  7. Active: active (running) since Tue 2019-09-03 01:13:17 PDT; 18min ago
  8. Process: 26437 ExecStart=/usr/local/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
  9. Main PID: 26438 (keepalived)
  10. Tasks: 2
  11. Memory: 652.0K
  12. CGroup: /system.slice/keepalived.service
  13. ├─26438 /usr/local/sbin/keepalived -D
  14. └─26439 /usr/local/sbin/keepalived -D
  15. Sep 03 01:15:35 master01 Keepalived_vrrp[26439]: (VI_1) ip address associated with VRID 51 not present in MASTER advert : 10.69.78.50
  16. Sep 03 01:15:36 master01 Keepalived_vrrp[26439]: (VI_1) ip address associated with VRID 51 not present in MASTER advert : 10.69.78.50
  17. Sep 03 01:15:37 master01 Keepalived_vrrp[26439]: (VI_1) ip address associated with VRID 51 not present in MASTER advert : 10.69.78.50
  18. Sep 03 01:15:38 master01 Keepalived_vrrp[26439]: (VI_1) ip address associated with VRID 51 not present in MASTER advert : 10.69.78.50
  19. Sep 03 01:15:39 master01 Keepalived_vrrp[26439]: (VI_1) ip address associated with VRID 51 not present in MASTER advert : 10.69.78.50
  20. Sep 03 01:15:40 master01 Keepalived_vrrp[26439]: (VI_1) ip address associated with VRID 51 not present in MASTER advert : 10.69.78.50
  21. Sep 03 01:15:41 master01 Keepalived_vrrp[26439]: (VI_1) ip address associated with VRID 51 not present in MASTER advert : 10.69.78.50
  22. Sep 03 01:15:42 master01 Keepalived_vrrp[26439]: (VI_1) ip address associated with VRID 51 not present in MASTER advert : 10.69.78.50
  23. Sep 03 01:15:43 master01 Keepalived_vrrp[26439]: (VI_1) ip address associated with VRID 51 not present in MASTER advert : 10.69.78.50
  24. Sep 03 01:15:43 master01 Keepalived_vrrp[26439]: (VI_1) ip address associated with VRID 51 not present in MASTER advert : 10.69.78.50
  25. keepalived.service - LVS and VRRP High Availability Monitor
  26. Loaded: loaded (/etc/systemd/system/keepalived.service; enabled; vendor preset: disabled)
  27. Active: active (running) since Tue 2019-09-03 01:17:24 PDT; 14min ago
  28. Process: 32672 ExecStart=/usr/local/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
  29. Main PID: 32673 (keepalived)
  30. Tasks: 2
  31. Memory: 652.0K
  32. CGroup: /system.slice/keepalived.service
  33. ├─32673 /usr/local/sbin/keepalived -D
  34. └─32674 /usr/local/sbin/keepalived -D
  35. Sep 03 01:17:24 master02 Keepalived_vrrp[32674]: (Line 19) Truncating auth_pass to 8 characters
  36. Sep 03 01:17:24 master02 Keepalived_vrrp[32674]: WARNING - script '/etc/keepalived/check_apiserver.sh' is not executable for uid:gid 0:0 - disabling.
  37. Sep 03 01:17:24 master02 Keepalived_vrrp[32674]: SECURITY VIOLATION - scripts are being executed but script_security not enabled.
  38. Sep 03 01:17:24 master02 Keepalived_vrrp[32674]: Assigned address 192.168.100.201 for interface eno1
  39. Sep 03 01:17:24 master02 Keepalived_vrrp[32674]: Assigned address fe80::250:56ff:fe88:fd2 for interface eno1
  40. Sep 03 01:17:24 master02 Keepalived_vrrp[32674]: Registering gratuitous ARP shared channel
  41. Sep 03 01:17:24 master02 Keepalived_vrrp[32674]: (VI_1) removing VIPs.
  42. Sep 03 01:17:24 master02 Keepalived_vrrp[32674]: (VI_1) Entering BACKUP STATE (init)
  43. Sep 03 01:17:24 master02 Keepalived_vrrp[32674]: VRRP sockpool: [ifindex(2), family(IPv4), proto(112), unicast(0), fd(11,12)]
  44. Sep 03 01:17:24 master02 systemd[1]: Started LVS and VRRP High Availability Monitor.
  45. keepalived.service - LVS and VRRP High Availability Monitor
  46. Loaded: loaded (/etc/systemd/system/keepalived.service; enabled; vendor preset: disabled)
  47. Active: active (running) since Tue 2019-09-03 01:17:35 PDT; 14min ago
  48. Process: 16830 ExecStart=/usr/local/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
  49. Main PID: 16831 (keepalived)
  50. Tasks: 2
  51. Memory: 648.0K
  52. CGroup: /system.slice/keepalived.service
  53. ├─16831 /usr/local/sbin/keepalived -D
  54. └─16832 /usr/local/sbin/keepalived -D
  55. Sep 03 01:17:35 master03 Keepalived_vrrp[16832]: (Line 19) Truncating auth_pass to 8 characters
  56. Sep 03 01:17:35 master03 Keepalived_vrrp[16832]: WARNING - script '/etc/keepalived/check_apiserver.sh' is not executable for uid:gid 0:0 - disabling.
  57. Sep 03 01:17:35 master03 Keepalived_vrrp[16832]: SECURITY VIOLATION - scripts are being executed but script_security not enabled.
  58. Sep 03 01:17:35 master03 Keepalived_vrrp[16832]: Assigned address 192.168.100.202 for interface eno1
  59. Sep 03 01:17:35 master03 Keepalived_vrrp[16832]: Assigned address fe80::250:56ff:fe88:9624 for interface eno1
  60. Sep 03 01:17:35 master03 Keepalived_vrrp[16832]: Registering gratuitous ARP shared channel
  61. Sep 03 01:17:35 master03 Keepalived_vrrp[16832]: (VI_1) removing VIPs.
  62. Sep 03 01:17:35 master03 Keepalived_vrrp[16832]: (VI_1) Entering BACKUP STATE (init)
  63. Sep 03 01:17:35 master03 Keepalived_vrrp[16832]: VRRP sockpool: [ifindex(2), family(IPv4), proto(112), unicast(0), fd(11,12)]
  64. Sep 03 01:17:35 master03 systemd[1]: Started LVS and VRRP High Availability Monitor.

haproxy

configuration

  • haproxy configure

    1. $ sudo bash -c 'cat /etc/haproxy/haproxy.cfg' << EOF
    2. #---------------------------------------------------------------------
    3. # Example configuration for a possible web application. See the
    4. # full configuration options online.
    5. #
    6. # http://haproxy.1wt.eu/download/2.0/doc/configuration.txt
    7. #
    8. #---------------------------------------------------------------------
    9. #---------------------------------------------------------------------
    10. # Global settings
    11. #---------------------------------------------------------------------
    12. global
    13. log 127.0.0.1 local2
    14. chroot /var/lib/haproxy
    15. pidfile /var/run/haproxy.pid
    16. maxconn 4000
    17. user haproxy
    18. group haproxy
    19. daemon
    20. # turn on stats unix socket
    21. stats socket /var/lib/haproxy/stats
    22. #---------------------------------------------------------------------
    23. # common defaults that all the 'listen' and 'backend' sections will
    24. # use if not designated in their block
    25. #---------------------------------------------------------------------
    26. defaults
    27. mode http
    28. log global
    29. option httplog
    30. option dontlognull
    31. option http-server-close
    32. option forwardfor except 127.0.0.0/8
    33. option redispatch
    34. retries 3
    35. timeout http-request 10s
    36. timeout queue 1m
    37. timeout connect 10s
    38. timeout client 1m
    39. timeout server 1m
    40. timeout http-keep-alive 10s
    41. timeout check 10s
    42. maxconn 3000
    43. #---------------------------------------------------------------------
    44. # kubernetes apiserver frontend which proxys to the backends
    45. #---------------------------------------------------------------------
    46. frontend kubernetes-apiserver
    47. mode tcp
    48. bind *:16443
    49. option tcplog
    50. default_backend kubernetes-apiserver
    51. #---------------------------------------------------------------------
    52. # round robin balancing between the various backends
    53. #---------------------------------------------------------------------
    54. backend kubernetes-apiserver
    55. mode tcp
    56. balance roundrobin
    57. option tcplog
    58. option tcp-check
    59. server ${master01Name} ${master01IP}:6443 check
    60. server ${master02Name} ${master02IP}:6443 check
    61. server ${master03Name} ${master03IP}:6443 check
    62. #---------------------------------------------------------------------
    63. # collection haproxy statistics message
    64. #---------------------------------------------------------------------
    65. listen stats
    66. bind :8000
    67. stats auth admin:devops
    68. maxconn 50
    69. stats refresh 10s
    70. stats realm HAProxy\ Statistics
    71. stats uri /healthy
    72. EOF
  • Service

    1. $ sudo bash -c 'cat > /lib/systemd/system/haproxy.service' << EOF
    2. [Unit]
    3. Description=HAProxy Load Balancer
    4. After=network.target syslog.service
    5. Wants=syslog.service
    6. [Service]
    7. Environment="CONFIG=/etc/haproxy/haproxy.cfg" "PIDFILE=/run/haproxy.pid"
    8. EnvironmentFile=-/etc/default/haproxy
    9. ExecStartPre=/usr/sbin/haproxy -f $CONFIG -c -q
    10. ExecStart=/usr/sbin/haproxy -W -f $CONFIG -p $PIDFILE $EXTRAOPTS
    11. ExecReload=/usr/sbin/haproxy -f $CONFIG -c -q $EXTRAOPTS $RELOADOPTS
    12. ExecReload=/bin/kill -USR2 $MAINPID
    13. KillMode=mixed
    14. Restart=always
    15. Type=forking
    16. [Install]
    17. WantedBy=multi-user.target
    18. EOF

    Start Service

    1. $ sudo systemctl enabled haproxy.service
    2. $ sudo systemctl start haproxy.service

verify

  1. $ sudo systemctl is-enabled haproxy.service
  2. enabled
  3. $ sudo systemctl is-active haproxy.service
  4. active
Powered By Valine
v1.5.2