Create a dynamic configuration (backends) for the load balancer in OpenShift 3.11.
All operations are performed on the management node.
I will focus on backend configuration as it is a more complicated and interesting one.
I assume that a router is configured and running on every infrastructure node.
Define load balancer configuration in hosts file.
# LB
openshift_loadbalancer_additional_frontends:
- name: "apps-http"
mode: "tcp"
options:
- "tcplog"
binds:
- "*:80"
default_backend: "apps-http"
- name: "apps-https"
mode: "tcp"
options:
- "tcplog"
binds:
- "*:443"
default_backend: "apps-https"
# do not define openshift_loadbalancer_additional_backends as it will be generated dynamically
Define load balancer in hosts file put infrastructure nodes as node-config-infra using openshift_node_group_name.
lb:
hosts:
openshift-example-lb-1.example.org:
masters:
hosts:
openshift-example-master-1.example.org:
etcd:
hosts:
openshift-example-master-1.example.org:
nodes:
hosts:
openshift-example-master-1.example.org:
openshift_node_group_name: node-config-master
openshift-example-infra-1.example.org:
openshift_node_group_name: node-config-infra
openshift-example-node-[1:2].example.org:
openshift_node_group_name: node-config-compute
Create a directory for an additional playbook (inside openshift-ansible).
$ mkdir playbooks/extra
Create an additional playbook that will update openshift_loadbalancer_additional_backends configuration variable.
$ cat playbooks/extra/openshift_loadbalancer_additional_backends.yml
---
- hosts: all
tasks:
- set_fact: backends_http="{{ [] }}"
- set_fact: backends_http="{{ backends_http + [{ 'name':item, 'address':hostvars[item]['ansible_default_ipv4']['address']+':80', 'opts':'check'}] }}"
with_items: "{{ groups['nodes'] }}"
when: hostvars[item]['openshift_node_group_name'] == 'node-config-infra'
- set_fact: backends_https="{{ [] }}"
- set_fact: backends_https="{{ backends_https + [{ 'name':item, 'address':hostvars[item]['ansible_default_ipv4']['address']+':443', 'opts':'check'}] }}"
with_items: "{{ groups['nodes'] }}"
when: hostvars[item]['openshift_node_group_name'] == 'node-config-infra'
- set_fact: openshift_loadbalancer_additional_backends="{{ [{'name':'apps-http', 'mode':'tcp', 'option':'tcplog', 'balance':'source', 'servers':backends_http}, {'name':'apps-https', 'mode':'tcp', 'option':'tcplog', 'balance':'source', 'servers':backends_https}] }}"
- debug: var=openshift_loadbalancer_additional_backends
Inspect dynamically generated openshift_loadbalancer_additional_backends configuration variable.
$ ansible-playbook -i hosts playbooks/extra/openshift_loadbalancer_additional_backends.yml --limit lb
[...]
TASK [debug] ***********************************************************************************************************************************************************************************************************************************************************************
Wednesday 15 April 2020 21:30:43 +0200 (0:00:00.109) 0:00:01.142 *******
ok: [openshift-example-lb-1.example.org] => {
"openshift_loadbalancer_additional_backends": [
{
"balance": "source",
"mode": "tcp",
"name": "apps-http",
"option": "tcplog",
"servers": [
{
"address": "192.0.2.11:80",
"name": "openshift-example-infra-1.example.org",
"opts": "check"
}
]
},
{
"balance": "source",
"mode": "tcp",
"name": "apps-https",
"option": "tcplog",
"servers": [
{
"address": "192.0.2.11:443",
"name": "openshift-example-infra-1.example.org",
"opts": "check"
}
]
}
]
}
[...]
Update the load balancer playbook to include the new code.
$ cat playbooks/openshift-loadbalancer/config.yml
---
- import_playbook: ../extra./openshift_loadbalancer_additional_backends.yml
- import_playbook: ../init/main.yml
vars:
l_init_fact_hosts: "oo_masters_to_config:oo_lb_to_config"
l_openshift_version_set_hosts: "oo_masters_to_config:!oo_first_master"
l_sanity_check_hosts: "{{ groups['oo_masters_to_config'] | union(groups['oo_lb_to_config']) }}"
- import_playbook: private/config.yml
Run playbook to update balancer configuration.
$ ansible-playbook -i hosts playbooks/openshift-loadbalancer/config.yml
PLAY RECAP ************************************************************************************************************************************************************************************************************************************************************************* localhost : ok=12 changed=0 unreachable=0 failed=0 skipped=4 rescued=0 ignored=0 openshift-example-infra-1.example.org : ok=6 changed=0 unreachable=0 failed=0 skipped=5 rescued=0 ignored=0 openshift-example-lb-1.example.org : ok=34 changed=4 unreachable=0 failed=0 skipped=30 rescued=0 ignored=0 openshift-example-master-1.example.org : ok=45 changed=0 unreachable=0 failed=0 skipped=36 rescued=0 ignored=0 openshift-example-node-1.example.org : ok=6 changed=0 unreachable=0 failed=0 skipped=5 rescued=0 ignored=0 openshift-example-node-2.example.org : ok=6 changed=0 unreachable=0 failed=0 skipped=5 rescued=0 ignored=0 INSTALLER STATUS ******************************************************************************************************************************************************************************************************************************************************************* Initialization : Complete (0:00:10) Load Balancer Install : Complete (0:00:10) Wednesday 15 April 2020 21:16:57 +0200 (0:00:00.035) 0:00:22.088 ******* =============================================================================== tuned : Restart tuned service ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 1.87s tuned : Ensure files are populated from templates --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 1.63s openshift_loadbalancer : Configure haproxy ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.87s openshift_loadbalancer : Enable and start haproxy --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.76s openshift_loadbalancer : Install haproxy ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 0.72s tuned : Ensure directory structure exists ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.61s get openshift_current_version ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.61s openshift_loadbalancer : Add iptables allow rules --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.61s Gather Cluster facts -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.61s set_fact -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.55s set_fact -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.54s tuned : Check for tuned package --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.43s openshift_control_plane : slurp --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.42s openshift_loadbalancer : Configure systemd service directory for haproxy ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.41s openshift_loadbalancer : Configure the nofile limits for haproxy ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 0.40s Detecting Operating System from ostree_booted ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.40s Initialize openshift.node.sdn_mtu ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.38s tuned : Make tuned use the recommended tuned profile on restart ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.33s openshift_loadbalancer : restart haproxy ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 0.33s set_fact -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0.31s
Configuration generated on the load balancer.
# Global settings
#---------------------------------------------------------------------
global
maxconn 20000
log /dev/log local0 info
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
user haproxy
group haproxy
daemon
# turn on stats unix socket
stats socket /var/lib/haproxy/stats
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
mode http
log global
option httplog
option dontlognull
# option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 300s
timeout server 300s
timeout http-keep-alive 10s
timeout check 10s
maxconn 20000
listen stats
bind :9000
mode http
stats enable
stats uri /
frontend atomic-openshift-api
bind *:8443
default_backend atomic-openshift-api
mode tcp
option tcplog
frontend apps-http
bind *:80
default_backend apps-http
mode tcp
option tcplog
frontend apps-https
bind *:443
default_backend apps-https
mode tcp
option tcplog
backend atomic-openshift-api
balance source
mode tcp
server master0 192.0.2.10:8443 check
backend apps-http
balance source
mode tcp
server openshift-example-infra-1.example.org 192.0.2.11:80 check
backend apps-https
balance source
mode tcp
server openshift-example-infra-1.example.org 192.0.2.11:443 check
You can automate frontend configuration in a similar way. It will be an easy task to automate after this (loop over simple port list).