High Availability Guide

Deploy a Keystore

oc -n keycloak create secret generic xsite-keystore-secret \
  --from-file=keystore.p12="./certs/keystore.p12" \ 1
  --from-literal=password=secret \ 2
  --from-literal=type=pkcs12 3

Deploy a Truststore

oc -n keycloak create secret generic xsite-truststore-secret \
        --from-file=truststore.p12="./certs/truststore.p12" \  1
        --from-literal=password=caSecret \  2
        --from-literal=type=pkcs12  3

The Infinispan CR for Site-A

apiVersion: infinispan.org/v1
kind: Infinispan
metadata:
  name: infinispan 1
  namespace: keycloak
  annotations:
    infinispan.org/monitoring: 'true' 2
spec:
  replicas: 3
  jmx:
    enabled: true
  security:
    endpointSecretName: connect-secret 3
  service:
    type: DataGrid
    sites:
      local:
        name: site-a 4
        expose:
          type: Route 5
        maxRelayNodes: 128
        encryption:
          transportKeyStore:
            secretName: xsite-keystore-secret 6
            alias: xsite 7
            filename: keystore.p12 8
          routerKeyStore:
            secretName: xsite-keystore-secret 9
            alias: xsite 10
            filename: keystore.p12 11
          trustStore:
            secretName: xsite-truststore-secret 12
            filename: truststore.p12 13
      locations:
        - name: site-b 14
          clusterName: infinispan
          namespace: keycloak 15
          url: openshift://api.site-b 16
          secretName: xsite-token-secret 17

The Infinispan CR for Site-B

apiVersion: infinispan.org/v1
kind: Infinispan
metadata:
  name: infinispan 1
  namespace: keycloak
  annotations:
    infinispan.org/monitoring: 'true' 2
spec:
  replicas: 3
  jmx:
    enabled: true
  security:
    endpointSecretName: connect-secret 3
  service:
    type: DataGrid
    sites:
      local:
        name: site-b 4
        expose:
          type: Route 5
        maxRelayNodes: 128
        encryption:
          transportKeyStore:
            secretName: xsite-keystore-secret 6
            alias: xsite 7
            filename: keystore.p12 8
          routerKeyStore:
            secretName: xsite-keystore-secret 9
            alias: xsite 10
            filename: keystore.p12 11
          trustStore:
            secretName: xsite-truststore-secret 12
            filename: truststore.p12 13
      locations:
        - name: site-a 14
          clusterName: infinispan
          namespace: keycloak 15
          url: openshift://api.site-a 16
          secretName: xsite-token-secret 17

authenticationSessions Cache CR in Site-B

apiVersion: infinispan.org/v2alpha1
kind: Cache
metadata:
  name: authenticationsessions
  namespace: keycloak
spec:
  clusterName: infinispan
  name: authenticationSessions
  template: |-
    distributedCache:
      mode: "SYNC"
      owners: "2"
      statistics: "true"
      remoteTimeout: "5000"
      encoding:
        media-type: "application/x-protostream"
      locking:
        acquireTimeout: "4000"
      transaction:
        mode: "NON_XA" 1
        locking: "PESSIMISTIC" 2
      stateTransfer:
        chunkSize: "16"
      backups:
        site-a: 3
          backup:
            strategy: "SYNC" 4
            timeout: "4500" 5
            failurePolicy: "FAIL" 6
            stateTransfer:
              chunkSize: "16"

Command:

aws globalaccelerator create-accelerator \
  --name example-accelerator \ 1
  --ip-address-type DUAL_STACK \ 2
  --region us-west-2 3

Output:

{
    "Accelerator": {
        "AcceleratorArn": "arn:aws:globalaccelerator::606671647913:accelerator/e35a94dd-391f-4e3e-9a3d-d5ad22a78c71", 1
        "Name": "example-accelerator",
        "IpAddressType": "DUAL_STACK",
        "Enabled": true,
        "IpSets": [
            {
                "IpFamily": "IPv4",
                "IpAddresses": [
                    "75.2.42.125",
                    "99.83.132.135"
                ],
                "IpAddressFamily": "IPv4"
            },
            {
                "IpFamily": "IPv6",
                "IpAddresses": [
                    "2600:9000:a400:4092:88f3:82e2:e5b2:e686",
                    "2600:9000:a516:b4ef:157e:4cbd:7b48:20f1"
                ],
                "IpAddressFamily": "IPv6"
            }
        ],
        "DnsName": "a099f799900e5b10d.awsglobalaccelerator.com", 2
        "Status": "IN_PROGRESS",
        "CreatedTime": "2023-11-13T15:46:40+00:00",
        "LastModifiedTime": "2023-11-13T15:46:42+00:00",
        "DualStackDnsName": "ac86191ca5121e885.dualstack.awsglobalaccelerator.com" 3
    }
}

Command:

aws globalaccelerator create-listener \
  --accelerator-arn 'arn:aws:globalaccelerator::606671647913:accelerator/e35a94dd-391f-4e3e-9a3d-d5ad22a78c71' \
  --port-ranges '[{"FromPort":443,"ToPort":443}]' \
  --protocol TCP \
  --region us-west-2

Output:

{
    "Listener": {
        "ListenerArn": "arn:aws:globalaccelerator::606671647913:accelerator/e35a94dd-391f-4e3e-9a3d-d5ad22a78c71/listener/1f396d40",
        "PortRanges": [
            {
                "FromPort": 443,
                "ToPort": 443
            }
        ],
        "Protocol": "TCP",
        "ClientAffinity": "NONE"
    }
}

Command:

CLUSTER_1_ENDPOINT_ARN=$(aws elbv2 describe-load-balancers \
    --query "LoadBalancers[?DNSName=='abab80a363ce8479ea9c4349d116bce2-6b65e8b4272fa4b5.elb.eu-west-1.amazonaws.com'].LoadBalancerArn" \ 1
    --region eu-west-1 \ 2
    --output text
)
CLUSTER_2_ENDPOINT_ARN=$(aws elbv2 describe-load-balancers \
    --query "LoadBalancers[?DNSName=='a1c76566e3c334e4ab7b762d9f8dcbcf-985941f9c8d108d4.elb.eu-west-1.amazonaws.com'].LoadBalancerArn" \ 3
    --region eu-west-1 \ 4
    --output text
)
ENDPOINTS='[
  {
    "EndpointId": "'${CLUSTER_1_ENDPOINT_ARN}'",
    "Weight": 128,
    "ClientIPPreservationEnabled": false
  },
  {
    "EndpointId": "'${CLUSTER_2_ENDPOINT_ARN}'",
    "Weight": 128,
    "ClientIPPreservationEnabled": false
  }
]'
aws globalaccelerator create-endpoint-group \
  --listener-arn 'arn:aws:globalaccelerator::606671647913:accelerator/e35a94dd-391f-4e3e-9a3d-d5ad22a78c71/listener/1f396d40' \ 5
  --traffic-dial-percentage 100 \
  --endpoint-configurations ${ENDPOINTS} \
  --endpoint-group-region eu-west-1 \ 6
  --region us-west-2

Output:

{
    "EndpointGroup": {
        "EndpointGroupArn": "arn:aws:globalaccelerator::606671647913:accelerator/e35a94dd-391f-4e3e-9a3d-d5ad22a78c71/listener/1f396d40/endpoint-group/2581af0dc700",
        "EndpointGroupRegion": "eu-west-1",
        "EndpointDescriptions": [
            {
                "EndpointId": "arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/abab80a363ce8479ea9c4349d116bce2/6b65e8b4272fa4b5",
                "Weight": 128,
                "HealthState": "HEALTHY",
                "ClientIPPreservationEnabled": false
            },
            {
                "EndpointId": "arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/a1c76566e3c334e4ab7b762d9f8dcbcf/985941f9c8d108d4",
                "Weight": 128,
                "HealthState": "HEALTHY",
                "ClientIPPreservationEnabled": false
            }
        ],
        "TrafficDialPercentage": 100.0,
        "HealthCheckPort": 443,
        "HealthCheckProtocol": "TCP",
        "HealthCheckPath": "undefined",
        "HealthCheckIntervalSeconds": 30,
        "ThresholdCount": 3
    }
}

Command:

oc apply -f - << EOF
apiVersion: v1
kind: ConfigMap
metadata:
  name: user-workload-monitoring-config
  namespace: openshift-user-workload-monitoring
data:
  config.yaml: |
    alertmanager:
      enabled: true
      enableAlertmanagerConfig: true
EOF
oc -n openshift-user-workload-monitoring rollout status --watch statefulset.apps/alertmanager-user-workload

Command:

aws secretsmanager create-secret \
  --name webhook-password \ 1
  --secret-string changeme \ 2
  --region eu-west-1 3

Command:

FUNCTION_NAME= 1
ROLE_ARN=$(aws iam create-role \
  --role-name ${FUNCTION_NAME} \
  --assume-role-policy-document \
  '{
    "Version": "2012-10-17",
    "Statement": [
      {
        "Effect": "Allow",
        "Principal": {
          "Service": "lambda.amazonaws.com"
        },
        "Action": "sts:AssumeRole"
      }
    ]
  }' \
  --query 'Role.Arn' \
  --region eu-west-1 \ 2
  --output text
)

Command:

POLICY_ARN=$(aws iam create-policy \
  --policy-name LambdaSecretManager \
  --policy-document \
  '{
      "Version": "2012-10-17",
      "Statement": [
          {
              "Effect": "Allow",
              "Action": [
                  "secretsmanager:GetSecretValue"
              ],
              "Resource": "*"
          }
      ]
  }' \
  --query 'Policy.Arn' \
  --output text
)
aws iam attach-role-policy \
  --role-name ${FUNCTION_NAME} \
  --policy-arn ${POLICY_ARN}

Command:

aws iam attach-role-policy \
  --role-name ${FUNCTION_NAME} \
  --policy-arn arn:aws:iam::aws:policy/ElasticLoadBalancingReadOnly

Command:

aws iam attach-role-policy \
  --role-name ${FUNCTION_NAME} \
  --policy-arn arn:aws:iam::aws:policy/GlobalAcceleratorFullAccess

Command:

LAMBDA_ZIP=/tmp/lambda.zip
cat << EOF > /tmp/lambda.py

from urllib.error import HTTPError

import boto3
import jmespath
import json
import os
import urllib3

from base64 import b64decode
from urllib.parse import unquote

# Prevent unverified HTTPS connection warning
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


class MissingEnvironmentVariable(Exception):
    pass


class MissingSiteUrl(Exception):
    pass


def env(name):
    if name in os.environ:
        return os.environ[name]
    raise MissingEnvironmentVariable(f"Environment Variable '{name}' must be set")


def handle_site_offline(labels):
    a_client = boto3.client('globalaccelerator', region_name='us-west-2')

    acceleratorDNS = labels['accelerator']
    accelerator = jmespath.search(f"Accelerators[?(DnsName=='{acceleratorDNS}'|| DualStackDnsName=='{acceleratorDNS}')]", a_client.list_accelerators())
    if not accelerator:
        print(f"Ignoring SiteOffline alert as accelerator with DnsName '{acceleratorDNS}' not found")
        return

    accelerator_arn = accelerator[0]['AcceleratorArn']
    listener_arn = a_client.list_listeners(AcceleratorArn=accelerator_arn)['Listeners'][0]['ListenerArn']

    endpoint_group = a_client.list_endpoint_groups(ListenerArn=listener_arn)['EndpointGroups'][0]
    endpoints = endpoint_group['EndpointDescriptions']

    # Only update accelerator endpoints if two entries exist
    if len(endpoints) > 1:
        # If the reporter endpoint is not healthy then do nothing for now
        # A Lambda will eventually be triggered by the other offline site for this reporter
        reporter = labels['reporter']
        reporter_endpoint = [e for e in endpoints if endpoint_belongs_to_site(e, reporter)][0]
        if reporter_endpoint['HealthState'] == 'UNHEALTHY':
            print(f"Ignoring SiteOffline alert as reporter '{reporter}' endpoint is marked UNHEALTHY")
            return

        offline_site = labels['site']
        endpoints = [e for e in endpoints if not endpoint_belongs_to_site(e, offline_site)]
        del reporter_endpoint['HealthState']
        a_client.update_endpoint_group(
            EndpointGroupArn=endpoint_group['EndpointGroupArn'],
            EndpointConfigurations=endpoints
        )
        print(f"Removed site={offline_site} from Accelerator EndpointGroup")

        take_infinispan_site_offline(reporter, offline_site)
        print(f"Backup site={offline_site} caches taken offline")
    else:
        print("Ignoring SiteOffline alert only one Endpoint defined in the EndpointGroup")


def endpoint_belongs_to_site(endpoint, site):
    lb_arn = endpoint['EndpointId']
    region = lb_arn.split(':')[3]
    client = boto3.client('elbv2', region_name=region)
    tags = client.describe_tags(ResourceArns=[lb_arn])['TagDescriptions'][0]['Tags']
    for tag in tags:
        if tag['Key'] == 'site':
            return tag['Value'] == site
    return false


def take_infinispan_site_offline(reporter, offlinesite):
    endpoints = json.loads(INFINISPAN_SITE_ENDPOINTS)
    if reporter not in endpoints:
        raise MissingSiteUrl(f"Missing URL for site '{reporter}' in 'INFINISPAN_SITE_ENDPOINTS' json")

    endpoint = endpoints[reporter]
    password = get_secret(INFINISPAN_USER_SECRET)
    url = f"https://{endpoint}/rest/v2/container/x-site/backups/{offlinesite}?action=take-offline"
    http = urllib3.PoolManager(cert_reqs='CERT_NONE')
    headers = urllib3.make_headers(basic_auth=f"{INFINISPAN_USER}:{password}")
    try:
        rsp = http.request("POST", url, headers=headers)
        if rsp.status >= 400:
            raise HTTPError(f"Unexpected response status '%d' when taking site offline", rsp.status)
        rsp.release_conn()
    except HTTPError as e:
        print(f"HTTP error encountered: {e}")


def get_secret(secret_name):
    session = boto3.session.Session()
    client = session.client(
        service_name='secretsmanager',
        region_name=SECRETS_REGION
    )
    return client.get_secret_value(SecretId=secret_name)['SecretString']


def decode_basic_auth_header(encoded_str):
    split = encoded_str.strip().split(' ')
    if len(split) == 2:
        if split[0].strip().lower() == 'basic':
            try:
                username, password = b64decode(split[1]).decode().split(':', 1)
            except:
                raise DecodeError
        else:
            raise DecodeError
    else:
        raise DecodeError

    return unquote(username), unquote(password)


def handler(event, context):
    print(json.dumps(event))

    authorization = event['headers'].get('authorization')
    if authorization is None:
        print("'Authorization' header missing from request")
        return {
            "statusCode": 401
        }

    expectedPass = get_secret(WEBHOOK_USER_SECRET)
    username, password = decode_basic_auth_header(authorization)
    if username != WEBHOOK_USER and password != expectedPass:
        print('Invalid username/password combination')
        return {
            "statusCode": 403
        }

    body = event.get('body')
    if body is None:
        raise Exception('Empty request body')

    body = json.loads(body)
    print(json.dumps(body))

    if body['status'] != 'firing':
        print("Ignoring alert as status is not 'firing', status was: '%s'" % body['status'])
        return {
            "statusCode": 204
        }

    for alert in body['alerts']:
        labels = alert['labels']
        if labels['alertname'] == 'SiteOffline':
            handle_site_offline(labels)

    return {
        "statusCode": 204
    }


INFINISPAN_USER = env('INFINISPAN_USER')
INFINISPAN_USER_SECRET = env('INFINISPAN_USER_SECRET')
INFINISPAN_SITE_ENDPOINTS = env('INFINISPAN_SITE_ENDPOINTS')
SECRETS_REGION = env('SECRETS_REGION')
WEBHOOK_USER = env('WEBHOOK_USER')
WEBHOOK_USER_SECRET = env('WEBHOOK_USER_SECRET')

EOF
zip -FS --junk-paths ${LAMBDA_ZIP} /tmp/lambda.py

Command:

aws lambda create-function \
  --function-name ${FUNCTION_NAME} \
  --zip-file fileb://${LAMBDA_ZIP} \
  --handler lambda.handler \
  --runtime python3.12 \
  --role ${ROLE_ARN} \
  --region eu-west-1 1

Command:

aws lambda create-function-url-config \
  --function-name ${FUNCTION_NAME} \
  --auth-type NONE \
  --region eu-west-1 1

Command:

aws lambda add-permission \
  --action "lambda:InvokeFunctionUrl" \
  --function-name ${FUNCTION_NAME} \
  --principal "*" \
  --statement-id FunctionURLAllowPublicAccess \
  --function-url-auth-type NONE \
  --region eu-west-1 1

Command:

aws lambda get-function-url-config \
  --function-name ${FUNCTION_NAME} \
  --query "FunctionUrl" \
  --region eu-west-1 \1
  --output text

Output:

https://tjqr2vgc664b6noj6vugprakoq0oausj.lambda-url.eu-west-1.on.aws

Command:

NAMESPACE= # The namespace containing your deployments
oc apply -n ${NAMESPACE} -f - << EOF
apiVersion: v1
kind: Secret
type: kubernetes.io/basic-auth
metadata:
  name: webhook-credentials
stringData:
  username: 'keycloak' 1
  password: 'changme' 2
---
apiVersion: monitoring.coreos.com/v1beta1
kind: AlertmanagerConfig
metadata:
  name: example-routing
spec:
  route:
    receiver: default
    groupBy:
      - accelerator
    groupInterval: 90s
    groupWait: 60s
    matchers:
      - matchType: =
        name: alertname
        value: SiteOffline
  receivers:
    - name: default
      webhookConfigs:
        - url: 'https://tjqr2vgc664b6noj6vugprakoq0oausj.lambda-url.eu-west-1.on.aws/' 3
          httpConfig:
            basicAuth:
              username:
                key: username
                name: webhook-credentials
              password:
                key: password
                name: webhook-credentials
            tlsConfig:
              insecureSkipVerify: true
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: xsite-status
spec:
  groups:
    - name: xsite-status
      rules:
        - alert: SiteOffline
          expr: 'min by (namespace, site) (vendor_jgroups_site_view_status{namespace="default",site="site-b"}) == 0' 4
          labels:
            severity: critical
            reporter: site-a 5
            accelerator: a3da6a6cbd4e27b02.awsglobalaccelerator.com 6

Command:

oc -n openshift-operators scale --replicas=0 deployment/infinispan-operator-controller-manager 1
oc -n openshift-operators rollout status -w deployment/infinispan-operator-controller-manager
oc -n ${NAMESPACE} scale --replicas=0 deployment/infinispan-router 2
oc -n ${NAMESPACE} rollout status -w deployment/infinispan-router

Command:

oc -n openshift-operators scale --replicas=1 deployment/infinispan-operator-controller-manager
oc -n openshift-operators rollout status -w deployment/infinispan-operator-controller-manager
oc -n ${NAMESPACE} scale --replicas=1 deployment/infinispan-router 1
oc -n ${NAMESPACE} rollout status -w deployment/infinispan-router