베어 메탈에 설치 프로그램이 프로비저닝 한 클러스터 배포

$ oc edit mc <machineconfig_custom_resource_name>

$ oc apply -f ./extraworker-secret.yaml

$ oc project openshift-machine-api

$ oc get machinesets

$ oc scale machineset <machineset_name> --replicas=<n> 

<load_balancer_ip_address>   A apps.<cluster_name>.<base_domain>
A record pointing to Load Balancer Front End

# ...
platform:
  baremetal:
    loadBalancer:
      type: UserManaged 

    apiVIPs:
    - <api_ip> 

    ingressVIPs:
    - <ingress_ip> 

# ...

apiVersion: v1
baseDomain: <domain>
metadata:
  name: <cluster_name>
networking:
  machineNetwork:
  - cidr: <public_cidr>
  networkType: OVNKubernetes
compute:
- name: worker
  replicas: 2 

controlPlane:
  name: master
  replicas: 3
  platform:
    baremetal: {}
platform:
  baremetal:
    apiVIPs:
      - <api_ip>
    ingressVIPs:
      - <wildcard_ip>
    provisioningNetworkCIDR: <CIDR>
    bootstrapExternalStaticIP: <bootstrap_static_ip_address> 

    bootstrapExternalStaticGateway: <bootstrap_static_gateway> 

    bootstrapExternalStaticDNS: <bootstrap_static_dns> 

    hosts:
      - name: openshift-master-0
        role: master
        bmc:
          address: ipmi://<out_of_band_ip> 

          username: <user>
          password: <password>
        bootMACAddress: <NIC1_mac_address>
        rootDeviceHints:
         deviceName: "<installation_disk_drive_path>" 

      - name: <openshift_master_1>
        role: master
        bmc:
          address: ipmi://<out_of_band_ip>
          username: <user>
          password: <password>
        bootMACAddress: <NIC1_mac_address>
        rootDeviceHints:
         deviceName: "<installation_disk_drive_path>"
      - name: <openshift_master_2>
        role: master
        bmc:
          address: ipmi://<out_of_band_ip>
          username: <user>
          password: <password>
        bootMACAddress: <NIC1_mac_address>
        rootDeviceHints:
         deviceName: "<installation_disk_drive_path>"
      - name: <openshift_worker_0>
        role: worker
        bmc:
          address: ipmi://<out_of_band_ip>
          username: <user>
          password: <password>
        bootMACAddress: <NIC1_mac_address>
      - name: <openshift_worker_1>
        role: worker
        bmc:
          address: ipmi://<out_of_band_ip>
          username: <user>
          password: <password>
        bootMACAddress: <NIC1_mac_address>
        rootDeviceHints:
         deviceName: "<installation_disk_drive_path>"
pullSecret: '<pull_secret>'
sshKey: '<ssh_pub_key>'

$ mkdir ~/clusterconfigs

$ cp install-config.yaml ~/clusterconfigs

$ ipmitool -I lanplus -U <user> -P <password> -H <management-server-ip> power off

for i in $(sudo virsh list | tail -n +3 | grep bootstrap | awk {'print $2'});
do
  sudo virsh destroy $i;
  sudo virsh undefine $i;
  sudo virsh vol-delete $i --pool $i;
  sudo virsh vol-delete $i.ign --pool $i;
  sudo virsh pool-destroy $i;
  sudo virsh pool-undefine $i;
done

$ export SERVER=<ip_address> 

$ export SystemID=<system_id> 

$ curl -u $USER:$PASS -X POST -H'Content-Type: application/json' -H'Accept: application/json' -d '{"ResetType": "On"}' https://$SERVER/redfish/v1/Systems/$SystemID/Actions/ComputerSystem.Reset

$ curl -u $USER:$PASS -X POST -H'Content-Type: application/json' -H'Accept: application/json' -d '{"ResetType": "ForceOff"}' https://$SERVER/redfish/v1/Systems/$SystemID/Actions/ComputerSystem.Reset

$ curl -u $USER:$PASS -X PATCH -H "Content-Type: application/json" -H "If-Match: <ETAG>"  https://$Server/redfish/v1/Systems/$SystemID/ -d '{"Boot": {"BootSourceOverrideTarget": "pxe", "BootSourceOverrideEnabled": "Once"}}

$ curl -u $USER:$PASS -X PATCH -H "Content-Type: application/json" -H "If-Match: <ETAG>"  https://$Server/redfish/v1/Systems/$SystemID/ -d '{"Boot": {"BootSourceOverrideMode":"UEFI"}}

$ curl -u $USER:$PASS -X PATCH -H "Content-Type: application/json" -H "If-Match: <ETAG>" https://$Server/redfish/v1/Systems/$SystemID/ -d '{"Boot": {"BootSourceOverrideTarget": "cd", "BootSourceOverrideEnabled": "Once"}}'

$ curl -u $USER:$PASS -X POST -H "Content-Type: application/json" https://$Server/redfish/v1/Managers/$ManagerID/VirtualMedia/$VmediaId -d '{"Image": "https://example.com/test.iso", "TransferProtocolType": "HTTPS", "UserName": "", "Password":""}'

$ curl -u $USER:$PASS -X PATCH -H "Content-Type: application/json" -H "If-Match: <ETAG>" https://$Server/redfish/v1/Managers/$ManagerID/VirtualMedia/$VmediaId -d '{"Image": "https://example.com/test.iso", "TransferProtocolType": "HTTPS", "UserName": "", "Password":""}'

apiVersion: v1
baseDomain: <domain>
proxy:
  httpProxy: http://USERNAME:PASSWORD@proxy.example.com:PORT
  httpsProxy: https://USERNAME:PASSWORD@proxy.example.com:PORT
  noProxy: <WILDCARD_OF_DOMAIN>,<PROVISIONING_NETWORK/CIDR>,<BMC_ADDRESS_RANGE/CIDR>

noProxy: .example.com,172.22.0.0/24,10.10.0.0/24

    hosts:
      - name: openshift-master-0
        role: master
        bmc:
          address: redfish+http://<out_of_band_ip>/redfish/v1/Systems/
          username: <user>
          password: <password>
          disableCertificateVerification: null
        bootMACAddress: <NIC1_mac_address>
        bootMode: UEFI
        rootDeviceHints:
          deviceName: "/dev/sda"
        networkConfig: 

          interfaces:
          - name: <nic1_name> 

            type: ethernet
            state: up
            ipv4:
              address:
              - ip: <ip_address> 

                prefix-length: 24
              enabled: true
          dns-resolver:
            config:
              server:
              - <dns_ip_address> 

          routes:
            config:
            - destination: 0.0.0.0/0
              next-hop-address: <next_hop_ip_address> 

              next-hop-interface: <next_hop_nic1_name> 

networking:
  machineNetwork:
  - cidr: 10.0.0.0/24
  - cidr: 192.168.0.0/24
  networkType: OVNKubernetes

networkConfig:
  interfaces:
  - name: <interface_name> 

    type: ethernet
    state: up
    ipv4:
      enabled: true
      dhcp: false
      address:
      - ip: <node_ip> 

        prefix-length: 24
      gateway: <gateway_ip> 

  dns-resolver:
    config:
      server:
      - <dns_ip> 

    hosts:
      - name: openshift-master-0
        role: master
        bmc:
          address: redfish+http://<out_of_band_ip>/redfish/v1/Systems/
          username: <user>
          password: <password>
          disableCertificateVerification: null
        bootMACAddress: <NIC1_mac_address>
        bootMode: UEFI
        rootDeviceHints:
          deviceName: "/dev/sda"
        networkConfig:
          interfaces:
          - name: eth0
            ipv6:
              addr-gen-mode: <address_mode> 

...

    hosts:
      - name: worker-0
        role: worker
        bmc:
          address: redfish+http://<out_of_band_ip>/redfish/v1/Systems/
          username: <user>
          password: <password>
          disableCertificateVerification: false
        bootMACAddress: <NIC1_mac_address>
        bootMode: UEFI
        networkConfig: 

          interfaces: 

           - name: eno1 

             type: ethernet 

             state: up
             mac-address: 0c:42:a1:55:f3:06
             ipv4:
               enabled: true
               dhcp: false 

             ethernet:
               sr-iov:
                 total-vfs: 2 

             ipv6:
               enabled: false
               dhcp: false
           - name: sriov:eno1:0
             type: ethernet
             state: up 

             ipv4:
               enabled: false 

             ipv6:
               enabled: false
           - name: sriov:eno1:1
             type: ethernet
             state: down
           - name: eno2
             type: ethernet
             state: up
             mac-address: 0c:42:a1:55:f3:07
             ipv4:
               enabled: true
             ethernet:
               sr-iov:
                 total-vfs: 2
             ipv6:
               enabled: false
           - name: sriov:eno2:0
             type: ethernet
             state: up
             ipv4:
               enabled: false
             ipv6:
               enabled: false
           - name: sriov:eno2:1
             type: ethernet
             state: down
           - name: bond0
             type: bond
             state: up
             min-tx-rate: 100 

             max-tx-rate: 200 

             link-aggregation:
               mode: active-backup 

               options:
                 primary: sriov:eno1:0 

               port:
                 - sriov:eno1:0
                 - sriov:eno2:0
             ipv4:
               address:
                 - ip: 10.19.16.57 

                   prefix-length: 23
               dhcp: false
               enabled: true
             ipv6:
               enabled: false
          dns-resolver:
            config:
              server:
                - 10.11.5.160
                - 10.2.70.215
          routes:
            config:
              - destination: 0.0.0.0/0
                next-hop-address: 10.19.17.254
                next-hop-interface: bond0 

                table-id: 254

$ ./openshift-baremetal-install --dir ~/clusterconfigs create manifests

INFO Consuming Install Config from target directory
WARNING Making control-plane schedulable by setting MastersSchedulable to true for Scheduler cluster settings
WARNING Discarding the OpenShift Manifest that was provided in the target directory because its dependencies are dirty and it needs to be regenerated

$ sudo dnf -y install butane

$ butane 99-master-chrony-conf-override.bu -o 99-master-chrony-conf-override.yaml

$ butane 99-worker-chrony-conf-override.bu -o 99-worker-chrony-conf-override.yaml

$ cd ~/clusterconfigs

$ cd manifests

$ touch cluster-network-avoid-workers-99-config.yaml

apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
  name: 50-worker-fix-ipi-rwn
  labels:
    machineconfiguration.openshift.io/role: worker
spec:
  config:
    ignition:
      version: 3.2.0
    storage:
      files:
        - path: /etc/kubernetes/manifests/keepalived.yaml
          mode: 0644
          contents:
            source: data:,

apiVersion: operator.openshift.io/v1
kind: IngressController
metadata:
  name: default
  namespace: openshift-ingress-operator
spec:
  nodePlacement:
    nodeSelector:
      matchLabels:
        node-role.kubernetes.io/master: ""

$ sed -i "s;mastersSchedulable: false;mastersSchedulable: true;g" clusterconfigs/manifests/cluster-scheduler-02-config.yml

apiVersion: operator.openshift.io/v1
kind: IngressController
metadata:
  name: default
  namespace: openshift-ingress-operator
spec:
  replicas: <num-of-router-pods>
  endpointPublishingStrategy:
    type: HostNetwork
  nodePlacement:
    nodeSelector:
      matchLabels:
        node-role.kubernetes.io/worker: ""

$ cp ~/router-replicas.yaml clusterconfigs/openshift/99_router-replicas.yaml

$ vim clusterconfigs/openshift/99_openshift-cluster-api_hosts-*.yaml

spec:
  firmware:
    simultaneousMultithreadingEnabled: true
    sriovEnabled: true
    virtualizationEnabled: true

$ vim clusterconfigs/openshift/99_openshift-cluster-api_hosts-*.yaml

apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
  labels:
    machineconfiguration.openshift.io/role: primary
  name: 10_primary_storage_config
spec:
  config:
    ignition:
      version: 3.2.0
    storage:
      disks:
        - device: </dev/xxyN>
          partitions:
            - label: recovery
              startMiB: 32768
              sizeMiB: 16384
      filesystems:
        - device: /dev/disk/by-partlabel/recovery
          label: recovery
          format: xfs

$ cp ~/<MachineConfig_manifest> ~/clusterconfigs/openshift

$ sudo firewall-cmd --add-port=5000/tcp --zone=libvirt  --permanent

$ sudo firewall-cmd --add-port=5000/tcp --zone=public   --permanent

$ sudo firewall-cmd --reload

$ sudo yum -y install python3 podman httpd httpd-tools jq

$ sudo mkdir -p /opt/registry/{auth,certs,data}

$ openshift-baremetal-install

$ echo "additionalTrustBundle: |" >> install-config.yaml

$ sed -e 's/^/  /' /opt/registry/certs/domain.crt >> install-config.yaml

$ echo "imageContentSources:" >> install-config.yaml

$ echo "- mirrors:" >> install-config.yaml

$ echo "  - registry.example.com:5000/ocp4/openshift4" >> install-config.yaml

$ echo "  source: quay.io/openshift-release-dev/ocp-release" >> install-config.yaml

$ echo "- mirrors:" >> install-config.yaml

$ echo "  - registry.example.com:5000/ocp4/openshift4" >> install-config.yaml

$ echo "  source: quay.io/openshift-release-dev/ocp-v4.0-art-dev" >> install-config.yaml