Red Hat Openstack Platform データプレーンの導入

apiVersion: core.openstack.org/v1beta1
kind: OpenStackControlPlane
metadata:
  name: openstack
spec:
  cinder:
    template:
      customServiceConfig: |
        [DEFAULT]
        debug = True
< . . . >

apiVersion: core.openstack.org/v1beta1
kind: OpenStackControlPlane
metadata:
  name: openstack
spec:
  cinder:
    template:
      cinderScheduler:
        customServiceConfig: |
          [DEFAULT]
          debug = True
< . . . >

apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfigPool
metadata:
  name: openstack
spec:
  machineConfigSelector:
    matchLabels:
      machineconfiguration.openshift.io/role: openstack
  nodeSelector:
    matchLabels:
      type: openstack

apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
  labels:
    machineconfiguration.openshift.io/role: openstack
< . . . >

CEPH_SSH="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100"
CEPH_KEY=$($CEPH_SSH "cat /etc/ceph/ceph.client.openstack.keyring | base64 -w 0")
CEPH_CONF=$($CEPH_SSH "cat /etc/ceph/ceph.conf | base64 -w 0")

$CEPH_SSH cephadm shell
ceph auth caps client.openstack \
  mgr 'allow *' \
  mon 'allow r, profile rbd' \
  osd 'profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images, allow rw pool manila_data'

oc apply -f - <<EOF
apiVersion: v1
data:
  ceph.client.openstack.keyring: $CEPH_KEY
  ceph.conf: $CEPH_CONF
kind: Secret
metadata:
  name: ceph-conf-files
  namespace: openstack
type: Opaque
EOF

oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  extraMounts:
    - name: v1
      region: r1
      extraVol:
        - propagation:
          - CinderVolume
          - CinderBackup
          - GlanceAPI
          - ManilaShare
          extraVolType: Ceph
          volumes:
          - name: ceph
            projected:
              sources:
              - secret:
                  name: ceph-conf-files
          mounts:
          - name: ceph
            mountPath: "/etc/ceph"
            readOnly: true
'

CEPH_FSID=$(oc get secret ceph-conf-files -o json | jq -r '.data."ceph.conf"' | base64 -d | grep fsid | sed -e 's/fsid = //')

CONTROLLER1_SSH="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100"
CONTROLLER2_SSH=""
CONTROLLER3_SSH=""

. stackrc ansible -i $(which tripleo-ansible-inventory) Controller -m shell -a "sudo systemctl stop tripleo_horizon.service" -b

openstack server list --all-projects -c ID -c Status |grep -E '\| .+ing \|'
openstack volume list --all-projects -c ID -c Status |grep -E '\| .+ing \|'| grep -vi error
openstack volume backup list --all-projects -c ID -c Status |grep -E '\| .+ing \|' | grep -vi error
openstack share list --all-projects -c ID -c Status |grep -E '\| .+ing \|'| grep -vi error
openstack image list -c ID -c Status |grep -E '\| .+ing \|'

# Update the services list to be stopped
ServicesToStop=("tripleo_horizon.service"
                "tripleo_keystone.service"
                "tripleo_cinder_api.service"
                "tripleo_cinder_api_cron.service"
                "tripleo_cinder_scheduler.service"
                "tripleo_cinder_backup.service"
                "tripleo_glance_api.service"
                "tripleo_manila_api.service"
                "tripleo_manila_api_cron.service"
                "tripleo_manila_scheduler.service"
                "tripleo_neutron_api.service"
                "tripleo_nova_api.service"
                "tripleo_placement_api.service"
                "tripleo_nova_api_cron.service"
                "tripleo_nova_api.service"
                "tripleo_nova_conductor.service"
                "tripleo_nova_metadata.service"
                "tripleo_nova_scheduler.service"
                "tripleo_nova_vnc_proxy.service"
                "tripleo_aodh_api.service"
                "tripleo_aodh_api_cron.service"
                "tripleo_aodh_evaluator.service"
                "tripleo_aodh_listener.service"
                "tripleo_aodh_notifier.service"
                "tripleo_ceilometer_agent_central.service"
                "tripleo_ceilometer_agent_compute.service"
                "tripleo_ceilometer_agent_ipmi.service"
                "tripleo_ceilometer_agent_notification.service"
                "tripleo_ovn_cluster_northd.service")

PacemakerResourcesToStop=("openstack-cinder-volume"
                          "openstack-cinder-backup"
                          "openstack-manila-share")

echo "Stopping systemd OpenStack services"
for service in ${ServicesToStop[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            echo "Stopping the $service in controller $i"
            if ${!SSH_CMD} sudo systemctl is-active $service; then
                ${!SSH_CMD} sudo systemctl stop $service
            fi
        fi
    done
done

echo "Checking systemd OpenStack services"
for service in ${ServicesToStop[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            echo "Checking status of $service in controller $i"
            if ! ${!SSH_CMD} systemctl show $service | grep ActiveState=inactive >/dev/null; then
                echo "ERROR: Service $service still running on controller $i"
            fi
        fi
    done
done

echo "Stopping pacemaker OpenStack services"
for i in {1..3}; do
    SSH_CMD=CONTROLLER${i}_SSH
    if [ ! -z "${!SSH_CMD}" ]; then
        echo "Using controller $i to run pacemaker commands"
        for resource in ${PacemakerResourcesToStop[*]}; do
            if ${!SSH_CMD} sudo pcs resource config $resource; then
                ${!SSH_CMD} sudo pcs resource disable $resource
            fi
        done
        break
    fi
done

PODIFIED_MARIADB_IP=$(oc get svc --selector "mariadb/name=openstack" -ojsonpath='{.items[0].spec.clusterIP}')
PODIFIED_CELL1_MARIADB_IP=$(oc get svc --selector "mariadb/name=openstack-cell1" -ojsonpath='{.items[0].spec.clusterIP}')
PODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d)

# The CHARACTER_SET and collation should match the source DB
# if the do not then it will break foreign key relationships
# for any tables that are created in the future as part of db sync
CHARACTER_SET=utf8
COLLATION=utf8_general_ci

MARIADB_IMAGE=registry.redhat.io/rhosp-dev-preview/openstack-mariadb-rhel9:18.0
# Replace with your environment's MariaDB Galera cluster VIP and backend IPs:
SOURCE_MARIADB_IP=192.168.122.99
declare -A SOURCE_GALERA_MEMBERS
SOURCE_GALERA_MEMBERS=(
  ["standalone.localdomain"]=192.168.122.100
  # ...
)
SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }')

STORAGE_CLASS_NAME=crc-csi-hostpath-provisioner
OVSDB_IMAGE=registry.redhat.io/rhosp-dev-preview/openstack-ovn-base-rhel9:18.0
SOURCE_OVSDB_IP=172.17.1.49

grep -rI 'ovn_[ns]b_conn' /var/lib/config-data/puppet-generated/

oc apply -f - <<EOF
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: ovn-data
spec:
  storageClassName: $STORAGE_CLASS_NAME
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
---
apiVersion: v1
kind: Pod
metadata:
  name: ovn-copy-data
  annotations:
    openshift.io/scc: anyuid
  labels:
    app: adoption
spec:
  containers:
  - image: $OVSDB_IMAGE
    command: [ "sh", "-c", "sleep infinity"]
    name: adoption
    volumeMounts:
    - mountPath: /backup
      name: ovn-data
  securityContext:
    allowPrivilegeEscalation: false
    capabilities:
      drop: ALL
    runAsNonRoot: true
    seccompProfile:
      type: RuntimeDefault
  volumes:
  - name: ovn-data
    persistentVolumeClaim:
      claimName: ovn-data
EOF

oc wait --for=condition=Ready pod/ovn-copy-data --timeout=30s

oc exec ovn-copy-data -- bash -c "ovsdb-client backup tcp:$SOURCE_OVSDB_IP:6641 > /backup/ovs-nb.db"
oc exec ovn-copy-data -- bash -c "ovsdb-client backup tcp:$SOURCE_OVSDB_IP:6642 > /backup/ovs-sb.db"

oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  ovn:
    enabled: true
    template:
      ovnDBCluster:
        ovndbcluster-nb:
          dbType: NB
          storageRequest: 10G
          networkAttachment: internalapi
        ovndbcluster-sb:
          dbType: SB
          storageRequest: 10G
          networkAttachment: internalapi
'

oc wait --for=jsonpath='{.status.phase}'=Running pod --selector=service=ovsdbserver-nb
oc wait --for=jsonpath='{.status.phase}'=Running pod --selector=service=ovsdbserver-sb

PODIFIED_OVSDB_NB_IP=$(oc get svc --selector "statefulset.kubernetes.io/pod-name=ovsdbserver-nb-0" -ojsonpath='{.items[0].spec.clusterIP}')
PODIFIED_OVSDB_SB_IP=$(oc get svc --selector "statefulset.kubernetes.io/pod-name=ovsdbserver-sb-0" -ojsonpath='{.items[0].spec.clusterIP}')

oc exec ovn-copy-data -- bash -c "ovsdb-client get-schema tcp:$PODIFIED_OVSDB_NB_IP:6641 > /backup/ovs-nb.ovsschema && ovsdb-tool convert /backup/ovs-nb.db /backup/ovs-nb.ovsschema"
oc exec ovn-copy-data -- bash -c "ovsdb-client get-schema tcp:$PODIFIED_OVSDB_SB_IP:6642 > /backup/ovs-sb.ovsschema && ovsdb-tool convert /backup/ovs-sb.db /backup/ovs-sb.ovsschema"

oc exec ovn-copy-data -- bash -c "ovsdb-client restore tcp:$PODIFIED_OVSDB_NB_IP:6641 < /backup/ovs-nb.db"
oc exec ovn-copy-data -- bash -c "ovsdb-client restore tcp:$PODIFIED_OVSDB_SB_IP:6642 < /backup/ovs-sb.db"

oc exec -it ovsdbserver-nb-0 -- ovn-nbctl show
oc exec -it ovsdbserver-sb-0 -- ovn-sbctl list Chassis

oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  ovn:
    enabled: true
    template:
      ovnNorthd:
        networkAttachment: internalapi
'

oc delete pod ovn-copy-data
oc delete pvc ovn-data

oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  neutron:
    enabled: true
    apiOverride:
      route: {}
    template:
      override:
        service:
          internal:
            metadata:
              annotations:
                metallb.universe.tf/address-pool: internalapi
                metallb.universe.tf/allow-shared-ip: internalapi
                metallb.universe.tf/loadBalancerIPs: 172.17.0.80
            spec:
              type: LoadBalancer
      databaseInstance: openstack
      secret: osp-secret
      networkAttachments:
      - internalapi
'

alias openstack="oc exec -t openstackclient -- openstack"

$CONTROLLER1_SSH cat /var/lib/config-data/puppet-generated/cinder/etc/cinder/cinder.conf > cinder.conf

apiVersion: core.openstack.org/v1beta1
kind: OpenStackControlPlane
metadata:
  name: openstack
spec:
  cinder:
    enabled: true
    template:
      cinderVolume:
        lvm:
          customServiceConfig: |
            [DEFAULT]
            debug = True
            [lvm]
< . . . >
        ceph:
          customServiceConfig: |
            [DEFAULT]
            debug = True
            [ceph]
< . . . >

   spec:
     cinder:
       enabled: true
       template:
         cinderAPI:
           customServiceConfigSecrets:
             - cinder-conf
         cinderScheduler:
           customServiceConfigSecrets:
             - cinder-conf
         cinderBackup:
           customServiceConfigSecrets:
             - cinder-conf
         cinderVolume:
           lvm1:
             customServiceConfig: |
               [DEFAULT]
               enabled_backends = lvm1
             customServiceConfigSecrets:
               - cinder-conf
           lvm2:
             customServiceConfig: |
               [DEFAULT]
               enabled_backends = lvm2
             customServiceConfigSecrets:
               - cinder-conf

oc patch openstackcontrolplane openstack --type=merge --patch-file=cinder.patch

spec:
  extraMounts:
  - extraVol:
    - extraVolType: Ceph
      mounts:
      - mountPath: /etc/ceph
        name: ceph
        readOnly: true
      propagation:
      - CinderVolume
      - CinderBackup
      - Glance
      volumes:
      - name: ceph
        projected:
          sources:
          - secret:
              name: ceph-conf-files
  cinder:
    enabled: true
    apiOverride:
      route: {}
    template:
      databaseInstance: openstack
      secret: osp-secret
      cinderAPI:
        override:
          service:
            internal:
              metadata:
                annotations:
                  metallb.universe.tf/address-pool: internalapi
                  metallb.universe.tf/allow-shared-ip: internalapi
                  metallb.universe.tf/loadBalancerIPs: 172.17.0.80
              spec:
                type: LoadBalancer
        replicas: 1
        customServiceConfig: |
          [DEFAULT]
          default_volume_type=tripleo
      cinderScheduler:
        replicas: 1
      cinderBackup:
        networkAttachments:
        - storage
        replicas: 1
        customServiceConfig: |
          [DEFAULT]
          backup_driver=cinder.backup.drivers.ceph.CephBackupDriver
          backup_ceph_conf=/etc/ceph/ceph.conf
          backup_ceph_user=openstack
          backup_ceph_pool=backups
      cinderVolumes:
        ceph:
          networkAttachments:
          - storage
          replicas: 1
          customServiceConfig: |
            [tripleo_ceph]
            backend_host=hostgroup
            volume_backend_name=tripleo_ceph
            volume_driver=cinder.volume.drivers.rbd.RBDDriver
            rbd_ceph_conf=/etc/ceph/ceph.conf
            rbd_user=openstack
            rbd_pool=volumes
            rbd_flatten_volume_from_snapshot=False
            report_discard_supported=True

openstack volume service list

+------------------+------------------------+------+---------+-------+----------------------------+
| Binary           | Host                   | Zone | Status  | State | Updated At                 |
+------------------+------------------------+------+---------+-------+----------------------------+
| cinder-backup    | standalone.localdomain | nova | enabled | down  | 2023-06-28T11:00:59.000000 |
| cinder-scheduler | standalone.localdomain | nova | enabled | down  | 2023-06-28T11:00:29.000000 |
| cinder-volume    | hostgroup@tripleo_ceph | nova | enabled | up    | 2023-06-28T17:00:03.000000 |
| cinder-scheduler | cinder-scheduler-0     | nova | enabled | up    | 2023-06-28T17:00:02.000000 |
| cinder-backup    | cinder-backup-0        | nova | enabled | up    | 2023-06-28T17:00:01.000000 |
+------------------+------------------------+------+---------+-------+----------------------------+

oc exec -it cinder-scheduler-0 -- cinder-manage service remove cinder-backup standalone.localdomain
oc exec -it cinder-scheduler-0 -- cinder-manage service remove cinder-scheduler standalone.localdomain

oc exec -it cinder-scheduler-0 -- cinder-manage db online_data_migrations

alias openstack="oc exec -t openstackclient -- openstack"

oc get horizon

PUBLIC_URL=$(oc get horizon horizon -o jsonpath='{.status.endpoint}')
curl --silent --output /dev/stderr --head --write-out "%{http_code}" "$PUBLIC_URL/dashboard/auth/login/?next=/dashboard/" | grep 200

[stack@rhosp17 ~]$ grep -E 'HeatPassword|HeatAuth' ~/overcloud-deploy/overcloud/overcloud-passwords.yaml
  HeatAuthEncryptionKey: Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2
  HeatPassword: dU2N0Vr2bdelYH7eQonAwPfI3

[stack@rhosp17 ~]$ ansible -i overcloud-deploy/overcloud/config-download/overcloud/tripleo-ansible-inventory.yaml overcloud-controller-0 -m shell -a "grep auth_encryption_key /var/lib/config-data/puppet-generated/heat/etc/heat/heat.conf | grep -Ev '^#|^$'" -b
overcloud-controller-0 | CHANGED | rc=0 >>
auth_encryption_key=Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2

❯ echo Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2 | base64
UTYwSGo4UHFickROdTJkRENieUlRRTJkaWJwUVVQZzIK

❯ oc patch secret osp-secret --type='json' -p='[{"op" : "replace" ,"path" : "/data/HeatAuthEncryptionKey" ,"value" : "UTYwSGo4UHFickROdTJkRENieUlRRTJkaWJwUVVQZzIK"}]'
secret/osp-secret patched

oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  heat:
    enabled: true
    apiOverride:
      route: {}
    template:
      databaseInstance: openstack
      secret: osp-secret
      memcachedInstance: memcached
      passwordSelectors:
        authEncryptionKey: HeatAuthEncryptionKey
        database: HeatDatabasePassword
        service: HeatPassword
'

❯ oc get Heat,HeatAPI,HeatEngine,HeatCFNAPI
NAME                           STATUS   MESSAGE
heat.heat.openstack.org/heat   True     Setup complete

NAME                                  STATUS   MESSAGE
heatapi.heat.openstack.org/heat-api   True     Setup complete

NAME                                        STATUS   MESSAGE
heatengine.heat.openstack.org/heat-engine   True     Setup complete

NAME                                        STATUS   MESSAGE
heatcfnapi.heat.openstack.org/heat-cfnapi   True     Setup complete

cat << EOF > ceilometer_patch.yaml
spec:
  ceilometer:
    enabled: true
    template:
      customServiceConfig: |
        [DEFAULT]
        debug=true
      secret: osp-secret
EOF

oc patch openstackcontrolplane openstack --type=merge --patch-file ceilometer_patch.yaml

cat << EOF > aodh_patch.yaml
spec:
  autoscaling:
    enabled: true
    prometheus:
      deployPrometheus: false
    aodh:
      customServiceConfig: |
        [DEFAULT]
        debug=true
      secret: osp-secret
      passwordSelectors:
      databaseUser: aodh
      databaseInstance: openstack
      memcachedInstance: memcached
EOF

oc patch openstackcontrolplane openstack --type=merge --patch-file aodh_patch.yaml

EDPM_PRIVATEKEY_PATH="~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa"
declare -A computes
computes=(
  ["standalone.localdomain"]="192.168.122.100"
  # ...
)

. stackrc
ansible -i $(which tripleo-ansible-inventory) Compute -m shell -a "sudo systemctl stop tripleo_virtqemud.service" -b

ComputeServicesToStop=(
                "tripleo_nova_compute.service"
                "tripleo_nova_libvirt.target"
                "tripleo_nova_migration_target.service"
                "tripleo_nova_virtlogd_wrapper.service"
                "tripleo_nova_virtnodedevd.service"
                "tripleo_nova_virtproxyd.service"
                "tripleo_nova_virtqemud.service"
                "tripleo_nova_virtsecretd.service"
                "tripleo_nova_virtstoraged.service")

PacemakerResourcesToStop=(
                "galera-bundle"
                "haproxy-bundle"
                "rabbitmq-bundle")

echo "Disabling systemd units and cleaning up for compute services"
for i in "${!computes[@]}"; do
    SSH_CMD="ssh -i $EDPM_PRIVATEKEY_PATH root@${computes[$i]}"
    for service in ${ComputeServicesToStop[*]}; do
        echo "Stopping the $service in compute $i"
        if ${SSH_CMD} sudo systemctl is-active $service; then
            ${SSH_CMD} sudo systemctl disable --now $service
            ${SSH_CMD} test -f /etc/systemd/system/$service '||' sudo systemctl mask $service
        fi
    done
done

echo "Stopping pacemaker services"
for i in {1..3}; do
    SSH_CMD=CONTROLLER${i}_SSH
    if [ ! -z "${!SSH_CMD}" ]; then
        echo "Using controller $i to run pacemaker commands"
        for resource in ${PacemakerResourcesToStop[*]}; do
            if ${!SSH_CMD} sudo pcs resource config $resource; then
                ${!SSH_CMD} sudo pcs resource disable $resource
            fi
        done
        break
    fi
done

PODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d)

alias openstack="oc exec -t openstackclient -- openstack"
FIP=192.168.122.20
declare -A computes
export computes=(
  ["standalone.localdomain"]="192.168.122.100"
  # ...
)

oc apply -f - <<EOF
apiVersion: network.openstack.org/v1beta1
kind: NetConfig
metadata:
  name: netconfig
spec:
  networks:
  - name: CtlPlane
    dnsDomain: ctlplane.example.com
    subnets:
    - name: subnet1
      allocationRanges:
      - end: 192.168.122.120
        start: 192.168.122.100
      - end: 192.168.122.200
        start: 192.168.122.150
      cidr: 192.168.122.0/24
      gateway: 192.168.122.1
  - name: InternalApi
    dnsDomain: internalapi.example.com
    subnets:
    - name: subnet1
      allocationRanges:
      - end: 172.17.0.250
        start: 172.17.0.100
      cidr: 172.17.0.0/24
      vlan: 20
  - name: External
    dnsDomain: external.example.com
    subnets:
    - name: subnet1
      allocationRanges:
      - end: 10.0.0.250
        start: 10.0.0.100
      cidr: 10.0.0.0/24
      gateway: 10.0.0.1
  - name: Storage
    dnsDomain: storage.example.com
    subnets:
    - name: subnet1
      allocationRanges:
      - end: 172.18.0.250
        start: 172.18.0.100
      cidr: 172.18.0.0/24
      vlan: 21
  - name: StorageMgmt
    dnsDomain: storagemgmt.example.com
    subnets:
    - name: subnet1
      allocationRanges:
      - end: 172.20.0.250
        start: 172.20.0.100
      cidr: 172.20.0.0/24
      vlan: 23
  - name: Tenant
    dnsDomain: tenant.example.com
    subnets:
    - name: subnet1
      allocationRanges:
      - end: 172.19.0.250
        start: 172.19.0.100
      cidr: 172.19.0.0/24
      vlan: 22
EOF

Failed to pull image "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0":
rpc error: code = Unknown desc = unable to retrieve auth token: invalid
username/password: unauthorized: Please login to the Red Hat Registry using
your Customer Portal credentials.

  Normal   Scheduled       3m40s                  default-scheduler  Successfully assigned openstack/rabbitmq-server-0 to worker0
  Normal   AddedInterface  3m38s                  multus             Add eth0 [10.101.0.41/23] from ovn-kubernetes
  Warning  Failed          2m16s (x6 over 3m38s)  kubelet            Error: ImagePullBackOff
  Normal   Pulling         2m5s (x4 over 3m38s)   kubelet            Pulling image "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0"
  Warning  Failed          2m5s (x4 over 3m38s)   kubelet            Failed to pull image "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0": rpc error: code  ... can be found here: https://access.redhat.com/RegistryAuthentication
  Warning  Failed          2m5s (x4 over 3m38s)   kubelet            Error: ErrImagePull
  Normal   BackOff         110s (x7 over 3m38s)   kubelet            Back-off pulling image "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0"

oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=<pull_secret_location.json>

kubectl delete pod rabbitmq-server-0 -n openstack

|    |                     |             |
|----|---------------------|-------------|
| osd | mon/mgr/crash      | rgw/ingress |
| osd | mon/mgr/crash      | rgw/ingress |
| osd | mon/mgr/crash      | rgw/ingress |

|     |                     |             |
|-----|---------------------|-------------|
| osd | mon/mgr/crash | rgw/ingress       |
| osd | mon/mgr/crash | rgw/ingress       |
| osd | mon/mgr/crash | dashboard/grafana |
| osd | rgw/ingress   | (free)            |

|     |                     |                         |
|-----|---------------------|-------------------------|
| osd | mon/mgr/crash       | rgw/ingress             |
| osd | mon/mgr/crash       | rgw/ingress             |
| osd | mon/mgr/crash       | mds/ganesha/ingress     |
| osd | rgw/ingress         | mds/ganesha/ingress     |
| osd | mds/ganesha/ingress | dashboard/grafana       |

(undercloud) [stack@undercloud-0 ~]$ metalsmith list


    +------------------------+    +----------------+
    | IP Addresses           |    |  Hostname      |
    +------------------------+    +----------------+
    | ctlplane=192.168.24.25 |    | cephstorage-0  |
    | ctlplane=192.168.24.10 |    | cephstorage-1  |
    | ctlplane=192.168.24.32 |    | cephstorage-2  |
    | ctlplane=192.168.24.28 |    | compute-0      |
    | ctlplane=192.168.24.26 |    | compute-1      |
    | ctlplane=192.168.24.43 |    | controller-0   |
    | ctlplane=192.168.24.7  |    | controller-1   |
    | ctlplane=192.168.24.41 |    | controller-2   |
    +------------------------+    +----------------+

Full List of Resources:
  * ip-192.168.24.46	(ocf:heartbeat:IPaddr2):     	Started controller-0
  * ip-10.0.0.103   	(ocf:heartbeat:IPaddr2):     	Started controller-1
  * ip-172.17.1.129 	(ocf:heartbeat:IPaddr2):     	Started controller-2
  * ip-172.17.3.68  	(ocf:heartbeat:IPaddr2):     	Started controller-0
  * ip-172.17.4.37  	(ocf:heartbeat:IPaddr2):     	Started controller-1
  * Container bundle set: haproxy-bundle

[undercloud-0.ctlplane.redhat.local:8787/rh-osbs/rhosp17-openstack-haproxy:pcmklatest]:
    * haproxy-bundle-podman-0   (ocf:heartbeat:podman):  Started controller-2
    * haproxy-bundle-podman-1   (ocf:heartbeat:podman):  Started controller-0
    * haproxy-bundle-podman-2   (ocf:heartbeat:podman):  Started controller-1

[heat-admin@controller-0 ~]$ ip -o -4 a

1: lo	inet 127.0.0.1/8 scope host lo\   	valid_lft forever preferred_lft forever
2: enp1s0	inet 192.168.24.45/24 brd 192.168.24.255 scope global enp1s0\   	valid_lft forever preferred_lft forever
2: enp1s0	inet 192.168.24.46/32 brd 192.168.24.255 scope global enp1s0\   	valid_lft forever preferred_lft forever
7: br-ex	inet 10.0.0.122/24 brd 10.0.0.255 scope global br-ex\   	valid_lft forever preferred_lft forever
8: vlan70	inet 172.17.5.22/24 brd 172.17.5.255 scope global vlan70\   	valid_lft forever preferred_lft forever
8: vlan70	inet 172.17.5.94/32 brd 172.17.5.255 scope global vlan70\   	valid_lft forever preferred_lft forever
9: vlan50	inet 172.17.2.140/24 brd 172.17.2.255 scope global vlan50\   	valid_lft forever preferred_lft forever
10: vlan30	inet 172.17.3.73/24 brd 172.17.3.255 scope global vlan30\   	valid_lft forever preferred_lft forever
10: vlan30	inet 172.17.3.68/32 brd 172.17.3.255 scope global vlan30\   	valid_lft forever preferred_lft forever
11: vlan20	inet 172.17.1.88/24 brd 172.17.1.255 scope global vlan20\   	valid_lft forever preferred_lft forever
12: vlan40	inet 172.17.4.24/24 brd 172.17.4.255 scope global vlan40\   	valid_lft forever preferred_lft forever

$ less /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg

...
...
listen ceph_rgw
  bind 10.0.0.103:8080 transparent
  bind 172.17.3.68:8080 transparent
  mode http
  balance leastconn
  http-request set-header X-Forwarded-Proto https if { ssl_fc }
  http-request set-header X-Forwarded-Proto http if !{ ssl_fc }
  http-request set-header X-Forwarded-Port %[dst_port]
  option httpchk GET /swift/healthcheck
  option httplog
  option forwardfor
  server controller-0.storage.redhat.local 172.17.3.73:8080 check fall 5 inter 2000 rise 2
  server controller-1.storage.redhat.local 172.17.3.146:8080 check fall 5 inter 2000 rise 2
  server controller-2.storage.redhat.local 172.17.3.156:8080 check fall 5 inter 2000 rise 2

[controller-0]$ ip -o -4 a

...
7: br-ex	inet 10.0.0.106/24 brd 10.0.0.255 scope global br-ex\   	valid_lft forever preferred_lft forever
...

---
network_config:
- type: interface
  name: nic1
  use_dhcp: false
  dns_servers: {{ ctlplane_dns_nameservers }}
  addresses:
  - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_subnet_cidr }}
  routes: {{ ctlplane_host_routes }}
- type: vlan
  vlan_id: {{ storage_mgmt_vlan_id }}
  device: nic1
  addresses:
  - ip_netmask: {{ storage_mgmt_ip }}/{{ storage_mgmt_cidr }}
  routes: {{ storage_mgmt_host_routes }}
- type: interface
  name: nic2
  use_dhcp: false
  defroute: false
- type: vlan
  vlan_id: {{ storage_vlan_id }}
  device: nic2
  addresses:
  - ip_netmask: {{ storage_ip }}/{{ storage_cidr }}
  routes: {{ storage_host_routes }}
- type: ovs_bridge
  name: {{ neutron_physical_bridge_name }}
  dns_servers: {{ ctlplane_dns_nameservers }}
  domain: {{ dns_search_domains }}
  use_dhcp: false
  addresses:
  - ip_netmask: {{ external_ip }}/{{ external_cidr }}
  routes: {{ external_host_routes }}
  members:
  - type: interface
    name: nic3
    primary: true

- name: CephStorage
  count: 3
  hostname_format: cephstorage-%index%
  instances:
  - hostname: cephstorage-0
  name: ceph-0
  - hostname: cephstorage-1
  name: ceph-1
  - hostname: cephstorage-2
  name: ceph-2
  defaults:
  profile: ceph-storage
  network_config:
      template: /home/stack/composable_roles/network/nic-configs/ceph-storage.j2
  networks:
  - network: ctlplane
      vif: true
  - network: storage
  - network: storage_mgmt
  - network: external

(undercloud) [stack@undercloud-0]$

openstack overcloud node provision
   -o overcloud-baremetal-deployed-0.yaml
   --stack overcloud
   --network-config -y
  $PWD/network/baremetal_deployment.yaml

[root@cephstorage-0 ~]# ip -o -4 a

1: lo	inet 127.0.0.1/8 scope host lo\   	valid_lft forever preferred_lft forever
2: enp1s0	inet 192.168.24.54/24 brd 192.168.24.255 scope global enp1s0\   	valid_lft forever preferred_lft forever
11: vlan40	inet 172.17.4.43/24 brd 172.17.4.255 scope global vlan40\   	valid_lft forever preferred_lft forever
12: vlan30	inet 172.17.3.23/24 brd 172.17.3.255 scope global vlan30\   	valid_lft forever preferred_lft forever
14: br-ex	inet 10.0.0.133/24 brd 10.0.0.255 scope global br-ex\   	valid_lft forever preferred_lft forever

for i in 0 1 2; {
    ceph orch host label add cephstorage-$i rgw;
}

[ceph: root@controller-0 /]#

for i in 0 1 2; {
    ceph orch host label add cephstorage-$i rgw;
}

Added label rgw to host cephstorage-0
Added label rgw to host cephstorage-1
Added label rgw to host cephstorage-2

[ceph: root@controller-0 /]# ceph orch host ls

HOST       	ADDR       	LABELS      	STATUS
cephstorage-0  192.168.24.54  osd rgw
cephstorage-1  192.168.24.44  osd rgw
cephstorage-2  192.168.24.30  osd rgw
controller-0   192.168.24.45  _admin mon mgr
controller-1   192.168.24.11  _admin mon mgr
controller-2   192.168.24.38  _admin mon mgr

6 hosts in cluster

[root@controller-0 heat-admin]# cat rgw

networks:
- 172.17.3.0/24
placement:
  hosts:
  - controller-0
  - controller-1
  - controller-2
service_id: rgw
service_name: rgw.rgw
service_type: rgw
spec:
  rgw_frontend_port: 8080
  rgw_realm: default
  rgw_zone: default

---
networks:
- 172.17.3.0/24
placement:
  label: rgw
service_id: rgw
service_name: rgw.rgw
service_type: rgw
spec:
  rgw_frontend_port: 8090
  rgw_realm: default
  rgw_zone: default

$ cephadm shell -m /home/ceph-admin/specs/rgw
$ cephadm shell -- ceph orch apply -i /mnt/rgw

...
osd.9                     	cephstorage-2
rgw.rgw.cephstorage-0.wsjlgx  cephstorage-0  172.17.3.23:8090   starting
rgw.rgw.cephstorage-1.qynkan  cephstorage-1  172.17.3.26:8090   starting
rgw.rgw.cephstorage-2.krycit  cephstorage-2  172.17.3.81:8090   starting
rgw.rgw.controller-1.eyvrzw   controller-1   172.17.3.146:8080  running (5h)
rgw.rgw.controller-2.navbxa   controller-2   172.17.3.66:8080   running (5h)

...
osd.9                     	cephstorage-2
rgw.rgw.cephstorage-0.wsjlgx  cephstorage-0  172.17.3.23:8090  running (19s)
rgw.rgw.cephstorage-1.qynkan  cephstorage-1  172.17.3.26:8090  running (16s)
rgw.rgw.cephstorage-2.krycit  cephstorage-2  172.17.3.81:8090  running (13s)

iptables -I INPUT -p tcp -m tcp --dport 8080 -m conntrack --ctstate NEW -m comment --comment "ceph rgw ingress" -j ACCEPT

iptables -I INPUT -p tcp -m tcp --dport 8090 -m conntrack --ctstate NEW -m comment --comment "ceph rgw backends" -j ACCEPT

for port in 8080 8090; {
    for i in 25 10 32; {
       ssh heat-admin@192.168.24.$i sudo iptables -I INPUT \
       -p tcp -m tcp --dport $port -m conntrack --ctstate NEW \
       -j ACCEPT;
   }
}

for i in 26 23 81; do {
    echo "---"
    curl 172.17.3.$i:8090;
    echo "---"
    echo
done

---
Query 172.17.3.23
<?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>
---

---
Query 172.17.3.26
<?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>
---

---
Query 172.17.3.81
<?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>
---

listen ceph_rgw
  bind 10.0.0.103:8080 transparent
  mode http
  balance leastconn
  http-request set-header X-Forwarded-Proto https if { ssl_fc }
  http-request set-header X-Forwarded-Proto http if !{ ssl_fc }
  http-request set-header X-Forwarded-Port %[dst_port]
  option httpchk GET /swift/healthcheck
  option httplog
  option forwardfor
   server controller-0.storage.redhat.local 172.17.3.73:8080 check fall 5 inter 2000 rise 2
  server controller-1.storage.redhat.local 172.17.3.146:8080 check fall 5 inter 2000 rise 2
  server controller-2.storage.redhat.local 172.17.3.156:8080 check fall 5 inter 2000 rise 2

[root@controller-0 ~]# sudo pcs resource restart haproxy-bundle
haproxy-bundle successfully restarted


[root@controller-0 ~]# sudo pcs status | grep haproxy

  * Container bundle set: haproxy-bundle [undercloud-0.ctlplane.redhat.local:8787/rh-osbs/rhosp17-openstack-haproxy:pcmklatest]:
    * haproxy-bundle-podman-0   (ocf:heartbeat:podman):  Started controller-0
    * haproxy-bundle-podman-1   (ocf:heartbeat:podman):  Started controller-1
    * haproxy-bundle-podman-2   (ocf:heartbeat:podman):  Started controller-2

[root@controller-0 ~]# ss -antop | grep 8080
[root@controller-0 ~]#

(overcloud) [root@cephstorage-0 ~]# swift list

HTTPConnectionPool(host='10.0.0.103', port=8080): Max retries exceeded with url: /swift/v1/AUTH_852f24425bb54fa896476af48cbe35d3?format=json (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fc41beb0430>: Failed to establish a new connection: [Errno 111] Connection refused'))

[ceph: root@controller-0 /]# ceph config set mgr mgr/cephadm/container_image_haproxy registry.redhat.io/rhceph/rhceph-haproxy-rhel9:latest
[ceph: root@controller-0 /]# ceph config set mgr mgr/cephadm/container_image_keepalived registry.redhat.io/rhceph/keepalived-rhel9:latest

$ sudo vim /home/ceph-admin/specs/rgw_ingress

---
service_type: ingress
service_id: rgw.rgw
placement:
  label: rgw
spec:
  backend_service: rgw.rgw
  virtual_ip: 10.0.0.89/24
  frontend_port: 8080
  monitor_port: 8898
  virtual_interface_networks:
    - 10.0.0.0/24

$ cephadm shell -m /home/ceph-admin/specs/rgw_ingress
$ cephadm shell -- ceph orch apply -i /mnt/rgw_ingress

[ceph: root@controller-0 /]# ceph orch ls

NAME                 	PORTS            	RUNNING  REFRESHED  AGE  PLACEMENT
crash                                         	6/6  6m ago 	3d   *
ingress.rgw.rgw      	10.0.0.89:8080,8898  	6/6  37s ago	60s  label:rgw
mds.mds                   3/3  6m ago 	3d   controller-0;controller-1;controller-2
mgr                       3/3  6m ago 	3d   controller-0;controller-1;controller-2
mon                       3/3  6m ago 	3d   controller-0;controller-1;controller-2
osd.default_drive_group   15  37s ago	3d   cephstorage-0;cephstorage-1;cephstorage-2
rgw.rgw   ?:8090          3/3  37s ago	4m   label:rgw

[ceph: root@controller-0 /]# curl  10.0.0.89:8080

---
<?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>[ceph: root@controller-0 /]#
—

(overcloud) [stack@undercloud-0 ~]$ openstack endpoint list | grep object

| 1326241fb6b6494282a86768311f48d1 | regionOne | swift    	| object-store   | True	| internal  | http://172.17.3.68:8080/swift/v1/AUTH_%(project_id)s |
| 8a34817a9d3443e2af55e108d63bb02b | regionOne | swift    	| object-store   | True	| public	| http://10.0.0.103:8080/swift/v1/AUTH_%(project_id)s  |
| fa72f8b8b24e448a8d4d1caaeaa7ac58 | regionOne | swift    	| object-store   | True	| admin 	| http://172.17.3.68:8080/swift/v1/AUTH_%(project_id)s |

(overcloud) [stack@undercloud-0 ~]$ openstack endpoint set --url "http://10.0.0.89:8080/swift/v1/AUTH_%(project_id)s" 95596a2d92c74c15b83325a11a4f07a3

(overcloud) [stack@undercloud-0 ~]$ openstack endpoint list | grep object-store
| 6c7244cc8928448d88ebfad864fdd5ca | regionOne | swift    	| object-store   | True	| internal  | http://172.17.3.79:8080/swift/v1/AUTH_%(project_id)s |
| 95596a2d92c74c15b83325a11a4f07a3 | regionOne | swift    	| object-store   | True	| public	| http://10.0.0.89:8080/swift/v1/AUTH_%(project_id)s   |
| e6d0599c5bf24a0fb1ddf6ecac00de2d | regionOne | swift    	| object-store   | True	| admin 	| http://172.17.3.79:8080/swift/v1/AUTH_%(project_id)s |

(overcloud) [stack@undercloud-0 ~]$ swift list --debug

DEBUG:swiftclient:Versionless auth_url - using http://10.0.0.115:5000/v3 as endpoint
DEBUG:keystoneclient.auth.identity.v3.base:Making authentication request to http://10.0.0.115:5000/v3/auth/tokens
DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): 10.0.0.115:5000
DEBUG:urllib3.connectionpool:http://10.0.0.115:5000 "POST /v3/auth/tokens HTTP/1.1" 201 7795
DEBUG:keystoneclient.auth.identity.v3.base:{"token": {"methods": ["password"], "user": {"domain": {"id": "default", "name": "Default"}, "id": "6f87c7ffdddf463bbc633980cfd02bb3", "name": "admin", "password_expires_at": null},


...
...
...

DEBUG:swiftclient:REQ: curl -i http://10.0.0.89:8080/swift/v1/AUTH_852f24425bb54fa896476af48cbe35d3?format=json -X GET -H "X-Auth-Token: gAAAAABj7KHdjZ95syP4c8v5a2zfXckPwxFQZYg0pgWR42JnUs83CcKhYGY6PFNF5Cg5g2WuiYwMIXHm8xftyWf08zwTycJLLMeEwoxLkcByXPZr7kT92ApT-36wTfpi-zbYXd1tI5R00xtAzDjO3RH1kmeLXDgIQEVp0jMRAxoVH4zb-DVHUos" -H "Accept-Encoding: gzip"
DEBUG:swiftclient:RESP STATUS: 200 OK
DEBUG:swiftclient:RESP HEADERS: {'content-length': '2', 'x-timestamp': '1676452317.72866', 'x-account-container-count': '0', 'x-account-object-count': '0', 'x-account-bytes-used': '0', 'x-account-bytes-used-actual': '0', 'x-account-storage-policy-default-placement-container-count': '0', 'x-account-storage-policy-default-placement-object-count': '0', 'x-account-storage-policy-default-placement-bytes-used': '0', 'x-account-storage-policy-default-placement-bytes-used-actual': '0', 'x-trans-id': 'tx00000765c4b04f1130018-0063eca1dd-1dcba-default', 'x-openstack-request-id': 'tx00000765c4b04f1130018-0063eca1dd-1dcba-default', 'accept-ranges': 'bytes', 'content-type': 'application/json; charset=utf-8', 'date': 'Wed, 15 Feb 2023 09:11:57 GMT'}
DEBUG:swiftclient:RESP BODY: b'[]'

(overcloud) [stack@undercloud-0 tempest-dir]$  tempest run --regex tempest.api.object_storage
...
...
...
======
Totals
======
Ran: 141 tests in 606.5579 sec.
 - Passed: 128
 - Skipped: 13
 - Expected Fail: 0
 - Unexpected Success: 0
 - Failed: 0
Sum of execute time for each test: 657.5183 sec.

==============
Worker Balance
==============
 - Worker 0 (1 tests) => 0:10:03.400561
 - Worker 1 (2 tests) => 0:00:24.531916
 - Worker 2 (4 tests) => 0:00:10.249889
 - Worker 3 (30 tests) => 0:00:32.730095
 - Worker 4 (51 tests) => 0:00:26.246044
 - Worker 5 (6 tests) => 0:00:20.114803
 - Worker 6 (20 tests) => 0:00:16.290323
 - Worker 7 (27 tests) => 0:00:17.103827