使用 Red Hat Advanced Cluster Management 的 multicluster engine operator

apiVersion: addon.open-cluster-management.io/v1alpha1
kind: AddOnDeploymentConfig
metadata:
  name: addon-ns-config
  namespace: multicluster-engine
spec:
  agentInstallNamespace: open-cluster-management-agent-addon-discovery

apiVersion: addon.open-cluster-management.io/v1alpha1
kind: ClusterManagementAddOn
metadata:
  name: work-manager
spec:
  addonMeta:
    displayName: work-manager
  installStrategy:
    placements:
    - name: global
      namespace: open-cluster-management-global-set
      rolloutStrategy:
        type: All
    type: Placements

apiVersion: addon.open-cluster-management.io/v1alpha1
kind: ClusterManagementAddOn
metadata:
  name: application-manager
spec:
  addOnMeta:
    displayName: application-manager
  installStrategy:
    placements:
    - name: global
      namespace: open-cluster-management-global-set
      rolloutStrategy:
        type: All
      configs:
      - group: addon.open-cluster-management.io
        name: addon-ns-config
        namespace: multicluster-engine
        resource: addondeploymentconfigs
    type: Placements

oc get deployment -n open-cluster-management-agent-addon-discovery

NAME                                                  READY   STATUS    RESTARTS   AGE
application-manager-6b7f74b8f7-7sd25                  1/1     Running   0          5d18h
cluster-proxy-proxy-agent-7985ddfdb6-kng5p            3/3     Running   0          5d18h
klusterlet-addon-workmgr-55fd575b4b-rs5vz             1/1     Running   0          5d18h
managed-serviceaccount-addon-agent-54bd989b94-g6gz9   1/1     Running   0          5d18h

kind: KlusterletConfig
apiVersion: config.open-cluster-management.io/v1alpha1
metadata:
  name: mce-import-klusterlet-config
spec:
  installMode:
    type: noOperator
    noOperator:
       postfix: mce-import

apiVersion: cluster.open-cluster-management.io/v1
kind: ManagedCluster
metadata:
  annotations:
    agent.open-cluster-management.io/klusterlet-config: mce-import-klusterlet-config 

  labels:
    cloud: auto-detect
    vendor: auto-detect
  name: mce-a 

spec:
  hubAcceptsClient: true
  leaseDurationSeconds: 60

oc get managedcluster

NAME           HUB ACCEPTED   MANAGED CLUSTER URLS            JOINED   AVAILABLE   AGE
local-cluster  true           https://<api.acm-hub.com:port>  True     True        44h
mce-a          true           https://<api.mce-a.com:port>    True     True        27s

oc patch addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine --type=merge -p '{"spec":{"agentInstallNamespace":"open-cluster-management-agent-addon-discovery"}}'

oc patch addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine --type=merge -p '{"spec":{"customizedVariables":[{"name":"disableMetrics","value": "true"},{"name":"disableHOManagement","value": "true"}]}}'

oc patch addondeploymentconfig hypershift-addon-deploy-config \
   -n multicluster-engine \
   --type=json \
   -p='[{"op":"add","path":"/spec/customizedVariables/-","value":{"name":"autoImportDisabled","value":"true"}}]'

clusteradm addon enable --names hypershift-addon --clusters <managed-cluster-names>

oc get managedcluster

oc get deployment -n open-cluster-management-agent-addon-discovery

NAME                                READY   UP-TO-DATE   AVAILABLE   AGE
cluster-proxy-proxy-agent           1/1     1            1           24h
klusterlet-addon-workmgr            1/1     1            1           24h
hypershift-addon-agent              1/1     1            1           24h
managed-serviceaccount-addon-agent  1/1     1            1           24h

apiVersion: policy.open-cluster-management.io/v1
kind: Policy
metadata:
  name: policy-mce-hcp-autoimport
  namespace: open-cluster-management-global-set
  annotations:
    policy.open-cluster-management.io/standards: NIST SP 800-53
    policy.open-cluster-management.io/categories: CM Configuration Management
    policy.open-cluster-management.io/controls: CM-2 Baseline Configuration
    policy.open-cluster-management.io/description: Discovered clusters that are of
      type MultiClusterEngineHCP can be automatically imported into ACM as managed clusters.
      This policy configure those discovered clusters so they are automatically imported.
      Fine tuning MultiClusterEngineHCP clusters to be automatically imported
      can be done by configure filters at the configMap or add annotation to the discoverd cluster.
spec:
  disabled: false
  policy-templates:
    - objectDefinition:
        apiVersion: policy.open-cluster-management.io/v1
        kind: ConfigurationPolicy
        metadata:
          name: mce-hcp-autoimport-config
        spec:
          object-templates:
            - complianceType: musthave
              objectDefinition:
                apiVersion: v1
                kind: ConfigMap
                metadata:
                  name: discovery-config
                  namespace: open-cluster-management-global-set
                data:
                  rosa-filter: ""
          remediationAction: enforce 

          severity: low
    - objectDefinition:
        apiVersion: policy.open-cluster-management.io/v1
        kind: ConfigurationPolicy
        metadata:
          name: policy-mce-hcp-autoimport
        spec:
          remediationAction: enforce
          severity: low
          object-templates-raw: |
            {{- /* find the MultiClusterEngineHCP DiscoveredClusters */ -}}
            {{- range $dc := (lookup "discovery.open-cluster-management.io/v1" "DiscoveredCluster" "" "").items }}
              {{- /* Check for the flag that indicates the import should be skipped */ -}}
              {{- $skip := "false" -}}
              {{- range $key, $value := $dc.metadata.annotations }}
                {{- if and (eq $key "discovery.open-cluster-management.io/previously-auto-imported")
                           (eq $value "true") }}
                  {{- $skip = "true" }}
                {{- end }}
              {{- end }}
              {{- /* if the type is MultiClusterEngineHCP and the status is Active */ -}}
              {{- if and (eq $dc.spec.status "Active")
                         (contains (fromConfigMap "open-cluster-management-global-set" "discovery-config" "mce-hcp-filter") $dc.spec.displayName)
                         (eq $dc.spec.type "MultiClusterEngineHCP")
                         (eq $skip "false") }}
            - complianceType: musthave
              objectDefinition:
                apiVersion: discovery.open-cluster-management.io/v1
                kind: DiscoveredCluster
                metadata:
                  name: {{ $dc.metadata.name }}
                  namespace: {{ $dc.metadata.namespace }}
                spec:
                  importAsManagedCluster: true 

              {{- end }}
            {{- end }}

apiVersion: cluster.open-cluster-management.io/v1beta1
kind: Placement
metadata:
  name: policy-mce-hcp-autoimport-placement
  namespace: open-cluster-management-global-set
spec:
  tolerations:
    - key: cluster.open-cluster-management.io/unreachable
      operator: Exists
    - key: cluster.open-cluster-management.io/unavailable
      operator: Exists
  clusterSets:
    - global
  predicates:
    - requiredClusterSelector:
        labelSelector:
          matchExpressions:
            - key: local-cluster
              operator: In
              values:
                - "true"

apiVersion: policy.open-cluster-management.io/v1
kind: PlacementBinding
metadata:
  name: policy-mce-hcp-autoimport-placement-binding
  namespace: open-cluster-management-global-set
placementRef:
  name: policy-mce-hcp-autoimport-placement
  apiGroup: cluster.open-cluster-management.io
  kind: Placement
subjects:
  - name: policy-mce-hcp-autoimport
    apiGroup: policy.open-cluster-management.io
    kind: Policy

oc get policies.policy.open-cluster-management.io policy-mce-hcp-autoimport -n <namespace>

apiVersion: policy.open-cluster-management.io/v1
kind: Policy
metadata:
  name: policy-rosa-autoimport
  annotations:
    policy.open-cluster-management.io/standards: NIST SP 800-53
    policy.open-cluster-management.io/categories: CM Configuration Management
    policy.open-cluster-management.io/controls: CM-2 Baseline Configuration
    policy.open-cluster-management.io/description: OpenShift Service on AWS discovered clusters can be automatically imported into
Red Hat Advanced Cluster Management as managed clusters with this policy. You can select and configure those managed clusters so you can import. Configure filters or add an annotation if you do not want all of your OpenShift Service on AWS clusters to be automatically imported.
spec:
  remediationAction: inform 

  disabled: false
  policy-templates:
    - objectDefinition:
        apiVersion: policy.open-cluster-management.io/v1
        kind: ConfigurationPolicy
        metadata:
          name: rosa-autoimport-config
        spec:
          object-templates:
            - complianceType: musthave
              objectDefinition:
                apiVersion: v1
                kind: ConfigMap
                metadata:
                  name: discovery-config
                  namespace: open-cluster-management-global-set
                data:
                  rosa-filter: "" 

          remediationAction: enforce
          severity: low
    - objectDefinition:
        apiVersion: policy.open-cluster-management.io/v1
        kind: ConfigurationPolicy
        metadata:
          name: policy-rosa-autoimport
        spec:
          remediationAction: enforce
          severity: low
          object-templates-raw: |
            {{- /* find the ROSA DiscoveredClusters */ -}}
            {{- range $dc := (lookup "discovery.open-cluster-management.io/v1" "DiscoveredCluster" "" "").items }}
              {{- /* Check for the flag that indicates the import should be skipped */ -}}
              {{- $skip := "false" -}}
              {{- range $key, $value := $dc.metadata.annotations }}
                {{- if and (eq $key "discovery.open-cluster-management.io/previously-auto-imported")
                           (eq $value "true") }}
                  {{- $skip = "true" }}
                {{- end }}
              {{- end }}
              {{- /* if the type is ROSA and the status is Active */ -}}
              {{- if and (eq $dc.spec.status "Active")
                         (contains (fromConfigMap "open-cluster-management-global-set" "discovery-config" "rosa-filter") $dc.spec.displayName)
                         (eq $dc.spec.type "ROSA")
                         (eq $skip "false") }}
            - complianceType: musthave
              objectDefinition:
                apiVersion: discovery.open-cluster-management.io/v1
                kind: DiscoveredCluster
                metadata:
                  name: {{ $dc.metadata.name }}
                  namespace: {{ $dc.metadata.namespace }}
                spec:
                  importAsManagedCluster: true
              {{- end }}
            {{- end }}
    - objectDefinition:
        apiVersion: policy.open-cluster-management.io/v1
        kind: ConfigurationPolicy
        metadata:
          name: policy-rosa-managedcluster-status
        spec:
          remediationAction: enforce
          severity: low
          object-templates-raw: |
            {{- /* Use the same DiscoveredCluster list to check ManagedCluster status */ -}}
            {{- range $dc := (lookup "discovery.open-cluster-management.io/v1" "DiscoveredCluster" "" "").items }}
              {{- /* Check for the flag that indicates the import should be skipped */ -}}
              {{- $skip := "false" -}}
              {{- range $key, $value := $dc.metadata.annotations }}
                {{- if and (eq $key "discovery.open-cluster-management.io/previously-auto-imported")
                           (eq $value "true") }}
                  {{- $skip = "true" }}
                {{- end }}
              {{- end }}
              {{- /* if the type is ROSA and the status is Active */ -}}
              {{- if and (eq $dc.spec.status "Active")
                         (contains (fromConfigMap "open-cluster-management-global-set" "discovery-config" "rosa-filter") $dc.spec.displayName)
                         (eq $dc.spec.type "ROSA")
                         (eq $skip "false") }}
            - complianceType: musthave
              objectDefinition:
                apiVersion: cluster.open-cluster-management.io/v1
                kind: ManagedCluster
                metadata:
                  name: {{ $dc.spec.displayName }}
                  namespace: {{ $dc.spec.displayName }}
                status:
                  conditions:
                    - type: ManagedClusterConditionAvailable
                      status: "True"
              {{- end }}
            {{- end }}

apiVersion: cluster.open-cluster-management.io/v1beta1
kind: Placement
metadata:
  name: placement-openshift-plus-hub
spec:
  predicates:
  - requiredClusterSelector:
      labelSelector:
        matchExpressions:
        - key: name
      	    operator: In
      	    values:
      	    - local-cluster

apiVersion: policy.open-cluster-management.io/v1
kind: PlacementBinding
metadata:
  name: binding-policy-rosa-autoimport
placementRef:
  apiGroup: cluster.open-cluster-management.io
  kind: Placement
  name: placement-policy-rosa-autoimport
subjects:
- apiGroup: policy.open-cluster-management.io
  kind: Policy
  name: policy-rosa-autoimport

oc get policies.policy.open-cluster-management.io policy-rosa-autoimport -n <namespace>

export MCH_NAMESPACE=<namespace>

oc patch multiclusterhubs.operator.open-cluster-management.io multiclusterhub -n ${MCH_NAMESPACE} --type json --patch '[{"op": "add", "path":"/spec/overrides/components/-", "value": {"name":"siteconfig","enabled": true}}]'

oc -n ${MCH_NAMESPACE} get po | grep siteconfig

siteconfig-controller-manager-6fdd86cc64-sdg87                    2/2     Running   0             43s

oc -n ${MCH_NAMESPACE} get cm

NAME                                DATA   AGE
ai-cluster-templates-v1             5      97s
ai-node-templates-v1                2      97s
...
ibi-cluster-templates-v1            3      97s
ibi-node-templates-v1               3      97s
...

apiVersion: v1
kind: Namespace
metadata:
  name: example-sno

oc apply -f clusterinstance-namespace.yaml

apiVersion: v1
kind: Secret
metadata:
  name: pull-secret
  namespace: example-sno 

data:
  .dockerconfigjson: <encoded_docker_configuration> 

type: kubernetes.io/dockerconfigjson

oc apply -f pull-secret.yaml

apiVersion: v1
data:
  password: <password>
  username: <username>
kind: Secret
metadata:
  name: example-bmh-secret
  namespace: "example-sno" 

type: Opaque

oc apply -f example-bmc-secret.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: enable-crun
  namespace: example-sno 

data:
  enable-crun-master.yaml: |
    apiVersion: machineconfiguration.openshift.io/v1
    kind: ContainerRuntimeConfig
    metadata:
      name: enable-crun-master
    spec:
      machineConfigPoolSelector:
        matchLabels:
          pools.operator.machineconfiguration.openshift.io/master: ""
      containerRuntimeConfig:
        defaultRuntime: crun
  enable-crun-worker.yaml: |
    apiVersion: machineconfiguration.openshift.io/v1
    kind: ContainerRuntimeConfig
    metadata:
      name: enable-crun-worker
    spec:
      machineConfigPoolSelector:
        matchLabels:
          pools.operator.machineconfiguration.openshift.io/worker: ""
      containerRuntimeConfig:
        defaultRuntime: crun

oc apply -f enable-crun.yaml

apiVersion: siteconfig.open-cluster-management.io/v1alpha1
kind: ClusterInstance
metadata:
  name: "example-clusterinstance"
  namespace: "example-sno" 

spec:
  #clusterType: "SNO" 

  holdInstallation: false
  extraManifestsRefs: 

    - name: extra-machine-configs
    - name: enable-crun
  pullSecretRef:
    name: "pull-secret" 

  [...]
  clusterName: "example-sno" 

  [...]
  clusterImageSetNameRef: "img4.17-x86-64"
  [...]
  reference: 

  [...]
    namespace: 

    [...]
  templateRefs: 

    - name: ibi-cluster-templates-v1
      namespace: rhacm
      hostRef: 

        - name: example-bmh
          namespace: example-sno
  [...]
  nodes:
      [...]
      bmcCredentialsName: 

        name: "example-bmh-secret"
      [...]
      templateRefs: 

        - name: ibi-node-templates-v1
          namespace: rhacm
      [...]

oc apply -f clusterinstance-ibi.yaml

oc get clusterinstance <cluster_name> -n <target_namespace> -o yaml

message: Applied site config manifests
reason: Completed
status: "True"
type: RenderedTemplatesApplied

oc get clusterinstance <cluster_name> -n <target_namespace> -o jsonpath='{.status.manifestsRendered}'

oc delete clusterinstance <cluster_name> -n <target_namespace>

oc get clusterinstance <cluster_name> -n <target_namespace>

Error from server (NotFound): clusterinstances.siteconfig.open-cluster-management.io "<cluster_name>" not found

apiVersion: v1
kind: ConfigMap
metadata:
  name: my-custom-secret
  namespace: rhacm
data:
  MySecret: |-
    apiVersion: v1
    kind: Secret
    metadata:
      name: "{{ .Spec.ClusterName }}-my-custom-secret-key"
      namespace: "clusters"
      annotations:
        siteconfig.open-cluster-management.io/sync-wave: "1" 

    type: Opaque
    data:
      key: <key>

oc apply -f my-custom-secret.yaml

spec:
    ...
  templateRefs:
    - name: ai-cluster-templates-v1.yaml
      namespace: rhacm
    - name: my-custom-secret.yaml
      namespace: rhacm
    ...

oc apply -f clusterinstance-my-custom-secret.yaml

kind: ConfigMap
apiVersion: v1
metadata:
  name: "idms-configmap"
  namespace: "example-sno"
data:
  99-example-idms.yaml: |
    apiVersion: config.openshift.io/v1
    kind: ImageDigestMirrorSet
    metadata:
      name: example-idms
    spec:
      imageDigestMirrors:
      - mirrors:
        - mirror.registry.example.com/image-repo/image
        source: registry.example.com/image-repo/image

oc apply -f idms-configmap.yaml

apiVersion: siteconfig.open-cluster-management.io/v1alpha1
kind: ClusterInstance
metadata:
  name: "example-sno"
  namespace: "example-sno"
spec:
  ...
  extraManifestsRefs:
    - name: idms-configmap
...