1.12. トラブルシューティング

<your-directory>/cluster-scoped-resources/gather-managed.log>

<your-directory>/cluster-scoped-resources/gather-managed.log>

"bmac.agent-install.openshift.io/installer-args": "[\"--append-karg\", \"coreos.force_persist_ip\"]"

"bmac.agent-install.openshift.io/installer-args": "[\"--append-karg\", \"coreos.force_persist_ip\"]"

oc get machine -n <hosted_cluster_namespace>

oc get machine -n <hosted_cluster_namespace>

NAMESPACE           NAME             CLUSTER          NODENAME   PROVIDERID   PHASE      AGE   VERSION
clusters-hosted-0   hosted-0-9gg8b   hosted-0-nhdbp                           Deleting   10h   <4.x.0>-rc.8

NAMESPACE           NAME             CLUSTER          NODENAME   PROVIDERID   PHASE      AGE   VERSION
clusters-hosted-0   hosted-0-9gg8b   hosted-0-nhdbp                           Deleting   10h   <4.x.0>-rc.8

oc edit machines -n <hosted_cluster_namespace>

oc edit machines -n <hosted_cluster_namespace>

oc get agentmachine -n <hosted_cluster_namespace>

oc get agentmachine -n <hosted_cluster_namespace>

hcp destroy cluster agent --name <cluster_name>

hcp destroy cluster agent --name <cluster_name>

MCE_NAMESPACE=<namespace>
oc delete multiclusterengine --all
oc delete apiservice v1.admission.cluster.open-cluster-management.io v1.admission.work.open-cluster-management.io
oc delete crd discoveredclusters.discovery.open-cluster-management.io discoveryconfigs.discovery.open-cluster-management.io
oc delete mutatingwebhookconfiguration ocm-mutating-webhook managedclustermutators.admission.cluster.open-cluster-management.io
oc delete validatingwebhookconfiguration ocm-validating-webhook
oc delete ns $MCE_NAMESPACE

MCE_NAMESPACE=<namespace>
oc delete multiclusterengine --all
oc delete apiservice v1.admission.cluster.open-cluster-management.io v1.admission.work.open-cluster-management.io
oc delete crd discoveredclusters.discovery.open-cluster-management.io discoveryconfigs.discovery.open-cluster-management.io
oc delete mutatingwebhookconfiguration ocm-mutating-webhook managedclustermutators.admission.cluster.open-cluster-management.io
oc delete validatingwebhookconfiguration ocm-validating-webhook
oc delete ns $MCE_NAMESPACE

kubectl -n multicluster-engine get pods -l app=managedcluster-import-controller-v2

kubectl -n multicluster-engine get pods -l app=managedcluster-import-controller-v2

kubectl -n multicluster-engine logs -l app=managedcluster-import-controller-v2 --tail=-1

kubectl -n multicluster-engine logs -l app=managedcluster-import-controller-v2 --tail=-1

kubectl -n <managed_cluster_name> get secrets <managed_cluster_name>-import

kubectl -n <managed_cluster_name> get secrets <managed_cluster_name>-import

kubectl -n multicluster-engine logs -l app=managedcluster-import-controller-v2 --tail=-1 | grep importconfig-controller

kubectl -n multicluster-engine logs -l app=managedcluster-import-controller-v2 --tail=-1 | grep importconfig-controller

kubectl get managedcluster <managed_cluster_name> -o=jsonpath='{range .status.conditions[*]}{.type}{"\t"}{.status}{"\t"}{.message}{"\n"}{end}' | grep ManagedClusterImportSucceeded

kubectl get managedcluster <managed_cluster_name> -o=jsonpath='{range .status.conditions[*]}{.type}{"\t"}{.status}{"\t"}{.message}{"\n"}{end}' | grep ManagedClusterImportSucceeded

cluster_name=<your-managed-cluster-name>

cluster_name=<your-managed-cluster-name>

kubeconfig_secret_name=$(oc -n ${cluster_name} get clusterdeployments ${cluster_name} -ojsonpath='{.spec.clusterMetadata.adminKubeconfigSecretRef.name}')

kubeconfig_secret_name=$(oc -n ${cluster_name} get clusterdeployments ${cluster_name} -ojsonpath='{.spec.clusterMetadata.adminKubeconfigSecretRef.name}')

oc -n ${cluster_name} get secret ${kubeconfig_secret_name} -ojsonpath={.data.kubeconfig} | base64 -d > kubeconfig.old

oc -n ${cluster_name} get secret ${kubeconfig_secret_name} -ojsonpath={.data.kubeconfig} | base64 -d > kubeconfig.old

export KUBECONFIG=kubeconfig.old

export KUBECONFIG=kubeconfig.old

oc get ns

oc get ns

cluster_name=<managed_cluster_name>
kubeconfig_file=<path_to_kubeconfig>

cluster_name=<managed_cluster_name>
kubeconfig_file=<path_to_kubeconfig>

kubeconfig=$(cat ${kubeconfig_file} | base64 -w0)

kubeconfig=$(cat ${kubeconfig_file} | base64 -w0)

kubeconfig=$(cat ${kubeconfig_file} | base64)

kubeconfig=$(cat ${kubeconfig_file} | base64)

kubeconfig_patch="[\{\"op\":\"replace\", \"path\":\"/data/kubeconfig\", \"value\":\"${kubeconfig}\"}, \{\"op\":\"replace\", \"path\":\"/data/raw-kubeconfig\", \"value\":\"${kubeconfig}\"}]"

kubeconfig_patch="[\{\"op\":\"replace\", \"path\":\"/data/kubeconfig\", \"value\":\"${kubeconfig}\"}, \{\"op\":\"replace\", \"path\":\"/data/raw-kubeconfig\", \"value\":\"${kubeconfig}\"}]"

kubeconfig_secret_name=$(oc -n ${cluster_name} get clusterdeployments ${cluster_name} -ojsonpath='{.spec.clusterMetadata.adminKubeconfigSecretRef.name}')

kubeconfig_secret_name=$(oc -n ${cluster_name} get clusterdeployments ${cluster_name} -ojsonpath='{.spec.clusterMetadata.adminKubeconfigSecretRef.name}')

oc -n ${cluster_name} patch secrets ${kubeconfig_secret_name} --type='json' -p="${kubeconfig_patch}"

oc -n ${cluster_name} patch secrets ${kubeconfig_secret_name} --type='json' -p="${kubeconfig_patch}"

kubectl get pod -n open-cluster-management-agent | grep klusterlet-agent

kubectl get pod -n open-cluster-management-agent | grep klusterlet-agent

kubectl logs <klusterlet_agent_pod> -n open-cluster-management-agent

kubectl logs <klusterlet_agent_pod> -n open-cluster-management-agent

oc get infrastructure cluster -o yaml | grep apiServerURL

oc get infrastructure cluster -o yaml | grep apiServerURL

oc delete secret -n <cluster_name> <cluster_name>-import

oc delete secret -n <cluster_name> <cluster_name>-import

oc get secret -n <cluster_name> <cluster_name>-import -ojsonpath='{.data.import\.yaml}' | base64 --decode  > import.yaml

oc get secret -n <cluster_name> <cluster_name>-import -ojsonpath='{.data.import\.yaml}' | base64 --decode  > import.yaml

oc apply -f import.yaml

oc apply -f import.yaml

oc edit managedcluster <cluster-name>

oc edit managedcluster <cluster-name>

kubectl get klusterlets klusterlet -oyaml

kubectl get klusterlets klusterlet -oyaml

oc api-resources --verbs=list --namespaced -o name | grep -E '^secrets|^serviceaccounts|^managedclusteraddons|^roles|^rolebindings|^manifestworks|^leases|^managedclusterinfo|^appliedmanifestworks'|^clusteroauths' | xargs -n 1 oc get --show-kind --ignore-not-found -n <cluster_name>

oc api-resources --verbs=list --namespaced -o name | grep -E '^secrets|^serviceaccounts|^managedclusteraddons|^roles|^rolebindings|^manifestworks|^leases|^managedclusterinfo|^appliedmanifestworks'|^clusteroauths' | xargs -n 1 oc get --show-kind --ignore-not-found -n <cluster_name>

oc edit <resource_kind> <resource_name> -n <namespace>

oc edit <resource_kind> <resource_name> -n <namespace>

oc delete ns <cluster-name>

oc delete ns <cluster-name>

oc delete secret auto-import-secret -n <cluster-namespace>

oc delete secret auto-import-secret -n <cluster-namespace>

kubectl describe placement <placement-name>

kubectl describe placement <placement-name>

Name:         demo-placement
Namespace:    default
Labels:       <none>
Annotations:  <none>
API Version:  cluster.open-cluster-management.io/v1beta1
Kind:         Placement
Status:
  Conditions:
    Last Transition Time:       2022-09-30T07:39:45Z
    Message:                    Placement configurations check pass
    Reason:                     Succeedconfigured
    Status:                     False
    Type:                       PlacementMisconfigured
    Last Transition Time:       2022-09-30T07:39:45Z
    Message:                    No valid ManagedClusterSetBindings found in placement namespace
    Reason:                     NoManagedClusterSetBindings
    Status:                     False
    Type:                       PlacementSatisfied
  Number Of Selected Clusters:  0

Name:         demo-placement
Namespace:    default
Labels:       <none>
Annotations:  <none>
API Version:  cluster.open-cluster-management.io/v1beta1
Kind:         Placement
Status:
  Conditions:
    Last Transition Time:       2022-09-30T07:39:45Z
    Message:                    Placement configurations check pass
    Reason:                     Succeedconfigured
    Status:                     False
    Type:                       PlacementMisconfigured
    Last Transition Time:       2022-09-30T07:39:45Z
    Message:                    No valid ManagedClusterSetBindings found in placement namespace
    Reason:                     NoManagedClusterSetBindings
    Status:                     False
    Type:                       PlacementSatisfied
  Number Of Selected Clusters:  0

Name:         demo-placement
Namespace:    default
Labels:       <none>
Annotations:  <none>
API Version:  cluster.open-cluster-management.io/v1beta1
Kind:         Placement
Events:
  Type    Reason          Age   From                 Message
  ----    ------          ----  ----                 -------
  Normal  DecisionCreate  2m10s   placementController  Decision demo-placement-decision-1 is created with placement demo-placement in namespace default
  Normal  DecisionUpdate  2m10s   placementController  Decision demo-placement-decision-1 is updated with placement demo-placement in namespace default
  Normal  ScoreUpdate     2m10s   placementController  cluster1:0 cluster2:100 cluster3:200
  Normal  DecisionUpdate  3s      placementController  Decision demo-placement-decision-1 is updated with placement demo-placement in namespace default
  Normal  ScoreUpdate     3s      placementController  cluster1:200 cluster2:145 cluster3:189 cluster4:200

Name:         demo-placement
Namespace:    default
Labels:       <none>
Annotations:  <none>
API Version:  cluster.open-cluster-management.io/v1beta1
Kind:         Placement
Events:
  Type    Reason          Age   From                 Message
  ----    ------          ----  ----                 -------
  Normal  DecisionCreate  2m10s   placementController  Decision demo-placement-decision-1 is created with placement demo-placement in namespace default
  Normal  DecisionUpdate  2m10s   placementController  Decision demo-placement-decision-1 is updated with placement demo-placement in namespace default
  Normal  ScoreUpdate     2m10s   placementController  cluster1:0 cluster2:100 cluster3:200
  Normal  DecisionUpdate  3s      placementController  Decision demo-placement-decision-1 is updated with placement demo-placement in namespace default
  Normal  ScoreUpdate     3s      placementController  cluster1:200 cluster2:145 cluster3:189 cluster4:200

oc get imagecontentsourcepolicy -o json | jq -r '.items[].spec.repositoryDigestMirrors[0].mirrors[0]'

oc get imagecontentsourcepolicy -o json | jq -r '.items[].spec.repositoryDigestMirrors[0].mirrors[0]'

oc get clusterversion version -ojsonpath='{.status.desired.image}'

oc get clusterversion version -ojsonpath='{.status.desired.image}'

oc image extract --file /release-manifests/0000_50_installer_coreos-bootimages.yaml <payload_image> --confirm

oc image extract --file /release-manifests/0000_50_installer_coreos-bootimages.yaml <payload_image> --confirm

cat 0000_50_installer_coreos-bootimages.yaml | yq -r .data.stream | jq -r '.architectures.x86_64.images.kubevirt."digest-ref"'

cat 0000_50_installer_coreos-bootimages.yaml | yq -r .data.stream | jq -r '.architectures.x86_64.images.kubevirt."digest-ref"'

oc image mirror <rhcos_image> <internal_registry>

oc image mirror <rhcos_image> <internal_registry>

apiVersion: config.openshift.io/v1
kind: ImageDigestMirrorSet
metadata:
  name: rhcos-boot-kubevirt
spec:
  repositoryDigestMirrors:
    - mirrors:
        - <rhcos_image_no_digest> 
      source: virthost.ostest.test.metalkube.org:5000/localimages/ocp-v4.0-art-dev 

apiVersion: config.openshift.io/v1
kind: ImageDigestMirrorSet
metadata:
  name: rhcos-boot-kubevirt
spec:
  repositoryDigestMirrors:
    - mirrors:
        - <rhcos_image_no_digest> 

      source: virthost.ostest.test.metalkube.org:5000/localimages/ocp-v4.0-art-dev 

oc apply -f rhcos-boot-kubevirt.yaml

oc apply -f rhcos-boot-kubevirt.yaml

apiVersion: config.open-cluster-management.io/v1alpha1
kind: KlusterletConfig
metadata:
  name: global
spec:
  hubKubeAPIServerConfig:
    serverVerificationStrategy: UseSystemTruststore

apiVersion: config.open-cluster-management.io/v1alpha1
kind: KlusterletConfig
metadata:
  name: global
spec:
  hubKubeAPIServerConfig:
    serverVerificationStrategy: UseSystemTruststore

oc apply -f <filename>

oc apply -f <filename>

oc get secret <cluster_name>-import -n <cluster_name> -o jsonpath={.data.import\.yaml} | base64 --decode > <cluster_name>-import.yaml

oc get secret <cluster_name>-import -n <cluster_name> -o jsonpath={.data.import\.yaml} | base64 --decode > <cluster_name>-import.yaml

oc apply -f <cluster_name>-import.yaml

oc apply -f <cluster_name>-import.yaml

oc get clusterversion version -o jsonpath='{.status.availableUpdates[*].version}'

oc get clusterversion version -o jsonpath='{.status.availableUpdates[*].version}'

oc -n <cluster_name> get managedclusterinfo <cluster_name> -o jsonpath='{.status.distributionInfo.ocp.availableUpdates[*]}'

oc -n <cluster_name> get managedclusterinfo <cluster_name> -o jsonpath='{.status.distributionInfo.ocp.availableUpdates[*]}'

oc -n <cluster_name> get ClusterCurator <cluster_name> -o yaml

oc -n <cluster_name> get ClusterCurator <cluster_name> -o yaml

oc -n <cluster_name> delete ClusterCurator <cluster_name>

oc -n <cluster_name> delete ClusterCurator <cluster_name>

oc -n open-cluster-management-agent-addon logs klusterlet-addon-workmgr-<your_pod_name>

oc -n open-cluster-management-agent-addon logs klusterlet-addon-workmgr-<your_pod_name>

export CA=<your_customized_CA>

export CA=<your_customized_CA>

oc create secret generic additional-ca \
  --from-literal=ca.crt="$CA" \
  --namespace hive

oc create secret generic additional-ca \
  --from-literal=ca.crt="$CA" \
  --namespace hive

oc patch hiveconfig hive --type=merge -p '
{
  "spec": {
    "additionalCertificateAuthoritiesSecretRef": [
      {
        "name": "additional-ca"
      }
    ]
  }
}'

oc patch hiveconfig hive --type=merge -p '
{
  "spec": {
    "additionalCertificateAuthoritiesSecretRef": [
      {
        "name": "additional-ca"
      }
    ]
  }
}'