git clone https://github.com/clusterpedia-io/clusterpedia.git
cd clusterpedia
git checkout v0.7.0

export STORAGE_NODE_NAME=<节点名称>
sed "s|__NODE_NAME__|$STORAGE_NODE_NAME|g" `grep __NODE_NAME__ -rl ./templates` > clusterpedia_internalstorage_pv.yaml

kubectl apply -f .

# 跳回 Clusterpedia 项目根目录
cd ../../../

# 部署 Clusterpedia CRD 与组件
kubectl apply -f ./deploy

kubectl -n clusterpedia-system get pods

kubectl get clusterimportpolicy

kubectl get clusterimportpolicy

kubectl get pediacluster

# delete compontents
kubectl delete -f ./deploy/clusterpedia_apiserver_apiservice.yaml
kubectl delete -f ./deploy/clusterpedia_apiserver_deployment.yaml
kubectl delete -f ./deploy/clusterpedia_clustersynchro_manager_deployment.yaml
kubectl delete -f ./deploy/clusterpedia_controller_manager_deployment.yaml
kubectl delete -f ./deploy/clusterpedia_apiserver_rbac.yaml

# delete crds
kubectl delete -f ./deploy/cluster.clusterpedia.io_clustersyncresources.yaml
kubectl delete -f ./deploy/cluster.clusterpedia.io_pediaclusers.yaml
kubectl delete -f ./deploy/policy.clusterpedia.io_clusterimportpolicies.yaml
kubectl delete -f ./deploy/policy.clusterpedia.io_pediaclusterlifecycles.yaml

kubectl delete -f ./deploy/internalstorage/<storage type>

kubectl get pv clusterpedia-internalstorage-<storage type>

kubectl delete pv clusterpedia-internalstorage-<storage type>

# 遗留数据所在节点
rm -rf /var/local/clusterpedia/internalstorage/<storage type>

# internalstorage configmap example
apiVersion: v1
kind: ConfigMap
metadata:
  name: clusterpedia-internalstorage
  namespace: clusterpedia-system
data:
  internalstorage-config.yaml: |
    type: "mysql"
    host: "clusterpedia-internalstorage-mysql"
    port: 3306
    user: root
    database: "clusterpedia"
    connPool:
      maxIdleConns: 10
      maxOpenConns: 100
      connMaxLifetime: 1h
    log:
      slowThreshold: "100ms"
      logger:
        filename: /var/log/clusterpedia/internalstorage.log
        maxbackups: 3    

type: "mysql"
host: "clusterpedia-internalstorage-mysql"
port: 3306
user: root
database: "clusterpedia"

kubectl -n clusterpedia-system create secret generic \
    internalstorage-password --from-literal=password=<存储组件访问密码>

$ kubectl get pediacluster
NAME        APISERVER                        VERSION            STATUS
demo1       https://10.6.101.100:6443        v1.22.3-aliyun.1   Healthy
demo2       https://10.6.101.100:6443        v1.21.0            Healthy

apiVersion: cluster.clusterpedia.io/v1alpha2
kind: PediaCluster
metadata:
  name: demo1
spec:
  apiserver: https://10.6.101.100:6443
  kubeconfig:
  caData:
  tokenData:
  certData:
  keyData:
  syncResources: []
  syncAllCustomResources: false
  syncResourcesRefName: ""

kubectl get clustersyncresources

apiVersion: cluster.clusterpedia.io/v1alpha2
kind: ClusterSyncResources
metadata:
  name: global-base
spec:
  syncResources:
    - group: ""
      resources:
        - pods
    - group: "apps"
      resources:
        - "*"

apiVersion: cluster.clusterpedia.io/v1alpha2
kind: PediaCluster
metadata:
  name: demo1
spec:
  syncResourcesRefName: "global-base"

apiVersion: cluster.clusterpedia.io/v1alpha2
kind: PediaCluster
metadata:
  name: demo1
spec:
  syncResourcesRefName: "global-base"
  syncResources:
    - group: ""
      resources:
        - pods
        - configmaps

kubectl get collectionresources

kubectl get collectionresources -o yaml

# 输出：
apiVersion: v1
items:
- apiVersion: clusterpedia.io/v1beta1
  kind: CollectionResource
  metadata:
    creationTimestamp: null
    name: any
  resourceTypes: []
- apiVersion: clusterpedia.io/v1beta1
  kind: CollectionResource
  metadata:
    creationTimestamp: null
    name: workloads
  resourceTypes:
  - group: apps
    resource: deployments
    version: v1
  - group: apps
    resource: daemonsets
    version: v1
  - group: apps
    resource: statefulsets
    version: v1
- apiVersion: clusterpedia.io/v1beta1
  kind: CollectionResource
  metadata:
    creationTimestamp: null
    name: kuberesources
  resourceTypes:
  - group: ""
  - group: admission.k8s.io
  - group: admissionregistration.k8s.io
  - group: apiextensions.k8s.io
  - group: apps
  - group: authentication.k8s.io
  - group: authorization.k8s.io
  - group: autoscaling
  - group: batch
  - group: certificates.k8s.io
  - group: coordination.k8s.io
  - group: discovery.k8s.io
  - group: events.k8s.io
  - group: extensions
  - group: flowcontrol.apiserver.k8s.io
  - group: imagepolicy.k8s.io
  - group: internal.apiserver.k8s.io
  - group: networking.k8s.io
  - group: node.k8s.io
  - group: policy
  - group: rbac.authorization.k8s.io
  - group: scheduling.k8s.io
  - group: storage.k8s.io
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

$ kubectl get collectionresources any
Error from server (BadRequest): url query - `groups` or `resources` is required

apiVersion: clusterpedia.io/v1beta1
kind: CollectionResource
metadata:
  name: workloads
resourceTypes:
- group: apps
  resource: deployments
- group: apps
  resource: daemonsets
- group: apps
  resource: statefulsets
- group: batch
  resource: cronjobs

apiVersion: policy.clusterpedia.io/v1alpha1
kind: ClusterImportPolicy
metadata:
  name: mcp
spec:
  source:
    group: "cluster.example.io"
    resource: clusters
    selectorTemplate: ""
  references:
    - group: ""
      resource: secrets
      namespaceTemplate: "{{ .source.spec.authSecretRef.namespace }}"
      nameTemplate: "{{ .source.spec.authSecretRef.name }}"
      key: authSecret
  nameTemplate: "mcp-{{ .source.metadata.name }}"
  template: |
    spec:
      apiserver: "{{ .source.spec.apiEndpoint }}"
      caData: "{{ .references.authSecret.data.ca }}"
      tokenData: "{{ .references.authSecret.data.token }}"
      syncResources:
        - group: ""
          resources:
          - "pods"
        - group: "apps"
          resources:
          - "*"    
  creationCondition: |
    {{ if ne .source.spec.apiEndpoint "" }}
      {{ range .source.status.conditions }}
        {{ if eq .type "Ready" }}
          {{ if eq .status "True" }} true {{ end }}
        {{ end }}
      {{ end }}
    {{ end }}    

apiVersion: policy.clusterpedia.io/v1alpha1
kind: ClusterImportPolicy
metadata:
  name: multi-cluster-polatform
spec:
  source:
    group: "example.io"
    resource: clusters
    versions: []
    selectorTemplate: |
            {{ if eq .source.metadata.namespace "default" }} true {{ end }}
  references:
    - group: ""
      resource: secrets
      versions: []
      namespaceTemplate: "{{ .source.spec.secretRef.namespace }}"
      nameTemplate: "{{ .source.spec.secretRef.name }}"
      key: secret

apiVersion: policy.clusterpedia.io/v1alpha1
kind: ClusterImportPolicy
metadata:
  name: kubevela
spec:
  source:
    group: ""
    resource: secrets
    selectorTemplate: |
      {{ if eq .source.metadata.namespace "vela-system" }}
        {{ if .source.metadata.labels }}
          {{ eq (index .source.metadata.labels "cluster.core.oam.dev/cluster-credential-type") "X509Certificate" }}
        {{ end }}
      {{ end }}      

spec:
  references:
  - group: a.example.io
    resource: aresource
    namespaceTemplate: "{{ .source.spec.aNamespace }}"
    nameTemplate: "{{ .source.spec.aName }}"
    key: refA
  - group: b.example.io
    resource: bresource
    namespaceTemplate: "{{ .references.refA.spec.bNamespace }}"
    nameTemplate: "{{ .references.refA.spec.bName }}"
    key: refB

apiVersion: policy.clusterpedia.io/v1alpha1
kind: PediaClusterLifecycle
metadata:
  name: <prefix>-example
spec:
  source:
    group: example.io
    version: v1beta1
    resource: clusters
    namespace: ""
    name: example
  references:
    - group: ""
      resource: secrets
      version: v1
      namespaceTemplate: "{{ .source.spec.secretRef.namespace }}"
      nameTemplate: "{{ .source.spec.secretRef.name }}"
      key: secret

apiVersion: policy.clusterpedia.io/v1alpha1
kind: ClusterImportPolicy
metadata:
  name: mcp
spec:
... other fields
  template: |
    spec:
      apiserver: "{{ .source.spec.apiEndpoint }}"
      caData: "{{ .references.authSecret.data.ca }}"
      tokenData: "{{ .references.tokenData.data.token }}"
      syncResources:
        - group: ""
          resources:
          - "pods"
        - group: "apps"
          resources:
          - "*"    
  creationCondition: |
    {{ if ne .source.spec.apiEndpoint "" }}
      {{ range .source.status.conditions }}
        {{ if eq .type "Ready" }}
          {{ if eq .status "True" }} true {{ end }}
        {{ end }}
      {{ end }}
    {{ end }}    

apiVersion: policy.clusterpedia.io/v1alpha1
kind: ClusterImportPolicy
metadata:
  name: mcp-example
spec:
... other fields
  template: |
    spec:
      apiserver: "{{ .source.spec.apiEndpoint }}"
      caData: "{{ .references.authSecret.data.ca }}"
      tokenData: "{{ .references.tokenData.data.token }}"
      syncResources:
        - group: ""
          resources:
          - "pods"
        - group: "apps"
          resources:
          - "*"    
  creationCondition: |
    {{ if ne .source.spec.apiEndpoint "" }}
      {{ range .source.status.conditions }}
        {{ if eq .type "Ready" }}
          {{ if eq .status "True" }} true {{ end }}
        {{ end }}
      {{ end }}
    {{ end }}    

apiVersion: cluster.clusterpedia.io/v1alpha2
kind: PediaCluster
metadata:
  name: mcp-example
  ownerReferences:
  - apiVersion: policy.clusterpedia.io/v1alpha1
    kind: PediaClusterLifecycle
    name: mcp-example
    uid: f932483a-b1c5-4894-a524-00f78ea34a9f

apiVersion: cluster.clusterpedia.io/v1alpha2
kind: PediaCluster
metadata:
  name: cluster-example
spec:
  apiserver: "https://10.30.43.43:6443"
  kubeconfig:
  caData:
  tokenData:
  certData:
  keyData:
  syncResources: []

# mac
cat ./kubeconfig | base64

# linux
cat ./kubeconfig | base64 -w 0

kubectl get pediacluster

# 注意：当前 kubectl 连接到被接入集群
kubectl apply -f https://raw.githubusercontent.com/clusterpedia-io/clusterpedia/main/examples/clusterpedia_synchro_rbac.yaml

# 获取 Service Account 对应 CA 和 Token
SYNCHRO_CA=$(kubectl -n default get secret $(kubectl -n default get serviceaccount clusterpedia-synchro -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.ca\.crt}')
SYNCHRO_TOKEN=$(kubectl -n default get secret $(kubectl -n default get serviceaccount clusterpedia-synchro -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.token}')

apiVersion: cluster.clusterpedia.io/v1alpha2
kind: PediaCluster
metadata:
  name: cluster-example
spec:
  apiserver: https://10.6.100.10:6443
  caData: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvakNDQWVhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1Ea3lOREV3TVRNeU5Gb1hEVE14TURreU1qRXdNVE15TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTy9TCnZta1U5bk1uUlRIT3lvK3hjdFRJY0lPYnMzc0F5cTI2djRQYkVtb3ZWM2xPOVQwMTF2cE96S0pyOUFxeVZMRnYKVXFBRHBTakM3WXd3MnZwSld3bDEySlBvUm1xZ1FBSFNkYlJpU3BDTDRudjlvR25VOWI2dllWSy9iRitkUVFCSApnQ1h6NnZoTGY4Wmd2N2tUQ2JBdkFPaE9OSlU3MllYTE8zT0lZQjJva1NCRGFVUjNvNnpwZGVWTkt5V0EyNVA3CkRobk8yTk01QzlpRERqTTRLY2FTa3JPSkJvbUlsSHFZRjRwVXdTTlFvcGVGRVRyZ3ZzcTkwSks2YUJVS0t5ajYKK2NGdjI3S0k4K1ZMUEtaSTE2c25Mbng2RXRTazZtZjJXTHdJZlhyQlgwREsvYXBEQ015R2pEb2dCaGpJSVhoVAp2bjVQZndFWUNsdGZFTEhKSkdVQ0F3RUFBYU5aTUZjd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZJVDhLRHdCbUVvMHladUFEZkhkKzQ1L3ZFYzdNQlVHQTFVZEVRUU8KTUF5Q0NtdDFZbVZ5Ym1WMFpYTXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBT0F5VHQ4S3ZFN0dvREhQT09pdgoyR2I2WWVsUU5KcUMza1dIOXc1NTFNaGZvS3ZiM21VaUV6ZVMwOUNwZUQrTFh5ZnlqQzhZYkJxQjZXSFhNZWMrCnpPdDNPazRYV0FmZVVZTXhOQ1FJblc4cjI4cmZnblErc1NCdHQyeERQN1RZY09oNVZGZkI2K3JtTmFTblZ1NjgKSFFxdlFMNEFXbVhkR09jRWNBRThYdkdiOWhwSjVNckRHdzQ0UTYyOG9YazZ0N01aWTFOMUNQdW9HZ1VmS1N3bgo1MUFWRTFOVVdNV0tEQXhaa2I4bEhvR3VWaDFzWmd3SnJRQjR5clh1cmxGN0Y2bVRlYm4rcDVKM0toT0V4KzlsCjFXdkwwbWkxL1J2bVJKNm11YmtjWUwzN1FJWjI1YXdyaEZMN0Z1ejNRSTFqTTdYMHZET2VUM2VuVUFCZW5SMS8KUnlnPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
  tokenData: ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklrMHRSalJtZGpSdVgxcFljMGxsU1ZneFlXMHpPSFZOY0Zwbk1UTkhiVFpsVFZwQ2JIWk9SbU5XYW5NaWZRLmV5SnBjM01pT2lKcmRXSmxjbTVsZEdWekwzTmxjblpwWTJWaFkyTnZkVzUwSWl3aWEzVmlaWEp1WlhSbGN5NXBieTl6WlhKMmFXTmxZV05qYjNWdWRDOXVZVzFsYzNCaFkyVWlPaUprWldaaGRXeDBJaXdpYTNWaVpYSnVaWFJsY3k1cGJ5OXpaWEoyYVdObFlXTmpiM1Z1ZEM5elpXTnlaWFF1Ym1GdFpTSTZJbU5zZFhOMFpYSndaV1JwWVMxemVXNWphSEp2TFhSdmEyVnVMVGsxYTJSNElpd2lhM1ZpWlhKdVpYUmxjeTVwYnk5elpYSjJhV05sWVdOamIzVnVkQzl6WlhKMmFXTmxMV0ZqWTI5MWJuUXVibUZ0WlNJNkltTnNkWE4wWlhKd1pXUnBZUzF6ZVc1amFISnZJaXdpYTNWaVpYSnVaWFJsY3k1cGJ5OXpaWEoyYVdObFlXTmpiM1Z1ZEM5elpYSjJhV05sTFdGalkyOTFiblF1ZFdsa0lqb2lNREl5WXpNMk5USXRPR1k0WkMwME5qSmtMV0l6TnpFdFpUVXhPREF3TnpFeE9HUTBJaXdpYzNWaUlqb2ljM2x6ZEdWdE9uTmxjblpwWTJWaFkyTnZkVzUwT21SbFptRjFiSFE2WTJ4MWMzUmxjbkJsWkdsaExYTjVibU5vY204aWZRLkF4ZjhmbG5oR0lDYjJaMDdkT0FKUW11aHVIX0ZzRzZRSVY5Sm5sSmtPUnF5aGpWSDMyMkVqWDk1bVhoZ2RVQ2RfZXphRFJ1RFFpLTBOWDFseGc5OXpYRks1MC10ZzNfYlh5NFA1QnRFOUpRNnNraUt4dDFBZVJHVUF4bG5fVFU3SHozLTU5Vnl5Q3NwckFZczlsQWQwRFB6bTRqb1dyS1lKUXpPaGl5VjkzOWpaX2ZkS1BVUmNaMVVKVGpXUTlvNEFFY0hMdDlyTEJNMTk2eDRkbzA4ZHFaUnVtTzJZRXFkQTB3ZnRxZ2NGQzdtTGlSVVhkWElkYW9CY1BuWXBwM01MU3B5QjJQMV9vSlRFNS1nd3k4N2Jwb3U1RXo2TElSSExIeW5NWXAtWVRLR2hBbDJwMXdJb0tDZUNnQng4RlRfdzM4Rnh1TnE0UDRoQW5RUUh6bU9Ndw==
  syncResources: []

kubectl get pediacluster

$ kubectl apply -f https://raw.githubusercontent.com/clusterpedia-io/clusterpedia/main/deploy/clusterimportpolicy/cluster_api.yaml
$ kubectl get clusterimportpolicy
NAME          AGE
cluster-api   4d19h

$ kubectl get cluster
NAME                PHASE         AGE     VERSION
capi-quickstart     Provisioned   3d23h   v1.24.2
capi-quickstart-2   Provisioned   3d23h   v1.24.2

$ kubectl get pediaclusterlifecycle
NAME                        AGE
default-capi-quickstart     3d23h
default-capi-quickstart-2   3d23h

$ kubectl get pediacluster
NAME                        READY   VERSION   APISERVER
default-capi-quickstart     True    v1.24.2
default-capi-quickstart-2   True    v1.24.2

NAME                    CLUSTER           INITIALIZED   API SERVER AVAILABLE   REPLICAS   READY   UPDATED   UNAVAILABLE   AGE   VERSION
capi-quickstart-2xcsz   capi-quickstart   true                                 1                  1         1             86s   v1.24.2

$ # Since CNI is not installed, the nodes are not ready.
$ kubectl --cluster clusterpedia get no
CLUSTER                     NAME                                            STATUS     ROLES           AGE   VERSION
default-capi-quickstart-2   capi-quickstart-2-ctm9k-g2m87                   NotReady   control-plane   12m   v1.24.2
default-capi-quickstart-2   capi-quickstart-2-md-0-s8hbx-7bd44554b5-kzcb6   NotReady   <none>          11m   v1.24.2
default-capi-quickstart     capi-quickstart-2xcsz-fxrrk                     NotReady   control-plane   21m   v1.24.2
default-capi-quickstart     capi-quickstart-md-0-9tw2g-b8b4f46cf-gggvq      NotReady   <none>          20m   v1.24.2

$ kubectl create -f https://raw.githubusercontent.com/clusterpedia-io/clusterpedia/main/deploy/clusterimportpolicy/karmada.yaml
$ kubectl get clusterimportpolicy
NAME      AGE
karmada   7d5h

$ kubectl get cluster
NAME      VERSION   MODE   READY   AGE
argocd              Push   False   8d
member1   v1.23.4   Push   True    22d
member3   v1.23.4   Pull   True    22d

$ kubectl get pediaclusterlifecycle
NAME              AGE
karmada-argocd    7d5h
karmada-member1   7d5h
karmada-member3   7d5h

NAME              APISERVER                 VERSION   STATUS
karmada-member1   https://172.18.0.4:6443   v1.23.4   Healthy

$ kubectl create -f https://raw.githubusercontent.com/clusterpedia-io/clusterpedia/main/deploy/clusterimportpolicy/vcluster.yaml
$ kubectl get clusterimportpolicy
NAME      AGE
vclsuter  5h

syncer:
  extraArgs:
  - --out-kube-config-server=https://<vcluster name>.<namespace>.svc
  - --tls-san=<vcluster name>.<namespace>.svc,127.0.0.1

# vcluster-1.yaml
syncer:
  extraArgs:
  - --out-kube-config-server=https://vcluster-1.default.svc
  - --tls-san=vcluster-1.default.svc,127.0.0.1

$ vcluster create -n default -f vcluster-1.yaml vcluster-1

# vcluster-2.yaml
syncer:
  extraArgs:
  - --out-kube-config-server=https://vcluster-2.default.svc
  - --tls-san=vcluster-2.default.svc,127.0.0.1

$ vcluster create -n default -f vcluster-2.yaml vcluster-2

$ vcluster list
 NAME              NAMESPACE         STATUS    CONNECTED   CREATED                         AGE
 caiwei-vcluster   caiwei-vcluster   Running               2022-08-26 16:10:52 +0800 CST   484h49m6s
 vcluster-1        default           Running               2022-09-15 20:57:59 +0800 CST   1m59s
 vcluster-2        default           Running               2022-09-15 20:59:34 +0800 CST   24s

 $ kubectl --cluster clusterpedia get po -A
NAMESPACE     CLUSTER                              NAME                       READY   STATUS    RESTARTS   AGE
default       vc-caiwei-vcluster-caiwei-vcluster   backend-77f8f45fc8-5ssww   1/1     Running   0          20d
default       vc-caiwei-vcluster-caiwei-vcluster   backend-77f8f45fc8-j5m4c   1/1     Running   0          20d
default       vc-caiwei-vcluster-caiwei-vcluster   backend-77f8f45fc8-vjzf6   1/1     Running   0          20d
kube-system   vc-default-vcluster-1                coredns-669fb9997d-cxktv   1/1     Running   0          3m40s
kube-system   vc-default-vcluster-2                coredns-669fb9997d-g7w8l   1/1     Running   0          2m6s
kube-system   vc-caiwei-vcluster-caiwei-vcluster   coredns-669fb9997d-x6vc2   1/1     Running   0          20d

$ kubectl --cluster clusterpedia get ns
CLUSTER                              NAME              STATUS   AGE
vc-default-vcluster-2                default           Active   2m49s
vc-default-vcluster-1                default           Active   4m24s
vc-caiwei-vcluster-caiwei-vcluster   default           Active   20d
vc-default-vcluster-2                kube-node-lease   Active   2m49s
vc-default-vcluster-1                kube-node-lease   Active   4m24s
vc-caiwei-vcluster-caiwei-vcluster   kube-node-lease   Active   20d
vc-default-vcluster-2                kube-public       Active   2m49s
vc-default-vcluster-1                kube-public       Active   4m24s
vc-caiwei-vcluster-caiwei-vcluster   kube-public       Active   20d
vc-default-vcluster-2                kube-system       Active   2m49s
vc-default-vcluster-1                kube-system       Active   4m24s
vc-caiwei-vcluster-caiwei-vcluster   kube-system       Active   20d

$ kubectl get pediaclusterlifecycle
NAME                                 AGE
vc-caiwei-vcluster-caiwei-vcluster   20d
vc-default-vcluster-1                5m57s
vc-default-vcluster-2                4m24s

$ kubectl get pediacluster
NAME                                 READY   VERSION        APISERVER
vc-caiwei-vcluster-caiwei-vcluster   True    v1.23.5+k3s1   https://caiwei-vcluster.caiwei-vcluster.svc
vc-default-vcluster-1                True    v1.23.5+k3s1   https://vcluster-1.default.svc
vc-default-vcluster-2                True    v1.23.5+k3s1   https://vcluster-2.default.svc

apiVersion: cluster.mcp.io
kind: Cluster
metadata:
  name: cluster-1
spec:
  apiEndpoint: "https://172.10.10.10:6443"
  authSecretRef:
    namespace: "default"
    name: "cluster-1"
status:
  conditions:
    - type: Ready
      status: True
---
apiVersion: v1
kind: Secret
metadata:
  name: cluster-1
data:
  ca: **cluster ca bundle**
  token: **cluster token**

apiVersion: policy.clusterpedia.io/v1alpha1
kind: ClusterImportPolicy
metadata:
  name: mcp
spec:
  source:
    group: "cluster.mcp.io"
    resource: clusters
    versions: []
  references:
    - group: ""
      resource: secrets
      versions: []
      namespaceTemplate: "{{ .source.spec.authSecretRef.namespace }}"
      nameTemplate: "{{ .source.spec.authSecretRef.name }}"
      key: authSecret
  nameTemplate: "mcp-{{ .source.metadata.name }}"
  template: |
    spec:
      apiserver: "{{ .source.spec.apiEndpoint }}"
      caData: "{{ .references.authSecret.data.ca }}"
      tokenData: "{{ .references.authSecret.data.token }}"
      syncResources:
        - group: ""
          resources:
          - "pods"
        - group: "apps"
          resources:
          - "*"
      syncResourcesRefName: ""    
  creationCondition: |
    {{ if ne .source.spec.apiEndpoint "" }}
      {{ range .source.status.conditions }}
        {{ if eq .type "Ready" }}
          {{ if eq .status "True" }} true {{ end }}
        {{ end }}
      {{ end }}
    {{ end }}    

# example
apiVersion: cluster.clusterpedia.io/v1alpha2
kind: PediaCluster
metadata:
  name: cluster-example
spec:
  apiserver: "https://10.30.43.43:6443"
  syncResources:
  - group: apps
    resources:
     - deployments
  - group: ""
    resources:
     - pods
     - configmaps
  - group: cert-manager.io
    versions:
      - v1
    resources:
      - certificates

syncResources:
 - group: apps
   versions: []
   resources:
     - deployments
     - daemonsets

kubectl get --raw="/apis/clusterpedia.io/v1beta1/resources/apis/apps" | jq

{
  "kind": "APIGroup",
  "apiVersion": "v1",
  "name": "apps",
  "versions": [
    {
      "groupVersion": "apps/v1",
      "version": "v1"
    },
    {
      "groupVersion": "apps/v1beta2",
      "version": "v1beta2"
    },
    {
      "groupVersion": "apps/v1beta1",
      "version": "v1beta1"
    }
  ],
  "preferredVersion": {
    "groupVersion": "apps/v1",
    "version": "v1"
  }
}

# 在被接入集群内执行
kubectl get --raw="/apis/cert-manager.io" | jq

{
  "kind": "APIGroup",
  "apiVersion": "v1",
  "name": "cert-manager.io",
  "versions": [
    {
      "groupVersion": "cert-manager.io/v1",
      "version": "v1"
    },
    {
      "groupVersion": "cert-manager.io/v1beta1",
      "version": "v1beta1"
    },
    {
      "groupVersion": "cert-manager.io/v1alpha3",
      "version": "v1alpha3"
    },
    {
      "groupVersion": "cert-manager.io/v1alpha2",
      "version": "v1alpha2"
    }
  ],
  "preferredVersion": {
    "groupVersion": "cert-manager.io/v1",
    "version": "v1"
  }
}

syncResources:
 - group: cert-manager.io
   versions:
    - v1beta1
   resources:
    - certificates

spec:
  syncAllCustomResources: true

spec:
  syncResources:
  - group: "apps"
    resources:
    - "*"

spec:
  syncResources:
  - group: "*"
    resources:
    - "*"

apiVersion: cluster.clusterpedia.io/v1alpha2
kind: ClusterSyncResources
metadata:
  name: global-base
spec:
  syncResources:
    - group: ""
      resources:
        - pods
    - group: "apps"
      resources:
        - "*"
---
apiVersion: cluster.clusterpedia.io/v1alpha2
kind: PediaCluster
metadata:
  name: demo1
spec:
  syncResourcesRefName: "global-base"
  syncResources:
    - group: ""
      resources:
        - pods
        - configmaps

status:
  syncResources:
  - group: apps
    resources:
    - name: deployments
      kind: Deployment
      namespaced: true
      syncConditions:
      - lastTransitionTime: "2022-01-13T04:34:08Z"
        status: Syncing
        storageVersion: v1
        version: v1

status:
  syncResources:
  - group: apps
    resources:
    - name: deployments
      kind: Deployment
      namespaced: true
      syncConditions:
      - lastTransitionTime: "2022-01-13T04:34:04Z"
        status: Syncing
        storageVersion: v1
        version: v1beta1

kubectl api-resources | grep clusterpedia.io

kubectl get --raw="/apis/clusterpedia.io/v1beta1/resources/version"

kubectl get --raw="/apis/clusterpedia.io/v1beta1/resources/clusters/cluster-1/version"

curl -sfL https://raw.githubusercontent.com/clusterpedia-io/clusterpedia/v0.7.0/hack/gen-clusterconfigs.sh | sh -

cat ~/.kube/config

# .kube/config
- cluster:
    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvakNDQWVhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1Ea3lOREV3TVRNeU5Gb1hEVE14TURreU1qRXdNVE15TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTy9TCnZta1U5bk1uUlRIT3lvK3hjdFRJY0lPYnMzc0F5cTI2djRQYkVtb3ZWM2xPOVQwMTF2cE96S0pyOUFxeVZMRnYKVXFBRHBTakM3WXd3MnZwSld3bDEySlBvUm1xZ1FBSFNkYlJpU3BDTDRudjlvR25VOWI2dllWSy9iRitkUVFCSApnQ1h6NnZoTGY4Wmd2N2tUQ2JBdkFPaE9OSlU3MllYTE8zT0lZQjJva1NCRGFVUjNvNnpwZGVWTkt5V0EyNVA3CkRobk8yTk01QzlpRERqTTRLY2FTa3JPSkJvbUlsSHFZRjRwVXdTTlFvcGVGRVRyZ3ZzcTkwSks2YUJVS0t5ajYKK2NGdjI3S0k4K1ZMUEtaSTE2c25Mbng2RXRTazZtZjJXTHdJZlhyQlgwREsvYXBEQ015R2pEb2dCaGpJSVhoVAp2bjVQZndFWUNsdGZFTEhKSkdVQ0F3RUFBYU5aTUZjd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZJVDhLRHdCbUVvMHladUFEZkhkKzQ1L3ZFYzdNQlVHQTFVZEVRUU8KTUF5Q0NtdDFZbVZ5Ym1WMFpYTXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBT0F5VHQ4S3ZFN0dvREhQT09pdgoyR2I2WWVsUU5KcUMza1dIOXc1NTFNaGZvS3ZiM21VaUV6ZVMwOUNwZUQrTFh5ZnlqQzhZYkJxQjZXSFhNZWMrCnpPdDNPazRYV0FmZVVZTXhOQ1FJblc4cjI4cmZnblErc1NCdHQyeERQN1RZY09oNVZGZkI2K3JtTmFTblZ1NjgKSFFxdlFMNEFXbVhkR09jRWNBRThYdkdiOWhwSjVNckRHdzQ0UTYyOG9YazZ0N01aWTFOMUNQdW9HZ1VmS1N3bgo1MUFWRTFOVVdNV0tEQXhaa2I4bEhvR3VWaDFzWmd3SnJRQjR5clh1cmxGN0Y2bVRlYm4rcDVKM0toT0V4KzlsCjFXdkwwbWkxL1J2bVJKNm11YmtjWUwzN1FJWjI1YXdyaEZMN0Z1ejNRSTFqTTdYMHZET2VUM2VuVUFCZW5SMS8KUnlnPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
    server: https://10.6.100.10:6443/apis/clusterpedia.io/v1beta1/resources/clusters/cluster-1
  name: cluster-1
- cluster:
    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvakNDQWVhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1Ea3lOREV3TVRNeU5Gb1hEVE14TURreU1qRXdNVE15TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTy9TCnZta1U5bk1uUlRIT3lvK3hjdFRJY0lPYnMzc0F5cTI2djRQYkVtb3ZWM2xPOVQwMTF2cE96S0pyOUFxeVZMRnYKVXFBRHBTakM3WXd3MnZwSld3bDEySlBvUm1xZ1FBSFNkYlJpU3BDTDRudjlvR25VOWI2dllWSy9iRitkUVFCSApnQ1h6NnZoTGY4Wmd2N2tUQ2JBdkFPaE9OSlU3MllYTE8zT0lZQjJva1NCRGFVUjNvNnpwZGVWTkt5V0EyNVA3CkRobk8yTk01QzlpRERqTTRLY2FTa3JPSkJvbUlsSHFZRjRwVXdTTlFvcGVGRVRyZ3ZzcTkwSks2YUJVS0t5ajYKK2NGdjI3S0k4K1ZMUEtaSTE2c25Mbng2RXRTazZtZjJXTHdJZlhyQlgwREsvYXBEQ015R2pEb2dCaGpJSVhoVAp2bjVQZndFWUNsdGZFTEhKSkdVQ0F3RUFBYU5aTUZjd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZJVDhLRHdCbUVvMHladUFEZkhkKzQ1L3ZFYzdNQlVHQTFVZEVRUU8KTUF5Q0NtdDFZbVZ5Ym1WMFpYTXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBT0F5VHQ4S3ZFN0dvREhQT09pdgoyR2I2WWVsUU5KcUMza1dIOXc1NTFNaGZvS3ZiM21VaUV6ZVMwOUNwZUQrTFh5ZnlqQzhZYkJxQjZXSFhNZWMrCnpPdDNPazRYV0FmZVVZTXhOQ1FJblc4cjI4cmZnblErc1NCdHQyeERQN1RZY09oNVZGZkI2K3JtTmFTblZ1NjgKSFFxdlFMNEFXbVhkR09jRWNBRThYdkdiOWhwSjVNckRHdzQ0UTYyOG9YazZ0N01aWTFOMUNQdW9HZ1VmS1N3bgo1MUFWRTFOVVdNV0tEQXhaa2I4bEhvR3VWaDFzWmd3SnJRQjR5clh1cmxGN0Y2bVRlYm4rcDVKM0toT0V4KzlsCjFXdkwwbWkxL1J2bVJKNm11YmtjWUwzN1FJWjI1YXdyaEZMN0Z1ejNRSTFqTTdYMHZET2VUM2VuVUFCZW5SMS8KUnlnPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
    server: https://10.6.100.10:6443/apis/clusterpedia.io/v1beta1/resources/clusters/cluster-2
  name: cluster-2
- cluster:
    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvakNDQWVhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1Ea3lOREV3TVRNeU5Gb1hEVE14TURreU1qRXdNVE15TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTy9TCnZta1U5bk1uUlRIT3lvK3hjdFRJY0lPYnMzc0F5cTI2djRQYkVtb3ZWM2xPOVQwMTF2cE96S0pyOUFxeVZMRnYKVXFBRHBTakM3WXd3MnZwSld3bDEySlBvUm1xZ1FBSFNkYlJpU3BDTDRudjlvR25VOWI2dllWSy9iRitkUVFCSApnQ1h6NnZoTGY4Wmd2N2tUQ2JBdkFPaE9OSlU3MllYTE8zT0lZQjJva1NCRGFVUjNvNnpwZGVWTkt5V0EyNVA3CkRobk8yTk01QzlpRERqTTRLY2FTa3JPSkJvbUlsSHFZRjRwVXdTTlFvcGVGRVRyZ3ZzcTkwSks2YUJVS0t5ajYKK2NGdjI3S0k4K1ZMUEtaSTE2c25Mbng2RXRTazZtZjJXTHdJZlhyQlgwREsvYXBEQ015R2pEb2dCaGpJSVhoVAp2bjVQZndFWUNsdGZFTEhKSkdVQ0F3RUFBYU5aTUZjd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZJVDhLRHdCbUVvMHladUFEZkhkKzQ1L3ZFYzdNQlVHQTFVZEVRUU8KTUF5Q0NtdDFZbVZ5Ym1WMFpYTXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBT0F5VHQ4S3ZFN0dvREhQT09pdgoyR2I2WWVsUU5KcUMza1dIOXc1NTFNaGZvS3ZiM21VaUV6ZVMwOUNwZUQrTFh5ZnlqQzhZYkJxQjZXSFhNZWMrCnpPdDNPazRYV0FmZVVZTXhOQ1FJblc4cjI4cmZnblErc1NCdHQyeERQN1RZY09oNVZGZkI2K3JtTmFTblZ1NjgKSFFxdlFMNEFXbVhkR09jRWNBRThYdkdiOWhwSjVNckRHdzQ0UTYyOG9YazZ0N01aWTFOMUNQdW9HZ1VmS1N3bgo1MUFWRTFOVVdNV0tEQXhaa2I4bEhvR3VWaDFzWmd3SnJRQjR5clh1cmxGN0Y2bVRlYm4rcDVKM0toT0V4KzlsCjFXdkwwbWkxL1J2bVJKNm11YmtjWUwzN1FJWjI1YXdyaEZMN0Z1ejNRSTFqTTdYMHZET2VUM2VuVUFCZW5SMS8KUnlnPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
    server: https://10.6.100.10:6443/apis/clusterpedia.io/v1beta1/resources
  name: clusterpedia
- cluster:
    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvakNDQWVhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1Ea3lOREV3TVRNeU5Gb1hEVE14TURreU1qRXdNVE15TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTy9TCnZta1U5bk1uUlRIT3lvK3hjdFRJY0lPYnMzc0F5cTI2djRQYkVtb3ZWM2xPOVQwMTF2cE96S0pyOUFxeVZMRnYKVXFBRHBTakM3WXd3MnZwSld3bDEySlBvUm1xZ1FBSFNkYlJpU3BDTDRudjlvR25VOWI2dllWSy9iRitkUVFCSApnQ1h6NnZoTGY4Wmd2N2tUQ2JBdkFPaE9OSlU3MllYTE8zT0lZQjJva1NCRGFVUjNvNnpwZGVWTkt5V0EyNVA3CkRobk8yTk01QzlpRERqTTRLY2FTa3JPSkJvbUlsSHFZRjRwVXdTTlFvcGVGRVRyZ3ZzcTkwSks2YUJVS0t5ajYKK2NGdjI3S0k4K1ZMUEtaSTE2c25Mbng2RXRTazZtZjJXTHdJZlhyQlgwREsvYXBEQ015R2pEb2dCaGpJSVhoVAp2bjVQZndFWUNsdGZFTEhKSkdVQ0F3RUFBYU5aTUZjd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZJVDhLRHdCbUVvMHladUFEZkhkKzQ1L3ZFYzdNQlVHQTFVZEVRUU8KTUF5Q0NtdDFZbVZ5Ym1WMFpYTXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBT0F5VHQ4S3ZFN0dvREhQT09pdgoyR2I2WWVsUU5KcUMza1dIOXc1NTFNaGZvS3ZiM21VaUV6ZVMwOUNwZUQrTFh5ZnlqQzhZYkJxQjZXSFhNZWMrCnpPdDNPazRYV0FmZVVZTXhOQ1FJblc4cjI4cmZnblErc1NCdHQyeERQN1RZY09oNVZGZkI2K3JtTmFTblZ1NjgKSFFxdlFMNEFXbVhkR09jRWNBRThYdkdiOWhwSjVNckRHdzQ0UTYyOG9YazZ0N01aWTFOMUNQdW9HZ1VmS1N3bgo1MUFWRTFOVVdNV0tEQXhaa2I4bEhvR3VWaDFzWmd3SnJRQjR5clh1cmxGN0Y2bVRlYm4rcDVKM0toT0V4KzlsCjFXdkwwbWkxL1J2bVJKNm11YmtjWUwzN1FJWjI1YXdyaEZMN0Z1ejNRSTFqTTdYMHZET2VUM2VuVUFCZW5SMS8KUnlnPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
    server: https://10.6.100.10:6443
  name: kubernetes

# 多集群检索时支持的资源
kubectl --cluster clusterpedia api-resources

# cluster-1 支持检索的资源
kubectl --cluster cluster-1 api-resources

kubectl --cluster clusterpedia api-resources

kubectl --cluster clusterpedia get deployments -A -l "internalstorage.clusterpedia.io/fuzzy-name=test"

kubectl --cluster clusterpedia get pods --field-selector="status.phase=Running"

# 也可以在首字符添加 `.`
kubectl --cluster clusterpedia get pods --field-selector=".status.phase notin (Running,Succeeded)"

kubectl --cluster clusterpedia get deploy \
    --field-selector="metadata.annotations['test.io'] in (value1,value2),spec.replica=3"

kubectl --cluster clusterpedia get pods --field-selector="status['phase']!=Running"

kubectl get po --field-selector="spec.containers[].name!=container1"

kubectl get po --field-selector="spec.containers[].name == container1"

kubectl get po --field-selector="spec.containers[1].name in (container1,container2)"

kubectl --cluster cluster-1 get deploy fake-deploy -o jsonpath="{.metadata.uid}"

$ kubectl get --raw "/apis/clusterpedia.io/v1beta1/collectionresources/workloads?onlyMetadata=true&limit=1" | jq
{
  "kind": "CollectionResource",
  "apiVersion": "clusterpedia.io/v1beta1",
  "metadata": {
    "name": "workloads",
    "creationTimestamp": null
  },
  "resourceTypes": [
    {
      "group": "apps",
      "version": "v1",
      "kind": "Deployment",
      "resource": "deployments"
    }
  ],
  "items": [
    {
      "apiVersion": "apps/v1",
      "kind": "Deployment",
      "metadata": {
        "annotations": {
          "deployment.kubernetes.io/revision": "1",
          "shadow.clusterpedia.io/cluster-name": "cluster-example"
        },
        "creationTimestamp": "2021-09-24T10:19:19Z",
        "generation": 1,
        "labels": {
          "k8s-app": "tigera-operator"
        },
        "name": "tigera-operator",
        "namespace": "tigera-operator",
        "resourceVersion": "125073610",
        "uid": "992f9d53-37cb-4184-a004-15b278b11f79"
      }
    }
  ]
}

$ kubectl get collectionresources
NAME            RESOURCES
any             *

$ kubectl get collectionresources any
Error from server (BadRequest): url query - `groups` or `resources` is required

$ kubectl get --raw "/apis/clusterpedia.io/v1beta1/collectionresources/any?onlyMetadata=true&groups=apps&resources=batch/jobs,batch/cronjobs" | jq

kube_deployment_created{cluster="test-14",namespace="clusterpedia-system",deployment="clusterpedia-apiserver"} 1.676557618e+09

$ helm repo update clusterpedia
$ helm search clusterpedia
 NAME                            CHART VERSION   APP VERSION     DESCRIPTION
 clusterpedia/clusterpedia       1.8.0           v0.7.0          A Helm chart for Kubernetes

$ helm -n clusterpedia-system get values clusterpedia > values.yaml

echo "clustersynchroManager:
  image:
    repository: iceber/clusterpedia/clustersynchro-manager
    tag: v0.8.0-ksm.1
  kubeStateMetrics:
    enabled: true
" > patch.yaml

$ helm -n clusterpedia-system upgrade -f values.yaml -f patch.yaml clusterpedia clusterpedia/clusterpedia

$ kubectl -n clusterpedia-system get svc
NAME                                          TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
clusterpedia-apiserver                        ClusterIP   10.97.129.238   <none>        443/TCP    150d
clusterpedia-clustersynchro-manager-metrics   ClusterIP   10.108.129.32   <none>        8081/TCP   51m
clusterpedia-kube-state-metrics               ClusterIP   10.108.130.62   <none>        8080/TCP   43m
clusterpedia-mysql                            ClusterIP   10.102.38.225   <none>        3306/TCP   150d
clusterpedia-mysql-headless                   ClusterIP   None            <none>        3306/TCP   150d

$ git clone --recursive https://github.com/clusterpedia-io/sample-storage.git && cd sample-storage
$ make build-plugin

$ # 在 sample-storage 目录
$ make build-components
$ ls -al ./bin
-rwxr-xr-x   1 icebergu  staff  90707488 11  7 11:15 apiserver
-rwxr-xr-x   1 icebergu  staff  91896016 11  7 11:16 binding-apiserver
-rwxr-xr-x   1 icebergu  staff  82769728 11  7 11:16 clustersynchro-manager
-rwxr-xr-x   1 icebergu  staff  45682000 11  7 11:17 controller-manager

# example-config.yaml
type: mysql
host: 127.0.0.1
port: "3306"
user: root
password: dangerous0
database: clusterpedia
log:
  stdout: true
  colorful: true
  slowThreshold: 100ms

$ STORAGE_PLUGINS=./plugins ./bin/clustersynchro-manager --kubeconfig ~/.kube/config \
    --storage-name=sample-storage-layer \
    --storage-config ./config.yaml

$ openssl req -nodes -new -x509 -keyout ca.key -out ca.crt
$ openssl req -out client.csr -new -newkey rsa:4096 -nodes -keyout client.key -subj "/CN=development/O=system:masters"
$ openssl x509 -req -days 365 -in client.csr -CA ca.crt -CAkey ca.key -set_serial 01 -sha256 -out client.crt

$ STORAGE_PLUGINS=./plugins ./bin/apiserver --client-ca-file ca.crt  --secure-port 8443 \
    --kubeconfig ~/.kube/config \
    --authentication-kubeconfig ~/.kube/config \
    --authorization-kubeconfig ~/.kube/config \
    --storage-name=sample-storage-layer \
    --storage-config ./config.yaml

# myvalues.yaml
storage:
  name: "sample-storage-layer"
  image:
    registry: ghcr.io
    repository: clusterpedia-io/clusterpedia/sample-storage-layer
    tag: v0.0.0-v0.6.0
  config:
    type: "mysql"
    host: "10.111.94.196"
    port: 3306
    user: root
    password: dangerous0
    database: clusterpedia

# helm template clusterpedia -n clusterpedia-system -f myvalues.yaml ./clusterpedia-core
...
      initContainers:
      - name: copy-storage-plugin
        image: ghcr.io/clusterpedia-io/clusterpedia/sample-storage-layer:v0.0.0-v0.6.0
        imagePullPolicy: IfNotPresent
        command:
          - /bin/sh
          - -ec
          - cp /plugins/* /var/lib/clusterpedia/plugins/
        volumeMounts:
        - name: storage-plugins
          mountPath: /var/lib/clusterpedia/plugins
      containers:
      - name: clusterpedia-clusterpedia-core-apiserver
        image: ghcr.io/clusterpedia-io/clusterpedia/apiserver:v0.6.0
        imagePullPolicy: IfNotPresent
        command:
        - /usr/local/bin/apiserver
        - --secure-port=443
        - --storage-name=sample-storage-layer
        - --storage-config=/etc/clusterpedia/storage/config.yaml
        env:
        - name: STORAGE_PLUGINS
          value: /var/lib/clusterpedia/plugins
        volumeMounts:
        - name: storage-config
          mountPath: /etc/clusterpedia/storage
          readOnly: true
        - name: storage-plugins
          mountPath: /var/lib/clusterpedia/plugins
          readOnly: true
      volumes:
      - name: storage-config
        configMap:
          name: clusterpedia-clusterpedia-core-sample-storage-layer-config
      - name: storage-plugins
        emptyDir: {}
...

# myvalues.yaml
storage:
  name: "sample-storage-layer"
  image:
    registry: ghcr.io
    repository: clusterpedia-io/clusterpedia/sample-storage-layer
    tag: v0.0.0-v0.6.0

  configMap: "sample-storage-config"
  componentEnv:
  - name: DB_PASSWORD
    valueFrom:
      secretKeyRef:
        name: "sample-storage-password"
        key: password

$ git clone --recursive https://github.com/clusterpedia-io/sample-storage.git && cd sample-storage

$ ls -al
...
-rw-r--r--   1 icebergu  staff    260 12 13 15:14 Dockerfile
-rw-r--r--   1 icebergu  staff   1836 12 13 16:03 Makefile
-rw-r--r--   1 icebergu  staff   2219 11 23 10:25 README.md
drwxr-xr-x  32 icebergu  staff   1024 11 23 10:30 clusterpedia
-rw-r--r--   1 icebergu  staff    156 11 23 10:25 example-config.yaml
-rw-r--r--   1 icebergu  staff   2376 12 13 15:33 go.mod
-rw-r--r--   1 icebergu  staff  46109 12 13 15:33 go.sum
-rw-r--r--   1 icebergu  staff    139 11 23 10:25 main.go
drwxr-xr-x  16 icebergu  staff    512 12 13 15:33 storage
drwxr-xr-x   9 icebergu  staff    288 12 13 15:33 vendor

package main

import (
	plugin "github.com/clusterpedia-io/sample-storage-layer/storage"
)

func init() {
	plugin.RegisterStorageLayer()
}

import (
	"gorm.io/gorm"

	"github.com/clusterpedia-io/clusterpedia/pkg/storage"
)

type StorageFactory struct {
	db *gorm.DB
}

var _ storage.StorageFactory = &StorageFactory{}

func NewStorageFactory(configPath string) (storage.StorageFactory, error)

const StorageName = "sample-storage-layer"

func RegisterStorageLayer() {
	storage.RegisterStorageFactoryFunc(StorageName, NewStorageFactory)
}

./bin/apiserver --storage-name=sample-storage-layer <other flags>

$ git submodule status
+4608c8d13101d82960525dfe39f51e4f64ed49b3 clusterpedia (v0.6.0)

$ make build-components
OUTPUT_DIR=/Users/icebergu/workspace/clusterpedia/sample-storage-layer ON_PLUGINS=true \
                /Library/Developer/CommandLineTools/usr/bin/make -C clusterpedia all
hack/builder.sh apiserver
hack/builder.sh binding-apiserver
hack/builder.sh clustersynchro-manager
hack/builder-nocgo.sh controller-manager

$ ls -al ./bin
-rwxr-xr-x   1 icebergu  staff  90724968 12 15 09:51 apiserver
-rwxr-xr-x   1 icebergu  staff  91936472 12 15 09:52 binding-apiserver
-rwxr-xr-x   1 icebergu  staff  82826584 12 15 09:52 clustersynchro-manager
-rwxr-xr-x   1 icebergu  staff  45677904 12 15 09:52 controller-manager

$ make build-plugin
CLUSTERPEDIA_REPO=/Users/icebergu/workspace/clusterpedia/sample-storage/clusterpedia \
                clusterpedia/hack/builder.sh plugins sample-storage-layer.so

$ ls -al ./plugins
-rw-r--r--   1 icebergu  staff  53354352 12 15 09:47 sample-storage-layer.so

$ BUILDER_IMAGE=ghcr.io/clusterpedia-io/clusterpedia/builder:v0.6.0 make image-plugin

$ cd clusterpedia
$ make image-builder
docker buildx build \
                -t "ghcr.io/clusterpedia-io/clusterpedia"/builder-amd64:4608c8d13101d82960525dfe39f51e4f64ed49b3 \
                --platform=linux/amd64 \
                --load \
                -f builder.dockerfile . ; \

# Makefile
CLUSTERPEDIA_VERSIONS = v0.6.0-beta.1 v0.6.0
RELEASE_ARCHS ?= amd64 arm64

# Chart.yaml
dependencies:
  - name: mysql
    repository: https://charts.bitnami.com/bitnami
    version: 9.x.x
  - name: common
    repository: https://charts.bitnami.com/bitnami
    version: 1.x.x
  - name: clusterpedia-core
    repository: https://clusterpedia-io.github.io/clusterpedia-helm/
    version: 0.1.x

# values.yaml
clusterpedia-core:
  storage:
    name: "sample-storage-layer"
    image:
      registry: "ghcr.io"
      repository: "clusterpedia-io/clusterpedia/sample-storage-layer"
      tag: "v0.0.0-v0.6.0"

# clusterpedia-core/templates/_storage_override.yaml
{{- define "clusterpedia.storage.override.initContainers" -}}
{{- end -}}

{{- define "clusterpedia.storage.override.configmap.name" -}}
{{- end -}}

{{- define "clusterpedia.storage.override.componentEnv" -}}
{{- end -}}

# _storage_override.yaml
{{- define "clusterpedia.storage.override.initContainers" -}}
- name: ensure-database
  image: docker.io/bitnami/mysql:8.0.28-debian-10-r23
  command:
  - /bin/sh
  - -ec
  - |
    if [ ${CREARE_DATABASE} = "ture" ]; then
      until mysql -u${STORAGE_USER} -p${DB_PASSWORD} --host=${STORAGE_HOST} --port=${STORAGE_PORT} -e 'CREATE DATABASE IF NOT EXISTS ${STORAGE_DATABASE}'; do
      echo waiting for database check && sleep 1;
      done;
      echo 'DataBase OK ✓'
    else
      until mysqladmin status -u${STORAGE_USER} -p${DB_PASSWORD} --host=${STORAGE_HOST} --port=${STORAGE_PORT}; do sleep 1; done
    fi    
  env:
  - name: DB_PASSWORD
    valueFrom:
      secretKeyRef:
        name: {{ include "clusterpedia.mysql.storage.fullname" . }}
        key: password
  envFrom:
  - configMapRef:
      name: {{ include "clusterpedia.mysql.storage.initContainer.env.name" . }}
{{- end -}}

apiVersion: v1
kind: ConfigMap
metadata:
  name: {{ include "clusterpedia.mysql.storage.initContainer.env.name" . }}
  namespace: {{ .Release.Namespace }}
  labels: {{ include "common.labels.standard" . | nindent 4 }}
data:
  STORAGE_HOST: {{ include "clusterpedia.mysql.storage.host" . | quote }}
  STORAGE_PORT: {{ include "clusterpedia.mysql.storage.port" . | quote }}
  STORAGE_USER: {{ include "clusterpedia.mysql.storage.user" . | quote }}
  STORAGE_DATABASE: {{ include "clusterpedia.mysql.storage.database" . | quote }}
  CREARE_DATABASE: {{ .Values.externalStorage.createDatabase | quote }}

# helm template clusterpedia -n clusterpedia-system --set persistenceMatchNode=None .
...
    spec:
      initContainers:
      - name: ensure-database
        image: docker.io/bitnami/mysql:8.0.28-debian-10-r23
        command:
        - /bin/sh
        - -ec
        - |
          if [ ${CREARE_DATABASE} = "ture" ]; then
            until mysql -u${STORAGE_USER} -p${DB_PASSWORD} --host=${STORAGE_HOST} --port=${STORAGE_PORT} -e 'CREATE DATABASE IF NOT EXISTS ${STORAGE_DATABASE}'; do
            echo waiting for database check && sleep 1;
            done;
            echo 'DataBase OK ✓'
          else
            until mysqladmin status -u${STORAGE_USER} -p${DB_PASSWORD} --host=${STORAGE_HOST} --port=${STORAGE_PORT}; do sleep 1; done
          fi          
        env:
        - name: DB_PASSWORD
          valueFrom:
            secretKeyRef:
              name: clusterpedia-mysql-storage
              key: password
        envFrom:
        - configMapRef:
            name: clusterpedia-mysql-storage-initcontainer-env

# _storage_override.yaml

{{- define "clusterpedia.storage.override.configmap.name" -}}
{{- printf "%s-mysql-storage-config" .Release.Name -}}
{{- end -}}

{{- define "clusterpedia.storage.override.componentEnv" -}}
- name: DB_PASSWORD
  valueFrom:
    secretKeyRef:
      name: {{ include "clusterpedia.mysql.storage.fullname" . }}
      key: password
{{- end -}}

# 忽略其他参数
./bin/clustersynchro-manager --feature-gates=SyncAllResources=true

kubectl get --raw="/apis/clusterpedia.io/v1beta1/resources/apis/apps/v1/deployments?withRemainingCount=false&limit=1" | jq

URL="/apis/clusterpedia.io/v1beta1/resources/apis/apps/v1/deployments"
kubectl get --raw="$URL?whereSQL=(cluster='global') OR (namespace IN ('kube-system','default'))"

spec:
  syncAllCustomResources: true

spec:
  syncResources:
  - group: "*"
    resources:
    - "*"