.github/pull_request_template.md

@@ -0,0 +1,22 @@
+<!--
+Thanks for creating a pull request!
+
+If this is your first time, please make sure to review CONTRIBUTING.MD.
+
+Please copy the appropriate `:text:` or icon to the beginning of your PR title:
+
+:sparkles: ✨ feature
+:bug: 🐛 bug fix
+:book: 📖 docs
+:memo: 📝 proposal
+:warning: ⚠️ breaking change
+:rocket: 🚀 release
+:seedling: 🌱 other/misc
+:question: ❓ requires manual review/categorization
+
+-->
+## Summary
+
+## Related issue(s)
+
+Fixes #

.github/workflows/cloudevents-integration.yml

@@ -0,0 +1,32 @@
+name: CloudEventsIntegration
+
+on:
+  workflow_dispatch: {}
+  pull_request:
+    paths:
+     - 'pkg/addonmanager/cloudevents/*.go'
+     - 'test/integration/cloudevents/**'
+    branches:
+      - main
+      - release-*
+
+env:
+  GO_VERSION: '1.21'
+  GO_REQUIRED_MIN_VERSION: ''
+
+permissions:
+  contents: read
+
+jobs:
+  integration:
+    name: cloudevents-integration
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v4
+      - name: install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: integration
+        run: make test-cloudevents-integration

.github/workflows/go-postsubmit.yml

@@ -24,7 +24,7 @@ jobs:
         arch: [ amd64, arm64 ]
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
           path: go/src/open-cluster-management.io/addon-framework
@@ -51,7 +51,7 @@ jobs:
     needs: [ images ]
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
           path: go/src/open-cluster-management.io/addon-framework

.github/workflows/go-presubmit.yml

@@ -11,21 +11,16 @@ env:
   # Common versions
   GO_VERSION: '1.21'
   GO_REQUIRED_MIN_VERSION: ''
-  GOPATH: '/home/runner/work/addon-framework/addon-framework/go'
-defaults:
-  run:
-    working-directory: go/src/open-cluster-management.io/addon-framework
 
 jobs:
   verify:
     name: verify
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          path: go/src/open-cluster-management.io/addon-framework
       - name: install Go
         uses: actions/setup-go@v3
         with:
@@ -38,10 +33,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          path: go/src/open-cluster-management.io/addon-framework
       - name: install Go
         uses: actions/setup-go@v3
         with:
@@ -54,10 +48,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          path: go/src/open-cluster-management.io/addon-framework
       - name: install Go
         uses: actions/setup-go@v3
         with:
@@ -70,10 +63,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          path: go/src/open-cluster-management.io/addon-framework
       - name: install Go
         uses: actions/setup-go@v3
         with:
@@ -86,10 +78,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          path: go/src/open-cluster-management.io/addon-framework
       - name: install Go
         uses: actions/setup-go@v3
         with:
@@ -102,10 +93,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          path: go/src/open-cluster-management.io/addon-framework
       - name: install Go
         uses: actions/setup-go@v3
         with:
@@ -129,15 +119,45 @@ jobs:
         env:
           KUBECONFIG: /home/runner/.kube/config
 
+  e2e-cloudevents:
+    name: e2e-cloudevents
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: install imagebuilder
+        run: go install github.com/openshift/imagebuilder/cmd/imagebuilder@v1.2.3
+      - name: addon-examples-image # will build and tag the examples image
+        run: imagebuilder --allow-pull -t quay.io/open-cluster-management/addon-examples:latest -t quay.io/ocm/addon-examples:latest -f ./build/Dockerfile.example .
+      - name: setup kind
+        uses: engineerd/setup-kind@v0.5.0
+        with:
+          version: v0.11.1
+          name: cluster1
+      - name: Load image on the nodes of the cluster
+        run: |
+          kind load docker-image --name=cluster1 quay.io/open-cluster-management/addon-examples:latest
+          kind load docker-image --name=cluster1 quay.io/ocm/addon-examples:latest
+      - name: Run e2e test with cloudevents
+        run: |
+          make test-e2e-cloudevents
+        env:
+          KUBECONFIG: /home/runner/.kube/config
+
   e2e-hosted:
     name: e2e-hosted
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          path: go/src/open-cluster-management.io/addon-framework
       - name: install Go
         uses: actions/setup-go@v3
         with:

.github/workflows/go-release.yml

@@ -8,23 +8,17 @@ env:
   # Common versions
   GO_VERSION: '1.21'
   GO_REQUIRED_MIN_VERSION: ''
-  GOPATH: '/home/runner/work/addon-framework/addon-framework/go'
   GITHUB_REF: ${{ github.ref }}
 
-defaults:
-  run:
-    working-directory: go/src/open-cluster-management.io/addon-framework
-
 jobs:
   env:
     name: prepare release env
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          path: go/src/open-cluster-management.io/addon-framework
       - name: get release version
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
@@ -44,10 +38,9 @@ jobs:
         arch: [ amd64, arm64 ]
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          path: go/src/open-cluster-management.io/addon-framework
       - name: install Go
         uses: actions/setup-go@v3
         with:
@@ -71,10 +64,9 @@ jobs:
     needs: [ env, images ]
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          path: go/src/open-cluster-management.io/addon-framework
       - name: create
         run: |
           echo ${{ secrets.DOCKER_PASSWORD }} | docker login quay.io --username ${{ secrets.DOCKER_USER }} --password-stdin
@@ -96,10 +88,9 @@ jobs:
     needs: [ env, image-manifest ]
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          path: go/src/open-cluster-management.io/addon-framework
       - name: generate changelog
         run: |
           echo "# Addon Framework ${{ needs.env.outputs.RELEASE_VERSION }}" > /home/runner/work/changelog.txt

.github/workflows/pr-verify.yml

@@ -0,0 +1,26 @@
+name: PR Verifier
+
+on:
+  # NB: using `pull_request_target` runs this in the context of
+  # the base repository, so it has permission to upload to the checks API.
+  # This means changes won't kick in to this file until merged onto the
+  # main branch.
+  pull_request_target:
+    types: [opened, edited, reopened, synchronize]
+
+permissions:
+  contents: read
+
+jobs:
+  verify:
+    name: verify PR contents
+    permissions:
+      checks: write
+      pull-requests: read
+    runs-on: ubuntu-latest
+    steps:
+      - name: Verifier action
+        id: verifier
+        uses: kubernetes-sigs/kubebuilder-release-tools@v0.4.3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}

Makefile

@@ -56,6 +56,9 @@ verify: verify-gocilint
 deploy-ocm:
 	examples/deploy/ocm/install.sh
 
+deploy-ocm-cloudevents:
+	examples/deploy/ocm-cloudevents/install.sh
+
 deploy-hosted-ocm:
 	examples/deploy/hosted-ocm/install.sh
 
@@ -71,6 +74,14 @@ deploy-helloworld: ensure-kustomize
 	$(KUSTOMIZE) build examples/deploy/addon/helloworld | $(KUBECTL) apply -f -
 	mv examples/deploy/addon/helloworld/kustomization.yaml.tmp examples/deploy/addon/helloworld/kustomization.yaml
 
+deploy-helloworld-cloudevents: ensure-kustomize
+	cp examples/deploy/addon/helloworld_cloudevents/kustomization.yaml examples/deploy/addon/helloworld_cloudevents/kustomization.yaml.tmp
+	cp -r examples/deploy/addon/helloworld/resources examples/deploy/addon/helloworld_cloudevents/
+	cd examples/deploy/addon/helloworld_cloudevents && ../../../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/addon-examples=$(EXAMPLE_IMAGE_NAME)
+	$(KUSTOMIZE) build examples/deploy/addon/helloworld_cloudevents | $(KUBECTL) apply -f -
+	mv examples/deploy/addon/helloworld_cloudevents/kustomization.yaml.tmp examples/deploy/addon/helloworld_cloudevents/kustomization.yaml
+	rm -rf examples/deploy/addon/helloworld_cloudevents/resources
+
 deploy-helloworld-helm: ensure-kustomize
 	cp examples/deploy/addon/helloworld-helm/kustomization.yaml examples/deploy/addon/helloworld-helm/kustomization.yaml.tmp
 	cd examples/deploy/addon/helloworld-helm && ../../../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/addon-examples=$(EXAMPLE_IMAGE_NAME)
@@ -111,6 +122,9 @@ undeploy-busybox: ensure-kustomize
 undeploy-helloworld: ensure-kustomize
 	$(KUSTOMIZE) build examples/deploy/addon/helloworld | $(KUBECTL) delete --ignore-not-found -f -
 
+undeploy-helloworld-cloudevents: ensure-kustomize
+	$(KUSTOMIZE) build examples/deploy/addon/helloworld-cloudevents | $(KUBECTL) delete --ignore-not-found -f -
+
 undeploy-helloworld-helm: ensure-kustomize
 	$(KUSTOMIZE) build examples/deploy/addon/helloworld-helm | $(KUBECTL) delete --ignore-not-found -f -
 
@@ -129,6 +143,9 @@ build-e2e:
 test-e2e: build-e2e deploy-ocm deploy-helloworld deploy-helloworld-helm
 	./e2e.test -test.v -ginkgo.v
 
+test-e2e-cloudevents: build-e2e deploy-ocm-cloudevents deploy-helloworld-cloudevents
+	./e2e.test -test.v -ginkgo.v -ginkgo.focus="install/uninstall helloworld addons"
+
 build-hosted-e2e:
 	go test -c ./test/e2ehosted
 

README.md

@@ -2,8 +2,6 @@
 
 This is to define an AddOn framework library.
 
-Still in PoC phase.
-
 ## Community, discussion, contribution, and support
 
 Check the [CONTRIBUTING Doc](CONTRIBUTING.md) for how to contribute to the repo.
@@ -36,6 +34,13 @@ So developers could be able to develop their own addon functions more easily.
 
 * [managed-serviceaccount](https://github.com/open-cluster-management-io/managed-serviceaccount)
 
+## AddOn Contributions
+
+A [collection of OCM addons](https://github.com/open-cluster-management-io/addon-contrib)
+for staging and testing PoC purposes:
+
+AI integration, IoT layer, cluster proxy, telemetry, resource usage collection and more.
+
 ## AddOn Examples 
 
 We have examples to implement AddOn using Helm Chart or Go Template. You can find more details in [examples](examples/README.md).

cmd/example/helloworld/main.go

@@ -21,7 +21,7 @@ import (
 	"open-cluster-management.io/addon-framework/examples/helloworld_agent"
 	"open-cluster-management.io/addon-framework/pkg/addonfactory"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
-	addonagent "open-cluster-management.io/addon-framework/pkg/agent"
+	"open-cluster-management.io/addon-framework/pkg/addonmanager/cloudevents"
 	cmdfactory "open-cluster-management.io/addon-framework/pkg/cmd/factory"
 	"open-cluster-management.io/addon-framework/pkg/utils"
 	"open-cluster-management.io/addon-framework/pkg/version"
@@ -67,24 +67,40 @@ func newCommand() *cobra.Command {
 }
 
 func newControllerCommand() *cobra.Command {
+	o := cloudevents.NewCloudEventsOptions()
+	c := &addManagerConfig{cloudeventsOptions: o}
 	cmd := cmdfactory.
-		NewControllerCommandConfig("helloworld-addon-controller", version.Get(), runController).
+		NewControllerCommandConfig("helloworld-addon-controller", version.Get(), c.runController).
 		NewCommand()
 	cmd.Use = "controller"
 	cmd.Short = "Start the addon controller"
+	o.AddFlags(cmd)
 
 	return cmd
 }
 
-func runController(ctx context.Context, kubeConfig *rest.Config) error {
+// addManagerConfig holds cloudevents configuration for addon manager
+type addManagerConfig struct {
+	cloudeventsOptions *cloudevents.CloudEventsOptions
+}
+
+func (c *addManagerConfig) runController(ctx context.Context, kubeConfig *rest.Config) error {
 	addonClient, err := addonv1alpha1client.NewForConfig(kubeConfig)
 	if err != nil {
 		return err
 	}
 
-	mgr, err := addonmanager.New(kubeConfig)
-	if err != nil {
-		return err
+	var mgr addonmanager.AddonManager
+	if c.cloudeventsOptions.WorkDriver == "kube" {
+		mgr, err = addonmanager.New(kubeConfig)
+		if err != nil {
+			return err
+		}
+	} else {
+		mgr, err = cloudevents.New(kubeConfig, c.cloudeventsOptions)
+		if err != nil {
+			return err
+		}
 	}
 
 	registrationOption := helloworld.NewRegistrationOption(
@@ -109,7 +125,11 @@ func runController(ctx context.Context, kubeConfig *rest.Config) error {
 			),
 		).
 		WithAgentRegistrationOption(registrationOption).
-		WithInstallStrategy(addonagent.InstallAllStrategy(helloworld.InstallationNamespace)).
+		WithAgentInstallNamespace(
+			utils.AgentInstallNamespaceFromDeploymentConfigFunc(
+				utils.NewAddOnDeploymentConfigGetter(addonClient),
+			),
+		).
 		WithAgentHealthProber(helloworld.AgentHealthProber()).
 		BuildTemplateAgentAddon()
 	if err != nil {

cmd/example/helloworld_helm/main.go

@@ -115,12 +115,12 @@ func runController(ctx context.Context, kubeConfig *rest.Config) error {
 		WithGetValuesFuncs(
 			helloworld_helm.GetDefaultValues,
 			addonfactory.GetAddOnDeploymentConfigValues(
-				addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
+				utils.NewAddOnDeploymentConfigGetter(addonClient),
 				addonfactory.ToAddOnNodePlacementValues,
 				addonfactory.ToAddOnProxyConfigValues,
 			),
 			addonfactory.GetAgentImageValues(
-				addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
+				utils.NewAddOnDeploymentConfigGetter(addonClient),
 				"global.imageOverrides.helloWorldHelm",
 				helloworld.DefaultHelloWorldExampleImage,
 			),

cmd/example/helloworld_hosted/main.go

@@ -23,6 +23,7 @@ import (
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
 	cmdfactory "open-cluster-management.io/addon-framework/pkg/cmd/factory"
 	"open-cluster-management.io/addon-framework/pkg/version"
+	clusterclientset "open-cluster-management.io/api/client/cluster/clientset/versioned"
 )
 
 func main() {
@@ -85,11 +86,16 @@ func runController(ctx context.Context, kubeConfig *rest.Config) error {
 		helloworld_hosted.AddonName,
 		utilrand.String(5))
 
+	clusterClient, err := clusterclientset.NewForConfig(kubeConfig)
+	if err != nil {
+		return err
+	}
 	agentAddon, err := addonfactory.NewAgentAddonFactory(
 		helloworld_hosted.AddonName, helloworld_hosted.FS, "manifests/templates").
 		WithGetValuesFuncs(helloworld.GetDefaultValues, addonfactory.GetValuesFromAddonAnnotation).
 		WithAgentRegistrationOption(registrationOption).
 		WithAgentHostedModeEnabledOption().
+		WithManagedClusterClient(clusterClient).
 		BuildTemplateAgentAddon()
 	if err != nil {
 		klog.Errorf("failed to build agent %v", err)

examples/deploy/addon/helloworld-template/resources/cluster_management_addon.yaml

@@ -2,8 +2,6 @@ apiVersion: addon.open-cluster-management.io/v1alpha1
 kind: ClusterManagementAddOn
 metadata:
   name: hello-template
-  annotations:
-    addon.open-cluster-management.io/lifecycle: "addon-manager"
 spec:
   addOnMeta:
     description: hello-template

examples/deploy/addon/helloworld/kustomization.yaml

@@ -4,10 +4,12 @@ resources:
 - resources/cluster_role.yaml
 - resources/cluster_role_binding.yaml
 - resources/service_account.yaml
+- resources/managed_clusterset_binding.yaml
+- resources/placement.yaml
+- resources/addon_deployment_config.yaml
 - resources/helloworld_clustermanagementaddon.yaml
 - resources/helloworld_controller.yaml
 
-
 images:
 - name: quay.io/open-cluster-management/addon-examples
   newName: quay.io/open-cluster-management/addon-examples

examples/deploy/addon/helloworld/resources/addon_deployment_config.yaml

@@ -0,0 +1,6 @@
+apiVersion: addon.open-cluster-management.io/v1alpha1
+kind: AddOnDeploymentConfig
+metadata:
+  name: global
+spec:
+  agentInstallNamespace: open-cluster-management-agent-addon

examples/deploy/addon/helloworld/resources/helloworld_clustermanagementaddon.yaml

@@ -9,3 +9,13 @@ spec:
   supportedConfigs:
   - group: addon.open-cluster-management.io
     resource: addondeploymentconfigs
+  installStrategy:
+    type: Placements
+    placements:
+      - name: global
+        namespace: open-cluster-management
+        configs:
+          - group: addon.open-cluster-management.io
+            resource: addondeploymentconfigs
+            name: global
+            namespace: open-cluster-management

examples/deploy/addon/helloworld/resources/managed_clusterset_binding.yaml

@@ -0,0 +1,6 @@
+apiVersion: cluster.open-cluster-management.io/v1beta2
+kind: ManagedClusterSetBinding
+metadata:
+  name: global
+spec:
+  clusterSet: global

examples/deploy/addon/helloworld/resources/placement.yaml

@@ -0,0 +1,10 @@
+apiVersion: cluster.open-cluster-management.io/v1beta1
+kind: Placement
+metadata:
+  name: global
+spec:
+  clusterSets:
+    - global
+  predicates:
+    - requiredClusterSelector:
+        labelSelector: {}

examples/deploy/addon/helloworld_cloudevents/config.yaml

@@ -0,0 +1,5 @@
+brokerHost: mosquitto.mqtt:1883
+topics:
+  sourceEvents: sources/addon-manager/clusters/+/sourceevents
+  agentEvents: sources/addon-manager/clusters/+/agentevents
+  sourceBroadcast: sources/addon-manager/sourcebroadcast

examples/deploy/addon/helloworld_cloudevents/kustomization.yaml

@@ -0,0 +1,38 @@
+namespace: open-cluster-management
+
+resources:
+- resources/cluster_role.yaml
+- resources/cluster_role_binding.yaml
+- resources/service_account.yaml
+- resources/managed_clusterset_binding.yaml
+- resources/placement.yaml
+- resources/addon_deployment_config.yaml
+- resources/helloworld_clustermanagementaddon.yaml
+- resources/helloworld_controller.yaml
+
+images:
+- name: quay.io/open-cluster-management/addon-examples
+  newName: quay.io/open-cluster-management/addon-examples
+  newTag: latest
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+vars:
+- fieldref:
+    fieldPath: spec.template.spec.containers.[name=helloworld-controller].image
+  name: EXAMPLE_IMAGE_NAME
+  objref:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: helloworld-controller
+
+secretGenerator:
+- files:
+  - config.yaml
+  name: work-driver-config
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+patches:
+- path: work_driver_config_patch.yaml

examples/deploy/addon/helloworld_cloudevents/work_driver_config_patch.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: helloworld-controller
+spec:
+  template:
+    spec:
+      containers:
+      - name: helloworld-controller
+        args:
+        - "/helloworld"
+        - "controller"
+        - "--work-driver=mqtt"
+        - "--work-driver-config=/var/run/secrets/hub/config.yaml"
+        - "--cloudevents-client-id=addon-manager-$(POD_NAME)"
+        - "--source-id=addon-manager"
+        volumeMounts:
+        - mountPath: /var/run/secrets/hub
+          name: workdriverconfig
+          readOnly: true
+      volumes:
+      - name: workdriverconfig
+        secret:
+          secretName: work-driver-config

examples/deploy/addon/kubernetes-dashboard/resources/cluster_management_addon.yaml

@@ -2,8 +2,6 @@ apiVersion: addon.open-cluster-management.io/v1alpha1
 kind: ClusterManagementAddOn
 metadata:
   name: kubernetes-dashboard
-  annotations:
-    addon.open-cluster-management.io/lifecycle: "addon-manager"
 spec:
   addOnMeta:
     description: kubernetes-dashboard

examples/deploy/hosted-ocm/install.sh

@@ -20,16 +20,6 @@ function wait_deployment() {
   set -e
 }
 
-function hub_approve_cluster(){
-  local cluster_name=$1
-  echo "approve ${cluster_name}"
-  ${KUBECTL} get csr -l open-cluster-management.io/cluster-name=${cluster_name} | \
-    grep -v NAME | awk '{print $1}' | xargs ${KUBECTL} certificate approve
-  ${KUBECTL} patch managedcluster "${cluster_name}" -p='{"spec":{"hubAcceptsClient":true}}' --type=merge
-  ${KUBECTL} get managedcluster "${cluster_name}"
-}
-
-
 BUILD_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 DEPLOY_DIR="$(dirname "$BUILD_DIR")"
 EXAMPLE_DIR="$(dirname "$DEPLOY_DIR")"
@@ -64,6 +54,15 @@ curl -s -f \
   -o "${KUBECTL}"
 chmod +x "${KUBECTL}"
 
+CLUSTERADM="${WORK_DIR}/bin/clusteradm"
+
+export PATH=$PATH:${WORK_DIR}/bin
+
+echo "############  Download clusteradm"
+mkdir -p "${WORK_DIR}/bin"
+wget -qO- https://github.com/open-cluster-management-io/clusteradm/releases/latest/download/clusteradm_${GOHOSTOS}_${GOHOSTARCH}.tar.gz | sudo tar -xvz -C ${WORK_DIR}/bin/
+chmod +x "${CLUSTERADM}"
+
 
 echo "###### installing e2e test cluster : ${REPO_DIR}/.kubeconfig"
 ${KIND} delete cluster --name ${MANAGED_CLUSTER_NAME}
@@ -78,35 +77,10 @@ echo "###### loading image: ${EXAMPLE_IMAGE_NAME}"
 ${KIND} load docker-image ${EXAMPLE_IMAGE_NAME} --name ${MANAGED_CLUSTER_NAME}
 
 echo "###### deploy operators"
-rm -rf "$WORK_DIR/_repo_ocm"
-git clone --depth 1 --branch main https://github.com/open-cluster-management-io/ocm.git "$WORK_DIR/_repo_ocm"
-
-${KUBECTL} apply -k "$WORK_DIR/_repo_ocm/deploy/cluster-manager/config/manifests"
-${KUBECTL} apply -k "$WORK_DIR/_repo_ocm/deploy/cluster-manager/config/samples"
-${KUBECTL} apply -k "$WORK_DIR/_repo_ocm/deploy/klusterlet/config/manifests"
-rm -rf "$WORK_DIR/_repo_ocm"
+${CLUSTERADM} init --wait --bundle-version=latest
+joincmd=$(${CLUSTERADM} get token | grep clusteradm)
 
-${KUBECTL} get ns open-cluster-management-agent || ${KUBECTL} create ns open-cluster-management-agent
-
-cat << EOF | ${KUBECTL} apply -f -
-apiVersion: operator.open-cluster-management.io/v1
-kind: Klusterlet
-metadata:
-  name: klusterlet
-spec:
-  deployOption:
-    mode: Default
-  registrationImagePullSpec: quay.io/open-cluster-management/registration
-  workImagePullSpec: quay.io/open-cluster-management/work
-  clusterName: ${MANAGED_CLUSTER_NAME}
-  namespace: open-cluster-management-agent
-  externalServerURLs:
-  - url: https://localhost
-  registrationConfiguration:
-    featureGates:
-    - feature: AddonManagement
-      mode: Enable
-EOF
+$(echo ${joincmd} --force-internal-endpoint-lookup --wait --bundle-version=latest | sed "s/<cluster_name>/${MANAGED_CLUSTER_NAME}/g")
 
 wait_deployment open-cluster-management cluster-manager
 ${KUBECTL} -n open-cluster-management rollout status deploy cluster-manager --timeout=120s
@@ -127,19 +101,12 @@ ${KUBECTL} -n open-cluster-management-hub scale --replicas=0 deployment/cluster-
 # scale replicas to save resources
 ${KUBECTL} -n open-cluster-management scale --replicas=1 deployment/klusterlet
 
-wait_deployment open-cluster-management-agent klusterlet-registration-agent
+${CLUSTERADM} accept --clusters ${MANAGED_CLUSTER_NAME} --wait
+
 echo "###### prepare bootstrap hub secret"
 cp "${KUBECONFIG}" "${WORK_DIR}"/e2e-kubeconfig
 ${KUBECTL} config set "clusters.${cluster_context}.server" "https://${cluster_ip}" \
   --kubeconfig "${WORK_DIR}"/e2e-kubeconfig
-${KUBECTL} delete secret bootstrap-hub-kubeconfig -n open-cluster-management-agent --ignore-not-found
-${KUBECTL} create secret generic bootstrap-hub-kubeconfig \
-  --from-file=kubeconfig="${WORK_DIR}"/e2e-kubeconfig \
-  -n open-cluster-management-agent
-${KUBECTL} -n open-cluster-management-agent rollout status deploy klusterlet-registration-agent --timeout=120s
-${KUBECTL} -n open-cluster-management-agent rollout status deploy klusterlet-work-agent --timeout=120s
-
-hub_approve_cluster ${MANAGED_CLUSTER_NAME}
 
 # prepare another managed cluster for hosted mode testing
 echo "###### installing e2e test managed cluster"
@@ -192,22 +159,13 @@ ${KUBECTL} create secret generic external-managed-kubeconfig \
   --from-file=kubeconfig="${WORK_DIR}"/e2e-managed-kubeconfig \
   -n ${HOSTED_MANAGED_KLUSTERLET_NAME}
 
-
 ${KUBECTL} delete secret ${HOSTED_MANAGED_KUBECONFIG_SECRET_NAME} \
   -n ${HOSTED_MANAGED_KLUSTERLET_NAME} --ignore-not-found
 ${KUBECTL} create secret generic ${HOSTED_MANAGED_KUBECONFIG_SECRET_NAME} \
   --from-file=kubeconfig="${WORK_DIR}"/e2e-managed-kubeconfig-public \
   -n ${HOSTED_MANAGED_KLUSTERLET_NAME}
 
-wait_deployment ${HOSTED_MANAGED_KLUSTERLET_NAME} ${HOSTED_MANAGED_KLUSTERLET_NAME}-registration-agent
-wait_deployment ${HOSTED_MANAGED_KLUSTERLET_NAME} ${HOSTED_MANAGED_KLUSTERLET_NAME}-work-agent
-
-${KUBECTL} -n ${HOSTED_MANAGED_KLUSTERLET_NAME} rollout status deploy \
-  ${HOSTED_MANAGED_KLUSTERLET_NAME}-registration-agent --timeout=120s
-${KUBECTL} -n ${HOSTED_MANAGED_KLUSTERLET_NAME} rollout status deploy \
-  ${HOSTED_MANAGED_KLUSTERLET_NAME}-work-agent --timeout=120s
-
-hub_approve_cluster ${HOSTED_MANAGED_CLUSTER_NAME}
+${CLUSTERADM} accept --clusters ${HOSTED_MANAGED_CLUSTER_NAME} --wait --skip-approve-check
 
 ${KUBECTL} wait --for=condition=ManagedClusterConditionAvailable=true \
   managedcluster/${MANAGED_CLUSTER_NAME} --timeout=120s

examples/deploy/mqtt/mqtt-broker.yaml

@@ -0,0 +1,68 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: mqtt
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mosquitto
+  namespace: mqtt
+spec:
+  ports:
+  - name: mosquitto
+    protocol: TCP
+    port: 1883
+    targetPort: 1883
+  selector:
+    name: mosquitto
+  sessionAffinity: None
+  type: ClusterIP
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: mosquitto
+  namespace: mqtt
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: mosquitto
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        name: mosquitto
+    spec:
+      containers:
+      - name: mosquitto
+        image: eclipse-mosquitto:2.0.18
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 1883
+          name: mosquitto
+        volumeMounts:
+        - name: mosquitto-persistent-storage
+          mountPath: /mosquitto/data
+        - name: mosquitto-config
+          mountPath: /mosquitto/config/mosquitto.conf
+          subPath: mosquitto.conf
+      volumes:
+      - name: mosquitto-persistent-storage
+        emptyDir: {}
+      - name: mosquitto-config
+        configMap:
+          name: mosquitto
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mosquitto
+  namespace: mqtt
+data:
+  mosquitto.conf: |
+    listener 1883 0.0.0.0
+    allow_anonymous true

examples/deploy/ocm-cloudevents/install.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+set -o nounset
+set -o pipefail
+
+KUBECTL=${KUBECTL:-kubectl}
+
+BUILD_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+DEPLOY_DIR="$(dirname "$BUILD_DIR")"
+EXAMPLE_DIR="$(dirname "$DEPLOY_DIR")"
+REPO_DIR="$(dirname "$EXAMPLE_DIR")"
+WORK_DIR="${REPO_DIR}/_output"
+CLUSTERADM="${WORK_DIR}/bin/clusteradm"
+
+export PATH=$PATH:${WORK_DIR}/bin
+
+echo "############  Download clusteradm"
+mkdir -p "${WORK_DIR}/bin"
+wget -qO- https://github.com/open-cluster-management-io/clusteradm/releases/latest/download/clusteradm_${GOHOSTOS}_${GOHOSTARCH}.tar.gz | sudo tar -xvz -C ${WORK_DIR}/bin/
+chmod +x "${CLUSTERADM}"
+
+echo "############ Init hub"
+${CLUSTERADM} init --wait --bundle-version=latest
+joincmd=$(${CLUSTERADM} get token | grep clusteradm)
+
+echo "############ Init agent as cluster1"
+$(echo ${joincmd} --force-internal-endpoint-lookup --wait --bundle-version=latest | sed "s/<cluster_name>/${MANAGED_CLUSTER_NAME}/g")
+
+echo "############ Accept join of cluster1"
+${CLUSTERADM} accept --clusters ${MANAGED_CLUSTER_NAME} --wait
+
+echo "############  All-in-one env is installed successfully!!"
+
+echo "############  Deploy mqtt broker"
+${KUBECTL} apply -f ${DEPLOY_DIR}/mqtt/mqtt-broker.yaml
+
+echo "############  Configure the work-agent"
+${KUBECTL} -n open-cluster-management scale --replicas=0 deployment/klusterlet
+
+cat << EOF | ${KUBECTL} -n open-cluster-management-agent apply -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: work-driver-config
+stringData:
+  config.yaml: |
+    brokerHost: mosquitto.mqtt:1883
+    topics:
+      sourceEvents: sources/addon-manager/clusters/${MANAGED_CLUSTER_NAME}/sourceevents
+      agentEvents: sources/addon-manager/clusters/${MANAGED_CLUSTER_NAME}/agentevents
+      agentBroadcast: clusters/${MANAGED_CLUSTER_NAME}/agentbroadcast
+EOF
+
+# patch klusterlet-work-agent deployment to use mqtt as workload source driver
+${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \
+    -p='[
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--cloudevents-client-codecs=manifestbundle"},
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--cloudevents-client-id=work-agent-$(POD_NAME)"},
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--workload-source-driver=mqtt"},
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--workload-source-config=/var/run/secrets/hub/config.yaml"}
+    ]'
+
+${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \
+    -p='[{"op": "add", "path": "/spec/template/spec/volumes/-", "value": {"name": "workdriverconfig","secret": {"secretName": "work-driver-config"}}}]'
+
+${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \
+    -p='[{"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", "value": {"name": "workdriverconfig","mountPath": "/var/run/secrets/hub"}}]'
+
+${KUBECTL} -n open-cluster-management-agent scale --replicas=1 deployment/klusterlet-work-agent
+${KUBECTL} -n open-cluster-management-agent rollout status deployment/klusterlet-work-agent --timeout=120s
+${KUBECTL} -n open-cluster-management-agent get pod -l app=klusterlet-manifestwork-agent
+
+# TODO: add live probe for the work-agent to check if it is connected to the mqtt broker
+isRunning=false
+for i in {1..20}; do
+    if ${KUBECTL} -n open-cluster-management-agent logs deployment/klusterlet-work-agent | grep "subscribing to topics"; then
+        echo "klusterlet-work-agent is subscribing to topics from mqtt broker"
+        isRunning=true
+        break
+    fi
+    sleep 12
+done
+
+if [ "$isRunning" = false ]; then
+    echo "timeout waiting for klusterlet-work-agent to subscribe to topics from mqtt broker"
+    exit 1
+fi

examples/deploy/ocm/install.sh

@@ -5,73 +5,29 @@ set -o pipefail
 
 KUBECTL=${KUBECTL:-kubectl}
 
-rm -rf _repo_ocm
 
-echo "############  Cloning ocm repo"
-git clone --depth 1 --branch main https://github.com/open-cluster-management-io/ocm.git _repo_ocm
+BUILD_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+DEPLOY_DIR="$(dirname "$BUILD_DIR")"
+EXAMPLE_DIR="$(dirname "$DEPLOY_DIR")"
+REPO_DIR="$(dirname "$EXAMPLE_DIR")"
+WORK_DIR="${REPO_DIR}/_output"
+CLUSTERADM="${WORK_DIR}/bin/clusteradm"
 
-cd _repo_ocm || {
-  printf "cd failed, _repo_ocm does not exist"
-  return 1
-}
+export PATH=$PATH:${WORK_DIR}/bin
 
-echo "############  Deploying operators"
-make deploy-hub cluster-ip deploy-spoke-operator apply-spoke-cr
-if [ $? -ne 0 ]; then
- echo "############  Failed to deploy"
- exit 1
-fi
+echo "############  Download clusteradm"
+mkdir -p "${WORK_DIR}/bin"
+wget -qO- https://github.com/open-cluster-management-io/clusteradm/releases/latest/download/clusteradm_${GOHOSTOS}_${GOHOSTARCH}.tar.gz | sudo tar -xvz -C ${WORK_DIR}/bin/
+chmod +x "${CLUSTERADM}"
 
-for i in {1..7}; do
-  echo "############$i  Checking cluster-manager-registration-controller"
-  RUNNING_POD=$($KUBECTL -n open-cluster-management-hub get pods | grep cluster-manager-registration-controller | grep -c "Running")
-  if [ "${RUNNING_POD}" -ge 1 ]; then
-    break
-  fi
+echo "############ Init hub"
+${CLUSTERADM} init --wait --bundle-version=latest
+joincmd=$(${CLUSTERADM} get token | grep clusteradm)
 
-  if [ $i -eq 7 ]; then
-    echo "!!!!!!!!!!  the cluster-manager-registration-controller is not ready within 3 minutes"
-    $KUBECTL -n open-cluster-management-hub get pods
+echo "############ Init agent as cluster1"
+$(echo ${joincmd} --force-internal-endpoint-lookup --wait --bundle-version=latest | sed "s/<cluster_name>/${MANAGED_CLUSTER_NAME}/g")
 
-    exit 1
-  fi
-  sleep 30
-done
-
-for i in {1..7}; do
-  echo "############$i  Checking cluster-manager-registration-webhook"
-  RUNNING_POD=$($KUBECTL -n open-cluster-management-hub get pods | grep cluster-manager-registration-webhook | grep -c "Running")
-  if [ "${RUNNING_POD}" -ge 1 ]; then
-    break
-  fi
-
-  if [ $i -eq 7 ]; then
-    echo "!!!!!!!!!!  the cluster-manager-registration-webhook is not ready within 3 minutes"
-    $KUBECTL -n open-cluster-management-hub get pods
-    exit 1
-  fi
-  sleep 30s
-done
-
-for i in {1..7}; do
-  echo "############$i  Checking klusterlet-registration-agent"
-  RUNNING_POD=$($KUBECTL -n open-cluster-management-agent get pods | grep klusterlet-registration-agent | grep -c "Running")
-  if [ ${RUNNING_POD} -ge 1 ]; then
-    break
-  fi
-
-  if [ $i -eq 7 ]; then
-    echo "!!!!!!!!!!  the klusterlet-registration-agent is not ready within 3 minutes"
-    $KUBECTL -n open-cluster-management-agent get pods
-    exit 1
-  fi
-  sleep 30
-done
+echo "############ Accept join of cluster1"
+${CLUSTERADM} accept --clusters ${MANAGED_CLUSTER_NAME} --wait
 
 echo "############  All-in-one env is installed successfully!!"
-
-echo "############  Cleanup"
-cd ../ || exist
-rm -rf _repo_ocm
-
-echo "############  Finished installation!!!"

examples/helloworld/helloworld.go

@@ -12,13 +12,12 @@ import (
 	"open-cluster-management.io/addon-framework/pkg/utils"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
-	workapiv1 "open-cluster-management.io/api/work/v1"
 )
 
 const (
 	DefaultHelloWorldExampleImage = "quay.io/open-cluster-management/addon-examples:latest"
 	AddonName                     = "helloworld"
-	InstallationNamespace         = "default"
+	InstallationNamespace         = "open-cluster-management-agent-addon"
 )
 
 //go:embed manifests
@@ -30,73 +29,32 @@ func NewRegistrationOption(kubeConfig *rest.Config, addonName, agentName string)
 		CSRConfigurations: agent.KubeClientSignerConfigurations(addonName, agentName),
 		CSRApproveCheck:   utils.DefaultCSRApprover(agentName),
 		PermissionConfig:  rbac.AddonRBAC(kubeConfig),
-		Namespace:         InstallationNamespace,
 	}
 }
 
 func GetDefaultValues(cluster *clusterv1.ManagedCluster,
 	addon *addonapiv1alpha1.ManagedClusterAddOn) (addonfactory.Values, error) {
-	installNamespace := addon.Spec.InstallNamespace
-	if len(installNamespace) == 0 {
-		installNamespace = InstallationNamespace
-	}
 
 	image := os.Getenv("EXAMPLE_IMAGE_NAME")
 	if len(image) == 0 {
 		image = DefaultHelloWorldExampleImage
 	}
 
 	manifestConfig := struct {
-		KubeConfigSecret      string
-		ClusterName           string
-		AddonInstallNamespace string
-		Image                 string
+		KubeConfigSecret string
+		ClusterName      string
+		Image            string
 	}{
-		KubeConfigSecret:      fmt.Sprintf("%s-hub-kubeconfig", addon.Name),
-		AddonInstallNamespace: installNamespace,
-		ClusterName:           cluster.Name,
-		Image:                 image,
+		KubeConfigSecret: fmt.Sprintf("%s-hub-kubeconfig", addon.Name),
+		ClusterName:      cluster.Name,
+		Image:            image,
 	}
 
 	return addonfactory.StructToValues(manifestConfig), nil
 }
 
 func AgentHealthProber() *agent.HealthProber {
 	return &agent.HealthProber{
-		Type: agent.HealthProberTypeWork,
-		WorkProber: &agent.WorkHealthProber{
-			ProbeFields: []agent.ProbeField{
-				{
-					ResourceIdentifier: workapiv1.ResourceIdentifier{
-						Group:     "apps",
-						Resource:  "deployments",
-						Name:      "helloworld-agent",
-						Namespace: InstallationNamespace,
-					},
-					ProbeRules: []workapiv1.FeedbackRule{
-						{
-							Type: workapiv1.WellKnownStatusType,
-						},
-					},
-				},
-			},
-			HealthCheck: func(identifier workapiv1.ResourceIdentifier, result workapiv1.StatusFeedbackResult) error {
-				if len(result.Values) == 0 {
-					return fmt.Errorf("no values are probed for deployment %s/%s", identifier.Namespace, identifier.Name)
-				}
-				for _, value := range result.Values {
-					if value.Name != "ReadyReplicas" {
-						continue
-					}
-
-					if *value.Value.Integer >= 1 {
-						return nil
-					}
-
-					return fmt.Errorf("readyReplica is %d for deployement %s/%s", *value.Value.Integer, identifier.Namespace, identifier.Name)
-				}
-				return fmt.Errorf("readyReplica is not probed")
-			},
-		},
+		Type: agent.HealthProberTypeDeploymentAvailability,
 	}
 }

examples/helloworld_helm/helloworld_helm.go

@@ -73,7 +73,13 @@ func GetImageValues(kubeClient kubernetes.Interface) addonfactory.GetValuesFunc
 				continue
 			}
 
-			configMap, err := kubeClient.CoreV1().ConfigMaps(config.Namespace).Get(context.Background(), config.Name, metav1.GetOptions{})
+			if config.DesiredConfig == nil {
+				continue
+			}
+
+			configMap, err := kubeClient.CoreV1().
+				ConfigMaps(config.DesiredConfig.Namespace).
+				Get(context.Background(), config.DesiredConfig.Name, metav1.GetOptions{})
 			if err != nil {
 				return nil, err
 			}

go.mod

@@ -3,72 +3,79 @@ module open-cluster-management.io/addon-framework
 go 1.21
 
 require (
-	github.com/evanphx/json-patch v5.6.0+incompatible
+	github.com/evanphx/json-patch v5.7.0+incompatible
 	github.com/fatih/structs v1.1.0
+	github.com/mochi-mqtt/server/v2 v2.4.6
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.29.0
-	github.com/openshift/build-machinery-go v0.0.0-20230306181456-d321ffa04533
+	github.com/onsi/gomega v1.31.1
+	github.com/openshift/build-machinery-go v0.0.0-20231128094528-1e9b1b0595c8
 	github.com/openshift/library-go v0.0.0-20240116081341-964bcb3f545c
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
-	helm.sh/helm/v3 v3.11.1
-	k8s.io/api v0.29.1
+	gopkg.in/yaml.v2 v2.4.0
+	helm.sh/helm/v3 v3.14.2
+	k8s.io/api v0.29.2
 	k8s.io/apiextensions-apiserver v0.29.0
-	k8s.io/apimachinery v0.29.1
+	k8s.io/apimachinery v0.29.2
 	k8s.io/apiserver v0.29.0
-	k8s.io/client-go v0.29.1
+	k8s.io/client-go v0.29.2
 	k8s.io/component-base v0.29.1
-	k8s.io/klog/v2 v2.110.1
-	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
+	k8s.io/klog/v2 v2.120.1
+	k8s.io/utils v0.0.0-20240310230437-4693a0247e57
 	open-cluster-management.io/api v0.13.0
-	open-cluster-management.io/sdk-go v0.13.0
-	sigs.k8s.io/controller-runtime v0.16.2
+	open-cluster-management.io/sdk-go v0.13.1-0.20240416062924-20307e6fe090
+	sigs.k8s.io/controller-runtime v0.17.2
 )
 
 require (
-	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/semver/v3 v3.2.0 // indirect
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/bwmarrin/snowflake v0.3.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cloudevents/sdk-go/protocol/mqtt_paho/v2 v2.0.0-20231030012137-0836a524e995 // indirect
+	github.com/cloudevents/sdk-go/v2 v2.15.2 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/eclipse/paho.golang v0.11.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.8.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/cel-go v0.17.7 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
-	github.com/huandu/xstrings v1.3.3 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -77,21 +84,22 @@ require (
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.16.0 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
-	github.com/shopspring/decimal v1.2.0 // indirect
-	github.com/spf13/cast v1.4.1 // indirect
+	github.com/prometheus/client_golang v1.18.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/rs/xid v1.4.0 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
-	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.10 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect
 	go.etcd.io/etcd/client/v3 v3.5.10 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
 	go.opentelemetry.io/otel v1.19.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect
@@ -101,30 +109,29 @@ require (
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
-	golang.org/x/crypto v0.16.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
-	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/oauth2 v0.10.0 // indirect
-	golang.org/x/sync v0.5.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/oauth2 v0.16.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/term v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
-	google.golang.org/grpc v1.58.3 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/grpc v1.62.1 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/kms v0.29.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )

pkg/addonfactory/addondeploymentconfig.go

@@ -1,7 +1,6 @@
 package addonfactory
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"strings"
@@ -23,74 +22,6 @@ var AddOnDeploymentConfigGVR = schema.GroupVersionResource{
 	Resource: "addondeploymentconfigs",
 }
 
-// AddOnDeloymentConfigToValuesFunc transform the AddOnDeploymentConfig object into Values object
-// The transformation logic depends on the definition of the addon template
-// Deprecated: use AddOnDeploymentConfigToValuesFunc instead.
-type AddOnDeloymentConfigToValuesFunc func(config addonapiv1alpha1.AddOnDeploymentConfig) (Values, error)
-
-// NewAddOnDeloymentConfigGetter returns a AddOnDeloymentConfigGetter with addon client
-// Deprecated: use NewAddOnDeploymentConfigGetter in pkg/utils package instead.
-func NewAddOnDeloymentConfigGetter(addonClient addonv1alpha1client.Interface) utils.AddOnDeploymentConfigGetter {
-	return utils.NewAddOnDeploymentConfigGetter(addonClient)
-}
-
-// GetAddOnDeloymentConfigValues uses AddOnDeloymentConfigGetter to get the AddOnDeploymentConfig object, then
-// uses AddOnDeloymentConfigToValuesFunc to transform the AddOnDeploymentConfig object to Values object
-// If there are multiple AddOnDeploymentConfig objects in the AddOn ConfigReferences, the big index object will
-// override the one from small index
-// Deprecated: use GetAddOnDeploymentConfigValues instead.
-func GetAddOnDeloymentConfigValues(
-	getter utils.AddOnDeploymentConfigGetter, toValuesFuncs ...AddOnDeloymentConfigToValuesFunc) GetValuesFunc {
-	return func(cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn) (Values, error) {
-		var lastValues = Values{}
-		for _, config := range addon.Status.ConfigReferences {
-			if config.ConfigGroupResource.Group != utils.AddOnDeploymentConfigGVR.Group ||
-				config.ConfigGroupResource.Resource != utils.AddOnDeploymentConfigGVR.Resource {
-				continue
-			}
-
-			addOnDeploymentConfig, err := getter.Get(context.Background(), config.Namespace, config.Name)
-			if err != nil {
-				return nil, err
-			}
-
-			for _, toValuesFunc := range toValuesFuncs {
-				values, err := toValuesFunc(*addOnDeploymentConfig)
-				if err != nil {
-					return nil, err
-				}
-				lastValues = MergeValues(lastValues, values)
-			}
-		}
-
-		return lastValues, nil
-	}
-}
-
-// ToAddOnDeloymentConfigValues transform the AddOnDeploymentConfig object into Values object that is a plain value map
-// for example: the spec of one AddOnDeploymentConfig is:
-//
-//	{
-//		customizedVariables: [{name: "Image", value: "img"}, {name: "ImagePullPolicy", value: "Always"}],
-//		nodePlacement: {nodeSelector: {"host": "ssd"}, tolerations: {"key": "test"}},
-//	}
-//
-// after transformed, the key set of Values object will be: {"Image", "ImagePullPolicy", "NodeSelector", "Tolerations"}
-// Deprecated: use ToAddOnDeploymentConfigValues instead.
-func ToAddOnDeloymentConfigValues(config addonapiv1alpha1.AddOnDeploymentConfig) (Values, error) {
-	values, err := ToAddOnCustomizedVariableValues(config)
-	if err != nil {
-		return nil, err
-	}
-
-	if config.Spec.NodePlacement != nil {
-		values["NodeSelector"] = config.Spec.NodePlacement.NodeSelector
-		values["Tolerations"] = config.Spec.NodePlacement.Tolerations
-	}
-
-	return values, nil
-}
-
 // ToAddOnNodePlacementValues only transform the AddOnDeploymentConfig NodePlacement part into Values object that has
 // a specific for helm chart values
 // for example: the spec of one AddOnDeploymentConfig is:
@@ -214,24 +145,21 @@ func GetAddOnDeploymentConfigValues(
 	getter utils.AddOnDeploymentConfigGetter, toValuesFuncs ...AddOnDeploymentConfigToValuesFunc) GetValuesFunc {
 	return func(cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn) (Values, error) {
 		var lastValues = Values{}
-		for _, config := range addon.Status.ConfigReferences {
-			if config.ConfigGroupResource.Group != utils.AddOnDeploymentConfigGVR.Group ||
-				config.ConfigGroupResource.Resource != utils.AddOnDeploymentConfigGVR.Resource {
-				continue
-			}
+		addOnDeploymentConfig, err := utils.GetDesiredAddOnDeploymentConfig(addon, getter)
+		if err != nil {
+			return lastValues, err
+		}
+
+		if addOnDeploymentConfig == nil {
+			return lastValues, nil
+		}
 
-			addOnDeploymentConfig, err := getter.Get(context.Background(), config.Namespace, config.Name)
+		for _, toValuesFunc := range toValuesFuncs {
+			values, err := toValuesFunc(*addOnDeploymentConfig)
 			if err != nil {
 				return nil, err
 			}
-
-			for _, toValuesFunc := range toValuesFuncs {
-				values, err := toValuesFunc(*addOnDeploymentConfig)
-				if err != nil {
-					return nil, err
-				}
-				lastValues = MergeValues(lastValues, values)
-			}
+			lastValues = MergeValues(lastValues, values)
 		}
 
 		return lastValues, nil
@@ -349,19 +277,12 @@ func getRegistriesFromClusterAnnotation(
 //   - Image registries configured in the addonDeploymentConfig will take precedence over the managed cluster annotation
 func GetAgentImageValues(getter utils.AddOnDeploymentConfigGetter, imageKey, image string) GetValuesFunc {
 	return func(cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn) (Values, error) {
-
+		addOnDeploymentConfig, err := utils.GetDesiredAddOnDeploymentConfig(addon, getter)
+		if err != nil {
+			return nil, err
+		}
 		// Get image from AddOnDeploymentConfig
-		for _, config := range addon.Status.ConfigReferences {
-			if config.ConfigGroupResource.Group != utils.AddOnDeploymentConfigGVR.Group ||
-				config.ConfigGroupResource.Resource != utils.AddOnDeploymentConfigGVR.Resource {
-				continue
-			}
-
-			addOnDeploymentConfig, err := getter.Get(context.Background(), config.Namespace, config.Name)
-			if err != nil {
-				return nil, err
-			}
-
+		if addOnDeploymentConfig != nil {
 			values, overrode, err := overrideImageWithKeyValue(imageKey, image,
 				getRegistriesFromAddonDeploymentConfig(*addOnDeploymentConfig))
 			if err != nil {

pkg/addonfactory/addondeploymentconfig_test.go

@@ -11,6 +11,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager/addontesting"
+	"open-cluster-management.io/addon-framework/pkg/utils"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	fakeaddon "open-cluster-management.io/api/client/addon/clientset/versioned/fake"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
@@ -39,9 +40,12 @@ func TestGetAddOnDeploymentConfigValues(t *testing.T) {
 								Group:    "config.test",
 								Resource: "testconfigs",
 							},
-							ConfigReferent: addonapiv1alpha1.ConfigReferent{
-								Namespace: "cluster1",
-								Name:      "testConfig",
+							DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
+								ConfigReferent: addonapiv1alpha1.ConfigReferent{
+									Namespace: "cluster1",
+									Name:      "testConfig",
+								},
+								SpecHash: "dummy",
 							},
 						},
 					}
@@ -50,68 +54,6 @@ func TestGetAddOnDeploymentConfigValues(t *testing.T) {
 			},
 			expectedValues: Values{},
 		},
-		{
-			name:          "mutiple addon deployment configs",
-			toValuesFuncs: []AddOnDeploymentConfigToValuesFunc{ToAddOnDeploymentConfigValues},
-			addOnObjs: []runtime.Object{
-				func() *addonapiv1alpha1.ManagedClusterAddOn {
-					addon := addontesting.NewAddon("test", "cluster1")
-					addon.Status.ConfigReferences = []addonapiv1alpha1.ConfigReference{
-						{
-							ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{
-								Group:    "addon.open-cluster-management.io",
-								Resource: "addondeploymentconfigs",
-							},
-							ConfigReferent: addonapiv1alpha1.ConfigReferent{
-								Namespace: "cluster1",
-								Name:      "config1",
-							},
-						},
-						{
-							ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{
-								Group:    "addon.open-cluster-management.io",
-								Resource: "addondeploymentconfigs",
-							},
-							ConfigReferent: addonapiv1alpha1.ConfigReferent{
-								Namespace: "cluster1",
-								Name:      "config2",
-							},
-						},
-					}
-					return addon
-				}(),
-				&addonapiv1alpha1.AddOnDeploymentConfig{
-					ObjectMeta: metav1.ObjectMeta{
-						Name:      "config1",
-						Namespace: "cluster1",
-					},
-					Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{
-						CustomizedVariables: []addonapiv1alpha1.CustomizedVariable{
-							{Name: "Test", Value: "test1"},
-						},
-					},
-				},
-				&addonapiv1alpha1.AddOnDeploymentConfig{
-					ObjectMeta: metav1.ObjectMeta{
-						Name:      "config2",
-						Namespace: "cluster1",
-					},
-					Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{
-						CustomizedVariables: []addonapiv1alpha1.CustomizedVariable{
-							{Name: "Test", Value: "test2"},
-						},
-						NodePlacement: &addonapiv1alpha1.NodePlacement{
-							NodeSelector: map[string]string{"test": "test"},
-						},
-					},
-				},
-			},
-			expectedValues: Values{
-				"Test":         "test2",
-				"NodeSelector": map[string]string{"test": "test"},
-				"Tolerations":  []corev1.Toleration{},
-			},
-		},
 		{
 			name:          "to addon node placement",
 			toValuesFuncs: []AddOnDeploymentConfigToValuesFunc{ToAddOnNodePlacementValues},
@@ -124,9 +66,12 @@ func TestGetAddOnDeploymentConfigValues(t *testing.T) {
 								Group:    "addon.open-cluster-management.io",
 								Resource: "addondeploymentconfigs",
 							},
-							ConfigReferent: addonapiv1alpha1.ConfigReferent{
-								Namespace: "cluster1",
-								Name:      "config",
+							DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
+								ConfigReferent: addonapiv1alpha1.ConfigReferent{
+									Namespace: "cluster1",
+									Name:      "config",
+								},
+								SpecHash: "dummy",
 							},
 						},
 					}
@@ -164,9 +109,12 @@ func TestGetAddOnDeploymentConfigValues(t *testing.T) {
 								Group:    "addon.open-cluster-management.io",
 								Resource: "addondeploymentconfigs",
 							},
-							ConfigReferent: addonapiv1alpha1.ConfigReferent{
-								Namespace: "cluster1",
-								Name:      "config",
+							DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
+								ConfigReferent: addonapiv1alpha1.ConfigReferent{
+									Namespace: "cluster1",
+									Name:      "config",
+								},
+								SpecHash: "dummy",
 							},
 						},
 					}
@@ -211,9 +159,12 @@ func TestGetAddOnDeploymentConfigValues(t *testing.T) {
 								Group:    "addon.open-cluster-management.io",
 								Resource: "addondeploymentconfigs",
 							},
-							ConfigReferent: addonapiv1alpha1.ConfigReferent{
-								Namespace: "cluster1",
-								Name:      "config",
+							DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
+								ConfigReferent: addonapiv1alpha1.ConfigReferent{
+									Namespace: "cluster1",
+									Name:      "config",
+								},
+								SpecHash: "dummy",
 							},
 						},
 					}
@@ -253,9 +204,12 @@ func TestGetAddOnDeploymentConfigValues(t *testing.T) {
 								Group:    "addon.open-cluster-management.io",
 								Resource: "addondeploymentconfigs",
 							},
-							ConfigReferent: addonapiv1alpha1.ConfigReferent{
-								Namespace: "cluster1",
-								Name:      "config",
+							DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
+								ConfigReferent: addonapiv1alpha1.ConfigReferent{
+									Namespace: "cluster1",
+									Name:      "config",
+								},
+								SpecHash: "dummy",
 							},
 						},
 					}
@@ -293,7 +247,7 @@ func TestGetAddOnDeploymentConfigValues(t *testing.T) {
 		t.Run(c.name, func(t *testing.T) {
 			fakeAddonClient := fakeaddon.NewSimpleClientset(c.addOnObjs...)
 
-			getter := NewAddOnDeploymentConfigGetter(fakeAddonClient)
+			getter := utils.NewAddOnDeploymentConfigGetter(fakeAddonClient)
 
 			addOn, ok := c.addOnObjs[0].(*addonapiv1alpha1.ManagedClusterAddOn)
 			if !ok {
@@ -606,36 +560,18 @@ func TestGetAgentImageValues(t *testing.T) {
 							Group:    "addon.open-cluster-management.io",
 							Resource: "addondeploymentconfigs",
 						},
-						ConfigReferent: addonapiv1alpha1.ConfigReferent{
-							Namespace: "cluster1",
-							Name:      "config1",
-						},
-					},
-					{
-						ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{
-							Group:    "addon.open-cluster-management.io",
-							Resource: "addondeploymentconfigs",
-						},
-						ConfigReferent: addonapiv1alpha1.ConfigReferent{
-							Namespace: "cluster1",
-							Name:      "config2",
+						DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
+							ConfigReferent: addonapiv1alpha1.ConfigReferent{
+								Namespace: "cluster1",
+								Name:      "config2",
+							},
+							SpecHash: "dummy",
 						},
 					},
 				}
 				return addon
 			}(),
 			addonDeploymentConfig: []runtime.Object{
-				&addonapiv1alpha1.AddOnDeploymentConfig{
-					ObjectMeta: metav1.ObjectMeta{
-						Name:      "config1",
-						Namespace: "cluster1",
-					},
-					Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{
-						CustomizedVariables: []addonapiv1alpha1.CustomizedVariable{
-							{Name: "Test", Value: "test1"},
-						},
-					},
-				},
 				&addonapiv1alpha1.AddOnDeploymentConfig{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      "config2",

pkg/addonfactory/addonfactory.go

@@ -10,7 +10,9 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/klog/v2"
+	"open-cluster-management.io/addon-framework/pkg/addonmanager/constants"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
+	clusterclientset "open-cluster-management.io/api/client/cluster/clientset/versioned"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 
 	"open-cluster-management.io/addon-framework/pkg/agent"
@@ -34,9 +36,12 @@ type AgentAddonFactory struct {
 	getValuesFuncs    []GetValuesFunc
 	agentAddonOptions agent.AgentAddonOptions
 	// trimCRDDescription flag is used to trim the description of CRDs in manifestWork. disabled by default.
-	trimCRDDescription    bool
+	trimCRDDescription bool
+	// Deprecated: use clusterClient to get the hosting cluster.
 	hostingCluster        *clusterv1.ManagedCluster
-	agentInstallNamespace func(addon *addonapiv1alpha1.ManagedClusterAddOn) string
+	clusterClient         clusterclientset.Interface
+	agentInstallNamespace func(addon *addonapiv1alpha1.ManagedClusterAddOn) (string, error)
+	helmEngineStrict      bool
 }
 
 // NewAgentAddonFactory builds an addonAgentFactory instance with addon name and fs.
@@ -53,12 +58,14 @@ func NewAgentAddonFactory(addonName string, fs embed.FS, dir string) *AgentAddon
 		agentAddonOptions: agent.AgentAddonOptions{
 			AddonName:           addonName,
 			Registration:        nil,
-			InstallStrategy:     nil,
 			HealthProber:        nil,
 			SupportedConfigGVRs: []schema.GroupVersionResource{},
+			// Set a default hosted mode info func.
+			HostedModeInfoFunc: constants.GetHostedModeInfo,
 		},
 		trimCRDDescription: false,
 		scheme:             s,
+		helmEngineStrict:   false,
 	}
 }
 
@@ -78,19 +85,6 @@ func (f *AgentAddonFactory) WithGetValuesFuncs(getValuesFuncs ...GetValuesFunc)
 	return f
 }
 
-// WithInstallStrategy defines the installation strategy of the manifests prescribed by Manifests(..).
-// Deprecated: add annotation "addon.open-cluster-management.io/lifecycle: addon-manager" to ClusterManagementAddon
-// and define install strategy in ClusterManagementAddon spec.installStrategy instead.
-// The migration plan refer to https://github.com/open-cluster-management-io/ocm/issues/355.
-func (f *AgentAddonFactory) WithInstallStrategy(strategy *agent.InstallStrategy) *AgentAddonFactory {
-	if strategy.InstallNamespace == "" {
-		strategy.InstallNamespace = AddonDefaultInstallNamespace
-	}
-	f.agentAddonOptions.InstallStrategy = strategy
-
-	return f
-}
-
 // WithAgentRegistrationOption defines how agent is registered to the hub cluster.
 func (f *AgentAddonFactory) WithAgentRegistrationOption(option *agent.RegistrationOption) *AgentAddonFactory {
 	f.agentAddonOptions.Registration = option
@@ -109,12 +103,25 @@ func (f *AgentAddonFactory) WithAgentHostedModeEnabledOption() *AgentAddonFactor
 	return f
 }
 
+// WithAgentHostedInfoFn sets the function to get the hosting cluster of an addon in the hosted mode.
+func (f *AgentAddonFactory) WithAgentHostedInfoFn(
+	infoFn func(*addonapiv1alpha1.ManagedClusterAddOn, *clusterv1.ManagedCluster) (string, string)) *AgentAddonFactory {
+	f.agentAddonOptions.HostedModeInfoFunc = infoFn
+	return f
+}
+
 // WithTrimCRDDescription is to enable trim the description of CRDs in manifestWork.
 func (f *AgentAddonFactory) WithTrimCRDDescription() *AgentAddonFactory {
 	f.trimCRDDescription = true
 	return f
 }
 
+// WithHelmEngineStrict is to enable script go template rendering for Helm charts to generate manifestWork.
+func (f *AgentAddonFactory) WithHelmEngineStrict() *AgentAddonFactory {
+	f.helmEngineStrict = true
+	return f
+}
+
 // WithConfigGVRs defines the addon supported configuration GroupVersionResource
 func (f *AgentAddonFactory) WithConfigGVRs(gvrs ...schema.GroupVersionResource) *AgentAddonFactory {
 	f.agentAddonOptions.SupportedConfigGVRs = append(f.agentAddonOptions.SupportedConfigGVRs, gvrs...)
@@ -123,11 +130,18 @@ func (f *AgentAddonFactory) WithConfigGVRs(gvrs ...schema.GroupVersionResource)
 
 // WithHostingCluster defines the hosting cluster used in hosted mode. An AgentAddon may use this to provide
 // additional metadata.
+// Deprecated: use WithManagedClusterClient to set a cluster client that can get the hosting cluster.
 func (f *AgentAddonFactory) WithHostingCluster(cluster *clusterv1.ManagedCluster) *AgentAddonFactory {
 	f.hostingCluster = cluster
 	return f
 }
 
+// WithManagedClusterClient defines the cluster client that can get the hosting cluster used in hosted mode.
+func (f *AgentAddonFactory) WithManagedClusterClient(c clusterclientset.Interface) *AgentAddonFactory {
+	f.clusterClient = c
+	return f
+}
+
 // WithAgentDeployTriggerClusterFilter defines the filter func to trigger the agent deploy/redploy when cluster info is
 // changed. Addons that need information from the ManagedCluster resource when deploying the agent should use this
 // function to set what information they need, otherwise the expected/up-to-date agent may be deployed delayed since the
@@ -150,7 +164,7 @@ func (f *AgentAddonFactory) WithAgentDeployTriggerClusterFilter(
 // override the default built-in namespace value; And if the registrationOption is not nil but the
 // registrationOption.AgentInstallNamespace is nil, this will also set it to this.
 func (f *AgentAddonFactory) WithAgentInstallNamespace(
-	nsFunc func(addon *addonapiv1alpha1.ManagedClusterAddOn) string,
+	nsFunc func(addon *addonapiv1alpha1.ManagedClusterAddOn) (string, error),
 ) *AgentAddonFactory {
 	f.agentInstallNamespace = nsFunc
 	return f

pkg/addonfactory/helm_agentaddon.go

@@ -2,6 +2,7 @@ package addonfactory
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"io"
 	"sort"
@@ -10,14 +11,18 @@ import (
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chartutil"
 	"helm.sh/helm/v3/pkg/engine"
+	"helm.sh/helm/v3/pkg/releaseutil"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/klog/v2"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
+	clusterclientset "open-cluster-management.io/api/client/cluster/clientset/versioned"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 
-	"open-cluster-management.io/addon-framework/pkg/addonmanager/constants"
 	"open-cluster-management.io/addon-framework/pkg/agent"
 )
 
@@ -42,13 +47,16 @@ type helmDefaultValues struct {
 }
 
 type HelmAgentAddon struct {
-	decoder               runtime.Decoder
-	chart                 *chart.Chart
-	getValuesFuncs        []GetValuesFunc
-	agentAddonOptions     agent.AgentAddonOptions
-	trimCRDDescription    bool
+	decoder            runtime.Decoder
+	chart              *chart.Chart
+	getValuesFuncs     []GetValuesFunc
+	agentAddonOptions  agent.AgentAddonOptions
+	trimCRDDescription bool
+	// Deprecated: use clusterClient to get the hosting cluster.
 	hostingCluster        *clusterv1.ManagedCluster
-	agentInstallNamespace func(addon *addonapiv1alpha1.ManagedClusterAddOn) string
+	clusterClient         clusterclientset.Interface
+	agentInstallNamespace func(addon *addonapiv1alpha1.ManagedClusterAddOn) (string, error)
+	helmEngineStrict      bool
 }
 
 func newHelmAgentAddon(factory *AgentAddonFactory, chart *chart.Chart) *HelmAgentAddon {
@@ -59,11 +67,40 @@ func newHelmAgentAddon(factory *AgentAddonFactory, chart *chart.Chart) *HelmAgen
 		agentAddonOptions:     factory.agentAddonOptions,
 		trimCRDDescription:    factory.trimCRDDescription,
 		hostingCluster:        factory.hostingCluster,
+		clusterClient:         factory.clusterClient,
 		agentInstallNamespace: factory.agentInstallNamespace,
+		helmEngineStrict:      factory.helmEngineStrict,
 	}
 }
 
 func (a *HelmAgentAddon) Manifests(
+	cluster *clusterv1.ManagedCluster,
+	addon *addonapiv1alpha1.ManagedClusterAddOn) ([]runtime.Object, error) {
+	objects, err := a.renderManifests(cluster, addon)
+	if err != nil {
+		return nil, err
+	}
+
+	manifests := make([]manifest, 0, len(objects))
+	for _, obj := range objects {
+		a, err := meta.TypeAccessor(obj)
+		if err != nil {
+			return nil, err
+		}
+		manifests = append(manifests, manifest{
+			Object: obj,
+			Kind:   a.GetKind(),
+		})
+	}
+	sortManifestsByKind(manifests, releaseutil.InstallOrder)
+
+	for i, manifest := range manifests {
+		objects[i] = manifest.Object
+	}
+	return objects, nil
+}
+
+func (a *HelmAgentAddon) renderManifests(
 	cluster *clusterv1.ManagedCluster,
 	addon *addonapiv1alpha1.ManagedClusterAddOn) ([]runtime.Object, error) {
 	var objects []runtime.Object
@@ -74,7 +111,7 @@ func (a *HelmAgentAddon) Manifests(
 	}
 
 	helmEngine := engine.Engine{
-		Strict:   true,
+		Strict:   a.helmEngineStrict,
 		LintMode: false,
 	}
 
@@ -93,16 +130,7 @@ func (a *HelmAgentAddon) Manifests(
 		return objects, err
 	}
 
-	// sort the filenames of the templates so the manifests are ordered consistently
-	keys := make([]string, 0, len(templates))
-	for k := range templates {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	for _, k := range keys {
-		data := templates[k]
-
+	for k, data := range templates {
 		if len(data) == 0 {
 			continue
 		}
@@ -131,7 +159,6 @@ func (a *HelmAgentAddon) Manifests(
 				objects = append(objects, object)
 			}
 		}
-
 	}
 
 	if a.trimCRDDescription {
@@ -177,8 +204,12 @@ func (a *HelmAgentAddon) getValues(
 
 	overrideValues = MergeValues(overrideValues, builtinValues)
 
+	releaseOptions, err := a.releaseOptions(addon)
+	if err != nil {
+		return nil, err
+	}
 	values, err := chartutil.ToRenderValues(a.chart, overrideValues,
-		a.releaseOptions(addon), a.capabilities(cluster, addon))
+		releaseOptions, a.capabilities(cluster, addon))
 	if err != nil {
 		klog.Errorf("failed to render helm chart with values %v. err:%v", overrideValues, err)
 		return values, err
@@ -187,18 +218,25 @@ func (a *HelmAgentAddon) getValues(
 	return values, nil
 }
 
-func (a *HelmAgentAddon) getValueAgentInstallNamespace(addon *addonapiv1alpha1.ManagedClusterAddOn) string {
+func (a *HelmAgentAddon) getValueAgentInstallNamespace(addon *addonapiv1alpha1.ManagedClusterAddOn) (string, error) {
 	installNamespace := addon.Spec.InstallNamespace
 	if len(installNamespace) == 0 {
 		installNamespace = AddonDefaultInstallNamespace
 	}
 	if a.agentInstallNamespace != nil {
-		ns := a.agentInstallNamespace(addon)
+		ns, err := a.agentInstallNamespace(addon)
+		if err != nil {
+			klog.Errorf("failed to get agentInstallNamespace from addon %s. err: %v", addon.Name, err)
+			return "", err
+		}
 		if len(ns) > 0 {
 			installNamespace = ns
+		} else {
+			klog.InfoS("Namespace for addon returned by agent install namespace func is empty",
+				"addonNamespace", addon.Namespace, "addonName", addon)
 		}
 	}
-	return installNamespace
+	return installNamespace, nil
 }
 
 func (a *HelmAgentAddon) getBuiltinValues(
@@ -207,9 +245,13 @@ func (a *HelmAgentAddon) getBuiltinValues(
 	builtinValues := helmBuiltinValues{}
 	builtinValues.ClusterName = cluster.GetName()
 
-	builtinValues.AddonInstallNamespace = a.getValueAgentInstallNamespace(addon)
+	addonInstallNamespace, err := a.getValueAgentInstallNamespace(addon)
+	if err != nil {
+		return nil, err
+	}
+	builtinValues.AddonInstallNamespace = addonInstallNamespace
 
-	builtinValues.InstallMode, _ = constants.GetHostedModeInfo(addon.GetAnnotations())
+	builtinValues.InstallMode, _ = a.agentAddonOptions.HostedModeInfoFunc(addon, cluster)
 
 	helmBuiltinValues, err := JsonStructToValues(builtinValues)
 	if err != nil {
@@ -219,6 +261,12 @@ func (a *HelmAgentAddon) getBuiltinValues(
 	return helmBuiltinValues, nil
 }
 
+// Deprecated: use "WithManagedClusterClient" in AgentAddonFactory to set a cluster client that
+// can be used to get the hosting cluster.
+func (a *HelmAgentAddon) SetHostingCluster(hostingCluster *clusterv1.ManagedCluster) {
+	a.hostingCluster = hostingCluster
+}
+
 func (a *HelmAgentAddon) getDefaultValues(
 	cluster *clusterv1.ManagedCluster,
 	addon *addonapiv1alpha1.ManagedClusterAddOn) (Values, error) {
@@ -233,6 +281,21 @@ func (a *HelmAgentAddon) getDefaultValues(
 
 	if a.hostingCluster != nil {
 		defaultValues.HostingClusterCapabilities = *a.capabilities(a.hostingCluster, addon)
+	} else if a.clusterClient != nil {
+		_, hostingClusterName := a.agentAddonOptions.HostedModeInfoFunc(addon, cluster)
+		if len(hostingClusterName) > 0 {
+			hostingCluster, err := a.clusterClient.ClusterV1().ManagedClusters().
+				Get(context.TODO(), hostingClusterName, metav1.GetOptions{})
+			if err == nil {
+				defaultValues.HostingClusterCapabilities = *a.capabilities(hostingCluster, addon)
+			} else if errors.IsNotFound(err) {
+				klog.Infof("hostingCluster %s not found, skip providing default value hostingClusterCapabilities",
+					hostingClusterName)
+			} else {
+				klog.Errorf("failed to get hostingCluster %s. err:%v", hostingClusterName, err)
+				return nil, err
+			}
+		}
 	}
 
 	helmDefaultValues, err := JsonStructToValues(defaultValues)
@@ -254,9 +317,58 @@ func (a *HelmAgentAddon) capabilities(
 
 // only support Release.Name, Release.Namespace
 func (a *HelmAgentAddon) releaseOptions(
-	addon *addonapiv1alpha1.ManagedClusterAddOn) chartutil.ReleaseOptions {
-	return chartutil.ReleaseOptions{
-		Name:      a.agentAddonOptions.AddonName,
-		Namespace: a.getValueAgentInstallNamespace(addon),
+	addon *addonapiv1alpha1.ManagedClusterAddOn) (chartutil.ReleaseOptions, error) {
+	releaseOptions := chartutil.ReleaseOptions{
+		Name: a.agentAddonOptions.AddonName,
+	}
+	namespace, err := a.getValueAgentInstallNamespace(addon)
+	if err != nil {
+		return releaseOptions, err
+	}
+	releaseOptions.Namespace = namespace
+	return releaseOptions, nil
+}
+
+// manifest represents a manifest file, which has a name and some content.
+type manifest struct {
+	Object runtime.Object
+	Kind   string
+}
+
+// sort manifests by kind.
+//
+// Results are sorted by 'ordering', keeping order of items with equal kind/priority
+func sortManifestsByKind(manifests []manifest, ordering releaseutil.KindSortOrder) []manifest {
+	sort.SliceStable(manifests, func(i, j int) bool {
+		return lessByKind(manifests[i], manifests[j], manifests[i].Kind, manifests[j].Kind, ordering)
+	})
+
+	return manifests
+}
+
+func lessByKind(a interface{}, b interface{}, kindA string, kindB string, o releaseutil.KindSortOrder) bool {
+	ordering := make(map[string]int, len(o))
+	for v, k := range o {
+		ordering[k] = v
+	}
+
+	first, aok := ordering[kindA]
+	second, bok := ordering[kindB]
+
+	if !aok && !bok {
+		// if both are unknown then sort alphabetically by kind, keep original order if same kind
+		if kindA != kindB {
+			return kindA < kindB
+		}
+		return first < second
+	}
+	// unknown kind is last
+	if !aok {
+		return false
+	}
+	if !bok {
+		return true
 	}
+	// sort different kinds, keep original order if same priority
+	return first < second
 }

pkg/addonfactory/helm_agentaddon_test.go

@@ -13,6 +13,7 @@ import (
 	"k8s.io/client-go/kubernetes/scheme"
 	"open-cluster-management.io/addon-framework/pkg/agent"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
+	clusterclientset "open-cluster-management.io/api/client/cluster/clientset/versioned/fake"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 	clusterv1apha1 "open-cluster-management.io/api/cluster/v1alpha1"
 )
@@ -66,6 +67,7 @@ func TestChartAgentAddon_Manifests(t *testing.T) {
 		scheme                          *runtime.Scheme
 		clusterName                     string
 		hostingCluster                  *clusterv1.ManagedCluster
+		getHostingClusterWithClient     bool
 		addonName                       string
 		installNamespace                string
 		annotationValues                string
@@ -146,6 +148,19 @@ func TestChartAgentAddon_Manifests(t *testing.T) {
 			expectedObjCnt:           5,
 			expectedNamespace:        true,
 		},
+		{
+			name:                        "template render ok, getting hosting cluster with client",
+			scheme:                      testScheme,
+			clusterName:                 "cluster1",
+			hostingCluster:              NewFakeManagedCluster("hosting-cluster", "1.25.0"),
+			getHostingClusterWithClient: true,
+			addonName:                   "helloworld",
+			installNamespace:            "myNs",
+			expectedInstallNamespace:    "myNs",
+			expectedImage:               "quay.io/testimage:test",
+			expectedObjCnt:              5,
+			expectedNamespace:           true,
+		},
 	}
 
 	for _, c := range cases {
@@ -166,13 +181,20 @@ func TestChartAgentAddon_Manifests(t *testing.T) {
 			cluster := NewFakeManagedCluster(c.clusterName, "1.10.1")
 			clusterAddon := NewFakeManagedClusterAddon(c.addonName, c.clusterName, c.installNamespace, c.annotationValues)
 
-			agentAddon, err := NewAgentAddonFactory(c.addonName, chartFS, "testmanifests/chart").
+			factory := NewAgentAddonFactory(c.addonName, chartFS, "testmanifests/chart").
 				WithGetValuesFuncs(getValuesFuncs...).
 				WithScheme(c.scheme).
 				WithTrimCRDDescription().
-				WithAgentRegistrationOption(&agent.RegistrationOption{}).
-				WithHostingCluster(c.hostingCluster).
-				BuildHelmAgentAddon()
+				WithAgentRegistrationOption(&agent.RegistrationOption{})
+			if c.getHostingClusterWithClient && c.hostingCluster != nil {
+				clusterClient := clusterclientset.NewSimpleClientset(c.hostingCluster)
+				factory = factory.WithManagedClusterClient(clusterClient)
+				clusterAddon.Annotations[addonapiv1alpha1.HostingClusterNameAnnotationKey] = c.hostingCluster.Name
+			}
+			if !c.getHostingClusterWithClient && c.hostingCluster != nil {
+				factory = factory.WithHostingCluster(c.hostingCluster)
+			}
+			agentAddon, err := factory.BuildHelmAgentAddon()
 			if err != nil {
 				t.Errorf("expected no error, got err %v", err)
 			}

pkg/addonfactory/template_agentaddon.go

@@ -9,7 +9,6 @@ import (
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 
-	"open-cluster-management.io/addon-framework/pkg/addonmanager/constants"
 	"open-cluster-management.io/addon-framework/pkg/agent"
 	"open-cluster-management.io/addon-framework/pkg/assets"
 )
@@ -42,7 +41,7 @@ type TemplateAgentAddon struct {
 	getValuesFuncs        []GetValuesFunc
 	agentAddonOptions     agent.AgentAddonOptions
 	trimCRDDescription    bool
-	agentInstallNamespace func(addon *addonapiv1alpha1.ManagedClusterAddOn) string
+	agentInstallNamespace func(addon *addonapiv1alpha1.ManagedClusterAddOn) (string, error)
 }
 
 func newTemplateAgentAddon(factory *AgentAddonFactory) *TemplateAgentAddon {
@@ -109,15 +108,18 @@ func (a *TemplateAgentAddon) getValues(
 			overrideValues = MergeValues(overrideValues, userValues)
 		}
 	}
-	builtinValues := a.getBuiltinValues(cluster, addon)
+	builtinValues, err := a.getBuiltinValues(cluster, addon)
+	if err != nil {
+		return overrideValues, err
+	}
 	overrideValues = MergeValues(overrideValues, builtinValues)
 
 	return overrideValues, nil
 }
 
 func (a *TemplateAgentAddon) getBuiltinValues(
 	cluster *clusterv1.ManagedCluster,
-	addon *addonapiv1alpha1.ManagedClusterAddOn) Values {
+	addon *addonapiv1alpha1.ManagedClusterAddOn) (Values, error) {
 	builtinValues := templateBuiltinValues{}
 	builtinValues.ClusterName = cluster.GetName()
 
@@ -126,16 +128,23 @@ func (a *TemplateAgentAddon) getBuiltinValues(
 		installNamespace = AddonDefaultInstallNamespace
 	}
 	if a.agentInstallNamespace != nil {
-		ns := a.agentInstallNamespace(addon)
+		ns, err := a.agentInstallNamespace(addon)
+		if err != nil {
+			klog.Errorf("failed to get agent install namespace for addon %s: %v", addon.Name, err)
+			return nil, err
+		}
 		if len(ns) > 0 {
 			installNamespace = ns
+		} else {
+			klog.InfoS("Namespace for addon returned by agent install namespace func is empty",
+				"addonNamespace", addon.Namespace, "addonName", addon)
 		}
 	}
 	builtinValues.AddonInstallNamespace = installNamespace
 
-	builtinValues.InstallMode, _ = constants.GetHostedModeInfo(addon.GetAnnotations())
+	builtinValues.InstallMode, _ = a.agentAddonOptions.HostedModeInfoFunc(addon, cluster)
 
-	return StructToValues(builtinValues)
+	return StructToValues(builtinValues), nil
 }
 
 func (a *TemplateAgentAddon) getDefaultValues(

pkg/addonmanager/base_manager.go

@@ -0,0 +1,200 @@
+package addonmanager
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic/dynamicinformer"
+	kubeinformers "k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	addonv1alpha1client "open-cluster-management.io/api/client/addon/clientset/versioned"
+	addoninformers "open-cluster-management.io/api/client/addon/informers/externalversions"
+	clusterv1informers "open-cluster-management.io/api/client/cluster/informers/externalversions"
+	workclientset "open-cluster-management.io/api/client/work/clientset/versioned"
+	workv1informers "open-cluster-management.io/api/client/work/informers/externalversions/work/v1"
+
+	"open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/addonconfig"
+	"open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/agentdeploy"
+	"open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/certificate"
+	"open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/cmaconfig"
+	"open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/cmamanagedby"
+	"open-cluster-management.io/addon-framework/pkg/addonmanager/controllers/registration"
+	"open-cluster-management.io/addon-framework/pkg/agent"
+	"open-cluster-management.io/addon-framework/pkg/basecontroller/factory"
+	"open-cluster-management.io/addon-framework/pkg/utils"
+)
+
+// BaseAddonManagerImpl is the base implementation of BaseAddonManager
+// that manages the addon agents and configs.
+type BaseAddonManagerImpl struct {
+	addonAgents  map[string]agent.AgentAddon
+	addonConfigs map[schema.GroupVersionResource]bool
+	config       *rest.Config
+	syncContexts []factory.SyncContext
+}
+
+// NewBaseAddonManagerImpl creates a new BaseAddonManagerImpl instance with the given config.
+func NewBaseAddonManagerImpl(config *rest.Config) *BaseAddonManagerImpl {
+	return &BaseAddonManagerImpl{
+		config:       config,
+		syncContexts: []factory.SyncContext{},
+		addonConfigs: map[schema.GroupVersionResource]bool{},
+		addonAgents:  map[string]agent.AgentAddon{},
+	}
+}
+
+func (a *BaseAddonManagerImpl) GetConfig() *rest.Config {
+	return a.config
+}
+
+func (a *BaseAddonManagerImpl) GetAddonAgents() map[string]agent.AgentAddon {
+	return a.addonAgents
+}
+
+func (a *BaseAddonManagerImpl) AddAgent(addon agent.AgentAddon) error {
+	addonOption := addon.GetAgentAddonOptions()
+	if len(addonOption.AddonName) == 0 {
+		return fmt.Errorf("addon name should be set")
+	}
+	if _, ok := a.addonAgents[addonOption.AddonName]; ok {
+		return fmt.Errorf("an agent is added for the addon already")
+	}
+	a.addonAgents[addonOption.AddonName] = addon
+	return nil
+}
+
+func (a *BaseAddonManagerImpl) Trigger(clusterName, addonName string) {
+	for _, syncContex := range a.syncContexts {
+		syncContex.Queue().Add(fmt.Sprintf("%s/%s", clusterName, addonName))
+	}
+}
+
+func (a *BaseAddonManagerImpl) StartWithInformers(ctx context.Context,
+	workClient workclientset.Interface,
+	workInformers workv1informers.ManifestWorkInformer,
+	kubeInformers kubeinformers.SharedInformerFactory,
+	addonInformers addoninformers.SharedInformerFactory,
+	clusterInformers clusterv1informers.SharedInformerFactory,
+	dynamicInformers dynamicinformer.DynamicSharedInformerFactory) error {
+
+	kubeClient, err := kubernetes.NewForConfig(a.config)
+	if err != nil {
+		return err
+	}
+
+	addonClient, err := addonv1alpha1client.NewForConfig(a.config)
+	if err != nil {
+		return err
+	}
+
+	v1CSRSupported, v1beta1Supported, err := utils.IsCSRSupported(kubeClient)
+	if err != nil {
+		return err
+	}
+
+	for _, agentImpl := range a.addonAgents {
+		for _, configGVR := range agentImpl.GetAgentAddonOptions().SupportedConfigGVRs {
+			a.addonConfigs[configGVR] = true
+		}
+	}
+
+	deployController := agentdeploy.NewAddonDeployController(
+		workClient,
+		addonClient,
+		clusterInformers.Cluster().V1().ManagedClusters(),
+		addonInformers.Addon().V1alpha1().ManagedClusterAddOns(),
+		workInformers,
+		a.addonAgents,
+	)
+
+	registrationController := registration.NewAddonRegistrationController(
+		addonClient,
+		clusterInformers.Cluster().V1().ManagedClusters(),
+		addonInformers.Addon().V1alpha1().ManagedClusterAddOns(),
+		a.addonAgents,
+	)
+
+	// This controller is used during migrating addons to be managed by addon-manager.
+	// This should be removed when the migration is done.
+	// The migration plan refer to https://github.com/open-cluster-management-io/ocm/issues/355.
+	managementAddonController := cmamanagedby.NewCMAManagedByController(
+		addonClient,
+		addonInformers.Addon().V1alpha1().ClusterManagementAddOns(),
+		a.addonAgents,
+		utils.FilterByAddonName(a.addonAgents),
+	)
+
+	var addonConfigController, managementAddonConfigController factory.Controller
+	if len(a.addonConfigs) != 0 {
+		addonConfigController = addonconfig.NewAddonConfigController(
+			addonClient,
+			addonInformers.Addon().V1alpha1().ManagedClusterAddOns(),
+			addonInformers.Addon().V1alpha1().ClusterManagementAddOns(),
+			dynamicInformers,
+			a.addonConfigs,
+			utils.FilterByAddonName(a.addonAgents),
+		)
+		managementAddonConfigController = cmaconfig.NewCMAConfigController(
+			addonClient,
+			addonInformers.Addon().V1alpha1().ClusterManagementAddOns(),
+			dynamicInformers,
+			a.addonConfigs,
+			utils.FilterByAddonName(a.addonAgents),
+		)
+	}
+
+	var csrApproveController factory.Controller
+	var csrSignController factory.Controller
+	// Spawn the following controllers only if v1 CSR api is supported in the
+	// hub cluster. Under v1beta1 CSR api, all the CSR objects will be signed
+	// by the kube-controller-manager so custom CSR controller should be
+	// disabled to avoid conflict.
+	if v1CSRSupported {
+		csrApproveController = certificate.NewCSRApprovingController(
+			kubeClient,
+			clusterInformers.Cluster().V1().ManagedClusters(),
+			kubeInformers.Certificates().V1().CertificateSigningRequests(),
+			nil,
+			addonInformers.Addon().V1alpha1().ManagedClusterAddOns(),
+			a.addonAgents,
+		)
+		csrSignController = certificate.NewCSRSignController(
+			kubeClient,
+			clusterInformers.Cluster().V1().ManagedClusters(),
+			kubeInformers.Certificates().V1().CertificateSigningRequests(),
+			addonInformers.Addon().V1alpha1().ManagedClusterAddOns(),
+			a.addonAgents,
+		)
+	} else if v1beta1Supported {
+		csrApproveController = certificate.NewCSRApprovingController(
+			kubeClient,
+			clusterInformers.Cluster().V1().ManagedClusters(),
+			nil,
+			kubeInformers.Certificates().V1beta1().CertificateSigningRequests(),
+			addonInformers.Addon().V1alpha1().ManagedClusterAddOns(),
+			a.addonAgents,
+		)
+	}
+
+	a.syncContexts = append(a.syncContexts, deployController.SyncContext())
+
+	go deployController.Run(ctx, 1)
+	go registrationController.Run(ctx, 1)
+	go managementAddonController.Run(ctx, 1)
+
+	if addonConfigController != nil {
+		go addonConfigController.Run(ctx, 1)
+	}
+	if managementAddonConfigController != nil {
+		go managementAddonConfigController.Run(ctx, 1)
+	}
+	if csrApproveController != nil {
+		go csrApproveController.Run(ctx, 1)
+	}
+	if csrSignController != nil {
+		go csrSignController.Run(ctx, 1)
+	}
+	return nil
+}

pkg/addonmanager/cloudevents/manager.go

@@ -0,0 +1,176 @@
+package cloudevents
+
+import (
+	"context"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/dynamic/dynamicinformer"
+	kubeinformers "k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
+	addonv1alpha1client "open-cluster-management.io/api/client/addon/clientset/versioned"
+	addoninformers "open-cluster-management.io/api/client/addon/informers/externalversions"
+	clusterv1client "open-cluster-management.io/api/client/cluster/clientset/versioned"
+	clusterv1informers "open-cluster-management.io/api/client/cluster/informers/externalversions"
+	workinformers "open-cluster-management.io/api/client/work/informers/externalversions"
+
+	"open-cluster-management.io/addon-framework/pkg/addonmanager"
+	"open-cluster-management.io/addon-framework/pkg/index"
+	cloudeventswork "open-cluster-management.io/sdk-go/pkg/cloudevents/work"
+	"open-cluster-management.io/sdk-go/pkg/cloudevents/work/source/codec"
+)
+
+// cloudeventsAddonManager is the implementation of AddonManager with
+// the base implementation and cloudevents options
+type cloudeventsAddonManager struct {
+	*addonmanager.BaseAddonManagerImpl
+	options *CloudEventsOptions
+}
+
+func (a *cloudeventsAddonManager) Start(ctx context.Context) error {
+	config := a.GetConfig()
+	addonAgents := a.GetAddonAgents()
+
+	var addonNames []string
+	for key := range addonAgents {
+		addonNames = append(addonNames, key)
+	}
+
+	// To support sending ManifestWorks to different drivers (like the Kubernetes apiserver or MQTT broker), we build
+	// ManifestWork client that implements the ManifestWorkInterface and ManifestWork informer based on different
+	// driver configuration.
+	// Refer to Event Based Manifestwork proposal in enhancements repo to get more details.
+	_, clientConfig, err := cloudeventswork.NewConfigLoader(a.options.WorkDriver, a.options.WorkDriverConfig).
+		WithKubeConfig(a.GetConfig()).
+		LoadConfig()
+	if err != nil {
+		return err
+	}
+
+	// we need a separated filtered manifestwork informers so we only watch the manifestworks that manifestworkreplicaset cares.
+	// This could reduce a lot of memory consumptions
+	workInformOption := workinformers.WithTweakListOptions(
+		func(listOptions *metav1.ListOptions) {
+			selector := &metav1.LabelSelector{
+				MatchExpressions: []metav1.LabelSelectorRequirement{
+					{
+						Key:      addonv1alpha1.AddonLabelKey,
+						Operator: metav1.LabelSelectorOpIn,
+						Values:   addonNames,
+					},
+				},
+			}
+			listOptions.LabelSelector = metav1.FormatLabelSelector(selector)
+		},
+	)
+
+	clientHolder, err := cloudeventswork.NewClientHolderBuilder(clientConfig).
+		WithClientID(a.options.CloudEventsClientID).
+		WithSourceID(a.options.SourceID).
+		WithInformerConfig(10*time.Minute, workInformOption).
+		WithCodecs(codec.NewManifestBundleCodec()).
+		NewSourceClientHolder(ctx)
+	if err != nil {
+		return err
+	}
+
+	kubeClient, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return err
+	}
+
+	dynamicClient, err := dynamic.NewForConfig(config)
+	if err != nil {
+		return err
+	}
+
+	addonClient, err := addonv1alpha1client.NewForConfig(config)
+	if err != nil {
+		return err
+	}
+
+	clusterClient, err := clusterv1client.NewForConfig(config)
+	if err != nil {
+		return err
+	}
+
+	addonInformers := addoninformers.NewSharedInformerFactory(addonClient, 10*time.Minute)
+	clusterInformers := clusterv1informers.NewSharedInformerFactory(clusterClient, 10*time.Minute)
+	dynamicInformers := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 10*time.Minute)
+
+	kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, 10*time.Minute,
+		kubeinformers.WithTweakListOptions(func(listOptions *metav1.ListOptions) {
+			selector := &metav1.LabelSelector{
+				MatchExpressions: []metav1.LabelSelectorRequirement{
+					{
+						Key:      addonv1alpha1.AddonLabelKey,
+						Operator: metav1.LabelSelectorOpIn,
+						Values:   addonNames,
+					},
+				},
+			}
+			listOptions.LabelSelector = metav1.FormatLabelSelector(selector)
+		}),
+	)
+
+	workClient := clientHolder.WorkInterface()
+	workInformers := clientHolder.ManifestWorkInformer()
+
+	// addonDeployController
+	err = workInformers.Informer().AddIndexers(
+		cache.Indexers{
+			index.ManifestWorkByAddon:           index.IndexManifestWorkByAddon,
+			index.ManifestWorkByHostedAddon:     index.IndexManifestWorkByHostedAddon,
+			index.ManifestWorkHookByHostedAddon: index.IndexManifestWorkHookByHostedAddon,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	err = addonInformers.Addon().V1alpha1().ManagedClusterAddOns().Informer().AddIndexers(
+		cache.Indexers{
+			index.ManagedClusterAddonByNamespace: index.IndexManagedClusterAddonByNamespace, // addonDeployController
+			index.ManagedClusterAddonByName:      index.IndexManagedClusterAddonByName,      // addonConfigController
+			index.AddonByConfig:                  index.IndexAddonByConfig,                  // addonConfigController
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	err = addonInformers.Addon().V1alpha1().ClusterManagementAddOns().Informer().AddIndexers(
+		cache.Indexers{
+			index.ClusterManagementAddonByConfig:    index.IndexClusterManagementAddonByConfig,    // managementAddonConfigController
+			index.ClusterManagementAddonByPlacement: index.IndexClusterManagementAddonByPlacement, // addonConfigController
+		})
+	if err != nil {
+		return err
+	}
+
+	err = a.StartWithInformers(ctx, workClient, workInformers, kubeInformers, addonInformers, clusterInformers, dynamicInformers)
+	if err != nil {
+		return err
+	}
+
+	kubeInformers.Start(ctx.Done())
+	go workInformers.Informer().Run(ctx.Done())
+	addonInformers.Start(ctx.Done())
+	clusterInformers.Start(ctx.Done())
+	dynamicInformers.Start(ctx.Done())
+	return nil
+}
+
+// New returns a new addon manager with the given config and optional options
+func New(config *rest.Config, opts *CloudEventsOptions) (addonmanager.AddonManager, error) {
+	cloudeventsAddonManager := &cloudeventsAddonManager{
+		BaseAddonManagerImpl: addonmanager.NewBaseAddonManagerImpl(config),
+		options:              opts,
+	}
+
+	return cloudeventsAddonManager, nil
+}

pkg/addonmanager/cloudevents/options.go

@@ -0,0 +1,35 @@
+package cloudevents
+
+import (
+	"github.com/spf13/cobra"
+	cloudeventswork "open-cluster-management.io/sdk-go/pkg/cloudevents/work"
+)
+
+// CloudEventsOptions defines the flags for addon manager
+type CloudEventsOptions struct {
+	WorkDriver          string
+	WorkDriverConfig    string
+	CloudEventsClientID string
+	SourceID            string
+}
+
+// NewCloudEventsOptions returns the flags with default value set
+func NewCloudEventsOptions() *CloudEventsOptions {
+	return &CloudEventsOptions{
+		// set default work driver to kube
+		WorkDriver: cloudeventswork.ConfigTypeKube,
+	}
+}
+
+// AddFlags adds the cloudevents options flags to the given command
+func (o *CloudEventsOptions) AddFlags(cmd *cobra.Command) {
+	flags := cmd.Flags()
+	flags.StringVar(&o.WorkDriver, "work-driver",
+		o.WorkDriver, "The type of work driver, currently it can be kube, mqtt or grpc")
+	flags.StringVar(&o.WorkDriverConfig, "work-driver-config",
+		o.WorkDriverConfig, "The config file path of current work driver")
+	flags.StringVar(&o.CloudEventsClientID, "cloudevents-client-id",
+		o.CloudEventsClientID, "The ID of the cloudevents client when publishing works with cloudevents")
+	flags.StringVar(&o.SourceID, "source-id",
+		o.SourceID, "The ID of the source when publishing works with cloudevents")
+}

pkg/addonmanager/constants/constants.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
+	clusterv1 "open-cluster-management.io/api/cluster/v1"
 )
 
 const (
@@ -35,8 +36,11 @@ func PreDeleteHookHostingWorkName(addonNamespace, addonName string) string {
 }
 
 // GetHostedModeInfo returns addon installation mode and hosting cluster name.
-func GetHostedModeInfo(annotations map[string]string) (string, string) {
-	hostingClusterName, ok := annotations[addonv1alpha1.HostingClusterNameAnnotationKey]
+func GetHostedModeInfo(addon *addonv1alpha1.ManagedClusterAddOn, _ *clusterv1.ManagedCluster) (string, string) {
+	if len(addon.Annotations) == 0 {
+		return InstallModeDefault, ""
+	}
+	hostingClusterName, ok := addon.Annotations[addonv1alpha1.HostingClusterNameAnnotationKey]
 	if !ok {
 		return InstallModeDefault, ""
 	}

pkg/addonmanager/controllers/addonconfig/controller_test.go

@@ -113,8 +113,8 @@ func TestSync(t *testing.T) {
 					t.Errorf("Expect addon config generation is 2, but got %v", addOn.Status.ConfigReferences[0].LastObservedGeneration)
 				}
 
-				if addOn.Status.ConfigReferences[0].DesiredConfig.SpecHash != "3e80b3778b3b03766e7be993131c0af2ad05630c5d96fb7fa132d05b77336e04" {
-					t.Errorf("Expect addon config spec hash is 3e80b3778b3b03766e7be993131c0af2ad05630c5d96fb7fa132d05b77336e04, but got %v", addOn.Status.ConfigReferences[0].DesiredConfig.SpecHash)
+				if addOn.Status.ConfigReferences[0].DesiredConfig.SpecHash != "5f0f8e6cc1574ce8832b6bcefac131d90ae93653fc7204c085bd6e91b6df0892" {
+					t.Errorf("Expect addon config spec hash is 5f0f8e6cc1574ce8832b6bcefac131d90ae93653fc7204c085bd6e91b6df0892, but got %v", addOn.Status.ConfigReferences[0].DesiredConfig.SpecHash)
 				}
 			},
 		},
@@ -347,8 +347,8 @@ func TestSync(t *testing.T) {
 				if addOn.Status.ConfigReferences[0].LastObservedGeneration != 3 {
 					t.Errorf("Expect addon config generation is 3, but got %v", addOn.Status.ConfigReferences[0].LastObservedGeneration)
 				}
-				if addOn.Status.ConfigReferences[0].DesiredConfig.SpecHash != "3e80b3778b3b03766e7be993131c0af2ad05630c5d96fb7fa132d05b77336e04" {
-					t.Errorf("Expect addon config spec hash is 3e80b3778b3b03766e7be993131c0af2ad05630c5d96fb7fa132d05b77336e04, but got %v", addOn.Status.ConfigReferences[0].DesiredConfig.SpecHash)
+				if addOn.Status.ConfigReferences[0].DesiredConfig.SpecHash != "5f0f8e6cc1574ce8832b6bcefac131d90ae93653fc7204c085bd6e91b6df0892" {
+					t.Errorf("Expect addon config spec hash is 5f0f8e6cc1574ce8832b6bcefac131d90ae93653fc7204c085bd6e91b6df0892, but got %v", addOn.Status.ConfigReferences[0].DesiredConfig.SpecHash)
 				}
 			},
 		},