Bundle stucked in modified loop caused by "app.kubernetes.io/managed-by":"helm"

[m3nax]

Is there an existing issue for this?

• I have searched the existing issues

Current Behavior

Fleet report some not ready resource after dapr helm chart installation and is stucked in "Modified" state.

Status:
  Conditions:
    Last Update Time:  2024-08-28T21:20:18Z
    Message:           Modified(1) [Cluster fleet-local/local]; clusterrole.rbac.authorization.k8s.io dapr-injector modified {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}; clusterrole.rbac.authorization.k8s.io dapr-operator-admin modified {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}; clusterrole.rbac.authorization.k8s.io dapr-placement modified {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}; clusterrole.rbac.authorization.k8s.io dapr-scheduler modified {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
    Status:            False
    Type:              Ready
  Display:
    Ready Clusters:            0/1
    State:                     Modified
  Max New:                     50
  Max Unavailable:             1
  Max Unavailable Partitions:  0
  Observed Generation:         3
  Partitions:
    Count:            1
    Max Unavailable:  1
    Name:             All
    Summary:
      Desired Ready:  1
      Modified:       1
      Non Ready Resources:
        Bundle State:  Modified
        Modified Status:
          API Version:  rbac.authorization.k8s.io/v1
          Kind:         ClusterRole
          Name:         dapr-injector
          Patch:        {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
          API Version:  rbac.authorization.k8s.io/v1
          Kind:         ClusterRole
          Name:         dapr-operator-admin
          Patch:        {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
          API Version:  rbac.authorization.k8s.io/v1
          Kind:         ClusterRole
          Name:         dapr-placement
          Patch:        {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
          API Version:  rbac.authorization.k8s.io/v1
          Kind:         ClusterRole
          Name:         dapr-scheduler
          Patch:        {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
          API Version:  rbac.authorization.k8s.io/v1
          Kind:         ClusterRole
          Name:         dapr-sentry
          Patch:        {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
          API Version:  rbac.authorization.k8s.io/v1
          Kind:         ClusterRoleBinding
          Name:         dapr-injector
          Patch:        {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
          API Version:  rbac.authorization.k8s.io/v1
          Kind:         ClusterRoleBinding
          Name:         dapr-operator-admin
          Patch:        {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
          API Version:  rbac.authorization.k8s.io/v1
          Kind:         ClusterRoleBinding
          Name:         dapr-placement
          Patch:        {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
          API Version:  rbac.authorization.k8s.io/v1
          Kind:         ClusterRoleBinding
          Name:         dapr-scheduler
          Patch:        {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
          API Version:  rbac.authorization.k8s.io/v1
          Kind:         ClusterRoleBinding
          Name:         dapr-sentry
          Patch:        {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
        Name:           fleet-local/local
      Ready:            0
  Resource Key:
    API Version:       policy/v1
    Kind:              PodDisruptionBudget
    Name:              dapr-scheduler-server-disruption-budget
    Namespace:         dapr-system
    API Version:       v1
    Kind:              ServiceAccount
    Name:              dapr-injector
    Namespace:         dapr-system
    API Version:       v1
    Kind:              ServiceAccount
    Name:              dapr-operator
    Namespace:         dapr-system
    API Version:       v1
    Kind:              ServiceAccount
    Name:              dapr-placement
    Namespace:         dapr-system
    API Version:       v1
    Kind:              ServiceAccount
    Name:              dapr-scheduler
    Namespace:         dapr-system
    API Version:       v1
    Kind:              ServiceAccount
    Name:              dapr-sentry
    Namespace:         dapr-system
    API Version:       v1
    Kind:              Secret
    Name:              dapr-trust-bundle
    Namespace:         dapr-system
    API Version:       v1
    Kind:              ConfigMap
    Name:              dapr-trust-bundle
    Namespace:         dapr-system
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              ClusterRole
    Name:              dapr-injector
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              ClusterRole
    Name:              dapr-operator-admin
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              ClusterRole
    Name:              dapr-placement
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              ClusterRole
    Name:              dapr-scheduler
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              ClusterRole
    Name:              dapr-sentry
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              ClusterRoleBinding
    Name:              dapr-injector
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              ClusterRoleBinding
    Name:              dapr-operator-admin
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              ClusterRoleBinding
    Name:              dapr-placement
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              ClusterRoleBinding
    Name:              dapr-scheduler
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              ClusterRoleBinding
    Name:              dapr-sentry
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              Role
    Name:              dapr-injector
    Namespace:         dapr-system
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              Role
    Name:              dapr-operator
    Namespace:         dapr-system
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              Role
    Name:              dapr-sentry
    Namespace:         dapr-system
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              Role
    Name:              secret-reader
    Namespace:         default
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              RoleBinding
    Name:              dapr-injector
    Namespace:         dapr-system
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              RoleBinding
    Name:              dapr-operator
    Namespace:         dapr-system
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              RoleBinding
    Name:              dapr-sentry
    Namespace:         dapr-system
    API Version:       rbac.authorization.k8s.io/v1
    Kind:              RoleBinding
    Name:              dapr-secret-reader
    Namespace:         default
    API Version:       v1
    Kind:              Service
    Name:              dapr-api
    Namespace:         dapr-system
    API Version:       v1
    Kind:              Service
    Name:              dapr-placement-server
    Namespace:         dapr-system
    API Version:       v1
    Kind:              Service
    Name:              dapr-scheduler-server
    Namespace:         dapr-system
    API Version:       v1
    Kind:              Service
    Name:              dapr-sentry
    Namespace:         dapr-system
    API Version:       v1
    Kind:              Service
    Name:              dapr-sidecar-injector
    Namespace:         dapr-system
    API Version:       v1
    Kind:              Service
    Name:              dapr-webhook
    Namespace:         dapr-system
    API Version:       apps/v1
    Kind:              Deployment
    Name:              dapr-operator
    Namespace:         dapr-system
    API Version:       apps/v1
    Kind:              Deployment
    Name:              dapr-sentry
    Namespace:         dapr-system
    API Version:       apps/v1
    Kind:              Deployment
    Name:              dapr-sidecar-injector
    Namespace:         dapr-system
    API Version:       apps/v1
    Kind:              StatefulSet
    Name:              dapr-placement-server
    Namespace:         dapr-system
    API Version:       apps/v1
    Kind:              StatefulSet
    Name:              dapr-scheduler-server
    Namespace:         dapr-system
    API Version:       dapr.io/v1alpha1
    Kind:              Configuration
    Name:              daprsystem
    Namespace:         dapr-system
    API Version:       admissionregistration.k8s.io/v1
    Kind:              MutatingWebhookConfiguration
    Name:              dapr-sidecar-injector
  resourcesSha256Sum:  c7b4eb50854ee495c0b55e2461ea95b2aaffa4b258f8d4f7672731c50be2f8a9
  Summary:
    Desired Ready:  1
    Modified:       1
    Non Ready Resources:
      Bundle State:  Modified
      Modified Status:
        API Version:       rbac.authorization.k8s.io/v1
        Kind:              ClusterRole
        Name:              dapr-injector
        Patch:             {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
        API Version:       rbac.authorization.k8s.io/v1
        Kind:              ClusterRole
        Name:              dapr-operator-admin
        Patch:             {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
        API Version:       rbac.authorization.k8s.io/v1
        Kind:              ClusterRole
        Name:              dapr-placement
        Patch:             {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
        API Version:       rbac.authorization.k8s.io/v1
        Kind:              ClusterRole
        Name:              dapr-scheduler
        Patch:             {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
        API Version:       rbac.authorization.k8s.io/v1
        Kind:              ClusterRole
        Name:              dapr-sentry
        Patch:             {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
        API Version:       rbac.authorization.k8s.io/v1
        Kind:              ClusterRoleBinding
        Name:              dapr-injector
        Patch:             {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
        API Version:       rbac.authorization.k8s.io/v1
        Kind:              ClusterRoleBinding
        Name:              dapr-operator-admin
        Patch:             {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
        API Version:       rbac.authorization.k8s.io/v1
        Kind:              ClusterRoleBinding
        Name:              dapr-placement
        Patch:             {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
        API Version:       rbac.authorization.k8s.io/v1
        Kind:              ClusterRoleBinding
        Name:              dapr-scheduler
        Patch:             {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
        API Version:       rbac.authorization.k8s.io/v1
        Kind:              ClusterRoleBinding
        Name:              dapr-sentry
        Patch:             {"metadata":{"labels":{"app.kubernetes.io/managed-by":"helm"}}}
      Name:                fleet-local/local
    Ready:                 0
  Unavailable:             0
  Unavailable Partitions:  0
Events:                    <none>

Expected Behavior

Dapr bundle deployment reach 1/1 BUNDLEDEPLOYMENTS-READY when executing kubectl get bundle -n fleet-local

Steps To Reproduce

1. Create k3s config:

sudo mkdir -p /etc/rancher/k3s
cat <<EOF | sudo tee /etc/rancher/k3s/config.yaml
write-kubeconfig-mode: "0644"
disable: traefik,servicelb
kubelet-arg:
  - "resolv-conf=/etc/resolv.conf"
EOF

2. Bootstrap a k3s node sudo curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.30.4+k3s1 sh -
3. Install fleet cdr helm -n cattle-fleet-system install --version 0.10.1 --create-namespace --wait fleet-crd fleet/fleet-crd
4. Install fleet helm -n cattle-fleet-system install --version 0.10.1 --create-namespace --wait fleet fleet/fleet
5. Add a GitRepo CDR
6. Add following fleet.yaml to the monitored GIT repo

# The default namespace to be applied to resources. This field is not used to
# enforce or lock down the deployment to a specific namespace, but instead
# provide the default value of the namespace field if one is not specified in
# the manifests.
#
# Default: default
defaultNamespace: dapr-system

helm:

  # These options control how "fleet apply" downloads the chart
  #
  # Use a custom location for the Helm chart. This can refer to any go-getter
  # URL or OCI registry based helm chart URL e.g.
  # "oci://ghcr.io/fleetrepoci/guestbook".  This allows one to download charts
  # from most any location.  Also know that go-getter URL supports adding a
  # digest to validate the download. If repo is set below this field is the name
  # of the chart to lookup.
  #
  # It is possible to download the chart from a Git repository, e.g.  by using
  # `git@github.com:rancher/fleet-examples//single-cluster/helm`. If a secret
  # for the SSH key was defined in the GitRepo via `helmSecretName`, it will be
  # injected into the chart URL.
  #
  # Git repositories can be downloaded via unauthenticated http, by using for
  # example:
  #
  # `git::http://github.com/rancher/fleet-examples/single-cluster/helm`.
  chart: dapr 

  # A https URL to a Helm repo to download the chart from. It's typically easier
  # to just use `chart` field and refer to a tgz file.  If repo is used the
  # value of `chart` will be used as the chart name to lookup in the Helm
  # repository.
  repo: https://dapr.github.io/helm-charts/

  # The version of the chart or semver constraint of the chart to find. If a
  # constraint is specified it is evaluated each time git changes.
  #
  # The version also determines which chart to download from OCI registries.
  # Note: OCI registries don't support the '+' character, which is supported by
  # semver.  When pushing a helm chart with a tag containing the '+' character
  # helm automatically replaces '+' to '_' before uploading it.
  #
  # You should use the version with the '+' in this file, as the '_' character
  # is not supported by semver and Fleet also replaces '+' to '_' when accessing
  # the OCI registry.
  version: 1.14.1

  # By default fleet downloads any dependency found in a helm chart.  Use
  # disableDependencyUpdate: true to disable this feature.
  disableDependencyUpdate: false

  ### These options control how fleet-agent deploys the bundle, they also apply
  ### for kustomize- and manifest-style bundles.
  #
  # A custom release name to deploy the chart as. If not specified a release name
  # will be generated by combining the invoking GitRepo.name + GitRepo.path.
  releaseName: dapr 
  #
  # Makes helm skip the check for its own annotations
  takeOwnership: false
  #
  # Override immutable resources. This could be dangerous.
  force: false
  #
  # Set the Helm --atomic flag when upgrading
  atomic: false
  #
  # Disable go template pre-processing on the fleet values
  disablePreProcess: false
  #
  # Disable DNS resolution in Helm's template functions
  disableDNS: false
  #
  # Skip evaluation of the values.schema.json file
  skipSchemaValidation: false
  #
  # If set and timeoutSeconds provided, will wait until all Jobs have been
  # completed before marking the GitRepo as ready.  It will wait for as long as
  # timeoutSeconds.
  waitForJobs: true

# A paused bundle will not update downstream clusters but instead mark the bundle
# as OutOfSync. One can then manually confirm that a bundle should be deployed to
# the downstream clusters.
#
# Default: false
paused: false

Environment

- Architecture: ARM64 (RPI5)
- Fleet Version: 0.10.1
- Cluster:
  - Provider: K3S
  - Options: 1 node
  - Kubernetes Version: v1.30.4+k3s1 (OS: Ubuntu)

Logs

No response

Anything else?

The k3s node was installed today and it is used for fleet gitops evaluation. Dapr control plane seems correctly installed.

get po --all-namespaces --sort-by=.metadata.namespace output:

NAMESPACE             NAME                                      READY   STATUS      RESTARTS   AGE
cattle-fleet-system   fleet-agent-0                             2/2     Running     0          90m
cattle-fleet-system   fleet-controller-5cbf7444d4-6vgq2         3/3     Running     0          90m
cattle-fleet-system   gitjob-994c56c64-xn8mw                    1/1     Running     0          90m
dapr-system           dapr-operator-79cbb85bf8-d8gh9            1/1     Running     0          30m
dapr-system           dapr-placement-server-0                   1/1     Running     0          30m
dapr-system           dapr-scheduler-server-0                   1/1     Running     0          30m
dapr-system           dapr-sentry-5768f86dc-vbp59               1/1     Running     0          30m
dapr-system           dapr-sidecar-injector-6579c6449d-fpmjn    1/1     Running     0          30m
fleet-local           k8r-fleet-test-05854-k2x74                0/1     Completed   0          25m
fleet-local           k8r-fleet-test-45bba-2bl9s                0/1     Completed   0          23m
fleet-local           k8r-fleet-test-75cce-n4bbb                0/1     Completed   0          30m
internal              internal-web-6876fd78cb-g4w4z             1/1     Running     0          30m
kube-system           coredns-576bfc4dc7-sdrg8                  1/1     Running     0          101m
kube-system           local-path-provisioner-6795b5f9d8-j55gc   1/1     Running     0          101m
kube-system           metrics-server-557ff575fb-r5jwh           1/1     Running     0          101m

[m3nax]

fleet.yaml with workaround:

# The default namespace to be applied to resources. This field is not used to
# enforce or lock down the deployment to a specific namespace, but instead
# provide the default value of the namespace field if one is not specified in
# the manifests.
#
# Default: default
defaultNamespace: dapr-system

helm:

  # These options control how "fleet apply" downloads the chart
  #
  # Use a custom location for the Helm chart. This can refer to any go-getter
  # URL or OCI registry based helm chart URL e.g.
  # "oci://ghcr.io/fleetrepoci/guestbook".  This allows one to download charts
  # from most any location.  Also know that go-getter URL supports adding a
  # digest to validate the download. If repo is set below this field is the name
  # of the chart to lookup.
  #
  # It is possible to download the chart from a Git repository, e.g.  by using
  # `git@github.com:rancher/fleet-examples//single-cluster/helm`. If a secret
  # for the SSH key was defined in the GitRepo via `helmSecretName`, it will be
  # injected into the chart URL.
  #
  # Git repositories can be downloaded via unauthenticated http, by using for
  # example:
  #
  # `git::http://github.com/rancher/fleet-examples/single-cluster/helm`.
  chart: dapr 

  # A https URL to a Helm repo to download the chart from. It's typically easier
  # to just use `chart` field and refer to a tgz file.  If repo is used the
  # value of `chart` will be used as the chart name to lookup in the Helm
  # repository.
  repo: https://dapr.github.io/helm-charts/

  # The version of the chart or semver constraint of the chart to find. If a
  # constraint is specified it is evaluated each time git changes.
  #
  # The version also determines which chart to download from OCI registries.
  # Note: OCI registries don't support the '+' character, which is supported by
  # semver.  When pushing a helm chart with a tag containing the '+' character
  # helm automatically replaces '+' to '_' before uploading it.
  #
  # You should use the version with the '+' in this file, as the '_' character
  # is not supported by semver and Fleet also replaces '+' to '_' when accessing
  # the OCI registry.
  version: 1.14.1

  # By default fleet downloads any dependency found in a helm chart.  Use
  # disableDependencyUpdate: true to disable this feature.
  disableDependencyUpdate: false

  ### These options only work for helm-type bundles.
  #
  # Any values that should be placed in the `values.yaml` and passed to helm
  # during install.
  values:
    global:
      k8sLabels:
        app.kubernetes.io/managed-by: "{{ .Release.Service }}"

  ### These options control how fleet-agent deploys the bundle, they also apply
  ### for kustomize- and manifest-style bundles.
  #
  # A custom release name to deploy the chart as. If not specified a release name
  # will be generated by combining the invoking GitRepo.name + GitRepo.path.
  releaseName: dapr 
  #
  # Makes helm skip the check for its own annotations
  takeOwnership: false
  #
  # Override immutable resources. This could be dangerous.
  force: false
  #
  # Set the Helm --atomic flag when upgrading
  atomic: false
  #
  # Disable go template pre-processing on the fleet values
  disablePreProcess: false
  #
  # Disable DNS resolution in Helm's template functions
  disableDNS: false
  #
  # Skip evaluation of the values.schema.json file
  skipSchemaValidation: false
  #
  # If set and timeoutSeconds provided, will wait until all Jobs have been
  # completed before marking the GitRepo as ready.  It will wait for as long as
  # timeoutSeconds.
  waitForJobs: true

# A paused bundle will not update downstream clusters but instead mark the bundle
# as OutOfSync. One can then manually confirm that a bundle should be deployed to
# the downstream clusters.
#
# Default: false
paused: false