Skip checks on logs in sharding end-to-end tests (#2832)

* Skip checks on logs in sharding tests This prevents test failures when: * tests are run more than once, without logs being cleared (eg. without deployments being scaled down and back up) between runs * debug logs are disabled * Add shard ID checks on bundle and bundle deployment Sharding end-to-end tests now check that Fleet resources created from a `GitRepo` bear the same shard ID value, through the `fleet.cattle.io/shard-ref` label, as specified in the `GitRepo` itself. User resources contained in bundle deployments are not labelled with the shard ID. * Fix check on configmap absence When checking for a config map not being created, we must check that it _consistently_ doesn't exist, instead of verifying that it eventually (at a unique point in time) doesn't exist.
rancher · Sep 13, 2024 · 4953fe7 · 4953fe7
1 parent decbf9a
commit 4953fe7
Showing 1 changed file with 44 additions and 95 deletions.
diff --git a/e2e/single-cluster/sharding_test.go b/e2e/single-cluster/sharding_test.go
@@ -7,47 +7,30 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"github.com/onsi/gomega/matchers"
 	"github.com/rancher/fleet/e2e/testenv"
 	"github.com/rancher/fleet/e2e/testenv/kubectl"
 )
 
 var shards = []string{"shard0", "shard1", "shard2"}
 
-var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func() {
+var _ = Describe("Filtering events by shard", Label("sharding"), func() {
 	var (
 		k               kubectl.Command
 		gitrepoName     string
 		r               = rand.New(rand.NewSource(GinkgoRandomSeed()))
 		targetNamespace string
 	)
 
-	BeforeAll(func() {
-		// No sharded gitjob controller should have reconciled any GitRepo until this point.
-		for _, shard := range shards {
-			logs, err := k.Namespace("cattle-fleet-system").Logs(
-				"-l",
-				"app=gitjob",
-				"-l",
-				fmt.Sprintf("fleet.cattle.io/shard-id=%s", shard),
-				"--tail=-1",
-			)
-			Expect(err).ToNot(HaveOccurred())
-			regexMatcher := matchers.MatchRegexpMatcher{Regexp: "Reconciling GitRepo.*"}
-			hasReconciledGitRepos, err := regexMatcher.Match(logs)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(hasReconciledGitRepos).To(BeFalse())
-		}
-
+	BeforeEach(func() {
 		k = env.Kubectl.Namespace(env.Namespace)
+		targetNamespace = testenv.NewNamespaceName("target", r)
+		gitrepoName = testenv.RandomFilename("sharding-test", r)
+
 	})
 
 	for _, shard := range shards {
 		When(fmt.Sprintf("deploying a gitrepo labeled with shard ID %s", shard), func() {
 			JustBeforeEach(func() {
-				targetNamespace = testenv.NewNamespaceName("target", r)
-				gitrepoName = testenv.RandomFilename("sharding-test", r)
-
 				err := testenv.ApplyTemplate(
 					k,
 					testenv.AssetPath("gitrepo/gitrepo_sharded.yaml"),
@@ -77,15 +60,46 @@ var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func()
 					return out
 				}).Should(ContainSubstring("test-simple-chart-config"))
 
-				By("checking the gitjob pod has the same nodeSelector as the sharded controller")
+				By("checking the bundle bears the shard label with the right shard ID")
+				bundleName := fmt.Sprintf("%s-simple-chart", gitrepoName)
+				Eventually(func(g Gomega) {
+					shardLabelValue, err := k.Get(
+						"bundle",
+						bundleName,
+						`-o jsonpath='{.metadata.labels.fleet\.cattle\.io/shard-ref}'`,
+					)
+					g.Expect(err).ToNot(HaveOccurred())
+					g.Expect(shardLabelValue).To(Equal(shard))
+				})
+
+				By("checking the bundle deployment bears the shard label with the right shard ID")
+				clusterNS, err := k.Get(
+					"cluster.fleet.cattle.io",
+					"local",
+					"-n",
+					"fleet-local",
+					`-o=jsonpath='{.status.namespace}'`,
+				)
+				Expect(err).ToNot(HaveOccurred())
+
+				Eventually(func(g Gomega) {
+					shardLabelValue, err := k.Get(
+						"bundledeployment",
+						bundleName,
+						"-n",
+						clusterNS,
+						`-o=jsonpath='{.metadata.labels.fleet\.cattle\.io/shard-ref}'`,
+					)
+					g.Expect(err).ToNot(HaveOccurred())
+					g.Expect(shardLabelValue).To(Equal(shard))
+				})
+
+				By("checking the gitjob pod has the same nodeSelector as the sharded controller deployment")
 				Eventually(func(g Gomega) {
 					shardNodeSelector, err := k.Namespace("cattle-fleet-system").Get(
-						"pods",
-						"-o=jsonpath={.items[0].spec.nodeSelector}",
-						"-l",
-						"app=fleet-controller",
-						"-l",
-						fmt.Sprintf("fleet.cattle.io/shard-id=%s", shard),
+						"deploy",
+						fmt.Sprintf("fleet-controller-shard-%s", shard),
+						"-o=jsonpath={.spec.template.spec.nodeSelector}",
 					)
 					g.Expect(err).ToNot(HaveOccurred())
 
@@ -108,42 +122,6 @@ var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func()
 					g.Expect(podNodeSelector).ToNot(BeEmpty())
 					g.Expect(podNodeSelector).To(Equal(shardNodeSelector))
 				}).Should(Succeed())
-
-				for _, s := range shards {
-					Eventually(func(g Gomega) {
-						logs, err := k.Namespace("cattle-fleet-system").Logs(
-							"-l",
-							"app=gitjob",
-							"-l",
-							fmt.Sprintf("fleet.cattle.io/shard-id=%s", s),
-							"--tail=100",
-						)
-						g.Expect(err).ToNot(HaveOccurred())
-						regexMatcher := matchers.MatchRegexpMatcher{
-							Regexp: fmt.Sprintf(`Reconciling GitRepo.*"name":"%s"`, gitrepoName),
-						}
-						hasReconciledGitRepo, err := regexMatcher.Match(logs)
-						g.Expect(err).ToNot(HaveOccurred())
-						if s == shard {
-							g.Expect(hasReconciledGitRepo).To(BeTrueBecause(
-								"GitRepo %q labeled with shard %q should have been"+
-									" deployed by gitjob for shard %q in namespace %q",
-								gitrepoName,
-								shard,
-								shard,
-								env.Namespace,
-							))
-						} else {
-							g.Expect(hasReconciledGitRepo).To(BeFalseBecause(
-								"GitRepo %q labeled with shard %q should not have been"+
-									" deployed by gitjob for shard %q",
-								gitrepoName,
-								shard,
-								s,
-							))
-						}
-					}).Should(Succeed())
-				}
 			})
 
 			AfterEach(func() {
@@ -155,9 +133,6 @@ var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func()
 
 	When("deploying a gitrepo labeled with an unknown shard ID", func() {
 		JustBeforeEach(func() {
-			targetNamespace = testenv.NewNamespaceName("target", r)
-			gitrepoName = testenv.RandomFilename("sharding-test", r)
-
 			err := testenv.ApplyTemplate(k, testenv.AssetPath("gitrepo/gitrepo_sharded.yaml"), struct {
 				Name            string
 				Repo            string
@@ -178,36 +153,10 @@ var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func()
 
 		It("does not deploy the gitrepo", func() {
 			By("checking the configmap does not exist")
-			Eventually(func() string {
+			Consistently(func() string {
 				out, _ := k.Namespace(targetNamespace).Get("configmaps")
 				return out
 			}).ShouldNot(ContainSubstring("test-simple-chart-config"))
-
-			for _, s := range shards {
-				logs, err := k.Namespace("cattle-fleet-system").Logs(
-					"-l",
-					"app=gitjob",
-					"-l",
-					fmt.Sprintf("fleet.cattle.io/shard-id=%s", s),
-					"--tail=100",
-				)
-				Expect(err).ToNot(HaveOccurred())
-				regexMatcher := matchers.MatchRegexpMatcher{
-					Regexp: fmt.Sprintf(
-						`Reconciling GitRepo.*"GitRepo": {"name":"%s","namespace":"%s"}`,
-						gitrepoName,
-						env.Namespace,
-					),
-				}
-				hasReconciledGitRepos, err := regexMatcher.Match(logs)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(hasReconciledGitRepos).To(BeFalseBecause(
-					"GitRepo labeled with shard %q should not have been deployed by"+
-						" gitjob for shard %q",
-					"unknown",
-					s,
-				))
-			}
 		})
 
 		AfterEach(func() {