Fixing tests and fix cleanup.

Signed-off-by: Christoph Fiehe <c.fiehe@eurodata.de>
aptly-dev · Oct 12, 2024 · 3523ba7 · 3523ba7
1 parent 5dd225f
commit 3523ba7
Show file tree

Hide file tree

Showing 35 changed files with 114 additions and 57 deletions.
diff --git a/deb/publish.go b/deb/publish.go
@@ -1462,73 +1462,119 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix
 
 // CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
 func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(publishedStorageProvider aptly.PublishedStorageProvider,
-	published *PublishedRepo, addedComponents, updatedComponents, removedComponents []string,
+	published *PublishedRepo, _, updatedComponents, removedComponents []string,
 	collectionFactory *CollectionFactory, progress aptly.Progress) error {
 
+	var err error
+
 	collection.loadList()
 
+	storage := published.Storage
 	prefix := published.Prefix
 	distribution := published.Distribution
-	multiDist := published.MultiDist
 	publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage)
-
-	components := make([]string, 0, len(addedComponents)+len(updatedComponents)+len(removedComponents))
-	components = append(append(append(components, addedComponents...), updatedComponents...), removedComponents...)
-	sort.Strings(components)
+	publishedComponents := published.Components()
+	rootPath := filepath.Join(prefix, "dists", distribution)
 
 	for _, component := range removedComponents {
 		if progress != nil {
 			progress.Printf("Removing component %q from prefix %q...\n", component, prefix)
 		}
 
-		err := publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), progress)
+		err = publishedStorage.RemoveDirs(filepath.Join(rootPath, component), progress)
 		if err != nil {
 			return err
 		}
 
-		if multiDist {
-			for _, component := range removedComponents {
-				err = publishedStorage.RemoveDirs(filepath.Join(prefix, "pool", distribution, component), progress)
+		// Ensure that component does not exist in multi distribution pool
+		err = publishedStorage.RemoveDirs(filepath.Join(prefix, "pool", distribution, component), progress)
+		if err != nil {
+			return err
+		}
+	}
+
+	referencedFiles := map[string][]string{}
+
+	if published.MultiDist {
+		rootPath = filepath.Join(prefix, "pool", distribution)
+
+		// Get all referenced files by component for determining orphaned pool files.
+		for _, component := range publishedComponents {
+			packageList, err := NewPackageListFromRefList(published.RefList(component), collectionFactory.PackageCollection(), progress)
+			if err != nil {
+				return err
+			}
+
+			packageList.ForEach(func(p *Package) error {
+				poolDir, err := p.PoolDirectory()
 				if err != nil {
 					return err
 				}
+
+				for _, file := range p.Files() {
+					referencedFiles[component] = append(referencedFiles[component], filepath.Join(poolDir, file.Filename))
+				}
+
+				return nil
+			})
+		}
+	} else {
+		rootPath = filepath.Join(prefix, "pool")
+
+		// In case of a shared component pool directory, we must check, if a component is no longer referenced by any other
+		// published repository within the same prefix.
+		referencedComponents := map[string]struct{}{}
+		for _, p := range collection.list {
+			if p.Prefix == prefix && p.Storage == storage && !p.MultiDist {
+				for _, component := range p.Components() {
+					referencedComponents[component] = struct{}{}
+				}
 			}
 		}
-	}
 
-	components = make([]string, 0, len(updatedComponents)+len(removedComponents))
-	components = append(append(components, addedComponents...), updatedComponents...)
+		// Remove orphaned component pool directories in the prefix.
+		for _, component := range removedComponents {
+			_, exists := referencedComponents[component]
+			if !exists {
+				err := publishedStorage.RemoveDirs(filepath.Join(rootPath, component), progress)
+				if err != nil {
+					return err
+				}
+			}
+		}
 
-	referencedFiles, err := collection.listReferencedFilesByComponent(prefix, components, collectionFactory, progress)
-	if err != nil {
-		return err
+		// Get all referenced files by component for determining orphaned pool files.
+		referencedFiles, err = collection.listReferencedFilesByComponent(prefix, publishedComponents, collectionFactory, progress)
+		if err != nil {
+			return err
+		}
 	}
 
-	for _, component := range components {
+	for _, component := range updatedComponents {
 		if progress != nil {
 			progress.Printf("Cleaning up component %q in prefix %q...\n", component, prefix)
 		}
 		sort.Strings(referencedFiles[component])
 
-		rootPath := filepath.Join(prefix, "pool", component)
-		existingFiles, err := publishedStorage.Filelist(rootPath)
+		path := filepath.Join(rootPath, component)
+		existingFiles, err := publishedStorage.Filelist(path)
 		if err != nil {
 			return err
 		}
 
 		sort.Strings(existingFiles)
 
-		filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles[component])
+		orphanedFiles := utils.StrSlicesSubstract(existingFiles, referencedFiles[component])
 
-		for _, file := range filesToDelete {
-			err = publishedStorage.Remove(filepath.Join(rootPath, file))
+		for _, file := range orphanedFiles {
+			err = publishedStorage.Remove(filepath.Join(path, file))
 			if err != nil {
 				return err
 			}
 		}
 	}
 
-	return nil
+	return err
 }
 
 // Remove removes published repository, cleaning up directories, files

diff --git a/system/t06_publish/AzurePublish2Test_gold b/system/t06_publish/AzurePublish2Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository azure:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.
diff --git a/system/t06_publish/AzurePublish3Test_gold b/system/t06_publish/AzurePublish3Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository azure:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.
diff --git a/system/t06_publish/AzurePublish5Test_gold b/system/t06_publish/AzurePublish5Test_gold
@@ -1,3 +1,3 @@
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published repository has been removed successfully.
diff --git a/system/t06_publish/PublishDrop3Test_gold b/system/t06_publish/PublishDrop3Test_gold
@@ -1,4 +1,4 @@
 Removing ${HOME}/.aptly/public/dists/sq1...
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published repository has been removed successfully.
diff --git a/system/t06_publish/PublishDrop5Test_gold b/system/t06_publish/PublishDrop5Test_gold
@@ -1,4 +1,4 @@
 Removing ${HOME}/.aptly/public/dists/sq2...
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published repository has been removed successfully.
diff --git a/system/t06_publish/PublishDrop9Test_gold b/system/t06_publish/PublishDrop9Test_gold
@@ -1,4 +1,4 @@
 Removing ${HOME}/.aptly/public/dists/sq1...
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published repository has been removed successfully.
diff --git a/system/t06_publish/PublishSwitch11Test_gold b/system/t06_publish/PublishSwitch11Test_gold
@@ -5,6 +5,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./maverick [i386, source] publishes {main: [snap2]: Snapshot from local repo [local-repo2]} has been successfully switched to new source.
diff --git a/system/t06_publish/PublishSwitch13Test_gold b/system/t06_publish/PublishSwitch13Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.
diff --git a/system/t06_publish/PublishSwitch15Test_gold b/system/t06_publish/PublishSwitch15Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.
diff --git a/system/t06_publish/PublishSwitch16Test_gold b/system/t06_publish/PublishSwitch16Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./bookworm (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.
diff --git a/system/t06_publish/PublishSwitch1Test_gold b/system/t06_publish/PublishSwitch1Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.
diff --git a/system/t06_publish/PublishSwitch2Test_gold b/system/t06_publish/PublishSwitch2Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "ppa" components main...
+Cleaning up component "main" in prefix "ppa"...
 
 Published snapshot repository ppa/maverick [amd64, i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully switched to new source.
diff --git a/system/t06_publish/PublishSwitch3Test_gold b/system/t06_publish/PublishSwitch3Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.
diff --git a/system/t06_publish/PublishSwitch4Test_gold b/system/t06_publish/PublishSwitch4Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "ppa" components main...
+Cleaning up component "main" in prefix "ppa"...
 
 Published snapshot repository ppa/maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully switched to new source.
diff --git a/system/t06_publish/PublishSwitch8Test_gold b/system/t06_publish/PublishSwitch8Test_gold
@@ -3,6 +3,7 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components b, c...
+Cleaning up component "b" in prefix "."...
+Cleaning up component "c" in prefix "."...
 
 Published snapshot repository ./maverick [amd64, i386, source] publishes {a: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {b: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'}, {c: [local2]: Snapshot from local repo [local-repo]} has been successfully switched to new source.
diff --git a/system/t06_publish/PublishUpdate10Test_gold b/system/t06_publish/PublishUpdate10Test_gold
@@ -5,6 +5,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate11Test_gold b/system/t06_publish/PublishUpdate11Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate13Test_gold b/system/t06_publish/PublishUpdate13Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate14Test_gold b/system/t06_publish/PublishUpdate14Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./bookworm [i386, source] publishes {main: [local-repo]} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate15Test_gold b/system/t06_publish/PublishUpdate15Test_gold
@@ -3,6 +3,7 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components ...
+Cleaning up component "other-test" in prefix "."...
+Cleaning up component "test" in prefix "."...
 
 Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {other-test: [snap3]: Created as empty}, {test: [snap2]: Created as empty} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate16Test_gold b/system/t06_publish/PublishUpdate16Test_gold
@@ -3,6 +3,9 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components ...
+Removing component "other-test" from prefix "."...
+Removing ${HOME}/.aptly/public/dists/maverick/other-test...
+Removing component "test" from prefix "."...
+Removing ${HOME}/.aptly/public/dists/maverick/test...
 
 Published snapshot repository ./maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate17Test_gold b/system/t06_publish/PublishUpdate17Test_gold
@@ -3,6 +3,7 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components other-test, test...
+Cleaning up component "other-test" in prefix "."...
+Cleaning up component "test" in prefix "."...
 
 Published snapshot repository ./maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {other-test: [snap5]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {test: [snap4]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate18Test_gold b/system/t06_publish/PublishUpdate18Test_gold
@@ -3,6 +3,9 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components test...
+Removing component "main" from prefix "."...
+Removing /home/runner/.aptly/public/dists/maverick/main...
+Cleaning up component "other-test" in prefix "."...
+Cleaning up component "test" in prefix "."...
 
 Published snapshot repository ./maverick [i386] publishes {other-test: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {test: [snap3]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate1Test_gold b/system/t06_publish/PublishUpdate1Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate2Test_gold b/system/t06_publish/PublishUpdate2Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate3Test_gold b/system/t06_publish/PublishUpdate3Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate4Test_gold b/system/t06_publish/PublishUpdate4Test_gold
@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./maverick [source] publishes {main: [local-repo]} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate7Test_gold b/system/t06_publish/PublishUpdate7Test_gold
@@ -3,6 +3,7 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components contrib, main...
+Cleaning up component "contrib" in prefix "."...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./maverick [i386, source] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated.
diff --git a/system/t06_publish/PublishUpdate8Test_gold b/system/t06_publish/PublishUpdate8Test_gold
@@ -1,6 +1,7 @@
 Loading packages...
 Generating metadata files and linking package files...
 Finalizing metadata files...
-Cleaning up prefix "." components contrib, main...
+Cleaning up component "contrib" in prefix "."...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./squeeze [i386] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated.