Fixing tests and fix cleanup.

Signed-off-by: Christoph Fiehe <c.fiehe@eurodata.de>

api/publish.go

@@ -438,8 +438,9 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		}
 
 		if b.SkipCleanup == nil || !*b.SkipCleanup {
-			err = collection.CleanupPrefixComponentFiles(context, published, result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(),
-				collectionFactory, out)
+			cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
+			cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
+			err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 			}
@@ -846,8 +847,9 @@ func apiPublishUpdate(c *gin.Context) {
 		}
 
 		if b.SkipCleanup == nil || !*b.SkipCleanup {
-			err = collection.CleanupPrefixComponentFiles(context, published,
-				result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(), collectionFactory, out)
+			cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
+			cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
+			err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 			}

cmd/publish_switch.go

@@ -115,8 +115,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 
 	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
 	if !skipCleanup {
-		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published,
-			[]string{}, components, []string{}, collectionFactory, context.Progress())
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, components, collectionFactory, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}

cmd/publish_update.go

@@ -76,8 +76,9 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 
 	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
 	if !skipCleanup {
-		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published,
-			result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(), collectionFactory, context.Progress())
+		cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
+		cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}

deb/publish.go

@@ -1462,73 +1462,127 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix
 
 // CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
 func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(publishedStorageProvider aptly.PublishedStorageProvider,
-	published *PublishedRepo, addedComponents, updatedComponents, removedComponents []string,
-	collectionFactory *CollectionFactory, progress aptly.Progress) error {
+	published *PublishedRepo, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
+
+	var err error
 
 	collection.loadList()
 
+	storage := published.Storage
 	prefix := published.Prefix
 	distribution := published.Distribution
-	multiDist := published.MultiDist
+
+	rootPath := filepath.Join(prefix, "dists", distribution)
 	publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage)
 
-	components := make([]string, 0, len(addedComponents)+len(updatedComponents)+len(removedComponents))
-	components = append(append(append(components, addedComponents...), updatedComponents...), removedComponents...)
-	sort.Strings(components)
+	sort.Strings(cleanComponents)
+	publishedComponents := published.Components()
+	removedComponents := utils.StrSlicesSubstract(cleanComponents, publishedComponents)
+	updatedComponents := utils.StrSlicesSubstract(cleanComponents, removedComponents)
+
+	if progress != nil {
+		progress.Printf("Cleaning up published repository %s/%s...\n", published.StoragePrefix(), distribution)
+	}
 
 	for _, component := range removedComponents {
 		if progress != nil {
-			progress.Printf("Removing component %q from prefix %q...\n", component, prefix)
+			progress.Printf("Removing component %q...\n", component)
 		}
 
-		err := publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), progress)
+		err = publishedStorage.RemoveDirs(filepath.Join(rootPath, component), progress)
 		if err != nil {
 			return err
 		}
 
-		if multiDist {
-			for _, component := range removedComponents {
-				err = publishedStorage.RemoveDirs(filepath.Join(prefix, "pool", distribution, component), progress)
+		// Ensure that component does not exist in multi distribution pool
+		err = publishedStorage.RemoveDirs(filepath.Join(prefix, "pool", distribution, component), nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	referencedFiles := map[string][]string{}
+
+	if published.MultiDist {
+		rootPath = filepath.Join(prefix, "pool", distribution)
+
+		// Get all referenced files by component for determining orphaned pool files.
+		for _, component := range publishedComponents {
+			packageList, err := NewPackageListFromRefList(published.RefList(component), collectionFactory.PackageCollection(), progress)
+			if err != nil {
+				return err
+			}
+
+			packageList.ForEach(func(p *Package) error {
+				poolDir, err := p.PoolDirectory()
 				if err != nil {
 					return err
 				}
+
+				for _, file := range p.Files() {
+					referencedFiles[component] = append(referencedFiles[component], filepath.Join(poolDir, file.Filename))
+				}
+
+				return nil
+			})
+		}
+	} else {
+		rootPath = filepath.Join(prefix, "pool")
+
+		// In case of a shared component pool directory, we must check, if a component is no longer referenced by any other
+		// published repository within the same prefix.
+		referencedComponents := map[string]struct{}{}
+		for _, p := range collection.list {
+			if p.Prefix == prefix && p.Storage == storage && !p.MultiDist {
+				for _, component := range p.Components() {
+					referencedComponents[component] = struct{}{}
+				}
 			}
 		}
-	}
 
-	components = make([]string, 0, len(updatedComponents)+len(removedComponents))
-	components = append(append(components, addedComponents...), updatedComponents...)
+		// Remove orphaned component pool directories in the prefix.
+		for _, component := range removedComponents {
+			_, exists := referencedComponents[component]
+			if !exists {
+				err := publishedStorage.RemoveDirs(filepath.Join(rootPath, component), progress)
+				if err != nil {
+					return err
+				}
+			}
+		}
 
-	referencedFiles, err := collection.listReferencedFilesByComponent(prefix, components, collectionFactory, progress)
-	if err != nil {
-		return err
+		// Get all referenced files by component for determining orphaned pool files.
+		referencedFiles, err = collection.listReferencedFilesByComponent(prefix, publishedComponents, collectionFactory, progress)
+		if err != nil {
+			return err
+		}
 	}
 
-	for _, component := range components {
+	for _, component := range updatedComponents {
 		if progress != nil {
-			progress.Printf("Cleaning up component %q in prefix %q...\n", component, prefix)
+			progress.Printf("Cleaning up component %q...\n", component)
 		}
 		sort.Strings(referencedFiles[component])
 
-		rootPath := filepath.Join(prefix, "pool", component)
-		existingFiles, err := publishedStorage.Filelist(rootPath)
+		path := filepath.Join(rootPath, component)
+		existingFiles, err := publishedStorage.Filelist(path)
 		if err != nil {
 			return err
 		}
 
 		sort.Strings(existingFiles)
 
-		filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles[component])
+		orphanedFiles := utils.StrSlicesSubstract(existingFiles, referencedFiles[component])
 
-		for _, file := range filesToDelete {
-			err = publishedStorage.Remove(filepath.Join(rootPath, file))
+		for _, file := range orphanedFiles {
+			err = publishedStorage.Remove(filepath.Join(path, file))
 			if err != nil {
 				return err
 			}
 		}
 	}
 
-	return nil
+	return err
 }
 
 // Remove removes published repository, cleaning up directories, files
@@ -1579,8 +1633,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
 	if !skipCleanup && len(cleanComponents) > 0 {
-		err = collection.CleanupPrefixComponentFiles(publishedStorageProvider, repo, []string{}, cleanComponents, []string{},
-			collectionFactory, progress)
+		err = collection.CleanupPrefixComponentFiles(publishedStorageProvider, repo, cleanComponents, collectionFactory, progress)
 		if err != nil {
 			if !force {
 				return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err)

system/t06_publish/AzurePublish2Test_gold

@@ -3,6 +3,7 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up published repository azure:test1:./maverick...
+Cleaning up component "main"...
 
 Published local repository azure:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.

system/t06_publish/AzurePublish3Test_gold

@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository azure:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.

system/t06_publish/AzurePublish5Test_gold

@@ -1,3 +1,3 @@
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published repository has been removed successfully.

system/t06_publish/PublishDrop3Test_gold

@@ -1,4 +1,4 @@
 Removing ${HOME}/.aptly/public/dists/sq1...
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published repository has been removed successfully.

system/t06_publish/PublishDrop5Test_gold

@@ -1,4 +1,4 @@
 Removing ${HOME}/.aptly/public/dists/sq2...
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published repository has been removed successfully.

system/t06_publish/PublishDrop9Test_gold

@@ -1,4 +1,4 @@
 Removing ${HOME}/.aptly/public/dists/sq1...
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published repository has been removed successfully.

system/t06_publish/PublishSwitch11Test_gold

@@ -5,6 +5,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./maverick [i386, source] publishes {main: [snap2]: Snapshot from local repo [local-repo2]} has been successfully switched to new source.

system/t06_publish/PublishSwitch13Test_gold

@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.

system/t06_publish/PublishSwitch15Test_gold

@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.

system/t06_publish/PublishSwitch16Test_gold

@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./bookworm (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.

system/t06_publish/PublishSwitch1Test_gold

@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.

system/t06_publish/PublishSwitch2Test_gold

@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "ppa" components main...
+Cleaning up component "main" in prefix "ppa"...
 
 Published snapshot repository ppa/maverick [amd64, i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully switched to new source.

system/t06_publish/PublishSwitch3Test_gold

@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source.

system/t06_publish/PublishSwitch4Test_gold

@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "ppa" components main...
+Cleaning up component "main" in prefix "ppa"...
 
 Published snapshot repository ppa/maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully switched to new source.

system/t06_publish/PublishSwitch8Test_gold

@@ -3,6 +3,7 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components b, c...
+Cleaning up component "b" in prefix "."...
+Cleaning up component "c" in prefix "."...
 
 Published snapshot repository ./maverick [amd64, i386, source] publishes {a: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {b: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'}, {c: [local2]: Snapshot from local repo [local-repo]} has been successfully switched to new source.

system/t06_publish/PublishUpdate10Test_gold

@@ -5,6 +5,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.

system/t06_publish/PublishUpdate11Test_gold

@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.

system/t06_publish/PublishUpdate13Test_gold

@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.

system/t06_publish/PublishUpdate14Test_gold

@@ -3,6 +3,6 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components main...
+Cleaning up component "main" in prefix "."...
 
 Published local repository ./bookworm [i386, source] publishes {main: [local-repo]} has been successfully updated.

system/t06_publish/PublishUpdate15Test_gold

@@ -3,6 +3,7 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components ...
+Cleaning up component "other-test" in prefix "."...
+Cleaning up component "test" in prefix "."...
 
 Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {other-test: [snap3]: Created as empty}, {test: [snap2]: Created as empty} has been successfully updated.

system/t06_publish/PublishUpdate16Test_gold

@@ -3,6 +3,9 @@ Generating metadata files and linking package files...
 Finalizing metadata files...
 Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
-Cleaning up prefix "." components ...
+Removing component "other-test" from prefix "."...
+Removing ${HOME}/.aptly/public/dists/maverick/other-test...
+Removing component "test" from prefix "."...
+Removing ${HOME}/.aptly/public/dists/maverick/test...
 
 Published snapshot repository ./maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully updated.