feat: report unknowns in sbom (#2998)

Signed-off-by: Keith Zantow <kzantow@gmail.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>

cmd/syft/internal/options/catalog.go

@@ -53,6 +53,9 @@ type Catalog struct {
 	Platform   string         `yaml:"platform" json:"platform" mapstructure:"platform"`
 	Source     sourceConfig   `yaml:"source" json:"source" mapstructure:"source"`
 	Exclusions []string       `yaml:"exclude" json:"exclude" mapstructure:"exclude"`
+
+	// configuration for inclusion of unknown information within elements
+	Unknowns unknownsConfig `yaml:"unknowns" mapstructure:"unknowns"`
 }
 
 var _ interface {
@@ -71,6 +74,7 @@ func DefaultCatalog() Catalog {
 		Java:          defaultJavaConfig(),
 		File:          defaultFileConfig(),
 		Relationships: defaultRelationshipsConfig(),
+		Unknowns:      defaultUnknowns(),
 		Source:        defaultSourceConfig(),
 		Parallelism:   1,
 	}
@@ -82,6 +86,7 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
 		WithParallelism(cfg.Parallelism).
 		WithRelationshipsConfig(cfg.ToRelationshipsConfig()).
 		WithComplianceConfig(cfg.ToComplianceConfig()).
+		WithUnknownsConfig(cfg.ToUnknownsConfig()).
 		WithSearchConfig(cfg.ToSearchConfig()).
 		WithPackagesConfig(cfg.ToPackagesConfig()).
 		WithFilesConfig(cfg.ToFilesConfig()).
@@ -114,6 +119,13 @@ func (cfg Catalog) ToComplianceConfig() cataloging.ComplianceConfig {
 	}
 }
 
+func (cfg Catalog) ToUnknownsConfig() cataloging.UnknownsConfig {
+	return cataloging.UnknownsConfig{
+		IncludeExecutablesWithoutPackages: cfg.Unknowns.ExecutablesWithoutPackages,
+		IncludeUnexpandedArchives:         cfg.Unknowns.UnexpandedArchives,
+	}
+}
+
 func (cfg Catalog) ToFilesConfig() filecataloging.Config {
 	hashers, err := intFile.Hashers(cfg.File.Metadata.Digests...)
 	if err != nil {

cmd/syft/internal/options/unknowns.go

@@ -0,0 +1,31 @@
+package options
+
+import (
+	"github.com/anchore/clio"
+	"github.com/anchore/syft/syft/cataloging"
+)
+
+type unknownsConfig struct {
+	RemoveWhenPackagesDefined  bool `json:"remove-when-packages-defined" yaml:"remove-when-packages-defined" mapstructure:"remove-when-packages-defined"`
+	ExecutablesWithoutPackages bool `json:"executables-without-packages" yaml:"executables-without-packages" mapstructure:"executables-without-packages"`
+	UnexpandedArchives         bool `json:"unexpanded-archives" yaml:"unexpanded-archives" mapstructure:"unexpanded-archives"`
+}
+
+var _ interface {
+	clio.FieldDescriber
+} = (*unknownsConfig)(nil)
+
+func (o *unknownsConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
+	descriptions.Add(&o.RemoveWhenPackagesDefined, `remove unknown errors on files with discovered packages`)
+	descriptions.Add(&o.ExecutablesWithoutPackages, `include executables without any identified packages`)
+	descriptions.Add(&o.UnexpandedArchives, `include archives which were not expanded and searched`)
+}
+
+func defaultUnknowns() unknownsConfig {
+	def := cataloging.DefaultUnknownsConfig()
+	return unknownsConfig{
+		RemoveWhenPackagesDefined:  def.RemoveWhenPackagesDefined,
+		ExecutablesWithoutPackages: def.IncludeExecutablesWithoutPackages,
+		UnexpandedArchives:         def.IncludeUnexpandedArchives,
+	}
+}

internal/constants.go

@@ -3,5 +3,5 @@ package internal
 const (
 	// JSONSchemaVersion is the current schema version output by the JSON encoder
 	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
-	JSONSchemaVersion = "16.0.17"
+	JSONSchemaVersion = "16.0.18"
 )

internal/file/zip_file_manifest.go

@@ -1,7 +1,6 @@
 package file
 
 import (
-	"fmt"
 	"os"
 	"sort"
 	"strings"
@@ -19,12 +18,13 @@ func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
 	zipReader, err := OpenZip(archivePath)
 	manifest := make(ZipFileManifest)
 	if err != nil {
-		return manifest, fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
+		log.Debugf("unable to open zip archive (%s): %v", archivePath, err)
+		return manifest, err
 	}
 	defer func() {
 		err = zipReader.Close()
 		if err != nil {
-			log.Warnf("unable to close zip archive (%s): %+v", archivePath, err)
+			log.Debugf("unable to close zip archive (%s): %+v", archivePath, err)
 		}
 	}()
 

internal/file/zip_read_closer.go

@@ -8,6 +8,8 @@ import (
 	"io"
 	"math"
 	"os"
+
+	"github.com/anchore/syft/internal/log"
 )
 
 // directoryEndLen, readByf, directoryEnd, and findSignatureInBlock were copied from the golang stdlib, specifically:
@@ -46,7 +48,8 @@ func OpenZip(filepath string) (*ZipReadCloser, error) {
 	// need to find the start of the archive and keep track of this offset.
 	offset, err := findArchiveStartOffset(f, fi.Size())
 	if err != nil {
-		return nil, fmt.Errorf("cannot find beginning of zip archive=%q : %w", filepath, err)
+		log.Debugf("cannot find beginning of zip archive=%q : %v", filepath, err)
+		return nil, err
 	}
 
 	if _, err := f.Seek(0, io.SeekStart); err != nil {
@@ -62,7 +65,8 @@ func OpenZip(filepath string) (*ZipReadCloser, error) {
 
 	r, err := zip.NewReader(io.NewSectionReader(f, offset64, size), size)
 	if err != nil {
-		return nil, fmt.Errorf("unable to open ZipReadCloser @ %q: %w", filepath, err)
+		log.Debugf("unable to open ZipReadCloser @ %q: %v", filepath, err)
+		return nil, err
 	}
 
 	return &ZipReadCloser{

internal/task/executor.go

@@ -11,8 +11,10 @@ import (
 
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/sbomsync"
+	"github.com/anchore/syft/internal/unknown"
 	"github.com/anchore/syft/syft/event/monitor"
 	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/sbom"
 )
 
 type Executor struct {
@@ -35,6 +37,12 @@ func NewTaskExecutor(tasks []Task, numWorkers int) *Executor {
 }
 
 func (p *Executor) Execute(ctx context.Context, resolver file.Resolver, s sbomsync.Builder, prog *monitor.CatalogerTaskProgress) error {
+	var lock sync.Mutex
+	withLock := func(fn func()) {
+		lock.Lock()
+		defer lock.Unlock()
+		fn()
+	}
 	var errs error
 	wg := &sync.WaitGroup{}
 	for i := 0; i < p.numWorkers; i++ {
@@ -48,9 +56,16 @@ func (p *Executor) Execute(ctx context.Context, resolver file.Resolver, s sbomsy
 					return
 				}
 
-				if err := runTaskSafely(ctx, tsk, resolver, s); err != nil {
-					errs = multierror.Append(errs, fmt.Errorf("failed to run task: %w", err))
-					prog.SetError(err)
+				err := runTaskSafely(ctx, tsk, resolver, s)
+				unknowns, err := unknown.ExtractCoordinateErrors(err)
+				if len(unknowns) > 0 {
+					appendUnknowns(s, tsk.Name(), unknowns)
+				}
+				if err != nil {
+					withLock(func() {
+						errs = multierror.Append(errs, fmt.Errorf("failed to run task: %w", err))
+						prog.SetError(err)
+					})
 				}
 				prog.Increment()
 			}
@@ -62,6 +77,19 @@ func (p *Executor) Execute(ctx context.Context, resolver file.Resolver, s sbomsy
 	return errs
 }
 
+func appendUnknowns(builder sbomsync.Builder, taskName string, unknowns []unknown.CoordinateError) {
+	if accessor, ok := builder.(sbomsync.Accessor); ok {
+		accessor.WriteToSBOM(func(sb *sbom.SBOM) {
+			for _, u := range unknowns {
+				if sb.Artifacts.Unknowns == nil {
+					sb.Artifacts.Unknowns = map[file.Coordinates][]string{}
+				}
+				sb.Artifacts.Unknowns[u.Coordinates] = append(sb.Artifacts.Unknowns[u.Coordinates], formatUnknown(u.Reason.Error(), taskName))
+			}
+		})
+	}
+}
+
 func runTaskSafely(ctx context.Context, t Task, resolver file.Resolver, s sbomsync.Builder) (err error) {
 	// handle individual cataloger panics
 	defer func() {

internal/task/file_tasks.go

@@ -3,7 +3,6 @@ package task
 import (
 	"context"
 	"crypto"
-	"fmt"
 
 	"github.com/anchore/syft/internal/sbomsync"
 	"github.com/anchore/syft/syft/artifact"
@@ -32,15 +31,12 @@ func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash
 		}
 
 		result, err := digestsCataloger.Catalog(ctx, resolver, coordinates...)
-		if err != nil {
-			return fmt.Errorf("unable to catalog file digests: %w", err)
-		}
 
 		accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
 			sbom.Artifacts.FileDigests = result
 		})
 
-		return nil
+		return err
 	}
 
 	return NewTask("file-digest-cataloger", fn)
@@ -62,15 +58,12 @@ func NewFileMetadataCatalogerTask(selection file.Selection) Task {
 		}
 
 		result, err := metadataCataloger.Catalog(ctx, resolver, coordinates...)
-		if err != nil {
-			return err
-		}
 
 		accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
 			sbom.Artifacts.FileMetadata = result
 		})
 
-		return nil
+		return err
 	}
 
 	return NewTask("file-metadata-cataloger", fn)
@@ -87,15 +80,12 @@ func NewFileContentCatalogerTask(cfg filecontent.Config) Task {
 		accessor := builder.(sbomsync.Accessor)
 
 		result, err := cat.Catalog(ctx, resolver)
-		if err != nil {
-			return err
-		}
 
 		accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
 			sbom.Artifacts.FileContents = result
 		})
 
-		return nil
+		return err
 	}
 
 	return NewTask("file-content-cataloger", fn)
@@ -112,15 +102,12 @@ func NewExecutableCatalogerTask(selection file.Selection, cfg executable.Config)
 		accessor := builder.(sbomsync.Accessor)
 
 		result, err := cat.Catalog(resolver)
-		if err != nil {
-			return err
-		}
 
 		accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
 			sbom.Artifacts.Executables = result
 		})
 
-		return nil
+		return err
 	}
 
 	return NewTask("file-executable-cataloger", fn)

internal/task/package_task_factory.go

@@ -103,9 +103,6 @@ func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string
 		t := bus.StartCatalogerTask(info, -1, "")
 
 		pkgs, relationships, err := c.Catalog(ctx, resolver)
-		if err != nil {
-			return fmt.Errorf("unable to catalog packages with %q: %w", catalogerName, err)
-		}
 
 		log.WithFields("cataloger", catalogerName).Debugf("discovered %d packages", len(pkgs))
 
@@ -120,7 +117,7 @@ func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string
 		t.SetCompleted()
 		log.WithFields("name", catalogerName).Trace("package cataloger completed")
 
-		return nil
+		return err
 	}
 	tags = append(tags, pkgcataloging.PackageTag)