diff --git a/go.mod b/go.mod index 3c345cef00c..07038673de1 100644 --- a/go.mod +++ b/go.mod @@ -87,6 +87,7 @@ require ( require google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect require ( + github.com/BurntSushi/toml v1.4.0 github.com/adrg/xdg v0.4.0 github.com/magiconair/properties v1.8.7 ) diff --git a/go.sum b/go.sum index 1605ba10bbd..b97838b4629 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,8 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg6 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CycloneDX/cyclonedx-go v0.9.0 h1:inaif7qD8bivyxp7XLgxUYtOXWtDez7+j72qKTMQTb8= github.com/CycloneDX/cyclonedx-go v0.9.0/go.mod h1:NE/EWvzELOFlG6+ljX/QeMlVt9VKcTwu8u0ccsACEsw= diff --git a/syft/pkg/cataloger/python/parse_poetry_lock.go b/syft/pkg/cataloger/python/parse_poetry_lock.go index dbc702d5383..84e67ddb589 100644 --- a/syft/pkg/cataloger/python/parse_poetry_lock.go +++ b/syft/pkg/cataloger/python/parse_poetry_lock.go @@ -4,10 +4,10 @@ import ( "context" "fmt" "sort" - "strings" - "github.com/pelletier/go-toml" + "github.com/BurntSushi/toml" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -19,7 +19,9 @@ import ( var _ generic.Parser = parsePoetryLock type poetryPackageSource struct { - URL string `toml:"url"` + URL string `toml:"url"` + Type string `toml:"type"` + Reference string `toml:"reference"` } type poetryPackages struct { @@ -27,14 +29,15 @@ type poetryPackages struct { } type poetryPackage struct { - Name string `toml:"name"` - Version string `toml:"version"` - Category string `toml:"category"` - Description string `toml:"description"` - Optional bool `toml:"optional"` - Source poetryPackageSource `toml:"source"` - Dependencies map[string]poetryPackageDependency `toml:"dependencies"` - Extras map[string][]string `toml:"extras"` + Name string `toml:"name"` + Version string `toml:"version"` + Category string `toml:"category"` + Description string `toml:"description"` + Optional bool `toml:"optional"` + Source poetryPackageSource `toml:"source"` + DependenciesUnmarshal map[string]toml.Primitive `toml:"dependencies"` + Extras map[string][]string `toml:"extras"` + Dependencies map[string][]poetryPackageDependency } type poetryPackageDependency struct { @@ -44,41 +47,6 @@ type poetryPackageDependency struct { Extras []string `toml:"extras"` } -func (d *poetryPackageDependency) UnmarshalText(data []byte) error { - // attempt to parse as a map first - var dep map[string]interface{} - if err := toml.Unmarshal(data, &dep); err == nil { - if extras, ok := dep["extras"]; ok { - if extrasList, ok := extras.([]string); ok { - d.Extras = extrasList - } - } - - if markers, ok := dep["markers"]; ok { - if markersString, ok := markers.(string); ok { - d.Markers = markersString - } - } - - if version, ok := dep["version"]; ok { - if versionString, ok := version.(string); ok { - d.Version = versionString - } - } - return nil - } - - if strings.ContainsAny(string(data), "[]{}") { - // odds are this is really a malformed toml array or object - return fmt.Errorf("unable to parse poetry dependency: version is malformed array/object: %q", string(data)) - } - - // assume this is a simple version string - d.Version = string(data) - - return nil -} - // parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered. func parsePoetryLock(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { pkgs, err := poetryLockPackages(reader) @@ -93,15 +61,33 @@ func parsePoetryLock(_ context.Context, _ file.Resolver, _ *generic.Environment, } func poetryLockPackages(reader file.LocationReadCloser) ([]pkg.Package, error) { - tree, err := toml.LoadReader(reader) + metadata := poetryPackages{} + md, err := toml.NewDecoder(reader).Decode(&metadata) if err != nil { - return nil, fmt.Errorf("unable to load poetry.lock for parsing: %w", err) + return nil, fmt.Errorf("failed to read poetry lock package: %w", err) } - metadata := poetryPackages{} - err = tree.Unmarshal(&metadata) - if err != nil { - return nil, fmt.Errorf("unable to parse poetry.lock: %w", err) + for i, p := range metadata.Packages { + dependencies := make(map[string][]poetryPackageDependency) + for pkgName, du := range p.DependenciesUnmarshal { + var ( + single string + singleObj poetryPackageDependency + multiObj []poetryPackageDependency + ) + + switch { + case md.PrimitiveDecode(du, &single) == nil: + dependencies[pkgName] = append(dependencies[pkgName], poetryPackageDependency{Version: single}) + case md.PrimitiveDecode(du, &singleObj) == nil: + dependencies[pkgName] = append(dependencies[pkgName], singleObj) + case md.PrimitiveDecode(du, &multiObj) == nil: + dependencies[pkgName] = append(dependencies[pkgName], multiObj...) + default: + log.Trace("failed to decode poetry lock package dependencies for %s; skipping", pkgName) + } + } + metadata.Packages[i].Dependencies = dependencies } var pkgs []pkg.Package @@ -137,13 +123,15 @@ func extractIndex(p poetryPackage) string { func extractPoetryDependencies(p poetryPackage) []pkg.PythonPoetryLockDependencyEntry { var deps []pkg.PythonPoetryLockDependencyEntry - for name, dep := range p.Dependencies { - deps = append(deps, pkg.PythonPoetryLockDependencyEntry{ - Name: name, - Version: dep.Version, - Extras: dep.Extras, - Markers: dep.Markers, - }) + for name, dependencies := range p.Dependencies { + for _, d := range dependencies { + deps = append(deps, pkg.PythonPoetryLockDependencyEntry{ + Name: name, + Version: d.Version, + Extras: d.Extras, + Markers: d.Markers, + }) + } } sort.Slice(deps, func(i, j int) bool { return deps[i].Name < deps[j].Name diff --git a/syft/pkg/cataloger/python/parse_poetry_lock_test.go b/syft/pkg/cataloger/python/parse_poetry_lock_test.go index b590be47f97..1a6c1beb797 100644 --- a/syft/pkg/cataloger/python/parse_poetry_lock_test.go +++ b/syft/pkg/cataloger/python/parse_poetry_lock_test.go @@ -24,7 +24,11 @@ func TestParsePoetryLock(t *testing.T) { Index: "https://test.pypi.org/simple", Dependencies: []pkg.PythonPoetryLockDependencyEntry{ {Name: "docutils", Version: "*"}, + {Name: "msal", Version: ">=0.4.1,<2.0.0"}, {Name: "natsort", Version: "*"}, + {Name: "packaging", Version: "*"}, + {Name: "portalocker", Version: ">=1.0,<3", Markers: `platform_system != "Windows"`}, + {Name: "portalocker", Version: ">=1.6,<3", Markers: `platform_system == "Windows"`}, {Name: "six", Version: "*"}, {Name: "sphinx", Version: "*"}, }, diff --git a/syft/pkg/cataloger/python/test-fixtures/poetry/dev-deps/poetry.lock b/syft/pkg/cataloger/python/test-fixtures/poetry/dev-deps/poetry.lock index 6d20352ff08..3b3f2078d0b 100644 --- a/syft/pkg/cataloger/python/test-fixtures/poetry/dev-deps/poetry.lock +++ b/syft/pkg/cataloger/python/test-fixtures/poetry/dev-deps/poetry.lock @@ -11,6 +11,13 @@ docutils = "*" natsort = "*" six = "*" sphinx = "*" +packaging = "*" +msal = {version = ">=0.4.1,<2.0.0"} +malformed = [ [ { version = "1.2" } ] ] +portalocker = [ + {version = ">=1.0,<3", markers = "platform_system != \"Windows\""}, + {version = ">=1.6,<3", markers = "platform_system == \"Windows\""}, +] [package.source] type = "legacy"