Skip to content

Commit

Permalink
Adding the ability to retrieve remote licenses from package.lock (#2708)
Browse files Browse the repository at this point in the history 
Signed-off-by: Colm O hEigeartaigh <coheigea@apache.org>
  • Loading branch information
@coheigea
coheigea committed Mar 21, 2024
1 parent 0d5ebed commit f4e1896
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 15 deletions.
3 changes: 2 additions & 1 deletion syft/pkg/cataloger/javascript/cataloger.go
View file
Expand Up @@ -17,8 +17,9 @@ func NewPackageCataloger() pkg.Cataloger {
// NewLockCataloger returns a new cataloger object for NPM (and NPM-adjacent, such as yarn) lock files.
func NewLockCataloger(cfg CatalogerConfig) pkg.Cataloger {
yarnLockAdapter := newGenericYarnLockAdapter(cfg)
packageLockAdapter := newGenericPackageLockAdapter(cfg)
return generic.NewCataloger("javascript-lock-cataloger").
WithParserByGlobs(parsePackageLock, "**/package-lock.json").
WithParserByGlobs(packageLockAdapter.parsePackageLock, "**/package-lock.json").
WithParserByGlobs(yarnLockAdapter.parseYarnLock, "**/yarn.lock").
WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml")
}
35 changes: 32 additions & 3 deletions syft/pkg/cataloger/javascript/package.go
View file
Expand Up @@ -47,7 +47,7 @@ func newPackageJSONPackage(u packageJSON, indexLocation file.Location) pkg.Packa
return p
}

func newPackageLockV1Package(resolver file.Resolver, location file.Location, name string, u lockDependency) pkg.Package {
func newPackageLockV1Package(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name string, u lockDependency) pkg.Package {
version := u.Version

const aliasPrefixPackageLockV1 = "npm:"
Expand All @@ -63,12 +63,26 @@ func newPackageLockV1Package(resolver file.Resolver, location file.Location, nam
version = canonicalPackageAndVersion[versionSeparator+1:]
}

var licenseSet pkg.LicenseSet

if cfg.SearchRemoteLicenses {
license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, version)
if err == nil && license != "" {
licenses := pkg.NewLicensesFromValues(license)
licenseSet = pkg.NewLicenseSet(licenses...)
}
if err != nil {
log.Warnf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, version, err)
}
}

return finalizeLockPkg(
resolver,
location,
pkg.Package{
Name: name,
Version: version,
Licenses: licenseSet,
Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
PURL: packageURL(name, version),
Language: pkg.JavaScript,
Expand All @@ -78,15 +92,30 @@ func newPackageLockV1Package(resolver file.Resolver, location file.Location, nam
)
}

func newPackageLockV2Package(resolver file.Resolver, location file.Location, name string, u lockPackage) pkg.Package {
func newPackageLockV2Package(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name string, u lockPackage) pkg.Package {
var licenseSet pkg.LicenseSet

if u.License != nil {
licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(location, u.License...)...)
} else if cfg.SearchRemoteLicenses {
license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, u.Version)
if err == nil && license != "" {
licenses := pkg.NewLicensesFromValues(license)
licenseSet = pkg.NewLicenseSet(licenses...)
}
if err != nil {
log.Warnf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, u.Version, err)
}
}

return finalizeLockPkg(
resolver,
location,
pkg.Package{
Name: name,
Version: u.Version,
Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromLocation(location, u.License...)...),
Licenses: licenseSet,
PURL: packageURL(name, u.Version),
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Expand Down
19 changes: 13 additions & 6 deletions syft/pkg/cataloger/javascript/parse_package_lock.go
View file
Expand Up @@ -15,9 +15,6 @@ import (
"github.com/anchore/syft/syft/pkg/cataloger/generic"
)

// integrity check
var _ generic.Parser = parsePackageLock

// packageLock represents a JavaScript package.lock json file
type packageLock struct {
Requires bool `json:"requires"`
Expand All @@ -44,8 +41,18 @@ type lockPackage struct {
// packageLockLicense
type packageLockLicense []string

type genericPackageLockAdapter struct {
cfg CatalogerConfig
}

func newGenericPackageLockAdapter(cfg CatalogerConfig) genericPackageLockAdapter {
return genericPackageLockAdapter{
cfg: cfg,
}
}

// parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages.
func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
func (a genericPackageLockAdapter) parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
// in the case we find package-lock.json files in the node_modules directories, skip those
// as the whole purpose of the lock file is for the specific dependencies of the root project
if pathContainsNodeModulesDirectory(reader.Path()) {
Expand All @@ -66,7 +73,7 @@ func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Envi

if lock.LockfileVersion == 1 {
for name, pkgMeta := range lock.Dependencies {
pkgs = append(pkgs, newPackageLockV1Package(resolver, reader.Location, name, pkgMeta))
pkgs = append(pkgs, newPackageLockV1Package(a.cfg, resolver, reader.Location, name, pkgMeta))
}
}

Expand All @@ -86,7 +93,7 @@ func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Envi

pkgs = append(
pkgs,
newPackageLockV2Package(resolver, reader.Location, getNameFromPath(name), pkgMeta),
newPackageLockV2Package(a.cfg, resolver, reader.Location, getNameFromPath(name), pkgMeta),
)
}
}
Expand Down
15 changes: 10 additions & 5 deletions syft/pkg/cataloger/javascript/parse_package_lock_test.go
View file
Expand Up @@ -106,7 +106,8 @@ func TestParsePackageLock(t *testing.T) {
expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
}

pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
adapter := newGenericPackageLockAdapter(CatalogerConfig{})
pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
}

func TestParsePackageLockV2(t *testing.T) {
Expand Down Expand Up @@ -169,7 +170,8 @@ func TestParsePackageLockV2(t *testing.T) {
for i := range expectedPkgs {
expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
}
pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
adapter := newGenericPackageLockAdapter(CatalogerConfig{})
pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
}

func TestParsePackageLockV3(t *testing.T) {
Expand Down Expand Up @@ -220,7 +222,8 @@ func TestParsePackageLockV3(t *testing.T) {
for i := range expectedPkgs {
expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
}
pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
adapter := newGenericPackageLockAdapter(CatalogerConfig{})
pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
}

func TestParsePackageLockAlias(t *testing.T) {
Expand Down Expand Up @@ -279,7 +282,8 @@ func TestParsePackageLockAlias(t *testing.T) {
for i := range expected {
expected[i].Locations.Add(file.NewLocation(pl))
}
pkgtest.TestFileParser(t, pl, parsePackageLock, expected, expectedRelationships)
adapter := newGenericPackageLockAdapter(CatalogerConfig{})
pkgtest.TestFileParser(t, pl, adapter.parsePackageLock, expected, expectedRelationships)
}
}

Expand Down Expand Up @@ -326,5 +330,6 @@ func TestParsePackageLockLicenseWithArray(t *testing.T) {
for i := range expectedPkgs {
expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
}
pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
adapter := newGenericPackageLockAdapter(CatalogerConfig{})
pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
}

0 comments on commit f4e1896

Please sign in to comment.