Guided Remediation: add npm registry clients & .npmrc parsing (#778)

The datasource and `DependencyClient` for querying the npm registry API
directly, instead of relying on deps.dev.
Also, parses `.npmrc` configs to allow resolution of requirements from
private registries.

Practically unchanged from what we had internally, besides a bunch of
linting complaints.

go.mod

@@ -28,6 +28,7 @@ require (
 	golang.org/x/vuln v1.0.1
 	google.golang.org/grpc v1.60.1
 	google.golang.org/protobuf v1.31.0
+	gopkg.in/ini.v1 v1.67.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -246,6 +246,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/remediation/relaxer/npm.go

@@ -8,9 +8,9 @@ import (
 	"deps.dev/util/semver"
 )
 
-type NPMRelaxer struct{}
+type NpmRelaxer struct{}
 
-func (r NPMRelaxer) Relax(ctx context.Context, cl resolve.Client, req resolve.RequirementVersion, allowMajor bool) (resolve.RequirementVersion, bool) {
+func (r NpmRelaxer) Relax(ctx context.Context, cl resolve.Client, req resolve.RequirementVersion, allowMajor bool) (resolve.RequirementVersion, bool) {
 	c, err := semver.NPM.ParseConstraint(req.Version)
 	if err != nil {
 		// The specified version is not a valid semver constraint

internal/remediation/relaxer/relaxer.go

@@ -27,7 +27,7 @@ func GetRelaxer(ecosystem resolve.System) (RequirementRelaxer, error) {
 	// TODO: is using ecosystem fine, or should this be per manifest?
 	switch ecosystem { //nolint:exhaustive
 	case resolve.NPM:
-		return NPMRelaxer{}, nil
+		return NpmRelaxer{}, nil
 	default:
 		return nil, errors.New("unsupported ecosystem")
 	}

internal/resolution/client/npm_registry_client.go

@@ -0,0 +1,286 @@
+package client
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/gob"
+	"fmt"
+	"os"
+	"slices"
+	"strings"
+
+	pb "deps.dev/api/v3alpha"
+	"deps.dev/util/resolve"
+	"deps.dev/util/resolve/dep"
+	"deps.dev/util/semver"
+	"github.com/google/osv-scanner/internal/resolution/datasource"
+	"github.com/google/osv-scanner/pkg/depsdev"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+)
+
+const npmRegistryCacheExt = ".resolve.npm"
+
+type NpmRegistryClient struct {
+	api *datasource.NpmRegistryAPIClient
+
+	// Fallback client for dealing with bundleDependencies.
+	ic       pb.InsightsClient
+	fallback *resolve.APIClient
+}
+
+func NewNpmRegistryClient(workdir string) (*NpmRegistryClient, error) {
+	api, err := datasource.NewNpmRegistryAPIClient(workdir)
+	if err != nil {
+		return nil, err
+	}
+
+	certPool, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, fmt.Errorf("getting system cert pool: %w", err)
+	}
+	creds := credentials.NewClientTLSFromCert(certPool, "")
+	conn, err := grpc.Dial(depsdev.DepsdevAPI, grpc.WithTransportCredentials(creds))
+	if err != nil {
+		return nil, fmt.Errorf("dialling %q: %w", depsdev.DepsdevAPI, err)
+	}
+	ic := pb.NewInsightsClient(conn)
+
+	return &NpmRegistryClient{
+		api:      api,
+		ic:       ic,
+		fallback: resolve.NewAPIClient(ic),
+	}, nil
+}
+
+func (c *NpmRegistryClient) Version(ctx context.Context, vk resolve.VersionKey) (resolve.Version, error) {
+	if isNpmBundle(vk.PackageKey) { // bundled dependencies, fallback to deps.dev client
+		return c.fallback.Version(ctx, vk)
+	}
+
+	return resolve.Version{VersionKey: vk}, nil
+}
+
+func (c *NpmRegistryClient) Versions(ctx context.Context, pk resolve.PackageKey) ([]resolve.Version, error) {
+	if isNpmBundle(pk) { // bundled dependencies, fallback to deps.dev client
+		return c.fallback.Versions(ctx, pk)
+	}
+
+	vers, err := c.api.Versions(ctx, pk.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	vks := make([]resolve.Version, len(vers.Versions))
+	for i, v := range vers.Versions {
+		vks[i] = resolve.Version{
+			VersionKey: resolve.VersionKey{
+				PackageKey:  pk,
+				Version:     v,
+				VersionType: resolve.Concrete,
+			}}
+	}
+
+	slices.SortFunc(vks, func(a, b resolve.Version) int { return semver.NPM.Compare(a.Version, b.Version) })
+
+	return vks, nil
+}
+
+func (c *NpmRegistryClient) Requirements(ctx context.Context, vk resolve.VersionKey) ([]resolve.RequirementVersion, error) {
+	if vk.System != resolve.NPM {
+		return nil, fmt.Errorf("unsupported system: %v", vk.System)
+	}
+
+	if isNpmBundle(vk.PackageKey) { // bundled dependencies, fallback to deps.dev client
+		return c.fallback.Requirements(ctx, vk)
+	}
+	dependencies, err := c.api.Dependencies(ctx, vk.Name, vk.Version)
+	if err != nil {
+		return nil, err
+	}
+
+	// Preallocate the dependency slice, which will hold all the dependencies of each type.
+	// The npm resolver expects bundled dependencies included twice in different forms:
+	// {foo@*|Scope="bundle"} and {mangled-name-of>0.1.2>foo@1.2.3}, hence the 2*len(bundled)
+	depCount := len(dependencies.Dependencies) + len(dependencies.DevDependencies) +
+		len(dependencies.OptionalDependencies) + len(dependencies.PeerDependencies) +
+		2*len(dependencies.BundleDependencies)
+	deps := make([]resolve.RequirementVersion, 0, depCount)
+	addDeps := func(ds map[string]string, t dep.Type) {
+		for name, req := range ds {
+			typ := t.Clone()
+			if r, ok := strings.CutPrefix(req, "npm:"); ok {
+				// This dependency is aliased, add it as a
+				// dependency on the actual name, with the
+				// KnownAs attribute set to the alias.
+				typ.AddAttr(dep.KnownAs, name)
+				name = r
+				req = ""
+				if i := strings.LastIndex(r, "@"); i > 0 {
+					name = r[:i]
+					req = r[i+1:]
+				}
+			}
+			deps = append(deps, resolve.RequirementVersion{
+				Type: typ,
+				VersionKey: resolve.VersionKey{
+					PackageKey: resolve.PackageKey{
+						System: resolve.NPM,
+						Name:   name,
+					},
+					VersionType: resolve.Requirement,
+					Version:     req,
+				},
+			})
+		}
+	}
+	addDeps(dependencies.Dependencies, dep.NewType())
+	addDeps(dependencies.DevDependencies, dep.NewType(dep.Dev))
+	addDeps(dependencies.OptionalDependencies, dep.NewType(dep.Opt))
+
+	peerType := dep.NewType()
+	peerType.AddAttr(dep.Scope, "peer")
+	addDeps(dependencies.PeerDependencies, peerType)
+
+	// The resolver expects bundleDependencies to be present as regular
+	// dependencies with a "*" version specifier, even if they were already
+	// in the regular dependencies.
+	bundleType := dep.NewType()
+	bundleType.AddAttr(dep.Scope, "bundle")
+	for _, name := range dependencies.BundleDependencies {
+		deps = append(deps, resolve.RequirementVersion{
+			Type: bundleType,
+			VersionKey: resolve.VersionKey{
+				PackageKey: resolve.PackageKey{
+					System: resolve.NPM,
+					Name:   name,
+				},
+				VersionType: resolve.Requirement,
+				Version:     "*",
+			},
+		})
+	}
+
+	// Correctly resolving the bundled dependencies would require downloading the package.
+	// Instead, call the fallback deps.dev client to get the bundled dependencies with mangled names.
+	if len(dependencies.BundleDependencies) > 0 {
+		fallbackReqs, err := c.fallback.Requirements(ctx, vk)
+		if err != nil {
+			// TODO: make some placeholder if the package doesn't exist on deps.dev
+			return nil, err
+		}
+		for _, req := range fallbackReqs {
+			if isNpmBundle(req.PackageKey) {
+				deps = append(deps, req)
+			}
+		}
+	}
+
+	resolve.SortDependencies(deps)
+
+	return deps, nil
+}
+
+func (c *NpmRegistryClient) MatchingVersions(ctx context.Context, vk resolve.VersionKey) ([]resolve.Version, error) {
+	if isNpmBundle(vk.PackageKey) { // bundled dependencies, fallback to deps.dev client
+		return c.fallback.MatchingVersions(ctx, vk)
+	}
+
+	versions, err := c.api.Versions(ctx, vk.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	if concVer, ok := versions.Tags[vk.Version]; ok {
+		// matched a tag, return just the concrete version of the tag
+		return []resolve.Version{{
+			VersionKey: resolve.VersionKey{
+				PackageKey:  vk.PackageKey,
+				Version:     concVer,
+				VersionType: resolve.Concrete,
+			},
+		}}, nil
+	}
+
+	resVersions := make([]resolve.Version, len(versions.Versions))
+	for i, v := range versions.Versions {
+		resVersions[i] = resolve.Version{
+			VersionKey: resolve.VersionKey{
+				PackageKey:  vk.PackageKey,
+				Version:     v,
+				VersionType: resolve.Concrete,
+			},
+		}
+	}
+
+	return resolve.MatchRequirement(vk, resVersions), nil
+}
+
+func isNpmBundle(pk resolve.PackageKey) bool {
+	// Bundles are represented in resolution with a 'mangled' name containing its origin e.g. "root-pkg>1.0.0>bundled-package"
+	// '>' is not a valid character for a npm package, so it'll only be found here.
+	return strings.Contains(pk.Name, ">")
+}
+
+func (c *NpmRegistryClient) PreFetch(ctx context.Context, imports []resolve.RequirementVersion, manifestPath string) {
+	// It doesn't matter if loading the cache fails
+	_ = c.LoadCache(manifestPath)
+
+	// Use the deps.dev client to fetch complete dependency graphs of our direct imports
+	for _, im := range imports {
+		// Get the preferred version of the import requirement
+		vks, err := c.MatchingVersions(ctx, im.VersionKey)
+		if err != nil || len(vks) == 0 {
+			continue
+		}
+
+		vk := vks[len(vks)-1]
+
+		// Make a request for the precomputed dependency tree
+		resp, err := c.ic.GetDependencies(ctx, &pb.GetDependenciesRequest{
+			VersionKey: &pb.VersionKey{
+				System:  pb.System(vk.System),
+				Name:    vk.Name,
+				Version: vk.Version,
+			},
+		})
+		if err != nil {
+			continue
+		}
+
+		// Send off queries to cache the packages in the dependency tree
+		for _, node := range resp.GetNodes() {
+			pbvk := node.GetVersionKey()
+			vk := resolve.VersionKey{
+				PackageKey: resolve.PackageKey{
+					System: resolve.System(pbvk.GetSystem()),
+					Name:   pbvk.GetName(),
+				},
+				Version:     pbvk.GetVersion(),
+				VersionType: resolve.Concrete,
+			}
+			go c.Requirements(ctx, vk) //nolint:errcheck
+		}
+	}
+	// don't bother waiting for goroutines to finish.
+}
+
+func (c *NpmRegistryClient) WriteCache(path string) error {
+	f, err := os.Create(path + npmRegistryCacheExt)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return gob.NewEncoder(f).Encode(c.api)
+}
+
+func (c *NpmRegistryClient) LoadCache(path string) error {
+	f, err := os.Open(path + npmRegistryCacheExt)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return gob.NewDecoder(f).Decode(&c.api)
+}