diff --git a/pkg/lockfile/fixtures/pip/file-format-example.txt b/pkg/lockfile/fixtures/pip/file-format-example.txt index 2fabc302..5bc417fa 100644 --- a/pkg/lockfile/fixtures/pip/file-format-example.txt +++ b/pkg/lockfile/fixtures/pip/file-format-example.txt @@ -12,7 +12,7 @@ coverage != 3.5 # Version Exclusion. Anything except version 3.5 Mopidy-Dirble ~= 1.1 # Compatible release. Same as >= 1.1, == 1.* ###### Refer to other requirements files ###### --r other-requirements.txt +-r other-file.txt ###### A particular file ###### ./downloads/numpy-1.9.2-cp34-none-win32.whl diff --git a/pkg/lockfile/fixtures/pip/other-file.txt b/pkg/lockfile/fixtures/pip/other-file.txt new file mode 100644 index 00000000..f44fd331 --- /dev/null +++ b/pkg/lockfile/fixtures/pip/other-file.txt @@ -0,0 +1 @@ +django==2.2.24 diff --git a/pkg/lockfile/fixtures/pip/with-bad-r-option.txt b/pkg/lockfile/fixtures/pip/with-bad-r-option.txt new file mode 100644 index 00000000..26629640 --- /dev/null +++ b/pkg/lockfile/fixtures/pip/with-bad-r-option.txt @@ -0,0 +1,3 @@ +requests==1.2.3 + +-r ./does-not-exist.txt diff --git a/pkg/lockfile/fixtures/pip/with-multiple-r-options.txt b/pkg/lockfile/fixtures/pip/with-multiple-r-options.txt new file mode 100644 index 00000000..132a3ccd --- /dev/null +++ b/pkg/lockfile/fixtures/pip/with-multiple-r-options.txt @@ -0,0 +1,5 @@ +-r ./one-package-constrained.txt +-r ./multiple-packages-mixed.txt + +requests==1.2.3 +pandas==0.23.4 diff --git a/pkg/lockfile/parse-requirements-txt.go b/pkg/lockfile/parse-requirements-txt.go index 40c853f1..78ed47f4 100644 --- a/pkg/lockfile/parse-requirements-txt.go +++ b/pkg/lockfile/parse-requirements-txt.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "os" + "path/filepath" "regexp" "strings" ) @@ -91,11 +92,11 @@ func isNotRequirementLine(line string) bool { } func ParseRequirementsTxt(pathToLockfile string) ([]PackageDetails, error) { - var packages []PackageDetails + packages := map[string]PackageDetails{} file, err := os.Open(pathToLockfile) if err != nil { - return packages, fmt.Errorf("could not open %s: %w", pathToLockfile, err) + return []PackageDetails{}, fmt.Errorf("could not open %s: %w", pathToLockfile, err) } defer file.Close() @@ -104,16 +105,33 @@ func ParseRequirementsTxt(pathToLockfile string) ([]PackageDetails, error) { for scanner.Scan() { line := removeComments(scanner.Text()) + if strings.HasPrefix(line, "-r ") { + details, err := ParseRequirementsTxt( + filepath.Join(filepath.Dir(pathToLockfile), strings.TrimPrefix(line, "-r ")), + ) + + if err != nil { + return []PackageDetails{}, fmt.Errorf("failed to include %s: %w", line, err) + } + + for _, detail := range details { + packages[detail.Name+"@"+detail.Version] = detail + } + + continue + } + if isNotRequirementLine(line) { continue } - packages = append(packages, parseLine(line)) + detail := parseLine(line) + packages[detail.Name+"@"+detail.Version] = detail } if err := scanner.Err(); err != nil { - return packages, fmt.Errorf("error while scanning %s: %w", pathToLockfile, err) + return []PackageDetails{}, fmt.Errorf("error while scanning %s: %w", pathToLockfile, err) } - return packages, nil + return pkgDetailsMapToSlice(packages), nil } diff --git a/pkg/lockfile/parse-requirements-txt_test.go b/pkg/lockfile/parse-requirements-txt_test.go index ed1026cc..aaaa08fb 100644 --- a/pkg/lockfile/parse-requirements-txt_test.go +++ b/pkg/lockfile/parse-requirements-txt_test.go @@ -292,6 +292,12 @@ func TestParseRequirementsTxt_FileFormatExample(t *testing.T) { Ecosystem: lockfile.PipEcosystem, CompareAs: lockfile.PipEcosystem, }, + { + Name: "django", + Version: "2.2.24", + Ecosystem: lockfile.PipEcosystem, + CompareAs: lockfile.PipEcosystem, + }, }) } @@ -344,3 +350,85 @@ func TestParseRequirementsTxt_NonNormalizedNames(t *testing.T) { }, }) } + +func TestParseRequirementsTxt_WithMultipleROptions(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/with-multiple-r-options.txt") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "flask", + Version: "0.0.0", + Ecosystem: lockfile.PipEcosystem, + CompareAs: lockfile.PipEcosystem, + }, + { + Name: "flask-cors", + Version: "0.0.0", + Ecosystem: lockfile.PipEcosystem, + CompareAs: lockfile.PipEcosystem, + }, + { + Name: "pandas", + Version: "0.23.4", + Ecosystem: lockfile.PipEcosystem, + CompareAs: lockfile.PipEcosystem, + }, + { + Name: "numpy", + Version: "1.16.0", + Ecosystem: lockfile.PipEcosystem, + CompareAs: lockfile.PipEcosystem, + }, + { + Name: "scikit-learn", + Version: "0.20.1", + Ecosystem: lockfile.PipEcosystem, + CompareAs: lockfile.PipEcosystem, + }, + { + Name: "sklearn", + Version: "0.0.0", + Ecosystem: lockfile.PipEcosystem, + CompareAs: lockfile.PipEcosystem, + }, + { + Name: "requests", + Version: "0.0.0", + Ecosystem: lockfile.PipEcosystem, + CompareAs: lockfile.PipEcosystem, + }, + { + Name: "gevent", + Version: "0.0.0", + Ecosystem: lockfile.PipEcosystem, + CompareAs: lockfile.PipEcosystem, + }, + { + Name: "requests", + Version: "1.2.3", + Ecosystem: lockfile.PipEcosystem, + CompareAs: lockfile.PipEcosystem, + }, + { + Name: "django", + Version: "2.2.24", + Ecosystem: lockfile.PipEcosystem, + CompareAs: lockfile.PipEcosystem, + }, + }) +} + +func TestParseRequirementsTxt_WithBadROption(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/with-bad-r-option.txt") + + expectErrContaining(t, err, "could not open") + expectPackages(t, packages, []lockfile.PackageDetails{}) +}