From 9bfcd157685279543e66614da600adab95698825 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Fri, 7 Oct 2022 07:31:14 +1300 Subject: [PATCH] feat: support parsing `pubspec.lock` (#159) --- README.md | 1 + internal/reporter/reporter_test.go | 1 + main_test.go | 1 + pkg/lockfile/ecosystems.go | 1 + pkg/lockfile/fixtures/pub/empty.lock | 0 pkg/lockfile/fixtures/pub/mixed-packages.lock | 31 +++ pkg/lockfile/fixtures/pub/no-packages.lock | 3 + pkg/lockfile/fixtures/pub/not-yaml.txt | 1 + .../fixtures/pub/one-package-dev.lock | 10 + pkg/lockfile/fixtures/pub/one-package.lock | 10 + pkg/lockfile/fixtures/pub/source-git.lock | 46 ++++ pkg/lockfile/fixtures/pub/source-path.lock | 10 + pkg/lockfile/fixtures/pub/source-sdk.lock | 11 + pkg/lockfile/fixtures/pub/two-packages.lock | 17 ++ pkg/lockfile/parse-pubspec-lock.go | 93 ++++++++ pkg/lockfile/parse-pubspec-lock_test.go | 221 ++++++++++++++++++ pkg/lockfile/parse.go | 1 + pkg/lockfile/parse_test.go | 2 + 18 files changed, 460 insertions(+) create mode 100644 pkg/lockfile/fixtures/pub/empty.lock create mode 100644 pkg/lockfile/fixtures/pub/mixed-packages.lock create mode 100644 pkg/lockfile/fixtures/pub/no-packages.lock create mode 100644 pkg/lockfile/fixtures/pub/not-yaml.txt create mode 100644 pkg/lockfile/fixtures/pub/one-package-dev.lock create mode 100644 pkg/lockfile/fixtures/pub/one-package.lock create mode 100644 pkg/lockfile/fixtures/pub/source-git.lock create mode 100644 pkg/lockfile/fixtures/pub/source-path.lock create mode 100644 pkg/lockfile/fixtures/pub/source-sdk.lock create mode 100644 pkg/lockfile/fixtures/pub/two-packages.lock create mode 100644 pkg/lockfile/parse-pubspec-lock.go create mode 100644 pkg/lockfile/parse-pubspec-lock_test.go diff --git a/README.md b/README.md index 1ab6e53d..95dd8963 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ The detector supports parsing the following lockfiles: | `go.mod` | `Go` | `go mod` | | `mix.lock` | `Hex` | `mix` | | `poetry.lock` | `PyPI` | `poetry` | +| `pubspec.lock` | `Pub` | `pub` | | `pom.xml`\* | `Maven` | `maven` | | `requirements.txt`\* | `PyPI` | `pip` | diff --git a/internal/reporter/reporter_test.go b/internal/reporter/reporter_test.go index 61478450..50a0137d 100644 --- a/internal/reporter/reporter_test.go +++ b/internal/reporter/reporter_test.go @@ -272,6 +272,7 @@ func TestReporter_PrintKnownEcosystems(t *testing.T) { " Hex", " Maven", " PyPI", + " Pub", "", }, "\n") diff --git a/main_test.go b/main_test.go index e2f2ac4b..76fb4d0d 100644 --- a/main_test.go +++ b/main_test.go @@ -131,6 +131,7 @@ func TestRun(t *testing.T) { pnpm-lock.yaml poetry.lock pom.xml + pubspec.lock requirements.txt yarn.lock csv-file diff --git a/pkg/lockfile/ecosystems.go b/pkg/lockfile/ecosystems.go index 587aae3d..adc5dc78 100644 --- a/pkg/lockfile/ecosystems.go +++ b/pkg/lockfile/ecosystems.go @@ -10,5 +10,6 @@ func KnownEcosystems() []Ecosystem { MixEcosystem, MavenEcosystem, PipEcosystem, + PubEcosystem, } } diff --git a/pkg/lockfile/fixtures/pub/empty.lock b/pkg/lockfile/fixtures/pub/empty.lock new file mode 100644 index 00000000..e69de29b diff --git a/pkg/lockfile/fixtures/pub/mixed-packages.lock b/pkg/lockfile/fixtures/pub/mixed-packages.lock new file mode 100644 index 00000000..763211ae --- /dev/null +++ b/pkg/lockfile/fixtures/pub/mixed-packages.lock @@ -0,0 +1,31 @@ +# Generated by pub +# See http://pub.dartlang.org/doc/glossary.html#lockfile +packages: + back_button_interceptor: + dependency: "direct main" + description: + name: back_button_interceptor + url: "https://pub.dartlang.org" + source: hosted + version: "6.0.1" + build_runner: + dependency: "direct dev" + description: + name: build_runner + url: "https://pub.dartlang.org" + source: hosted + version: "2.2.1" + shelf: + dependency: transitive + description: + name: shelf + url: "https://pub.dartlang.org" + source: hosted + version: "1.3.2" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + url: "https://pub.dartlang.org" + source: hosted + version: "1.0.2" diff --git a/pkg/lockfile/fixtures/pub/no-packages.lock b/pkg/lockfile/fixtures/pub/no-packages.lock new file mode 100644 index 00000000..e7b03b50 --- /dev/null +++ b/pkg/lockfile/fixtures/pub/no-packages.lock @@ -0,0 +1,3 @@ +# Generated by pub +# See http://pub.dartlang.org/doc/glossary.html#lockfile +packages: {} diff --git a/pkg/lockfile/fixtures/pub/not-yaml.txt b/pkg/lockfile/fixtures/pub/not-yaml.txt new file mode 100644 index 00000000..a2fce73d --- /dev/null +++ b/pkg/lockfile/fixtures/pub/not-yaml.txt @@ -0,0 +1 @@ +this is not valid yaml! diff --git a/pkg/lockfile/fixtures/pub/one-package-dev.lock b/pkg/lockfile/fixtures/pub/one-package-dev.lock new file mode 100644 index 00000000..caf5d623 --- /dev/null +++ b/pkg/lockfile/fixtures/pub/one-package-dev.lock @@ -0,0 +1,10 @@ +# Generated by pub +# See http://pub.dartlang.org/doc/glossary.html#lockfile +packages: + build_runner: + dependency: "direct dev" + description: + name: build_runner + url: "https://pub.dartlang.org" + source: hosted + version: "2.2.1" diff --git a/pkg/lockfile/fixtures/pub/one-package.lock b/pkg/lockfile/fixtures/pub/one-package.lock new file mode 100644 index 00000000..3e0d42e5 --- /dev/null +++ b/pkg/lockfile/fixtures/pub/one-package.lock @@ -0,0 +1,10 @@ +# Generated by pub +# See http://pub.dartlang.org/doc/glossary.html#lockfile +packages: + back_button_interceptor: + dependency: "direct main" + description: + name: back_button_interceptor + url: "https://pub.dartlang.org" + source: hosted + version: "6.0.1" diff --git a/pkg/lockfile/fixtures/pub/source-git.lock b/pkg/lockfile/fixtures/pub/source-git.lock new file mode 100644 index 00000000..e99ecc92 --- /dev/null +++ b/pkg/lockfile/fixtures/pub/source-git.lock @@ -0,0 +1,46 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + flutter_rust_bridge: + dependency: "direct main" + description: + path: frb_dart + ref: master + resolved-ref: e5adce55eea0b74d3680e66a2c5252edf17b07e1 + url: "https://github.com/SoLongAndThanksForAllThePizza/flutter_rust_bridge" + source: git + version: "1.32.0" + screen_retriever: + dependency: transitive + description: + path: "." + ref: "406b9b0" + resolved-ref: "406b9b038b2c1d779f1e7bf609c8c248be247372" + url: "https://github.com/Kingtous/rustdesk_screen_retriever.git" + source: git + version: "0.1.2" + toggle_switch: + dependency: "direct main" + description: + name: toggle_switch + url: "https://pub.dartlang.org" + source: hosted + version: "1.4.0" + tray_manager: + dependency: "direct main" + description: + path: "." + ref: "3aa37c86e47ea748e7b5507cbe59f2c54ebdb23a" + resolved-ref: "3aa37c86e47ea748e7b5507cbe59f2c54ebdb23a" + url: "https://github.com/Kingtous/rustdesk_tray_manager" + source: git + version: "0.1.8" + window_manager: + dependency: "direct main" + description: + path: "." + ref: "88487257cbafc501599ab4f82ec343b46acec020" + resolved-ref: "88487257cbafc501599ab4f82ec343b46acec020" + url: "https://github.com/Kingtous/rustdesk_window_manager" + source: git + version: "0.2.7" diff --git a/pkg/lockfile/fixtures/pub/source-path.lock b/pkg/lockfile/fixtures/pub/source-path.lock new file mode 100644 index 00000000..bee16188 --- /dev/null +++ b/pkg/lockfile/fixtures/pub/source-path.lock @@ -0,0 +1,10 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + maa_core: + dependency: "direct main" + description: + path: ".." + relative: true + source: path + version: "0.0.1" diff --git a/pkg/lockfile/fixtures/pub/source-sdk.lock b/pkg/lockfile/fixtures/pub/source-sdk.lock new file mode 100644 index 00000000..7415a298 --- /dev/null +++ b/pkg/lockfile/fixtures/pub/source-sdk.lock @@ -0,0 +1,11 @@ +# Generated by pub +# See http://pub.dartlang.org/doc/glossary.html#lockfile +packages: + flutter_web_plugins: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" +sdks: + dart: ">=2.17.1 <3.0.0" + flutter: ">=3.0.0" diff --git a/pkg/lockfile/fixtures/pub/two-packages.lock b/pkg/lockfile/fixtures/pub/two-packages.lock new file mode 100644 index 00000000..fad426a8 --- /dev/null +++ b/pkg/lockfile/fixtures/pub/two-packages.lock @@ -0,0 +1,17 @@ +# Generated by pub +# See http://pub.dartlang.org/doc/glossary.html#lockfile +packages: + shelf: + dependency: transitive + description: + name: shelf + url: "https://pub.dartlang.org" + source: hosted + version: "1.3.2" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + url: "https://pub.dartlang.org" + source: hosted + version: "1.0.2" diff --git a/pkg/lockfile/parse-pubspec-lock.go b/pkg/lockfile/parse-pubspec-lock.go new file mode 100644 index 00000000..367103b9 --- /dev/null +++ b/pkg/lockfile/parse-pubspec-lock.go @@ -0,0 +1,93 @@ +package lockfile + +import ( + "fmt" + "gopkg.in/yaml.v2" + "os" +) + +type PubspecLockDescription struct { + Name string `yaml:"name"` + URL string `yaml:"url"` + Path string `yaml:"path"` + Ref string `yaml:"resolved-ref"` +} + +var _ yaml.Unmarshaler = &PubspecLockDescription{} + +func (pld *PubspecLockDescription) UnmarshalYAML(unmarshal func(interface{}) error) error { + var m struct { + Name string `yaml:"name"` + URL string `yaml:"url"` + Path string `yaml:"path"` + Ref string `yaml:"resolved-ref"` + } + + err := unmarshal(&m) + + if err == nil { + pld.Name = m.Name + pld.Path = m.Path + pld.URL = m.URL + pld.Ref = m.Ref + + return nil + } + + var str *string + + err = unmarshal(&str) + + if err != nil { + return err + } + + pld.Path = *str + + return nil +} + +type PubspecLockPackage struct { + Source string `yaml:"source"` + Description PubspecLockDescription `yaml:"description"` + Version string `yaml:"version"` +} + +type PubspecLockfile struct { + Packages map[string]PubspecLockPackage `yaml:"packages,omitempty"` + Sdks map[string]string `yaml:"sdks"` +} + +const PubEcosystem Ecosystem = "Pub" + +func ParsePubspecLock(pathToLockfile string) ([]PackageDetails, error) { + var parsedLockfile *PubspecLockfile + + lockfileContents, err := os.ReadFile(pathToLockfile) + + if err != nil { + return []PackageDetails{}, fmt.Errorf("could not read %s: %w", pathToLockfile, err) + } + + err = yaml.Unmarshal(lockfileContents, &parsedLockfile) + + if err != nil { + return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err) + } + if parsedLockfile == nil { + return []PackageDetails{}, nil + } + + packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) + + for name, pkg := range parsedLockfile.Packages { + packages = append(packages, PackageDetails{ + Name: name, + Version: pkg.Version, + Commit: pkg.Description.Ref, + Ecosystem: PubEcosystem, + }) + } + + return packages, nil +} diff --git a/pkg/lockfile/parse-pubspec-lock_test.go b/pkg/lockfile/parse-pubspec-lock_test.go new file mode 100644 index 00000000..c196736a --- /dev/null +++ b/pkg/lockfile/parse-pubspec-lock_test.go @@ -0,0 +1,221 @@ +package lockfile_test + +import ( + "github.com/g-rath/osv-detector/pkg/lockfile" + "testing" +) + +func TestParsePubspecLock_FileDoesNotExist(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePubspecLock("fixtures/pub/does-not-exist") + + expectErrContaining(t, err, "could not read") + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParsePubspecLock_InvalidYaml(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePubspecLock("fixtures/pub/not-yaml.txt") + + expectErrContaining(t, err, "could not parse") + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParsePubspecLock_Empty(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePubspecLock("fixtures/pub/empty.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParsePubspecLock_NoPackages(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePubspecLock("fixtures/pub/no-packages.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParsePubspecLock_OnePackage(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePubspecLock("fixtures/pub/one-package.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "back_button_interceptor", + Version: "6.0.1", + Ecosystem: lockfile.PubEcosystem, + }, + }) +} + +func TestParsePubspecLock_OnePackageDev(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePubspecLock("fixtures/pub/one-package-dev.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "build_runner", + Version: "2.2.1", + Ecosystem: lockfile.PubEcosystem, + }, + }) +} + +func TestParsePubspecLock_TwoPackages(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePubspecLock("fixtures/pub/two-packages.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "shelf", + Version: "1.3.2", + Ecosystem: lockfile.PubEcosystem, + }, + { + Name: "shelf_web_socket", + Version: "1.0.2", + Ecosystem: lockfile.PubEcosystem, + }, + }) +} + +func TestParsePubspecLock_MixedPackages(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePubspecLock("fixtures/pub/mixed-packages.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "back_button_interceptor", + Version: "6.0.1", + Ecosystem: lockfile.PubEcosystem, + }, + { + Name: "build_runner", + Version: "2.2.1", + Ecosystem: lockfile.PubEcosystem, + }, + { + Name: "shelf", + Version: "1.3.2", + Ecosystem: lockfile.PubEcosystem, + }, + { + Name: "shelf_web_socket", + Version: "1.0.2", + Ecosystem: lockfile.PubEcosystem, + }, + }) +} + +func TestParsePubspecLock_PackageWithGitSource(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePubspecLock("fixtures/pub/source-git.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "flutter_rust_bridge", + Version: "1.32.0", + Ecosystem: lockfile.PubEcosystem, + Commit: "e5adce55eea0b74d3680e66a2c5252edf17b07e1", + }, + { + Name: "screen_retriever", + Version: "0.1.2", + Ecosystem: lockfile.PubEcosystem, + Commit: "406b9b038b2c1d779f1e7bf609c8c248be247372", + }, + { + Name: "tray_manager", + Version: "0.1.8", + Ecosystem: lockfile.PubEcosystem, + Commit: "3aa37c86e47ea748e7b5507cbe59f2c54ebdb23a", + }, + { + Name: "window_manager", + Version: "0.2.7", + Ecosystem: lockfile.PubEcosystem, + Commit: "88487257cbafc501599ab4f82ec343b46acec020", + }, + { + Name: "toggle_switch", + Version: "1.4.0", + Ecosystem: lockfile.PubEcosystem, + Commit: "", + }, + }) +} + +func TestParsePubspecLock_PackageWithSdkSource(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePubspecLock("fixtures/pub/source-sdk.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "flutter_web_plugins", + Version: "0.0.0", + Ecosystem: lockfile.PubEcosystem, + Commit: "", + }, + }) +} + +func TestParsePubspecLock_PackageWithPathSource(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePubspecLock("fixtures/pub/source-path.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "maa_core", + Version: "0.0.1", + Ecosystem: lockfile.PubEcosystem, + Commit: "", + }, + }) +} diff --git a/pkg/lockfile/parse.go b/pkg/lockfile/parse.go index 73c27d7d..42788a13 100644 --- a/pkg/lockfile/parse.go +++ b/pkg/lockfile/parse.go @@ -27,6 +27,7 @@ var parsers = map[string]PackageDetailsParser{ "pnpm-lock.yaml": ParsePnpmLock, "poetry.lock": ParsePoetryLock, "pom.xml": ParseMavenLock, + "pubspec.lock": ParsePubspecLock, "requirements.txt": ParseRequirementsTxt, "yarn.lock": ParseYarnLock, } diff --git a/pkg/lockfile/parse_test.go b/pkg/lockfile/parse_test.go index a032f466..4bb6d469 100644 --- a/pkg/lockfile/parse_test.go +++ b/pkg/lockfile/parse_test.go @@ -44,6 +44,7 @@ func TestFindParser(t *testing.T) { "package-lock.json", "yarn.lock", "pnpm-lock.yaml", + "pubspec.lock", "composer.lock", "Gemfile.lock", "go.mod", @@ -92,6 +93,7 @@ func TestParse_FindsExpectedParsers(t *testing.T) { "mix.lock", "pom.xml", "poetry.lock", + "pubspec.lock", "requirements.txt", }