Check deprecated packages (#588)

* First steps

Basic skeleton + notes of what to do next

* Move git code into test-runner

* Remove unused utils

* Add and refactor tests

* Tests working

* Package-level checks are done

* Part 1 is done

* Update package-lock.json too?

* Update jest. This changes EVERYTHING???!

package.json

@@ -40,7 +40,7 @@
     "@types/tar": "^1.0.27",
     "@types/travis-fold": "^0.1.0",
     "@types/yargs": "^8.0.1",
-    "jest": "^23.6.0",
+    "jest": "^24.3.0",
     "ts-jest": "^23.10.5",
     "tslint": "^5.13.0"
   },

src/get-definitely-typed.test.ts

@@ -1,7 +1,7 @@
 import { Dir, FS, getDefinitelyTyped, InMemoryDT } from "./get-definitely-typed";
 import { Options } from "./lib/common";
 import { loggerWithErrors } from "./util/logging";
-import { testo } from "./util/util";
+import { testo } from "./util/test";
 
 testo({
     async downloadDefinitelyTyped() {

src/lib/packages.ts

@@ -2,7 +2,7 @@ import assert = require("assert");
 import { Author, TypeScriptVersion } from "definitelytyped-header-parser";
 
 import { FS } from "../get-definitely-typed";
-import { assertDefined, assertSorted, joinPaths, mapValues, unmangleScopedPackage } from "../util/util";
+import { assertSorted, joinPaths, mapValues, unmangleScopedPackage } from "../util/util";
 
 import { readDataFile } from "./common";
 import { outputDirPath, scopeName } from "./settings";
@@ -51,8 +51,8 @@ export class AllPackages {
         private readonly data: ReadonlyMap<string, TypingsVersions>,
         private readonly notNeeded: ReadonlyArray<NotNeededPackage>) {}
 
-    getNotNeededPackage(name: string): NotNeededPackage {
-        return assertDefined(this.notNeeded.find(p => p.name === name));
+    getNotNeededPackage(name: string): NotNeededPackage | undefined {
+        return this.notNeeded.find(p => p.name === name);
     }
 
     hasTypingFor(dep: PackageId): boolean {

src/lib/versions.ts

@@ -32,7 +32,7 @@ export async function readChangedPackages(allPackages: AllPackages): Promise<Cha
     return {
         changedTypings: json.changedTypings.map(({ id, version, latestVersion }): ChangedTyping =>
             ({ pkg: allPackages.getTypingsData(id), version, latestVersion })),
-        changedNotNeededPackages: json.changedNotNeededPackages.map(id => allPackages.getNotNeededPackage(id)),
+        changedNotNeededPackages: json.changedNotNeededPackages.map(id => assertDefined(allPackages.getNotNeededPackage(id))),
     };
 }
 

src/parse-definitions.test.ts

@@ -1,7 +1,7 @@
 import parseDefinitions from "./parse-definitions";
 import { Dir, InMemoryDT } from "./get-definitely-typed";
 import { loggerWithErrors } from "./util/logging";
-import { testo } from "./util/util";
+import { testo } from "./util/test";
 
 function createMockDT() {
     const root = new Dir(undefined);

src/tester/get-affected-packages.test.ts

@@ -1,32 +1,6 @@
-import { testo } from "../util/util";
-import { AllPackages, License, NotNeededPackage, TypesDataFile, TypingsVersionsRaw, PackageId } from "../lib/packages";
-import getAffectedPackages from "./get-affected-packages";
-
-function createTypingsVersionRaw(
-    name: string, dependencies: PackageId[], testDependencies: string[]
-): TypingsVersionsRaw {
-    return {
-        "1": {
-            libraryName: name,
-            typingsPackageName: name,
-            dependencies,
-            testDependencies,
-            files: ["index.d.ts"],
-            libraryMajorVersion: 1,
-            libraryMinorVersion: 0,
-            pathMappings: [],
-            contributors: [{ name: "Bender", url: "futurama.com", githubUsername: "bender" },],
-            minTsVersion: "2.3",
-            typesVersions: [],
-            license: License.MIT,
-            packageJsonDependencies: [],
-            contentHash: "11111111111111",
-            projectName: "zombo.com",
-            globals: [],
-            declaredModules: [],
-        },
-    }
-}
+import { testo, createTypingsVersionRaw } from "../util/test";
+import { AllPackages, NotNeededPackage, TypesDataFile } from "../lib/packages";
+import { getAffectedPackages } from "./get-affected-packages";
 const typesData: TypesDataFile = {
     jquery: createTypingsVersionRaw("jquery", [], []),
     known: createTypingsVersionRaw("known", [{ name: "jquery", majorVersion: 1 }], []),

src/tester/get-affected-packages.ts

@@ -1,28 +1,13 @@
-import { getDefinitelyTyped } from "../get-definitely-typed";
-import { Options, TesterOptions } from "../lib/common";
-import { parseMajorVersionFromDirectoryName } from "../lib/definition-parser";
-import { AllPackages, PackageBase, TypingsData, PackageId, DependencyVersion, getMangledNameForScopedPackage } from "../lib/packages";
-import { sourceBranch, typesDirectoryName } from "../lib/settings";
-import { consoleLogger, Logger, loggerWithErrors } from "../util/logging";
-import { execAndThrowErrors, flatMap, logUncaughtErrors, mapDefined, mapIter, sort } from "../util/util";
-
-if (!module.parent) {
-    logUncaughtErrors(main(Options.defaults));
-}
-async function main(options: TesterOptions): Promise<void> {
-    const changes = getAffectedPackages(
-        await AllPackages.read(await getDefinitelyTyped(options, loggerWithErrors()[0])),
-        await gitChanges(consoleLogger.info, options.definitelyTypedPath));
-    console.log({ changedPackages: changes.changedPackages.map(t => t.desc), dependersLength: changes.dependentPackages.map(t => t.desc).length });
-}
+import { AllPackages, PackageBase, TypingsData, PackageId, getMangledNameForScopedPackage } from "../lib/packages";
+import { mapDefined, mapIter, sort } from "../util/util";
 
 export interface Affected {
     readonly changedPackages: ReadonlyArray<TypingsData>;
     readonly dependentPackages: ReadonlyArray<TypingsData>;
 }
 
 /** Gets all packages that have changed on this branch, plus all packages affected by the change. */
-export default function getAffectedPackages(allPackages: AllPackages, changedPackageIds: PackageId[]): Affected {
+export function getAffectedPackages(allPackages: AllPackages, changedPackageIds: PackageId[]): Affected {
     const resolved = changedPackageIds.map(id => allPackages.tryResolve(id));
     // If a package doesn't exist, that's because it was deleted.
     const changed = mapDefined(resolved, id => allPackages.tryGetTypingsData(id));
@@ -106,90 +91,3 @@ function getReverseDependencies(allPackages: AllPackages, changedPackages: Packa
 function packageIdToKey(pkg: PackageId): string {
     return getMangledNameForScopedPackage(pkg.name) + "/v" + pkg.majorVersion;
 }
-
-/** Returns all immediate subdirectories of the root directory that have changed. */
-export async function gitChanges(log: Logger, definitelyTypedPath: string): Promise<Array<PackageId>> {
-    const changedPackages = new Map<string, Set<DependencyVersion>>();
-
-    for (const fileName of await gitDiff(log, definitelyTypedPath)) {
-        const dep = getDependencyFromFile(fileName);
-        if (dep) {
-            const versions = changedPackages.get(dep.name);
-            if (!versions) {
-                changedPackages.set(dep.name, new Set([dep.majorVersion]));
-            } else {
-                versions.add(dep.majorVersion);
-            }
-        }
-    }
-
-    return Array.from(flatMap(changedPackages, ([name, versions]) =>
-        mapIter(versions, majorVersion => ({ name, majorVersion }))));
-}
-
-/*
-We have to be careful about how we get the diff because travis uses a shallow clone.
-
-Travis runs:
-    git clone --depth=50 https://github.com/DefinitelyTyped/DefinitelyTyped.git DefinitelyTyped
-    cd DefinitelyTyped
-    git fetch origin +refs/pull/123/merge
-    git checkout -qf FETCH_HEAD
-
-If editing this code, be sure to test on both full and shallow clones.
-*/
-async function gitDiff(log: Logger, definitelyTypedPath: string): Promise<string[]> {
-    try {
-        await run(`git rev-parse --verify ${sourceBranch}`);
-        // If this succeeds, we got the full clone.
-    } catch (_) {
-        // This is a shallow clone.
-        await run(`git fetch origin ${sourceBranch}`);
-        await run(`git branch ${sourceBranch} FETCH_HEAD`);
-    }
-
-    // `git diff foo...bar` gets all changes from X to `bar` where X is the common ancestor of `foo` and `bar`.
-    // Source: https://git-scm.com/docs/git-diff
-    let diff = (await run(`git diff ${sourceBranch} --name-only`)).trim();
-    if (diff === "") {
-        // We are probably already on master, so compare to the last commit.
-        diff = (await run(`git diff ${sourceBranch}~1 --name-only`)).trim();
-    }
-    return diff.split("\n");
-
-    async function run(cmd: string): Promise<string> {
-        log(`Running: ${cmd}`);
-        const stdout = await execAndThrowErrors(cmd, definitelyTypedPath);
-        log(stdout);
-        return stdout;
-    }
-}
-
-/**
- * For "types/a/b/c", returns { name: "a", version: "latest" }.
- * For "types/a/v3/c", returns { name: "a", version: 3 }.
- * For "x", returns undefined.
- */
-function getDependencyFromFile(fileName: string): PackageId | undefined {
-    const parts = fileName.split("/");
-    if (parts.length <= 2) {
-        // It's not in a typings directory at all.
-        return undefined;
-    }
-
-    const [typesDirName, name, subDirName] = parts; // Ignore any other parts
-
-    if (typesDirName !== typesDirectoryName) {
-        return undefined;
-    }
-
-    if (subDirName) {
-        // Looks like "types/a/v3/c"
-        const majorVersion = parseMajorVersionFromDirectoryName(subDirName);
-        if (majorVersion !== undefined) {
-            return { name,  majorVersion };
-        }
-    }
-
-    return { name, majorVersion: "*" };
-}

src/tester/test-runner.test.ts

@@ -0,0 +1,62 @@
+import { testo, createTypingsVersionRaw } from "../util/test";
+import { AllPackages, NotNeededPackage, TypesDataFile } from "../lib/packages";
+import { checkDeletedFiles, GitDiff } from "./test-runner";
+
+const typesData: TypesDataFile = {
+    jquery: createTypingsVersionRaw("jquery", [], []),
+    known: createTypingsVersionRaw("known", [{ name: "jquery", majorVersion: 1 }], []),
+    "known-test": createTypingsVersionRaw("known-test", [], ["jquery"]),
+    "most-recent": createTypingsVersionRaw("most-recent", [{ name: "jquery", majorVersion: "*" }], []),
+    unknown: createTypingsVersionRaw("unknown", [{ name: "COMPLETELY-UNKNOWN", majorVersion: 1 }], []),
+    "unknown-test": createTypingsVersionRaw("unknown-test", [], ["WAT"]),
+};
+
+const jestNotNeeded = [
+    new NotNeededPackage({ typingsPackageName: "jest", libraryName: "jest", asOfVersion: "100.0.0", sourceRepoURL: "jest.com" })
+];
+const allPackages = AllPackages.from(typesData, jestNotNeeded);
+
+const deleteJestDiffs: GitDiff[] = [
+    { status: "M", file: "notNeededPackages.json" },
+    { status: "D", file: "types/jest/index.d.ts" },
+    { status: "D", file: "types/jest/jest-tests.d.ts" },
+];
+
+
+testo({
+    ok() {
+        checkDeletedFiles(allPackages, deleteJestDiffs);
+    },
+    forgotToDeleteFiles() {
+        expect(() =>
+        checkDeletedFiles(
+            AllPackages.from({ jest: createTypingsVersionRaw("jest", [], []) }, jestNotNeeded),
+            deleteJestDiffs)).toThrow('Please delete all files in jest');
+
+    },
+    tooManyDeletes() {
+        expect(() => checkDeletedFiles(allPackages, [{ status: "D", file: "oops.txt" }])).toThrow(
+            "Unexpected file deleted: oops.txt");
+    },
+    extraneousFile() {
+        checkDeletedFiles(allPackages, [
+            { status: "A", file: "oooooooooooops.txt" },
+            { status: "M", file: "notNeededPackages.json" },
+            { status: "D", file: "types/jest/index.d.ts" },
+            { status: "D", file: "types/jest/jest-tests.d.ts" },
+        ]);
+    },
+    forgotToUpdateNotNeededJson() {
+        expect(() => checkDeletedFiles(AllPackages.from(typesData, []), [{status: "D", file: "types/jest/index.d.ts" }])).toThrow(
+            "Deleted package jest is not in notNeededPackages.json.");
+    },
+    scoped() {
+        checkDeletedFiles(
+            AllPackages.from(
+                typesData,
+                [new NotNeededPackage({ typingsPackageName: "ember__object", libraryName: "@ember/object", asOfVersion: "1.0.0", sourceRepoURL: "ember.js" })]),
+            [{ status: "D", file: "types/ember__object/index.d.ts" }]);
+    },
+    // TODO: Test npm info (and with scoped names)
+    // TODO: Test with dependents, etc etc
+});