fix(pip): fix compatibility with some simple pip indexes (#6649)

renovatebot · Jul 7, 2020 · 7d3b475 · 7d3b475
1 parent 634e553
commit 7d3b475
Show file tree

Hide file tree

Showing 3 changed files with 115 additions and 1 deletion.
diff --git a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
@@ -1,5 +1,69 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`datasource/pypi getReleases fall back from json and process data from simple endpoint 1`] = `
+Object {
+  "releases": Array [
+    Object {
+      "version": "0.1.2",
+    },
+    Object {
+      "version": "0.1.3",
+    },
+    Object {
+      "version": "0.1.4",
+    },
+    Object {
+      "version": "0.2.0",
+    },
+    Object {
+      "version": "0.2.1",
+    },
+    Object {
+      "version": "0.2.2",
+    },
+    Object {
+      "version": "0.3.0",
+    },
+    Object {
+      "version": "0.4.0",
+    },
+    Object {
+      "version": "0.4.1",
+    },
+    Object {
+      "version": "0.4.2",
+    },
+    Object {
+      "version": "0.5.0",
+    },
+  ],
+}
+`;
+
+exports[`datasource/pypi getReleases fall back from json and process data from simple endpoint 2`] = `
+Array [
+  Object {
+    "headers": Object {
+      "accept": "application/json",
+      "accept-encoding": "gzip, deflate",
+      "host": "custom.pypi.net",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "GET",
+    "url": "https://custom.pypi.net/foo/dj-database-url/json",
+  },
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate",
+      "host": "custom.pypi.net",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "GET",
+    "url": "https://custom.pypi.net/foo/dj-database-url",
+  },
+]
+`;
+
 exports[`datasource/pypi getReleases find url from project_urls 1`] = `
 Array [
   Object {
@@ -318,6 +382,15 @@ Array [
     "method": "GET",
     "url": "https://pypi.org/pypi/something/json",
   },
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate",
+      "host": "pypi.org",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "GET",
+    "url": "https://pypi.org/pypi/something",
+  },
 ]
 `;
 

diff --git a/lib/datasource/pypi/index.spec.ts b/lib/datasource/pypi/index.spec.ts
@@ -46,6 +46,7 @@ describe('datasource/pypi', () => {
     });
     it('returns null for 404', async () => {
       httpMock.scope(baseUrl).get('/something/json').reply(404);
+      httpMock.scope(baseUrl).get('/something').reply(404);
       expect(
         await getPkgReleases({
           datasource,
@@ -303,5 +304,25 @@ describe('datasource/pypi', () => {
         })
       ).toBeNull();
     });
+    it('fall back from json and process data from simple endpoint', async () => {
+      httpMock
+        .scope('https://custom.pypi.net/foo')
+        .get('/dj-database-url/json')
+        .reply(404);
+      httpMock
+        .scope('https://custom.pypi.net/foo')
+        .get('/dj-database-url')
+        .reply(200, htmlResponse + '');
+      const config = {
+        registryUrls: ['https://custom.pypi.net/foo'],
+      };
+      const result = await getPkgReleases({
+        datasource,
+        ...config,
+        depName: 'dj-database-url',
+      });
+      expect(result).toMatchSnapshot();
+      expect(httpMock.getTrace()).toMatchSnapshot();
+    });
   });
 });
diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts
@@ -191,10 +191,30 @@ export async function getReleases({
   registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   const hostUrl = ensureTrailingSlash(registryUrl);
+
+  // not all simple indexes use this identifier, but most do
   if (hostUrl.endsWith('/simple/') || hostUrl.endsWith('/+simple/')) {
     logger.trace({ lookupName, hostUrl }, 'Looking up pypi simple dependency');
     return getSimpleDependency(lookupName, hostUrl);
   }
+
   logger.trace({ lookupName, hostUrl }, 'Looking up pypi api dependency');
-  return getDependency(lookupName, hostUrl, compatibility);
+  try {
+    // we need to resolve early here so we can catch any 404s and fallback to a simple lookup
+    const releases = await getDependency(lookupName, hostUrl, compatibility);
+    // the dep was found in the json api, return as-is
+    return releases;
+  } catch (err) {
+    if (err.statusCode !== 404) {
+      throw err;
+    }
+
+    // error contacting json-style api -- attempt to fallback to a simple-style api
+    logger.trace(
+      { lookupName, hostUrl },
+      'Looking up pypi simple dependency via fallback'
+    );
+    const releases = await getSimpleDependency(lookupName, hostUrl);
+    return releases;
+  }
 }