From 7d3b475c5199ca168a02ff8a650ac663876b6d6c Mon Sep 17 00:00:00 2001
From: Kevin James <KevinJames@thekev.in>
Date: Mon, 6 Jul 2020 21:56:31 -0700
Subject: [PATCH] fix(pip): fix compatibility with some simple pip indexes
 (#6649)

---
 .../pypi/__snapshots__/index.spec.ts.snap     | 73 +++++++++++++++++++
 lib/datasource/pypi/index.spec.ts             | 21 ++++++
 lib/datasource/pypi/index.ts                  | 22 +++++-
 3 files changed, 115 insertions(+), 1 deletion(-)

diff --git a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
index 957cebdae4987f..15bbd461e9a218 100644
--- a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
@@ -1,5 +1,69 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`datasource/pypi getReleases fall back from json and process data from simple endpoint 1`] = `
+Object {
+  "releases": Array [
+    Object {
+      "version": "0.1.2",
+    },
+    Object {
+      "version": "0.1.3",
+    },
+    Object {
+      "version": "0.1.4",
+    },
+    Object {
+      "version": "0.2.0",
+    },
+    Object {
+      "version": "0.2.1",
+    },
+    Object {
+      "version": "0.2.2",
+    },
+    Object {
+      "version": "0.3.0",
+    },
+    Object {
+      "version": "0.4.0",
+    },
+    Object {
+      "version": "0.4.1",
+    },
+    Object {
+      "version": "0.4.2",
+    },
+    Object {
+      "version": "0.5.0",
+    },
+  ],
+}
+`;
+
+exports[`datasource/pypi getReleases fall back from json and process data from simple endpoint 2`] = `
+Array [
+  Object {
+    "headers": Object {
+      "accept": "application/json",
+      "accept-encoding": "gzip, deflate",
+      "host": "custom.pypi.net",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "GET",
+    "url": "https://custom.pypi.net/foo/dj-database-url/json",
+  },
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate",
+      "host": "custom.pypi.net",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "GET",
+    "url": "https://custom.pypi.net/foo/dj-database-url",
+  },
+]
+`;
+
 exports[`datasource/pypi getReleases find url from project_urls 1`] = `
 Array [
   Object {
@@ -318,6 +382,15 @@ Array [
     "method": "GET",
     "url": "https://pypi.org/pypi/something/json",
   },
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate",
+      "host": "pypi.org",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "GET",
+    "url": "https://pypi.org/pypi/something",
+  },
 ]
 `;
 
diff --git a/lib/datasource/pypi/index.spec.ts b/lib/datasource/pypi/index.spec.ts
index e3c577caf38275..622e323003d1a2 100644
--- a/lib/datasource/pypi/index.spec.ts
+++ b/lib/datasource/pypi/index.spec.ts
@@ -46,6 +46,7 @@ describe('datasource/pypi', () => {
     });
     it('returns null for 404', async () => {
       httpMock.scope(baseUrl).get('/something/json').reply(404);
+      httpMock.scope(baseUrl).get('/something').reply(404);
       expect(
         await getPkgReleases({
           datasource,
@@ -303,5 +304,25 @@ describe('datasource/pypi', () => {
         })
       ).toBeNull();
     });
+    it('fall back from json and process data from simple endpoint', async () => {
+      httpMock
+        .scope('https://custom.pypi.net/foo')
+        .get('/dj-database-url/json')
+        .reply(404);
+      httpMock
+        .scope('https://custom.pypi.net/foo')
+        .get('/dj-database-url')
+        .reply(200, htmlResponse + '');
+      const config = {
+        registryUrls: ['https://custom.pypi.net/foo'],
+      };
+      const result = await getPkgReleases({
+        datasource,
+        ...config,
+        depName: 'dj-database-url',
+      });
+      expect(result).toMatchSnapshot();
+      expect(httpMock.getTrace()).toMatchSnapshot();
+    });
   });
 });
diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts
index 56c3a9a605a4ba..2945a31047117e 100644
--- a/lib/datasource/pypi/index.ts
+++ b/lib/datasource/pypi/index.ts
@@ -191,10 +191,30 @@ export async function getReleases({
   registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   const hostUrl = ensureTrailingSlash(registryUrl);
+
+  // not all simple indexes use this identifier, but most do
   if (hostUrl.endsWith('/simple/') || hostUrl.endsWith('/+simple/')) {
     logger.trace({ lookupName, hostUrl }, 'Looking up pypi simple dependency');
     return getSimpleDependency(lookupName, hostUrl);
   }
+
   logger.trace({ lookupName, hostUrl }, 'Looking up pypi api dependency');
-  return getDependency(lookupName, hostUrl, compatibility);
+  try {
+    // we need to resolve early here so we can catch any 404s and fallback to a simple lookup
+    const releases = await getDependency(lookupName, hostUrl, compatibility);
+    // the dep was found in the json api, return as-is
+    return releases;
+  } catch (err) {
+    if (err.statusCode !== 404) {
+      throw err;
+    }
+
+    // error contacting json-style api -- attempt to fallback to a simple-style api
+    logger.trace(
+      { lookupName, hostUrl },
+      'Looking up pypi simple dependency via fallback'
+    );
+    const releases = await getSimpleDependency(lookupName, hostUrl);
+    return releases;
+  }
 }