feat(github): Shrink GraphQL nodes count on 50x errors (#8394)

renovatebot · Feb 6, 2021 · 27a494f · 27a494f
1 parent be47f40
commit 27a494f
Show file tree

Hide file tree

Showing 3 changed files with 202 additions and 3 deletions.
diff --git a/lib/util/http/__snapshots__/github.spec.ts.snap b/lib/util/http/__snapshots__/github.spec.ts.snap
@@ -0,0 +1,157 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`util/http/github GraphQL shrinks items count on 50x 1`] = `
+Array [
+  Object {
+    "graphql": Object {
+      "query": Object {
+        "repository": Object {
+          "__args": Object {
+            "name": "testName",
+            "owner": "testOwner",
+          },
+          "testItem": Object {
+            "__args": Object {
+              "first": "100",
+            },
+            "nodes": Object {
+              "body": null,
+              "number": null,
+              "state": null,
+              "title": null,
+            },
+            "pageInfo": Object {
+              "endCursor": null,
+              "hasNextPage": null,
+            },
+          },
+        },
+      },
+    },
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate",
+      "content-length": "408",
+      "content-type": "application/json",
+      "host": "api.github.com",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "POST",
+    "url": "https://api.github.com/graphql",
+  },
+  Object {
+    "graphql": Object {
+      "query": Object {
+        "repository": Object {
+          "__args": Object {
+            "name": "testName",
+            "owner": "testOwner",
+          },
+          "testItem": Object {
+            "__args": Object {
+              "after": "cursor1",
+              "first": "100",
+            },
+            "nodes": Object {
+              "body": null,
+              "number": null,
+              "state": null,
+              "title": null,
+            },
+            "pageInfo": Object {
+              "endCursor": null,
+              "hasNextPage": null,
+            },
+          },
+        },
+      },
+    },
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate",
+      "content-length": "428",
+      "content-type": "application/json",
+      "host": "api.github.com",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "POST",
+    "url": "https://api.github.com/graphql",
+  },
+  Object {
+    "graphql": Object {
+      "query": Object {
+        "repository": Object {
+          "__args": Object {
+            "name": "testName",
+            "owner": "testOwner",
+          },
+          "testItem": Object {
+            "__args": Object {
+              "after": "cursor1",
+              "first": "50",
+            },
+            "nodes": Object {
+              "body": null,
+              "number": null,
+              "state": null,
+              "title": null,
+            },
+            "pageInfo": Object {
+              "endCursor": null,
+              "hasNextPage": null,
+            },
+          },
+        },
+      },
+    },
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate",
+      "content-length": "427",
+      "content-type": "application/json",
+      "host": "api.github.com",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "POST",
+    "url": "https://api.github.com/graphql",
+  },
+  Object {
+    "graphql": Object {
+      "query": Object {
+        "repository": Object {
+          "__args": Object {
+            "name": "testName",
+            "owner": "testOwner",
+          },
+          "testItem": Object {
+            "__args": Object {
+              "after": "cursor2",
+              "first": "50",
+            },
+            "nodes": Object {
+              "body": null,
+              "number": null,
+              "state": null,
+              "title": null,
+            },
+            "pageInfo": Object {
+              "endCursor": null,
+              "hasNextPage": null,
+            },
+          },
+        },
+      },
+    },
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate",
+      "content-length": "427",
+      "content-type": "application/json",
+      "host": "api.github.com",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "POST",
+    "url": "https://api.github.com/graphql",
+  },
+]
+`;
diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts
@@ -388,5 +388,32 @@ describe(getName(__filename), () => {
       expect(httpMock.getTrace()).toHaveLength(2);
       expect(items).toHaveLength(2);
     });
+    it('shrinks items count on 50x', async () => {
+      httpMock
+        .scope(githubApiHost)
+        .post('/graphql')
+        .reply(200, page1)
+        .post('/graphql')
+        .reply(500)
+        .post('/graphql')
+        .reply(200, page2)
+        .post('/graphql')
+        .reply(200, page3);
+
+      const items = await githubApi.queryRepoField(query, 'testItem');
+      expect(items).toHaveLength(3);
+
+      const trace = httpMock.getTrace();
+      expect(trace).toHaveLength(4);
+      expect(trace).toMatchSnapshot();
+    });
+    it('throws on 50x if count < 10', async () => {
+      httpMock.scope(githubApiHost).post('/graphql').reply(500);
+      await expect(
+        githubApi.queryRepoField(query, 'testItem', {
+          count: 9,
+        })
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
+    });
   });
 });
diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts
@@ -235,8 +235,23 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
         opts
       );
       result = res?.body?.data?.repository;
-    } catch (gotErr) {
-      handleGotError(gotErr, path, opts);
+    } catch (err) {
+      if (err instanceof ExternalHostError) {
+        const gotError = err.err as GotLegacyError;
+        const statusCode = gotError?.statusCode;
+        const count = options.count;
+        if (
+          count &&
+          count > 10 &&
+          statusCode &&
+          statusCode >= 500 &&
+          statusCode < 600
+        ) {
+          logger.info('Reducing pagination count to workaround graphql 5xx');
+          return null;
+        }
+      }
+      handleGotError(err, path, opts);
     }
     return result;
   }
@@ -263,7 +278,7 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
         replacement += cursor ? `, after: "${cursor}", ` : ', ';
         query = query.replace(regex, replacement);
       }
-      const gqlRes = await this.queryRepo<T>(query, options);
+      const gqlRes = await this.queryRepo<T>(query, { ...options, count });
       if (gqlRes?.[fieldName]) {
         const { nodes = [], edges = [], pageInfo } = gqlRes[fieldName];
         result.push(...nodes);