Dev environment runs on M1 mac (#188)

* Dev environment runs on M1 mac Pupeteer does not play well with M1 and ARM in general. While they [just released](puppeteer/puppeteer#7546) experimental support for ARM macs, they still don't support [ARM machines for Debian/Docker](puppeteer/puppeteer#7546 (comment)). This PR switches from puppeteer to [zombie](https://github.com/assaf/zombie). Ideally I would like to move back, as Zombie does not appear to be actively maintained. * Replaced puppeteer mock with zombie + removed commented out puppeteer code
olaven · May 13, 2022 · 59d3663 · 59d3663
1 parent 7a5824b
commit 59d3663
Show file tree

Hide file tree

Showing 13 changed files with 320 additions and 143 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -19,9 +19,7 @@ services:
       - LOG_LEVEL=$LOG_LEVEL
     working_dir: /paperpod
   api:
-    build:
-      context: .
-      dockerfile: ./packages/api/Dockerfile
+    build: .
     env_file:
       - .env
     ports:
@@ -78,7 +76,7 @@ services:
       - .env
     ports:
       - $WEB_PORT:$WEB_PORT
-    command: yarn web dev
+    command: yarn web dev --port $WEB_PORT
     volumes:
       - .:/paperpod
     environment:

diff --git a/dockerfiles/Dockerfile.api b/dockerfiles/Dockerfile.api
@@ -1,4 +1,4 @@
-FROM buildkite/puppeteer:8.0.0
+FROM node:12-alpine
 
 WORKDIR /app
 

diff --git a/packages/api/Dockerfile b/packages/api/Dockerfile
diff --git a/packages/authentication/src/app.ts b/packages/authentication/src/app.ts
@@ -5,7 +5,6 @@ import { subscriptionManagementRoutes } from "./routes/public/subscription-manag
 import { userRoutes } from "./routes/public/user-routes";
 
 export const publicAuthenticationApp = server.app
-
   .appWithEnvironment(
     server.app.appWithBodyParser(server.app.appWithCookieParser())
   )

diff --git a/packages/converter/__mocks__/puppeteer.ts b/packages/converter/__mocks__/puppeteer.ts
diff --git a/packages/converter/__mocks__/zombie.ts b/packages/converter/__mocks__/zombie.ts
@@ -0,0 +1,36 @@
+import faker from "faker";
+
+export default class Zombie {
+  private doneVisiting: boolean = false;
+  constructor(_options) {}
+  public visit(_url: string, callback: () => void) {
+    this.doneVisiting = true;
+    callback();
+  }
+
+  public html() {
+    if (!this.doneVisiting) {
+      throw `Should not have called HTML before done with visit`;
+    }
+
+    return `
+      <!DOCTYPE html>
+
+        <head>
+            <title>${faker.lorem.sentence()}</title>
+            <meta name="author" content="${faker.name.firstName()}"/>
+        </head>
+
+        <body>
+            <p>${faker.lorem.paragraphs()}</p>
+            <p>${faker.lorem.paragraphs()}</p>
+            <p>${faker.lorem.paragraphs()}</p>
+            <p>${faker.lorem.paragraphs()}</p>
+            <p>${faker.lorem.paragraphs()}</p>
+            <p>${faker.lorem.paragraphs()}</p>
+        </body>
+
+      </html>
+    `;
+  }
+}
diff --git a/packages/converter/package.json b/packages/converter/package.json
@@ -14,9 +14,9 @@
     "@paperpod/common": "0.0.1",
     "node-kall": "^1.0.83",
     "pdf-parse": "^1.1.1",
-    "puppeteer": "^5.4.1",
     "serialize-xml": "^0.4.0",
-    "unfluff": "^3.2.0"
+    "unfluff": "^3.2.0",
+    "zombie": "^6.1.4"
   },
   "devDependencies": {
     "@types/faker": "^5.5.3",

diff --git a/packages/converter/src/index.test.ts b/packages/converter/src/index.test.ts
@@ -3,7 +3,7 @@ import { withStorageUri } from ".";
 import { withTextualData, getRSSFeed } from "./index";
 
 describe("converter", () => {
-  jest.mock("puppeteer");
+  jest.mock("zombie");
 
   describe("withTextualData", () => {
     it("Does not throw", () => {

diff --git a/packages/converter/src/rss.test.ts b/packages/converter/src/rss.test.ts
@@ -85,10 +85,6 @@ describe("Conversion from articles to RSS", () => {
       expect(rss).toContain("<link>");
     });
 
-    it("Print to show", () => {
-      console.log(convertToRSSFeed([test.mocks.article()]));
-    });
-
     describe("The channel cover image", () => {
       const getImageTag = () => {
         const rss = convertToRSSFeed([]);

diff --git a/packages/converter/src/text/web.test.ts b/packages/converter/src/text/web.test.ts
@@ -1,10 +1,10 @@
-import { models, test } from "@paperpod/common";
+import { test } from "@paperpod/common";
 import { extractTextFromWeb } from "./web";
 
 describe("extracting text", () => {
-  jest.mock("puppeteer");
+  jest.mock("zombie");
 
-  it("Does mock puppeteer", async () => {
+  it("Does mock zombie", async () => {
     const content = extractTextFromWeb(test.mocks.article());
     expect(content).toBeDefined();
   });

diff --git a/packages/converter/src/text/web.ts b/packages/converter/src/text/web.ts
@@ -1,6 +1,7 @@
 import unfluff from "unfluff";
-import puppeteer from "puppeteer";
 import { models } from "@paperpod/common";
+import { logger } from "@paperpod/common";
+import zombie from "zombie";
 
 /**
  * returns the publication timestamp, if any.
@@ -22,30 +23,24 @@ export const extractTextFromWeb = async (
     author: extracted.author().join(", "),
     description: extracted.description(),
     publication_time: date(extracted),
-    added_time: new Date()
+    added_time: new Date(),
   };
 };
 
 /**
- * Using Puppeteer (or a browser-emulator in general) makes it possible for
- * me to access text that is not directly provided, but client side rendered.
+ * Using Zombie (or a browser-emulator in general) makes it possible
+ * to access text that is not directly provided, but client side rendered.
  */
-const getHtml = async (url: string) => {
-  const browser = await puppeteer.launch({
-    executablePath: process.env.PUPPETEER_EXEC_PATH,
-    headless: true,
-    //FIXME: security considerations without sandbox? Read up on this.
-    args: ["--disable-setuid-sandbox", "--no-sandbox"],
-    ignoreHTTPSErrors: true,
+const getHtml = (url: string): Promise<string> =>
+  new Promise((resolve, _reject) => {
+    const browser = new zombie({
+      debug: true,
+      waitFor: 15000,
+    });
+
+    browser.visit(url, function () {
+      const html = browser.html();
+      logger.debug({ message: `Got HTML from zombie`, html, url });
+      resolve(html);
+    });
   });
-
-  const page = await browser.newPage();
-  await page.goto(url);
-
-  await page.waitFor("*");
-
-  const html = await page.content();
-  await browser.close();
-
-  return html;
-};
diff --git a/packages/extension/src/background/fetch_token.ts b/packages/extension/src/background/fetch_token.ts
@@ -25,7 +25,6 @@ const run = async () => {
   }
 };
 
-logger.debug("AM her at all");
 run().then(() => {
   logger.debug(`Running token background script`);
 });