feat: add support for markdown

README.md

@@ -35,27 +35,35 @@ $ linkinator LOCATION [ --arguments ]
 
   Flags
 
-    --config
-        Path to the config file to use. Looks for `linkinator.config.json` by default.
-
     --concurrency
-          The number of connections to make simultaneously. Defaults to 100.
-
-    --recurse, -r
-        Recursively follow links on the same root domain.
+        The number of connections to make simultaneously. Defaults to 100.
 
-    --skip, -s
-        List of urls in regexy form to not include in the check.
+    --config
+        Path to the config file to use. Looks for `linkinator.config.json` by default.
+
+    --format, -f
+        Return the data in CSV or JSON format.
 
     --include, -i
         List of urls in regexy form to include.  The opposite of --skip.
 
-    --format, -f
-        Return the data in CSV or JSON format.
+    --inputType
+        Type of input to process.  One of HTTP, LOCAL_HTML, or LOCAL_MARKDOWN. 
+        Defaults to HTTP or LOCAL_HTML depending on the provided LOCATION.
+
+    --markdownFiles
+        Only valid when using \`--inputType LOCAL_MARKDOWN\`. Array of globs 
+        to compile when processing markdown.  Defaults to \`**/*.md\`.
+
+    --recurse, -r
+        Recursively follow links on the same root domain.
 
     --silent
         Only output broken links.
 
+    --skip, -s
+        List of urls in regexy form to not include in the check.
+
     --timeout
         Request timeout in ms.  Defaults to 0 (no timeout).
 

src/cli.ts

@@ -5,7 +5,7 @@ import * as updateNotifier from 'update-notifier';
 import chalk = require('chalk');
 import {LinkChecker, LinkState, LinkResult, CheckOptions} from './index';
 import {promisify} from 'util';
-import {Flags, getConfig} from './config';
+import {Flags, getConfig, InputType} from './config';
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const toCSV = promisify(require('jsonexport'));
@@ -31,6 +31,14 @@ const cli = meow(
       --concurrency
           The number of connections to make simultaneously. Defaults to 100.
 
+      --inputType
+          Type of input to process.  One of HTTP, LOCAL_HTML, or LOCAL_MARKDOWN. 
+          Defaults to HTTP or LOCAL_HTML depending on the provided LOCATION.
+
+      --markdownFiles
+          Only valid when using \`--inputType LOCAL_MARKDOWN\`. Array of globs 
+          to compile when processing markdown.  Defaults to \`**/*.md\`.
+
       --recurse, -r
           Recursively follow links on the same root domain.
 
@@ -59,6 +67,8 @@ const cli = meow(
   {
     flags: {
       config: {type: 'string'},
+      inputType: {type: 'string'},
+      markdownFiles: {type: 'string'},
       concurrency: {type: 'number'},
       recurse: {type: 'boolean', alias: 'r'},
       skip: {type: 'string', alias: 's'},
@@ -111,11 +121,19 @@ async function main() {
     }
     log(`${state} ${chalk.gray(link.url)}`);
   });
+
+  // Inform the user of auto-detected input type
+  checker.on('inputTypeDetected', (type: InputType) => {
+    log(`Input type autodetected: ${type}`);
+  });
+
   const opts: CheckOptions = {
     path: cli.input[0],
     recurse: flags.recurse,
     timeout: Number(flags.timeout),
     concurrency: Number(flags.concurrency),
+    inputType: flags.inputType as InputType,
+    markdownFiles: flags.markdownFiles,
   };
   if (flags.skip) {
     if (typeof flags.skip === 'string') {

src/config.ts

@@ -3,6 +3,8 @@ import {promisify} from 'util';
 
 const readFile = promisify(fs.readFile);
 
+export type InputType = 'LOCAL_HTML' | 'HTTP' | 'LOCAL_MARKDOWN';
+
 export interface Flags {
   concurrency?: number;
   config?: string;
@@ -11,6 +13,8 @@ export interface Flags {
   format?: string;
   silent?: boolean;
   timeout?: number;
+  inputType?: string;
+  markdownFiles?: string;
 }
 
 export async function getConfig(flags: Flags) {

src/index.ts

@@ -7,9 +7,15 @@ import PQueue, {DefaultAddOptions} from 'p-queue';
 import {getLinks} from './links';
 import {URL} from 'url';
 import PriorityQueue from 'p-queue/dist/priority-queue';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as util from 'util';
 
 import finalhandler = require('finalhandler');
 import serveStatic = require('serve-static');
+import {InputType} from './config';
+
+const readdir = util.promisify(fs.readdir);
 
 export interface CheckOptions {
   concurrency?: number;
@@ -18,6 +24,8 @@ export interface CheckOptions {
   recurse?: boolean;
   timeout?: number;
   linksToSkip?: string[] | ((link: string) => Promise<boolean>);
+  inputType?: InputType;
+  markdownFiles?: string;
 }
 
 export enum LinkState {
@@ -60,7 +68,8 @@ export class LinkChecker extends EventEmitter {
   async check(options: CheckOptions) {
     options.linksToSkip = options.linksToSkip || [];
     let server: http.Server | undefined;
-    if (!options.path.startsWith('http')) {
+    options.inputType = await this.detectInputType(options);
+    if (options.inputType === 'LOCAL_HTML') {
       const port = options.port || 5000 + Math.round(Math.random() * 1000);
       server = await this.startWebServer(options.path, port);
       enableDestroy(server);
@@ -97,6 +106,45 @@ export class LinkChecker extends EventEmitter {
     return result;
   }
 
+  /**
+   * Given user flags, determine if preprocessing is needed before the scan.
+   * Options include:
+   * - HTTP - user provided an http:// path
+   * - HTML_LOCAL - user provided a local filesystem path, and HTML is in the root
+   * - MARKDOWN_LOCAL - user provided a local filesystem path, and MD is in the root
+   * If none of these conditions are met, throw.
+   * @param options CheckOptions derived from user flags]
+   */
+  private async detectInputType(options: CheckOptions): Promise<InputType> {
+    if (options.inputType) {
+      return options.inputType;
+    }
+    if (options.path.startsWith('http')) {
+      this.emit('inputTypeDetected', 'HTTP');
+      return 'HTTP';
+    }
+    const files = await readdir(options.path);
+    const hasHtml = !!files.find(f => {
+      const ext = path.extname(f).toLowerCase();
+      return ext === '.html' || ext === '.htm';
+    });
+    if (hasHtml) {
+      this.emit('inputTypeDetected', 'LOCAL_HTML');
+      return 'LOCAL_HTML';
+    }
+    const hasMarkdown = !!files.find(f => {
+      const ext = path.extname(f).toLowerCase();
+      return ext === '.md';
+    });
+    if (hasMarkdown) {
+      this.emit('inputTypeDetected', 'LOCAL_MARKDOWN');
+      return 'LOCAL_MARKDOWN';
+    }
+    throw new Error(
+      'Unable to find HTML or markdown in the provided path. Consider specifying an inputType or double check your path.'
+    );
+  }
+
   /**
    * Spin up a local HTTP server to serve static requests from disk
    * @param root The local path that should be mounted as a static web server

tsconfig.json

@@ -7,7 +7,6 @@
   },
   "include": [
     "src/*.ts",
-    "test/*.ts",
-    "system-test/*.ts"
+    "test/*.ts"
   ]
 }