Add new --proxy-cache option

README.md

@@ -59,6 +59,7 @@ This will install `http-server` globally so that it may be run from the command
 |`--log-ip` |Enable logging of the client's IP address |`false` |
 |`-P` or `--proxy` |Proxies all requests which can't be resolved locally to the given url. e.g.: -P http://someurl.com | |
 |`--proxy-options` |Pass proxy [options](https://github.com/http-party/node-http-proxy#options) using nested dotted objects. e.g.: --proxy-options.secure false |
+|`--proxy-cache` |Enable disk caching of proxy responses to specified folder | |
 |`--username` |Username for basic authentication | |
 |`--password` |Password for basic authentication | |
 |`-S`, `--tls` or `--ssl` |Enable secure request serving with TLS/SSL (HTTPS)|`false`|

bin/http-server

@@ -10,6 +10,7 @@ var chalk     = require('chalk'),
 
     fs         = require('fs'),
     url        = require('url');
+const {getPort} = require("../lib/core/get-port.js");
 var argv = require('minimist')(process.argv.slice(2), {
   alias: {
     tls: 'ssl'
@@ -46,6 +47,7 @@ if (argv.h || argv.help) {
     '',
     '  -P --proxy       Fallback proxy if the request cannot be resolved. e.g.: http://someurl.com',
     '  --proxy-options  Pass options to proxy using nested dotted objects. e.g.: --proxy-options.secure false',
+    '  --proxy-cache    Enable disk caching of proxy responses to specified folder',
     '',
     '  --username   Username for basic authentication [none]',
     '               Can also be specified with the env variable NODE_HTTP_SERVER_USERNAME',
@@ -71,6 +73,7 @@ var port = argv.p || argv.port || parseInt(process.env.PORT, 10),
     sslPassphrase = process.env.NODE_HTTP_SERVER_SSL_PASSPHRASE,
     proxy = argv.P || argv.proxy,
     proxyOptions = argv['proxy-options'],
+    proxyCache = argv['proxy-cache'],
     utc = argv.U || argv.utc,
     version = argv.v || argv.version,
     logger;
@@ -126,11 +129,7 @@ if (version) {
 }
 
 if (!port) {
-  portfinder.basePort = 8080;
-  portfinder.getPort(function (err, port) {
-    if (err) { throw err; }
-    listen(port);
-  });
+  getPort(8080).then(listen)
 }
 else {
   listen(port);
@@ -150,6 +149,7 @@ function listen(port) {
     logFn: logger.request,
     proxy: proxy,
     proxyOptions: proxyOptions,
+    proxyCache: proxyCache,
     showDotfiles: argv.dotfiles,
     mimetypes: argv.mimetypes,
     username: argv.username || process.env.NODE_HTTP_SERVER_USERNAME,
@@ -240,6 +240,7 @@ function listen(port) {
       else {
         logger.info('Unhandled requests will be served from: ' + proxy);
       }
+      logger.info('And will cached to: ' + proxyCache);
     }
 
     logger.info('Hit CTRL-C to stop the server');

doc/http-server.1

@@ -89,6 +89,10 @@ Fallback proxy if the request cannot be resolved.
 .BI \-\-proxy\-options
 Pass proxy options using nested dotted objects.
 
+.TP
+.BI \-\-proxy\-cache
+Enable disk caching of proxy responses to specified folder.
+
 .TP
 .BI \-\-username " " \fIUSERNAME\fR
 Username for basic authentication.

lib/core/get-port.js

@@ -0,0 +1,14 @@
+const portfinder = require('portfinder');
+
+exports.getPort = function (basePort) {
+  return new Promise((resolve, reject) => {
+    portfinder.basePort = basePort || 8080;
+    portfinder.getPort(function (err, port) {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(port);
+      }
+    });
+  });
+}

lib/http-server.js

@@ -1,6 +1,8 @@
 'use strict';
 
 var fs = require('fs'),
+  path = require('path'),
+  zlib = require('zlib'),
   union = require('union'),
   httpServerCore = require('./core'),
   auth = require('basic-auth'),
@@ -142,6 +144,50 @@ function HttpServer(options) {
   if (typeof options.proxy === 'string') {
     var proxyOptions = options.proxyOptions || {};
     var proxy = httpProxy.createProxyServer(proxyOptions);
+
+    if (typeof options.proxyCache === 'string') {
+      var proxyCache = options.proxyCache;
+      var proxyCacheLogFn = options.proxyCacheLogFn || function () {};
+      proxy.on('proxyRes', async function (proxyRes, req, res) {
+        var localFile = path.isAbsolute(proxyCache) ? proxyCache : path.join(process.cwd(), proxyCache, req.url.split('?')[0]);
+        var localDir = path.dirname(localFile);
+        if (proxyRes.statusCode !== 200) {
+          proxyCacheLogFn({
+            status: proxyRes.statusCode,
+            message: proxyRes.statusMessage
+          }, proxyRes, req, res, localFile);
+          return;
+        }
+        var contentEncoding = proxyRes.headers['content-encoding'];
+        if (!fs.existsSync(localDir)) {
+          fs.mkdirSync(localDir, { recursive: true });
+        }
+
+        await new Promise((resolve, reject) => {
+          var stream = fs.createWriteStream(localFile);
+
+          if (contentEncoding === 'gzip' || contentEncoding === 'deflate') {
+            proxyRes.pipe(zlib.createGunzip()).pipe(stream);
+          } else if (contentEncoding === 'br') {
+            proxyRes.pipe(zlib.createBrotliDecompress()).pipe(stream);
+          } else {
+            proxyRes.pipe(stream);
+          }
+
+          stream.on('finish', () => {
+            proxyCacheLogFn(null, proxyRes, req, res, localFile);
+            resolve();
+          });
+          stream.on('error', (err) => {
+            proxyCacheLogFn({
+              status: proxyRes.statusCode,
+              message: err.message
+            }, proxyRes, req, res, localFile);
+            reject(err);
+          });
+        });
+      });
+    }
     before.push(function (req, res) {
       proxy.web(req, res, {
         target: options.proxy,

test/proxy-cache.test.js

@@ -0,0 +1,210 @@
+const test = require('tap').test
+const promisify = require('util').promisify
+const httpServer = require('../lib/http-server')
+const request = require('request')
+const fs = require("fs");
+const path = require("path");
+const {getPort} = require("../lib/core/get-port.js");
+const requestAsync = promisify(request)
+const fsReadFile = promisify(fs.readFile)
+
+// Prevent errors from being swallowed
+process.on('uncaughtException', console.error)
+
+test('cache response when configured', (t) => {
+  t.plan(4);
+  new Promise((resolve) => {
+    const remoteServerRoot = path.join(__dirname, 'fixtures', 'root');
+    const serverRoot = path.join(__dirname, 'cache');
+    const cachedFilePath = path.join(serverRoot, 'file');
+    const remoteFilePath = path.join(remoteServerRoot, 'file');
+
+    const remoteServer = httpServer.createServer({
+      root: remoteServerRoot,
+    });
+
+    getPort().then(remotePort => {
+      remoteServer.listen(remotePort, async () => {
+        try {
+          const serverWithCache = httpServer.createServer({
+            root: serverRoot,
+            proxy: `http://localhost:${remotePort}`,
+            proxyCache: './',
+          });
+
+          const serverPort = await getPort();
+
+          await new Promise((resolve) => {
+            serverWithCache.listen(serverPort, async () => {
+              try {
+                await requestAsync(`http://localhost:${serverPort}/file`).then(async (res) => {
+                  t.ok(res)
+                  t.equal(res.statusCode, 200);
+
+                  const cachedFile = await fsReadFile(cachedFilePath, 'utf8');
+                  const remoteFile = await fsReadFile(remoteFilePath, 'utf8');
+
+                  t.equal(res.body.trim(), cachedFile.trim(), 'cached file content matches');
+                  t.equal(cachedFile.trim(), remoteFile.trim(), 'cached file content matches remote file content');
+                })
+              } catch (err) {
+                t.fail(err.toString())
+              } finally {
+                fs.rmSync(cachedFilePath);
+                serverWithCache.close();
+                resolve();
+              }
+            });
+          });
+        } catch (err) {
+          t.fail(err.toString())
+        } finally {
+          remoteServer.close();
+          resolve();
+        }
+      });
+    })
+
+  })
+    .then(() => t.end())
+    .catch(err => {
+      t.fail(err.toString())
+      t.end()
+    })
+});
+
+test('decompress gzipped response before caching', (t) => {
+  new Promise(resolve => {
+    const remoteServerRoot = path.join(__dirname, 'public', 'gzip');
+    const serverRoot = path.join(__dirname, 'cache');
+
+    const remoteFilePath = path.join(remoteServerRoot, 'real_ecstatic');
+    const cachedFilePath = path.join(serverRoot, 'real_ecstatic');
+
+
+    const remoteServer = httpServer.createServer({
+      root: remoteServerRoot,
+      gzip: true,
+    });
+
+    getPort().then(remotePort => {
+      remoteServer.listen(remotePort, async () => {
+        try {
+          const serverWithCache = httpServer.createServer({
+            root: serverRoot,
+            proxy: `http://localhost:${remotePort}`,
+            proxyCache: './',
+          });
+
+          const serverPort = await getPort();
+          await new Promise((resolve) => {
+            serverWithCache.listen(serverPort, async () => {
+              try {
+                await requestAsync({
+                  uri: `http://localhost:${serverPort}/real_ecstatic`,
+                  headers: {
+                    'accept-encoding': 'gzip, deflate, br'
+                  }
+                }).then(async (res) => {
+                  t.ok(res)
+                  t.equal(res.statusCode, 200, 'response is 200');
+                  t.equal(res.headers['content-encoding'], 'gzip', 'response is gzipped');
+
+                  const cachedFile = await fsReadFile(cachedFilePath, 'utf8');
+                  const remoteFile = await fsReadFile(remoteFilePath, 'utf8');
+
+                  t.equal(cachedFile.trim(), remoteFile.trim(), 'cached file content matches remote file content');
+                })
+              } catch (err) {
+                t.fail(err.toString())
+              } finally {
+                fs.rmSync(cachedFilePath);
+                serverWithCache.close();
+                resolve();
+              }
+            });
+          });
+        } catch (err) {
+          t.fail(err.toString())
+        } finally {
+          remoteServer.close();
+          resolve();
+        }
+
+      });
+    })
+  })
+    .then(() => t.end())
+    .catch(err => {
+      t.fail(err.toString())
+      t.end()
+    })
+});
+
+test('decompress brotli response before caching', (t) => {
+  new Promise(resolve => {
+    const remoteServerRoot = path.join(__dirname, 'public', 'brotli');
+    const serverRoot = path.join(__dirname, 'cache');
+
+    const remoteFilePath = path.join(remoteServerRoot, 'real_ecstatic');
+    const cachedFilePath = path.join(serverRoot, 'real_ecstatic');
+
+
+    const remoteServer = httpServer.createServer({
+      root: remoteServerRoot,
+      brotli: true,
+    });
+
+    getPort().then(remotePort => {
+      remoteServer.listen(remotePort, async () => {
+        try {
+          const serverWithCache = httpServer.createServer({
+            root: serverRoot,
+            proxy: `http://localhost:${remotePort}`,
+            proxyCache: './',
+          });
+
+          const serverPort = await getPort();
+          await new Promise((resolve) => {
+            serverWithCache.listen(serverPort, async () => {
+              try {
+                await requestAsync({
+                  uri: `http://localhost:${serverPort}/real_ecstatic`,
+                  headers: {
+                    'accept-encoding': 'gzip, deflate, br'
+                  }
+                }).then(async (res) => {
+                  t.ok(res)
+                  t.equal(res.statusCode, 200, 'response is 200');
+                  t.equal(res.headers['content-encoding'], 'br', 'response is brotli');
+
+                  const cachedFile = await fsReadFile(cachedFilePath, 'utf8');
+                  const remoteFile = await fsReadFile(remoteFilePath, 'utf8');
+
+                  t.equal(cachedFile.trim(), remoteFile.trim(), 'cached file content matches remote file content');
+                })
+              } catch (err) {
+                t.fail(err.toString())
+              } finally {
+                fs.rmSync(cachedFilePath);
+                serverWithCache.close();
+                resolve();
+              }
+            });
+          });
+        } catch (err) {
+          t.fail(err.toString())
+        } finally {
+          remoteServer.close();
+          resolve();
+        }
+
+      });
+    })
+  })
+    .then(() => t.end())
+    .catch(err => {
+      t.fail(err.toString())
+      t.end()
+    })
+});