Keep buffer for long text during transformation

packages/parse5-html-rewriting-stream/lib/index.js

@@ -5,7 +5,7 @@ const { escapeString } = require('parse5/lib/serializer');
 
 class RewritingStream extends SAXParser {
     constructor() {
-        super({ sourceCodeLocationInfo: true });
+        super({ sourceCodeLocationInfo: true, bufferSafekeeping: true });
 
         this.posTracker = this.locInfoMixin.posTracker;
     }
@@ -21,7 +21,9 @@ class RewritingStream extends SAXParser {
         const start = location.startOffset - droppedBufferSize;
         const end = location.endOffset - droppedBufferSize;
 
-        return this.tokenizer.preprocessor.html.slice(start, end);
+        const slice = this.tokenizer.preprocessor.html.slice(start, end);
+        this.bufferSafekeepingMixin.safekeepingOffset = location.endOffset;
+        return slice;
     }
 
     // Events

packages/parse5-html-rewriting-stream/test/rewriting-stream.test.js

@@ -270,3 +270,21 @@ exports['Regression - RewritingStream - Should not accept binary input (GH-269)'
 
     assert.throws(() => stream.write(buf), TypeError);
 };
+
+exports['Regression - RewritingStream - should pass long text correctly (GH-292)'] = done => {
+    const source = 'a'.repeat(65540);
+    const parser = new RewritingStream();
+    let output = '';
+
+    parser.on('data', data => {
+        output += data.toString();
+    });
+
+    parser.once('finish', () => {
+        assert.strictEqual(output.length, source.length);
+        done();
+    });
+
+    parser.write(source);
+    parser.end();
+};

packages/parse5-sax-parser/lib/index.js

@@ -7,6 +7,7 @@ const Mixin = require('parse5/lib/utils/mixin');
 const mergeOptions = require('parse5/lib/utils/merge-options');
 const DevNullStream = require('./dev-null-stream');
 const ParserFeedbackSimulator = require('./parser-feedback-simulator');
+const BufferSafekeepingPreprocessorMixin = require('../../parse5/lib/extensions/buffer-safekeeping/preprocessor-mixin');
 
 const DEFAULT_OPTIONS = {
     sourceCodeLocationInfo: false
@@ -20,9 +21,16 @@ class SAXParser extends Transform {
 
         this.tokenizer = new Tokenizer(options);
         this.locInfoMixin = null;
+        this.bufferSafekeepingMixin = null;
 
         if (this.options.sourceCodeLocationInfo) {
             this.locInfoMixin = Mixin.install(this.tokenizer, LocationInfoTokenizerMixin);
+            if (this.options.bufferSafekeeping) {
+                this.bufferSafekeepingMixin = Mixin.install(
+                    this.tokenizer.preprocessor,
+                    BufferSafekeepingPreprocessorMixin
+                );
+            }
         }
 
         this.parserFeedbackSimulator = new ParserFeedbackSimulator(this.tokenizer);

packages/parse5/lib/extensions/buffer-safekeeping/preprocessor-mixin.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const Mixin = require('../../utils/mixin');
+
+class BufferSafekeepingPreprocessorMixin extends Mixin {
+    constructor(preprocessor) {
+        super(preprocessor);
+
+        this.safekeepingOffset = 0;
+    }
+
+    _getOverriddenMethods(mxn, orig) {
+        return {
+            getDropPosition() {
+                return Math.min(orig.getDropPosition(), mxn.safekeepingOffset - mxn.droppedBufferSize);
+            }
+        };
+    }
+}
+
+module.exports = BufferSafekeepingPreprocessorMixin;

packages/parse5/lib/tokenizer/preprocessor.js

@@ -67,15 +67,32 @@ class Preprocessor {
     }
 
     dropParsedChunk() {
-        if (this.pos > this.bufferWaterline) {
-            this.lastCharPos -= this.pos;
-            this.html = this.html.substring(this.pos);
-            this.pos = 0;
-            this.lastGapPos = -1;
-            this.gapStack = [];
+        const lastPos = this.getDropPosition();
+        if (lastPos > this.bufferWaterline) {
+            this.lastCharPos -= lastPos;
+            this.html = this.html.substring(lastPos);
+            this.pos -= lastPos;
+            if (this.pos === 0) {
+                this.lastGapPos = -1;
+                this.gapStack = [];
+            } else {
+                this.gapStack.push(this.lastGapPos);
+                const newGapStack = [-1];
+                for (let i = 1; i < this.gapStack.length; i++) {
+                    if (this.gapStack[i] >= lastPos) {
+                        newGapStack.push(this.gapStack[i] >= lastPos);
+                    }
+                }
+                this.gapStack = newGapStack;
+                this.lastGapPos = this.gapStack.pop();
+            }
         }
     }
 
+    getDropPosition() {
+        return this.pos;
+    }
+
     write(chunk, isLastChunk) {
         if (this.html) {
             this.html += chunk;