.github/workflows/nodejs.yml

@@ -19,4 +19,4 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - run: npm ci
     - run: npm run build --if-present
-    - run: npm test
+    - run: npm run test:full

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 6.1.7
+
+- Improve documentation and error messaging on ending a stream too early #317
+- bump dependencies
+
 ## 6.1.6
 
 - support allow_embed #314

README.md

@@ -84,10 +84,11 @@ app.get('/sitemap.xml', function(req, res) {
     smStream.write({ url: '/page-2/',  changefreq: 'monthly',  priority: 0.7 })
     smStream.write({ url: '/page-3/'})    // changefreq: 'weekly',  priority: 0.5
     smStream.write({ url: '/page-4/',   img: "http://urlTest.com" })
-    smStream.end()
 
     // cache the response
     streamToPromise(pipeline).then(sm => sitemap = sm)
+    // make sure to attach a write stream such as streamToPromise before ending
+    smStream.end()
     // stream write the response
     pipeline.pipe(res).on('error', (e) => {throw e})
   } catch (e) {
@@ -121,7 +122,7 @@ const sms = new SitemapAndIndexStream({
   // it needs to create a new sitemap file. You merely need to return a stream
   // for it to write the sitemap urls to and the expected url where that sitemap will be hosted
   getSitemapStream: (i) => {
-    const sitemapStream = new SitemapStream();
+    const sitemapStream = new SitemapStream({ hostname: 'https://example.com' });
     const path = `./sitemap-${i}.xml`;
 
     sitemapStream
@@ -147,6 +148,7 @@ sms
 
 const arrayOfSitemapItems = [{ url: '/page-1/', changefreq: 'daily'}, ...]
 arrayOfSitemapItems.forEach(item => sms.write(item))
+sms.end()
 ```
 
 ### Options you can pass

examples/sitemapAndIndex.js

@@ -0,0 +1,45 @@
+const { /* createReadStream, */ createWriteStream } = require('fs');
+const { resolve } = require('path');
+const { createGzip } = require('zlib');
+const {
+  SitemapAndIndexStream,
+  SitemapStream,
+  // lineSeparatedURLsToSitemapOptions,
+} = require('../dist/index');
+
+const sms = new SitemapAndIndexStream({
+  limit: 10000, // defaults to 45k
+  // SitemapAndIndexStream will call this user provided function every time
+  // it needs to create a new sitemap file. You merely need to return a stream
+  // for it to write the sitemap urls to and the expected url where that sitemap will be hosted
+  getSitemapStream: (i) => {
+    const sitemapStream = new SitemapStream({
+      hostname: 'https://example.com',
+    });
+    const path = `./sitemap-${i}.xml`;
+
+    sitemapStream
+      .pipe(createGzip()) // compress the output of the sitemap
+      .pipe(createWriteStream(resolve(path + '.gz'))); // write it to sitemap-NUMBER.xml
+
+    return [
+      new URL(path, 'https://example.com/subdir/').toString(),
+      sitemapStream,
+    ];
+  },
+});
+
+// // when reading from a file
+// lineSeparatedURLsToSitemapOptions(createReadStream('./your-data.json.txt'))
+//   .pipe(sms)
+//   .pipe(createGzip())
+//   .pipe(createWriteStream(resolve('./sitemap-index.xml.gz')));
+
+// or reading straight from an in-memory array
+sms
+  .pipe(createGzip())
+  .pipe(createWriteStream(resolve('./sitemap-index.xml.gz')));
+
+const arrayOfSitemapItems = [{ url: '/page-1/', changefreq: 'daily' }];
+arrayOfSitemapItems.forEach((item) => sms.write(item));
+sms.end();

lib/errors.ts

@@ -252,3 +252,21 @@ export class InvalidVideoPriceCurrency extends Error {
     Error.captureStackTrace(this, InvalidVideoPriceCurrency);
   }
 }
+
+export class EmptyStream extends Error {
+  constructor() {
+    super(
+      'You have ended the stream before anything was written. streamToPromise MUST be called before ending the stream.'
+    );
+    this.name = 'EmptyStream';
+    Error.captureStackTrace(this, EmptyStream);
+  }
+}
+
+export class EmptySitemap extends Error {
+  constructor() {
+    super('You ended the stream without writing anything.');
+    this.name = 'EmptySitemap';
+    Error.captureStackTrace(this, EmptyStream);
+  }
+}

lib/sitemap-stream.ts

@@ -9,6 +9,7 @@ import {
 import { SitemapItemLoose, ErrorLevel } from './types';
 import { validateSMIOptions, normalizeURL } from './utils';
 import { SitemapItemStream } from './sitemap-item-stream';
+import { EmptyStream, EmptySitemap } from './errors';
 
 const xmlDec = '<?xml version="1.0" encoding="UTF-8"?>';
 export const stylesheetInclude = (url: string): string => {
@@ -125,8 +126,12 @@ export class SitemapStream extends Transform {
   }
 
   _flush(cb: TransformCallback): void {
-    this.push(closetag);
-    cb();
+    if (!this.hasHeadOutput) {
+      cb(new EmptySitemap());
+    } else {
+      this.push(closetag);
+      cb();
+    }
   }
 }
 
@@ -136,21 +141,23 @@ export class SitemapStream extends Transform {
  */
 export function streamToPromise(stream: Readable): Promise<Buffer> {
   return new Promise((resolve, reject): void => {
-    let drain: Buffer[];
+    const drain: Buffer[] = [];
     stream
       .pipe(
         new Writable({
           write(chunk, enc, next): void {
-            if (!drain) {
-              drain = [chunk];
-            } else {
-              drain.push(chunk);
-            }
+            drain.push(chunk);
             next();
           },
         })
       )
       .on('error', reject)
-      .on('finish', () => resolve(Buffer.concat(drain)));
+      .on('finish', () => {
+        if (!drain.length) {
+          reject(new EmptyStream());
+        } else {
+          resolve(Buffer.concat(drain));
+        }
+      });
   });
 }