custom rss parser

rss-to-tana/index.js

@@ -1,16 +1,9 @@
-const RSSParser = require('rss-parser');
 const cron = require('node-cron');
 
 const Store = require('./store');
 const Item = require('./item');
 const Tana = require('./tana');
-
-const parser = new RSSParser({
-  defaultRSS: 2.0,
-  xml2js: {
-    strict: true,
-  }
-});
+const RSS = require('./rss');
 
 const schedules = {
   twiceAtNight: '0 0 23,4 * * *', // 23:00 and 04:00 every day
@@ -108,9 +101,9 @@ function dateDiffInDays(a, b) {
 async function extractItems(feed) {
   console.log(feed.url, '- parsing')
   try {
-    const parsedFeed = await parser.parseURL(feed.url);
+    const items = await RSS.parse(feed.url);
 
-    return parsedFeed.items.map(rssItem => Item.create(rssItem, feed))
+    return items.map(rssItem => Item.create(rssItem, feed))
   } catch (error) {
     console.error(feed.url, `parsing error`, error);
 

rss-to-tana/item.js

@@ -91,7 +91,7 @@ function website(feedUrl, item) {
 const create = (rssItem, feed) => ({
   id: rssItem.link,
   title: rssItem.title,
-  publishedAt: new Date(rssItem.isoDate),
+  publishedAt: rssItem.publishedAt,
   tanaNode: feed.toTana(feed.url, rssItem),
   feed,
 })

rss-to-tana/package.json

@@ -2,9 +2,9 @@
   "name": "rss-to-tana",
   "main": "index.js",
   "dependencies": {
+    "htmlparser2": "^9.0.0",
     "node-cron": "^3.0.2",
-    "redis": "^4.6.7",
-    "rss-parser": "^3.9.0"
+    "redis": "^4.6.7"
   },
   "scripts": {
     "start": "node index.js"

rss-to-tana/rss.js

@@ -0,0 +1,34 @@
+const htmlparser2 = require('htmlparser2')
+
+function parsePubDate(pubDateString) {
+  return new Date(pubDateString)
+}
+
+async function parse(feedUrl) {
+  const response = await fetch(feedUrl)
+
+  if (!response.ok) {
+    throw new Error(`Error fetching ${feedUrl}: ${response.status} ${response.statusText}`)
+  }
+
+  const content = await response.text()
+
+  const feed = htmlparser2.parseFeed(content);
+
+  const items = feed.items || []
+
+  try {
+    return items.map((item) => ({
+      title: item.title,
+      link: item.link,
+      publishedAt: parsePubDate(item.pubDate || item.date),
+    }))
+  } catch (err) {
+    console.log(err)
+    return []
+  }
+}
+
+module.exports = {
+  parse,
+}