add log

rss-to-tana/index.js

@@ -4,6 +4,7 @@ const Store = require('./store');
 const Item = require('./item');
 const Tana = require('./tana');
 const RSS = require('./rss');
+const Log = require('./log');
 
 const schedules = {
   twiceAtNight: '0 0 23,4 * * *', // 23:00 and 04:00 every day
@@ -99,13 +100,13 @@ function dateDiffInDays(a, b) {
 }
 
 async function extractItems(feed) {
-  console.log(feed.url, '- parsing')
+  Log.debug(feed.url, '- parsing')
   try {
     const items = await RSS.parse(feed.url);
 
     return items.map(rssItem => Item.create(rssItem, feed))
   } catch (error) {
-    console.error(feed.url, `parsing error`, error);
+    Log.error(feed.url, `parsing error`, error);
 
     return []
   }
@@ -133,17 +134,35 @@ async function filterSavedItems(items) {
 
 async function parseFeed(feed) {
   const items = await extractItems(feed)
-  console.log(feed.url, `- ${items.length} items in feed`)
+  Log.debug(feed.url, `- ${items.length} items in feed`)
 
   const notOldItems = await filterOlderItems(items)
-  console.log(feed.url, `- ${notOldItems.length} items young enough`)
+  Log.debug(feed.url, `- ${notOldItems.length} items young enough`)
 
   const notAlreadySaved = await filterSavedItems(notOldItems)
-  console.log(feed.url, `- ${notAlreadySaved.length} new items`)
+  Log.info(feed.url, `- ${notAlreadySaved.length} new items`)
 
   Tana.saveItems(notAlreadySaved);
 }
 
+async function parseFeeds() {
+  for (const feed of rssFeeds) {
+      await parseFeed(feed)
+  }
+}
+
+async function scheduleFeeds() {
+  for (const feed of rssFeeds) {
+    Log.info('Scheduling', feed.url, 'on', feed.cron)
+
+    if (!cron.validate(feed.cron)) {
+      throw new Error(`${feed.cron} not a valid cron expression`)
+    }
+
+    cron.schedule(feed.cron, () => parseFeed(feed))
+  }
+}
+
 // 1. Parses feed to retrive their items
 // 2. Filters out those that
 //    - either are older than 3 days
@@ -154,21 +173,7 @@ async function parseFeed(feed) {
 (async () => {
   await Store.initialize()
 
-  for (const feed of rssFeeds) {
-    /**
-     * We can use FORCE=true env var to run the feeds parsing directly, without
-     * cron schedule
-     */
-    if (process.env.FORCE === 'true') {
-      await parseFeed(feed)
-    } else {
-      console.log('Scheduling', feed.url, 'on', feed.cron)
-
-      if (!cron.validate(feed.cron)) {
-        throw new Error(`${feed.cron} not a valid cron expression`)
-      }
-
-      cron.schedule(feed.cron, () => parseFeed(feed))
-    }
-  }
+  // we parse all feeds at app startup
+  await parseFeeds()
+  await scheduleFeeds()
 })();

rss-to-tana/log.js

@@ -0,0 +1,19 @@
+function info(...message) {
+  console.info(...message)
+}
+
+function debug(...message) {
+  if (process.env.DEBUG === 'true') {
+    console.debug(...message)
+  }
+}
+
+function error(...error) {
+  console.error(...error)
+}
+
+module.exports = {
+  info,
+  debug,
+  error,
+}

rss-to-tana/rss.js

@@ -23,7 +23,6 @@ async function parse(feedUrl) {
       publishedAt: parsePubDate(item.pubDate || item.date),
     }))
   } catch (err) {
-    console.log(err)
     return []
   }
 }

rss-to-tana/store.js

@@ -1,20 +1,22 @@
 const { createClient } = require('redis');
+const Log = require('./log');
 
 const REDIS_URL = `redis://${process.env.REDIS_HOST}`
 
 const client = createClient({
   url: REDIS_URL,
 });
 
-client.on('error', error => console.error('Redis Client Error', error));
+client.on('error', error => Log.error('Redis Client Error', error.message));
+client.on('reconnecting', () => Log.debug('Redis Client reconnecting...'));
+client.on('ready', () => Log.info('Redis Client Ready!'));
 
 // change the prefix to trash all existing ids
 const storeId = (id) => `1-${id}`
 
 const initialize = async () => {
-  console.log('Connecting to Redis', REDIS_URL)
+  Log.info('Connecting to Redis', REDIS_URL)
   await client.connect();
-  console.log('Connected!')
 }
 
 const savedAlready = async (itemId) => {

rss-to-tana/tana.js

@@ -1,3 +1,4 @@
+const Log = require('./log');
 const API_KEY = process.env.TANA_API_KEY
 
 const Store = require('./store');
@@ -45,19 +46,19 @@ const queue = []
 setInterval(
   () => {
     if (queue.length) {
-      console.log(`Posting ${queue.length} items to Tana`)
+      Log.debug(`Posting ${queue.length} items to Tana`)
 
       // extracting all items from the queue
       const items = queue.splice(0, Infinity)
 
       postItems(items)
         .then(() => Store.saveItemsSaved(items.map(item => item.id)))
         .then(() => {
-          console.log(`${items.length} items saved`);
+          Log.debug(`${items.length} items saved`);
         })
         // in case of failure, we put back items in the queue
         .catch(error => {
-          console.error(error);
+          Log.error(error);
           queue.push(...items)
         });
     }
@@ -66,8 +67,10 @@ setInterval(
 )
 
 function saveItems(items) {
-  queue.push(...items)
-  console.log(`Added ${items.length} items to queue (${queue.length} items)`)
+  if (items.length) {
+    queue.push(...items)
+    Log.info(`Added ${items.length} items to queue (${queue.length} items)`)
+  }
 }
 
 module.exports = { saveItems };