Merge branch 'main' into glitch

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.eslintrc.json

@@ -0,0 +1,50 @@
+{
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": [
+        "eslint:recommended",
+        "plugin:@typescript-eslint/recommended"
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+        "ecmaVersion": 6,
+        "sourceType": "module"
+    },
+    "plugins": [
+        "@typescript-eslint"
+    ],
+    "rules": {
+        "indent": [
+            "error",
+            2
+        ],
+        "linebreak-style": [
+            "error",
+            "unix"
+        ],
+        "quotes": [
+            "error",
+            "double"
+        ],
+        "semi": [
+            "error",
+            "always"
+        ],
+        "no-trailing-spaces": [
+            2,
+            {
+                "skipBlankLines": false
+            }
+        ],
+        "no-multiple-empty-lines": [
+            "error",
+            {
+                "max": 2,
+                "maxEOF": 1
+            }
+        ],
+        "@typescript-eslint/no-var-requires": 0
+    }
+}

.github/FUNDING.yml

@@ -1,7 +1,7 @@
 # These are supported funding model platforms
 
 github: georgemandis
-patreon: georgemandis
+patreon: #
 open_collective: # Replace with a single Open Collective username
 ko_fi: # Replace with a single Ko-fi username
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
@@ -9,4 +9,4 @@ community_bridge: # Replace with a single Community Bridge project-name e.g., cl
 liberapay: # Replace with a single Liberapay username
 issuehunt: # Replace with a single IssueHunt username
 otechie: # Replace with a single Otechie username
-custom: https://george.mand.is/sponsor
+custom: #

.gitignore

@@ -1,2 +1,4 @@
 node_modules/*
-output/index.html
+public/index.html
+dist/*
+.DS_Store

.nvmrc

@@ -0,0 +1 @@
+18.12.1

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 George Mandis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,35 +1,104 @@
+[![Netlify Status](https://api.netlify.com/api/v1/badges/81dd219c-51cb-4418-a18c-42c8b104c689/deploy-status)](https://app.netlify.com/sites/bubo-rss-demo/deploys)
+
 # 🦉 Bubo Reader
 
-Bubo Reader is a somewhat irrationally minimalist <acronym title="Really Simple Syndication">RSS</acronym> and <acronym title="JavaScript Object Notation">JSON</acronym> feed reader you can deploy on [Netlify](https://netlify.com) in a few steps or [Glitch](https://glitch.com) in even fewer steps! The goal of the project is to generate a webpage that shows a list of links from a collection of feeds organized by category and website. That's it.
+Bubo Reader is a hyper-minimalist <acronym title="Really Simple Syndication">RSS</acronym> and <acronym title="JavaScript Object Notation">JSON</acronym> feed reader you can deploy on your own server, [Netlify](https://netlify.com) in a few steps or [Glitch](https://glitch.com) in even fewer steps! The goal of the project is to generate a webpage that shows a list of links from a collection of feeds organized by category and website. That's it.
 
 It is named after this [silly robot owl](https://www.youtube.com/watch?v=MYSeCfo9-NI) from Clash of the Titans (1981).
 
-You can read more about how this project came about in my blog post '[Introducing Bubo RSS: An Absurdly Minimalist RSS Feed Reader](https://george.mand.is/2019/11/introducing-bubo-rss-an-absurdly-minimalist-rss-feed-reader/)'
+You can read more about this project on my blog:
 
-## Getting Started
+- [Introducing Bubo RSS: An Absurdly Minimalist RSS Feed Reader](https://george.mand.is/2019/11/introducing-bubo-rss-an-absurdly-minimalist-rss-feed-reader/).
+- [Publishing Bubos RSS to Netlify with GitHub Actions](https://george.mand.is/2020/02/publishing-bubos-rss-to-netlify-with-github-actions/)
 
-How to deploy Bubo Reader in a few easy steps with Netlify or Glitch:
+## Get Started
 
-### Deploying to Glitch
+- Clone or fork the repo and run `npm install` to install the dependencies.
+- Update `feeds.json` to include categories and links to feeds you would like to see.
+- Run `npm run build:bubo`
+
+That's it! You should now have a static page with links to the latest content from your feeds in the `public` folder, ready to serve.
+
+<details>
+  <summary>
+    <strong>Anatomy of Bubo Reader</strong>
+  </summary>
+
+The static pieces:
+
+- `conf/feeds.json` - a JSON file containing your feed URLS separated into categories.
+- `config/template.html` - a [Nunjucks](https://mozilla.github.io/nunjucks/) template that lets you change how the feeds are displayed. This can be changed to anything else you like— see below.
+- `public/style.css` - a CSS file to stylize your feed output.
+- `public/index.html` - The HTML file that gets automatically generated when Bubo is run.
+
+The engine:
+
+- `src/index.ts` - The primary script you run when you want to build a new version of Bubo. It will automatically fetch the latest content from your feeds and build a new static file at `public/index.html`.
+- `src/renderer.ts` — The renderer that loads Nunjucks, the template and understands how to process the incoming feed data. Prefer something else? This is the place to change it!
+- `src/utilities.ts` — A variety of parsing and normalization utilities for Bubo, hidden away to try and keep things clean.
+
+</details>
+
+<details>
+  <summary>
+    <strong>Throttling</strong>
+  </summary>
+
+In the main `index.ts` file you will find two values that allow you to batch and throttle your feed requests:
+
+- `MAX_CONNECTIONS` dictates the maximum number of requests a batch can have going at once.
+- `DELAY_MS` dictates the amount of delay time between each batch.
+
+The default configuration is **no batching or throttling** because `MAX_CONNECTIONS` is set to `Infinity`. If you wanted to change Bubo to only fetch one feed at a time every second you could set these values to:
+
+```javascript
+const MAX_CONNECTIONS = 1;
+const DELAY_MS = 1000;
+```
+
+If you wanted to limit things to 10 simultaneous requests every 2.5 seconds you could set it like so:
+
+```javascript
+const MAX_CONNECTIONS = 10;
+const DELAY_MS = 2500;
+```
+
+In practice, I've never _really_ run into an issue leaving `MAX_CONNECTIONS` set to `Infinity` but this feels like a sensible safeguard to design.
+
+</details>
+
+<details>
+  <summary>
+    <strong>Getting Started</strong>
+  </summary>
+
+- [Deploying to Glitch](#glitch)
+- [Deploying to Netlify](#netlify)
+- [Keeping feeds updated](#updated)
+
+<a id="glitch"></a>
+
+## Deploying to Glitch
 
 The quickest way is to remix the project on Glitch:
 [https://glitch.com/edit/#!/bubo-rss](https://glitch.com/edit/#!/bubo-rss)
 
-Just changed some feeds in `./src/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./output/style.css` file or the `./src/template.html` file respectively.
+There is also a `glitch` branch on this repo if you'd prefer to start there.
 
-There is also a special `glitch` branch you can clone if you prefer:
+Just change some feeds in `./config/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./public/style.css` file or the `./config/template.html` file respectively.
-[https://github.com/georgemandis/bubo-rss/tree/glitch](https://github.com/georgemandis/bubo-rss/tree/glitch)
 
-The only difference between this branch and `master` is that it spins up a server using [Express](https://expressjs.com/) to serve your `./output/index.html` file on Glitch. Everything else is the same.
+<a id="netlify"></a>
 
-### Deploying to Netlify
+## Deploying to Netlify
 
 - [Fork the repository](https://github.com/georgemandis/bubo-rss/fork)
-- From your forked repository go to and edcit `src/feeds.json` to manage your feeds and categories
+- From your forked repository edit `config/feeds.json` to manage your feeds and categories
 - [Create a new site](https://app.netlify.com/start) on Netlify from GitHub
 
 The deploy settings should automatically import from the `netlify.toml` file. All you'll need to do is confirm and you're ready to go!
 
+<a id="updated"></a>
+
 ### Keeping Feeds Updated
 
 #### Using Netlify Webhooks
@@ -40,24 +109,11 @@ To keep your feeds up to date you'll want to [setup a Build Hook](https://www.ne
 - [Zapier](https://zapier.com/)
 - [EasyCron](https://www.easycron.com/)
 
+#### Rolling Your Own
+
 If you already have a server running Linux and some command-line experience it might be simpler to setup a [cron job](https://en.wikipedia.org/wiki/Cron).
 
-#### Using GitHub Actions
+</details>
-
-This approach is a little different and requires some modifications to the repository. Netlify started billing for [build minutes](https://www.netlify.com/pricing/faq/) very shortly after I published this project. Running `npm build` and downloading all of the RSS feeds took up a substantial number of this minutes, particulary if you had some kind of process pinging the webhook and trigger a build every 15 minutes or so.
-
-How is the The GitHub Action-based approach different? The same build process runs, but this time it's on GitHub's servers via the Action. It then **commits** the newly created file generated at `./output/index.html` back into the repository. Netlify still gets pinged when the repository is updated, but skips the `npm run build` step on their end, which significantly reduces the number of build minutes required.
-
-**Short Answer**: use the [`github-action-publishing`](https://github.com/georgemandis/bubo-rss/tree/github-action-publishing) branch for now if you'd prefer to use GitHub Actions to run your builds. 
-
-The GitHub Action is setup to build and commit directly to the `master` branch, which is not the best practice. I'd suggest creating a separate branch to checkout and commit changes to in the Action. You could then specify that same branch as the one to checkout and publish on Netlify.
-
-## Anatomy of Bubo Reader
-
-- `src/index.html` - a [Nunjucks](https://mozilla.github.io/nunjucks/) template that lets you change how the feeds are displayed
-- `output/style.css` - a CSS file to stylize your feed output
-- `src/feeds.json` - a JSON file containing the URLs for various site's feeds separated into categories
-- `src/index.js` - the script that loads the feeds and does the actual parsinga and rendering
 
 ## Demos
 
@@ -66,8 +122,16 @@ You can view live demos here:
 - [https://bubo-rss-demo.netlify.com/](https://bubo-rss-demo.netlify.com/)
 - [http://bubo-rss.glitch.me/](http://bubo-rss.glitch.me/)
 
-Not the most exciting-looking demos, I'll admit, but they work!
-
 ## Support
 
-If you found this useful please consider sponsoring me or this project. If you'd rather run this on your own server please consider using one of these affiliate links to setup a micro instance on [Linode](https://www.linode.com/?r=8729957ab02b50a695dcea12a5ca55570979d8b9), [Digital Ocean](https://m.do.co/c/31f58d367777) or [Vultr](https://www.vultr.com/?ref=8403978).
+If you found this useful please consider [sponsoring me or this project](https://github.com/sponsors/georgemandis).
+
+If you'd rather run this on your own server please consider using one of these affiliate links to setup a micro instance on [Linode](https://www.linode.com/?r=8729957ab02b50a695dcea12a5ca55570979d8b9), [Digital Ocean](https://m.do.co/c/31f58d367777) or [Vultr](https://www.vultr.com/?ref=8403978).
+
+## Showcase
+
+Here are some websites using Bubo Reader:
+
+- [Kevin Fiol](https://kevinfiol.com/reader/) ([repo](https://github.com/kevinfiol/reader))
+
+Please share if you would like to be featured!

config/feeds.json

@@ -1,24 +1,26 @@
 {
-  "Web Development": [
+  "Developer News": [
     "https://hacks.mozilla.org/feed/",
-    "https://blog.mozilla.org/feed/",
     "https://web.dev/feed.xml",
     "https://v8.dev/blog.atom",
     "https://alistapart.com/main/feed/",
     "https://css-tricks.com/feed/",
-    "https://dev.to/feed"
+    "https://dev.to/feed",
+    "https://changelog.com/feed"
   ],
   "Blogs": [
     "https://george.mand.is/feed.xml",
     "https://joy.recurse.com/feed.atom"
   ],
+  "Social": [
+    "https://social.mandis.dev/@georgemandis.rss"
+  ],
   "My GitHub Projects": [
     "https://github.com/georgemandis.atom",
-    "https://github.com/snaptortoise/konami-js/releases.atom",
+    "https://github.com/georgemandis/bubo-rss/releases.atom",
-    "https://github.com/snaptortoise/konami-js/commits/master.atom",
+    "https://github.com/georgemandis/konami-js/releases.atom",
+    "https://github.com/georgemandis/konami-js/commits/main.atom",
     "https://github.com/javascriptforartists/cheer-me-up-and-sing-me-a-song/commits/master.atom",
-    "https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom",
+    "https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom"
-    "https://github.com/georgemandis/remote-working-list/commits/master.atom",
-    "https://github.com/georgemandis/tweeter-totter/commits/master.atom"
   ]
 }

config/template.html

@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta http-equiv="X-UA-Compatible" content="ie=edge" />
+    <title>🦉 Bubo Reader</title>
+    <link rel="stylesheet" href="/style.css" />
+  </head>
+  <body>
+    <h1>🦉 Bubo Reader</h1>
+
+    {% for group, feeds in data %}
+    <h2>{{ group }}</h2>
+    {% for feed in feeds %}
+    <details>
+      <summary>
+        <span class="feed-title">{{ feed.title }}</span>
+        <span class="feed-url">({{ feed.feed }})</span>
+      </summary>
+      <ul>
+        {% for item in feed.items %}
+        <li>
+          {{ item.timestamp | formatDate }} -
+          <a
+            href="{{ item.link }}"
+            target="_blank"
+            rel="noopener norefferer nofollow"
+            >{{ item.title }}</a
+          >
+        </li>
+        {% endfor %}
+      </ul>
+    </details>
+    {% endfor %} {% endfor %} {% if errors | length > 0 %}
+    <h2>Errors</h2>
+    <p>There were errors trying to parse these feeds:</p>
+    <ul>
+      {% for error in errors %}
+      <li>{{ error }}</li>
+      {% endfor %}
+    </ul>
+    {% endif %}
+
+    <br />
+    <hr />
+    <p>Last updated {{ now }}.</p>
+    <p>
+      Powered by
+      <a href="https://github.com/georgemandis/bubo-rss"
+        >Bubo Reader (v{{ info.version }})</a
+      >, a project by <a href="https://george.mand.is">George Mandis</a>. ❤️
+      <a href="{{ info.funding.url }}">Sponsor on GitHub</a>
+    </p>
+  </body>
+</html>

netlify.toml

@@ -1,4 +0,0 @@
-[build]  
-  command = "npm run build"
-  publish = "output/"
-

output/style.css

@@ -1,30 +0,0 @@
-body {
-  font-family:system-ui;
-  font-size: 18px;
-}
-
-
-
-  details:focus,
-  details:focus-within,
-  details:hover {
-    /* background:#ffeb3b; */
-    /* outline:2px #000 solid; */
-  }
-
-
-  details ul li {
-  
-  }
-
-  summary {
-    cursor:pointer;
-  }
-
-  summary:hover {
-    opacity:.75;
-  }
-
-  .feed-url {
-    color:#aaa;
-  }

package.json

@@ -1,17 +1,49 @@
 {
   "name": "bubo-reader",
-  "version": "1.0.0",
+  "version": "2.0.1",
-  "description": "A somewhat dumb but effective feed reader (RSS, JSON & Twitter)",
+  "description": "A simple but effective feed reader (RSS, JSON)",
+  "homepage": "https://github.com/georgemandis/bubo-rss",
   "main": "src/index.js",
+  "type": "module",
   "scripts": {
-    "build": "node src/index.js > output/index.html",
+    "dev": "tsc --watch",
+    "clean": "rm -rf dist",
+    "build": "tsc",
+    "bubo": "node dist/index.js",
+    "build:bubo": "tsc && node dist/index.js",
+    "start": "npm run build:bubo; node server.js",    
     "test": "echo \"Error: no test specified\" && exit 1"
   },
-  "author": "",
+  "author": {
-  "license": "ISC",
+    "name": "George Mandis",
+    "email": "george@mand.is",
+    "url": "https://george.mand.is"
+  },
+  "funding": {
+    "type": "github",
+    "url": "https://github.com/sponsors/georgemandis"
+  },
+  "bugs": {
+    "url": "https://github.com/georgemandis/bubo-rss/issues",
+    "email": "george+bubo@mand.is"
+  },
+  "license": "MIT",
   "dependencies": {
-    "node-fetch": "^2.6.1",
+    "chalk": "^5.1.2",
-    "nunjucks": "^3.2.0",
+    "node-fetch": "^3.3.0",
-    "rss-parser": "^3.6.3"
+    "nunjucks": "^3.2.3",
+    "rss-parser": "^3.12.0",
+    "express": "^4.17.1"
+  },
+  "engines": { "node": "16.x" },
+  "devDependencies": {
+    "@types/node": "^16.18.4",
+    "@types/nunjucks": "^3.2.1",
+    "@types/xml2js": "^0.4.11",
+    "@typescript-eslint/eslint-plugin": "^5.45.0",
+    "@typescript-eslint/parser": "^5.45.0",
+    "eslint": "^8.29.0",
+    "tslib": "^2.4.1",
+    "typescript": "^4.9.3"
   }
 }

public/style.css

@@ -0,0 +1,28 @@
+body {
+  font-family: system-ui;
+  font-size: 18px;
+}
+
+
+
+details:focus,
+details:focus-within,
+details:hover {
+  /* background:#ffeb3b; */
+  /* outline:2px #000 solid; */
+}
+
+
+details ul li {}
+
+summary {
+  cursor: pointer;
+}
+
+summary:hover {
+  opacity: .75;
+}
+
+.feed-url {
+  color: #aaa;
+}

server.js

@@ -0,0 +1,24 @@
+// Bubo RSS (on Glitch!)
+
+// init project
+import express from "express";
+import { URL } from 'url';
+const __dirname = new URL('.', import.meta.url).pathname;
+
+const app = express();
+
+// we've started you off with Express,
+// but feel free to use whatever libs or frameworks you'd like through `package.json`.
+
+// http://expressjs.com/en/starter/static-files.html
+app.use(express.static("public"));
+
+// http://expressjs.com/en/starter/basic-routing.html
+app.get("/", function(request, response) {
+  response.sendFile(__dirname + "/public/index.html");
+});
+
+// listen for requests :)
+const listener = app.listen(process.env.PORT, function() {
+  console.log("Your app is listening on port " + listener.address().port);
+});

src/@types/bubo.d.ts

@@ -0,0 +1,15 @@
+export interface Feeds {
+  [key: string]: object[]
+}
+export interface FeedItem {
+  [key: string]: string | number | Date | FeedItem[];
+  items: FeedItem[]
+}
+
+//NEW WAY
+export type JSONValue =
+  | string
+  | number
+  | boolean
+  | { [x: string]: JSONValue }
+  | Array<JSONValue>;

src/index.js

@@ -1,89 +0,0 @@
-/**
- * 🦉 Bubo RSS Reader
- * ====
- * Dead, dead simple feed reader that renders an HTML
- * page with links to content from feeds organized by site
- *
- */
-
-const fetch = require("node-fetch");
-const Parser = require("rss-parser");
-const parser = new Parser();
-
-const nunjucks = require("nunjucks");
-const env = nunjucks.configure({ autoescape: true });
-
-const feeds = require("./feeds.json");
-
-env.addFilter("formatDate", function(dateString) {
-  const formattedDate = new Date(dateString).toLocaleDateString()
-  return formattedDate !== 'Invalid Date' ? formattedDate : dateString;
-});
-
-env.addGlobal('now', (new Date()).toUTCString() );
-
-// parse XML or JSON feeds
-function parseFeed(response) {
-  const contentType = response.headers.get("content-type")
-    ? response.headers.get("content-type").split(";")[0]
-    : false;
-
-  const rssFeed = [contentType]
-    .map(item =>
-      [
-        "application/atom+xml",
-        "application/rss+xml",
-        "application/xml",
-        "text/xml",
-        "text/html" // this is kind of a gamble
-      ].includes(item)
-        ? response.text()
-        : false
-    )
-    .filter(_ => _)[0];
-
-  const jsonFeed = [contentType]
-    .map(item =>
-      ["application/json"].includes(item) ? response.json() : false
-    )
-    .filter(_ => _)[0];
-
-  return rssFeed || jsonFeed || false;
-}
-
-(async () => {
-  const contentFromAllFeeds = {};
-  const errors = [];
-
-  for (group in feeds) {
-    contentFromAllFeeds[group] = [];
-
-    for (let index = 0; index < feeds[group].length; index++) {
-      try {
-        const response = await fetch(feeds[group][index]);
-        const body = await parseFeed(response);
-        const contents =
-          typeof body === "string" ? await parser.parseString(body) : body;
-
-        contents.feed = feeds[group][index];
-        contents.title = contents.title ? contents.title : contents.link;
-        contentFromAllFeeds[group].push(contents);
-        
-        // try to normalize date attribute naming
-        contents.items.forEach(item => {
-          const timestamp = new Date(item.pubDate || item.isoDate || item.date).getTime();          
-          item.timestamp = isNaN(timestamp) ? (item.pubDate || item.isoDate || item.date) : timestamp;
-        });        
-
-      } catch (error) {
-        errors.push(feeds[group][index]);
-      }
-    }
-  }
-
-  const output = env.render("./src/template.html", {
-    data: contentFromAllFeeds,
-    errors: errors
-  });
-  console.log(output);
-})();

src/index.ts

@@ -0,0 +1,169 @@
+/*
+ * 🦉 Bubo Reader
+ * ====
+ * Dead simple feed reader (RSS + JSON) that renders an HTML
+ * page with links to content from feeds organized by site
+ *
+ * Code: https://github.com/georgemandis/bubo-rss
+ * Copyright (c) 2019 George Mandis (https://george.mand.is)
+ * Version: 1.0.1 (11/14/2021)
+ * Licensed under the MIT License (http://opensource.org/licenses/MIT)
+ */
+
+import fetch from "node-fetch";
+import Parser from "rss-parser";
+import { Feeds, FeedItem } from "./@types/bubo";
+import { Response } from "node-fetch";
+import { render } from "./renderer.js";
+import {
+  getLink,
+  getTitle,
+  getTimestamp,
+  parseFeed,
+  getFeedList,
+  getBuboInfo
+} from "./utilities.js";
+import { writeFile } from "fs/promises";
+import chalk from "chalk";
+
+const buboInfo = await getBuboInfo();
+const parser = new Parser();
+const feedList = await getFeedList();
+const feedListLength =
+  Object.entries(feedList).flat(2).length - Object.keys(feedList).length;
+
+/**
+ * contentFromAllFeeds = Contains normalized, aggregated feed data and is passed to template renderer at the end
+ * errors = Contains errors from parsing feeds and is also passed to template.
+ */
+const contentFromAllFeeds: Feeds = {};
+const errors: unknown[] = [];
+
+// benchmarking data + utility
+const initTime = Date.now();
+const benchmark = (startTime: number) =>
+  chalk.cyanBright.bold(`${(Date.now() - startTime) / 1000} seconds`);
+
+/**
+ * These values are used to control throttling/batching the fetches:
+ *  - MAX_CONNECTION = max number of fetches to contain in a batch
+ *  - DELAY_MS = the delay in milliseconds between batches
+ */
+const MAX_CONNECTIONS = Infinity;
+const DELAY_MS = 850;
+
+const error = chalk.bold.red;
+const success = chalk.bold.green;
+
+// keeping tally of total feeds fetched and parsed so we can compare
+// to feedListLength and know when we're finished.
+let completed = 0;
+
+/**
+ * finishBuild
+ * --
+ * function that gets called when all the feeds are through fetching
+ * and we want to build the static output.
+ */
+const finishBuild: () => void = async () => {
+  console.log("\nDone fetching everything!");
+
+  // generate the static HTML output from our template renderer
+  const output = render({
+    data: contentFromAllFeeds,
+    errors: errors,
+    info: buboInfo
+  });
+
+  // write the output to public/index.html
+  await writeFile("./public/index.html", output);
+  console.log(
+    `\nFinished writing to output:\n- ${feedListLength} feeds in ${benchmark(
+      initTime
+    )}\n- ${errors.length} errors`
+  );
+};
+
+/**
+ * processFeed
+ * --
+ * Process an individual feed and normalize its items
+ * @param { group, feed, startTime}
+ * @returns Promise<void>
+ */
+const processFeed =
+  ({
+    group,
+    feed,
+    startTime
+  }: {
+    group: string;
+    feed: string;
+    startTime: number;
+  }) =>
+    async (response: Response): Promise<void> => {
+      const body = await parseFeed(response);
+      completed++;
+      // skip to the next one if this didn't work out
+      if (!body) return;
+
+      try {
+        const contents: FeedItem = (
+          typeof body === "string" ? await parser.parseString(body) : body
+        ) as FeedItem;
+
+        contents.feed = feed;
+        contents.title = getTitle(contents);
+        contents.link = getLink(contents);
+
+        // try to normalize date attribute naming
+        contents?.items?.forEach(item => {
+          item.timestamp = getTimestamp(item);
+          item.title = getTitle(item);
+          item.link = getLink(item);
+        });
+
+        contentFromAllFeeds[group].push(contents as object);
+        console.log(
+          `${success("Successfully fetched:")} ${feed} - ${benchmark(startTime)}`
+        );
+      } catch (err) {
+        console.log(
+          `${error("Error processing:")} ${feed} - ${benchmark(
+            startTime
+          )}\n${err}`
+        );
+        errors.push(`Error processing: ${feed}\n\t${err}`);
+      }
+
+      // if this is the last feed, go ahead and build the output
+      completed === feedListLength && finishBuild();
+    };
+
+// go through each group of feeds and process
+const processFeeds = () => {
+  let idx = 0;
+
+  for (const [group, feeds] of Object.entries(feedList)) {
+    contentFromAllFeeds[group] = [];
+
+    for (const feed of feeds) {
+      const startTime = Date.now();
+      setTimeout(() => {
+        console.log(`Fetching: ${feed}...`);
+
+        fetch(feed)
+          .then(processFeed({ group, feed, startTime }))
+          .catch(err => {
+            console.log(
+              error(`Error fetching ${feed} ${benchmark(startTime)}`)
+            );
+            errors.push(`Error fetching ${feed} ${err.toString()}`);
+          });
+      }, (idx % (feedListLength / MAX_CONNECTIONS)) * DELAY_MS);
+      idx++;
+    }
+  }
+};
+
+processFeeds();

src/renderer.ts

@@ -0,0 +1,44 @@
+/*
+ * Return our renderer.
+ * Using Nunjucks out of the box.
+ * https://mozilla.github.io/nunjucks/
+ */
+
+import nunjucks from "nunjucks";
+const env: nunjucks.Environment = nunjucks.configure({ autoescape: true });
+import { readFile } from "fs/promises";
+import { Feeds, JSONValue } from "./@types/bubo";
+
+/**
+ * Global filters for my Nunjucks templates
+ */
+env.addFilter("formatDate", function (dateString): string {
+  const date: Date = new Date(parseInt(dateString));
+  return !isNaN(date.getTime()) ? date.toLocaleDateString() : dateString;
+});
+
+env.addGlobal("now", new Date().toUTCString());
+
+// load the template
+const template: string = (
+  await readFile(new URL("../config/template.html", import.meta.url))
+).toString();
+
+// generate the static HTML output from our template renderer
+const render = ({
+  data,
+  errors,
+  info
+}: {
+  data: Feeds;
+  errors: unknown[];
+  info?: JSONValue;
+}) => {
+  return env.renderString(template, {
+    data,
+    errors,
+    info
+  });
+};
+
+export { render };

src/template.html

@@ -1,48 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <meta http-equiv="X-UA-Compatible" content="ie=edge">
-  <title>🦉 Bubo Reader</title>
-  <link rel="stylesheet" href="/style.css">
-</head>
-<body>
-
-  <h1>🦉 Bubo Reader</h1>
-
-  {% for group, feeds in data %}    
-  <h2>{{ group }}</h2>
-    {% for feed  in feeds %} 
-      <details>
-        <summary>
-          <span class="feed-title">{{ feed.title }}</span> 
-          <span class="feed-url">({{ feed.feed }})</span>
-        </summary>
-        <ul>
-          {% for item in feed.items %}
-            <li>              
-              {{ item.timestamp | formatDate }} - <a href="{{ item.link }}" target='_blank' rel='noopener norefferer nofollow'>{{ item.title }}</a>
-            </li>
-          {% endfor %}
-        </ul>
-      </details>
-    {% endfor %}
-  {% endfor %}  
-
-  {% if errors | length > 0 %}
-    <h2>Errors</h2>
-    <p>There were errors trying to parse these feeds:</p>
-    <ul>
-    {% for error in errors %}
-      <li>{{ error }}</li>
-    {% endfor %}
-    </ul>
-  {% endif %}
-
-  <hr>
-  <p>
-    Last updated {{ now }}. Powered by <a href="https://github.com/georgemandis/bubo-rss">Bubo Reader</a>, a project by <a href="https://george.mand.is">George Mandis</a>
-  </p>
-</body>
-</html>

src/utilities.ts

@@ -0,0 +1,90 @@
+/*
+  There's a little inconsistency with how feeds report certain things like
+  title, links and timestamps. These helpers try to normalize that bit and
+  provide an order-of-operations list of properties to look for.
+
+  Note: these are tightly-coupled to the template and a personal preference.
+*/
+
+import { Response } from "node-fetch";
+import { readFile } from "fs/promises";
+import { FeedItem, JSONValue } from "./@types/bubo";
+
+export const getLink = (obj: FeedItem): string => {
+  const link_values: string[] = ["link", "url", "guid", "home_page_url"];
+  const keys: string[] = Object.keys(obj);
+  const link_property: string | undefined = link_values.find(link_value =>
+    keys.includes(link_value)
+  );
+  return link_property ? (obj[link_property] as string) : "";
+};
+
+// fallback to URL for the title if not present
+// (title -> url -> link)
+export const getTitle = (obj: FeedItem): string => {
+  const title_values: string[] = ["title", "url", "link"];
+  const keys: string[] = Object.keys(obj);
+
+  // if title is empty for some reason, fall back on url or link
+  const title_property: string | undefined = title_values.find(
+    title_value => keys.includes(title_value) && obj[title_value]
+  );
+  return title_property ? (obj[title_property] as string) : "";
+};
+
+// More dependable way to get timestamps
+export const getTimestamp = (obj: FeedItem): string => {
+  const dateString: string = (
+    obj.pubDate ||
+    obj.isoDate ||
+    obj.date ||
+    obj.date_published
+  ).toString();
+  const timestamp: number = new Date(dateString).getTime();
+  return isNaN(timestamp) ? dateString : timestamp.toString();
+};
+
+// parse RSS/XML or JSON feeds
+export async function parseFeed(response: Response): Promise<JSONValue> {
+  const contentType = response.headers.get("content-type")?.split(";")[0];
+
+  if (!contentType) return {};
+
+  const rssFeed = [contentType]
+    .map(item =>
+      [
+        "application/atom+xml",
+        "application/rss+xml",
+        "application/xml",
+        "text/xml",
+        "text/html" // this is kind of a gamble
+      ].includes(item)
+        ? response.text()
+        : false
+    )
+    .filter(_ => _)[0];
+
+  const jsonFeed = [contentType]
+    .map(item =>
+      ["application/json", "application/feed+json"].includes(item)
+        ? (response.json() as Promise<JSONValue>)
+        : false
+    )
+    .filter(_ => _)[0];
+
+  return (rssFeed && rssFeed) || (jsonFeed && jsonFeed) || {};
+}
+
+export const getFeedList = async (): Promise<JSONValue> => {
+  return JSON.parse(
+    (
+      await readFile(new URL("../config/feeds.json", import.meta.url))
+    ).toString()
+  );
+};
+
+export const getBuboInfo = async (): Promise<JSONValue> => {
+  return JSON.parse(
+    (await readFile(new URL("../package.json", import.meta.url))).toString()
+  );
+};

tsconfig.json

@@ -0,0 +1,30 @@
+{
+  "compilerOptions": {
+    "module": "esnext",
+    "forceConsistentCasingInFileNames": true,
+    "removeComments": true,
+    "strict": true,
+    "importHelpers": true,
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "target": "ES2021",
+    "noImplicitAny": true,
+    "moduleResolution": "node",
+    "sourceMap": false,
+    "resolveJsonModule": true,
+    "outDir": "dist",
+    "baseUrl": ".",
+    "typeRoots": [
+      "src/@types"
+    ],
+    "paths": {
+      "*": [
+        "node_modules/*",
+        "src/@types"
+      ]
+    }
+  },
+  "include": [
+    "src/**/*"
+  ]
+}