Merging 2.0 in and tweaking for Glitch

This commit is contained in:
George Mandis 2021-11-29 01:06:07 -08:00
commit 1a9c60a4d6
18 changed files with 3315 additions and 1992 deletions

50
.eslintrc.json Normal file
View File

@ -0,0 +1,50 @@
{
"env": {
"browser": true,
"es2021": true
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 6,
"sourceType": "module"
},
"plugins": [
"@typescript-eslint"
],
"rules": {
"indent": [
"error",
2
],
"linebreak-style": [
"error",
"unix"
],
"quotes": [
"error",
"double"
],
"semi": [
"error",
"always"
],
"no-trailing-spaces": [
2,
{
"skipBlankLines": false
}
],
"no-multiple-empty-lines": [
"error",
{
"max": 2,
"maxEOF": 1
}
],
"@typescript-eslint/no-var-requires": 0
}
}

4
.github/FUNDING.yml vendored
View File

@ -1,7 +1,7 @@
# These are supported funding model platforms # These are supported funding model platforms
github: georgemandis github: georgemandis
patreon: georgemandis patreon: #
open_collective: # Replace with a single Open Collective username open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username ko_fi: # Replace with a single Ko-fi username
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
@ -9,4 +9,4 @@ community_bridge: # Replace with a single Community Bridge project-name e.g., cl
liberapay: # Replace with a single Liberapay username liberapay: # Replace with a single Liberapay username
issuehunt: # Replace with a single IssueHunt username issuehunt: # Replace with a single IssueHunt username
otechie: # Replace with a single Otechie username otechie: # Replace with a single Otechie username
custom: https://george.mand.is/sponsor custom: #

4
.gitignore vendored
View File

@ -1,2 +1,4 @@
node_modules/* node_modules/*
output/index.html public/index.html
dist/*
.DS_Store

1
.nvmrc Normal file
View File

@ -0,0 +1 @@
16.4.1

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2021 George Mandis
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

121
README.md
View File

@ -1,35 +1,102 @@
# 🦉 Bubo Reader # 🦉 Bubo Reader (2.0)
Bubo Reader is a somewhat irrationally minimalist <acronym title="Really Simple Syndication">RSS</acronym> and <acronym title="JavaScript Object Notation">JSON</acronym> feed reader you can deploy on [Netlify](https://netlify.com) in a few steps or [Glitch](https://glitch.com) in even fewer steps! The goal of the project is to generate a webpage that shows a list of links from a collection of feeds organized by category and website. That's it. Bubo Reader is a hyper-minimalist <acronym title="Really Simple Syndication">RSS</acronym> and <acronym title="JavaScript Object Notation">JSON</acronym> feed reader you can deploy on your own server, [Netlify](https://netlify.com) in a few steps or [Glitch](https://glitch.com) in even fewer steps! The goal of the project is to generate a webpage that shows a list of links from a collection of feeds organized by category and website. That's it.
It is named after this [silly robot owl](https://www.youtube.com/watch?v=MYSeCfo9-NI) from Clash of the Titans (1981). It is named after this [silly robot owl](https://www.youtube.com/watch?v=MYSeCfo9-NI) from Clash of the Titans (1981).
You can read more about how this project came about in my blog post '[Introducing Bubo RSS: An Absurdly Minimalist RSS Feed Reader](https://george.mand.is/2019/11/introducing-bubo-rss-an-absurdly-minimalist-rss-feed-reader/)' You can read more about how this project came about on my blog:
- [Introducing Bubo RSS: An Absurdly Minimalist RSS Feed Reader](https://george.mand.is/2019/11/introducing-bubo-rss-an-absurdly-minimalist-rss-feed-reader/).
## Getting Started ## Getting Started
How to deploy Bubo Reader in a few easy steps with Netlify or Glitch: - Clone or fork the repo and run `npm install` to install the dependencies.
- Update `feeds.json` to include categories and links to feeds you would like to see.
- Run `npm run build:bubo`
### Deploying to Glitch That's it! You should now have a static page with links to the latest content from your feeds in the `public` folder, ready to serve.
## Differences in Bubo 2.0
Version 2.0 has introduced some substantial changes for Bubo! While the static output remains endearingly spartan, the engine that builds it has changed a bit.
Hopefully all of these changes are in services of making this project more useful to others and encouraging outside contributions.
Changes of note:
- Bubo has been rewritten in [TypeScript](https://www.typescriptlang.org/). It's pretty slick! I anticipate the typing could be improved, but it's a start.
- You fill find an `.nvmrc` file in the root of this project. Learn more [about nvm](https://github.com/nvm-sh/nvm) if you're unfamiliar.
- The script will actually write your `index.html` file for you (Previously the build script simply ran `node src/index.js > output/index.html`). It makes a strong assumption that this file lives in the `public` folder.
- There is a somewhat sophisticated mechansim in-place for batching & throttling your requests, if needed.
## Anatomy of Bubo Reader
The static pieces:
- `conf/feeds.json` - a JSON file containing your feed URLS separated into categories.
- `config/template.html` - a [Nunjucks](https://mozilla.github.io/nunjucks/) template that lets you change how the feeds are displayed. This can be changed to anything else you like— see below.
- `public/style.css` - a CSS file to stylize your feed output.
- `public/index.html` - The HTML file that gets automatically generated when Bubo is run.
The engine:
- `src/index.ts` - The primary script you run when you want to build a new version of Bubo. It will automatically fetch the latest content from your feeds and build a new static file at `public/index.html`.
- `src/renderer.ts` — The renderer that loads Nunjucks, the template and understands how to process the incoming feed data. Prefer something else? This is the place to change it!
- `src/utilities.ts` — A variety of parsing and normalization utilities for Bubo, hidden away to try and keep things clean.
## Throttling
In the main `index.ts` file you will find two values that allow yout to batch and throttle your feed requests:
- `MAX_CONNECTIONS` dictates the maximium number of requests a batch can have going at once.
- `DELAY_MS` dictates the amount of de;ay time between each batch.
The default configuration is **no batching or throttling** beacuse `MAX_CONNECTIONS` is set to `Infinity`. If you wanted to change Bubo to only fetch one feed at a time every second you could set these values to:
```javascript
const MAX_CONNECTIONS = 1;
const DELAY_MS = 1000;
```
If you wanted to limit things to 10 simultaneous requests every 2.5 seconds you could set it like so:
```javascript
const MAX_CONNECTIONS = 10;
const DELAY_MS = 2500;
```
In practice, I've never _really_ run into an issue leaving `MAX_CONNECTIONS` set to `Infinity` but this feels like a sensible safeguard to design.
## Demos
You can view live demos here:
- [https://bubo-rss-demo.netlify.com/](https://bubo-rss-demo.netlify.com/)
- [http://bubo-rss.glitch.me/](http://bubo-rss.glitch.me/)
Not the most exciting-looking demos, I'll admit, but they work!
**Getting Started**
- [Deploying to Glitch](#glitch)
- [Deploying to Netlify](#netlify)
- [Keeping feeds updated](#updated)
<a id="glitch"></a>
## Deploying to Glitch
The quickest way is to remix the project on Glitch: The quickest way is to remix the project on Glitch:
[https://glitch.com/edit/#!/bubo-rss](https://glitch.com/edit/#!/bubo-rss) [https://glitch.com/edit/#!/bubo-rss](https://glitch.com/edit/#!/bubo-rss)
Just changed some feeds in `./src/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./output/style.css` file or the `./src/template.html` file respectively. Just changed some feeds in `./config/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./public/style.css` file or the `./config/template.html` file respectively.
There is also a special `glitch` branch you can clone if you prefer: <a id="netlify"></a>
[https://github.com/georgemandis/bubo-rss/tree/glitch](https://github.com/georgemandis/bubo-rss/tree/glitch) ## Deploying to Netlify
The only difference between this branch and `master` is that it spins up a server using [Express](https://expressjs.com/) to serve your `./output/index.html` file on Glitch. Everything else is the same.
### Deploying to Netlify
- [Fork the repository](https://github.com/georgemandis/bubo-rss/fork) - [Fork the repository](https://github.com/georgemandis/bubo-rss/fork)
- From your forked repository go to and edcit `src/feeds.json` to manage your feeds and categories - From your forked repository edit `config/feeds.json` to manage your feeds and categories
- [Create a new site](https://app.netlify.com/start) on Netlify from GitHub - [Create a new site](https://app.netlify.com/start) on Netlify from GitHub
The deploy settings should automatically import from the `netlify.toml` file. All you'll need to do is confirm and you're ready to go! The deploy settings should automatically import from the `netlify.toml` file. All you'll need to do is confirm and you're ready to go!
<a id="updated"></a>
### Keeping Feeds Updated ### Keeping Feeds Updated
#### Using Netlify Webhooks #### Using Netlify Webhooks
@ -40,25 +107,21 @@ To keep your feeds up to date you'll want to [setup a Build Hook](https://www.ne
- [Zapier](https://zapier.com/) - [Zapier](https://zapier.com/)
- [EasyCron](https://www.easycron.com/) - [EasyCron](https://www.easycron.com/)
If you already have a server running Linux and some command-line experience it might be simpler to setup a [cron job](https://en.wikipedia.org/wiki/Cron).
#### Using GitHub Actions #### Using GitHub Actions
This approach is a little different and requires some modifications to the repository. Netlify started billing for [build minutes](https://www.netlify.com/pricing/faq/) very shortly after I published this project. Running `npm build` and downloading all of the RSS feeds took up a substantial number of this minutes, particulary if you had some kind of process pinging the webhook and trigger a build every 15 minutes or so. Coming soon—there is an old branch that demonstrates this, but it needs to be revisisted in light of Bubo 2.0.
How is the The GitHub Action-based approach different? The same build process runs, but this time it's on GitHub's servers via the Action. It then **commits** the newly created file generated at `./output/index.html` back into the repository. Netlify still gets pinged when the repository is updated, but skips the `npm run build` step on their end, which significantly reduces the number of build minutes required. #### Rolling Your Own
**Short Answer**: use the [`github-action-publishing`](https://github.com/georgemandis/bubo-rss/tree/github-action-publishing) branch for now if you'd prefer to use GitHub Actions to run your builds. If you already have a server running Linux and some command-line experience it might be simpler to setup a [cron job](https://en.wikipedia.org/wiki/Cron).
The GitHub Action is setup to build and commit directly to the `master` branch, which is not the best practice. I'd suggest creating a separate branch to checkout and commit changes to in the Action. You could then specify that same branch as the one to checkout and publish on Netlify.
## Anatomy of Bubo Reader
- `src/index.html` - a [Nunjucks](https://mozilla.github.io/nunjucks/) template that lets you change how the feeds are displayed
- `output/style.css` - a CSS file to stylize your feed output
- `src/feeds.json` - a JSON file containing the URLs for various site's feeds separated into categories
- `src/index.js` - the script that loads the feeds and does the actual parsinga and rendering
## Support ## Support
If you found this useful please consider sponsoring me or this project. If you'd rather run this on your own server please consider using one of these affiliate links to setup a micro instance on [Linode](https://www.linode.com/?r=8729957ab02b50a695dcea12a5ca55570979d8b9), [Digital Ocean](https://m.do.co/c/31f58d367777) or [Vultr](https://www.vultr.com/?ref=8403978). If you found this useful please consider sponsoring me or this project.
If you'd rather run this on your own server please consider using one of these affiliate links to setup a micro instance on [Linode](https://www.linode.com/?r=8729957ab02b50a695dcea12a5ca55570979d8b9), [Digital Ocean](https://m.do.co/c/31f58d367777) or [Vultr](https://www.vultr.com/?ref=8403978).
## Showcase
Here are some websites using Bubo Reader:
- [Kevin Fiol](https://kevinfiol.com/reader/) ([repo](https://github.com/kevinfiol/reader))

View File

@ -1,7 +1,6 @@
{ {
"Web Development": [ "Web Development": [
"https://hacks.mozilla.org/feed/", "https://hacks.mozilla.org/feed/",
"https://blog.mozilla.org/feed/",
"https://web.dev/feed.xml", "https://web.dev/feed.xml",
"https://v8.dev/blog.atom", "https://v8.dev/blog.atom",
"https://alistapart.com/main/feed/", "https://alistapart.com/main/feed/",
@ -14,11 +13,10 @@
], ],
"My GitHub Projects": [ "My GitHub Projects": [
"https://github.com/georgemandis.atom", "https://github.com/georgemandis.atom",
"https://github.com/georgemandis/bubo-rss/releases.atom",
"https://github.com/georgemandis/konami-js/releases.atom", "https://github.com/georgemandis/konami-js/releases.atom",
"https://github.com/georgemandis/konami-js/commits/main.atom", "https://github.com/georgemandis/konami-js/commits/main.atom",
"https://github.com/javascriptforartists/cheer-me-up-and-sing-me-a-song/commits/master.atom", "https://github.com/javascriptforartists/cheer-me-up-and-sing-me-a-song/commits/master.atom",
"https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom", "https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom"
"https://github.com/georgemandis/remote-working-list/commits/master.atom",
"https://github.com/georgemandis/tweeter-totter/commits/master.atom"
] ]
} }

View File

@ -40,9 +40,17 @@
</ul> </ul>
{% endif %} {% endif %}
<br>
<hr> <hr>
<p> <p>
Last updated {{ now }}. Powered by <a href="https://github.com/georgemandis/bubo-rss">Bubo Reader</a>, a project by <a href="https://george.mand.is">George Mandis</a> Last updated {{ now }}.
</p> </p>
<p>
Powered by <a href="https://github.com/georgemandis/bubo-rss">Bubo Reader (2.0.0)</a>, a project by <a href="https://george.mand.is">George Mandis</a>.
</p>
<p>
<a href='https://github.com/sponsors/georgemandis'>❤️ Sponsor on GitHub</a>
</p>
</body> </body>
</html> </html>

View File

@ -1,30 +0,0 @@
body {
font-family:system-ui;
font-size: 18px;
}
details:focus,
details:focus-within,
details:hover {
/* background:#ffeb3b; */
/* outline:2px #000 solid; */
}
details ul li {
}
summary {
cursor:pointer;
}
summary:hover {
opacity:.75;
}
.feed-url {
color:#aaa;
}

4563
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,19 +1,44 @@
{ {
"name": "bubo-reader", "name": "bubo-reader",
"version": "1.0.1", "version": "2.0.0",
"description": "A Glitch-ready implementation of Bubo Reader", "description": "A simple but effective feed reader (RSS, JSON)",
"main": "src/index.js", "main": "src/index.js",
"type": "module",
"scripts": { "scripts": {
"start": "node src/index.js > output/index.html; node server.js", "dev": "tsc --watch",
"build": "node src/index.js > output/index.html", "clean": "rm -rf dist",
"build": "tsc",
"bubo": "node dist/index.js",
"build:bubo": "tsc && node dist/index.js",
"start": "npm run build:bubo; node server.js",
"test": "echo \"Error: no test specified\" && exit 1" "test": "echo \"Error: no test specified\" && exit 1"
}, },
"author": "", "author": {
"name": "George Mandis",
"email": "george@mand.is",
"url": "https://george.mand.is"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/georgemandis"
},
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"node-fetch": "^2.6.0", "chalk": "^5.0.0",
"node-fetch": "^3.1.0",
"nunjucks": "^3.2.0", "nunjucks": "^3.2.0",
"rss-parser": "^3.7.5", "rss-parser": "^3.6.3",
"express": "^4.17.1" "express": "^4.17.1"
},
"engines": { "node": "16.x" },
"devDependencies": {
"@types/node": "^16.11.10",
"@types/nunjucks": "^3.2.0",
"@types/xml2js": "^0.4.9",
"@typescript-eslint/eslint-plugin": "^5.4.0",
"@typescript-eslint/parser": "^5.4.0",
"eslint": "^8.3.0",
"tslib": "^2.3.1",
"typescript": "^4.5.2"
} }
} }

28
public/style.css Normal file
View File

@ -0,0 +1,28 @@
body {
font-family: system-ui;
font-size: 18px;
}
details:focus,
details:focus-within,
details:hover {
/* background:#ffeb3b; */
/* outline:2px #000 solid; */
}
details ul li {}
summary {
cursor: pointer;
}
summary:hover {
opacity: .75;
}
.feed-url {
color: #aaa;
}

15
src/@types/bubo.d.ts vendored Normal file
View File

@ -0,0 +1,15 @@
export interface Feeds {
[key: string]: object[]
}
export interface FeedItem {
[key: string]: string | number | Date | FeedItem[];
items: FeedItem[]
}
//NEW WAY
export type JSONValue =
| string
| number
| boolean
| { [x: string]: JSONValue }
| Array<JSONValue>;

View File

@ -1,129 +0,0 @@
/*
* 🦉 Bubo RSS Reader
* ====
* Dead simple feed reader that renders an HTML
* page with links to content from feeds organized by site
*
* Code: https://github.com/georgemandis/bubo-rss
* Copyright (c) 2019 George Mandis (https://george.mand.is)
* Version: 1.0.1 (11/14/2021)
* Licensed under the MIT License (http://opensource.org/licenses/MIT)
*/
const fetch = require("node-fetch");
const Parser = require("rss-parser");
const parser = new Parser();
const nunjucks = require("nunjucks");
const env = nunjucks.configure({ autoescape: true });
const feeds = require("./feeds.json");
/**
* Global filters for my Nunjucks templates
*/
env.addFilter("formatDate", function (dateString) {
const formattedDate = new Date(dateString).toLocaleDateString()
return formattedDate !== 'Invalid Date' ? formattedDate : dateString;
});
env.addGlobal('now', (new Date()).toUTCString());
// parse RSS/XML or JSON feeds
function parseFeed(response) {
const contentType = response.headers.get("content-type")
? response.headers.get("content-type").split(";")[0]
: false;
const rssFeed = [contentType]
.map(item =>
[
"application/atom+xml",
"application/rss+xml",
"application/xml",
"text/xml",
"text/html" // this is kind of a gamble
].includes(item)
? response.text()
: false
)
.filter(_ => _)[0];
const jsonFeed = [contentType]
.map(item =>
["application/json", "application/feed+json"].includes(item) ? response.json() : false
)
.filter(_ => _)[0];
return rssFeed || jsonFeed || false;
}
/*
There's a little inconcistency with how feeds report certain things like
title, links and timestamps. These helpers try to normalize that bit and
provide an order-of-operations list of properties to look for.
Note: these are tightly-coupled to the template and a personal preference.
*/
const getLink = (obj) => {
const link_values = ["link", "url", "guid", "home_page_url"];
const keys = Object.keys(obj);
const link_property = link_values.find(link_value => keys.includes(link_value));
return obj[link_property];
}
// fallback to URL for the title if not present (coupled to my template)
const getTitle = (obj) => {
const title_values = ["title", "url", "link"]; // fallback to url/link as title if omitted
const keys = Object.keys(obj);
const title_property = title_values.find(title_value => keys.includes(title_value));
return obj[title_property];
}
// More dependable way to get timestamps
const getTimestamp = (obj) => {
const timestamp = new Date(obj.pubDate || obj.isoDate || obj.date || obj.date_published).getTime();
return isNaN(timestamp) ? (obj.pubDate || obj.isoDate || obj.date || obj.date_published) : timestamp;
}
// fetch the feeds and build the object for our template
(async () => {
const contentFromAllFeeds = {};
const errors = [];
for (const group in feeds) {
contentFromAllFeeds[group] = [];
for (let index = 0; index < feeds[group].length; index++) {
try {
const response = await fetch(feeds[group][index]);
const body = await parseFeed(response);
const contents =
typeof body === "string" ? await parser.parseString(body) : body;
contents.feed = feeds[group][index];
contents.title = getTitle(contents);
contents.link = getLink(contents);
contentFromAllFeeds[group].push(contents);
// try to normalize date attribute naming
contents?.items?.forEach(item => {
item.timestamp = getTimestamp(item);
item.title = getTitle(item);
item.link = getLink(item);
});
} catch (error) {
errors.push(feeds[group][index]);
}
}
}
const output = env.render("./src/template.html", {
data: contentFromAllFeeds,
errors: errors
});
console.log(output);
})();

143
src/index.ts Normal file
View File

@ -0,0 +1,143 @@
/*
* 🦉 Bubo Reader
* ====
* Dead simple feed reader (RSS + JSON) that renders an HTML
* page with links to content from feeds organized by site
*
* Code: https://github.com/georgemandis/bubo-rss
* Copyright (c) 2019 George Mandis (https://george.mand.is)
* Version: 1.0.1 (11/14/2021)
* Licensed under the MIT License (http://opensource.org/licenses/MIT)
*/
import fetch from "node-fetch";
import Parser from "rss-parser";
import { Feeds, FeedItem } from "./@types/bubo";
import { Response } from "node-fetch";
import { render } from "./renderer.js";
import { getLink, getTitle, getTimestamp, parseFeed, getFeedList } from "./utilities.js";
import { writeFile } from "fs/promises";
import chalk from "chalk";
const parser = new Parser();
const feedList = await getFeedList();
const feedListLength = Object.entries(feedList).flat(2).length - Object.keys(feedList).length;
/**
* contentFromAllFeeds = Contains normalized, aggregated feed data and is passed to template renderer at the end
* errors = Contains errors from parsing feeds and is also passed to template.
*/
const contentFromAllFeeds: Feeds = {};
const errors: unknown[] = [];
// benchmarking data + utility
const initTime = Date.now();
const benchmark = (startTime: number) => chalk.cyanBright.bold(`(${(Date.now() - startTime) / 1000} seconds)`);
/**
* These values are used to control throttling/batching the fetches:
* - MAX_CONNECTION = max number of fetches to contain in a batch
* - DELAY_MS = the delay in milliseconds between batches
*/
const MAX_CONNECTIONS = Infinity;
const DELAY_MS = 850;
const error = chalk.bold.red;
const success = chalk.bold.green;
// keeping tally of total feeds fetched and parsed so we can compare
// to feedListLength and know when we're finished.
let completed = 0;
/**
* finishBuild
* --
* function that gets called when all the feeds are through fetching
* and we want to build the static output.
*/
const finishBuild: () => void = async () => {
console.log("\nDone fetching everything!");
// generate the static HTML output from our template renderer
const output = render({
data: contentFromAllFeeds,
errors: errors
});
// write the output to public/index.html
await writeFile("./public/index.html", output);
console.log(`Finished writing to output. ${benchmark(initTime)}`);
};
/**
* processFeed
* --
* Process an individual feed and normalize its items
* @param { group, feed, startTime}
* @returns Promise<void>
*/
const processFeed = (
{
group, feed, startTime
}: { group: string; feed: string, startTime: number }
) => async (response: Response): Promise<void> => {
const body = await parseFeed(response);
completed++;
// skip to the next one if this didn't work out
if (!body) return;
try {
const contents: FeedItem =
(typeof body === "string" ? (await parser.parseString(body)) : body) as FeedItem;
contents.feed = feed;
contents.title = getTitle(contents);
contents.link = getLink(contents);
// try to normalize date attribute naming
contents?.items?.forEach((item) => {
item.timestamp = getTimestamp(item);
item.title = getTitle(item);
item.link = getLink(item);
});
contentFromAllFeeds[group].push(contents as object);
console.log(`${success("Successfully fetched:")} ${feed} ${benchmark(startTime)}`);
} catch (err) {
console.log(`${error("Error processing:")} ${feed} ${benchmark(startTime)}`);
errors.push(err);
}
// if this is the last feed, go ahead and build the output
(completed === feedListLength) && finishBuild();
};
// go through each group of feeds and process
const processFeeds = () => {
let idx = 0;
for (const [group, feeds] of Object.entries(feedList)) {
contentFromAllFeeds[group] = [];
for (const feed of feeds) {
const startTime = Date.now();
setTimeout(() => {
console.log(`Fetching: ${feed}...`);
fetch(feed).then(processFeed({ group, feed, startTime })).catch(err => {
console.log(error(`Error fetching ${feed} ${benchmark(startTime)}`));
errors.push(`Error fetching ${feed} ${err.toString()}`);
});
}, (idx % (feedListLength / MAX_CONNECTIONS)) * DELAY_MS);
idx++;
}
}
};
processFeeds();

37
src/renderer.ts Normal file
View File

@ -0,0 +1,37 @@
/*
* Return our renderer.
* Using Nunjucks out of the box.
* https://mozilla.github.io/nunjucks/
*/
import nunjucks from "nunjucks";
const env: nunjucks.Environment = nunjucks.configure({ autoescape: true });
import { readFile } from "fs/promises";
import { Feeds } from "./@types/bubo";
/**
* Global filters for my Nunjucks templates
*/
env.addFilter("formatDate", function (dateString): string {
const date: Date = new Date(parseInt(dateString));
return !isNaN(date.getTime()) ? date.toLocaleDateString() : dateString;
});
env.addGlobal("now", (new Date()).toUTCString());
// load the template
const template: string =
(await readFile(
new URL("../config/template.html", import.meta.url)
)).toString();
// generate the static HTML output from our template renderer
const render = ({ data, errors }: { data: Feeds; errors: unknown[] }) => {
return env.renderString(template, {
data,
errors
});
};
export { render };

74
src/utilities.ts Normal file
View File

@ -0,0 +1,74 @@
/*
There's a little inconcistency with how feeds report certain things like
title, links and timestamps. These helpers try to normalize that bit and
provide an order-of-operations list of properties to look for.
Note: these are tightly-coupled to the template and a personal preference.
*/
import { Response } from "node-fetch";
import { readFile } from "fs/promises";
import { FeedItem, JSONValue } from "./@types/bubo";
export const getLink = (obj: FeedItem): string => {
const link_values: string[] = ["link", "url", "guid", "home_page_url"];
const keys: string[] = Object.keys(obj);
const link_property: string | undefined = link_values.find(link_value => keys.includes(link_value));
return link_property ? obj[link_property] as string : "";
};
// fallback to URL for the title if not present (coupled to my template)
export const getTitle = (obj: FeedItem): string => {
const title_values: string[] = ["title", "url", "link"]; // fallback to url/link as title if omitted
const keys: string[] = Object.keys(obj);
const title_property: string | undefined = title_values.find(title_value => keys.includes(title_value));
return title_property ? obj[title_property] as string : "";
};
// More dependable way to get timestamps
export const getTimestamp = (obj: FeedItem): string => {
const dateString: string = (obj.pubDate || obj.isoDate || obj.date || obj.date_published).toString();
const timestamp: number = new Date(dateString).getTime();
return isNaN(timestamp) ? dateString : timestamp.toString();
};
// parse RSS/XML or JSON feeds
export async function parseFeed(response: Response): Promise<JSONValue> {
const contentType = response.headers.get("content-type")?.split(";")[0];
if (!contentType) return {};
const rssFeed = [contentType]
.map(item =>
[
"application/atom+xml",
"application/rss+xml",
"application/xml",
"text/xml",
"text/html" // this is kind of a gamble
].includes(item)
? response.text()
: false
)
.filter(_ => _)[0];
const jsonFeed = [contentType]
.map(item =>
["application/json", "application/feed+json"].includes(item) ? response.json() as Promise<JSONValue> : false
)
.filter(_ => _)[0];
return (rssFeed && rssFeed) || (jsonFeed && jsonFeed) || {};
}
export const getFeedList = async (): Promise<JSONValue> => {
return JSON.parse(
(await readFile(
new URL("../config/feeds.json", import.meta.url)
)).toString()
);
};

30
tsconfig.json Normal file
View File

@ -0,0 +1,30 @@
{
"compilerOptions": {
"module": "esnext",
"forceConsistentCasingInFileNames": true,
"removeComments": true,
"strict": true,
"importHelpers": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"target": "ES2021",
"noImplicitAny": true,
"moduleResolution": "node",
"sourceMap": false,
"resolveJsonModule": true,
"outDir": "dist",
"baseUrl": ".",
"typeRoots": [
"src/@types"
],
"paths": {
"*": [
"node_modules/*",
"src/@types"
]
}
},
"include": [
"src/**/*"
]
}