1
0
forked from notBrad/bubo-rss

Compare commits

..

12 Commits

Author SHA1 Message Date
George Mandis
b0741837d5 Merge branch 'main' into github-action-publishing 2021-05-09 11:49:43 -07:00
George Mandis
92a9012b7e Merge branch 'master' into github-action-publishing 2020-02-04 10:09:44 -05:00
George Mandis
6cf062e42d
Update and rename publish.yml to publish.yml--example
Added "--example" to extension to prevent workflow from running on this repo
2020-02-04 10:05:58 -05:00
George Mandis
6a5dee1f41
Update publish.yml
Commenting out "on" block to prevent trigger
2020-02-04 10:04:35 -05:00
George Mandis
987a0445f1
Update publish.yml
Commenting out the cron scheduling because I don't actually want this to run on the repo
2020-02-04 10:02:29 -05:00
George Mandis
1b665ea9ba Merge branch 'master' into github-action-publishing 2020-02-04 09:44:35 -05:00
George Mandis
72062d5335 Merge branch 'master' into github-action-publishing 2020-02-04 09:43:18 -05:00
George Mandis
6bcb0ca5e1 Merge branch 'master' into github-action-publishing 2020-02-04 09:28:33 -05:00
George Mandis
e805fcc954 Merge branch 'master' into github-action-publishing 2020-02-04 09:01:51 -05:00
George Mandis
2c056603f0 Merge branch 'master' into github-action-publishing 2020-02-04 08:59:48 -05:00
George Mandis
33761d7e20 Updating README 2020-02-04 08:57:48 -05:00
George Mandis
e4a26fef91 Maded changes to GitHub Actions based publishing 2020-02-04 08:45:40 -05:00
24 changed files with 2060 additions and 3625 deletions

View File

@ -1,50 +0,0 @@
{
"env": {
"browser": true,
"es2021": true
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 6,
"sourceType": "module"
},
"plugins": [
"@typescript-eslint"
],
"rules": {
"indent": [
"error",
2
],
"linebreak-style": [
"error",
"unix"
],
"quotes": [
"error",
"double"
],
"semi": [
"error",
"always"
],
"no-trailing-spaces": [
2,
{
"skipBlankLines": false
}
],
"no-multiple-empty-lines": [
"error",
{
"max": 2,
"maxEOF": 1
}
],
"@typescript-eslint/no-var-requires": 0
}
}

4
.github/FUNDING.yml vendored
View File

@ -1,7 +1,7 @@
# These are supported funding model platforms
github: georgemandis
patreon: #
patreon: georgemandis
open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
@ -9,4 +9,4 @@ community_bridge: # Replace with a single Community Bridge project-name e.g., cl
liberapay: # Replace with a single Liberapay username
issuehunt: # Replace with a single IssueHunt username
otechie: # Replace with a single Otechie username
custom: #
custom: https://george.mand.is/sponsor

View File

@ -1,38 +0,0 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS: [e.g. iOS]
- Browser [e.g. chrome, safari]
- Version [e.g. 22]
**Smartphone (please complete the following information):**
- Device: [e.g. iPhone6]
- OS: [e.g. iOS8.1]
- Browser [e.g. stock browser, safari]
- Version [e.g. 22]
**Additional context**
Add any other context about the problem here.

View File

@ -1,20 +0,0 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: ''
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

36
.github/workflows/publish.yml--example vendored Normal file
View File

@ -0,0 +1,36 @@
name: Publish with GitHub Actions
# Uncommen the "on" block below to use
on:
push:
branches:
- master
schedule:
# Run this script every 15 minutes
- cron: '*/15 * * * *'
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout master branch
uses: actions/checkout@v2
with:
ref: master
- name: Setup Node.js and install dependencies
uses: actions/setup-node@v1
with:
node-version: 11.4.0
- run: npm install
- name: Run build and parse RSS feeds
- run: npm run build --if-present
- name: Commit latest build to ./output folder
uses: github-actions-x/commit@v2.3
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
push-branch: 'master'
commit-message: 'Published latest changes to RSS feeds'
force-add: 'true'
files: output/*
name: GitHub Action Bubo Bot
email: action@github.com

5
.gitignore vendored
View File

@ -1,4 +1 @@
node_modules/*
public/index.html
dist/*
.DS_Store
node_modules/*

1
.nvmrc
View File

@ -1 +0,0 @@
18.12.1

View File

@ -1,26 +0,0 @@
# Use an official Node.js runtime as a parent image
FROM node:14
# Set the working directory in the container
WORKDIR /usr/src/app
# Copy package.json and package-lock.json
COPY package*.json ./
# Install any dependencies
RUN npm install
# Bundle the source code inside the Docker container
COPY . .
# Build the project
RUN npm run build:bubo
# Install 'serve' to serve the static site
RUN npm install -g serve
# Make port 5000 available to the world outside this container
EXPOSE 5000
# Run 'serve' to serve the static site on port 5000
CMD ["serve", "-s", "public", "-l", "5000"]

21
LICENSE
View File

@ -1,21 +0,0 @@
MIT License
Copyright (c) 2021 George Mandis
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

126
README.md
View File

@ -1,104 +1,35 @@
[![Netlify Status](https://api.netlify.com/api/v1/badges/81dd219c-51cb-4418-a18c-42c8b104c689/deploy-status)](https://app.netlify.com/sites/bubo-rss-demo/deploys)
# 🦉 Bubo Reader
Bubo Reader is a hyper-minimalist feed reader (RSS, Atom, JSON) you can deploy on your own server, [Netlify](https://netlify.com) in a few steps or [Glitch](https://glitch.com) in even fewer steps! The goal of the project is to generate a webpage that shows a list of links from a collection of feeds organized by category and website. That's it.
Bubo Reader is a somewhat irrationally minimalist <acronym title="Really Simple Syndication">RSS</acronym> and <acronym title="JavaScript Object Notation">JSON</acronym> feed reader you can deploy on [Netlify](https://netlify.com) in a few steps or [Glitch](https://glitch.com) in even fewer steps! The goal of the project is to generate a webpage that shows a list of links from a collection of feeds organized by category and website. That's it.
It is named after this [silly robot owl](https://www.youtube.com/watch?v=MYSeCfo9-NI) from Clash of the Titans (1981).
You can read more about this project on my blog:
You can read more about how this project came about in my blog post '[Introducing Bubo RSS: An Absurdly Minimalist RSS Feed Reader](https://george.mand.is/2019/11/introducing-bubo-rss-an-absurdly-minimalist-rss-feed-reader/)'
- [Introducing Bubo RSS: An Absurdly Minimalist RSS Feed Reader](https://george.mand.is/2019/11/introducing-bubo-rss-an-absurdly-minimalist-rss-feed-reader/).
- [Publishing Bubos RSS to Netlify with GitHub Actions](https://george.mand.is/2020/02/publishing-bubos-rss-to-netlify-with-github-actions/)
## Getting Started
## Get Started
How to deploy Bubo Reader in a few easy steps with Netlify or Glitch:
- Clone or fork the repo and run `npm install` to install the dependencies.
- Update `feeds.json` to include categories and links to feeds you would like to see.
- Run `npm run build:bubo`
That's it! You should now have a static page with links to the latest content from your feeds in the `public` folder, ready to serve.
<details>
<summary>
<strong>Anatomy of Bubo Reader</strong>
</summary>
The static pieces:
- `conf/feeds.json` - a JSON file containing your feed URLS separated into categories.
- `config/template.html` - a [Nunjucks](https://mozilla.github.io/nunjucks/) template that lets you change how the feeds are displayed. This can be changed to anything else you like— see below.
- `public/style.css` - a CSS file to stylize your feed output.
- `public/index.html` - The HTML file that gets automatically generated when Bubo is run.
The engine:
- `src/index.ts` - The primary script you run when you want to build a new version of Bubo. It will automatically fetch the latest content from your feeds and build a new static file at `public/index.html`.
- `src/renderer.ts` — The renderer that loads Nunjucks, the template and understands how to process the incoming feed data. Prefer something else? This is the place to change it!
- `src/utilities.ts` — A variety of parsing and normalization utilities for Bubo, hidden away to try and keep things clean.
</details>
<details>
<summary>
<strong>Throttling</strong>
</summary>
In the main `index.ts` file you will find two values that allow you to batch and throttle your feed requests:
- `MAX_CONNECTIONS` dictates the maximum number of requests a batch can have going at once.
- `DELAY_MS` dictates the amount of delay time between each batch.
The default configuration is **no batching or throttling** because `MAX_CONNECTIONS` is set to `Infinity`. If you wanted to change Bubo to only fetch one feed at a time every second you could set these values to:
```javascript
const MAX_CONNECTIONS = 1;
const DELAY_MS = 1000;
```
If you wanted to limit things to 10 simultaneous requests every 2.5 seconds you could set it like so:
```javascript
const MAX_CONNECTIONS = 10;
const DELAY_MS = 2500;
```
In practice, I've never _really_ run into an issue leaving `MAX_CONNECTIONS` set to `Infinity` but this feels like a sensible safeguard to design.
</details>
<details>
<summary>
<strong>Getting Started</strong>
</summary>
- [Deploying to Glitch](#glitch)
- [Deploying to Netlify](#netlify)
- [Keeping feeds updated](#updated)
<a id="glitch"></a>
## Deploying to Glitch
### Deploying to Glitch
The quickest way is to remix the project on Glitch:
[https://glitch.com/edit/#!/bubo-rss](https://glitch.com/edit/#!/bubo-rss)
There is also a `glitch` branch on this repo if you'd prefer to start there.
Just changed some feeds in `./src/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./output/style.css` file or the `./src/template.html` file respectively.
Just change some feeds in `./config/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./public/style.css` file or the `./config/template.html` file respectively.
There is also a special `glitch` branch you can clone if you prefer:
[https://github.com/georgemandis/bubo-rss/tree/glitch](https://github.com/georgemandis/bubo-rss/tree/glitch)
<a id="netlify"></a>
The only difference between this branch and `master` is that it spins up a server using [Express](https://expressjs.com/) to serve your `./output/index.html` file on Glitch. Everything else is the same.
## Deploying to Netlify
### Deploying to Netlify
- [Fork the repository](https://github.com/georgemandis/bubo-rss/fork)
- From your forked repository edit `config/feeds.json` to manage your feeds and categories
- [Create a new site](https://app.netlify.com/start) on Netlify from GitHub
- From your forked repository go to and edcit `src/feeds.json` to manage your feeds and categories
- [Create a new site](https://app.netlify.com/start) on Netlify from GitHub
The deploy settings should automatically import from the `netlify.toml` file. All you'll need to do is confirm and you're ready to go!
<a id="updated"></a>
### Keeping Feeds Updated
#### Using Netlify Webhooks
@ -109,11 +40,24 @@ To keep your feeds up to date you'll want to [setup a Build Hook](https://www.ne
- [Zapier](https://zapier.com/)
- [EasyCron](https://www.easycron.com/)
#### Rolling Your Own
If you already have a server running Linux and some command-line experience it might be simpler to setup a [cron job](https://en.wikipedia.org/wiki/Cron).
If you already have a server running Linux and some command-line experience it might be simpler to setup a [cron job](https://en.wikipedia.org/wiki/Cron).
#### Using GitHub Actions
</details>
This approach is a little different and requires some modifications to the repository. Netlify started billing for [build minutes](https://www.netlify.com/pricing/faq/) very shortly after I published this project. Running `npm build` and downloading all of the RSS feeds took up a substantial number of this minutes, particulary if you had some kind of process pinging the webhook and trigger a build every 15 minutes or so.
How is the The GitHub Action-based approach different? The same build process runs, but this time it's on GitHub's servers via the Action. It then **commits** the newly created file generated at `./output/index.html` back into the repository. Netlify still gets pinged when the repository is updated, but skips the `npm run build` step on their end, which significantly reduces the number of build minutes required.
**Short Answer**: use the [`github-action-publishing`](https://github.com/georgemandis/bubo-rss/tree/github-action-publishing) branch for now if you'd prefer to use GitHub Actions to run your builds.
The GitHub Action is setup to build and commit directly to the `master` branch, which is not the best practice. I'd suggest creating a separate branch to checkout and commit changes to in the Action. You could then specify that same branch as the one to checkout and publish on Netlify.
## Anatomy of Bubo Reader
- `src/index.html` - a [Nunjucks](https://mozilla.github.io/nunjucks/) template that lets you change how the feeds are displayed
- `output/style.css` - a CSS file to stylize your feed output
- `src/feeds.json` - a JSON file containing the URLs for various site's feeds separated into categories
- `src/index.js` - the script that loads the feeds and does the actual parsinga and rendering
## Demos
@ -122,16 +66,8 @@ You can view live demos here:
- [https://bubo-rss-demo.netlify.com/](https://bubo-rss-demo.netlify.com/)
- [http://bubo-rss.glitch.me/](http://bubo-rss.glitch.me/)
Not the most exciting-looking demos, I'll admit, but they work!
## Support
If you found this useful please consider [sponsoring me or this project](https://github.com/sponsors/georgemandis).
If you'd rather run this on your own server please consider using one of these affiliate links to setup a micro instance on [Linode](https://www.linode.com/?r=8729957ab02b50a695dcea12a5ca55570979d8b9), [Digital Ocean](https://m.do.co/c/31f58d367777) or [Vultr](https://www.vultr.com/?ref=8403978).
## Showcase
Here are some websites using Bubo Reader:
- [Kevin Fiol](https://kevinfiol.com/reader/) ([repo](https://github.com/kevinfiol/reader))
Please share if you would like to be featured!
If you found this useful please consider sponsoring me or this project. If you'd rather run this on your own server please consider using one of these affiliate links to setup a micro instance on [Linode](https://www.linode.com/?r=8729957ab02b50a695dcea12a5ca55570979d8b9), [Digital Ocean](https://m.do.co/c/31f58d367777) or [Vultr](https://www.vultr.com/?ref=8403978).

View File

@ -1,56 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta http-equiv="X-UA-Compatible" content="ie=edge" />
<title>🦉 Bubo Reader</title>
<link rel="stylesheet" href="/style.css" />
</head>
<body>
<h1>🦉 Bubo Reader</h1>
{% for group, feeds in data %}
<h2>{{ group }}</h2>
{% for feed in feeds %}
<details>
<summary>
<span class="feed-title">{{ feed.title }}</span>
<span class="feed-url">({{ feed.feed }})</span>
</summary>
<ul>
{% for item in feed.items %}
<li>
{{ item.timestamp | formatDate }} -
<a
href="{{ item.link }}"
target="_blank"
rel="noopener norefferer nofollow"
>{{ item.title }}</a
>
</li>
{% endfor %}
</ul>
</details>
{% endfor %} {% endfor %} {% if errors | length > 0 %}
<h2>Errors</h2>
<p>There were errors trying to parse these feeds:</p>
<ul>
{% for error in errors %}
<li>{{ error }}</li>
{% endfor %}
</ul>
{% endif %}
<br />
<hr />
<p>Last updated {{ now }}.</p>
<p>
Powered by
<a href="https://github.com/georgemandis/bubo-rss"
>Bubo Reader (v{{ info.version }})</a
>, a project by <a href="https://george.mand.is">George Mandis</a>. ❤️
<a href="{{ info.funding.url }}">Sponsor on GitHub</a>
</p>
</body>
</html>

View File

@ -1,4 +1,3 @@
[build]
command = "npm run build:bubo"
publish = "./public/"
[build]
publish = "output/"

30
output/style.css Normal file
View File

@ -0,0 +1,30 @@
body {
font-family:system-ui;
font-size: 18px;
}
details:focus,
details:focus-within,
details:hover {
/* background:#ffeb3b; */
/* outline:2px #000 solid; */
}
details ul li {
}
summary {
cursor:pointer;
}
summary:hover {
opacity:.75;
}
.feed-url {
color:#aaa;
}

4690
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,45 +1,17 @@
{
"name": "bubo-reader",
"version": "2.0.2",
"description": "A simple but effective feed reader (RSS, JSON)",
"homepage": "https://github.com/georgemandis/bubo-rss",
"version": "1.0.0",
"description": "A somewhat dumb but effective feed reader (RSS, JSON & Twitter)",
"main": "src/index.js",
"type": "module",
"scripts": {
"dev": "tsc --watch",
"clean": "rm -rf dist",
"build": "tsc",
"bubo": "node dist/index.js",
"build:bubo": "tsc && node dist/index.js"
"build": "node src/index.js > output/index.html",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": {
"name": "George Mandis",
"email": "george@mand.is",
"url": "https://george.mand.is"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/georgemandis"
},
"bugs": {
"url": "https://github.com/georgemandis/bubo-rss/issues",
"email": "george+bubo@mand.is"
},
"license": "MIT",
"author": "",
"license": "ISC",
"dependencies": {
"chalk": "^5.2.0",
"node-fetch": "^3.3.1",
"nunjucks": "^3.2.4",
"rss-parser": "^3.13.0"
},
"devDependencies": {
"@types/node": "^20.2.5",
"@types/nunjucks": "^3.2.2",
"@types/xml2js": "^0.4.11",
"@typescript-eslint/eslint-plugin": "^5.59.8",
"@typescript-eslint/parser": "^5.59.8",
"eslint": "^8.42.0",
"tslib": "^2.5.3",
"typescript": "^5.1.3"
"node-fetch": "^2.6.1",
"nunjucks": "^3.2.0",
"rss-parser": "^3.6.3"
}
}

View File

@ -1,28 +0,0 @@
body {
font-family: system-ui;
font-size: 18px;
}
details:focus,
details:focus-within,
details:hover {
/* background:#ffeb3b; */
/* outline:2px #000 solid; */
}
details ul li {}
summary {
cursor: pointer;
}
summary:hover {
opacity: .75;
}
.feed-url {
color: #aaa;
}

15
src/@types/bubo.d.ts vendored
View File

@ -1,15 +0,0 @@
export interface Feeds {
[key: string]: object[]
}
export interface FeedItem {
[key: string]: string | number | Date | FeedItem[];
items: FeedItem[]
}
//NEW WAY
export type JSONValue =
| string
| number
| boolean
| { [x: string]: JSONValue }
| Array<JSONValue>;

View File

@ -1,12 +1,12 @@
{
"Developer News": [
"Web Development": [
"https://hacks.mozilla.org/feed/",
"https://blog.mozilla.org/feed/",
"https://web.dev/feed.xml",
"https://v8.dev/blog.atom",
"https://alistapart.com/main/feed/",
"https://css-tricks.com/feed/",
"https://dev.to/feed",
"https://changelog.com/feed"
"https://dev.to/feed"
],
"Blogs": [
"https://george.mand.is/feed.xml",
@ -14,10 +14,11 @@
],
"My GitHub Projects": [
"https://github.com/georgemandis.atom",
"https://github.com/georgemandis/bubo-rss/releases.atom",
"https://github.com/georgemandis/konami-js/releases.atom",
"https://github.com/georgemandis/konami-js/commits/main.atom",
"https://github.com/snaptortoise/konami-js/releases.atom",
"https://github.com/snaptortoise/konami-js/commits/master.atom",
"https://github.com/javascriptforartists/cheer-me-up-and-sing-me-a-song/commits/master.atom",
"https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom"
"https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom",
"https://github.com/georgemandis/remote-working-list/commits/master.atom",
"https://github.com/georgemandis/tweeter-totter/commits/master.atom"
]
}

89
src/index.js Normal file
View File

@ -0,0 +1,89 @@
/**
* 🦉 Bubo RSS Reader
* ====
* Dead, dead simple feed reader that renders an HTML
* page with links to content from feeds organized by site
*
*/
const fetch = require("node-fetch");
const Parser = require("rss-parser");
const parser = new Parser();
const nunjucks = require("nunjucks");
const env = nunjucks.configure({ autoescape: true });
const feeds = require("./feeds.json");
env.addFilter("formatDate", function(dateString) {
const formattedDate = new Date(dateString).toLocaleDateString()
return formattedDate !== 'Invalid Date' ? formattedDate : dateString;
});
env.addGlobal('now', (new Date()).toUTCString() );
// parse XML or JSON feeds
function parseFeed(response) {
const contentType = response.headers.get("content-type")
? response.headers.get("content-type").split(";")[0]
: false;
const rssFeed = [contentType]
.map(item =>
[
"application/atom+xml",
"application/rss+xml",
"application/xml",
"text/xml",
"text/html" // this is kind of a gamble
].includes(item)
? response.text()
: false
)
.filter(_ => _)[0];
const jsonFeed = [contentType]
.map(item =>
["application/json"].includes(item) ? response.json() : false
)
.filter(_ => _)[0];
return rssFeed || jsonFeed || false;
}
(async () => {
const contentFromAllFeeds = {};
const errors = [];
for (group in feeds) {
contentFromAllFeeds[group] = [];
for (let index = 0; index < feeds[group].length; index++) {
try {
const response = await fetch(feeds[group][index]);
const body = await parseFeed(response);
const contents =
typeof body === "string" ? await parser.parseString(body) : body;
contents.feed = feeds[group][index];
contents.title = contents.title ? contents.title : contents.link;
contentFromAllFeeds[group].push(contents);
// try to normalize date attribute naming
contents.items.forEach(item => {
const timestamp = new Date(item.pubDate || item.isoDate || item.date).getTime();
item.timestamp = isNaN(timestamp) ? (item.pubDate || item.isoDate || item.date) : timestamp;
});
} catch (error) {
errors.push(feeds[group][index]);
}
}
}
const output = env.render("./src/template.html", {
data: contentFromAllFeeds,
errors: errors
});
console.log(output);
})();

View File

@ -1,172 +0,0 @@
/*
* 🦉 Bubo Reader
* ====
* Dead simple feed reader (RSS + JSON) that renders an HTML
* page with links to content from feeds organized by site
*
* Code: https://github.com/georgemandis/bubo-rss
* Copyright (c) 2019 George Mandis (https://george.mand.is)
* Version: 1.0.1 (11/14/2021)
* Licensed under the MIT License (http://opensource.org/licenses/MIT)
*/
import fetch from "node-fetch";
import Parser from "rss-parser";
import { Feeds, FeedItem } from "./@types/bubo";
import { Response } from "node-fetch";
import { render } from "./renderer.js";
import {
getLink,
getTitle,
getTimestamp,
parseFeed,
getFeedList,
getBuboInfo
} from "./utilities.js";
import { writeFile } from "fs/promises";
import chalk from "chalk";
const buboInfo = await getBuboInfo();
const parser = new Parser();
const feedList = await getFeedList();
const feedListLength =
Object.entries(feedList).flat(2).length - Object.keys(feedList).length;
/**
* contentFromAllFeeds = Contains normalized, aggregated feed data and is passed to template renderer at the end
* errors = Contains errors from parsing feeds and is also passed to template.
*/
const contentFromAllFeeds: Feeds = {};
const errors: unknown[] = [];
// benchmarking data + utility
const initTime = Date.now();
const benchmark = (startTime: number) =>
chalk.cyanBright.bold(`${(Date.now() - startTime) / 1000} seconds`);
/**
* These values are used to control throttling/batching the fetches:
* - MAX_CONNECTION = max number of fetches to contain in a batch
* - DELAY_MS = the delay in milliseconds between batches
*/
const MAX_CONNECTIONS = Infinity;
const DELAY_MS = 850;
const error = chalk.bold.red;
const success = chalk.bold.green;
// keeping tally of total feeds fetched and parsed so we can compare
// to feedListLength and know when we're finished.
let completed = 0;
/**
* finishBuild
* --
* function that gets called when all the feeds are through fetching
* and we want to build the static output.
*/
const finishBuild: () => void = async () => {
completed++;
// if this isn't the last feed, just return early
if (completed !== feedListLength) return;
process.stdout.write("\nDone fetching everything!\n");
// generate the static HTML output from our template renderer
const output = render({
data: contentFromAllFeeds,
errors: errors,
info: buboInfo
});
// write the output to public/index.html
await writeFile("./public/index.html", output);
process.stdout.write(
`\nFinished writing to output:\n- ${feedListLength} feeds in ${benchmark(
initTime
)}\n- ${errors.length} errors\n`
);
};
/**
* processFeed
* --
* Process an individual feed and normalize its items
* @param { group, feed, startTime}
* @returns Promise<void>
*/
const processFeed =
({
group,
feed,
startTime
}: {
group: string;
feed: string;
startTime: number;
}) =>
async (response: Response): Promise<void> => {
const body = await parseFeed(response);
//skip to the next one if this didn't work out
if (!body) return;
try {
const contents: FeedItem = (
typeof body === "string" ? await parser.parseString(body) : body
) as FeedItem;
contents.feed = feed;
contents.title = getTitle(contents);
contents.link = getLink(contents);
// try to normalize date attribute naming
contents?.items?.forEach(item => {
item.timestamp = getTimestamp(item);
item.title = getTitle(item);
item.link = getLink(item);
});
contentFromAllFeeds[group].push(contents as object);
process.stdout.write(
`${success("Successfully fetched:")} ${feed} - ${benchmark(startTime)}\n`
);
} catch (err) {
process.stdout.write(
`${error("Error processing:")} ${feed} - ${benchmark(
startTime
)}\n${err}\n`
);
errors.push(`Error processing: ${feed}\n\t${err}`);
}
finishBuild();
};
// go through each group of feeds and process
const processFeeds = () => {
let idx = 0;
for (const [group, feeds] of Object.entries(feedList)) {
contentFromAllFeeds[group] = [];
for (const feed of feeds) {
const startTime = Date.now();
setTimeout(() => {
process.stdout.write(`Fetching: ${feed}...\n`);
fetch(feed)
.then(processFeed({ group, feed, startTime }))
.catch(err => {
process.stdout.write(
error(`Error fetching ${feed} ${benchmark(startTime)}\n`)
);
errors.push(`Error fetching ${feed} ${err.toString()}\n`);
finishBuild();
});
}, (idx % (feedListLength / MAX_CONNECTIONS)) * DELAY_MS);
idx++;
}
}
};
processFeeds();

View File

@ -1,44 +0,0 @@
/*
* Return our renderer.
* Using Nunjucks out of the box.
* https://mozilla.github.io/nunjucks/
*/
import nunjucks from "nunjucks";
const env: nunjucks.Environment = nunjucks.configure({ autoescape: true });
import { readFile } from "fs/promises";
import { Feeds, JSONValue } from "./@types/bubo";
/**
* Global filters for my Nunjucks templates
*/
env.addFilter("formatDate", function (dateString): string {
const date: Date = new Date(parseInt(dateString));
return !isNaN(date.getTime()) ? date.toLocaleDateString() : dateString;
});
env.addGlobal("now", new Date().toUTCString());
// load the template
const template: string = (
await readFile(new URL("../config/template.html", import.meta.url))
).toString();
// generate the static HTML output from our template renderer
const render = ({
data,
errors,
info
}: {
data: Feeds;
errors: unknown[];
info?: JSONValue;
}) => {
return env.renderString(template, {
data,
errors,
info
});
};
export { render };

48
src/template.html Normal file
View File

@ -0,0 +1,48 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>🦉 Bubo Reader</title>
<link rel="stylesheet" href="/style.css">
</head>
<body>
<h1>🦉 Bubo Reader</h1>
{% for group, feeds in data %}
<h2>{{ group }}</h2>
{% for feed in feeds %}
<details>
<summary>
<span class="feed-title">{{ feed.title }}</span>
<span class="feed-url">({{ feed.feed }})</span>
</summary>
<ul>
{% for item in feed.items %}
<li>
{{ item.timestamp | formatDate }} - <a href="{{ item.link }}" target='_blank' rel='noopener norefferer nofollow'>{{ item.title }}</a>
</li>
{% endfor %}
</ul>
</details>
{% endfor %}
{% endfor %}
{% if errors | length > 0 %}
<h2>Errors</h2>
<p>There were errors trying to parse these feeds:</p>
<ul>
{% for error in errors %}
<li>{{ error }}</li>
{% endfor %}
</ul>
{% endif %}
<hr>
<p>
Last updated {{ now }}. Powered by <a href="https://github.com/georgemandis/bubo-rss">Bubo Reader</a>, a project by <a href="https://george.mand.is">George Mandis</a>
</p>
</body>
</html>

View File

@ -1,90 +0,0 @@
/*
There's a little inconsistency with how feeds report certain things like
title, links and timestamps. These helpers try to normalize that bit and
provide an order-of-operations list of properties to look for.
Note: these are tightly-coupled to the template and a personal preference.
*/
import { Response } from "node-fetch";
import { readFile } from "fs/promises";
import { FeedItem, JSONValue } from "./@types/bubo";
export const getLink = (obj: FeedItem): string => {
const link_values: string[] = ["link", "url", "guid", "home_page_url"];
const keys: string[] = Object.keys(obj);
const link_property: string | undefined = link_values.find(link_value =>
keys.includes(link_value)
);
return link_property ? (obj[link_property] as string) : "";
};
// fallback to URL for the title if not present
// (title -> url -> link)
export const getTitle = (obj: FeedItem): string => {
const title_values: string[] = ["title", "url", "link"];
const keys: string[] = Object.keys(obj);
// if title is empty for some reason, fall back on url or link
const title_property: string | undefined = title_values.find(
title_value => keys.includes(title_value) && obj[title_value]
);
return title_property ? (obj[title_property] as string) : "";
};
// More dependable way to get timestamps
export const getTimestamp = (obj: FeedItem): string => {
const dateString: string = (
obj.pubDate ||
obj.isoDate ||
obj.date ||
obj.date_published
).toString();
const timestamp: number = new Date(dateString).getTime();
return isNaN(timestamp) ? dateString : timestamp.toString();
};
// parse RSS/XML or JSON feeds
export async function parseFeed(response: Response): Promise<JSONValue> {
const contentType = response.headers.get("content-type")?.split(";")[0];
if (!contentType) return {};
const rssFeed = [contentType]
.map(item =>
[
"application/atom+xml",
"application/rss+xml",
"application/xml",
"text/xml",
"text/html" // this is kind of a gamble
].includes(item)
? response.text()
: false
)
.filter(_ => _)[0];
const jsonFeed = [contentType]
.map(item =>
["application/json", "application/feed+json"].includes(item)
? (response.json() as Promise<JSONValue>)
: false
)
.filter(_ => _)[0];
return (rssFeed && rssFeed) || (jsonFeed && jsonFeed) || {};
}
export const getFeedList = async (): Promise<JSONValue> => {
return JSON.parse(
(
await readFile(new URL("../config/feeds.json", import.meta.url))
).toString()
);
};
export const getBuboInfo = async (): Promise<JSONValue> => {
return JSON.parse(
(await readFile(new URL("../package.json", import.meta.url))).toString()
);
};

View File

@ -1,30 +0,0 @@
{
"compilerOptions": {
"module": "esnext",
"forceConsistentCasingInFileNames": true,
"removeComments": true,
"strict": true,
"importHelpers": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"target": "ES2021",
"noImplicitAny": true,
"moduleResolution": "node",
"sourceMap": false,
"resolveJsonModule": true,
"outDir": "dist",
"baseUrl": ".",
"typeRoots": [
"src/@types"
],
"paths": {
"*": [
"node_modules/*",
"src/@types"
]
}
},
"include": [
"src/**/*"
]
}