mirror of
https://github.com/georgemandis/bubo-rss.git
synced 2024-11-23 12:24:23 +00:00
Merging 2.0 in and tweaking for Glitch
This commit is contained in:
George Mandis
commit
1a9c60a4d6
50
.eslintrc.json
Normal file
50
.eslintrc.json
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
{
|
||||||
|
"env": {
|
||||||
|
"browser": true,
|
||||||
|
"es2021": true
|
||||||
|
},
|
||||||
|
"extends": [
|
||||||
|
"eslint:recommended",
|
||||||
|
"plugin:@typescript-eslint/recommended"
|
||||||
|
],
|
||||||
|
"parser": "@typescript-eslint/parser",
|
||||||
|
"parserOptions": {
|
||||||
|
"ecmaVersion": 6,
|
||||||
|
"sourceType": "module"
|
||||||
|
},
|
||||||
|
"plugins": [
|
||||||
|
"@typescript-eslint"
|
||||||
|
],
|
||||||
|
"rules": {
|
||||||
|
"indent": [
|
||||||
|
"error",
|
||||||
|
2
|
||||||
|
],
|
||||||
|
"linebreak-style": [
|
||||||
|
"error",
|
||||||
|
"unix"
|
||||||
|
],
|
||||||
|
"quotes": [
|
||||||
|
"error",
|
||||||
|
"double"
|
||||||
|
],
|
||||||
|
"semi": [
|
||||||
|
"error",
|
||||||
|
"always"
|
||||||
|
],
|
||||||
|
"no-trailing-spaces": [
|
||||||
|
2,
|
||||||
|
{
|
||||||
|
"skipBlankLines": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"no-multiple-empty-lines": [
|
||||||
|
"error",
|
||||||
|
{
|
||||||
|
"max": 2,
|
||||||
|
"maxEOF": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"@typescript-eslint/no-var-requires": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
4
.github/FUNDING.yml
vendored
4
.github/FUNDING.yml
vendored
@ -1,7 +1,7 @@
|
|||||||
# These are supported funding model platforms
|
# These are supported funding model platforms
|
||||||
|
|
||||||
github: georgemandis
|
github: georgemandis
|
||||||
patreon: georgemandis
|
patreon: #
|
||||||
open_collective: # Replace with a single Open Collective username
|
open_collective: # Replace with a single Open Collective username
|
||||||
ko_fi: # Replace with a single Ko-fi username
|
ko_fi: # Replace with a single Ko-fi username
|
||||||
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
||||||
@ -9,4 +9,4 @@ community_bridge: # Replace with a single Community Bridge project-name e.g., cl
|
|||||||
liberapay: # Replace with a single Liberapay username
|
liberapay: # Replace with a single Liberapay username
|
||||||
issuehunt: # Replace with a single IssueHunt username
|
issuehunt: # Replace with a single IssueHunt username
|
||||||
otechie: # Replace with a single Otechie username
|
otechie: # Replace with a single Otechie username
|
||||||
custom: https://george.mand.is/sponsor
|
custom: #
|
||||||
|
|||||||
4
.gitignore
vendored
4
.gitignore
vendored
@ -1,2 +1,4 @@
|
|||||||
node_modules/*
|
node_modules/*
|
||||||
output/index.html
|
public/index.html
|
||||||
|
dist/*
|
||||||
|
.DS_Store
|
||||||
|
|||||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2021 George Mandis
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
121
README.md
121
README.md
@ -1,35 +1,102 @@
|
|||||||
# 🦉 Bubo Reader
|
# 🦉 Bubo Reader (2.0)
|
||||||
|
|
||||||
Bubo Reader is a somewhat irrationally minimalist <acronym title="Really Simple Syndication">RSS</acronym> and <acronym title="JavaScript Object Notation">JSON</acronym> feed reader you can deploy on [Netlify](https://netlify.com) in a few steps or [Glitch](https://glitch.com) in even fewer steps! The goal of the project is to generate a webpage that shows a list of links from a collection of feeds organized by category and website. That's it.
|
Bubo Reader is a hyper-minimalist <acronym title="Really Simple Syndication">RSS</acronym> and <acronym title="JavaScript Object Notation">JSON</acronym> feed reader you can deploy on your own server, [Netlify](https://netlify.com) in a few steps or [Glitch](https://glitch.com) in even fewer steps! The goal of the project is to generate a webpage that shows a list of links from a collection of feeds organized by category and website. That's it.
|
||||||
|
|
||||||
It is named after this [silly robot owl](https://www.youtube.com/watch?v=MYSeCfo9-NI) from Clash of the Titans (1981).
|
It is named after this [silly robot owl](https://www.youtube.com/watch?v=MYSeCfo9-NI) from Clash of the Titans (1981).
|
||||||
|
|
||||||
You can read more about how this project came about in my blog post '[Introducing Bubo RSS: An Absurdly Minimalist RSS Feed Reader](https://george.mand.is/2019/11/introducing-bubo-rss-an-absurdly-minimalist-rss-feed-reader/)'
|
You can read more about how this project came about on my blog:
|
||||||
|
- [Introducing Bubo RSS: An Absurdly Minimalist RSS Feed Reader](https://george.mand.is/2019/11/introducing-bubo-rss-an-absurdly-minimalist-rss-feed-reader/).
|
||||||
|
|
||||||
## Getting Started
|
## Getting Started
|
||||||
|
|
||||||
How to deploy Bubo Reader in a few easy steps with Netlify or Glitch:
|
- Clone or fork the repo and run `npm install` to install the dependencies.
|
||||||
|
- Update `feeds.json` to include categories and links to feeds you would like to see.
|
||||||
|
- Run `npm run build:bubo`
|
||||||
|
|
||||||
### Deploying to Glitch
|
That's it! You should now have a static page with links to the latest content from your feeds in the `public` folder, ready to serve.
|
||||||
|
|
||||||
|
## Differences in Bubo 2.0
|
||||||
|
|
||||||
|
Version 2.0 has introduced some substantial changes for Bubo! While the static output remains endearingly spartan, the engine that builds it has changed a bit.
|
||||||
|
|
||||||
|
Hopefully all of these changes are in services of making this project more useful to others and encouraging outside contributions.
|
||||||
|
|
||||||
|
Changes of note:
|
||||||
|
|
||||||
|
- Bubo has been rewritten in [TypeScript](https://www.typescriptlang.org/). It's pretty slick! I anticipate the typing could be improved, but it's a start.
|
||||||
|
- You fill find an `.nvmrc` file in the root of this project. Learn more [about nvm](https://github.com/nvm-sh/nvm) if you're unfamiliar.
|
||||||
|
- The script will actually write your `index.html` file for you (Previously the build script simply ran `node src/index.js > output/index.html`). It makes a strong assumption that this file lives in the `public` folder.
|
||||||
|
- There is a somewhat sophisticated mechansim in-place for batching & throttling your requests, if needed.
|
||||||
|
## Anatomy of Bubo Reader
|
||||||
|
|
||||||
|
The static pieces:
|
||||||
|
|
||||||
|
- `conf/feeds.json` - a JSON file containing your feed URLS separated into categories.
|
||||||
|
- `config/template.html` - a [Nunjucks](https://mozilla.github.io/nunjucks/) template that lets you change how the feeds are displayed. This can be changed to anything else you like— see below.
|
||||||
|
- `public/style.css` - a CSS file to stylize your feed output.
|
||||||
|
- `public/index.html` - The HTML file that gets automatically generated when Bubo is run.
|
||||||
|
|
||||||
|
The engine:
|
||||||
|
|
||||||
|
- `src/index.ts` - The primary script you run when you want to build a new version of Bubo. It will automatically fetch the latest content from your feeds and build a new static file at `public/index.html`.
|
||||||
|
- `src/renderer.ts` — The renderer that loads Nunjucks, the template and understands how to process the incoming feed data. Prefer something else? This is the place to change it!
|
||||||
|
- `src/utilities.ts` — A variety of parsing and normalization utilities for Bubo, hidden away to try and keep things clean.
|
||||||
|
|
||||||
|
## Throttling
|
||||||
|
|
||||||
|
In the main `index.ts` file you will find two values that allow yout to batch and throttle your feed requests:
|
||||||
|
|
||||||
|
- `MAX_CONNECTIONS` dictates the maximium number of requests a batch can have going at once.
|
||||||
|
- `DELAY_MS` dictates the amount of de;ay time between each batch.
|
||||||
|
|
||||||
|
The default configuration is **no batching or throttling** beacuse `MAX_CONNECTIONS` is set to `Infinity`. If you wanted to change Bubo to only fetch one feed at a time every second you could set these values to:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
const MAX_CONNECTIONS = 1;
|
||||||
|
const DELAY_MS = 1000;
|
||||||
|
```
|
||||||
|
|
||||||
|
If you wanted to limit things to 10 simultaneous requests every 2.5 seconds you could set it like so:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
const MAX_CONNECTIONS = 10;
|
||||||
|
const DELAY_MS = 2500;
|
||||||
|
```
|
||||||
|
|
||||||
|
In practice, I've never _really_ run into an issue leaving `MAX_CONNECTIONS` set to `Infinity` but this feels like a sensible safeguard to design.
|
||||||
|
## Demos
|
||||||
|
|
||||||
|
You can view live demos here:
|
||||||
|
|
||||||
|
- [https://bubo-rss-demo.netlify.com/](https://bubo-rss-demo.netlify.com/)
|
||||||
|
- [http://bubo-rss.glitch.me/](http://bubo-rss.glitch.me/)
|
||||||
|
|
||||||
|
Not the most exciting-looking demos, I'll admit, but they work!
|
||||||
|
|
||||||
|
**Getting Started**
|
||||||
|
|
||||||
|
- [Deploying to Glitch](#glitch)
|
||||||
|
- [Deploying to Netlify](#netlify)
|
||||||
|
- [Keeping feeds updated](#updated)
|
||||||
|
|
||||||
|
<a id="glitch"></a>
|
||||||
|
## Deploying to Glitch
|
||||||
|
|
||||||
The quickest way is to remix the project on Glitch:
|
The quickest way is to remix the project on Glitch:
|
||||||
[https://glitch.com/edit/#!/bubo-rss](https://glitch.com/edit/#!/bubo-rss)
|
[https://glitch.com/edit/#!/bubo-rss](https://glitch.com/edit/#!/bubo-rss)
|
||||||
|
|
||||||
Just changed some feeds in `./src/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./output/style.css` file or the `./src/template.html` file respectively.
|
Just changed some feeds in `./config/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./public/style.css` file or the `./config/template.html` file respectively.
|
||||||
|
|
||||||
There is also a special `glitch` branch you can clone if you prefer:
|
<a id="netlify"></a>
|
||||||
[https://github.com/georgemandis/bubo-rss/tree/glitch](https://github.com/georgemandis/bubo-rss/tree/glitch)
|
## Deploying to Netlify
|
||||||
|
|
||||||
The only difference between this branch and `master` is that it spins up a server using [Express](https://expressjs.com/) to serve your `./output/index.html` file on Glitch. Everything else is the same.
|
|
||||||
|
|
||||||
### Deploying to Netlify
|
|
||||||
|
|
||||||
- [Fork the repository](https://github.com/georgemandis/bubo-rss/fork)
|
- [Fork the repository](https://github.com/georgemandis/bubo-rss/fork)
|
||||||
- From your forked repository go to and edcit `src/feeds.json` to manage your feeds and categories
|
- From your forked repository edit `config/feeds.json` to manage your feeds and categories
|
||||||
- [Create a new site](https://app.netlify.com/start) on Netlify from GitHub
|
- [Create a new site](https://app.netlify.com/start) on Netlify from GitHub
|
||||||
|
|
||||||
The deploy settings should automatically import from the `netlify.toml` file. All you'll need to do is confirm and you're ready to go!
|
The deploy settings should automatically import from the `netlify.toml` file. All you'll need to do is confirm and you're ready to go!
|
||||||
|
|
||||||
|
<a id="updated"></a>
|
||||||
### Keeping Feeds Updated
|
### Keeping Feeds Updated
|
||||||
|
|
||||||
#### Using Netlify Webhooks
|
#### Using Netlify Webhooks
|
||||||
@ -40,25 +107,21 @@ To keep your feeds up to date you'll want to [setup a Build Hook](https://www.ne
|
|||||||
- [Zapier](https://zapier.com/)
|
- [Zapier](https://zapier.com/)
|
||||||
- [EasyCron](https://www.easycron.com/)
|
- [EasyCron](https://www.easycron.com/)
|
||||||
|
|
||||||
If you already have a server running Linux and some command-line experience it might be simpler to setup a [cron job](https://en.wikipedia.org/wiki/Cron).
|
|
||||||
|
|
||||||
#### Using GitHub Actions
|
#### Using GitHub Actions
|
||||||
|
|
||||||
This approach is a little different and requires some modifications to the repository. Netlify started billing for [build minutes](https://www.netlify.com/pricing/faq/) very shortly after I published this project. Running `npm build` and downloading all of the RSS feeds took up a substantial number of this minutes, particulary if you had some kind of process pinging the webhook and trigger a build every 15 minutes or so.
|
Coming soon—there is an old branch that demonstrates this, but it needs to be revisisted in light of Bubo 2.0.
|
||||||
|
|
||||||
How is the The GitHub Action-based approach different? The same build process runs, but this time it's on GitHub's servers via the Action. It then **commits** the newly created file generated at `./output/index.html` back into the repository. Netlify still gets pinged when the repository is updated, but skips the `npm run build` step on their end, which significantly reduces the number of build minutes required.
|
#### Rolling Your Own
|
||||||
|
|
||||||
**Short Answer**: use the [`github-action-publishing`](https://github.com/georgemandis/bubo-rss/tree/github-action-publishing) branch for now if you'd prefer to use GitHub Actions to run your builds.
|
If you already have a server running Linux and some command-line experience it might be simpler to setup a [cron job](https://en.wikipedia.org/wiki/Cron).
|
||||||
|
|
||||||
The GitHub Action is setup to build and commit directly to the `master` branch, which is not the best practice. I'd suggest creating a separate branch to checkout and commit changes to in the Action. You could then specify that same branch as the one to checkout and publish on Netlify.
|
|
||||||
|
|
||||||
## Anatomy of Bubo Reader
|
|
||||||
|
|
||||||
- `src/index.html` - a [Nunjucks](https://mozilla.github.io/nunjucks/) template that lets you change how the feeds are displayed
|
|
||||||
- `output/style.css` - a CSS file to stylize your feed output
|
|
||||||
- `src/feeds.json` - a JSON file containing the URLs for various site's feeds separated into categories
|
|
||||||
- `src/index.js` - the script that loads the feeds and does the actual parsinga and rendering
|
|
||||||
|
|
||||||
## Support
|
## Support
|
||||||
|
|
||||||
If you found this useful please consider sponsoring me or this project. If you'd rather run this on your own server please consider using one of these affiliate links to setup a micro instance on [Linode](https://www.linode.com/?r=8729957ab02b50a695dcea12a5ca55570979d8b9), [Digital Ocean](https://m.do.co/c/31f58d367777) or [Vultr](https://www.vultr.com/?ref=8403978).
|
If you found this useful please consider sponsoring me or this project.
|
||||||
|
|
||||||
|
If you'd rather run this on your own server please consider using one of these affiliate links to setup a micro instance on [Linode](https://www.linode.com/?r=8729957ab02b50a695dcea12a5ca55570979d8b9), [Digital Ocean](https://m.do.co/c/31f58d367777) or [Vultr](https://www.vultr.com/?ref=8403978).
|
||||||
|
|
||||||
|
## Showcase
|
||||||
|
|
||||||
|
Here are some websites using Bubo Reader:
|
||||||
|
- [Kevin Fiol](https://kevinfiol.com/reader/) ([repo](https://github.com/kevinfiol/reader))
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
{
|
{
|
||||||
"Web Development": [
|
"Web Development": [
|
||||||
"https://hacks.mozilla.org/feed/",
|
"https://hacks.mozilla.org/feed/",
|
||||||
"https://blog.mozilla.org/feed/",
|
|
||||||
"https://web.dev/feed.xml",
|
"https://web.dev/feed.xml",
|
||||||
"https://v8.dev/blog.atom",
|
"https://v8.dev/blog.atom",
|
||||||
"https://alistapart.com/main/feed/",
|
"https://alistapart.com/main/feed/",
|
||||||
@ -14,11 +13,10 @@
|
|||||||
],
|
],
|
||||||
"My GitHub Projects": [
|
"My GitHub Projects": [
|
||||||
"https://github.com/georgemandis.atom",
|
"https://github.com/georgemandis.atom",
|
||||||
|
"https://github.com/georgemandis/bubo-rss/releases.atom",
|
||||||
"https://github.com/georgemandis/konami-js/releases.atom",
|
"https://github.com/georgemandis/konami-js/releases.atom",
|
||||||
"https://github.com/georgemandis/konami-js/commits/main.atom",
|
"https://github.com/georgemandis/konami-js/commits/main.atom",
|
||||||
"https://github.com/javascriptforartists/cheer-me-up-and-sing-me-a-song/commits/master.atom",
|
"https://github.com/javascriptforartists/cheer-me-up-and-sing-me-a-song/commits/master.atom",
|
||||||
"https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom",
|
"https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom"
|
||||||
"https://github.com/georgemandis/remote-working-list/commits/master.atom",
|
|
||||||
"https://github.com/georgemandis/tweeter-totter/commits/master.atom"
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@ -40,9 +40,17 @@
|
|||||||
</ul>
|
</ul>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
<br>
|
||||||
<hr>
|
<hr>
|
||||||
<p>
|
<p>
|
||||||
Last updated {{ now }}. Powered by <a href="https://github.com/georgemandis/bubo-rss">Bubo Reader</a>, a project by <a href="https://george.mand.is">George Mandis</a>
|
Last updated {{ now }}.
|
||||||
</p>
|
</p>
|
||||||
|
<p>
|
||||||
|
Powered by <a href="https://github.com/georgemandis/bubo-rss">Bubo Reader (2.0.0)</a>, a project by <a href="https://george.mand.is">George Mandis</a>.
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
<a href='https://github.com/sponsors/georgemandis'>❤️ Sponsor on GitHub</a>
|
||||||
|
</p>
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
@ -1,30 +0,0 @@
|
|||||||
body {
|
|
||||||
font-family:system-ui;
|
|
||||||
font-size: 18px;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
details:focus,
|
|
||||||
details:focus-within,
|
|
||||||
details:hover {
|
|
||||||
/* background:#ffeb3b; */
|
|
||||||
/* outline:2px #000 solid; */
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
details ul li {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
summary {
|
|
||||||
cursor:pointer;
|
|
||||||
}
|
|
||||||
|
|
||||||
summary:hover {
|
|
||||||
opacity:.75;
|
|
||||||
}
|
|
||||||
|
|
||||||
.feed-url {
|
|
||||||
color:#aaa;
|
|
||||||
}
|
|
||||||
4563
package-lock.json
generated
4563
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
39
package.json
39
package.json
@ -1,19 +1,44 @@
|
|||||||
{
|
{
|
||||||
"name": "bubo-reader",
|
"name": "bubo-reader",
|
||||||
"version": "1.0.1",
|
"version": "2.0.0",
|
||||||
"description": "A Glitch-ready implementation of Bubo Reader",
|
"description": "A simple but effective feed reader (RSS, JSON)",
|
||||||
"main": "src/index.js",
|
"main": "src/index.js",
|
||||||
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node src/index.js > output/index.html; node server.js",
|
"dev": "tsc --watch",
|
||||||
"build": "node src/index.js > output/index.html",
|
"clean": "rm -rf dist",
|
||||||
|
"build": "tsc",
|
||||||
|
"bubo": "node dist/index.js",
|
||||||
|
"build:bubo": "tsc && node dist/index.js",
|
||||||
|
"start": "npm run build:bubo; node server.js",
|
||||||
"test": "echo \"Error: no test specified\" && exit 1"
|
"test": "echo \"Error: no test specified\" && exit 1"
|
||||||
},
|
},
|
||||||
"author": "",
|
"author": {
|
||||||
|
"name": "George Mandis",
|
||||||
|
"email": "george@mand.is",
|
||||||
|
"url": "https://george.mand.is"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/georgemandis"
|
||||||
|
},
|
||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"node-fetch": "^2.6.0",
|
"chalk": "^5.0.0",
|
||||||
|
"node-fetch": "^3.1.0",
|
||||||
"nunjucks": "^3.2.0",
|
"nunjucks": "^3.2.0",
|
||||||
"rss-parser": "^3.7.5",
|
"rss-parser": "^3.6.3",
|
||||||
"express": "^4.17.1"
|
"express": "^4.17.1"
|
||||||
|
},
|
||||||
|
"engines": { "node": "16.x" },
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^16.11.10",
|
||||||
|
"@types/nunjucks": "^3.2.0",
|
||||||
|
"@types/xml2js": "^0.4.9",
|
||||||
|
"@typescript-eslint/eslint-plugin": "^5.4.0",
|
||||||
|
"@typescript-eslint/parser": "^5.4.0",
|
||||||
|
"eslint": "^8.3.0",
|
||||||
|
"tslib": "^2.3.1",
|
||||||
|
"typescript": "^4.5.2"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
28
public/style.css
Normal file
28
public/style.css
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
body {
|
||||||
|
font-family: system-ui;
|
||||||
|
font-size: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
details:focus,
|
||||||
|
details:focus-within,
|
||||||
|
details:hover {
|
||||||
|
/* background:#ffeb3b; */
|
||||||
|
/* outline:2px #000 solid; */
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
details ul li {}
|
||||||
|
|
||||||
|
summary {
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
summary:hover {
|
||||||
|
opacity: .75;
|
||||||
|
}
|
||||||
|
|
||||||
|
.feed-url {
|
||||||
|
color: #aaa;
|
||||||
|
}
|
||||||
15
src/@types/bubo.d.ts
vendored
Normal file
15
src/@types/bubo.d.ts
vendored
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
export interface Feeds {
|
||||||
|
[key: string]: object[]
|
||||||
|
}
|
||||||
|
export interface FeedItem {
|
||||||
|
[key: string]: string | number | Date | FeedItem[];
|
||||||
|
items: FeedItem[]
|
||||||
|
}
|
||||||
|
|
||||||
|
//NEW WAY
|
||||||
|
export type JSONValue =
|
||||||
|
| string
|
||||||
|
| number
|
||||||
|
| boolean
|
||||||
|
| { [x: string]: JSONValue }
|
||||||
|
| Array<JSONValue>;
|
||||||
129
src/index.js
129
src/index.js
@ -1,129 +0,0 @@
|
|||||||
/*
|
|
||||||
* 🦉 Bubo RSS Reader
|
|
||||||
* ====
|
|
||||||
* Dead simple feed reader that renders an HTML
|
|
||||||
* page with links to content from feeds organized by site
|
|
||||||
*
|
|
||||||
* Code: https://github.com/georgemandis/bubo-rss
|
|
||||||
* Copyright (c) 2019 George Mandis (https://george.mand.is)
|
|
||||||
* Version: 1.0.1 (11/14/2021)
|
|
||||||
* Licensed under the MIT License (http://opensource.org/licenses/MIT)
|
|
||||||
*/
|
|
||||||
|
|
||||||
const fetch = require("node-fetch");
|
|
||||||
const Parser = require("rss-parser");
|
|
||||||
const parser = new Parser();
|
|
||||||
|
|
||||||
const nunjucks = require("nunjucks");
|
|
||||||
const env = nunjucks.configure({ autoescape: true });
|
|
||||||
|
|
||||||
const feeds = require("./feeds.json");
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Global filters for my Nunjucks templates
|
|
||||||
*/
|
|
||||||
env.addFilter("formatDate", function (dateString) {
|
|
||||||
const formattedDate = new Date(dateString).toLocaleDateString()
|
|
||||||
return formattedDate !== 'Invalid Date' ? formattedDate : dateString;
|
|
||||||
});
|
|
||||||
|
|
||||||
env.addGlobal('now', (new Date()).toUTCString());
|
|
||||||
|
|
||||||
// parse RSS/XML or JSON feeds
|
|
||||||
function parseFeed(response) {
|
|
||||||
const contentType = response.headers.get("content-type")
|
|
||||||
? response.headers.get("content-type").split(";")[0]
|
|
||||||
: false;
|
|
||||||
|
|
||||||
const rssFeed = [contentType]
|
|
||||||
.map(item =>
|
|
||||||
[
|
|
||||||
"application/atom+xml",
|
|
||||||
"application/rss+xml",
|
|
||||||
"application/xml",
|
|
||||||
"text/xml",
|
|
||||||
"text/html" // this is kind of a gamble
|
|
||||||
].includes(item)
|
|
||||||
? response.text()
|
|
||||||
: false
|
|
||||||
)
|
|
||||||
.filter(_ => _)[0];
|
|
||||||
|
|
||||||
const jsonFeed = [contentType]
|
|
||||||
.map(item =>
|
|
||||||
["application/json", "application/feed+json"].includes(item) ? response.json() : false
|
|
||||||
)
|
|
||||||
.filter(_ => _)[0];
|
|
||||||
|
|
||||||
return rssFeed || jsonFeed || false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
There's a little inconcistency with how feeds report certain things like
|
|
||||||
title, links and timestamps. These helpers try to normalize that bit and
|
|
||||||
provide an order-of-operations list of properties to look for.
|
|
||||||
|
|
||||||
Note: these are tightly-coupled to the template and a personal preference.
|
|
||||||
*/
|
|
||||||
|
|
||||||
const getLink = (obj) => {
|
|
||||||
const link_values = ["link", "url", "guid", "home_page_url"];
|
|
||||||
const keys = Object.keys(obj);
|
|
||||||
const link_property = link_values.find(link_value => keys.includes(link_value));
|
|
||||||
return obj[link_property];
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// fallback to URL for the title if not present (coupled to my template)
|
|
||||||
const getTitle = (obj) => {
|
|
||||||
const title_values = ["title", "url", "link"]; // fallback to url/link as title if omitted
|
|
||||||
const keys = Object.keys(obj);
|
|
||||||
const title_property = title_values.find(title_value => keys.includes(title_value));
|
|
||||||
return obj[title_property];
|
|
||||||
}
|
|
||||||
|
|
||||||
// More dependable way to get timestamps
|
|
||||||
const getTimestamp = (obj) => {
|
|
||||||
const timestamp = new Date(obj.pubDate || obj.isoDate || obj.date || obj.date_published).getTime();
|
|
||||||
return isNaN(timestamp) ? (obj.pubDate || obj.isoDate || obj.date || obj.date_published) : timestamp;
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetch the feeds and build the object for our template
|
|
||||||
(async () => {
|
|
||||||
const contentFromAllFeeds = {};
|
|
||||||
const errors = [];
|
|
||||||
|
|
||||||
for (const group in feeds) {
|
|
||||||
contentFromAllFeeds[group] = [];
|
|
||||||
|
|
||||||
for (let index = 0; index < feeds[group].length; index++) {
|
|
||||||
try {
|
|
||||||
const response = await fetch(feeds[group][index]);
|
|
||||||
const body = await parseFeed(response);
|
|
||||||
const contents =
|
|
||||||
typeof body === "string" ? await parser.parseString(body) : body;
|
|
||||||
|
|
||||||
contents.feed = feeds[group][index];
|
|
||||||
contents.title = getTitle(contents);
|
|
||||||
contents.link = getLink(contents);
|
|
||||||
contentFromAllFeeds[group].push(contents);
|
|
||||||
|
|
||||||
// try to normalize date attribute naming
|
|
||||||
contents?.items?.forEach(item => {
|
|
||||||
item.timestamp = getTimestamp(item);
|
|
||||||
item.title = getTitle(item);
|
|
||||||
item.link = getLink(item);
|
|
||||||
});
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
errors.push(feeds[group][index]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const output = env.render("./src/template.html", {
|
|
||||||
data: contentFromAllFeeds,
|
|
||||||
errors: errors
|
|
||||||
});
|
|
||||||
console.log(output);
|
|
||||||
})();
|
|
||||||
143
src/index.ts
Normal file
143
src/index.ts
Normal file
@ -0,0 +1,143 @@
|
|||||||
|
/*
|
||||||
|
* 🦉 Bubo Reader
|
||||||
|
* ====
|
||||||
|
* Dead simple feed reader (RSS + JSON) that renders an HTML
|
||||||
|
* page with links to content from feeds organized by site
|
||||||
|
*
|
||||||
|
* Code: https://github.com/georgemandis/bubo-rss
|
||||||
|
* Copyright (c) 2019 George Mandis (https://george.mand.is)
|
||||||
|
* Version: 1.0.1 (11/14/2021)
|
||||||
|
* Licensed under the MIT License (http://opensource.org/licenses/MIT)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import fetch from "node-fetch";
|
||||||
|
import Parser from "rss-parser";
|
||||||
|
import { Feeds, FeedItem } from "./@types/bubo";
|
||||||
|
import { Response } from "node-fetch";
|
||||||
|
import { render } from "./renderer.js";
|
||||||
|
import { getLink, getTitle, getTimestamp, parseFeed, getFeedList } from "./utilities.js";
|
||||||
|
import { writeFile } from "fs/promises";
|
||||||
|
import chalk from "chalk";
|
||||||
|
|
||||||
|
const parser = new Parser();
|
||||||
|
const feedList = await getFeedList();
|
||||||
|
const feedListLength = Object.entries(feedList).flat(2).length - Object.keys(feedList).length;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* contentFromAllFeeds = Contains normalized, aggregated feed data and is passed to template renderer at the end
|
||||||
|
* errors = Contains errors from parsing feeds and is also passed to template.
|
||||||
|
*/
|
||||||
|
const contentFromAllFeeds: Feeds = {};
|
||||||
|
const errors: unknown[] = [];
|
||||||
|
|
||||||
|
// benchmarking data + utility
|
||||||
|
const initTime = Date.now();
|
||||||
|
const benchmark = (startTime: number) => chalk.cyanBright.bold(`(${(Date.now() - startTime) / 1000} seconds)`);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* These values are used to control throttling/batching the fetches:
|
||||||
|
* - MAX_CONNECTION = max number of fetches to contain in a batch
|
||||||
|
* - DELAY_MS = the delay in milliseconds between batches
|
||||||
|
*/
|
||||||
|
const MAX_CONNECTIONS = Infinity;
|
||||||
|
const DELAY_MS = 850;
|
||||||
|
|
||||||
|
const error = chalk.bold.red;
|
||||||
|
const success = chalk.bold.green;
|
||||||
|
|
||||||
|
// keeping tally of total feeds fetched and parsed so we can compare
|
||||||
|
// to feedListLength and know when we're finished.
|
||||||
|
let completed = 0;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* finishBuild
|
||||||
|
* --
|
||||||
|
* function that gets called when all the feeds are through fetching
|
||||||
|
* and we want to build the static output.
|
||||||
|
*/
|
||||||
|
const finishBuild: () => void = async () => {
|
||||||
|
console.log("\nDone fetching everything!");
|
||||||
|
|
||||||
|
// generate the static HTML output from our template renderer
|
||||||
|
const output = render({
|
||||||
|
data: contentFromAllFeeds,
|
||||||
|
errors: errors
|
||||||
|
});
|
||||||
|
|
||||||
|
// write the output to public/index.html
|
||||||
|
await writeFile("./public/index.html", output);
|
||||||
|
console.log(`Finished writing to output. ${benchmark(initTime)}`);
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* processFeed
|
||||||
|
* --
|
||||||
|
* Process an individual feed and normalize its items
|
||||||
|
* @param { group, feed, startTime}
|
||||||
|
* @returns Promise<void>
|
||||||
|
*/
|
||||||
|
const processFeed = (
|
||||||
|
{
|
||||||
|
group, feed, startTime
|
||||||
|
}: { group: string; feed: string, startTime: number }
|
||||||
|
) => async (response: Response): Promise<void> => {
|
||||||
|
const body = await parseFeed(response);
|
||||||
|
completed++;
|
||||||
|
// skip to the next one if this didn't work out
|
||||||
|
if (!body) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const contents: FeedItem =
|
||||||
|
(typeof body === "string" ? (await parser.parseString(body)) : body) as FeedItem;
|
||||||
|
|
||||||
|
contents.feed = feed;
|
||||||
|
contents.title = getTitle(contents);
|
||||||
|
contents.link = getLink(contents);
|
||||||
|
|
||||||
|
// try to normalize date attribute naming
|
||||||
|
contents?.items?.forEach((item) => {
|
||||||
|
item.timestamp = getTimestamp(item);
|
||||||
|
item.title = getTitle(item);
|
||||||
|
item.link = getLink(item);
|
||||||
|
});
|
||||||
|
|
||||||
|
contentFromAllFeeds[group].push(contents as object);
|
||||||
|
console.log(`${success("Successfully fetched:")} ${feed} ${benchmark(startTime)}`);
|
||||||
|
|
||||||
|
} catch (err) {
|
||||||
|
console.log(`${error("Error processing:")} ${feed} ${benchmark(startTime)}`);
|
||||||
|
errors.push(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
// if this is the last feed, go ahead and build the output
|
||||||
|
(completed === feedListLength) && finishBuild();
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
// go through each group of feeds and process
|
||||||
|
const processFeeds = () => {
|
||||||
|
let idx = 0;
|
||||||
|
|
||||||
|
for (const [group, feeds] of Object.entries(feedList)) {
|
||||||
|
contentFromAllFeeds[group] = [];
|
||||||
|
|
||||||
|
for (const feed of feeds) {
|
||||||
|
const startTime = Date.now();
|
||||||
|
setTimeout(() => {
|
||||||
|
console.log(`Fetching: ${feed}...`);
|
||||||
|
|
||||||
|
fetch(feed).then(processFeed({ group, feed, startTime })).catch(err => {
|
||||||
|
console.log(error(`Error fetching ${feed} ${benchmark(startTime)}`));
|
||||||
|
errors.push(`Error fetching ${feed} ${err.toString()}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
}, (idx % (feedListLength / MAX_CONNECTIONS)) * DELAY_MS);
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
processFeeds();
|
||||||
37
src/renderer.ts
Normal file
37
src/renderer.ts
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
/*
|
||||||
|
* Return our renderer.
|
||||||
|
* Using Nunjucks out of the box.
|
||||||
|
* https://mozilla.github.io/nunjucks/
|
||||||
|
*/
|
||||||
|
|
||||||
|
import nunjucks from "nunjucks";
|
||||||
|
const env: nunjucks.Environment = nunjucks.configure({ autoescape: true });
|
||||||
|
import { readFile } from "fs/promises";
|
||||||
|
import { Feeds } from "./@types/bubo";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Global filters for my Nunjucks templates
|
||||||
|
*/
|
||||||
|
env.addFilter("formatDate", function (dateString): string {
|
||||||
|
const date: Date = new Date(parseInt(dateString));
|
||||||
|
return !isNaN(date.getTime()) ? date.toLocaleDateString() : dateString;
|
||||||
|
});
|
||||||
|
|
||||||
|
env.addGlobal("now", (new Date()).toUTCString());
|
||||||
|
|
||||||
|
// load the template
|
||||||
|
const template: string =
|
||||||
|
(await readFile(
|
||||||
|
new URL("../config/template.html", import.meta.url)
|
||||||
|
)).toString();
|
||||||
|
|
||||||
|
// generate the static HTML output from our template renderer
|
||||||
|
const render = ({ data, errors }: { data: Feeds; errors: unknown[] }) => {
|
||||||
|
return env.renderString(template, {
|
||||||
|
data,
|
||||||
|
errors
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
export { render };
|
||||||
74
src/utilities.ts
Normal file
74
src/utilities.ts
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
/*
|
||||||
|
There's a little inconcistency with how feeds report certain things like
|
||||||
|
title, links and timestamps. These helpers try to normalize that bit and
|
||||||
|
provide an order-of-operations list of properties to look for.
|
||||||
|
|
||||||
|
Note: these are tightly-coupled to the template and a personal preference.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Response } from "node-fetch";
|
||||||
|
import { readFile } from "fs/promises";
|
||||||
|
import { FeedItem, JSONValue } from "./@types/bubo";
|
||||||
|
|
||||||
|
export const getLink = (obj: FeedItem): string => {
|
||||||
|
const link_values: string[] = ["link", "url", "guid", "home_page_url"];
|
||||||
|
const keys: string[] = Object.keys(obj);
|
||||||
|
const link_property: string | undefined = link_values.find(link_value => keys.includes(link_value));
|
||||||
|
return link_property ? obj[link_property] as string : "";
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
// fallback to URL for the title if not present (coupled to my template)
|
||||||
|
export const getTitle = (obj: FeedItem): string => {
|
||||||
|
const title_values: string[] = ["title", "url", "link"]; // fallback to url/link as title if omitted
|
||||||
|
const keys: string[] = Object.keys(obj);
|
||||||
|
const title_property: string | undefined = title_values.find(title_value => keys.includes(title_value));
|
||||||
|
return title_property ? obj[title_property] as string : "";
|
||||||
|
};
|
||||||
|
|
||||||
|
// More dependable way to get timestamps
|
||||||
|
export const getTimestamp = (obj: FeedItem): string => {
|
||||||
|
const dateString: string = (obj.pubDate || obj.isoDate || obj.date || obj.date_published).toString();
|
||||||
|
const timestamp: number = new Date(dateString).getTime();
|
||||||
|
return isNaN(timestamp) ? dateString : timestamp.toString();
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
// parse RSS/XML or JSON feeds
|
||||||
|
export async function parseFeed(response: Response): Promise<JSONValue> {
|
||||||
|
const contentType = response.headers.get("content-type")?.split(";")[0];
|
||||||
|
|
||||||
|
if (!contentType) return {};
|
||||||
|
|
||||||
|
const rssFeed = [contentType]
|
||||||
|
.map(item =>
|
||||||
|
[
|
||||||
|
"application/atom+xml",
|
||||||
|
"application/rss+xml",
|
||||||
|
"application/xml",
|
||||||
|
"text/xml",
|
||||||
|
"text/html" // this is kind of a gamble
|
||||||
|
].includes(item)
|
||||||
|
? response.text()
|
||||||
|
: false
|
||||||
|
)
|
||||||
|
.filter(_ => _)[0];
|
||||||
|
|
||||||
|
const jsonFeed = [contentType]
|
||||||
|
.map(item =>
|
||||||
|
["application/json", "application/feed+json"].includes(item) ? response.json() as Promise<JSONValue> : false
|
||||||
|
)
|
||||||
|
.filter(_ => _)[0];
|
||||||
|
|
||||||
|
return (rssFeed && rssFeed) || (jsonFeed && jsonFeed) || {};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export const getFeedList = async (): Promise<JSONValue> => {
|
||||||
|
return JSON.parse(
|
||||||
|
(await readFile(
|
||||||
|
new URL("../config/feeds.json", import.meta.url)
|
||||||
|
)).toString()
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
30
tsconfig.json
Normal file
30
tsconfig.json
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"module": "esnext",
|
||||||
|
"forceConsistentCasingInFileNames": true,
|
||||||
|
"removeComments": true,
|
||||||
|
"strict": true,
|
||||||
|
"importHelpers": true,
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"allowSyntheticDefaultImports": true,
|
||||||
|
"target": "ES2021",
|
||||||
|
"noImplicitAny": true,
|
||||||
|
"moduleResolution": "node",
|
||||||
|
"sourceMap": false,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"outDir": "dist",
|
||||||
|
"baseUrl": ".",
|
||||||
|
"typeRoots": [
|
||||||
|
"src/@types"
|
||||||
|
],
|
||||||
|
"paths": {
|
||||||
|
"*": [
|
||||||
|
"node_modules/*",
|
||||||
|
"src/@types"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"include": [
|
||||||
|
"src/**/*"
|
||||||
|
]
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user