notBrad/bubo-rss
1
0
Fork 1
mirror of https://github.com/georgemandis/bubo-rss.git synced 2024-11-23 12:24:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into glitch

Browse Source
This commit is contained in:
George Mandis 2021-12-05 13:41:43 -08:00
commit b9a7da4287
7 changed files with 186 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@ -1,3 +1,5 @@
[![Netlify Status](https://api.netlify.com/api/v1/badges/81dd219c-51cb-4418-a18c-42c8b104c689/deploy-status)](https://app.netlify.com/sites/bubo-rss-demo/deploys)
# 🦉 Bubo Reader (2.0) # 🦉 Bubo Reader (2.0)
Bubo Reader is a hyper-minimalist <acronym title="Really Simple Syndication">RSS</acronym> and <acronym title="JavaScript Object Notation">JSON</acronym> feed reader you can deploy on your own server, [Netlify](https://netlify.com) in a few steps or [Glitch](https://glitch.com) in even fewer steps! The goal of the project is to generate a webpage that shows a list of links from a collection of feeds organized by category and website. That's it. Bubo Reader is a hyper-minimalist <acronym title="Really Simple Syndication">RSS</acronym> and <acronym title="JavaScript Object Notation">JSON</acronym> feed reader you can deploy on your own server, [Netlify](https://netlify.com) in a few steps or [Glitch](https://glitch.com) in even fewer steps! The goal of the project is to generate a webpage that shows a list of links from a collection of feeds organized by category and website. That's it.
@ -7,6 +9,7 @@ It is named after this [silly robot owl](https://www.youtube.com/watch?v=MYSeCfo
You can read more about how this project came about on my blog: You can read more about how this project came about on my blog:
- [Introducing Bubo RSS: An Absurdly Minimalist RSS Feed Reader](https://george.mand.is/2019/11/introducing-bubo-rss-an-absurdly-minimalist-rss-feed-reader/). - [Introducing Bubo RSS: An Absurdly Minimalist RSS Feed Reader](https://george.mand.is/2019/11/introducing-bubo-rss-an-absurdly-minimalist-rss-feed-reader/).
## Getting Started ## Getting Started
- Clone or fork the repo and run `npm install` to install the dependencies. - Clone or fork the repo and run `npm install` to install the dependencies.
@ -85,7 +88,9 @@ Not the most exciting-looking demos, I'll admit, but they work!
The quickest way is to remix the project on Glitch: The quickest way is to remix the project on Glitch:
[https://glitch.com/edit/#!/bubo-rss](https://glitch.com/edit/#!/bubo-rss) [https://glitch.com/edit/#!/bubo-rss](https://glitch.com/edit/#!/bubo-rss)
Just changed some feeds in `./config/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./public/style.css` file or the `./config/template.html` file respectively. There is also a `glitch` branch on this repo if you'd prefer to start there.
Just change some feeds in `./config/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./public/style.css` file or the `./config/template.html` file respectively.
<a id="netlify"></a> <a id="netlify"></a>
## Deploying to Netlify ## Deploying to Netlify

48
config/template.html
View File

@ -1,14 +1,13 @@
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta http-equiv="X-UA-Compatible" content="ie=edge"> <meta http-equiv="X-UA-Compatible" content="ie=edge" />
<title>🦉 Bubo Reader</title> <title>🦉 Bubo Reader</title>
<link rel="stylesheet" href="/style.css"> <link rel="stylesheet" href="/style.css" />
</head> </head>
<body> <body>
<h1>🦉 Bubo Reader</h1> <h1>🦉 Bubo Reader</h1>
{% for group, feeds in data %} {% for group, feeds in data %}
@ -22,15 +21,18 @@
<ul> <ul>
{% for item in feed.items %} {% for item in feed.items %}
<li> <li>
{{ item.timestamp | formatDate }} - <a href="{{ item.link }}" target='_blank' rel='noopener norefferer nofollow'>{{ item.title }}</a> {{ item.timestamp | formatDate }} -
<a
href="{{ item.link }}"
target="_blank"
rel="noopener norefferer nofollow"
>{{ item.title }}</a
>
</li> </li>
{% endfor %} {% endfor %}
</ul> </ul>
</details> </details>
{% endfor %} {% endfor %} {% endfor %} {% if errors | length > 0 %}
{% endfor %}
{% if errors | length > 0 %}
<h2>Errors</h2> <h2>Errors</h2>
<p>There were errors trying to parse these feeds:</p> <p>There were errors trying to parse these feeds:</p>
<ul> <ul>
@ -40,17 +42,15 @@
</ul> </ul>
{% endif %} {% endif %}
<br> <br />
<hr> <hr />
<p>Last updated {{ now }}.</p>
<p> <p>
Last updated {{ now }}. Powered by
<a href="https://github.com/georgemandis/bubo-rss"
>Bubo Reader (v{{ info.version }})</a
>, a project by <a href="https://george.mand.is">George Mandis</a>. ❤️
<a href="{{ info.funding.url }}">Sponsor on GitHub</a>
</p> </p>
<p> </body>
Powered by <a href="https://github.com/georgemandis/bubo-rss">Bubo Reader (2.0.0)</a>, a project by <a href="https://george.mand.is">George Mandis</a>.
</p>
<p>
<a href='https://github.com/sponsors/georgemandis'>❤️ Sponsor on GitHub</a>
</p>
</body>
</html> </html>

6
package-lock.json generated
View File

@ -1,13 +1,13 @@
{ {
"name": "bubo-reader", "name": "bubo-reader",
"version": "1.0.3", "version": "2.0.1",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "bubo-reader", "name": "bubo-reader",
"version": "1.0.3", "version": "2.0.1",
"license": "ISC", "license": "MIT",
"dependencies": { "dependencies": {
"chalk": "^5.0.0", "chalk": "^5.0.0",
"node-fetch": "^3.1.0", "node-fetch": "^3.1.0",

9
package.json
View File

@ -1,7 +1,8 @@
{ {
"name": "bubo-reader", "name": "bubo-reader",
"version": "2.0.0", "version": "2.0.1",
"description": "A simple but effective feed reader (RSS, JSON)", "description": "A simple but effective feed reader (RSS, JSON)",
"homepage": "https://github.com/georgemandis/bubo-rss",
"main": "src/index.js", "main": "src/index.js",
"type": "module", "type": "module",
"scripts": { "scripts": {
@ -22,7 +23,11 @@
"type": "github", "type": "github",
"url": "https://github.com/sponsors/georgemandis" "url": "https://github.com/sponsors/georgemandis"
}, },
"license": "ISC", "bugs": {
"url": "https://github.com/georgemandis/bubo-rss/issues",
"email": "george+bubo@mand.is"
},
"license": "MIT",
"dependencies": { "dependencies": {
"chalk": "^5.0.0", "chalk": "^5.0.0",
"node-fetch": "^3.1.0", "node-fetch": "^3.1.0",

78
src/index.ts
View File

@ -15,13 +15,22 @@ import Parser from "rss-parser";
import { Feeds, FeedItem } from "./@types/bubo"; import { Feeds, FeedItem } from "./@types/bubo";
import { Response } from "node-fetch"; import { Response } from "node-fetch";
import { render } from "./renderer.js"; import { render } from "./renderer.js";
import { getLink, getTitle, getTimestamp, parseFeed, getFeedList } from "./utilities.js"; import {
getLink,
getTitle,
getTimestamp,
parseFeed,
getFeedList,
getBuboInfo
} from "./utilities.js";
import { writeFile } from "fs/promises"; import { writeFile } from "fs/promises";
import chalk from "chalk"; import chalk from "chalk";
const buboInfo = await getBuboInfo();
const parser = new Parser(); const parser = new Parser();
const feedList = await getFeedList(); const feedList = await getFeedList();
const feedListLength = Object.entries(feedList).flat(2).length - Object.keys(feedList).length; const feedListLength =
Object.entries(feedList).flat(2).length - Object.keys(feedList).length;
/** /**
* contentFromAllFeeds = Contains normalized, aggregated feed data and is passed to template renderer at the end * contentFromAllFeeds = Contains normalized, aggregated feed data and is passed to template renderer at the end
@ -32,7 +41,8 @@ const errors: unknown[] = [];
// benchmarking data + utility // benchmarking data + utility
const initTime = Date.now(); const initTime = Date.now();
const benchmark = (startTime: number) => chalk.cyanBright.bold(`(${(Date.now() - startTime) / 1000} seconds)`); const benchmark = (startTime: number) =>
chalk.cyanBright.bold(`${(Date.now() - startTime) / 1000} seconds`);
/** /**
* These values are used to control throttling/batching the fetches: * These values are used to control throttling/batching the fetches:
@ -49,7 +59,6 @@ const success = chalk.bold.green;
// to feedListLength and know when we're finished. // to feedListLength and know when we're finished.
let completed = 0; let completed = 0;
/** /**
* finishBuild * finishBuild
* -- * --
@ -62,12 +71,17 @@ const finishBuild: () => void = async () => {
// generate the static HTML output from our template renderer // generate the static HTML output from our template renderer
const output = render({ const output = render({
data: contentFromAllFeeds, data: contentFromAllFeeds,
errors: errors errors: errors,
info: buboInfo
}); });
// write the output to public/index.html // write the output to public/index.html
await writeFile("./public/index.html", output); await writeFile("./public/index.html", output);
console.log(`Finished writing to output. ${benchmark(initTime)}`); console.log(
`\nFinished writing to output:\n- ${feedListLength} feeds in ${benchmark(
initTime
)}\n- ${errors.length} errors`
);
}; };
/** /**
@ -77,43 +91,54 @@ const finishBuild: () => void = async () => {
* @param { group, feed, startTime} * @param { group, feed, startTime}
* @returns Promise<void> * @returns Promise<void>
*/ */
const processFeed = ( const processFeed =
{ ({
group, feed, startTime group,
}: { group: string; feed: string, startTime: number } feed,
) => async (response: Response): Promise<void> => { startTime
}: {
group: string;
feed: string;
startTime: number;
}) =>
async (response: Response): Promise<void> => {
const body = await parseFeed(response); const body = await parseFeed(response);
completed++; completed++;
// skip to the next one if this didn't work out // skip to the next one if this didn't work out
if (!body) return; if (!body) return;
try { try {
const contents: FeedItem = const contents: FeedItem = (
(typeof body === "string" ? (await parser.parseString(body)) : body) as FeedItem; typeof body === "string" ? await parser.parseString(body) : body
) as FeedItem;
contents.feed = feed; contents.feed = feed;
contents.title = getTitle(contents); contents.title = getTitle(contents);
contents.link = getLink(contents); contents.link = getLink(contents);
// try to normalize date attribute naming // try to normalize date attribute naming
contents?.items?.forEach((item) => { contents?.items?.forEach(item => {
item.timestamp = getTimestamp(item); item.timestamp = getTimestamp(item);
item.title = getTitle(item); item.title = getTitle(item);
item.link = getLink(item); item.link = getLink(item);
}); });
contentFromAllFeeds[group].push(contents as object); contentFromAllFeeds[group].push(contents as object);
console.log(`${success("Successfully fetched:")} ${feed} ${benchmark(startTime)}`); console.log(
`${success("Successfully fetched:")} ${feed} - ${benchmark(startTime)}`
);
} catch (err) { } catch (err) {
console.log(`${error("Error processing:")} ${feed} ${benchmark(startTime)}`); console.log(
errors.push(err); `${error("Error processing:")} ${feed} - ${benchmark(
startTime
)}\n${err}`
);
errors.push(`Error processing: ${feed}\n\t${err}`);
} }
// if this is the last feed, go ahead and build the output // if this is the last feed, go ahead and build the output
(completed === feedListLength) && finishBuild(); completed === feedListLength && finishBuild();
}; };
// go through each group of feeds and process // go through each group of feeds and process
const processFeeds = () => { const processFeeds = () => {
@ -127,17 +152,18 @@ const processFeeds = () => {
setTimeout(() => { setTimeout(() => {
console.log(`Fetching: ${feed}...`); console.log(`Fetching: ${feed}...`);
fetch(feed).then(processFeed({ group, feed, startTime })).catch(err => { fetch(feed)
console.log(error(`Error fetching ${feed} ${benchmark(startTime)}`)); .then(processFeed({ group, feed, startTime }))
.catch(err => {
console.log(
error(`Error fetching ${feed} ${benchmark(startTime)}`)
);
errors.push(`Error fetching ${feed} ${err.toString()}`); errors.push(`Error fetching ${feed} ${err.toString()}`);
}); });
}, (idx % (feedListLength / MAX_CONNECTIONS)) * DELAY_MS); }, (idx % (feedListLength / MAX_CONNECTIONS)) * DELAY_MS);
idx++; idx++;
} }
} }
}; };
processFeeds(); processFeeds();

33
src/renderer.ts
View File

@ -1,13 +1,13 @@
/* /*
* Return our renderer. * Return our renderer.
* Using Nunjucks out of the box. * Using Nunjucks out of the box.
* https://mozilla.github.io/nunjucks/ * https://mozilla.github.io/nunjucks/
*/ */
import nunjucks from "nunjucks"; import nunjucks from "nunjucks";
const env: nunjucks.Environment = nunjucks.configure({ autoescape: true }); const env: nunjucks.Environment = nunjucks.configure({ autoescape: true });
import { readFile } from "fs/promises"; import { readFile } from "fs/promises";
import { Feeds } from "./@types/bubo"; import { Feeds, JSONValue } from "./@types/bubo";
/** /**
* Global filters for my Nunjucks templates * Global filters for my Nunjucks templates
@ -17,21 +17,28 @@ env.addFilter("formatDate", function (dateString): string {
return !isNaN(date.getTime()) ? date.toLocaleDateString() : dateString; return !isNaN(date.getTime()) ? date.toLocaleDateString() : dateString;
}); });
env.addGlobal("now", (new Date()).toUTCString()); env.addGlobal("now", new Date().toUTCString());
// load the template // load the template
const template: string = const template: string = (
(await readFile( await readFile(new URL("../config/template.html", import.meta.url))
new URL("../config/template.html", import.meta.url) ).toString();
)).toString();
// generate the static HTML output from our template renderer // generate the static HTML output from our template renderer
const render = ({ data, errors }: { data: Feeds; errors: unknown[] }) => { const render = ({
data,
errors,
info
}: {
data: Feeds;
errors: unknown[];
info?: JSONValue;
}) => {
return env.renderString(template, { return env.renderString(template, {
data, data,
errors errors,
info
}); });
}; };
export { render }; export { render };

46
src/utilities.ts
View File

@ -1,5 +1,5 @@
/* /*
There's a little inconcistency with how feeds report certain things like There's a little inconsistency with how feeds report certain things like
title, links and timestamps. These helpers try to normalize that bit and title, links and timestamps. These helpers try to normalize that bit and
provide an order-of-operations list of properties to look for. provide an order-of-operations list of properties to look for.
@ -13,27 +13,37 @@ import { FeedItem, JSONValue } from "./@types/bubo";
export const getLink = (obj: FeedItem): string => { export const getLink = (obj: FeedItem): string => {
const link_values: string[] = ["link", "url", "guid", "home_page_url"]; const link_values: string[] = ["link", "url", "guid", "home_page_url"];
const keys: string[] = Object.keys(obj); const keys: string[] = Object.keys(obj);
const link_property: string | undefined = link_values.find(link_value => keys.includes(link_value)); const link_property: string | undefined = link_values.find(link_value =>
return link_property ? obj[link_property] as string : ""; keys.includes(link_value)
);
return link_property ? (obj[link_property] as string) : "";
}; };
// fallback to URL for the title if not present
// fallback to URL for the title if not present (coupled to my template) // (title -> url -> link)
export const getTitle = (obj: FeedItem): string => { export const getTitle = (obj: FeedItem): string => {
const title_values: string[] = ["title", "url", "link"]; // fallback to url/link as title if omitted const title_values: string[] = ["title", "url", "link"];
const keys: string[] = Object.keys(obj); const keys: string[] = Object.keys(obj);
const title_property: string | undefined = title_values.find(title_value => keys.includes(title_value));
return title_property ? obj[title_property] as string : ""; // if title is empty for some reason, fall back on url or link
const title_property: string | undefined = title_values.find(
title_value => keys.includes(title_value) && obj[title_value]
);
return title_property ? (obj[title_property] as string) : "";
}; };
// More dependable way to get timestamps // More dependable way to get timestamps
export const getTimestamp = (obj: FeedItem): string => { export const getTimestamp = (obj: FeedItem): string => {
const dateString: string = (obj.pubDate || obj.isoDate || obj.date || obj.date_published).toString(); const dateString: string = (
obj.pubDate ||
obj.isoDate ||
obj.date ||
obj.date_published
).toString();
const timestamp: number = new Date(dateString).getTime(); const timestamp: number = new Date(dateString).getTime();
return isNaN(timestamp) ? dateString : timestamp.toString(); return isNaN(timestamp) ? dateString : timestamp.toString();
}; };
// parse RSS/XML or JSON feeds // parse RSS/XML or JSON feeds
export async function parseFeed(response: Response): Promise<JSONValue> { export async function parseFeed(response: Response): Promise<JSONValue> {
const contentType = response.headers.get("content-type")?.split(";")[0]; const contentType = response.headers.get("content-type")?.split(";")[0];
@ -56,19 +66,25 @@ export async function parseFeed(response: Response): Promise<JSONValue> {
const jsonFeed = [contentType] const jsonFeed = [contentType]
.map(item => .map(item =>
["application/json", "application/feed+json"].includes(item) ? response.json() as Promise<JSONValue> : false ["application/json", "application/feed+json"].includes(item)
? (response.json() as Promise<JSONValue>)
: false
) )
.filter(_ => _)[0]; .filter(_ => _)[0];
return (rssFeed && rssFeed) || (jsonFeed && jsonFeed) || {}; return (rssFeed && rssFeed) || (jsonFeed && jsonFeed) || {};
} }
export const getFeedList = async (): Promise<JSONValue> => { export const getFeedList = async (): Promise<JSONValue> => {
return JSON.parse( return JSON.parse(
(await readFile( (
new URL("../config/feeds.json", import.meta.url) await readFile(new URL("../config/feeds.json", import.meta.url))
)).toString() ).toString()
); );
}; };
export const getBuboInfo = async (): Promise<JSONValue> => {
return JSON.parse(
(await readFile(new URL("../package.json", import.meta.url))).toString()
);
};