mirror of
https://github.com/georgemandis/bubo-rss.git
synced 2024-11-23 12:24:23 +00:00
Merge branch 'main' into glitch
This commit is contained in:
George Mandis
commit
b9a7da4287
@ -1,3 +1,5 @@
|
||||
[](https://app.netlify.com/sites/bubo-rss-demo/deploys)
|
||||
|
||||
# 🦉 Bubo Reader (2.0)
|
||||
|
||||
Bubo Reader is a hyper-minimalist <acronym title="Really Simple Syndication">RSS</acronym> and <acronym title="JavaScript Object Notation">JSON</acronym> feed reader you can deploy on your own server, [Netlify](https://netlify.com) in a few steps or [Glitch](https://glitch.com) in even fewer steps! The goal of the project is to generate a webpage that shows a list of links from a collection of feeds organized by category and website. That's it.
|
||||
@ -7,6 +9,7 @@ It is named after this [silly robot owl](https://www.youtube.com/watch?v=MYSeCfo
|
||||
You can read more about how this project came about on my blog:
|
||||
- [Introducing Bubo RSS: An Absurdly Minimalist RSS Feed Reader](https://george.mand.is/2019/11/introducing-bubo-rss-an-absurdly-minimalist-rss-feed-reader/).
|
||||
|
||||
|
||||
## Getting Started
|
||||
|
||||
- Clone or fork the repo and run `npm install` to install the dependencies.
|
||||
@ -85,7 +88,9 @@ Not the most exciting-looking demos, I'll admit, but they work!
|
||||
The quickest way is to remix the project on Glitch:
|
||||
[https://glitch.com/edit/#!/bubo-rss](https://glitch.com/edit/#!/bubo-rss)
|
||||
|
||||
Just changed some feeds in `./config/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./public/style.css` file or the `./config/template.html` file respectively.
|
||||
There is also a `glitch` branch on this repo if you'd prefer to start there.
|
||||
|
||||
Just change some feeds in `./config/feeds.json` file and you're set! If you'd like to modify the style or the template you can changed `./public/style.css` file or the `./config/template.html` file respectively.
|
||||
|
||||
<a id="netlify"></a>
|
||||
## Deploying to Netlify
|
||||
|
||||
@ -1,14 +1,13 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<meta http-equiv="X-UA-Compatible" content="ie=edge">
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta http-equiv="X-UA-Compatible" content="ie=edge" />
|
||||
<title>🦉 Bubo Reader</title>
|
||||
<link rel="stylesheet" href="/style.css">
|
||||
<link rel="stylesheet" href="/style.css" />
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<h1>🦉 Bubo Reader</h1>
|
||||
|
||||
{% for group, feeds in data %}
|
||||
@ -22,15 +21,18 @@
|
||||
<ul>
|
||||
{% for item in feed.items %}
|
||||
<li>
|
||||
{{ item.timestamp | formatDate }} - <a href="{{ item.link }}" target='_blank' rel='noopener norefferer nofollow'>{{ item.title }}</a>
|
||||
{{ item.timestamp | formatDate }} -
|
||||
<a
|
||||
href="{{ item.link }}"
|
||||
target="_blank"
|
||||
rel="noopener norefferer nofollow"
|
||||
>{{ item.title }}</a
|
||||
>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</details>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
{% if errors | length > 0 %}
|
||||
{% endfor %} {% endfor %} {% if errors | length > 0 %}
|
||||
<h2>Errors</h2>
|
||||
<p>There were errors trying to parse these feeds:</p>
|
||||
<ul>
|
||||
@ -40,17 +42,15 @@
|
||||
</ul>
|
||||
{% endif %}
|
||||
|
||||
<br>
|
||||
<hr>
|
||||
<br />
|
||||
<hr />
|
||||
<p>Last updated {{ now }}.</p>
|
||||
<p>
|
||||
Last updated {{ now }}.
|
||||
Powered by
|
||||
<a href="https://github.com/georgemandis/bubo-rss"
|
||||
>Bubo Reader (v{{ info.version }})</a
|
||||
>, a project by <a href="https://george.mand.is">George Mandis</a>. ❤️
|
||||
<a href="{{ info.funding.url }}">Sponsor on GitHub</a>
|
||||
</p>
|
||||
<p>
|
||||
Powered by <a href="https://github.com/georgemandis/bubo-rss">Bubo Reader (2.0.0)</a>, a project by <a href="https://george.mand.is">George Mandis</a>.
|
||||
</p>
|
||||
<p>
|
||||
<a href='https://github.com/sponsors/georgemandis'>❤️ Sponsor on GitHub</a>
|
||||
</p>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
6
package-lock.json
generated
6
package-lock.json
generated
@ -1,13 +1,13 @@
|
||||
{
|
||||
"name": "bubo-reader",
|
||||
"version": "1.0.3",
|
||||
"version": "2.0.1",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "bubo-reader",
|
||||
"version": "1.0.3",
|
||||
"license": "ISC",
|
||||
"version": "2.0.1",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"chalk": "^5.0.0",
|
||||
"node-fetch": "^3.1.0",
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
{
|
||||
"name": "bubo-reader",
|
||||
"version": "2.0.0",
|
||||
"version": "2.0.1",
|
||||
"description": "A simple but effective feed reader (RSS, JSON)",
|
||||
"homepage": "https://github.com/georgemandis/bubo-rss",
|
||||
"main": "src/index.js",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
@ -22,7 +23,11 @@
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/georgemandis"
|
||||
},
|
||||
"license": "ISC",
|
||||
"bugs": {
|
||||
"url": "https://github.com/georgemandis/bubo-rss/issues",
|
||||
"email": "george+bubo@mand.is"
|
||||
},
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"chalk": "^5.0.0",
|
||||
"node-fetch": "^3.1.0",
|
||||
|
||||
76
src/index.ts
76
src/index.ts
@ -15,13 +15,22 @@ import Parser from "rss-parser";
|
||||
import { Feeds, FeedItem } from "./@types/bubo";
|
||||
import { Response } from "node-fetch";
|
||||
import { render } from "./renderer.js";
|
||||
import { getLink, getTitle, getTimestamp, parseFeed, getFeedList } from "./utilities.js";
|
||||
import {
|
||||
getLink,
|
||||
getTitle,
|
||||
getTimestamp,
|
||||
parseFeed,
|
||||
getFeedList,
|
||||
getBuboInfo
|
||||
} from "./utilities.js";
|
||||
import { writeFile } from "fs/promises";
|
||||
import chalk from "chalk";
|
||||
|
||||
const buboInfo = await getBuboInfo();
|
||||
const parser = new Parser();
|
||||
const feedList = await getFeedList();
|
||||
const feedListLength = Object.entries(feedList).flat(2).length - Object.keys(feedList).length;
|
||||
const feedListLength =
|
||||
Object.entries(feedList).flat(2).length - Object.keys(feedList).length;
|
||||
|
||||
/**
|
||||
* contentFromAllFeeds = Contains normalized, aggregated feed data and is passed to template renderer at the end
|
||||
@ -32,7 +41,8 @@ const errors: unknown[] = [];
|
||||
|
||||
// benchmarking data + utility
|
||||
const initTime = Date.now();
|
||||
const benchmark = (startTime: number) => chalk.cyanBright.bold(`(${(Date.now() - startTime) / 1000} seconds)`);
|
||||
const benchmark = (startTime: number) =>
|
||||
chalk.cyanBright.bold(`${(Date.now() - startTime) / 1000} seconds`);
|
||||
|
||||
/**
|
||||
* These values are used to control throttling/batching the fetches:
|
||||
@ -49,7 +59,6 @@ const success = chalk.bold.green;
|
||||
// to feedListLength and know when we're finished.
|
||||
let completed = 0;
|
||||
|
||||
|
||||
/**
|
||||
* finishBuild
|
||||
* --
|
||||
@ -62,12 +71,17 @@ const finishBuild: () => void = async () => {
|
||||
// generate the static HTML output from our template renderer
|
||||
const output = render({
|
||||
data: contentFromAllFeeds,
|
||||
errors: errors
|
||||
errors: errors,
|
||||
info: buboInfo
|
||||
});
|
||||
|
||||
// write the output to public/index.html
|
||||
await writeFile("./public/index.html", output);
|
||||
console.log(`Finished writing to output. ${benchmark(initTime)}`);
|
||||
console.log(
|
||||
`\nFinished writing to output:\n- ${feedListLength} feeds in ${benchmark(
|
||||
initTime
|
||||
)}\n- ${errors.length} errors`
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
@ -77,44 +91,55 @@ const finishBuild: () => void = async () => {
|
||||
* @param { group, feed, startTime}
|
||||
* @returns Promise<void>
|
||||
*/
|
||||
const processFeed = (
|
||||
{
|
||||
group, feed, startTime
|
||||
}: { group: string; feed: string, startTime: number }
|
||||
) => async (response: Response): Promise<void> => {
|
||||
const processFeed =
|
||||
({
|
||||
group,
|
||||
feed,
|
||||
startTime
|
||||
}: {
|
||||
group: string;
|
||||
feed: string;
|
||||
startTime: number;
|
||||
}) =>
|
||||
async (response: Response): Promise<void> => {
|
||||
const body = await parseFeed(response);
|
||||
completed++;
|
||||
// skip to the next one if this didn't work out
|
||||
if (!body) return;
|
||||
|
||||
try {
|
||||
const contents: FeedItem =
|
||||
(typeof body === "string" ? (await parser.parseString(body)) : body) as FeedItem;
|
||||
const contents: FeedItem = (
|
||||
typeof body === "string" ? await parser.parseString(body) : body
|
||||
) as FeedItem;
|
||||
|
||||
contents.feed = feed;
|
||||
contents.title = getTitle(contents);
|
||||
contents.link = getLink(contents);
|
||||
|
||||
// try to normalize date attribute naming
|
||||
contents?.items?.forEach((item) => {
|
||||
contents?.items?.forEach(item => {
|
||||
item.timestamp = getTimestamp(item);
|
||||
item.title = getTitle(item);
|
||||
item.link = getLink(item);
|
||||
});
|
||||
|
||||
contentFromAllFeeds[group].push(contents as object);
|
||||
console.log(`${success("Successfully fetched:")} ${feed} ${benchmark(startTime)}`);
|
||||
|
||||
console.log(
|
||||
`${success("Successfully fetched:")} ${feed} - ${benchmark(startTime)}`
|
||||
);
|
||||
} catch (err) {
|
||||
console.log(`${error("Error processing:")} ${feed} ${benchmark(startTime)}`);
|
||||
errors.push(err);
|
||||
console.log(
|
||||
`${error("Error processing:")} ${feed} - ${benchmark(
|
||||
startTime
|
||||
)}\n${err}`
|
||||
);
|
||||
errors.push(`Error processing: ${feed}\n\t${err}`);
|
||||
}
|
||||
|
||||
// if this is the last feed, go ahead and build the output
|
||||
(completed === feedListLength) && finishBuild();
|
||||
completed === feedListLength && finishBuild();
|
||||
};
|
||||
|
||||
|
||||
// go through each group of feeds and process
|
||||
const processFeeds = () => {
|
||||
let idx = 0;
|
||||
@ -127,17 +152,18 @@ const processFeeds = () => {
|
||||
setTimeout(() => {
|
||||
console.log(`Fetching: ${feed}...`);
|
||||
|
||||
fetch(feed).then(processFeed({ group, feed, startTime })).catch(err => {
|
||||
console.log(error(`Error fetching ${feed} ${benchmark(startTime)}`));
|
||||
fetch(feed)
|
||||
.then(processFeed({ group, feed, startTime }))
|
||||
.catch(err => {
|
||||
console.log(
|
||||
error(`Error fetching ${feed} ${benchmark(startTime)}`)
|
||||
);
|
||||
errors.push(`Error fetching ${feed} ${err.toString()}`);
|
||||
});
|
||||
|
||||
}, (idx % (feedListLength / MAX_CONNECTIONS)) * DELAY_MS);
|
||||
idx++;
|
||||
}
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
processFeeds();
|
||||
@ -7,7 +7,7 @@
|
||||
import nunjucks from "nunjucks";
|
||||
const env: nunjucks.Environment = nunjucks.configure({ autoescape: true });
|
||||
import { readFile } from "fs/promises";
|
||||
import { Feeds } from "./@types/bubo";
|
||||
import { Feeds, JSONValue } from "./@types/bubo";
|
||||
|
||||
/**
|
||||
* Global filters for my Nunjucks templates
|
||||
@ -17,21 +17,28 @@ env.addFilter("formatDate", function (dateString): string {
|
||||
return !isNaN(date.getTime()) ? date.toLocaleDateString() : dateString;
|
||||
});
|
||||
|
||||
env.addGlobal("now", (new Date()).toUTCString());
|
||||
env.addGlobal("now", new Date().toUTCString());
|
||||
|
||||
// load the template
|
||||
const template: string =
|
||||
(await readFile(
|
||||
new URL("../config/template.html", import.meta.url)
|
||||
)).toString();
|
||||
const template: string = (
|
||||
await readFile(new URL("../config/template.html", import.meta.url))
|
||||
).toString();
|
||||
|
||||
// generate the static HTML output from our template renderer
|
||||
const render = ({ data, errors }: { data: Feeds; errors: unknown[] }) => {
|
||||
const render = ({
|
||||
data,
|
||||
errors,
|
||||
info
|
||||
}: {
|
||||
data: Feeds;
|
||||
errors: unknown[];
|
||||
info?: JSONValue;
|
||||
}) => {
|
||||
return env.renderString(template, {
|
||||
data,
|
||||
errors
|
||||
errors,
|
||||
info
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
export { render };
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
There's a little inconcistency with how feeds report certain things like
|
||||
There's a little inconsistency with how feeds report certain things like
|
||||
title, links and timestamps. These helpers try to normalize that bit and
|
||||
provide an order-of-operations list of properties to look for.
|
||||
|
||||
@ -13,27 +13,37 @@ import { FeedItem, JSONValue } from "./@types/bubo";
|
||||
export const getLink = (obj: FeedItem): string => {
|
||||
const link_values: string[] = ["link", "url", "guid", "home_page_url"];
|
||||
const keys: string[] = Object.keys(obj);
|
||||
const link_property: string | undefined = link_values.find(link_value => keys.includes(link_value));
|
||||
return link_property ? obj[link_property] as string : "";
|
||||
const link_property: string | undefined = link_values.find(link_value =>
|
||||
keys.includes(link_value)
|
||||
);
|
||||
return link_property ? (obj[link_property] as string) : "";
|
||||
};
|
||||
|
||||
|
||||
// fallback to URL for the title if not present (coupled to my template)
|
||||
// fallback to URL for the title if not present
|
||||
// (title -> url -> link)
|
||||
export const getTitle = (obj: FeedItem): string => {
|
||||
const title_values: string[] = ["title", "url", "link"]; // fallback to url/link as title if omitted
|
||||
const title_values: string[] = ["title", "url", "link"];
|
||||
const keys: string[] = Object.keys(obj);
|
||||
const title_property: string | undefined = title_values.find(title_value => keys.includes(title_value));
|
||||
return title_property ? obj[title_property] as string : "";
|
||||
|
||||
// if title is empty for some reason, fall back on url or link
|
||||
const title_property: string | undefined = title_values.find(
|
||||
title_value => keys.includes(title_value) && obj[title_value]
|
||||
);
|
||||
return title_property ? (obj[title_property] as string) : "";
|
||||
};
|
||||
|
||||
// More dependable way to get timestamps
|
||||
export const getTimestamp = (obj: FeedItem): string => {
|
||||
const dateString: string = (obj.pubDate || obj.isoDate || obj.date || obj.date_published).toString();
|
||||
const dateString: string = (
|
||||
obj.pubDate ||
|
||||
obj.isoDate ||
|
||||
obj.date ||
|
||||
obj.date_published
|
||||
).toString();
|
||||
const timestamp: number = new Date(dateString).getTime();
|
||||
return isNaN(timestamp) ? dateString : timestamp.toString();
|
||||
};
|
||||
|
||||
|
||||
// parse RSS/XML or JSON feeds
|
||||
export async function parseFeed(response: Response): Promise<JSONValue> {
|
||||
const contentType = response.headers.get("content-type")?.split(";")[0];
|
||||
@ -56,19 +66,25 @@ export async function parseFeed(response: Response): Promise<JSONValue> {
|
||||
|
||||
const jsonFeed = [contentType]
|
||||
.map(item =>
|
||||
["application/json", "application/feed+json"].includes(item) ? response.json() as Promise<JSONValue> : false
|
||||
["application/json", "application/feed+json"].includes(item)
|
||||
? (response.json() as Promise<JSONValue>)
|
||||
: false
|
||||
)
|
||||
.filter(_ => _)[0];
|
||||
|
||||
return (rssFeed && rssFeed) || (jsonFeed && jsonFeed) || {};
|
||||
}
|
||||
|
||||
|
||||
export const getFeedList = async (): Promise<JSONValue> => {
|
||||
return JSON.parse(
|
||||
(await readFile(
|
||||
new URL("../config/feeds.json", import.meta.url)
|
||||
)).toString()
|
||||
(
|
||||
await readFile(new URL("../config/feeds.json", import.meta.url))
|
||||
).toString()
|
||||
);
|
||||
};
|
||||
|
||||
export const getBuboInfo = async (): Promise<JSONValue> => {
|
||||
return JSON.parse(
|
||||
(await readFile(new URL("../package.json", import.meta.url))).toString()
|
||||
);
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user