FOSS
/
urltomarkdown
mirror of https://github.com/macsplit/urltomarkdown
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

process question and answers separately for stackoverflow

main
Lee Hanken 2022-11-22 22:12:45 +00:00
parent faf75dc6fb
commit 990d1d3e71
1 changed files with 23 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
index.js
View File

@ -18,7 +18,7 @@ const service = new turndown();
const apple_dev_prefix = "https://developer.apple.com"; const apple_dev_prefix = "https://developer.apple.com";
const stackoverflow_prefix = "https://stackoverflow.com"; const stackoverflow_prefix = "https://stackoverflow.com/questions";
const rateLimiter = rateLimit({ const rateLimiter = rateLimit({
windowMs: 30 * 1000, windowMs: 30 * 1000,
@ -50,6 +50,8 @@ app.get('/', (req, res) => {
send_headers(res); send_headers(res);
if (url.startsWith(apple_dev_prefix)) { if (url.startsWith(apple_dev_prefix)) {
read_apple_url(url, res, inline_title, ignore_links); read_apple_url(url, res, inline_title, ignore_links);
} else if (url.startsWith(stackoverflow_prefix)) {
read_stack_url(url, res, inline_title, ignore_links);
} else { } else {
read_url(url, res, inline_title, ignore_links); read_url(url, res, inline_title, ignore_links);
} }
@ -73,7 +75,7 @@ app.post('/', function(req, res) {
} }
if (url && validURL(url) && url.startsWith(stackoverflow_prefix)) { if (url && validURL(url) && url.startsWith(stackoverflow_prefix)) {
send_headers(res); send_headers(res);
read_url(url, res, inline_title, ignore_links); read_stack_url(url, res, inline_title, ignore_links);
return; return;
} }
if (!html) { if (!html) {
@ -100,10 +102,13 @@ function send_headers(res) {
res.header("Content-Type", 'text/markdown'); res.header("Content-Type", 'text/markdown');
} }
function process_dom(url, document, res, inline_title, ignore_links) { function process_dom(url, document, res, inline_title, ignore_links, id="") {
let title = document.window.document.querySelector('title'); let title = document.window.document.querySelector('title');
if (title) if (title)
res.header("X-Title", encodeURIComponent(title.textContent)); res.header("X-Title", encodeURIComponent(title.textContent));
if (id) {
document = new JSDOM('<!DOCTYPE html>'+ document.window.document.querySelector("#"+id).innerHTML);
}
let reader = new Readability(document.window.document); let reader = new Readability(document.window.document);
let readable = reader.parse().content; let readable = reader.parse().content;
let replacements = [] let replacements = []
@ -129,6 +134,21 @@ function read_url(url, res, inline_title, ignore_links) {
}); });
} }
function read_stack_url(url, res, inline_title, ignore_links) {
JSDOM.fromURL(url).then((document)=>{
let markdown_q = process_dom(url, document, res, inline_title, ignore_links, 'question');
let markdown_a = process_dom(url, document, res, false, ignore_links, 'answers');
if (markdown_a.startsWith('Your Answer')) {
res.send(markdown_q);
}
else {
res.send(markdown_q + "\n\n## Answer\n"+ markdown_a);
}
}).catch((error)=> {
res.status(400).send("Sorry, could not fetch and convert that URL");
});
}
function read_apple_url(url, res, inline_title, ignore_links) { function read_apple_url(url, res, inline_title, ignore_links) {
json_url = apple_dev_parser.dev_doc_url(url); json_url = apple_dev_parser.dev_doc_url(url);
https.get(json_url,(apple_res) => { https.get(json_url,(apple_res) => {