From 990d1d3e718ccadb568d23814ea9f1602906da04 Mon Sep 17 00:00:00 2001 From: Lee Hanken Date: Tue, 22 Nov 2022 22:12:45 +0000 Subject: [PATCH] process question and answers separately for stackoverflow --- index.js | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index b4db08a..67cf0db 100755 --- a/index.js +++ b/index.js @@ -18,7 +18,7 @@ const service = new turndown(); const apple_dev_prefix = "https://developer.apple.com"; -const stackoverflow_prefix = "https://stackoverflow.com"; +const stackoverflow_prefix = "https://stackoverflow.com/questions"; const rateLimiter = rateLimit({ windowMs: 30 * 1000, @@ -50,6 +50,8 @@ app.get('/', (req, res) => { send_headers(res); if (url.startsWith(apple_dev_prefix)) { read_apple_url(url, res, inline_title, ignore_links); + } else if (url.startsWith(stackoverflow_prefix)) { + read_stack_url(url, res, inline_title, ignore_links); } else { read_url(url, res, inline_title, ignore_links); } @@ -73,7 +75,7 @@ app.post('/', function(req, res) { } if (url && validURL(url) && url.startsWith(stackoverflow_prefix)) { send_headers(res); - read_url(url, res, inline_title, ignore_links); + read_stack_url(url, res, inline_title, ignore_links); return; } if (!html) { @@ -100,10 +102,13 @@ function send_headers(res) { res.header("Content-Type", 'text/markdown'); } -function process_dom(url, document, res, inline_title, ignore_links) { +function process_dom(url, document, res, inline_title, ignore_links, id="") { let title = document.window.document.querySelector('title'); if (title) res.header("X-Title", encodeURIComponent(title.textContent)); + if (id) { + document = new JSDOM(''+ document.window.document.querySelector("#"+id).innerHTML); + } let reader = new Readability(document.window.document); let readable = reader.parse().content; let replacements = [] @@ -129,6 +134,21 @@ function read_url(url, res, inline_title, ignore_links) { }); } +function read_stack_url(url, res, inline_title, ignore_links) { + JSDOM.fromURL(url).then((document)=>{ + let markdown_q = process_dom(url, document, res, inline_title, ignore_links, 'question'); + let markdown_a = process_dom(url, document, res, false, ignore_links, 'answers'); + if (markdown_a.startsWith('Your Answer')) { + res.send(markdown_q); + } + else { + res.send(markdown_q + "\n\n## Answer\n"+ markdown_a); + } + }).catch((error)=> { + res.status(400).send("Sorry, could not fetch and convert that URL"); + }); +} + function read_apple_url(url, res, inline_title, ignore_links) { json_url = apple_dev_parser.dev_doc_url(url); https.get(json_url,(apple_res) => {