allow POST of html as alternative to GET of url
parent
ce97090936
commit
fa2903d466
69
index.js
69
index.js
|
@ -4,10 +4,12 @@ const { Readability } = require('@mozilla/readability');
|
||||||
const JSDOM = require('jsdom').JSDOM;
|
const JSDOM = require('jsdom').JSDOM;
|
||||||
const common_filters = require('./url_to_markdown_common_filters');
|
const common_filters = require('./url_to_markdown_common_filters');
|
||||||
const validURL = require('@7c/validurl');
|
const validURL = require('@7c/validurl');
|
||||||
|
const express = require('express');
|
||||||
service = new turndown();
|
const app = express();
|
||||||
|
|
||||||
const rateLimit = require('express-rate-limit');
|
const rateLimit = require('express-rate-limit');
|
||||||
|
const service = new turndown();
|
||||||
|
|
||||||
|
const port = process.env.PORT;
|
||||||
|
|
||||||
const rateLimiter = rateLimit({
|
const rateLimiter = rateLimit({
|
||||||
windowMs: 30 * 1000,
|
windowMs: 30 * 1000,
|
||||||
|
@ -16,37 +18,66 @@ const rateLimiter = rateLimit({
|
||||||
headers: true
|
headers: true
|
||||||
});
|
});
|
||||||
|
|
||||||
const express = require('express')
|
app.use(rateLimiter);
|
||||||
const app = express()
|
|
||||||
const port = process.env.PORT
|
|
||||||
|
|
||||||
app.use(rateLimiter)
|
app.use(express.urlencoded({
|
||||||
|
extended: true
|
||||||
|
}));
|
||||||
|
|
||||||
app.get('/', (req, res) => {
|
app.get('/', (req, res) => {
|
||||||
url = req.query.url;
|
url = req.query.url;
|
||||||
res.header("Access-Control-Allow-Origin", '*');
|
|
||||||
res.header("Access-Control-Expose-Headers", 'X-Title');
|
|
||||||
res.header("Content-Type", 'text/markdown');
|
|
||||||
if (url && validURL(url)) {
|
if (url && validURL(url)) {
|
||||||
read_url(url, res);
|
send_headers(res);
|
||||||
|
let markdown = read_url(url, res);
|
||||||
|
let result = common_filters.filter(url, markdown);
|
||||||
|
res.send(result);
|
||||||
} else {
|
} else {
|
||||||
res.status(400).send("Please specify a valid url query parameter");
|
res.status(400).send("Please specify a valid url query parameter");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
app.post('/', function(req, res) {
|
||||||
|
let html;
|
||||||
|
|
||||||
|
if (req.body) {
|
||||||
|
html = req.body.html;
|
||||||
|
if (!html) {
|
||||||
|
res.status(400).send("Please provide a POST parameter called html");
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
let document = new JSDOM(html);
|
||||||
|
let markdown = process_dom(document, res);
|
||||||
|
res.send(markdown);
|
||||||
|
} catch (error) {
|
||||||
|
res.status(400).send("Could not parse that document");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
app.listen(port, () => {
|
app.listen(port, () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
|
function send_headers(res) {
|
||||||
|
res.header("Access-Control-Allow-Origin", '*');
|
||||||
|
res.header("Access-Control-Expose-Headers", 'X-Title');
|
||||||
|
res.header("Content-Type", 'text/markdown');
|
||||||
|
}
|
||||||
|
|
||||||
|
function process_dom(document, res) {
|
||||||
|
let title = document.window.document.querySelector('title');
|
||||||
|
if (title)
|
||||||
|
res.header("X-Title", encodeURIComponent(title.textContent));
|
||||||
|
let reader = new Readability(document.window.document);
|
||||||
|
let article = reader.parse();
|
||||||
|
let markdown = service.turndown(article.content);
|
||||||
|
return markdown;
|
||||||
|
}
|
||||||
|
|
||||||
function read_url(url, res) {
|
function read_url(url, res) {
|
||||||
JSDOM.fromURL(url).then((document)=>{
|
JSDOM.fromURL(url).then((document)=>{
|
||||||
let title = document.window.document.querySelector('title');
|
process_dom(document, res);
|
||||||
if (title)
|
|
||||||
res.header("X-Title", encodeURIComponent(title.textContent));
|
|
||||||
let reader = new Readability(document.window.document);
|
|
||||||
let article = reader.parse();
|
|
||||||
let markdown = service.turndown(article.content);
|
|
||||||
let result = common_filters.filter(url, markdown);
|
|
||||||
res.send(result);
|
|
||||||
}).catch((error)=> {
|
}).catch((error)=> {
|
||||||
res.status(400).send("Sorry, could not fetch and convert that URL");
|
res.status(400).send("Sorry, could not fetch and convert that URL");
|
||||||
});
|
});
|
||||||
|
|
Loading…
Reference in New Issue