aboutsummaryrefslogtreecommitdiffstats
path: root/db/mongo
diff options
context:
space:
mode:
authorterminaldweller <devi@terminaldweller.com>2023-07-13 18:10:26 +0000
committerterminaldweller <devi@terminaldweller.com>2023-07-13 18:10:26 +0000
commitb595ac2150e42a22e92b954188ae374d290b3f53 (patch)
treeb5e9d5e421c6d266e669476e2a1dfa4e89383953 /db/mongo
parentupdate (diff)
downloadscripts-b595ac2150e42a22e92b954188ae374d290b3f53.tar.gz
scripts-b595ac2150e42a22e92b954188ae374d290b3f53.zip
updateHEADmain
Diffstat (limited to 'db/mongo')
-rwxr-xr-xdb/mongo/scraper.js56
1 files changed, 56 insertions, 0 deletions
diff --git a/db/mongo/scraper.js b/db/mongo/scraper.js
new file mode 100755
index 0000000..d28330f
--- /dev/null
+++ b/db/mongo/scraper.js
@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+"use strict";
+
+const x = require("x-ray")();
+
+async function getHTML(url) {
+ const response = await fetch(url, {
+ credentials: "include",
+ headers: {
+ "User-Agent":
+ "Mozilla/5.0 (Windows NT 10.0; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5666.197 Safari/537.36",
+ Accept:
+ "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.5",
+ "Upgrade-Insecure-Requests": "1",
+ "Sec-Fetch-Dest": "document",
+ "Sec-Fetch-Mode": "navigate",
+ "Sec-Fetch-Site": "none",
+ "Sec-Fetch-User": "?1",
+ "Sec-GPC": "1",
+ "sec-ch-ua-platform": '"Windows"',
+ "sec-ch-ua":
+ '"Google Chrome";v="113", "Chromium";v="113", "Not=A?Brand";v="24"',
+ "sec-ch-ua-mobile": "?0",
+ Pragma: "no-cache",
+ "Cache-Control": "no-cache",
+ },
+ method: "GET",
+ mode: "cors",
+ });
+
+ if (response.ok) {
+ console.log(response.body);
+ const chunks = [];
+
+ for await (const chunk of response.body) {
+ chunks.push(Buffer.from(chunk));
+ }
+
+ return Buffer.concat(chunks).toString("utf-8");
+ }
+}
+
+async function main() {
+ const resultHTML = await getHTML("https://chapmanganato.com/manga-kd987838");
+
+ x(
+ resultHTML,
+ "a.chapter-name.text-nowrap"
+ )(function (err, lastChapter) {
+ console.log(lastChapter);
+ return lastChapter;
+ });
+}
+
+main();