robots.txt and xml sitemap.WIP

4 files changed, 221 insertions, 10 deletions

diff --git a/package-lock.json b/package-lock.json
index c79e271..5ac375b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10,13 +10,14 @@
       "dependencies": {
         "ejs": "^3.1.6",
         "express": "^4.17.1",
+        "express-sitemap": "^1.8.0",
+        "express-sitemap-xml": "^3.0.0",
         "jsdom": "^16.2.2",
         "katex": "^0.11.1",
         "markdown-it": "^11.0.0",
         "markdown-it-highlightjs": "^3.4.0",
         "markdown-it-multimd-table": "^4.0.2",
-        "markdown-it-texmath": "^0.7.0",
-        "memory-cache": "^0.2.0"
+        "markdown-it-texmath": "^0.7.0"
       }
     },
     "node_modules/abab": {
@@ -565,6 +566,41 @@
         "node": ">= 0.10.0"
       }
     },
+    "node_modules/express-sitemap": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/express-sitemap/-/express-sitemap-1.8.0.tgz",
+      "integrity": "sha512-J5a5Zyhr+2xszhf0pIxUFHwz+Bwy8fgts4WIx+b+TJsw+ZyMZUXb4O2PYcPS+4Vwbv74Pz1pu5GMMrwd8XlndQ==",
+      "dependencies": {
+        "setheaders": "0.3.0",
+        "tickle": "1.4.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/express-sitemap-xml": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/express-sitemap-xml/-/express-sitemap-xml-3.0.0.tgz",
+      "integrity": "sha512-5cim7WY3g5C2HpK0sJAS749PSEhCV/NtQzdkdor1PzTdTHqS405h/obWJy6rz4UpU+s07+awK9uRQTIbZyQCsQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "p-memoize": "^4.0.1",
+        "xmlbuilder": "^15.1.1"
+      }
+    },
     "node_modules/express/node_modules/qs": {
       "version": "6.7.0",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
@@ -928,6 +964,17 @@
       "resolved": "https://registry.npmjs.org/lodash.flow/-/lodash.flow-3.5.0.tgz",
       "integrity": "sha1-h79AKSuM+D5OjOGjrkIJ4gBxZ1o="
     },
+    "node_modules/map-age-cleaner": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz",
+      "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==",
+      "dependencies": {
+        "p-defer": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/markdown-it": {
       "version": "11.0.1",
       "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-11.0.1.tgz",
@@ -978,10 +1025,20 @@
         "node": ">= 0.6"
       }
     },
-    "node_modules/memory-cache": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/memory-cache/-/memory-cache-0.2.0.tgz",
-      "integrity": "sha1-eJCwHVLADI68nVM+H46xfjA0hxo="
+    "node_modules/mem": {
+      "version": "6.1.1",
+      "resolved": "https://registry.npmjs.org/mem/-/mem-6.1.1.tgz",
+      "integrity": "sha512-Ci6bIfq/UgcxPTYa8dQQ5FY3BzKkT894bwXWXxC/zqs0XgMO2cT20CGkOqda7gZNkmK5VP4x89IGZ6K7hfbn3Q==",
+      "dependencies": {
+        "map-age-cleaner": "^0.1.3",
+        "mimic-fn": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/mem?sponsor=1"
+      }
     },
     "node_modules/merge-descriptors": {
       "version": "1.0.1",
@@ -1026,6 +1083,14 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/mimic-fn": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-3.1.0.tgz",
+      "integrity": "sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/minimatch": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
@@ -1090,6 +1155,29 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/p-defer": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz",
+      "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/p-memoize": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/p-memoize/-/p-memoize-4.0.1.tgz",
+      "integrity": "sha512-km0sP12uE0dOZ5qP+s7kGVf07QngxyG0gS8sYFvFWhqlgzOsSy+m71aUejf/0akxj5W7gE//2G74qTv6b4iMog==",
+      "dependencies": {
+        "mem": "^6.0.1",
+        "mimic-fn": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/p-memoize?sponsor=1"
+      }
+    },
     "node_modules/parse5": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz",
@@ -1339,6 +1427,14 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/setheaders": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/setheaders/-/setheaders-0.3.0.tgz",
+      "integrity": "sha1-PnSfM3ou58GGwuDdB/RftHJIjhM=",
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/setprototypeof": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
@@ -1414,6 +1510,14 @@
       "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
       "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="
     },
+    "node_modules/tickle": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/tickle/-/tickle-1.4.0.tgz",
+      "integrity": "sha1-uBkLxqcEXo3+ePQJewTWedpCQu0=",
+      "engines": {
+        "node": ">=4.0"
+      }
+    },
     "node_modules/toidentifier": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@@ -1637,6 +1741,14 @@
       "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz",
       "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw=="
     },
+    "node_modules/xmlbuilder": {
+      "version": "15.1.1",
+      "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz",
+      "integrity": "sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg==",
+      "engines": {
+        "node": ">=8.0"
+      }
+    },
     "node_modules/xmlchars": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
@@ -2095,6 +2207,24 @@
         }
       }
     },
+    "express-sitemap": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/express-sitemap/-/express-sitemap-1.8.0.tgz",
+      "integrity": "sha512-J5a5Zyhr+2xszhf0pIxUFHwz+Bwy8fgts4WIx+b+TJsw+ZyMZUXb4O2PYcPS+4Vwbv74Pz1pu5GMMrwd8XlndQ==",
+      "requires": {
+        "setheaders": "0.3.0",
+        "tickle": "1.4.0"
+      }
+    },
+    "express-sitemap-xml": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/express-sitemap-xml/-/express-sitemap-xml-3.0.0.tgz",
+      "integrity": "sha512-5cim7WY3g5C2HpK0sJAS749PSEhCV/NtQzdkdor1PzTdTHqS405h/obWJy6rz4UpU+s07+awK9uRQTIbZyQCsQ==",
+      "requires": {
+        "p-memoize": "^4.0.1",
+        "xmlbuilder": "^15.1.1"
+      }
+    },
     "extend": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@@ -2372,6 +2502,14 @@
       "resolved": "https://registry.npmjs.org/lodash.flow/-/lodash.flow-3.5.0.tgz",
       "integrity": "sha1-h79AKSuM+D5OjOGjrkIJ4gBxZ1o="
     },
+    "map-age-cleaner": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz",
+      "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==",
+      "requires": {
+        "p-defer": "^1.0.0"
+      }
+    },
     "markdown-it": {
       "version": "11.0.1",
       "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-11.0.1.tgz",
@@ -2416,10 +2554,14 @@
       "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
       "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
     },
-    "memory-cache": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/memory-cache/-/memory-cache-0.2.0.tgz",
-      "integrity": "sha1-eJCwHVLADI68nVM+H46xfjA0hxo="
+    "mem": {
+      "version": "6.1.1",
+      "resolved": "https://registry.npmjs.org/mem/-/mem-6.1.1.tgz",
+      "integrity": "sha512-Ci6bIfq/UgcxPTYa8dQQ5FY3BzKkT894bwXWXxC/zqs0XgMO2cT20CGkOqda7gZNkmK5VP4x89IGZ6K7hfbn3Q==",
+      "requires": {
+        "map-age-cleaner": "^0.1.3",
+        "mimic-fn": "^3.0.0"
+      }
     },
     "merge-descriptors": {
       "version": "1.0.1",
@@ -2449,6 +2591,11 @@
         "mime-db": "1.47.0"
       }
     },
+    "mimic-fn": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-3.1.0.tgz",
+      "integrity": "sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ=="
+    },
     "minimatch": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
@@ -2498,6 +2645,20 @@
         "word-wrap": "~1.2.3"
       }
     },
+    "p-defer": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz",
+      "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww="
+    },
+    "p-memoize": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/p-memoize/-/p-memoize-4.0.1.tgz",
+      "integrity": "sha512-km0sP12uE0dOZ5qP+s7kGVf07QngxyG0gS8sYFvFWhqlgzOsSy+m71aUejf/0akxj5W7gE//2G74qTv6b4iMog==",
+      "requires": {
+        "mem": "^6.0.1",
+        "mimic-fn": "^3.0.0"
+      }
+    },
     "parse5": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz",
@@ -2686,6 +2847,11 @@
         "send": "0.17.1"
       }
     },
+    "setheaders": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/setheaders/-/setheaders-0.3.0.tgz",
+      "integrity": "sha1-PnSfM3ou58GGwuDdB/RftHJIjhM="
+    },
     "setprototypeof": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
@@ -2741,6 +2907,11 @@
       "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
       "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="
     },
+    "tickle": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/tickle/-/tickle-1.4.0.tgz",
+      "integrity": "sha1-uBkLxqcEXo3+ePQJewTWedpCQu0="
+    },
     "toidentifier": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@@ -2902,6 +3073,11 @@
       "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz",
       "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw=="
     },
+    "xmlbuilder": {
+      "version": "15.1.1",
+      "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz",
+      "integrity": "sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg=="
+    },
     "xmlchars": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
diff --git a/package.json b/package.json
index ab27f5d..307cddd 100644
--- a/package.json
+++ b/package.json
@@ -19,6 +19,8 @@
   "dependencies": {
     "ejs": "^3.1.6",
     "express": "^4.17.1",
+    "express-sitemap": "^1.8.0",
+    "express-sitemap-xml": "^3.0.0",
     "jsdom": "^16.2.2",
     "katex": "^0.11.1",
     "markdown-it": "^11.0.0",
diff --git a/run.sh b/run.sh
new file mode 100755
index 0000000..178c319
--- /dev/null
+++ b/run.sh
@@ -0,0 +1,2 @@
+#!/usr/bin/env sh
+docker run --entrypoint /server/server.js -p 9909:9000 web
diff --git a/server.js b/server.js
index a9f2973..b51bb52 100755
--- a/server.js
+++ b/server.js
@@ -2,6 +2,7 @@
 "use strict";
 
 const express = require("express");
+const sitemap = require("express-sitemap-xml");
 const path = require("path");
 const fs = require("fs");
 const mit = require("markdown-it")({ html: true })
@@ -37,6 +38,7 @@ function renderAndSend(req, res) {
       path.join(__dirname, viewPath),
       "utf-8"
     );
+    //FIXME-this can obviously fail
     readStream.on("data", (chunk) => {
       res.render("index", {
         cache: true,
@@ -51,6 +53,21 @@ function renderAndSend(req, res) {
   }
 }
 
+app.get("/health", (req, res) => {
+  res.type("application/json");
+  let response = { isOK: "True", error: "" };
+  res.send(response);
+});
+
+app.get("/robots.txt", (req, res) => {
+  res.type("text/plain");
+  let robots_txt = "Sitemap: http://blog.terminaldweller.com\n";
+  robots_txt += "User-agent: *\n";
+  robots_txt += "Disallow: \n";
+  robots_txt += "Crawl-Delay: 20";
+  res.send(robots_txt);
+});
+
 app.get("/$", (req, res) => {
   renderAndSend(req, res);
 });
@@ -62,4 +79,18 @@ app.get("/mds/:mdname$", (req, res) => {
   renderAndSend(req, res);
 });
 
+async function enumerateDir() {
+  return await fs.readdirSync(path.join(__dirname, "mds"));
+}
+
+app.use(sitemap(enumerateDir, "http://blog.terminaldweller.com"));
+
+app.use((req, res) => {
+  return res.status(404).send({ message: "Path" + req.url + "not found!" });
+});
+
+app.use((err, req, res) => {
+  return res.status(500).send({ error: err });
+});
+
 app.listen(9000);