robots.txt and xml sitemap.WIP

1 files changed, 186 insertions, 10 deletions

diff --git a/package-lock.json b/package-lock.json
index c79e271..5ac375b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10,13 +10,14 @@
       "dependencies": {
         "ejs": "^3.1.6",
         "express": "^4.17.1",
+        "express-sitemap": "^1.8.0",
+        "express-sitemap-xml": "^3.0.0",
         "jsdom": "^16.2.2",
         "katex": "^0.11.1",
         "markdown-it": "^11.0.0",
         "markdown-it-highlightjs": "^3.4.0",
         "markdown-it-multimd-table": "^4.0.2",
-        "markdown-it-texmath": "^0.7.0",
-        "memory-cache": "^0.2.0"
+        "markdown-it-texmath": "^0.7.0"
       }
     },
     "node_modules/abab": {
@@ -565,6 +566,41 @@
         "node": ">= 0.10.0"
       }
     },
+    "node_modules/express-sitemap": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/express-sitemap/-/express-sitemap-1.8.0.tgz",
+      "integrity": "sha512-J5a5Zyhr+2xszhf0pIxUFHwz+Bwy8fgts4WIx+b+TJsw+ZyMZUXb4O2PYcPS+4Vwbv74Pz1pu5GMMrwd8XlndQ==",
+      "dependencies": {
+        "setheaders": "0.3.0",
+        "tickle": "1.4.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/express-sitemap-xml": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/express-sitemap-xml/-/express-sitemap-xml-3.0.0.tgz",
+      "integrity": "sha512-5cim7WY3g5C2HpK0sJAS749PSEhCV/NtQzdkdor1PzTdTHqS405h/obWJy6rz4UpU+s07+awK9uRQTIbZyQCsQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "p-memoize": "^4.0.1",
+        "xmlbuilder": "^15.1.1"
+      }
+    },
     "node_modules/express/node_modules/qs": {
       "version": "6.7.0",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
@@ -928,6 +964,17 @@
       "resolved": "https://registry.npmjs.org/lodash.flow/-/lodash.flow-3.5.0.tgz",
       "integrity": "sha1-h79AKSuM+D5OjOGjrkIJ4gBxZ1o="
     },
+    "node_modules/map-age-cleaner": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz",
+      "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==",
+      "dependencies": {
+        "p-defer": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/markdown-it": {
       "version": "11.0.1",
       "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-11.0.1.tgz",
@@ -978,10 +1025,20 @@
         "node": ">= 0.6"
       }
     },
-    "node_modules/memory-cache": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/memory-cache/-/memory-cache-0.2.0.tgz",
-      "integrity": "sha1-eJCwHVLADI68nVM+H46xfjA0hxo="
+    "node_modules/mem": {
+      "version": "6.1.1",
+      "resolved": "https://registry.npmjs.org/mem/-/mem-6.1.1.tgz",
+      "integrity": "sha512-Ci6bIfq/UgcxPTYa8dQQ5FY3BzKkT894bwXWXxC/zqs0XgMO2cT20CGkOqda7gZNkmK5VP4x89IGZ6K7hfbn3Q==",
+      "dependencies": {
+        "map-age-cleaner": "^0.1.3",
+        "mimic-fn": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/mem?sponsor=1"
+      }
     },
     "node_modules/merge-descriptors": {
       "version": "1.0.1",
@@ -1026,6 +1083,14 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/mimic-fn": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-3.1.0.tgz",
+      "integrity": "sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/minimatch": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
@@ -1090,6 +1155,29 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/p-defer": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz",
+      "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/p-memoize": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/p-memoize/-/p-memoize-4.0.1.tgz",
+      "integrity": "sha512-km0sP12uE0dOZ5qP+s7kGVf07QngxyG0gS8sYFvFWhqlgzOsSy+m71aUejf/0akxj5W7gE//2G74qTv6b4iMog==",
+      "dependencies": {
+        "mem": "^6.0.1",
+        "mimic-fn": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/p-memoize?sponsor=1"
+      }
+    },
     "node_modules/parse5": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz",
@@ -1339,6 +1427,14 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/setheaders": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/setheaders/-/setheaders-0.3.0.tgz",
+      "integrity": "sha1-PnSfM3ou58GGwuDdB/RftHJIjhM=",
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/setprototypeof": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
@@ -1414,6 +1510,14 @@
       "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
       "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="
     },
+    "node_modules/tickle": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/tickle/-/tickle-1.4.0.tgz",
+      "integrity": "sha1-uBkLxqcEXo3+ePQJewTWedpCQu0=",
+      "engines": {
+        "node": ">=4.0"
+      }
+    },
     "node_modules/toidentifier": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@@ -1637,6 +1741,14 @@
       "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz",
       "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw=="
     },
+    "node_modules/xmlbuilder": {
+      "version": "15.1.1",
+      "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz",
+      "integrity": "sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg==",
+      "engines": {
+        "node": ">=8.0"
+      }
+    },
     "node_modules/xmlchars": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
@@ -2095,6 +2207,24 @@
         }
       }
     },
+    "express-sitemap": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/express-sitemap/-/express-sitemap-1.8.0.tgz",
+      "integrity": "sha512-J5a5Zyhr+2xszhf0pIxUFHwz+Bwy8fgts4WIx+b+TJsw+ZyMZUXb4O2PYcPS+4Vwbv74Pz1pu5GMMrwd8XlndQ==",
+      "requires": {
+        "setheaders": "0.3.0",
+        "tickle": "1.4.0"
+      }
+    },
+    "express-sitemap-xml": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/express-sitemap-xml/-/express-sitemap-xml-3.0.0.tgz",
+      "integrity": "sha512-5cim7WY3g5C2HpK0sJAS749PSEhCV/NtQzdkdor1PzTdTHqS405h/obWJy6rz4UpU+s07+awK9uRQTIbZyQCsQ==",
+      "requires": {
+        "p-memoize": "^4.0.1",
+        "xmlbuilder": "^15.1.1"
+      }
+    },
     "extend": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@@ -2372,6 +2502,14 @@
       "resolved": "https://registry.npmjs.org/lodash.flow/-/lodash.flow-3.5.0.tgz",
       "integrity": "sha1-h79AKSuM+D5OjOGjrkIJ4gBxZ1o="
     },
+    "map-age-cleaner": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz",
+      "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==",
+      "requires": {
+        "p-defer": "^1.0.0"
+      }
+    },
     "markdown-it": {
       "version": "11.0.1",
       "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-11.0.1.tgz",
@@ -2416,10 +2554,14 @@
       "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
       "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
     },
-    "memory-cache": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/memory-cache/-/memory-cache-0.2.0.tgz",
-      "integrity": "sha1-eJCwHVLADI68nVM+H46xfjA0hxo="
+    "mem": {
+      "version": "6.1.1",
+      "resolved": "https://registry.npmjs.org/mem/-/mem-6.1.1.tgz",
+      "integrity": "sha512-Ci6bIfq/UgcxPTYa8dQQ5FY3BzKkT894bwXWXxC/zqs0XgMO2cT20CGkOqda7gZNkmK5VP4x89IGZ6K7hfbn3Q==",
+      "requires": {
+        "map-age-cleaner": "^0.1.3",
+        "mimic-fn": "^3.0.0"
+      }
     },
     "merge-descriptors": {
       "version": "1.0.1",
@@ -2449,6 +2591,11 @@
         "mime-db": "1.47.0"
       }
     },
+    "mimic-fn": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-3.1.0.tgz",
+      "integrity": "sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ=="
+    },
     "minimatch": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
@@ -2498,6 +2645,20 @@
         "word-wrap": "~1.2.3"
       }
     },
+    "p-defer": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz",
+      "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww="
+    },
+    "p-memoize": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/p-memoize/-/p-memoize-4.0.1.tgz",
+      "integrity": "sha512-km0sP12uE0dOZ5qP+s7kGVf07QngxyG0gS8sYFvFWhqlgzOsSy+m71aUejf/0akxj5W7gE//2G74qTv6b4iMog==",
+      "requires": {
+        "mem": "^6.0.1",
+        "mimic-fn": "^3.0.0"
+      }
+    },
     "parse5": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz",
@@ -2686,6 +2847,11 @@
         "send": "0.17.1"
       }
     },
+    "setheaders": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/setheaders/-/setheaders-0.3.0.tgz",
+      "integrity": "sha1-PnSfM3ou58GGwuDdB/RftHJIjhM="
+    },
     "setprototypeof": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
@@ -2741,6 +2907,11 @@
       "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
       "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="
     },
+    "tickle": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/tickle/-/tickle-1.4.0.tgz",
+      "integrity": "sha1-uBkLxqcEXo3+ePQJewTWedpCQu0="
+    },
     "toidentifier": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@@ -2902,6 +3073,11 @@
       "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz",
       "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw=="
     },
+    "xmlbuilder": {
+      "version": "15.1.1",
+      "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz",
+      "integrity": "sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg=="
+    },
     "xmlchars": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",