Add sitemap and robots

astro.config.mjs

@@ -8,6 +8,8 @@ import icon from "astro-icon";
 
 import mdx from "@astrojs/mdx";
 
+import sitemap from "@astrojs/sitemap";
+
 // https://astro.build/config
 export default defineConfig({
   vite: {
@@ -24,7 +26,7 @@ export default defineConfig({
     }
   },
 
-  integrations: [icon(), mdx()],
+  integrations: [icon(), mdx(), sitemap()],
 
   site: "https://www.nathancummins.com.au",
 
@@ -35,4 +37,4 @@ export default defineConfig({
   },
 
   output: "static"
-});
+});

package-lock.json

@@ -9,6 +9,7 @@
       "version": "0.0.1",
       "dependencies": {
         "@astrojs/mdx": "^4.3.1",
+        "@astrojs/sitemap": "^3.4.2",
         "@iconify-json/fa": "^1.2.1",
         "@iconify-json/fa7-brands": "^1.2.0",
         "@iconify-json/fa7-solid": "^1.2.0",
@@ -134,6 +135,17 @@
         "node": "18.20.8 || ^20.3.0 || >=22.0.0"
       }
     },
+    "node_modules/@astrojs/sitemap": {
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/@astrojs/sitemap/-/sitemap-3.4.2.tgz",
+      "integrity": "sha512-wfN2dZzdkto6yaMtOFa/J9gc60YE3wl3rgSBoNJ+MU3lJVUMsDY9xf9uAVi8Mp/zEQKFDSJlQzBvqQUpw0Hf6g==",
+      "license": "MIT",
+      "dependencies": {
+        "sitemap": "^8.0.0",
+        "stream-replace-string": "^2.0.0",
+        "zod": "^3.24.4"
+      }
+    },
     "node_modules/@astrojs/telemetry": {
       "version": "3.3.0",
       "license": "MIT",
@@ -988,6 +1000,15 @@
         "undici-types": "~7.8.0"
       }
     },
+    "node_modules/@types/sax": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz",
+      "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/tar": {
       "version": "6.1.13",
       "license": "MIT",
@@ -1345,6 +1366,12 @@
         "url": "https://github.com/sponsors/jonschlinkert"
       }
     },
+    "node_modules/arg": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
+      "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==",
+      "license": "MIT"
+    },
     "node_modules/argparse": {
       "version": "2.0.1",
       "license": "Python-2.0"
@@ -5791,6 +5818,12 @@
         "suf-log": "^2.5.3"
       }
     },
+    "node_modules/sax": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz",
+      "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==",
+      "license": "ISC"
+    },
     "node_modules/semver": {
       "version": "7.7.2",
       "license": "ISC",
@@ -5884,6 +5917,31 @@
       "version": "1.0.5",
       "license": "MIT"
     },
+    "node_modules/sitemap": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-8.0.0.tgz",
+      "integrity": "sha512-+AbdxhM9kJsHtruUF39bwS/B0Fytw6Fr1o4ZAIAEqA6cke2xcoO2GleBw9Zw7nRzILVEgz7zBM5GiTJjie1G9A==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "^17.0.5",
+        "@types/sax": "^1.2.1",
+        "arg": "^5.0.0",
+        "sax": "^1.2.4"
+      },
+      "bin": {
+        "sitemap": "dist/cli.js"
+      },
+      "engines": {
+        "node": ">=14.0.0",
+        "npm": ">=6.0.0"
+      }
+    },
+    "node_modules/sitemap/node_modules/@types/node": {
+      "version": "17.0.45",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz",
+      "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==",
+      "license": "MIT"
+    },
     "node_modules/smol-toml": {
       "version": "1.4.1",
       "license": "BSD-3-Clause",
@@ -5916,6 +5974,12 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
+    "node_modules/stream-replace-string": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/stream-replace-string/-/stream-replace-string-2.0.0.tgz",
+      "integrity": "sha512-TlnjJ1C0QrmxRNrON00JvaFFlNh5TTG00APw23j74ET7gkQpTASi6/L2fuiav8pzK715HXtUeClpBTw2NPSn6w==",
+      "license": "MIT"
+    },
     "node_modules/string-width": {
       "version": "7.2.0",
       "license": "MIT",

package.json

@@ -10,6 +10,7 @@
   },
   "dependencies": {
     "@astrojs/mdx": "^4.3.1",
+    "@astrojs/sitemap": "^3.4.2",
     "@iconify-json/fa": "^1.2.1",
     "@iconify-json/fa7-brands": "^1.2.0",
     "@iconify-json/fa7-solid": "^1.2.0",

src/layouts/MainHead.astro

@@ -33,6 +33,7 @@ const headDescription: string = description || site.description;
   <link rel="icon" type="image/png" href="/favicon.png" />
   <link rel="apple-touch-icon" href="/favicon.png" />
 
+  <link rel="sitemap" href="/sitemap-index.xml" />
   <SEO title={headTitle} description={headDescription} {image} />
 
   <ThemeManager defaultTheme="auto" />

src/pages/robots.txt.ts

@@ -0,0 +1,13 @@
+import type { APIRoute } from "astro";
+
+const getRobotsTxt = (sitemapURL: URL) => `\
+User-agent: *
+Allow: /
+
+Sitemap: ${sitemapURL.href}
+`;
+
+export const GET: APIRoute = ({ site }) => {
+  const sitemapURL = new URL("sitemap-index.xml", site);
+  return new Response(getRobotsTxt(sitemapURL));
+};