feat: robots

astro.config.ts

@@ -5,6 +5,7 @@ import sitemap from '@astrojs/sitemap';
 import image from '@astrojs/image';
 import vercel from '@astrojs/vercel/static';
 import compress from 'astro-compress';
+import robotsTxt from 'astro-robots-txt';
 
 import { SITE } from './src/config';
 
@@ -24,8 +25,14 @@ export default defineConfig({
 			img: false,
 			js: true,
 			svg: false,
-
+		}),
-			logger: 1,
+		robotsTxt({
+			policy: [
+				{
+					userAgent: '*',
+				},
+			],
+			sitemap: true,
 		}),
 	],
 	adapter: vercel(),

package-lock.json

@@ -17,6 +17,9 @@
         "autoprefixer": "^10.4.13",
         "cssnano": "^5.1.14",
         "tailwindcss": "^3.2.4"
+      },
+      "devDependencies": {
+        "astro-robots-txt": "^0.3.10"
       }
     },
     "node_modules/@altano/tiny-async-pool": {
@@ -1444,6 +1447,17 @@
         "url": "https://opencollective.com/svgo"
       }
     },
+    "node_modules/astro-robots-txt": {
+      "version": "0.3.10",
+      "resolved": "https://registry.npmjs.org/astro-robots-txt/-/astro-robots-txt-0.3.10.tgz",
+      "integrity": "sha512-fVQOdiVFPgMJEILUFBpNsup3q/RBvYiiecQVyPeWo3Ss+CzjMCkbWBflbFr86+9tTkiX0q1dufNem03QnorJ/w==",
+      "dev": true,
+      "dependencies": {
+        "deepmerge": "^4.2.2",
+        "valid-filename": "^4.0.0",
+        "zod": "^3.19.1"
+      }
+    },
     "node_modules/async-sema": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/async-sema/-/async-sema-3.1.1.tgz",
@@ -3000,6 +3014,18 @@
       "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
       "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
     },
+    "node_modules/filename-reserved-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-3.0.0.tgz",
+      "integrity": "sha512-hn4cQfU6GOT/7cFHXBqeBg2TbrMBgdD0kcjLhvSQYYwm3s4B6cjvBfb7nBALJLAXqmU5xajSa7X2NnUud/VCdw==",
+      "dev": true,
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/files-pipeline": {
       "version": "0.0.2",
       "resolved": "https://registry.npmjs.org/files-pipeline/-/files-pipeline-0.0.2.tgz",
@@ -7775,6 +7801,21 @@
         "node": ">=8"
       }
     },
+    "node_modules/valid-filename": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/valid-filename/-/valid-filename-4.0.0.tgz",
+      "integrity": "sha512-VEYTpTVPMgO799f2wI7zWf0x2C54bPX6NAfbZ2Z8kZn76p+3rEYCTYVYzMUcVSMvakxMQTriBf24s3+WeXJtEg==",
+      "dev": true,
+      "dependencies": {
+        "filename-reserved-regex": "^3.0.0"
+      },
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/vfile": {
       "version": "5.3.6",
       "resolved": "https://registry.npmjs.org/vfile/-/vfile-5.3.6.tgz",
@@ -9332,6 +9373,17 @@
         }
       }
     },
+    "astro-robots-txt": {
+      "version": "0.3.10",
+      "resolved": "https://registry.npmjs.org/astro-robots-txt/-/astro-robots-txt-0.3.10.tgz",
+      "integrity": "sha512-fVQOdiVFPgMJEILUFBpNsup3q/RBvYiiecQVyPeWo3Ss+CzjMCkbWBflbFr86+9tTkiX0q1dufNem03QnorJ/w==",
+      "dev": true,
+      "requires": {
+        "deepmerge": "^4.2.2",
+        "valid-filename": "^4.0.0",
+        "zod": "^3.19.1"
+      }
+    },
     "async-sema": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/async-sema/-/async-sema-3.1.1.tgz",
@@ -10309,6 +10361,12 @@
       "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
       "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
     },
+    "filename-reserved-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-3.0.0.tgz",
+      "integrity": "sha512-hn4cQfU6GOT/7cFHXBqeBg2TbrMBgdD0kcjLhvSQYYwm3s4B6cjvBfb7nBALJLAXqmU5xajSa7X2NnUud/VCdw==",
+      "dev": true
+    },
     "files-pipeline": {
       "version": "0.0.2",
       "resolved": "https://registry.npmjs.org/files-pipeline/-/files-pipeline-0.0.2.tgz",
@@ -13495,6 +13553,15 @@
         "sade": "^1.7.3"
       }
     },
+    "valid-filename": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/valid-filename/-/valid-filename-4.0.0.tgz",
+      "integrity": "sha512-VEYTpTVPMgO799f2wI7zWf0x2C54bPX6NAfbZ2Z8kZn76p+3rEYCTYVYzMUcVSMvakxMQTriBf24s3+WeXJtEg==",
+      "dev": true,
+      "requires": {
+        "filename-reserved-regex": "^3.0.0"
+      }
+    },
     "vfile": {
       "version": "5.3.6",
       "resolved": "https://registry.npmjs.org/vfile/-/vfile-5.3.6.tgz",

package.json

@@ -20,5 +20,8 @@
     "autoprefixer": "^10.4.13",
     "cssnano": "^5.1.14",
     "tailwindcss": "^3.2.4"
+  },
+  "devDependencies": {
+    "astro-robots-txt": "^0.3.10"
   }
 }