feat: add robots.txt & use main tag

astro.config.mjs

@@ -14,11 +14,13 @@ import compress from "astro-compress";
 
 // https://astro.build/config
 import prefetch from "@astrojs/prefetch";
+
+import robotsTxt from "astro-robots-txt";
 import minify from "./astro-integrations/minify/index.mjs";
 
 // https://astro.build/config
 export default defineConfig({
-    site: "https://stargazers.club", //prepisat uwu
+    site: "https://hyrousek.tk", //prepisat uwu
     integrations: [
         tailwind(),
         image(),
@@ -27,7 +29,15 @@ export default defineConfig({
             svg: false,
             html: false,
         }),
+        prefetch(),
+        robotsTxt({
+            policy: [
+                {
+                    userAgent: "*",
+                }
+            ],
+            sitemap: true,
+        }),
         minify(),
-        prefetch()
     ],
 });

package-lock.json

@@ -14,6 +14,7 @@
         "@astrojs/tailwind": "^2.1.3",
         "astro": "^1.6.12",
         "astro-compress": "^1.1.17",
+        "astro-robots-txt": "^0.3.10",
         "fireworks-js": "^2.10.0",
         "html-minifier": "^4.0.0",
         "tailwindcss": "^3.2.4"
@@ -1200,6 +1201,16 @@
         "@nikolarhristov/pipeline": "0.0.12"
       }
     },
+    "node_modules/astro-robots-txt": {
+      "version": "0.3.10",
+      "resolved": "https://registry.npmjs.org/astro-robots-txt/-/astro-robots-txt-0.3.10.tgz",
+      "integrity": "sha512-fVQOdiVFPgMJEILUFBpNsup3q/RBvYiiecQVyPeWo3Ss+CzjMCkbWBflbFr86+9tTkiX0q1dufNem03QnorJ/w==",
+      "dependencies": {
+        "deepmerge": "^4.2.2",
+        "valid-filename": "^4.0.0",
+        "zod": "^3.19.1"
+      }
+    },
     "node_modules/autoprefixer": {
       "version": "10.4.13",
       "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz",
@@ -2652,6 +2663,17 @@
         "node": "^12.20 || >= 14.13"
       }
     },
+    "node_modules/filename-reserved-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-3.0.0.tgz",
+      "integrity": "sha512-hn4cQfU6GOT/7cFHXBqeBg2TbrMBgdD0kcjLhvSQYYwm3s4B6cjvBfb7nBALJLAXqmU5xajSa7X2NnUud/VCdw==",
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/fill-range": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -7183,6 +7205,20 @@
         "node": ">=8"
       }
     },
+    "node_modules/valid-filename": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/valid-filename/-/valid-filename-4.0.0.tgz",
+      "integrity": "sha512-VEYTpTVPMgO799f2wI7zWf0x2C54bPX6NAfbZ2Z8kZn76p+3rEYCTYVYzMUcVSMvakxMQTriBf24s3+WeXJtEg==",
+      "dependencies": {
+        "filename-reserved-regex": "^3.0.0"
+      },
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/vfile": {
       "version": "5.3.6",
       "resolved": "https://registry.npmjs.org/vfile/-/vfile-5.3.6.tgz",
@@ -8849,6 +8885,16 @@
         "@nikolarhristov/pipeline": "0.0.12"
       }
     },
+    "astro-robots-txt": {
+      "version": "0.3.10",
+      "resolved": "https://registry.npmjs.org/astro-robots-txt/-/astro-robots-txt-0.3.10.tgz",
+      "integrity": "sha512-fVQOdiVFPgMJEILUFBpNsup3q/RBvYiiecQVyPeWo3Ss+CzjMCkbWBflbFr86+9tTkiX0q1dufNem03QnorJ/w==",
+      "requires": {
+        "deepmerge": "^4.2.2",
+        "valid-filename": "^4.0.0",
+        "zod": "^3.19.1"
+      }
+    },
     "autoprefixer": {
       "version": "10.4.13",
       "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz",
@@ -9740,6 +9786,11 @@
         "web-streams-polyfill": "^3.0.3"
       }
     },
+    "filename-reserved-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-3.0.0.tgz",
+      "integrity": "sha512-hn4cQfU6GOT/7cFHXBqeBg2TbrMBgdD0kcjLhvSQYYwm3s4B6cjvBfb7nBALJLAXqmU5xajSa7X2NnUud/VCdw=="
+    },
     "fill-range": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -12678,6 +12729,14 @@
         "sade": "^1.7.3"
       }
     },
+    "valid-filename": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/valid-filename/-/valid-filename-4.0.0.tgz",
+      "integrity": "sha512-VEYTpTVPMgO799f2wI7zWf0x2C54bPX6NAfbZ2Z8kZn76p+3rEYCTYVYzMUcVSMvakxMQTriBf24s3+WeXJtEg==",
+      "requires": {
+        "filename-reserved-regex": "^3.0.0"
+      }
+    },
     "vfile": {
       "version": "5.3.6",
       "resolved": "https://registry.npmjs.org/vfile/-/vfile-5.3.6.tgz",

package.json

@@ -17,6 +17,7 @@
     "@astrojs/tailwind": "^2.1.3",
     "astro": "^1.6.12",
     "astro-compress": "^1.1.17",
+    "astro-robots-txt": "^0.3.10",
     "fireworks-js": "^2.10.0",
     "html-minifier": "^4.0.0",
     "tailwindcss": "^3.2.4"

src/layouts/BlogLayout.astro

@@ -21,8 +21,8 @@ const {
     </h1>
 
     <Container>
-        <div class="text-white text-2xl">
+        <main class="text-white text-2xl">
             <slot />
-        </div>
+        </main>
     </Container>
 </Layout>

src/pages/blog/index.astro

@@ -15,6 +15,7 @@ const posts = (await Astro.glob('./posts/*.md')).sort(
     </h1>
 
     <Container>
+        <main>
             {posts.map(post => {
                 return (
                     <a class="text-2xl font-bold text-white mb-4 break-all" href={post.url}>{post.frontmatter.title}</a>
@@ -28,5 +29,6 @@ const posts = (await Astro.glob('./posts/*.md')).sort(
                     <hr class="text-gray" />
                 )
             })}
+        </main>
     </Container>
 </Layout>