robots.txt: add more explicit allow rules while disallowing xml

Mikaela · Jun 2, 2024 · b23e622 · b23e622
1 parent b87a66d
commit b23e622
Showing 1 changed file with 11 additions and 0 deletions.
diff --git a/robots.txt b/robots.txt
@@ -1,10 +1,21 @@
 User-agent: *
 Allow: /
+Allow: /$
+Allow: /blog/$
+Allow: /blog/feed.xml
+Allow: /blog/feed.en.xml
+Allow: /blog/feed.fi.xml
+Allow: /irc/$
+Allow: /matrix/$
+Allow: /n/$
+Allow: /sitemap.xml
+Allow: /sitemaps.xml
 Disallow: /*.asc$
 Disallow: /*.json$
 Disallow: /*.sig$
 Disallow: /*.txt$
 Disallow: /*.toml$
+Disallow: /*.xml$
 Disallow: /txt/
 Disallow: /php/
 Disallow: /r/