git: 7808bd27ab - main - Enable robots.txt in docs.freebsd.org

[Sergio Carlavilla Delgado]

The branch main has been updated by carlavilla:

URL: https://cgit.FreeBSD.org/doc/commit/?id=7808bd27abab51b4602005cdcd593ac18e59256c

commit 7808bd27abab51b4602005cdcd593ac18e59256c
Author:     Sergio Carlavilla Delgado <carlavilla at FreeBSD.org>
AuthorDate: 2021-01-26 19:12:37 +0000
Commit:     Sergio Carlavilla Delgado <carlavilla at FreeBSD.org>
CommitDate: 2021-01-26 19:17:05 +0000

    Enable robots.txt in docs.freebsd.org
    
    Enable again robots.txt for docs.freebsd.org
    
    PR:             253031
    Submitted by:   wosch@
---
 documentation/config/_default/config.toml       |  1 +
 documentation/themes/beastie/layouts/robots.txt | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/documentation/config/_default/config.toml b/documentation/config/_default/config.toml
index ed455cd1b0..9d0acfa24e 100644
--- a/documentation/config/_default/config.toml
+++ b/documentation/config/_default/config.toml
@@ -12,6 +12,7 @@ disableKinds = [ "taxonomy", "taxonomyTerm" ]
 authors = [ "carlavilla at FreeBSD.org" ]
 preserveTOC = true
 ignoreFiles = [ "chapters-order.adoc$", "toc.adoc$", "toc-tables.adoc$", "toc-figures.adoc$", "toc-examples.adoc$", "toc-1.adoc$", "toc-2.adoc$", "toc-3.adoc$", "toc-4.adoc$", "toc-5.adoc$", "books.adoc$", "chapter.adoc$" ]
+enableRobotsTXT = true
 
 [params]
   websiteURL = "https://www-dev.freebsd.org/"
diff --git a/documentation/themes/beastie/layouts/robots.txt b/documentation/themes/beastie/layouts/robots.txt
new file mode 100644
index 0000000000..b94bcd43cf
--- /dev/null
+++ b/documentation/themes/beastie/layouts/robots.txt
@@ -0,0 +1,19 @@
+# $FreeBSD$
+
+User-agent: *
+#Crawl-delay: 10
+Disallow: /cgi/confirm-code.cgi
+Disallow: /cgi/cvsweb.cgi
+Disallow: /cgi/dosendpr.cgi
+Disallow: /cgi/getmsg.cgi
+Disallow: /cgi/mailindex.cgi
+Disallow: /cgi/man.cgi?apropos=2&
+Disallow: /cgi/mid.cgi
+Disallow: /cgi/mirror.cgi
+Disallow: /cgi/missing_handler.cgi
+Disallow: /cgi/ports.cgi
+Disallow: /cgi/query-pr-summary.cgi
+Disallow: /cgi/query-pr.cgi
+Disallow: /cgi/search.cgi
+Disallow: /cgi/url.cgi
+Disallow: /statistic