[evolvis-commits] r18609: revive using a robots.txt file to, by default, slow down crawlers
mirabilos at evolvis.org
mirabilos at evolvis.org
Fri Sep 21 16:37:11 CEST 2012
Author: mirabilos
Date: 2012-09-21 16:37:11 +0200 (Fri, 21 Sep 2012)
New Revision: 18609
Added:
trunk/gforge_base/evolvisforge-5.1/src/etc/robots.txt
Modified:
trunk/gforge_base/evolvisforge-5.1/src/debian/changelog
trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-list.inc
trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-main.inc
trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-list.inc
trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-main.inc
trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-list.inc
trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-main.inc
trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-list.inc
trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-main.inc
trunk/gforge_base/evolvisforge-5.1/src/packaging/install/web-apache2
Log:
revive using a robots.txt file to, by default, slow down crawlers
now as dpkg conffile
Modified: trunk/gforge_base/evolvisforge-5.1/src/debian/changelog
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/debian/changelog 2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/debian/changelog 2012-09-21 14:37:11 UTC (rev 18609)
@@ -14,8 +14,9 @@
* [#3205] Use gitweb from wheezy
* Some packaging fixes
* Project Summary: sort usernames in Project Members widget
+ * Revive robots.txt file from (4.8) 35 except now /etc/gforge/custom
- -- Thorsten Glaser <t.glaser at tarent.de> Fri, 21 Sep 2012 11:00:52 +0200
+ -- Thorsten Glaser <t.glaser at tarent.de> Fri, 21 Sep 2012 14:23:57 +0200
fusionforge (1:5.1.1+evolvis69) unstable; urgency=medium
Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-list.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-list.inc 2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-list.inc 2012-09-21 14:37:11 UTC (rev 18609)
@@ -1,5 +1,7 @@
ServerName {core/lists_host}
+Alias /robots.txt {core/custom_path}/robots.txt
+
RedirectMatch permanent ^/$ https://{core/web_host}/mailman/listinfo
RedirectMatch permanent ^/(pipermail/.*) https://{core/web_host}/$1
RedirectMatch permanent ^/(mailman/.*) https://{core/web_host}/$1
Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-main.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-main.inc 2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-main.inc 2012-09-21 14:37:11 UTC (rev 18609)
@@ -26,6 +26,7 @@
Include {core/config_path}/httpd.conf.d/plugin-*.inc
Alias /images/ {core/source_path}/www/images/
+Alias /robots.txt {core/custom_path}/robots.txt
# For FusionForge without DNS delegation, uncomment this
# and remove 30-vhosts-projects.conf
Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-list.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-list.inc 2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-list.inc 2012-09-21 14:37:11 UTC (rev 18609)
@@ -1 +1,3 @@
ServerName {core/lists_host}
+
+Alias /robots.txt /etc/gforge/custom/robots.txt
Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-main.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-main.inc 2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-main.inc 2012-09-21 14:37:11 UTC (rev 18609)
@@ -26,6 +26,7 @@
Include /etc/gforge/httpd.conf.d/plugin-*.inc
Alias /images/ /usr/share/gforge/www/images/
+Alias /robots.txt /etc/gforge/custom/robots.txt
# For FusionForge without DNS delegation, uncomment this
# and remove 30-vhosts-projects.conf
Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-list.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-list.inc 2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-list.inc 2012-09-21 14:37:11 UTC (rev 18609)
@@ -1 +1,3 @@
ServerName {core/lists_host}
+
+Alias /robots.txt /etc/gforge/custom/robots.txt
Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-main.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-main.inc 2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-main.inc 2012-09-21 14:37:11 UTC (rev 18609)
@@ -26,6 +26,7 @@
Include /etc/gforge/httpd.conf.d/plugin-*.inc
Alias /images/ /opt/gforge/www/images/
+Alias /robots.txt /etc/gforge/custom/robots.txt
# For FusionForge without DNS delegation, uncomment this
# and remove 30-vhosts-projects.conf
Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-list.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-list.inc 2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-list.inc 2012-09-21 14:37:11 UTC (rev 18609)
@@ -1 +1,3 @@
ServerName {core/lists_host}
+
+Alias /robots.txt /etc/gforge/custom/robots.txt
Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-main.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-main.inc 2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-main.inc 2012-09-21 14:37:11 UTC (rev 18609)
@@ -26,6 +26,7 @@
Include /etc/gforge/httpd.conf.d/plugin-*.inc
Alias /images/ /usr/local/share/gforge/www/images/
+Alias /robots.txt /etc/gforge/custom/robots.txt
# For FusionForge without DNS delegation, uncomment this
# and remove 30-vhosts-projects.conf
Added: trunk/gforge_base/evolvisforge-5.1/src/etc/robots.txt
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/robots.txt (rev 0)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/robots.txt 2012-09-21 14:37:11 UTC (rev 18609)
@@ -0,0 +1,15 @@
+# Allow downloads
+User-Agent: wget
+Crawl-delay: 30
+Disallow:
+
+# Disallow all spiders
+#User-Agent: *
+#Disallow: /
+
+# - or -
+
+# Ask spiders to slow down
+User-Agent: *
+Crawl-delay: 7
+Disallow: /admin/
Modified: trunk/gforge_base/evolvisforge-5.1/src/packaging/install/web-apache2
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/packaging/install/web-apache2 2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/packaging/install/web-apache2 2012-09-21 14:37:11 UTC (rev 18609)
@@ -5,4 +5,5 @@
utils/manage-apache-config.sh usr/share/@OLDPACKAGE@/bin/
etc/httpd.conf.d-fhs usr/share/@OLDPACKAGE@/etc/
etc/httpd.conf-fhs usr/share/@OLDPACKAGE@/etc/
-evolvis-captcha.flf usr/share/figlet/
\ No newline at end of file
+evolvis-captcha.flf usr/share/figlet/
+etc/robots.txt etc/@OLDPACKAGE@/custom/
More information about the evolvis-commits
mailing list