[evolvis-commits] r18609: revive using a robots.txt file to, by default, slow down crawlers

mirabilos at evolvis.org mirabilos at evolvis.org
Fri Sep 21 16:37:11 CEST 2012


Author: mirabilos
Date: 2012-09-21 16:37:11 +0200 (Fri, 21 Sep 2012)
New Revision: 18609

Added:
   trunk/gforge_base/evolvisforge-5.1/src/etc/robots.txt
Modified:
   trunk/gforge_base/evolvisforge-5.1/src/debian/changelog
   trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-list.inc
   trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-main.inc
   trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-list.inc
   trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-main.inc
   trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-list.inc
   trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-main.inc
   trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-list.inc
   trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-main.inc
   trunk/gforge_base/evolvisforge-5.1/src/packaging/install/web-apache2
Log:
revive using a robots.txt file to, by default, slow down crawlers

now as dpkg conffile

Modified: trunk/gforge_base/evolvisforge-5.1/src/debian/changelog
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/debian/changelog	2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/debian/changelog	2012-09-21 14:37:11 UTC (rev 18609)
@@ -14,8 +14,9 @@
   * [#3205] Use gitweb from wheezy
   * Some packaging fixes
   * Project Summary: sort usernames in Project Members widget
+  * Revive robots.txt file from (4.8) 35 except now /etc/gforge/custom
 
- -- Thorsten Glaser <t.glaser at tarent.de>  Fri, 21 Sep 2012 11:00:52 +0200
+ -- Thorsten Glaser <t.glaser at tarent.de>  Fri, 21 Sep 2012 14:23:57 +0200
 
 fusionforge (1:5.1.1+evolvis69) unstable; urgency=medium
 

Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-list.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-list.inc	2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-list.inc	2012-09-21 14:37:11 UTC (rev 18609)
@@ -1,5 +1,7 @@
 ServerName {core/lists_host}
 
+Alias /robots.txt {core/custom_path}/robots.txt
+
 RedirectMatch permanent ^/$ https://{core/web_host}/mailman/listinfo
 RedirectMatch permanent ^/(pipermail/.*) https://{core/web_host}/$1
 RedirectMatch permanent ^/(mailman/.*) https://{core/web_host}/$1

Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-main.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-main.inc	2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d/vhost-main.inc	2012-09-21 14:37:11 UTC (rev 18609)
@@ -26,6 +26,7 @@
 Include {core/config_path}/httpd.conf.d/plugin-*.inc
 
 Alias /images/ {core/source_path}/www/images/
+Alias /robots.txt {core/custom_path}/robots.txt
 
 # For FusionForge without DNS delegation, uncomment this
 # and remove 30-vhosts-projects.conf

Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-list.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-list.inc	2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-list.inc	2012-09-21 14:37:11 UTC (rev 18609)
@@ -1 +1,3 @@
 ServerName {core/lists_host}
+
+Alias /robots.txt /etc/gforge/custom/robots.txt

Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-main.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-main.inc	2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-fhs/vhost-main.inc	2012-09-21 14:37:11 UTC (rev 18609)
@@ -26,6 +26,7 @@
 Include /etc/gforge/httpd.conf.d/plugin-*.inc
 
 Alias /images/ /usr/share/gforge/www/images/
+Alias /robots.txt /etc/gforge/custom/robots.txt
 
 # For FusionForge without DNS delegation, uncomment this
 # and remove 30-vhosts-projects.conf

Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-list.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-list.inc	2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-list.inc	2012-09-21 14:37:11 UTC (rev 18609)
@@ -1 +1,3 @@
 ServerName {core/lists_host}
+
+Alias /robots.txt /etc/gforge/custom/robots.txt

Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-main.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-main.inc	2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-opt/vhost-main.inc	2012-09-21 14:37:11 UTC (rev 18609)
@@ -26,6 +26,7 @@
 Include /etc/gforge/httpd.conf.d/plugin-*.inc
 
 Alias /images/ /opt/gforge/www/images/
+Alias /robots.txt /etc/gforge/custom/robots.txt
 
 # For FusionForge without DNS delegation, uncomment this
 # and remove 30-vhosts-projects.conf

Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-list.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-list.inc	2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-list.inc	2012-09-21 14:37:11 UTC (rev 18609)
@@ -1 +1,3 @@
 ServerName {core/lists_host}
+
+Alias /robots.txt /etc/gforge/custom/robots.txt

Modified: trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-main.inc
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-main.inc	2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/httpd.conf.d-usrlocal/vhost-main.inc	2012-09-21 14:37:11 UTC (rev 18609)
@@ -26,6 +26,7 @@
 Include /etc/gforge/httpd.conf.d/plugin-*.inc
 
 Alias /images/ /usr/local/share/gforge/www/images/
+Alias /robots.txt /etc/gforge/custom/robots.txt
 
 # For FusionForge without DNS delegation, uncomment this
 # and remove 30-vhosts-projects.conf

Added: trunk/gforge_base/evolvisforge-5.1/src/etc/robots.txt
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/etc/robots.txt	                        (rev 0)
+++ trunk/gforge_base/evolvisforge-5.1/src/etc/robots.txt	2012-09-21 14:37:11 UTC (rev 18609)
@@ -0,0 +1,15 @@
+# Allow downloads
+User-Agent: wget
+Crawl-delay: 30
+Disallow:
+
+# Disallow all spiders
+#User-Agent: *
+#Disallow: /
+
+# - or -
+
+# Ask spiders to slow down
+User-Agent: *
+Crawl-delay: 7
+Disallow: /admin/

Modified: trunk/gforge_base/evolvisforge-5.1/src/packaging/install/web-apache2
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/src/packaging/install/web-apache2	2012-09-21 09:45:21 UTC (rev 18608)
+++ trunk/gforge_base/evolvisforge-5.1/src/packaging/install/web-apache2	2012-09-21 14:37:11 UTC (rev 18609)
@@ -5,4 +5,5 @@
 utils/manage-apache-config.sh	    usr/share/@OLDPACKAGE@/bin/
 etc/httpd.conf.d-fhs		    usr/share/@OLDPACKAGE@/etc/
 etc/httpd.conf-fhs		    usr/share/@OLDPACKAGE@/etc/
-evolvis-captcha.flf		    usr/share/figlet/
\ No newline at end of file
+evolvis-captcha.flf		    usr/share/figlet/
+etc/robots.txt			    etc/@OLDPACKAGE@/custom/



More information about the evolvis-commits mailing list