[Evolvis-commits] r248: This is a mega script making cvs stats tables from history files↵

mirabilos at evolvis.org mirabilos at evolvis.org
Thu Feb 25 14:12:23 CET 2010


Author: mirabilos
Date: 2010-02-25 13:12:22 +0000 (Thu, 25 Feb 2010)
New Revision: 248

Added:
   trunk/gforge_base/evolvisforge/gforge/deb-specific/stats_cvs.pl
Modified:
   trunk/gforge_base/evolvisforge/
Log:
This is a mega script making cvs stats tables from history files



Property changes on: trunk/gforge_base/evolvisforge
___________________________________________________________________
Name: bzr:revision-info
   - timestamp: 2001-08-15 21:01:35.000000000 +0000
committer: cbayle

   + timestamp: 2001-08-17 14:02:07.000000000 +0000
committer: cbayle

Name: bzr:file-ids
   - gforge/cronjobs/stats/README	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2FREADME
gforge/cronjobs/stats/README.debian	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2FREADME.debian
gforge/cronjobs/stats/backfill.sh	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fbackfill.sh
gforge/cronjobs/stats/create.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fcreate.sql
gforge/cronjobs/stats/cvsserver/stats_tables.sql	15 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fcvsserver%2Fstats_tables.sql
gforge/cronjobs/stats/cvsserver/stats_tables_drop.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fcvsserver%2Fstats_tables_drop.sql
gforge/cronjobs/stats/db_stats_agg.php	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fdb_stats_agg.php
gforge/cronjobs/stats/db_stats_agg.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fdb_stats_agg.sql
gforge/cronjobs/stats/db_stats_agg_drop.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fdb_stats_agg_drop.sql
gforge/cronjobs/stats/download	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fdownload
gforge/cronjobs/stats/download/stats_ftp_logparse.pl	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fdownload%2Fstats_ftp_logparse.pl
gforge/cronjobs/stats/download/stats_http_logparse.pl	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fdownload%2Fstats_http_logparse.pl
gforge/cronjobs/stats/download/stats_logparse.sh	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fdownload%2Fstats_logparse.sh
gforge/cronjobs/stats/download/stats_sum.pl	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fdownload%2Fstats_sum.pl
gforge/cronjobs/stats/download/stats_tables.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fdownload%2Fstats_tables.sql
gforge/cronjobs/stats/download/stats_tables_drop.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fdownload%2Fstats_tables_drop.sql
gforge/cronjobs/stats/drop.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fdrop.sql
gforge/cronjobs/stats/include	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Finclude
gforge/cronjobs/stats/include/stats_projects.inc	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Finclude%2Fstats_projects.inc
gforge/cronjobs/stats/include/stats_site.inc	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Finclude%2Fstats_site.inc
gforge/cronjobs/stats/site_stats.php	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fsite_stats.php
gforge/cronjobs/stats/site_stats.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fsite_stats.sql
gforge/cronjobs/stats/site_stats_drop.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fsite_stats_drop.sql
gforge/cronjobs/stats/stats.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fstats.sql
gforge/cronjobs/stats/stats_drop.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fstats_drop.sql
gforge/cronjobs/stats/stats_projects-backfill.php	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fstats_projects-backfill.php
gforge/cronjobs/stats/subdomains	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fsubdomains
gforge/cronjobs/stats/subdomains/stats_projects_logparse.pl	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fsubdomains%2Fstats_projects_logparse.pl
gforge/cronjobs/stats/subdomains/stats_tables.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fsubdomains%2Fstats_tables.sql
gforge/cronjobs/stats/subdomains/stats_tables_drop.sql	18 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fcronjobs%2Fstats%2Fsubdomains%2Fstats_tables_drop.sql

   + gforge/deb-specific/stats_cvs.pl	19 at 9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk%2Fgforge%2Fdeb-specific%2Fstats_cvs.pl

Name: bzr:revision-id:v4
   - 1 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:1
2 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:2
3 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:7
4 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:9
5 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:10
6 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:11
7 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:12
8 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:13
9 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:14
10 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:15
11 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:16
12 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:17
13 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:18

   + 1 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:1
2 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:2
3 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:7
4 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:9
5 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:10
6 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:11
7 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:12
8 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:13
9 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:14
10 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:15
11 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:16
12 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:17
13 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:18
14 svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:19

Name: bzr:text-parents
   - gforge/cronjobs/stats/cvsserver/stats_tables.sql	svn-v4:9d84d37e-dcb1-4aad-b103-6f3d92f53bf6:trunk:16

   + 

Added: trunk/gforge_base/evolvisforge/gforge/deb-specific/stats_cvs.pl
===================================================================
--- trunk/gforge_base/evolvisforge/gforge/deb-specific/stats_cvs.pl	                        (rev 0)
+++ trunk/gforge_base/evolvisforge/gforge/deb-specific/stats_cvs.pl	2010-02-25 13:12:22 UTC (rev 248)
@@ -0,0 +1,114 @@
+#!/usr/bin/perl
+#/**
+#  *
+#  * cvsserver_debian.pl - NIGHTLY SCRIPT
+#  *
+#  * Recurses through the /cvsroot directory tree and parses each projects
+#  * '~/CVSROOT/history' file, and create and fill the sql table with 
+#  * checkouts, commits, and adds to each project.
+#  *
+#  * @version   $Id$
+#  *
+#  */
+
+# For the files
+#use strict;
+use Time::Local;
+use POSIX qw( strftime );
+
+# For the database
+use DBI;
+require("/usr/lib/sourceforge/lib/include.pl");
+&db_connect;
+
+my ($sql);
+$sql = "DROP TABLE deb_cvs_dump";
+$dbh->do( $sql );
+$sql = "CREATE TABLE deb_cvs_dump (
+		type char(1),
+		year integer NOT NULL,
+		month integer NOT NULL,
+		day integer NOT NULL,
+		time integer NOT NULL,
+		cvsuser text,
+		cvsgroup text
+	)";
+$dbh->do( $sql );
+
+
+my ($year, $month, $day, $day_begin, $day_end);
+my $verbose = 1;
+my $cvsroot = "/var/lib/sourceforge/chroot/cvsroot";
+
+$|=0 if $verbose;
+$|++;
+
+print "Running tree at $cvsroot/\n";
+
+chdir( "$cvsroot" ) || die("Unable to make $cvsroot the working directory.\n");
+
+foreach $group ( glob("*") ) {
+	next if ( ! -d "$group" );
+	my ($cvs_co, $cvs_commit, $cvs_add, %usr_commit, %usr_add );
+	print "Parsing $group/\n";
+
+	open(HISTORY, "< $cvsroot/$group/CVSROOT/history") or print "E::Unable to open history for $group\n";
+	while ( <HISTORY> ) {
+		my ($time_parsed, $type, $cvstime, $user, $curdir, $module, $rev, $file );
+ 
+		## Split the cvs history entry into it's 6 fields.
+		($cvstime,$user,$curdir,$module,$rev,$file) = split(/\|/, $_, 6 );
+
+		## log commits  $type eq "M" 
+		## log adds  $type eq "A"
+		## log checkouts  $type eq "O" 
+		$type = substr($cvstime, 0, 1);
+		$time_parsed = hex( substr($cvstime, 1, 8) );
+		$year	= strftime("%Y", gmtime( $time_parsed ) );
+		$month	= strftime("%m", gmtime( $time_parsed ) );
+		$day	= strftime("%d", gmtime( $time_parsed ) );
+		$sql = "INSERT INTO deb_cvs_dump 
+		(type,year,month,day,time,cvsuser,cvsgroup)
+		VALUES ('$type','$year','$month','$day','$time_parsed','$user','$group')";
+		
+		#print "$sql";
+		$dbh->do( $sql );
+	}
+	close( HISTORY );
+}
+$sql = "DROP TABLE deb_cvs_group";
+$dbh->do( $sql );
+$sql = "
+CREATE TABLE deb_cvs_group AS
+        SELECT agg.cvsgroup,agg.year,agg.month,agg.day,agg.total AS total,c.commits AS commits,a.adds AS adds,ch.checkouts AS checkouts,e.errors AS errors
+        FROM (
+        	SELECT cvsgroup,year,month,day,COUNT(*) AS total
+        	FROM deb_cvs_dump
+        	GROUP BY year,month,day,cvsgroup
+	) agg
+	LEFT JOIN (
+        SELECT cvsgroup,COUNT(*) AS commits
+        FROM deb_cvs_dump
+	WHERE type='A'
+        GROUP BY year,month,day,cvsgroup
+	) c USING (cvsgroup)
+	LEFT JOIN (
+        SELECT cvsgroup,COUNT(*) AS adds
+        FROM deb_cvs_dump
+	WHERE type='M'
+        GROUP BY year,month,day,cvsgroup
+	) a USING (cvsgroup)
+	LEFT JOIN (
+        SELECT cvsgroup,COUNT(*) AS checkouts
+        FROM deb_cvs_dump
+	WHERE type='O'
+        GROUP BY year,month,day,cvsgroup
+	) ch USING (cvsgroup)
+	LEFT JOIN (
+        SELECT cvsgroup,COUNT(*) AS errors
+        FROM deb_cvs_dump
+	WHERE type='E'
+        GROUP BY year,month,day,cvsgroup
+	) e USING (cvsgroup)
+";
+$dbh->do( $sql );


Property changes on: trunk/gforge_base/evolvisforge/gforge/deb-specific/stats_cvs.pl
___________________________________________________________________
Name: svn:executable
   + *




More information about the evolvis-commits mailing list