[evolvis-commits] r7231: Files for cvs stats in project page↵

mirabilos at evolvis.org mirabilos at evolvis.org
Thu Feb 24 15:36:49 CET 2011


Author: mirabilos
Date: 2011-02-24 15:36:49 +0100 (Thu, 24 Feb 2011)
New Revision: 7231

Added:
   trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/
   trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/cvsserver/
   trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/cvsserver/cvs_history_parse.pl
   trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/cvsserver/db_stats_cvs_history.pl
Log:
Files for cvs stats in project page


Added: trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/cvsserver/cvs_history_parse.pl
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/cvsserver/cvs_history_parse.pl	                        (rev 0)
+++ trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/cvsserver/cvs_history_parse.pl	2011-02-24 14:36:49 UTC (rev 7231)
@@ -0,0 +1,151 @@
+#!/usr/bin/perl
+#/**
+#  *
+#  * cvs_history_parse.pl - NIGHTLY SCRIPT
+#  *
+#  * Recurses through the /cvsroot directory tree and parses each projects
+#  * '~/CVSROOT/history' file, building agregate stats on the number of
+#  * checkouts, commits, and adds to each project over the past 24 hours.
+#  *
+#  * SourceForge: Breaking Down the Barriers to Open Source Development
+#  * Copyright 1999-2001 (c) VA Linux Systems
+#  * http://sourceforge.net
+#  *
+#  * @version   $Id$
+#  *
+#  */
+
+use strict;
+use Time::Local;
+use POSIX qw( strftime );
+
+my ($year, $month, $day, $day_begin, $day_end);
+my ($group, $histline, $daily_log_file, $key, $verbose);
+my $verbose = 1;
+my $base_log_dir = "/var/log/sourceforge/cvs";
+my $cvsroot = "/var/lib/sourceforge/chroot/cvsroot";
+
+$|=0 if $verbose;
+$|++;
+
+   ## Set the time to collect stats for
+if ( $ARGV[0] && $ARGV[1] && $ARGV[2] ) {
+
+        $day_begin = timegm( 0, 0, 0, $ARGV[2], $ARGV[1] - 1, $ARGV[0] - 1900 );
+        $day_end = timegm( 0, 0, 0, (gmtime( $day_begin + 86400 ))[3,4,5] );
+	
+	$year = $ARGV[0];
+	$month = $ARGV[1];
+	$day = $ARGV[2];
+
+} else {
+
+           ## Start at midnight last night.
+        $day_end = timegm( 0, 0, 0, (gmtime( time() ))[3,4,5] );
+           ## go until midnight yesterday.
+        $day_begin = timegm( 0, 0, 0, (gmtime( time() - 86400 ))[3,4,5] );
+
+	$year	= strftime("%Y", gmtime( $day_begin ) );
+	$month	= strftime("%m", gmtime( $day_begin ) );
+	$day	= strftime("%d", gmtime( $day_begin ) );
+
+}
+
+my $daily_log_file;
+
+print "Parsing cvs logs looking for traffic on day $day, month $month, year $year.\n" if $verbose;
+
+if ( -d $base_log_dir ) {
+	$daily_log_file = $base_log_dir . "/" . sprintf("%04d", $year);
+	if ( ! -d $daily_log_file ) {
+		print "Making dest dir \'$daily_log_file\'\n";
+		mkdir( $daily_log_file, 0755 ) || die("Could not mkdir $daily_log_file");
+	} 
+	$daily_log_file .= "/" . sprintf("%02d", $month);
+	if ( ! -d $daily_log_file ) {
+		print "Making dest dir \'$daily_log_file\'\n";
+		mkdir( $daily_log_file, 0755 ) || die("Could not mkdir $daily_log_file");
+	}
+	$daily_log_file .= "/cvs_traffic_" . sprintf("%04d%02d%02d",$year,$month,$day) . ".log";
+} else {
+	die("Base log directory \'$base_log_dir\' does not exist!");
+}
+
+open(DAYS_LOG, "> $daily_log_file") || die "Unable to open the log file \'$daily_log_file\'";
+print "Opened log file at \'$daily_log_file\' for writing...\n";
+print "Running tree at $cvsroot/\n";
+
+chdir( "$cvsroot" ) || die("Unable to make $cvsroot the working directory.\n");
+foreach $group ( glob("*") ) {
+	
+	next if ( ! -d "$group" );
+
+	my ($cvs_co, $cvs_commit, $cvs_add, %usr_commit, %usr_add );
+
+	open(HISTORY, "< $cvsroot/$group/CVSROOT/history") or print "E::Unable to open history for $group\n";
+	while ( <HISTORY> ) {
+		my ($time_parsed, $type, $cvstime, $user, $curdir, $module, $rev, $file );
+ 
+		   ## Split the cvs history entry into it's 6 fields.
+		($cvstime,$user,$curdir,$module,$rev,$file) = split(/\|/, $_, 6 );
+
+		$type = substr($cvstime, 0, 1);
+		$time_parsed = hex( substr($cvstime, 1, 8) );
+
+		   ## If the entry was made in the past 24 hours 
+		   ## (i.e. - since the last run of this script...)
+		if ( ($time_parsed > $day_begin) && ($time_parsed < $day_end) ) {
+
+			   ## log commits
+			if ( $type eq "M" ) {
+				$cvs_commit++;
+				$usr_commit{$user}++;
+				next;
+			}
+
+			   ## log adds
+			if ( $type eq "A" ) {
+				$cvs_add++;
+				$usr_add{$user}++;
+				next;
+			}
+
+			   ## log checkouts
+			if ( $type eq "O" ) {
+				$cvs_co++;
+				## we don't care about checkouts on a per-user
+				## most of them will be anon anyhow.
+				next;
+			}
+		
+		} elsif ( $time_parsed > $day_end ) {
+			if ( $verbose >= 2 ) {
+				print "Short circuting execution, parsed date exceeded current threshold.\n";
+			}
+			last;
+		}
+
+	}
+	close( HISTORY );
+
+	   ## Now, we'll print all of the results for that project, in the following format:
+	   ## (G|U|E)::proj_name::user_name::checkouts::commits::adds
+	   ## If 'G', then record is group statistics, and field 2 is a space...
+	   ## If 'U', then record is per-user stats, and field 2 is the user name...
+	   ## If 'E', then record is an error, and field 1 is a description, there are no other fields.
+	if ( $cvs_co || $cvs_commit || $cvs_add ) {
+		print DAYS_LOG "G::" . $group . ":: ::" . ($cvs_co?$cvs_co:"0") . "::"
+			. ($cvs_commit?$cvs_commit:"0") . "::" . ($cvs_add?$cvs_add:"0") . "\n";
+	
+		foreach $key ( keys %usr_commit ) {
+	
+			print DAYS_LOG "U::" . $group . "::" . $key . "::0::" . ($usr_commit{$key}?$usr_commit{$key}:"0") 
+				. "::" . ($usr_add{$key}?$usr_add{$key}:"0") . "\n";
+		}
+	}
+}
+print "Done processing cvs history file for this date.\n" if $verbose;
+
+##
+## EOF
+##


Property changes on: trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/cvsserver/cvs_history_parse.pl
___________________________________________________________________
Added: svn:executable
   + *

Added: trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/cvsserver/db_stats_cvs_history.pl
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/cvsserver/db_stats_cvs_history.pl	                        (rev 0)
+++ trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/cvsserver/db_stats_cvs_history.pl	2011-02-24 14:36:49 UTC (rev 7231)
@@ -0,0 +1,125 @@
+#!/usr/bin/perl
+#/**
+#  *
+#  * db_cvs_history.pl - NIGHTLY SCRIPT
+#  *
+#  * Pulls the parsed CVS datafile (generated by cvs_history_parse.pl )
+#  * from the cvs server, and parses it into the database.
+#  *
+#  * SourceForge: Breaking Down the Barriers to Open Source Development
+#  * Copyright 1999-2001 (c) VA Linux Systems
+#  * http://sourceforge.net
+#  *
+#  * @version   $Id$
+#  * @author  Matthew Snelham <matthew at valinux.com>
+#  *
+#  */
+
+#use strict; ## annoying include requirements
+use DBI;
+use Time::Local;
+use POSIX qw( strftime );
+require("/usr/lib/sourceforge/lib/include.pl");  # Include all the predefined functions
+&db_connect;
+
+my ($logfile, $sql, $res, $temp, %groups, $group_id, $errors );
+my $verbose = 1;
+
+##
+## Set begin and end times (in epoch seconds) of day to be run
+## Either specified on the command line, or auto-calculated
+## to run yesterday's data.
+##
+if ( $ARGV[0] && $ARGV[1] && $ARGV[2] ) {
+
+	$day_begin = timegm( 0, 0, 0, $ARGV[2], $ARGV[1] - 1, $ARGV[0] - 1900 );
+	$day_end = timegm( 0, 0, 0, (gmtime( $day_begin + 86400 ))[3,4,5] );
+
+} else {
+
+	   ## Start at midnight last night.
+	#$day_end = timegm( 0, 0, 0, (gmtime( time() ))[3,4,5] );
+	$day_end = timelocal( 0, 0, 0, (localtime( time() ))[3,4,5] );
+	   ## go until midnight yesterday.
+	#$day_begin = timegm( 0, 0, 0, (gmtime( time() - 86400 ))[3,4,5] );
+	$day_begin = timelocal( 0, 0, 0, (localtime( time() - 86400 ))[3,4,5] );
+
+}
+
+   ## Preformat the important date strings.
+$year   = strftime("%Y", gmtime( $day_begin ) );
+$mon    = strftime("%m", gmtime( $day_begin ) );
+$week   = strftime("%U", gmtime( $day_begin ) );    ## GNU ext.
+$day    = strftime("%d", gmtime( $day_begin ) );
+print "Running week $week, day $day month $mon year $year \n" if $verbose;
+
+
+   ## We'll pull down the parsed CVS log from the CVS server via http?! <sigh>
+$logfile = "/var/log/sourceforge/cvs/$year/$mon/cvs_traffic_$year$mon$day.log";
+print "Using $logfile\n";
+
+   ## Now, we will pull all of the project ID's and names into a *massive*
+   ## hash, because it will save us some real time in the log processing.
+print "Caching group information from groups table.\n" if $verbose;
+$sql = "SELECT group_id,unix_group_name FROM groups";
+$res = $dbh->prepare($sql);
+$res->execute();
+while ( $temp = $res->fetchrow_arrayref() ) {
+	$groups{${$temp}[1]} = ${$temp}[0];
+}
+
+
+# TODO
+#
+#   Need to do the same thing for users
+#
+
+
+$dbh->{AutoCommit} = 0;
+
+$dbh->do("DELETE FROM stats_cvs_group WHERE month='$year$month' AND day='$day'");
+
+## begin parsing the log file line by line.
+print "Parsing the information into the database..." if $verbose;
+open( LOGFILE, $logfile ) or die "Cannot open $logfile";
+while(<LOGFILE>) {
+
+	if ( $_ =~ /^G::/ ) {
+		chomp($_);
+
+		   ## (G|U|E)::proj_name::user_name::checkouts::commits::adds
+		my ($type, $group, $user, $checkouts, $commits, $adds) = split( /::/, $_, 6 );
+
+		$group_id = $groups{$group};
+
+		if ( $group_id == 0 ) {
+			print STDERR "db_cvs_history.pl: bad unix_group_name \'$name\' ($_)\n";
+			next;
+		}
+			
+		$sql = "INSERT INTO stats_cvs_group
+			(month,day,group_id,checkouts,commits,adds)
+			VALUES ('$year$month','$day','$group_id','$checkouts','$commits','$adds')";
+
+		$dbh->do( $sql );
+
+	} elsif ( $_ =~ /^U::/ ) {
+#
+#   TODO - process user cvs stats
+#
+#   table: stats_cvs_user
+#
+	} elsif ( $_ =~ /^E::/ ) {
+		$errors++;
+	}
+
+}
+close( LOGFILE );
+
+$dbh->commit;
+
+print " done.\n" if $verbose;
+
+##
+## EOF
+##


Property changes on: trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/cvsserver/db_stats_cvs_history.pl
___________________________________________________________________
Added: svn:executable
   + *



More information about the evolvis-commits mailing list