[evolvis-commits] r8557: further cleanup of the backend scripts - almost done↵

mirabilos at evolvis.org mirabilos at evolvis.org
Thu Feb 24 16:45:44 CET 2011


Author: mirabilos
Date: 2011-02-24 16:45:43 +0100 (Thu, 24 Feb 2011)
New Revision: 8557

Added:
   trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/README.root
   trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/history_parse.php
   trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/usergroup.php
Removed:
   trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/group.php
   trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/user.php
   trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/stats/
Log:
further cleanup of the backend scripts - almost done


Added: trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/README.root
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/README.root	                        (rev 0)
+++ trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/README.root	2011-02-24 15:45:43 UTC (rev 8557)
@@ -0,0 +1,43 @@
+Several scripts in cvs-cron/ will allow you to automate the process of adding
+users, groups, and cvs trees to your unix machine. These scripts build 
+the /etc/passwd /etc/shadow and /etc/group files based on the contents of your 
+gforge database.
+
+WARNING: There are potential security risks. Any gforge project admin could 
+add a user to a project, who would then have unix-level access to your 
+machine. For best security, you should separate the CVS hosting machine from 
+the web/database machine. To do this, simply copy the gforge tarball 
+and configure a new /etc/gforge/local.inc on the cvs machine. You can create 
+a second, read-only database user, and place that user's name/password in 
+the local.inc file. This user only needs read access to the users,groups, and
+user_group tables. See the postgres manual for information on managing 
+database users and permissions.
+
+To use these files, you should first create any default users/groups you 
+want on your machine. Then copy /etc/group to /etc/group.org, and do the 
+same for /etc/shadow -> /etc/shadow.org /etc/passwd -> /etc/passwd.org
+
+The usergroup.php script will build on the .org file and place them in .new
+files.
+
+SAMPLE CRON ENTRIES
+
+#
+#	Create the /etc/* files
+#
+10 * * * * ~/alexandria/cronjobs/cvs-cron/usergroup.php
+
+#
+#	Upload the SSH key files
+#
+15 * * * * ~/alexandria/cronjobs/cvs-cron/ssh_create.php
+
+#
+#	Create the blank cvs trees
+#
+20 * * * * ~/alexandria/cronjobs/cvs-cron/cvs.php
+
+#
+#	Daily stats collection
+#
+###incomplete 20 1 * * * ~/alexandria/cronjobs/cvs-cron/history_parse.php

Deleted: trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/group.php
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/group.php	2011-02-24 15:45:41 UTC (rev 8556)
+++ trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/group.php	2011-02-24 15:45:43 UTC (rev 8557)
@@ -1,61 +0,0 @@
-#!/usr/bin/php
-<?php
-//
-//	unix_group_id is database_group_id+50000
-//
-//this reads the database and creates groups in /etc/group
-//this script must be ran before you run the add users to groups first,
-//because you need a group to be a member of it
-
-require_once('squal_pre.php');
-
-//1) read in groups from db
-$groups = array();
-$res=db_query("SELECT group_id,unix_group_name FROM groups");
-for($i = 0; $i < db_numrows($res); $i++) {
-	$groups[] = db_result($res,$i,'unix_group_name');
-	$gids[db_result($res,$i,'unix_group_name')]=db_result($res,$i,'group_id')+50000;
-}
-
-//2) read in groups from /etc/group
-$h = fopen("/etc/group.backup","r");
-
-if(!$h) {
-	die("Groups.php -- unable to open /etc/group for reading");
-}
-
-$filecontent = fread($h, filesize("/etc/group.backup"));
-fclose($h);
-$lines = explode("\n",$filecontent);
-
-//3) if group is listed in the db and not /etc/group add
-$h2 = fopen("/etc/group.backup","w");
-
-if(!h2) {
-	die("Groups.php -- unable to open /etc/group for writing");
-}
-
-//write the group file out again, followed by new gforge stuff
-$i = 0;
-for($i; $i < count($lines)-1; $i++) {
-	fwrite($h2,$lines[$i]."\n");
-}
-fwrite($h2,$lines[$i]);
-
-//see if there is no group with same name, if not add group, if so don't add group	
-foreach($groups as $group) {
-	foreach($lines as $line) {
-		$etcline = explode(":",$line);
-
-		if($group == $etcline[0]) {
-			continue 2;
-		}
-	}
-
-	$gid = $gids[$group];
-	$writegrouptofile = "$group:x:$gid:\n";
-	fwrite($h2,$writegrouptofile);
-}
-
-fclose($h2);	
-?>

Added: trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/history_parse.php
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/history_parse.php	                        (rev 0)
+++ trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/history_parse.php	2011-02-24 15:45:43 UTC (rev 8557)
@@ -0,0 +1,151 @@
+#!/usr/bin/php
+<?php
+
+require_once('squal_pre.php');
+/**
+ *
+ * Recurses through the /cvsroot directory tree and parses each projects
+ * '~/CVSROOT/history' file, building agregate stats on the number of
+ * checkouts, commits, and adds to each project over the past 24 hours.
+ *
+ * @version   $Id$
+ */
+
+/*
+
+//
+//	TRANSLATE TO PHP
+//
+
+
+my ($year, $month, $day, $day_begin, $day_end);
+my ($group, $histline, $daily_log_file, $key, $verbose);
+my $verbose = 1;
+my $base_log_dir = "/var/log/gforge/cvs";
+my $cvsroot = "/var/lib/gforge/chroot/cvsroot";
+
+$|=0 if $verbose;
+$|++;
+
+   ## Set the time to collect stats for
+if ( $ARGV[0] && $ARGV[1] && $ARGV[2] ) {
+
+        $day_begin = timegm( 0, 0, 0, $ARGV[2], $ARGV[1] - 1, $ARGV[0] - 1900 );
+        $day_end = timegm( 0, 0, 0, (gmtime( $day_begin + 86400 ))[3,4,5] );
+	
+	$year = $ARGV[0];
+	$month = $ARGV[1];
+	$day = $ARGV[2];
+
+} else {
+
+           ## Start at midnight last night.
+        $day_end = timegm( 0, 0, 0, (gmtime( time() ))[3,4,5] );
+           ## go until midnight yesterday.
+        $day_begin = timegm( 0, 0, 0, (gmtime( time() - 86400 ))[3,4,5] );
+
+	$year	= strftime("%Y", gmtime( $day_begin ) );
+	$month	= strftime("%m", gmtime( $day_begin ) );
+	$day	= strftime("%d", gmtime( $day_begin ) );
+
+}
+
+my $daily_log_file;
+
+print "Parsing cvs logs looking for traffic on day $day, month $month, year $year.\n" if $verbose;
+
+if ( -d $base_log_dir ) {
+	$daily_log_file = $base_log_dir . "/" . sprintf("%04d", $year);
+	if ( ! -d $daily_log_file ) {
+		print "Making dest dir \'$daily_log_file\'\n";
+		mkdir( $daily_log_file, 0755 ) || die("Could not mkdir $daily_log_file");
+	} 
+	$daily_log_file .= "/" . sprintf("%02d", $month);
+	if ( ! -d $daily_log_file ) {
+		print "Making dest dir \'$daily_log_file\'\n";
+		mkdir( $daily_log_file, 0755 ) || die("Could not mkdir $daily_log_file");
+	}
+	$daily_log_file .= "/cvs_traffic_" . sprintf("%04d%02d%02d",$year,$month,$day) . ".log";
+} else {
+	die("Base log directory \'$base_log_dir\' does not exist!");
+}
+
+open(DAYS_LOG, "> $daily_log_file") || die "Unable to open the log file \'$daily_log_file\'";
+print "Opened log file at \'$daily_log_file\' for writing...\n";
+print "Running tree at $cvsroot/\n";
+
+chdir( "$cvsroot" ) || die("Unable to make $cvsroot the working directory.\n");
+foreach $group ( glob("*") ) {
+	
+	next if ( ! -d "$group" );
+
+	my ($cvs_co, $cvs_commit, $cvs_add, %usr_commit, %usr_add );
+
+	open(HISTORY, "< $cvsroot/$group/CVSROOT/history") or print "E::Unable to open history for $group\n";
+	while ( <HISTORY> ) {
+		my ($time_parsed, $type, $cvstime, $user, $curdir, $module, $rev, $file );
+ 
+		   ## Split the cvs history entry into it's 6 fields.
+		($cvstime,$user,$curdir,$module,$rev,$file) = split(/\|/, $_, 6 );
+
+		$type = substr($cvstime, 0, 1);
+		$time_parsed = hex( substr($cvstime, 1, 8) );
+
+		   ## If the entry was made in the past 24 hours 
+		   ## (i.e. - since the last run of this script...)
+		if ( ($time_parsed > $day_begin) && ($time_parsed < $day_end) ) {
+
+			   ## log commits
+			if ( $type eq "M" ) {
+				$cvs_commit++;
+				$usr_commit{$user}++;
+				next;
+			}
+
+			   ## log adds
+			if ( $type eq "A" ) {
+				$cvs_add++;
+				$usr_add{$user}++;
+				next;
+			}
+
+			   ## log checkouts
+			if ( $type eq "O" ) {
+				$cvs_co++;
+				## we don't care about checkouts on a per-user
+				## most of them will be anon anyhow.
+				next;
+			}
+		
+		} elsif ( $time_parsed > $day_end ) {
+			if ( $verbose >= 2 ) {
+				print "Short circuting execution, parsed date exceeded current threshold.\n";
+			}
+			last;
+		}
+
+	}
+	close( HISTORY );
+
+                $sql = "INSERT INTO stats_cvs_group
+                        (month,day,group_id,checkouts,commits,adds)
+                        VALUES ('$year$mon','$day','$group_id','$checkouts','$commits','$adds')";
+
+	   ## Now, we'll print all of the results for that project, in the following format:
+	   ## (G|U|E)::proj_name::user_name::checkouts::commits::adds
+	   ## If 'G', then record is group statistics, and field 2 is a space...
+	   ## If 'U', then record is per-user stats, and field 2 is the user name...
+	   ## If 'E', then record is an error, and field 1 is a description, there are no other fields.
+	if ( $cvs_co || $cvs_commit || $cvs_add ) {
+		print DAYS_LOG "G::" . $group . ":: ::" . ($cvs_co?$cvs_co:"0") . "::"
+			. ($cvs_commit?$cvs_commit:"0") . "::" . ($cvs_add?$cvs_add:"0") . "\n";
+	
+		foreach $key ( keys %usr_commit ) {
+	
+			print DAYS_LOG "U::" . $group . "::" . $key . "::0::" . ($usr_commit{$key}?$usr_commit{$key}:"0") 
+				. "::" . ($usr_add{$key}?$usr_add{$key}:"0") . "\n";
+		}
+	}
+}
+*/
+?>


Property changes on: trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/history_parse.php
___________________________________________________________________
Added: svn:executable
   + *

Deleted: trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/user.php
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/user.php	2011-02-24 15:45:41 UTC (rev 8556)
+++ trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/user.php	2011-02-24 15:45:43 UTC (rev 8557)
@@ -1,163 +0,0 @@
-#!/usr/bin/php
-<?php
-/*
-
-
-	PROBLEM - user's home dir is not owned by user or correct perms
-
-
-*/
-require_once('squal_pre.php');
-
-//
-//	Default shell for the user should be grap (cvs only)
-//
-define('DEFAULT_SHELL','/bin/grap');
-
-//
-//	Get the users' unix_name and password out of the database
-//	ONLY USERS WITH CVS COMMIT PRIVS ARE ADDED
-//
-$res = db_query("SELECT distinct users.user_name,users.unix_pw,users.user_id 
-	FROM users,user_group
-	WHERE users.user_id=user_group.user_id 
-	AND user_group.cvs_flags='1'
-	ORDER BY user_id ASC");
-
-$users = array();
-$user_ids = array();
-$user_pws = array();
-
-$users    =& util_result_column_to_array($res,'user_name');
-$user_ids =& util_result_column_to_array($res,'user_id');
-$user_pws =& util_result_column_to_array($res,'unix_pw');
-
-//
-//	this is where we add users to /etc/passwd
-//
-$h = fopen("/etc/passwd.backup","r");
-$passwdcontents = fread($h,filesize("/etc/passwd.backup"));
-fclose($h);
-$passwdlines = explode("\n",$passwdcontents);
-
-$h2 = fopen("/etc/passwd.backup","a");
-
-for($i = 0; $i < count($users); $i++) {
-	for($k = 0; $k < count($passwdlines); $k++) {
-		$passwdline = explode(":",$passwdlines[$k]);
-
-		if($passwdline[0] == $users[$i]) {
-			continue 2;
-		}
-	}
-
-	$line = $users[$i] . ":x:" . ($user_ids[$i] + 1000) . ":" . ($user_ids[$i] + 1000) . "::/home/$users[$i]:".DEFAULT_SHELL."\n";
-	fwrite($h2,$line);
-
-}
-
-fclose($h2);
-
-//
-//	this is where we add users to /etc/shadow
-//
-$h3 = fopen("/etc/shadow.backup","r");
-$shadowcontents = fread($h3,filesize("/etc/shadow.backup"));
-fclose($h3);
-$shadowlines = explode("\n",$shadowcontents);
-
-$h4 = fopen("/etc/shadow.backup","a");
-
-for($i = 0; $i < count($users); $i++) {
-	for($k = 0; $k < count($shadowlines); $k++) {
-		$shadowline = explode(":",$shadowlines[$k]);
-		if($shadowline[0] == $users[$i])
-			continue 2;
-	}
-
-	$line = $users[$i] . ":" . $user_pws[$i] . ":12090:0:99999:7:::\n";
-	fwrite($h4,$line);
-}
-
-fclose($h4);
-
-//
-//	this is where we give a user a home
-//
-foreach($users as $user) {
-	@mkdir("/home/".$user);
-}
-
-//
-//	this is where we add user primary groups (redhat specific right now)
-//
-$h5 = fopen("/etc/group.backup","r");
-$groupcontents = fread($h5,filesize("/etc/group.backup"));
-fclose($h5);
-$grouplines = explode("\n",$groupcontents);
-
-$h6 = fopen("/etc/group.backup","a");
-
-for($i = 0; $i < count($users); $i++) {
-	for($k = 0; $k < count($grouplines);$k++) {
-		$groupline = explode(":", $grouplines[$k]);
-		if($groupline[0] == $users[$i])
-			continue 2;
-	}
-
-	$line = $users[$i] . ":x:" . ($user_ids[$i]+1000) . ":\n";
-
-	fwrite($h6, $line);
-}	
-
-fclose($h6);
-
-//
-//	have to re-read the group file since we just modified it
-//
-$h7 = fopen("/etc/group.backup","r");
-$groupcontent = fread($h7,filesize("/etc/group.backup"));
-fclose($h7);
-
-$grouplines = explode("\n",$groupcontent);
-
-//
-//	this is where we add users to groups in /etc/groups	
-//
-for($i = 0; $i < count($users); $i++) {
-	$res6 = db_query("select groups.group_id,groups.unix_group_name 
-		FROM user_group,groups 
-		WHERE user_group.user_id='$user_ids[$i]'	
-		AND groups.group_id=user_group.group_id");
-	$rows = db_numrows($res6);
-
-	for($k = 0; $k < $rows; $k++) {
-		$group_id = db_result($res6,$k,'group_id');
-		$group = db_result($res6,$k,'unix_group_name');
-
-		for($j = 0; $j < count($grouplines); $j++) {
-			list($group_name,$group_pw,$group_id,$members) = explode(":",$grouplines[$j]);
-
-			if($group_name == $group) {
-				$memberslist = explode(",",$members);
-
-				foreach($memberslist as $member) {
-					if($member == $users[$i]) {
-						continue 3;
-					}
-				}
-				if($memberslist[0] == "" && count($memberslist) == 1)
-					$grouplines[$j] = $grouplines[$j] . "$users[$i]";
-				else
-					$grouplines[$j] = $grouplines[$j] . ",$users[$i]";
-			}
-		}
-	}
-}
-
-$h8 = fopen("/etc/group.backup","w");
-foreach($grouplines as $line)
-fwrite($h8,$line."\n");
-fclose($h8);
-
-?>

Added: trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/usergroup.php
===================================================================
--- trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/usergroup.php	                        (rev 0)
+++ trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/usergroup.php	2011-02-24 15:45:43 UTC (rev 8557)
@@ -0,0 +1,210 @@
+#! /usr/bin/php4 -f
+<?php
+
+require_once('squal_pre.php');
+
+//
+//	Default values for the script
+//
+define('DEFAULT_SHELL','/bin/grap');
+define('USER_ID_ADD',10000);
+define('GROUP_ID_ADD',50000);
+define('USER_DEFAULT_GROUP','users');
+
+//
+//	Get the users' unix_name and password out of the database
+//	ONLY USERS WITH CVS COMMIT PRIVS ARE ADDED
+//
+$res = db_query("SELECT distinct users.user_name,users.unix_pw,users.user_id 
+	FROM users,user_group
+	WHERE users.user_id=user_group.user_id 
+	AND user_group.cvs_flags='1'
+	AND users.status='A'
+	ORDER BY user_id ASC");
+
+$users    =& util_result_column_to_array($res,'user_name');
+$user_ids =& util_result_column_to_array($res,'user_id');
+$user_pws =& util_result_column_to_array($res,'unix_pw');
+
+//
+//	Read in the "default" users
+//
+$h = fopen("/etc/passwd.org","r");
+$passwdcontents = fread($h,filesize("/etc/passwd.org"));
+fclose($h);
+$passwdlines = explode("\n",$passwdcontents);
+
+//
+//	Write the "default" users to a temp file
+//
+$h2 = fopen("/etc/passwd.new","w");
+for($k = 0; $k < count($passwdlines); $k++) {
+	$passwdline = explode(":",$passwdlines[$k]);
+	$def_users[$passwdline[0]]=1;
+	fwrite($h2,$passwdlines[$k]."\n");
+}
+
+//
+//	Now append the users from the gforge database
+//
+for($i = 0; $i < count($users); $i++) {
+
+	if ($def_users[$users[$i]]) {
+
+		//this username was already existing in the "default" file
+
+	} else {
+
+		$line = $users[$i] . ":x:" . ($user_ids[$i] + USER_ID_ADD) . ":" . ($user_ids[$i] + USER_ID_ADD) . "::/home/$users[$i]:".DEFAULT_SHELL."\n";
+		fwrite($h2,$line);
+
+	}
+
+}
+
+fclose($h2);
+
+//
+//	this is where we add users to /etc/shadow
+//
+$h3 = fopen("/etc/shadow.org","r");
+$shadowcontents = fread($h3,filesize("/etc/shadow.org"));
+fclose($h3);
+$shadowlines = explode("\n",$shadowcontents);
+
+//
+//	Write the "default" shadow to a temp file
+//
+$h4 = fopen("/etc/shadow.new","w");
+for($k = 0; $k < count($shadowlines); $k++) {
+    $shadowline = explode(":",$shadowlines[$k]);
+    $def_shadow[$shadowline[0]]=1;
+    fwrite($h4,$shadowlines[$k]."\n");
+}
+
+//
+//  Now append the users from the gforge database
+//
+for($i = 0; $i < count($users); $i++) {
+
+    if ($def_shadow[$users[$i]]) {
+
+        //this username was already existing in the "default" file
+
+    } else {
+
+		$line = $users[$i] . ":" . $user_pws[$i] . ":12090:0:99999:7:::\n";
+		fwrite($h4,$line);
+
+	}
+
+}
+
+fclose($h4);
+
+//
+//	Read the groups from the "default" file
+//
+$h5 = fopen("/etc/group.org","r");
+$groupcontents = fread($h5,filesize("/etc/group.org"));
+fclose($h5);
+$grouplines = explode("\n",$groupcontents);
+
+//
+//	Write the "default" groups to a temp file
+//
+$h6 = fopen("/etc/group.new","w");
+for($k = 0; $k < count($grouplines); $k++) {
+    $groupline = explode(":",$grouplines[$k]);
+    $def_group[$groupline[0]]=1;
+    fwrite($h6,$grouplines[$k]."\n");
+}
+
+//
+//	Add the groups from the gforge database
+//
+$res=db_query("SELECT group_id,unix_group_name FROM groups WHERE status='A'");
+for($i = 0; $i < db_numrows($res); $i++) {
+    $groups[] = db_result($res,$i,'unix_group_name');
+    $gids[db_result($res,$i,'unix_group_name')]=db_result($res,$i,'group_id')+GROUP_ID_ADD;
+}
+
+for($i = 0; $i < count($users); $i++) {
+
+    if ($def_group[$groups[$i]]) {
+
+        //this username was already existing in the "default" file
+
+    } else {
+
+		$line = $groups[$i] . ":x:" . ($gids[$groups[$i]]) . ":\n";
+
+		fwrite($h6, $line);
+
+	}
+
+}
+
+fclose($h6);
+
+//
+//	have to re-read the group file since we just modified it
+//
+$h7 = fopen("/etc/group.new","r");
+$groupcontent = fread($h7,filesize("/etc/group.new"));
+fclose($h7);
+
+$grouplines = explode("\n",$groupcontent);
+
+//
+//	this is where we add users to groups in /etc/groups	
+//
+for($i = 0; $i < count($users); $i++) {
+	$res6 = db_query("select groups.group_id,groups.unix_group_name 
+		FROM user_group,groups 
+		WHERE user_group.user_id='$user_ids[$i]'	
+		AND groups.group_id=user_group.group_id");
+	$rows = db_numrows($res6);
+
+	for($k = 0; $k < $rows; $k++) {
+		$group_id = db_result($res6,$k,'group_id');
+		$group = db_result($res6,$k,'unix_group_name');
+
+		for($j = 0; $j < count($grouplines); $j++) {
+			list($group_name,$group_pw,$group_id,$members) = explode(":",$grouplines[$j]);
+
+			if($group_name == $group) {
+				$memberslist = explode(",",$members);
+
+				foreach($memberslist as $member) {
+					if($member == $users[$i]) {
+						continue 3;
+					}
+				}
+				if($memberslist[0] == "" && count($memberslist) == 1)
+					$grouplines[$j] = $grouplines[$j] . "$users[$i]";
+				else
+					$grouplines[$j] = $grouplines[$j] . ",$users[$i]";
+			}
+		}
+	}
+}
+
+$h8 = fopen("/etc/group.new","w");
+foreach($grouplines as $line)
+fwrite($h8,$line."\n");
+fclose($h8);
+
+//
+//	this is where we give a user a home
+//
+foreach($users as $user) {
+	if (is_dir("/home/".$user)) {
+		
+	} else {
+		@mkdir("/home/".$user);
+		system("chown $user:".USER_DEFAULT_GROUP." /home/".$user);
+	}
+}
+
+?>


Property changes on: trunk/gforge_base/evolvisforge-5.1/gforge/cronjobs/cvs-cron/usergroup.php
___________________________________________________________________
Added: svn:executable
   + *



More information about the evolvis-commits mailing list