postprocess-log

<<<<<<< HEAD
#!/disks/patric-common/runtime/bin/perl
#!/usr/bin/perl
=======
#!/usr/bin/env perl
>>>>>>> origin/master

use File::Basename;
use strict;
use Cwd qw(abs_path getcwd);
use Getopt::Long;
use Pod::Usage;
use Data::Dumper;
use Template;
<<<<<<< HEAD
=======
use File::Temp;
>>>>>>> origin/master

my($help, $dest, $module_dat);
my $rpm_name;
my $build_rpm;
my $rpm_sandbox = "rpm-sandbox";
my $rpm_version;
my @added_path;
<<<<<<< HEAD
=======
my @modules_to_build;
>>>>>>> origin/master
GetOptions('h'    => \$help,
	   'help' => \$help,
	   'd=s'    => \$dest,
	   'm=s'    => \$module_dat,
	   "build-rpm" => \$build_rpm,
	   "rpm-name=s" => \$rpm_name,
	   "rpm-version=s" => \$rpm_version,
	   "rpm-sandbox=s" => \$rpm_sandbox,
	   "path=s" => \@added_path,
<<<<<<< HEAD
=======
	   "modules=s" => \@modules_to_build,
>>>>>>> origin/master
    ) or pod2usage(0);

pod2usage(-exitstatus => 0,
          -output => \*STDOUT,
          -verbose => 2,
          -noperldoc => 1,
    ) if (defined $help or (!defined $dest) or (!defined $module_dat));

<<<<<<< HEAD
=======
my %modules_to_build;
$modules_to_build{$_} = 1 foreach @modules_to_build;
>>>>>>> origin/master

if ($build_rpm)
{
    if (! -d $rpm_sandbox)
    {
	die "RPM sandbox directory $rpm_sandbox does not exist\n";
    }
    $rpm_sandbox = abs_path($rpm_sandbox);

    for my $p (qw(BUILD RPMS SOURCES SPECS SRPMS))
    {
	if (! -d "$rpm_sandbox/$p")
	{
	    mkdir($p) or die "Cannot mkdir $p: $!\n";
	}
    }
}

$ENV{PATH} = join(":", "$dest/bin", @added_path, $ENV{PATH});
$ENV{CPATH} = "$dest/include";
$ENV{LIBRARY_PATH} = "$dest/lib";

my %target = (open => 0,
	      open_read => 0,
	      symlink => 1,
	      fopen64 => 0,
	      fopen64_read => 0,
	      fopen => 0,
	      fopen_read => 0,
	      rename => 1,
	      link => 1,
	      creat => 0,
	      chown => -1,
	      fchmod => -1,
	      ftruncate => -1,
	      rmdir => -1,
	      mkdir => -1,
	      unlink => -1,
	      chmod => -1,
	     );
<<<<<<< HEAD
=======
my %rm_target = (open => -1,
	      open_read => -1,
	      symlink => -1,
	      fopen64 => -1,
	      fopen64_read => -1,
	      fopen => -1,
	      fopen_read => -1,
	      rename => 0,
	      link => -1,
	      creat => -1,
	      chown => -1,
	      fchmod => -1,
	      ftruncate => -1,
	      rmdir => -1,
	      mkdir => -1,
	      unlink => 0,
	      chmod => -1,
	     );
>>>>>>> origin/master

my $mbase = basename($module_dat);

my $start_cwd = getcwd();
my $log_dir = "$start_cwd/logs.$mbase";
-d $log_dir || mkdir $log_dir || die "Cannot mkdir $log_dir: $!";
my $log_dir = abs_path($log_dir);

$ENV{TARGET} = $dest;

if (! -d $dest)
{
    mkdir $dest or die "Cannot mkdir $dest: $!";
}

for my $dir (qw(bin lib etc man))
{
    if (! -d "$dest/$dir")
    {
	mkdir "$dest/$dir" or die "Cannot mkdir $dest/$dir: $!";
    }
}

open(DAT, "<", $module_dat) or die "Cannot open $module_dat: $!";

$| = 1;

my @modules;

my %modules;
my %attribs;
<<<<<<< HEAD
=======
my $module_attribs = {};
>>>>>>> origin/master

while (<DAT>)
{
    chomp;
    s/^\s*//;
    if (/^\#\s+(\S+)\s+(.*)$/)
    {
	$attribs{$1} = $2;
    }
<<<<<<< HEAD
=======
    if (/^\#\#\s+(\S+)\s+(.*)$/)
    {
	push(@{$module_attribs->{$1}}, $2);
    }
>>>>>>> origin/master
    next if /^\#/;
    my($dir, $cmd) = split(/\s+/, $_, 2);
    die "error parsing $module_dat" unless ($dir && $cmd);
    die "directory $dir does not exist" unless -d $dir;
    die "directory $dir is not executable" unless -e $dir;
    die "directory $dir is not writable" unless -w $dir;

<<<<<<< HEAD
    my $rec = [$dir, $dir, $cmd];
    push(@modules, $rec);
    push(@{$modules{$dir}}, $rec);
=======
    my $rec = [$dir, $dir, $cmd, $module_attribs];
    push(@modules, $rec);
    push(@{$modules{$dir}}, $rec);
    $module_attribs = {};
>>>>>>> origin/master
}
close(DAT);

$rpm_version = $attribs{'rpm-version'} if $attribs{'rpm-version'} && !$rpm_version;
$rpm_name = $attribs{'rpm-name'} if $attribs{'rpm-name'} && !$rpm_name;

#
# Rewrite dir-tag element ($rec[1]) for the
# directories that have more than one build record.
#

<<<<<<< HEAD
for my $dir (keys %modules)
=======
for my $dir (sort keys %modules)
>>>>>>> origin/master
{
    my $l = $modules{$dir};
    if (@$l > 1)
    {
	for my $i (0..$#$l)
	{
	    my $n = $i + 1;
	    $l->[$i]->[1] = "${dir}_$n";
	}
    }
}

<<<<<<< HEAD
# print Dumper(\@modules);
=======
# die Dumper(\@modules);
>>>>>>> origin/master

my %belongs_to;
my %uses;
my %files_of;

for my $mod (@modules)
{
<<<<<<< HEAD
    my($dir, $tag, $cmd) = @$mod;
=======
    my($dir, $tag, $cmd, $attribs) = @$mod;
>>>>>>> origin/master

    if (!-f "$log_dir/built.$tag")
    {
	warn "$tag not successfully built\n";
	last;
    }

    my $log = "$log_dir/install_data.$tag";

    open(L, "<", $log) or die "Cannot open $log: $!";

<<<<<<< HEAD
=======
    #
    # One pass to process writes, another to process
    # reads to identify orphans.
    #
    while (<L>)
    {
	chomp;
	my(@a) = split(/\t/);
	my $err = pop @a;
	if ($err ne '#success')
	{
	    next;
	}
	my $rc = shift @a;
	my $cmd = shift @a;

	my $tfield = $target{$cmd};
	if (!defined($tfield))
	{
	    warn "No target defined for $cmd at $log:$.\n$_\n";
	    next;
	}
	if ($tfield >= 0)
	{
	    my $target = $a[$tfield];
	    
	    if ($target =~ m/^$dest/)
	    {
		my $pkg = $belongs_to{$target};
		if ($cmd !~ /read/)
		{
		    if ($pkg && $pkg ne $tag)
		    {
			warn "OVERWRITE: $tag overwrites $pkg: $target\n";
		    }
		    $belongs_to{$target} = $tag;
		    # print "ADD $tag $target\n";
		    $files_of{$tag}->{$target} = 1;
		}
	    }
	}

	my $rmfield = $rm_target{$cmd};
	if ($rmfield >= 0)
	{
	    my $rmfile = $a[$rmfield];
	    if ($rmfile =~ m/^$dest/)
	    {
		# print "DELETE $tag $rmfile\n";
		delete $files_of{$tag}->{$rmfile};
	    }
	}
    }
    
    seek(L, 0, 0);
>>>>>>> origin/master
    while (<L>)
    {
	chomp;
	my(@a) = split(/\t/);
	my $err = pop @a;
	if ($err ne '#success')
	{
	    next;
	}
	my $rc = shift @a;
	my $cmd = shift @a;

	my $tfield = $target{$cmd};
	if (!defined($tfield))
	{
	    warn "No target defined for $cmd at $log:$.\n$_\n";
	    next;
	}
	next if $tfield < 0;
	my $target = $a[$tfield];

	next unless $target =~ m/^$dest/;

<<<<<<< HEAD
	if ($cmd =~ /read/)
	{
	    my $pkg = $belongs_to{$target};
	    if ($pkg)
	    {
		$uses{$tag} = $pkg;
	    }
	    else
	    {
		"$tag uses orphan $target\n";
	    }
	}
	else
	{
	    $belongs_to{$target} = $tag;
	    $files_of{$tag}->{$target} = 1;
	}
    }
}

for my $tag (keys %files_of)
{
    build_rpm();
=======
	my $pkg = $belongs_to{$target};
	if ($cmd =~ /read/)
	{
	    if ($pkg)
	    {
		if ($pkg ne $tag)
		{
		    $uses{$tag}->{$pkg}->{$target} = 1;
		}
	    }
	    else
	    {
		# print "$tag uses orphan $target\n";
	    }
	}
    }
}

exit unless $build_rpm;

my @alldeps;

for my $mod (@modules)
{
    my($dir, $tag, $cmd, $attribs) = @$mod;

    next if (@modules_to_build && !$modules_to_build{$tag});
    my @uses = map { "$rpm_name-$_" } keys %{$uses{$tag}};
    print "$tag uses @uses\n";

    my $deps = $attribs->{'rpm-dep'} || [];
    push(@uses, @$deps);

    my @files = sort grep { -f $_ } keys %{$files_of{$tag}};

    build_rpm($tag, "$rpm_name-$tag", \@uses, \@files);

    push(@alldeps, "$rpm_name-$tag");
}

if (!@modules_to_build)
{
    build_rpm("rollup of all modules", $rpm_name, \@alldeps, []);
>>>>>>> origin/master
}

sub build_rpm
{
<<<<<<< HEAD
    my($files_of, $belongs_to, $uses) @_;
    my $spec_dir = "$rpm_sandbox/SPECS";
    my $rpm_name_base = "$rpm_name-$rpm_version";
=======
    my($tag, $this_rpm, $dependencies, $files) = @_;
    
    my $spec_dir = "$rpm_sandbox/SPECS";
    my $rpm_name_base = "$this_rpm-$rpm_version";
>>>>>>> origin/master
    my $rel;
    for my $p (<$spec_dir/$rpm_name_base*>)
    {
	my($r) = $p =~ /$rpm_name_base-(\d+)/;
	$rel = $r if ($r > $rel);
    }
    $rel++;
<<<<<<< HEAD
    my $spec = "$spec_dir/$rpm_name-$rpm_version-$rel.spec";
    print "Create $spec\n";

=======
    my $spec = "$spec_dir/$this_rpm-$rpm_version-$rel.spec";
    print "Create $spec\n";

    my $tmpfile = File::Temp->new();
    print $tmpfile "$_\n" foreach @$files;
    close($tmpfile);

>>>>>>> origin/master
    my $templ = Template->new(RELATIVE => 1);

    my $build_root = dirname($dest);
    my $base_name = basename($dest);
    my $vars = {
<<<<<<< HEAD
	name => $rpm_name,
	version => $rpm_version,
	release => $rel,
	summary => "Bootstrap build from $module_dat",
=======
	name => $this_rpm,
	version => $rpm_version,
	release => $rel,
	files => [],
	file_list => "$tmpfile",
	dependencies => $dependencies,
	summary => "Bootstrap build for $tag from $module_dat",
>>>>>>> origin/master
	root_path => $build_root,
	base_name => $base_name,
    };
    $templ->process("bootstrap_spec.tt", $vars, $spec);

    my @cmd = ("rpmbuild",
	       -D => "_topdir $rpm_sandbox",
	       -D => "_builddir $rpm_sandbox/BUILD",
	       -D => "_rpmdir $rpm_sandbox/RPMS",
	       -D => "_sourcedir $rpm_sandbox/SOURCES",
	       -D => "_specdir $rpm_sandbox/SPECS",
	       -D => "_srcrpmdir $rpm_sandbox/SRPMS",
	       '-bb', $spec);
    print "@cmd\n";
    my $rc = system(@cmd);
    if ($rc != 0)
    {
	die "Error $rc building RPM  with @cmd\n";
    }
}

=pod

=head1  NAME

bootstrap_modules.pl

=head1  SYNOPSIS

=over

=item bootstrap_modules.pl -d /kb/runtime -m ./my_modules.dat

=back

=head1  DESCRIPTION

The bootstrap.pl script reads a module.dat and runs a builder for each module. A module is defined as a directory that contains a builder script. For example, kb_blast is a directory that represents a blast module. In that directory is a script that installs blast.

=head1  CONVENTIONS

The module directory may contain a suffix. If there is no suffix, it is assumed that the installer inside that directory will properly install the module on all supported operating systems. If there is a suffix on the module directory in the form _ubuntu and _centos, then the installer inside that directory will properly install the module on the named operating system.

Examples include bootstrap_ubuntu and bootstrap_centos. In side bootstrap_ubuntu the builder script could use apt-get to install ubuntu modules and inside the bootstrap_centos directory the builder script could use yum to install centos modules.

=head1 MODULES FILE

The modules.dat file contains a space delimited set of module directories and builer scripts. You can think of this as the module directory being the key and the name of the builder script being the value if you want. In sort, the named builder script associated with each module directory will be executed.

=head1  COMMAND-LINE OPTIONS

=over

=item -h, --help  This documentation

=item -d Destination target for runtime (ie /kb/runtime)

=item -m Name of the modules.dat file

=back

=head1  AUTHORS

Robert Olson, Tom Brettin

=cut