RetroZilla/modules/oji/tests/script/autorun.pl

#!/usr/bin/perl
# 
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Sun Microsystems, Inc.
# Portions created by the Initial Developer are Copyright (C) 1999
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****

# Created By:     Raju Pallath
# Creation Date:  Aug 2nd 1999
#
# Ported by:	  Konstantin S. Ermakov
# Ported date:	  Aug 24th 1999
#
# This script is used to  invoke all test case for OJI API
# through apprunner and to recored their results
#
#

# Attach Perl Libraries
###################################################################
use POSIX ":sys_wait_h"; #used by waitpid
use Cwd;
use File::Copy;
#use Win32::Process;
#use IO::Handle;
###################################################################


########################
#                      #
#     User's settings  #
#                      #
########################

#Mozilla's executable
$MOZILLA_EXECUTABLE="mozilla-bin";
if ($^O =~ /Win32/i) {
  $MOZILLA_EXECUTABLE="mozilla.exe";
}

#sometimes we need to specify additional parameters for mozilla
$ADDITIONAL_PARAMETERS="-P default";

# time in seconds after which the apprunner has to be killed.
# by default the apprunner will be up for so much time regardless of
# whether over or not. User can either decrease it or increase it.
#
$DELAY_FACTOR = 50;

$FULL_TEST_LIST="OJITestsList.lst";

$TEST_RESULTS="OJITestResults.txt";

$TEST_DESCRIPTIONS="../OJIAPITestsDescr.html";
$TARGET_WINDOW="Test Descriptions";

$curdir = cwd();           
$DEFAULT_TEST_URL="file:///$curdir";

# time period in seconds of periodically checking: is the apprunner still alive
$DELAY_OF_CYCLE = 1;


##############################################################
##############################################################
##############################################################


$DOCROOT = $ENV{"TEST_URL"} ? $ENV{"TEST_URL"} : $DEFAULT_TEST_URL;

# delimiter for logfile
$delimiter="###################################################\n";
$nosort=0;


##################################################################
# Usage
##################################################################
sub usage() {

    print "\n";
    print "##################################################################\n";
    print " perl autorun.pl [ -help ] [ -genhtml [ ALL/PASS/FAILED/<test_type> ] ] [ -compare [ <output_file> <file1> <file2> ] ] [ -t <test_case> ]\n";
    print "\n";
    print "  -help              This message.\n\n";
    print "  -compare <output_file> <file1> <file2> \n";
    print "            Compares two test result files and generates the diff files\n";
    print "              <output_file>  the resulting diff files in text (.txt)\n";
    print "                             and HTML (.html) formats\n";
    print "              <file1(2)>     files to be compared (optional parameters)\n";
    print "                             should contain test results in text format\n";
    print "                             (generated by autorun.pl as BWTest.txt);\n";
    print "                             if ommited the default values are used: \n";
    print "                               <file1> => log/BWTest.txt \n";
    print "                               <file2> => log/BWTest.txt.bak \n\n";
    print "  -genhtml <option>  Generates report in HTML format (without tests execution).\n";
    print "            ALL         Generates report for all executed tests. \n";
    print "            PASS        Generates report for passed tests\n";
    print "            FAILED      Generates report for failed tests\n";
    print "            <test_type> Generates report for the specific <test_type>\n";
    print "                        <test_type> = JNIEnv/LCM/ThreadManager/JVMManager\n";
    print "                 If specified all other parameters are ignored\n\n";
    print "  -t <test case> Executes the specified <test_case>.\n";
    print "                 test_case example: JVMManager_CreateProxyJNI_1\n";
    print "\n";
    print "##################################################################\n";
    print "\n";
   
}

##################################################################
# Title display
##################################################################
sub title() {
	
    print "\n";
    print "################################################\n";
    print "   Automated Execution of OJI API TestSuite\n";
    print "################################################\n";
    print "\n\n";

}

##################################################################
#
# check which tests to run. XML/HTML or both
#
##################################################################
sub checkRun() {
    $runtype = "0";
    while( true ) {
	print "Run 1) HTML Test suite.\n";
	print "    2) XML Test suite.\n";
	print "    3) BOTH HTML and XML.\n";
	print "Enter choice (1-3) :\n";
	$runtype = getc;
	  
	if(( $runtype ne "1" ) &&
	   ( $runtype ne "2" ) &&
	   ( $runtype ne "3" ) ) 
	{
	    print "Invaid choice. Range is from 1-3...\n";
	    print "\n";
	    next;
	} else {
	    last;
	}
    }
}


#########################################################################
#
# Append table entries to Output HTML File 
#
#########################################################################
sub appendEntries() {
   print LOGHTML "<tr><td></td><td></td></tr><td></td>\n";
   print LOGHTML "<tr><td></td><td></td></tr><td></td>\n";
   print LOGHTML "<tr bgcolor=\"#FF6666\"><td>Test Status (XML)</td><td>Result</td><td>Comment</td></tr>\n";

}

#########################################################################
#
# Construct Output HTML file Header
#
#########################################################################
sub constructHTMLHeader() {
   print LOGHTML "<html><head><title>\n";
   print LOGHTML "OJI API Test Status\n";
   print LOGHTML "</title></head><body bgcolor=\"white\">\n";
   print LOGHTML "<center><h1>\n";
   print LOGHTML "OJI API Automated TestRun Results\n";
   $date = localtime;
   print LOGHTML "</h1><h2>", $date;
   print LOGHTML "</h2></center>\n";
   print LOGHTML "<hr noshade>";
   print LOGHTML "<table bgcolor=\"lightgreen\" border=1 cellpadding=10>\n";
   print LOGHTML "<tr bgcolor=\"lightblue\">\n";
   print LOGHTML "<td>Test Case</td>\n";
   print LOGHTML "<td>Result</td>\n";
   print LOGHTML "<td>Comment</td></tr>\n";
}

#########################################################################
#
# Construct Output HTML file indicating status of each run
#
#########################################################################
sub constructHTMLBody() {
    open( MYLOG, $LOGTXT ) || print("WARNING: can't open log file $LOGTXT: $!\n");
    @all_lines = <MYLOG>;
    close( MYLOG );
    
    my $line;
    my $f_cnt = 0;
    my $p_cnt = 0;
    foreach $line ( ($nosort) ? @all_lines : sort @all_lines ) {
        # avoid linebreaks
	chop $line;
	if ($line eq "" || $line =~ /^\0*$/ || $line =~ /^\s*$/) {
	    next;
	}   
	$comment = "---";
	# assuming that all lines are kind'of 'aaa=bbb'
	($class, $status) = split /\: /, $line;	
	if ($status =~ /(.*?) \((.*?)\)$/) {
		$status = $1;
		$comment = $2 ? $2 : "---";
	}
	if ($status =~ /FAIL/) {
		$status = "<font color=\"red\">".$status;
		$status = $status."<\/font>";
		$f_cnt++;
	} elsif ($status =~ /PASS/) {
		$p_cnt++;
	}
	if ($genparam eq "ALL" || ($genparam ne "" && $line =~ /$genparam/)) {
		print LOGHTML "<tr><td><a target=\"$TARGET_WINDOW\" href=\"$TEST_DESCRIPTIONS#$class\">",$class,"</a></td><td>",$status,"</td><td>",$comment,"</td></tr>\n";
	}
    } 
    if ($p_cnt+$f_cnt) {
	    $pp = sprintf "%.2f", $p_cnt/($p_cnt+$f_cnt)*100;  
	    $pf = sprintf "%.2f", $f_cnt/($p_cnt+$f_cnt)*100;
    }
    print LOGHTML "<tr><td colspan=3>Total: $p_cnt($pp\%) tests passed and $f_cnt($pf\%) tests failed.</td></tr>\n";
}
		
#########################################################################
#
# Construct Output HTML file Footer
#
#########################################################################
sub constructHTMLFooter() {
    print LOGHTML "</table></body></html>\n";
}

#########################################################################
#
# Construct Output HTML file indicating status of each run
#
#########################################################################
sub constructHTML() {
	constructHTMLHeader();
	constructHTMLBody();
	constructHTMLFooter();
}

#########################################################################
#
# Construct LogFile Header. The Log file is always appended with entries
#
#########################################################################
sub constructLogHeader() {
    print LOGFILE "\n";
    print LOGFILE "\n";    
    print LOGFILE $delimiter;
    $date = localtime;
    print LOGFILE "Logging Test Run on $date ...\n";
    print LOGFILE $delimiter;
    print LOGFILE "\n";    	

    print "All Log Entries are maintained in LogFile $LOGFILE\n";
    print "\n";
}

#########################################################################
#
# Construct LogFile Footer. 
#
#########################################################################
sub constructLogFooter() {
    print "\n";
    print LOGFILE $delimiter;
    $date = localtime;
    print LOGFILE "End of Logging Test $date ...\n";
    print LOGFILE $delimiter;
    print "\n"; 
}

########################################################################
#
# Construct Log String
#
########################################################################
sub constructLogString {
    my $logstring = shift(@_);
    print LOGFILE "$logstring\n";
    print "$logstring\n";

}

########################################################################
#
# Safely append to file : open, append, close.
#
########################################################################
sub safeAppend {
    my $file = shift(@_);
    my $line = shift(@_);
    open (FILE, ">>$file") or die ("Cann't open $file");
    print FILE $line;
    close FILE;
}


#######################################################################
#
# Running Test case under Win32
#
#######################################################################

sub RunTestCaseWin {
# win32 specific values 
# STILL_ALIVE is a constant defined in winbase.h and indicates what
# process still alive
    $STILL_ALIVE = 0x103;
    do 'Win32/Process.pm';

    open(SAVEOUT, ">&STDOUT" );
    open(SAVEERR, ">&STDERR" );
    open(STDOUT, ">$testlog" ) || die "Can't redirect stdout";
    open(STDERR, ">&STDOUT" ) || die "Can't dup STDOUT";
    select STDERR; $|=1; select STDOUT; $|=1; 
    Win32::Process::Create($ProcessObj,
			   "$mozhome/$MOZILLA_EXECUTABLE",
			   "$mozhome/$MOZILLA_EXECUTABLE $ADDITIONAL_PARAMETERS $DOCFILE",
			   1,
			   NORMAL_PRIORITY_CLASS,
			   "$mozhome" ) || die "cann't start apprunner";
    close(STDOUT);
    close(STDERR);
    open(STDOUT, ">&SAVEOUT");
    open(STDERR, ">&SAVEERR");
    
    $crashed = 0;
    $cnt = 0;
    while (true) {
	sleep($DELAY_OF_CYCLE);
	system("$curdir\\killer.exe");
	$ProcessObj->GetExitCode($exit_code);
	if ($exit_code != $STILL_ALIVE ) {
	    $crashed = 1;
	    $logstr = "Test FAILED...";
	    constructLogString "$logstr";
	    $logstr = "Mozilla terminated with exit code $exit_code.";
	    constructLogString "$logstr";
	    $logstr = "Check ErrorLog File $testlog ";
	    constructLogString "$logstr";
	    constructLogString "========================================\n";	
	    safeAppend $LOGTXT, "$testcase: FAILED (Mozilla crashed with exitcode $exit_code)\n";
	    last;
	}	    
	$cnt += $DELAY_OF_CYCLE;
	if ( $cnt >= $DELAY_FACTOR ) {
	    $ProcessObj->Kill(0);
	    last;
	}
    } # while with sleep
    $crashed;
}


#######################################################################
#
# Running Test case under Unix
#
#######################################################################

sub RunTestCaseUnix {
    $exit_code = 0;
    $pid = 0;
    open(SAVEOUT, ">&STDOUT" );
    open(SAVEERR, ">&STDERR" );
    open(STDOUT, ">$testlog" ) || die "Can't redirect stdout";
    open(STDERR, ">&STDOUT" ) || die "Can't dup STDOUT";
    select STDERR; $|=1; select STDOUT; $|=1; 
    if (!($pid = fork())) {
	#this is a child process - we should start mozilla in it
        # the second exec to overide any shell parsing
        # If argument list had any metacharacter (like ~) then it is pass
        # to /bin/sh -c as a separate process. To avoid that we have the
        # second exec.
	exec(" exec $mozhome/$MOZILLA_EXECUTABLE $ADDITIONAL_PARAMETERS $DOCFILE"); 
	close(STDOUT);
	close(STDERR);
	open(STDOUT, ">&SAVEOUT");
	open(STDERR, ">&SAVEERR");
	die "ERROR: Can't start mozilla: $!\n";	
    }
    #this is a parent process - we should control mozilla execution from it
    close(STDOUT);
    close(STDERR);
    open(STDOUT, ">&SAVEOUT");
    open(STDERR, ">&SAVEERR");
    $cnt = 0;
    $crashed = 0;
    print("Mozilla's pid: $pid\n");
    while (true) {
	sleep($DELAY_OF_CYCLE);
	#print "Try call waitpid ...\n";
	if (waitpid($pid,&WNOHANG)) { 
	    $crashed = 1;
	    $logstr = "Test FAILED...";
	    constructLogString "$logstr";
	    $logstr = "Mozilla terminated by signal ".($? & 127)." with exitcode ".($? >> 8);
	    constructLogString "$logstr";		
	    $logstr = "Check ErrorLog File $testlog ";
	    constructLogString "$logstr";
	    constructLogString "========================================\n";
	    safeAppend $LOGTXT, "$testcase: FAILED (Mozilla terminated by signal ".($? & 127)." with exitcode ".($? >> 8).")\n";
	    last;
	}		
	$cnt += $DELAY_OF_CYCLE;
	if ( $cnt >= $DELAY_FACTOR ) {	
	    kill(9, $pid);
            #now we should take exitcode to avoid zombie appearance
            sleep(1);
            waitpid($pid,&WNOHANG);
	    last;
	}
    } # while with sleep
    $crashed;
}


sub compareResults {
	$out_fn = shift;
	$fn1 = shift;
	$fn2 = shift;
	if (!defined($out_fn)) {
		$out_fn = "$curdir/log/diff";
	}
	if (!defined($fn2)) {
		$fn2 = "$curdir/log/BWTest.txt.bak";
	}
	if (!defined($fn1)) {
		$fn1 = "$curdir/log/BWTest.txt";
	}

	(-f $fn1 && open(IN1, "<$fn1")) || die "Can't open file $fn1: $!\n";
	(-f $fn2 && open(IN2, "<$fn2")) || die "Can't open file $fn2: $!\n";
	open(OUT_TXT, ">$out_fn".".txt") || die "Can't open output file $out_fn.txt: $!\n";
	$genparam = "ALL";
	$nosort = 1;
	
	my %res1;
	my %line1;
	my %res2;
	my $ln = 0;
	my $j=0;
	
	while($line = <IN1>) {
		$ln ++;
		chop $line;
		if ($line =~ /(.*): (.*)/) {
			$res1{$1} = $2;
			$line1{$1} = $line;
		} else {
			die "File format error in file $fn1, line $ln: $line\n";
		}
	}
	$ln = 0;
	while($line = <IN2>) {
		$ln ++;
		chop $line;
		if ($line =~ /(.*): (.*)/) {
			if (defined($res1{$1})) {
				if ($res1{$1} ne $2) {
					print OUT_TXT $line1{$1}."\n";
					print OUT_TXT $line."\n";
				}
				delete $res1{$1};
			} else {
				$new_in_2[$j++] = $line;
				#file2 contain extra tests (that file1 doesn't contain)
				#file1 can also contain extra tests (that file2 doen't contain)
				#do we need to report about it ?
			}
		} else {
			die "File format error in file $fn2, line $ln: $line\n";
		}
	}

	#report about extra tests in file1
	@remained_keys = keys %res1;
	if ($#remained_keys >= 0) {
		print OUT_TXT " : EXTRA (TESTS IN $fn1:)\n";
		foreach $key (@remained_keys) {
			print OUT_TXT "$line1{$key}\n";
		}
	}
	#report about extra tests in file2
	if ($j > 0) {
		print OUT_TXT " : EXTRA (TESTS IN $fn2:)\n";
		for($i=0; $i<$j; $i++) {
			print OUT_TXT "$new_in_2[$i]\n";
		}
	}
	if (<OUT_TXT>) {
		close OUT_TXT;
		$LOGTXT = "$out_fn".".txt";
		open(LOGHTML, ">$out_fn.html") || die "Can't open output file $out_fn.html: $!\n";
		constructHTML();
		close LOGHTML;
	} else {
		close OUT_TXT;
		print "Results are equivalent !\n";
	}
	close IN1;
	close IN2;
}

##################################################################
# main
##################################################################

title;

$curdir = cwd();           

# Prepare file names
$x=`mkdir -p $curdir/log`; 
$LOGFILE = "$curdir/log/BWTestRun.log";
$LOGTXT = "$curdir/log/BWTest.txt";
$LOGHTML = "$curdir/log/BWTest.html";


# process command-line parameters
# and check for valid usage

$testparam = "";
$genparam = "";

if ($ARGV[0] =~ /help/ || $ARGV[0] =~ /\?/) {
	usage();
	exit(0);
}

if ($ARGV[0] =~ "-genhtml") {
	$genparam = $ARGV[1];
        if ($genparam eq "") {
           $genparam = "ALL";
         }

	open( LOGHTML, ">$LOGHTML" ) or die("Can't open HTML file $LOGHTML ($!)...\n");
	print("Generating result HTML page ...\n");
	constructHTML();
	exit(0);
}

if ($ARGV[0] eq "-compare") {
	compareResults($ARGV[1], $ARGV[2], $ARGV[3]);
	exit(0);	
}	

if ( $#ARGV >= 0 ) {
    if ( $ARGV[0] != "-t" ) {
	usage;
	die;
    } elsif ($ARGV[1] ne "") {
	$testparam = $ARGV[1];
    } else {
	usage();
	die;
    }
}

## Default behaviour is to generate the HTML file
if ($genparam eq "") {
  $genparam = "ALL";
}

$mozhome = $ENV{"MOZILLA_FIVE_HOME"};

if ( $mozhome eq "" ) {
    print "MOZILLA_FIVE_HOME is not set. Please set it and rerun this script....\n";
    die;
}

if ( ! -f "$mozhome/$MOZILLA_EXECUTABLE" ) {
    print "Could not find $MOZILLA_EXECUTABLE in MOZILLA_FIVE_HOME.\n";
    print "Please check your setting...\n";
    die;
}

# Here must come a piece of code, that determinates 
# apprunner instance, removes core, but there's no
# core under win32

# Backup existing .lst file
if ( ! -f "$curdir/$FULL_TEST_LIST" ) {
	print "Can't find list of OJI API tests to be run.";
	die;
}


$id=$$;


# check if output text file of previous run exists.
# if so, then save it as .bak
#
if ( -f "$LOGTXT" ) {
    $newfile="$LOGTXT.bak";
    rename $LOGTXT, $newfile;
}

# check if output html file of previous run exists.
# if so, then save it as .bak
#
if ( -f "$LOGHTML" ) {
    $newfile="$LOGHTML.bak";
    rename $LOGHTML, $newfile;
}

# construct DOCFILE
$DOCFILE = "$DOCROOT/test.html";
$runcnt = 1;
$filename = "$curdir/$FULL_TEST_LIST";

# Prepare log streams

open( LOGHTML, ">$LOGHTML" ) or die("Can't open HTML file...\n");
open( LOGFILE, ">>$LOGFILE" ) or die("Can't open LOG file...\n");
select LOGFILE; $| = 1; select STDOUT; 




constructLogHeader();

$LST_OUT = "$mozhome/OJITests.lst";

$currcnt = 0;
while (true) {
    open( file, $filename ) or die("Can't open $filename...\n");
    while( $line = <file> ) {
	chop $line;
	if ( $testparam ne "" ) {
	    $testcase = $testparam;
	} else {
	    $testcase = $line;
	}

	if ( $testcase eq "" || $testcase =~ /^\s*#/ ) {
	    next;
	} 

	open(LST_OUT, ">$LST_OUT") or die ("Can't open LST_OUT file...\n");
	print LST_OUT $testcase;
	close(LST_OUT);

	chdir( $mozhome );
	#deleting old result ...
	unlink("$mozhome/$TEST_RESULTS");
	$logstr="Running TestCase $testcase....";
	constructLogString "========================================";
	constructLogString "$logstr";

	($nom) = ($testcase =~ /([^\.]*)$/);
	$testlog = "$curdir/log/$nom.$id.log";

	if ($^O =~ /Win32/i) {
	    $crashed = RunTestCaseWin();
	} else {
	    $crashed = RunTestCaseUnix();
	}

	if (!$crashed) {
	    if (!open (TEST_RES, "$mozhome/$TEST_RESULTS") || ($logstr = <TEST_RES>) =~ /^\s*$/) {
		$logstr = "$testcase: FAILED (undefined result)";
	    }
      	    close TEST_RES;
	    chomp $logstr;
	    constructLogString "$logstr";
	    constructLogString "========================================\n";
	    safeAppend "$LOGTXT", "$logstr\n";
	}

	( $testparam eq "" ) || last;
    
    } # while ( $line
    
    ( ++$currcnt < $runcnt ) || last;
    
} # while(true)
constructLogFooter;

constructHTML();

if (-f "$curdir/log/BWTest.txt.bak") {
	print("Old result file found => generating diff files: \nTXT:->$curdir/log/diff.txt \nHTML:->$curdir/log/diff.html");
	compareResults();
}

chdir($curdir);