#!/usr/bin/perl -w

# Tool to summarise a journal file generated by TET
# or to determine the difference in test results between two
# journals.
#
# (C) Copyright 2001 The Free Standards Group  Inc
#
# 21/5/2001 Chris Yeoh, IBM
#
# This is $Revision: 1.1 $
#
# $Log: tjreport,v $
# Revision 1.1  2005/06/07 07:53:09  phost
# initial check in using the latest files in tests/lsb-runtime-test/scripts/support
#
# Revision 1.18  2005/03/29 14:30:15  ajosey
# allow for TEST_VERS and TEST_ARCH not to be set
#
# Revision 1.17  2005/03/29 14:17:26  ajosey
# revamp tjreport to get waiver files over the network if no other option specified, bug 650
#
# Revision 1.16  2005/03/29 07:35:01  ajosey
# correct test counts, bug 213, have to count twice for os sections with M section tests generated
#
# Revision 1.15  2005/02/25 21:17:08  mats
# Add a line counter so you can figure out what line the journal caused a failure
#
# Revision 1.14  2005/02/24 13:32:35  mats
# Output cleanup per bug 698
#
# Revision 1.13  2004/05/12 07:14:40  cyeoh
# Checks that expected number of test results matches with number of
# test results seen. It only checks if a non zero number of results is
# expected to support journal files which do not specify the expected
# number of test results
#
# Revision 1.12  2004/03/28 15:59:17  mats
# Collected tweaks: improve processing loop, accept build and clean journals
# as well (110, 300 lines), quit if journal is corrupt (e.g. hand-edited)
#
# Revision 1.11  2003/10/31 02:34:37  cyeoh
# Only print test system information if it exists
# Adds verbose flag which causes error information to be printed
#
# Revision 1.10  2003/07/17 07:17:49  cyeoh
# change error about test inconsistency to warning as lsblibchk
# is creating duplicates at the moment
#
# Revision 1.9  2002/01/16 05:14:00  cyeoh
# clean up help information
# remove dependency on DBI when not used
#
# Revision 1.8  2001/12/04 07:39:41  cyeoh
# Add support for waiver files
#
# Revision 1.7  2001/11/05 07:31:19  cyeoh
# Fixes bug that caused the missing of some testcases
# Adds cross check of testcase count
# Removes debug of DBI calls
#
# Revision 1.6  2001/08/22 03:52:32  cyeoh
# Fixes die error reporting
# Adds storage and update of testsuite to test case mapping
#
# Revision 1.5  2001/08/14 08:31:59  cyeoh
# Fix generation of description information so that
# journal prefix ids are removed
#
# Revision 1.4  2001/08/14 04:59:55  cyeoh
# Rework for change in db format.
# Now keeps track of how many results available for each testcase
#
# Revision 1.3  2001/08/14 02:18:07  cyeoh
# Adds ability to add error information for failed tests into database
#
# Revision 1.2  2001/08/13 04:51:04  cyeoh
# Adds ability to add data into SQL database for further analysis
#
# Revision 1.1  2001/08/11 11:46:13  cyeoh
# Initial version
#
#

use strict;
use Getopt::Std;

my (%StateMap) = (
    0   => "PASS",
    1   => "FAIL",
    2   => "UNRESOLVED",
    3   => "NOTINUSE",
    4   => "UNSUPPORTED",
    5   => "UNTESTED",
    6   => "UNINITIATED",
    7   => "UNREPORTED",
    101 => "WARNING",
    102 => "FIP",
    103 => "NOTIMP",
    104 => "UNAPPROVE"
);

my (%PFMap) = (
    "PASS"        => "PASS",
    "FAIL"        => "FAIL",
    "UNRESOLVED"  => "FAIL",
    "NOTINUSE"    => "PASS",
    "UNSUPPORTED" => "PASS",
    "UNTESTED"    => "PASS",
    "UNINITIATED" => "FAIL",
    "UNREPORTED"  => "FAIL",
    "WARNING"     => "PASS",
    "FIP"         => "PASS",
    "NOTIMP"      => "PASS",
    "UNAPPROVE"   => "PASS",
    "MISSING"     => "FAIL",
    "UNKNOWN"     => "FAIL"
);

my ($VerboseSummary) = 0;

# Analyses one journal file to get statistics on pass/failure
#
sub GatherStats($) {
    my ($journalFile) = shift;
    my ($stats)       = {};
    my ($loop);
    my ($line);
    local (*JFILE);

    # Initialise
    $stats->{STATE_SUMMARY} = {};
    foreach $loop ( keys %StateMap ) {
        $stats->{STATE_SUMMARY}{ $StateMap{$loop} } = 0;
    }
    $stats->{STATE_SUMMARY}{TEST_ERROR} = 0;
    $stats->{STATE_SUMMARY}{UNKNOWN}    = 0;
    $stats->{TOTAL_TESTS_PASSED}        = 0;
    $stats->{TOTAL_TESTS_FAILED}        = 0;
    $stats->{TOTAL_TESTS_EXPECTED}      = 0;
    $stats->{TEST_VERS} 		= "unset";
    $stats->{TEST_ARCH} 		= "unset";

    # Analyse file
    open( JFILE, $journalFile ) || die "Could not open file: $journalFile\n";

    my ($testName);
    my ($testNum);
    my (@line);
    my ($testState);
    my ($errorMessage);
    my ($tmp);
    my ($lineno) = 0;
    while ( defined( $line = <JFILE> ) ) {

        $lineno++;
        # Look for system info
        if ( $line =~ /^0\|(.*)\|/ ) {
            @line = split( / /, $1 );
            $stats->{TEST_DATE} = $line[2];
            $stats->{TEST_TIME} = $line[1];
            next;
        }
        elsif ( $line =~ /^30\|.*VSX_SYS=(.*)$/ ) {
            $stats->{TEST_SYSTEM} = $1;
            next;
        }

# record test version and also architecture
        elsif ( $line =~ /^30\|.*VSX_NAME=LSB Certification Version (.*) (.*)$/ ) {
            $stats->{TEST_VERS} = $1;
            $stats->{TEST_ARCH} = $2;
            next;
        }
# for the .os sections need to double expected counts to allow
# for macro versions.
        elsif ( $line =~ /^70\|.*total tests in (ANSI|POSIX|LSB|PTHR).os ([0-9]+)/ ) {
            $stats->{TOTAL_TESTS_EXPECTED} += $2;
            $stats->{TOTAL_TESTS_EXPECTED} += $2;
        }
        elsif ( $line =~ /^70\|.*total tests in ([A-Za-z.0-9]+) ([0-9]+)/ ) {
            $stats->{TOTAL_TESTS_EXPECTED} += $2;
        }

        # Look for test results
        @line = split( / /, $line );
        if ( $line[0] =~ /^10/ || $line[0] =~ /^110/ || $line[0] =~ /^300/ ) {
            $testName = $line[1];
        }
        elsif ( $line[0] =~ /^200/ ) {
            $testNum      = $line[1];
            $errorMessage = "";
        }
        elsif ( $line[0] =~ /^220/ ) {

            # Test state report
            $testState =
              exists( $StateMap{ $line[2] } )
              ? $StateMap{ $line[2] }
              : "UNKNOWN";

            if ( exists( $stats->{TESTS}{$testName}{$testNum}{STATE} ) ) {
                die "Error: dup entry at ${testName} ${testNum}, line $lineno, journal invalid\n";
            }
            $stats->{STATE_SUMMARY}{$testState}++;

            $stats->{TESTS}{$testName}{$testNum}{STATE} = $testState;

            if ( $PFMap{$testState} eq "PASS" ) {
                $stats->{TOTAL_TESTS_PASSED}++;
            }
            else {
                $stats->{TOTAL_TESTS_FAILED}++;
                $stats->{TESTS}{$testName}{$testNum}{INFO} = $errorMessage;
            }
        }
        elsif ( $line[0] =~ /^520/ ) {

            # Accumulate any error/info messages associated with test
            $line =~ /\|([^\|]*)$/;
            $tmp = $1;
            chomp($tmp);
            $errorMessage .= "$tmp\n";
        }
    }

    close(JFILE);
    return $stats;
}

#----------------------------------------------------------------------
# Find the difference in test results between the two journals
sub DiffJournals($$) {
    my ($j1)        = shift;
    my ($j2)        = shift;
    my ($diffStats) = {};
    my ($testName);
    my ($testNum);

    $diffStats->{TESTS} = {};

    foreach $testName ( sort keys %{ $j1->{TESTS} } ) {
        foreach $testNum ( sort { $a <=> $b }
            keys %{ $j1->{TESTS}{$testName} } )
        {
            if (   exists( $j2->{TESTS}{$testName} )
                && exists( $j2->{TESTS}{$testName}{$testNum} ) )
            {
                if ( $j1->{TESTS}{$testName}{$testNum} ne
                    $j2->{TESTS}{$testName}{$testNum} )
                {
                    $diffStats->{TESTS}{$testName}{$testNum} =
                        "$j1->{TESTS}{$testName}{$testNum}{STATE},"
                      . "$j2->{TESTS}{$testName}{$testNum}{STATE}";
                }
            }
            else {
                $diffStats->{TESTS}{$testName}{$testNum} =
                  "$j1->{TESTS}{$testName}{$testNum}{STATE},MISSING";
            }
        }
    }

    # Check reverse
    foreach $testName ( sort keys %{ $j2->{TESTS} } ) {
        foreach $testNum ( sort { $a <=> $b }
            keys %{ $j2->{TESTS}{$testName} } )
        {
            if (
                !(
                       exists( $j2->{TESTS}{$testName} )
                    && exists( $j2->{TESTS}{$testName}{$testNum} )
                )
              )
            {
                $diffStats->{TESTS}{$testName}{$testNum} =
                  "MISSING,$j2->{TESTS}{$testName}{$testNum}{STATE}";
            }
        }
    }

    return $diffStats;
}

#----------------------------------------------------------------------
#
sub PrintDiffSummary($$) {
    my ($diffStats)  = shift;
    my ($isDetailed) = shift;

    my ($testName);
    my ($testNum);
    my ( $state1, $state2 );

    foreach $testName ( sort keys %{ $diffStats->{TESTS} } ) {
        foreach $testNum (
            sort { $a <=> $b }
            keys %{ $diffStats->{TESTS}{$testName} }
          )
        {
            ( $state1, $state2 ) =
              split( /,/, $diffStats->{TESTS}{$testName}{$testNum} );
            if ( ( $PFMap{$state1} ne $PFMap{$state2} ) || $isDetailed ) {
                print "$testName $testNum $diffStats->{TESTS}{$testName}{$testNum}\n";
            }
        }
    }
}

#----------------------------------------------------------------------
#
sub PrintSummary($$$) {
    my ($stats)      = shift;
    my ($isDetailed) = shift;
    my ($waivers)    = shift;

    my ($testState);

    my ($testName);
    my ($testNum);
    my ($numTests);
    my ($secondCount) = 0;
    my ($waived)      = 0;

    foreach $testName ( sort keys %{ $stats->{TESTS} } ) {
        foreach $testNum ( sort { $a <=> $b }
            keys %{ $stats->{TESTS}{$testName} } )
        {
            $secondCount++;

            #      print "$testName $testNum\n";
            if (
                (
                    $PFMap{ $stats->{TESTS}{$testName}{$testNum}{STATE} } eq
                    "FAIL"
                )
                || $isDetailed
              )
            {
                print "$testName $testNum $stats->{TESTS}{$testName}{$testNum}{STATE}";
                if ( defined($waivers) && $waivers->{"$testName-$testNum"} ) {
                    print " (WAIVED)";
                    $waived++;
                }
                print "\n";
                if ($VerboseSummary) {
                    print "  $stats->{TESTS}{$testName}{$testNum}{INFO}";
                }
            }
        }
    }

    $numTests = $stats->{TOTAL_TESTS_PASSED} + $stats->{TOTAL_TESTS_FAILED};

    print "Warning: Inconsistency in test count. "
      . "This is probably due to a bug in this program. "
      . "($numTests, $secondCount).\n"
      unless $numTests == $secondCount;

    print "\n\n";
    print "Test system: $stats->{TEST_SYSTEM}\n"
      unless ( !defined( $stats->{TEST_SYSTEM} ) );
    print "Test was run: $stats->{TEST_DATE} $stats->{TEST_TIME} \n";
    print "Test Suite Version: $stats->{TEST_VERS}\n";
    print "Test Suite Architecture: $stats->{TEST_ARCH}\n";

    print "Total Tests Passed: $stats->{TOTAL_TESTS_PASSED}\n";
    print
      "Total Tests Failed (including waived): $stats->{TOTAL_TESTS_FAILED}\n";
    print "Total Tests Failed (excluding waived): "
      . ( $stats->{TOTAL_TESTS_FAILED} - $waived ) . "\n";

    print "\nTest Result Breakdown:\n";
    foreach $testState ( keys %{ $stats->{STATE_SUMMARY} } ) {
        print "$testState: $stats->{STATE_SUMMARY}{$testState}\n";
    }

    if (   $stats->{TOTAL_TESTS_EXPECTED} > 0
        && $stats->{TOTAL_TESTS_EXPECTED} != $numTests )
    {
        print
          "\nJournal indicates expected $stats->{TOTAL_TESTS_EXPECTED} tests,"
          . " found $numTests\n";
    }

}

#----------------------------------------------------------------------
#
sub LoadIntoDatabase($$$$) {
    my ($dbh)         = shift;
    my ($stats)       = shift;
    my ($vendorName)  = shift;
    my ($versionName) = shift;
    my ($testName);
    my ($testNum);
    my ($sth);
    my ($distroId);

    print "Adding information to DB\n";

    $sth =
      $dbh->prepare(
            "SELECT DistroId from distributions where VendorName='$vendorName'"
          . " and VersionName='$versionName'" )
      || die $dbh->errstr;
    $sth->execute() || die $dbh->errstr;
    if ( $sth->rows == 0 ) {

        # Add distribution information to database
        print "Adding distribution information to database\n";
        $sth =
          $dbh->prepare( "INSERT INTO distributions (VendorName, VersionName)"
              . " Values ('$vendorName', '$versionName')" )
          || die $dbh->errstr;
        $sth->execute() || die $dbh->errstr;
        $sth =
          $dbh->prepare(
            "SELECT DistroId from distributions where VendorName='$vendorName'"
              . " and VersionName='$versionName'" )
          || die $dbh->errstr;
        $sth->execute() || die $dbh->errstr;
    }
    die "found " . $sth->rows . " occurences of distro name\n"
      unless $sth->rows == 1;

    # Get distro id
    $distroId = $sth->fetchrow_hashref->{DistroId};
    print "Distribution id: $distroId\n";

    $sth =
      $dbh->prepare(
            "INSERT INTO results (Testcase, Distro, State, Description)"
          . "Values (?, ?, ?, ?)" )
      || die $dbh->errstr;

    my ($errorMessage);
    my ($testcaseId);
    my ($id_sth);
    foreach $testName ( sort keys %{ $stats->{TESTS} } ) {
        foreach $testNum ( sort { $a <=> $b }
            keys %{ $stats->{TESTS}{$testName} } )
        {

            # Get testcase id
            $id_sth =
              $dbh->prepare( "SELECT TestcaseId from testcases"
                  . " where Name='$testName-$testNum'" )
              || die $dbh->errstr;
            $id_sth->execute();
            if ( $id_sth->rows == 0 ) {
                $id_sth =
                  $dbh->prepare( "INSERT INTO testcases (Name)"
                      . " Values ('$testName-$testNum')" )
                  || die $dbh->errstr;
                $id_sth->execute() || die $dbh->errstr;
                $id_sth =
                  $dbh->prepare( "SELECT TestcaseId from testcases"
                      . " where Name='$testName-$testNum'" )
                  || die $dbh->errstr;
                $id_sth->execute() || die $dbh->errstr;
            }
            $testcaseId = $id_sth->fetchrow_hashref->{TestcaseId};

            if ( exists( $stats->{TESTS}{$testName}{$testNum}{INFO} ) ) {
                $errorMessage = $stats->{TESTS}{$testName}{$testNum}{INFO};
            }
            else {
                $errorMessage = "";
            }
            $sth->execute( $testcaseId, $distroId,
                $stats->{TESTS}{$testName}{$testNum}{STATE},
                $errorMessage )
              || die "$dbh->errstr\n$testName\n$testNum";
        }
    }

    # Update test case count
    $sth =
      $dbh->prepare( "REPLACE INTO testcase_count (Testcase, NumberEntries) "
          . "SELECT Testcase, Count(*) FROM results GROUP BY Testcase" )
      || $sth->errstr;
    $sth->execute() || die $sth->errstr;

    # Commit changes
    $dbh->commit || die $dbh->errstr;

}

#----------------------------------------------------------------------
# Set test suite values for each testcase
sub SetTestSuiteForTestcases($) {
    my ($dbh) = shift;
    my ($sth);
    my ($row);
    my ($update);

    $sth =
      $dbh->prepare( "SELECT * from testsuites, testsuite_idstrings "
          . "where TSItestsuite=TSid" )
      || die $dbh->errstr;
    $sth->execute() || die $sth->errstr;

    while ( $row = $sth->fetchrow_hashref ) {
        print "$row->{TSname}\n";

        #    $dbh->trace(1);
        $update =
          $dbh->prepare( "UPDATE testcases SET testsuiteid=$row->{TSid} "
              . "WHERE Name LIKE '%/$row->{TSIidstring}/%'" )
          || die $dbh->errstr;
        $update->execute() || die $update->errstr;
    }

    # Commit changes
    $dbh->commit() || die $dbh->errstr;
}

#----------------------------------------------------------------------
# Load waiver file
# Returns hash of waived tests
sub LoadWaiverFile($) {
    my ($waiverFilename) = shift;
    local (*WAIVERFILE);
    my ($waived) = {};

    die "Could not open waiver file $waiverFilename"
      unless open( WAIVERFILE, $waiverFilename );
    my ($line);
    while ( defined( $line = <WAIVERFILE> ) ) {
        chomp($line);

        # '#' is the comment character for the waiver file
        if ( $line !~ /^\#/ ) {
            $waived->{$line} = 1;
        }
    }
    return $waived;
}

#----------------------------------------------------------------------
# Load networked waiver file via wget
# Returns hash of waived tests
sub LoadNetWaiverFile($) {
    my $filename=shift;
    local (*WAIVERFILE);
    my $ret;
    my ($waived) = {};

#  if a test version is not determined
    if ( $filename =~ /unset/ ) {
       return $waived;
    }

    print "Loading networked waivers information -- file: http://www.opengroup.org/infosrv/lsb/waivers/$filename\n";
    $ret = system "wget -nv --output-document=waivers.txt http://www.opengroup.org/infosrv/lsb/waivers/$filename 2>&1 >/dev/null" ;
    if ( $ret != 0 )
    {
    	print "Warning, wget was unable to locate network waivers to retrieve..\n";
        return;
    }

    die "Could not open waiver file waivers.txt"
      unless open( WAIVERFILE, "./waivers.txt" );
    my ($line);
    while ( defined( $line = <WAIVERFILE> ) ) {
        chomp($line);

        # '#' is the comment character for the waiver file
        if ( $line !~ /^\#/ ) {
            $waived->{$line} = 1;
        }
    }
    return $waived;
}


#----------------------------------------------------------------------
# Main bit

my (%options);

getopts( 'dhu:p:b:e:r:o:w:v', \%options );

if ( exists( $options{'h'} ) || ( $#ARGV != 0 && $#ARGV != 1 ) ) {
    print STDOUT <<"EOM"
Usage: $0 [-h] [-d] [-u username] [-p password] [-b db_name] 
          [-e vendor_name] [-r version] journal [journal2]

    -h               Display Help
    -d               Display a detailed summary of test results. The
                     test result is shown even if the test passed
    -u username      Username for connection to Mysql db
    -p password      Password for connection to Mysql db
    -b db_name       Name of Mysql Database name
    -o host          Host for Mysql databse (defaults to localhost)
    -e vendor_name   Vendor name of distribution
    -r version       Version name for distribution
    -w waiver file   Take into account waived failures for test suite
    -v               Verbose mode (show error messages)

    When one journal file is supplied a summary of the tests
    is output. When two journal files are supplied the difference
    between the two is shown.

EOM
      ;
    exit(0);
}

my ($dbh);
if ( exists( $options{'b'} ) ) {
    require DBI;
    if ( !exists( $options{'e'} ) || !exists( $options{'r'} ) ) {
        die "Must specify vendor and version of distribution\n";
    }

    my ($data_source) = "DBI:mysql:database=$options{'b'}";
    if ( exists( $options{'o'} ) ) {
        $data_source .= ";host=$options{'o'}";
    }

    print "Using datasource: $data_source\n";

    # DB Init
    $dbh = DBI->connect(
        $data_source,
        exists( $options{'u'} ) ? $options{'u'} : "",
        exists( $options{'p'} ) ? $options{'p'} : "",
        { AutoCommit => 0 }
    );
    die "Could not connect to database\n" unless defined($dbh);
}

if ( exists( $options{'v'} ) ) {
    $VerboseSummary = 1;
}

# Diff or summary
if ( $#ARGV == 1 ) {
    my ($stats1);
    my ($stats2);
    my ($diffStats);
    $stats1    = GatherStats( $ARGV[0] );
    $stats2    = GatherStats( $ARGV[1] );
    $diffStats = DiffJournals( $stats1, $stats2 );
    PrintDiffSummary( $diffStats, exists( $options{'d'} ) );
}
else {
    my ($stats);
    $stats = GatherStats( $ARGV[0] );
    if ( exists( $options{'b'} ) ) {
        LoadIntoDatabase( $dbh, $stats, $options{'e'}, $options{'r'} );
        SetTestSuiteForTestcases($dbh);
        $dbh->disconnect();
    }
    else {
        my ($waivers);
        if ( exists( $options{'w'} ) ) {
            $waivers = LoadWaiverFile( $options{'w'} );
        }
        else  {
        $waivers = LoadNetWaiverFile($stats->{TEST_VERS});
        }
        PrintSummary( $stats, exists( $options{'d'} ), $waivers );
    }
}

