#! /usr/bin/perl -w
##**************************************************************
##
## Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
## University of Wisconsin-Madison, WI.
## 
## Licensed under the Apache License, Version 2.0 (the "License"); you
## may not use this file except in compliance with the License.  You may
## obtain a copy of the License at
## 
##    http://www.apache.org/licenses/LICENSE-2.0
## 
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
##**************************************************************


##********************************************************************
## Tests whether the undertaker works
##********************************************************************

use strict;
use FindBin;
use lib ($FindBin::Bin, "$FindBin::Bin/lib", "$FindBin::Bin/../lib");
use Execute;
use File::Copy;
use Getopt::Long;

#***
# Constant Static Variables
#***
my $UNIQ_PID_MIDWIFE_CMD = 'uniq_pid_midwife';
my $UNIQ_PID_UNDERTAKER_CMD = 'uniq_pid_undertaker';
my $FILE_OPT = '--file';
my $BLOCK_OPT = '--block';
my $NO_BLOCK_OPT = '--noblock';
my $PRECISION_OPT = '--precision';
my $PRECISION = 2;
my $SLEEP_CMD = 'sleep';
my $LONG_SLEEP = 28800; #8 hrs
my $SHORT_SLEEP = 1;
my $PID_FILE = "pid.file.test";

my $PID_HEADER = 'PID';
my $BDAY_HEADER = 'BDAY';
my $CONFIRM_HEADER = 'CONFIRM';
my $CONTROL_HEADER = 'CONTROL_TIME';

my @ALIVE_TESTS = ( \&Test1Alive,
		   \&Test2Alive,
		   \&Test3Alive,
		   \&Test4Alive,
		   \&Test5Alive,
		   \&Test6Alive,
		   \&Test7Alive
		  );

my @DEAD_TESTS = ( \&Test1Dead,
		   \&Test2Dead,
		   \&Test3Dead,
		   \&Test4Dead,
		   \&Test5Dead,
		   \&Test6Dead,
		   \&Test7Dead,
		   \&Test8Dead,
		   \&Test9Dead,
		   \&Test10Dead,
		   \&Test11Dead
		  );

my @ANOMALY_TESTS = ( \&Test1Anomaly,
		      \&Test2Anomaly,
		      \&Test3Anomaly,
		      \&Test4Anomaly,
		      \&Test5Anomaly,
		      \&Test6Anomaly
		     );


#***
# Non-constant Static Variables
#***
my $undertaker_path = 0;
my $anomaly_test = 0;
my $alive_test = 1;
my $dead_test = 1;


#***
# Main Function
#***

GetOptions('anomaly'=>\$anomaly_test,
	   );

# Stats
my $sucesses = 0;
my $failures = 0;

# Perform Alive Tests if neccessary
if( $alive_test ){
    # Setup Alive Tests
    &SetupAlive($PID_FILE);
    
    # Run Alive Tests
    my @tests = @ALIVE_TESTS;
    foreach( @tests ){
	
	if( &$_($PID_FILE) ){
	    $sucesses++;
	} else{
	    $failures++;
	}
    }
    

    # Tear Down Alive
    &TearDownAlive($PID_FILE);
}

# Perform Dead Tests if neccessary
if( $dead_test ){
    # Setup Dead
    &SetupDead($PID_FILE);

    # Run Dead Tests
    my @tests = @DEAD_TESTS;
    foreach( @tests ){
	if( &$_($PID_FILE) ){
	    $sucesses++;
	} else{
	    $failures++;
	}
    }

    # Tear Down Dead Test
    &TearDownDead($PID_FILE);
}

# Perform Anomaly Tests if neccessary
# User must have 'sudo date' access
if( $anomaly_test ){

    &SetupAnomaly($PID_FILE);

    # Run Anomaly Tests
    my @tests = @ANOMALY_TESTS;
    foreach( @tests ){
	if( &$_($PID_FILE) ){
	    $sucesses++;
	} else{
	    $failures++;
	}
    }
    &TearDownAnomaly($PID_FILE);	
}

# Print Stats
print "Sucesses[$sucesses]\n";
print "Failures[$failures]\n";

# 0 for all clear
# 1 for failures
exit ($failures != 0);

######################################################################
# Alive Tests: Tests where the monitored process is definitly alive.
######################################################################
sub SetupAlive(){
    my( $pid_file ) = @_; #Name the parameters
    
    # Find uniq_pid_midwife
    my $midwife_path = File::Spec->catpath(0, 
					   $FindBin::Bin, 
					   $UNIQ_PID_MIDWIFE_CMD);

    # Execute it
    my @args = ($NO_BLOCK_OPT,
		$FILE_OPT, $pid_file     ,
		$PRECISION_OPT, $PRECISION,
		$SLEEP_CMD, $LONG_SLEEP);
    !system $midwife_path, @args
	or die "FAILED: Could not execute the midwife\n";
}

sub TearDownAlive(){
    my( $pid_file ) = @_; #Name the parameters
    
    my $pid = &GetChildPid($pid_file);
    
    # Kill the process
    kill 9, $pid
	or die "FAILED: Could not kill $pid\n";

    # Unlink the pid file
    unlink $pid_file
	or die "FAILED: Could not unlink $pid_file:$!";
}


# Confirmed
# (Alive)
sub Test1Alive(){
    my( $pid_file ) = @_; #Name the parameters
    
    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test1alive';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 1 ){
	print STDERR "TEST1(Alive): Undertaker returned $exit_value instead of 1\n";
    }

    # clean up
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 1);
}

# Midwife Failure After Confirm Time (no control time)
# (Uncertain)
sub Test2Alive(){
    my( $pid_file ) = @_; #Name the parameters

    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test2alive';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 1);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 2 ){
	print STDERR "TEST2Alive: Undertaker returned $exit_value instead of 2\n";
    }

    # clean up
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 2);
}

# Midwife Failure After BDay Control Time
# (Uncertain)
sub Test3Alive(){
    my( $pid_file ) = @_; #Name the parameters

    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test3alive';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 2);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 2 ){
	print STDERR "TEST3Alive: Undertaker returned $exit_value instead of 2\n";
    }

    # clean up
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 2);
}

# Midwife Failure After BDay
# (Uncertain)
sub Test4Alive(){
    my( $pid_file ) = @_; #Name the parameters

    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test4alive';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 3);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 2 ){
	print STDERR "TEST4Alive: Undertaker returned $exit_value instead of 4\n";
    }

    # clean up
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 2);
}

# Midwife Failure After Child Pid
# (Uncertain)
sub Test5Alive(){
    my( $pid_file ) = @_; #Name the parameters
	
    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test5Alive';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 6);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);
    
    # failure
    if( $exit_value != 2 ){
	print STDERR "TEST5Alive: Undertaker returned $exit_value instead of 2\n";
    }

    # clean up
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 2);
}

# Midwife Failure After Midwife Pid
# (Failure)
sub Test6Alive(){
    my( $pid_file ) = @_; #Name the parameters
	
    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test6alive';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 7);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);
    
    # failure
    if( $exit_value != 255 ){
	print STDERR "TEST6Alive: Undertaker returned $exit_value instead of 255\n";
    }

    # clean up
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 255);
}

# Midwife Failure Empty file
# (Failure)
sub Test7Alive(){
    my( $pid_file ) = @_; #Name the parameters

    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test7alive';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 8);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 255 ){
	print STDERR "TEST7Alive: Undertaker returned $exit_value instead of 255\n";
    }
    
    # clean up
    unlink $test_pid_file
	or die "FAILED: $!";
    
    return( $exit_value == 255);
}

######################################################################
# Dead Tests: Tests where the monitored process is definitly dead
######################################################################
sub SetupDead(){
    my( $pid_file ) = @_; #Name the parameters
    
    # Find uniq_pid_midwife
    my $midwife_path = File::Spec->catpath(0, 
					   $FindBin::Bin, 
					   $UNIQ_PID_MIDWIFE_CMD);

    # Execute it (block to ensure the process is dead
    my @args = ($FILE_OPT, $pid_file,
		$PRECISION_OPT, $PRECISION,
		$BLOCK_OPT,
		$SLEEP_CMD, $SHORT_SLEEP );
    !system $midwife_path, @args
	or die "FAILED: Could not execute the midwife\n";
}

sub TearDownDead(){
    my( $pid_file ) = @_; #Name the parameters

    # Unlink the pid file
    unlink $pid_file
	or die "FAILED: Could not unlink $pid_file:$!";
}

# Confirmed
# (Dead)
sub Test1Dead(){
    my( $pid_file ) = @_; #Name the parameters
    
    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'test1dead';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 0 ){
	print STDERR "TEST1Dead: Undertaker returned $exit_value instead of 0\n";
    }

    # clean up
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 0);
}

# Confirmed, with Reuse
# new parent reserved
# (Dead)
sub Test2Dead(){
    my( $pid_file ) = @_; #Name the parameters

    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'test2dead';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";


    # Create a new process to be the reuse
    my( $new_pid, $new_bday, $new_ctl) = &CreateNewOrphan($pid_file);
    
    # Change pid file to appear as though the pid is being reused
    &ResetPid($test_pid_file, $new_pid);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);
 
    # failure
    if( $exit_value != 0 ){
	print STDERR "TEST2Dead: Undertaker returned $exit_value instead of 0\n";
    }

    # clean up
    kill 9, $new_pid
	or die "FAILED: $!";
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 0);
}

# Midwife Failure After BDay Control Time
# Reuse
# new parent reserved [init]
# bday not in range
# (Dead)
sub Test3Dead(){
    my( $pid_file ) = @_; #Name the parameters

    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test3dead';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 2);

    # Create an orphan
    my ($new_pid, $new_bday, $new_ctl) = &CreateNewOrphan($pid_file);
    
    # Change the pid file to appear as though the pid is being reused
    &ResetPid($test_pid_file, $new_pid);
    
    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 0 ){
	print STDERR "TEST3Dead: Undertaker returned $exit_value instead of 0\n";
    }

    # clean up
    kill 9, $new_pid
	or die "FAILED: $!";
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 0);
}

# Midwife Failure After BDay Control Time
# Reuse
# new parent non-reserved
# bday not in range
# (Dead)
sub Test4Dead(){
    my( $pid_file ) = @_; #Name the parameters
    
    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test4dead';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 2);

    # Create a new child
    my $new_pid = &CreateNewChild();
    
    # Change the pid file to appear as though the pid is being reused
    &ResetPid($test_pid_file, $new_pid);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 0 ){
	print STDERR "TEST4Dead: Undertaker returned $exit_value instead of 0\n";
    }

    # clean up
    kill 9, $new_pid
	or die "FAILED: $!";
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 0);
}

# Midwife Failure After BDay Control Time
# Reuse
# new parent reserved
# bday in range
# (Uncertain)
sub Test5Dead(){
    my( $pid_file ) = @_; #Name the parameters
    
    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test5dead';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 2);

    # Create a new orphan
    my( $new_pid, $new_bday, $new_ctl ) = &CreateNewOrphan($pid_file);

    # Move the birtday into "old" control time
    my $old_ctl = &GetControl($test_pid_file);
    my $shift_amt = $old_ctl - $new_ctl;
    $new_bday += $shift_amt;
    
    # Change the pid file to appear as though the pid is being reused
    &ResetPid($test_pid_file, $new_pid);
    &ResetBday($test_pid_file, $new_bday + 1);
    
    

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 2 ){
	print STDERR "TEST5Dead: Undertaker returned $exit_value instead of 2\n";
    }

    # clean up
    kill 9, $new_pid
	or die "FAILED: $!";
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 2);
}

# Midwife Failure After BDay
# Reuse
# new parent non-reserved
# bday not in range
# (Dead)
sub Test6Dead(){
    my( $pid_file ) = @_; #Name the parameters
    
    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test6dead';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 3);

    # Create a new child
    my $new_pid = &CreateNewChild();
    
    # Change the pid file to appear as though the pid is being reused
    &ResetPid($test_pid_file, $new_pid);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 0 ){
	print STDERR "TEST6Dead: Undertaker returned $exit_value instead of 0\n";
    }

    # clean up
    kill 9, $new_pid
	or die "FAILED: $!";
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 0);
}

# Midwife Failure After BDay
# Reuse
# new parent reserved
# bday not in range
# (Uncertain)
sub Test7Dead(){
    my( $pid_file ) = @_; #Name the parameters

    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test7dead';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 3);

    # Create a new orphan
    my( $new_pid, $new_bday, $new_ctl ) = &CreateNewOrphan($pid_file);
    
    # Change the pid file to appear as though the pid is being reused
    &ResetPid($test_pid_file, $new_pid);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 2 ){
	print STDERR "TEST7Dead: Undertaker returned $exit_value instead of 2\n";
    }

    # clean up
    kill 9, $new_pid
	or die "FAILED: $!";
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 2);
}

# Midwife Failure After Child Pid
# Reuse
# new parent reserved
# (Uncertain)
sub Test8Dead(){
    my( $pid_file ) = @_; #Name the parameters

    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test8dead';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 6);

    # Create a new orphan
    my( $new_pid, $new_bday, $new_ctl ) = &CreateNewOrphan($pid_file);
    
    # Change the pid file to appear as though the pid is being reused
    &ResetPid($test_pid_file, $new_pid);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 2 ){
	print STDERR "TEST8Dead: Undertaker returned $exit_value instead of 2\n";
    }

    # clean up
    kill 9, $new_pid
	or die "FAILED: $!";
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 2);
}

# Midwife Failure After Child Pid
# Reuse
# new parent non-reserved
# (Dead)
sub Test9Dead(){
    my( $pid_file ) = @_; #Name the parameters

    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test9dead';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 6);

    # Create a new orphan
    my $new_pid = &CreateNewChild();
    
    # Change the pid file to appear as though the pid is being reused
    &ResetPid($test_pid_file, $new_pid);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 0 ){
	print STDERR "TEST9Dead: Undertaker returned $exit_value instead of 0\n";
    }

    # clean up
    kill 9, $new_pid
	or die "FAILED: $!";
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 0);
}

# Midwife Failure After Midwife Pid
# (Failure)
sub Test10Dead(){
    my( $pid_file ) = @_; #Name the parameters

    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test10dead';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 7);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 255 ){
	print STDERR "TEST10Dead: Undertaker returned $exit_value instead of 255\n";
    }

    # clean up
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 255);
}

# Midwife Failure Empty file
# (Failure)
sub Test11Dead(){
    my( $pid_file ) = @_; #Name the parameters

    # Create a copy of the pid file to muck with
    my $test_pid_file = $pid_file.'.test11dead';
    copy($pid_file, $test_pid_file)
	or die "FAILED: $!";

    # Damage the pid file to make it appear as though midwife failure occured
    &RemoveLines($test_pid_file, 8);

    # Run the undertaker
    my $exit_value = &CallUndertaker($test_pid_file);

    # failure
    if( $exit_value != 255 ){
	print STDERR "TEST11Dead: Undertaker returned $exit_value instead of 255\n";
    }

    # clean up
    unlink $test_pid_file
	or die "FAILED: $!";

    return( $exit_value == 255);
}
######################################################################
# Anomaly Tests: Tests where time has an anomaly.
######################################################################
sub SetupAnomaly(){
    #Nothing Yet
}

sub TearDownAnomaly(){
   #Nothing Yet
}

# Forward
# After Confirmation
sub Test1Anomaly(){
    my( $pid_file) = @_; #name the parameters

    my $sucesses = 0;
    my $failures = 0;
    
    # Setup For Tests
    &SetupAlive($pid_file);
    
    # Move time forward 30min
    &ShiftTime(30);
    
    # Run Alive Tests
    print STDERR "STARTING TIME ANOMALY PHASE\n";
    my @tests = @ALIVE_TESTS;
    foreach( @tests ){
	
	if( &$_($pid_file) ){
	    $sucesses++;
	} else {
	    $failures++;
	}
    }
    # Tear down Alive
    &TearDownAlive($pid_file);
    &ShiftTime(-30);
	
    # Setup Dead Tests
    &SetupDead($pid_file);
    &ShiftTime(30);
    
    # Run Dead Tests
    @tests = @DEAD_TESTS;
    foreach( @tests ){
	if( &$_($pid_file) ){
	    $sucesses++;
	} else{
	    $failures++;
	}
    }
    
    print STDERR "FINISHED TIME ANOMALY PHASE\n";

    # Tear down Dead
    &TearDownDead($pid_file);
    &ShiftTime(-30);
     
    # failure
    if( $failures > 0 ){
	print STDERR "TEST1Anomaly: sucesses[$sucesses] failure[$failures]";
    }

    return( $failures == 0 );
}

# Backward
# After Confirmation
sub Test2Anomaly(){
    my( $pid_file) = @_; #name the parameters

    my $sucesses = 0;
    my $failures = 0;
    
    # Setup For Tests
    &SetupAlive($pid_file);
    
    # Move time forward 30min
    &ShiftTime(-30);
    
    # Run Alive Tests
    print STDERR "STARTING TIME ANOMALY PHASE\n";
    my @tests = @ALIVE_TESTS;
    foreach ( @tests ){
	
	if( &$_($pid_file) ){
	    $sucesses++;
	} else {
	    $failures++;
	}
    }
    # Tear down Alive
    &TearDownAlive($pid_file);
    &ShiftTime(30);
	
    # Setup Dead Tests
    &SetupDead($pid_file);
    &ShiftTime(-30);
    
    # Run Dead Tests
    @tests = @DEAD_TESTS;
    foreach( @tests ){
	if( &$_($pid_file) ){
	    $sucesses++;
	} else{
	    $failures++;
	}
    }
    
    print STDERR "FINISHED TIME ANOMALY PHASE\n";

    # Tear down Dead
    &TearDownDead($pid_file);
    &ShiftTime(30);

    # failure
    if( $failures > 0 ){
	print STDERR "TEST2Anomaly: sucesses[$sucesses] failure[$failures]";
    }

    return( $failures == 0 );
}

# Forward
# Before Confirmation
sub Test3Anomaly(){
    my( $pid_file) = @_; #name the parameters

    my $sucesses = 0;
    my $failures = 0;
    
    # Setup For Tests
    &SetupAlive($pid_file);
    &ShiftTime(30);

    # Modify the pid file to make it appear the shift occurred
    # while before the confirmation
    my($conf, $ctl) = &GenerateConfirmation($pid_file);
    &RemoveLines($pid_file, 2);
    &AddLines($pid_file, 
	      ("$CONFIRM_HEADER = $conf", "$CONTROL_HEADER = $ctl")
	      );
    
    # Run Alive Tests
    print STDERR "STARTING TIME ANOMALY PHASE\n";
    my @tests = @ALIVE_TESTS;
    foreach( @tests ){
	
	if( &$_($pid_file) ){
	    $sucesses++;
	} else {
	    $failures++;
	}
    }
    # Tear down Alive
    &TearDownAlive($pid_file);
    &ShiftTime(-30);
	
    # Setup Dead Tests
    &SetupDead($pid_file);
    &ShiftTime(30);
    
    # Modify the pid file to make it appear the shift occurred
    # while before the confirmation
    ($conf, $ctl) = &GenerateConfirmation($pid_file);
    &RemoveLines($pid_file, 2);
    &AddLines($pid_file, 
	      ("$CONFIRM_HEADER = $conf", "$CONTROL_HEADER = $ctl")
	      );
    
    # Run Dead Tests
    @tests = @DEAD_TESTS;
    foreach( @tests ){
	if( &$_($pid_file) ){
	    $sucesses++;
	} else{
	    $failures++;
	}
    }
    
    print STDERR "FINISHED TIME ANOMALY PHASE\n";

    # Tear down Dead
    &TearDownDead($pid_file);
    &ShiftTime(-30);

    # failure
    if( $failures > 0 ){
	print STDERR "TEST3Anomaly: sucesses[$sucesses] failure[$failures]";
    }

    return( $failures == 0 );
}




# Backward
# After Confirmation
sub Test4Anomaly(){
    my( $pid_file) = @_; #name the parameters

    my $sucesses = 0;
    my $failures = 0;
    
    # Setup For Tests
    &SetupAlive($pid_file);
    &ShiftTime(-30);

    # Modify the pid file to make it appear the shift occurred
    # while before the confirmation
    my($conf, $ctl) = &GenerateConfirmation($pid_file);
    &RemoveLines($pid_file, 2);
    &AddLines($pid_file, 
	      ("$CONFIRM_HEADER = $conf", "$CONTROL_HEADER = $ctl")
	      );
    
    # Run Alive Tests
    print STDERR "STARTING TIME ANOMALY PHASE\n";
    my @tests = @ALIVE_TESTS;
    foreach( @tests ){
	
	if( &$_($pid_file) ){
	    $sucesses++;
	} else {
	    $failures++;
	}
    }
    # Tear down Alive
    &TearDownAlive($pid_file);
    &ShiftTime(30);
	
    # Setup Dead Tests
    &SetupDead($pid_file);
    &ShiftTime(-30);
    
    # Modify the pid file to make it appear the shift occurred
    # while before the confirmation
    ($conf, $ctl) = &GenerateConfirmation($pid_file);
    &RemoveLines($pid_file, 2);
    &AddLines($pid_file, 
	      ("$CONFIRM_HEADER = $conf", "$CONTROL_HEADER = $ctl")
	      );
    
    # Run Dead Tests
    @tests = @DEAD_TESTS;
    foreach( @tests ){
	if( &$_($pid_file) ){
	    $sucesses++;
	} else{
	    $failures++;
	}
    }
    
    print STDERR "FINISHED TIME ANOMALY PHASE\n";

    # Tear down Dead
    &TearDownDead($pid_file);
    &ShiftTime(30);

    # failure
    if( $failures > 0 ){
	print STDERR "TEST4Anomaly: sucesses[$sucesses] failure[$failures]";
    }

    return( $failures == 0 );
}

# Forward
# Before and After
sub Test5Anomaly(){
    my( $pid_file) = @_; #name the parameters

    my $sucesses = 0;
    my $failures = 0;
    
    # Setup For Tests
    &SetupAlive($pid_file);

    # Modify the pid file to make it appear the shift occurred
    # while before the confirmation
    &ShiftTime(30);
    my($conf, $ctl) = &GenerateConfirmation($pid_file);
    &RemoveLines($pid_file, 2);
    &AddLines($pid_file, 
	      ("$CONFIRM_HEADER = $conf", "$CONTROL_HEADER = $ctl")
	      );

    # Shift Again
    &ShiftTime(60);

    # Run Alive Tests
    print STDERR "STARTING TIME ANOMALY PHASE\n";
    my @tests = @ALIVE_TESTS;
    foreach( @tests ){
	
	if( &$_($pid_file) ){
	    $sucesses++;
	} else {
	    $failures++;
	}
    }
    # Tear down Alive
    &TearDownAlive($pid_file);
    &ShiftTime(-90);
	
    # Setup Dead Tests
    &SetupDead($pid_file);
    
    # Modify the pid file to make it appear the shift occurred
    # while before the confirmation
    &ShiftTime(30);
    ($conf, $ctl) = &GenerateConfirmation($pid_file);
    &RemoveLines($pid_file, 2);
    &AddLines($pid_file, 
	      ("$CONFIRM_HEADER = $conf", "$CONTROL_HEADER = $ctl")
	      );

    # Shift Again
    &ShiftTime(60);
    
    # Run Dead Tests
    @tests = @DEAD_TESTS;
    foreach( @tests ){
	if( &$_($pid_file) ){
	    $sucesses++;
	} else{
	    $failures++;
	}
    }
    
    print STDERR "FINISHED TIME ANOMALY PHASE\n";

    # Tear down Dead
    &TearDownDead($pid_file);
    &ShiftTime(-90);

    # failure
    if( $failures > 0 ){
	print STDERR "TEST5Anomaly: sucesses[$sucesses] failure[$failures]";
    }

    return( $failures == 0 );
}

# Backward
# Before and After
sub Test6Anomaly(){
    my( $pid_file) = @_; #name the parameters

    my $sucesses = 0;
    my $failures = 0;
    
    # Setup For Tests
    &SetupAlive($pid_file);

    # Modify the pid file to make it appear the shift occurred
    # while before the confirmation
    &ShiftTime(-30);
    my($conf, $ctl) = &GenerateConfirmation($pid_file);
    &RemoveLines($pid_file, 2);
    &AddLines($pid_file, 
	      ("$CONFIRM_HEADER = $conf", "$CONTROL_HEADER = $ctl")
	      );

    # Shift Again
    &ShiftTime(-60);

    # Run Alive Tests
    print STDERR "STARTING TIME ANOMALY PHASE\n";
    my @tests = @ALIVE_TESTS;
    foreach( @tests ){
	
	if( &$_($pid_file) ){
	    $sucesses++;
	} else {
	    $failures++;
	}
    }
    # Tear down Alive
    &TearDownAlive($pid_file);
    &ShiftTime(90);
	
    # Setup Dead Tests
    &SetupDead($pid_file);
    
    # Modify the pid file to make it appear the shift occurred
    # while before the confirmation
    &ShiftTime(-30);
    ($conf, $ctl) = &GenerateConfirmation($pid_file);
    &RemoveLines($pid_file, 2);
    &AddLines($pid_file, 
	      ("$CONFIRM_HEADER = $conf", "$CONTROL_HEADER = $ctl")
	      );

    # Shift Again
    &ShiftTime(-60);
    
    # Run Dead Tests
    @tests = @DEAD_TESTS;
    foreach( @tests ){
	if( &$_($pid_file) ){
	    $sucesses++;
	} else{
	    $failures++;
	}
    }
    
    print STDERR "FINISHED TIME ANOMALY PHASE\n";

    # Tear down Dead
    &TearDownDead($pid_file);
    &ShiftTime(90);

    # failure
    if( $failures > 0 ){
	print STDERR "TEST6Anomaly: sucesses[$sucesses] failure[$failures]";
    }

    return( $failures == 0 );
}

######################################################################
# Helper methods
######################################################################
sub CallUndertaker(){
    my( $pid_file ) = @_;

    # Find uniq_pid_undertaker, if necessary
    if( !$undertaker_path ){
	$undertaker_path = File::Spec->catpath(0,
					       $FindBin::Bin,
					       $UNIQ_PID_UNDERTAKER_CMD);
    }
    
    # Construct the arguments
    my @args = ($PRECISION_OPT, $PRECISION,
		$FILE_OPT, $pid_file);
    
    # Execute the undertaker
    my $returnHashRef = ExecuteAndCapture($undertaker_path, @args);
    
    # Return the exit value
    return( $returnHashRef->{EXIT_VALUE} );
}

sub CreateNewChild(){
    my $child_pid = fork();
    if (!defined $child_pid ){
	die "cannot fork: $!";
    } elsif( $child_pid == 0)  {
	# Child
	exec "$SLEEP_CMD $LONG_SLEEP\n"
	    or die "FAILED: $!";
    }

    # Parent
    return( $child_pid );
}



sub CreateNewOrphan(){
    my( $pid_file ) = @_; #Name the parameters
    
    # Find uniq_pid_midwife
    my $midwife_path = File::Spec->catpath(0, 
					   $FindBin::Bin, 
					   $UNIQ_PID_MIDWIFE_CMD);
    # Execute it
    my $temp_pid_file = $pid_file.'.orphan';
    my @args = ($NO_BLOCK_OPT,
		$FILE_OPT, $temp_pid_file,
		$PRECISION_OPT, $PRECISION,
		$SLEEP_CMD, $LONG_SLEEP);
    !system $midwife_path, @args
	or die "FAILED: Could not execute the midwife";

    # get the orphaned child's pid
    my $orphan_pid = &GetChildPid($temp_pid_file);
    my $orphan_bday = &GetBday($temp_pid_file);
    my $orphan_ctl = &GetControl($temp_pid_file);
    
    # unlink the tmp pid file
    unlink $temp_pid_file
	or die "FAILED: $!";

    return( ($orphan_pid,$orphan_bday, $orphan_ctl) );
}

sub GetChildPid(){
    my ($pid_file) = @_; 

    my $pid = &GetValue($pid_file, $PID_HEADER);

    return $pid;
}

sub GetBday(){
    my($pid_file) = @_;
    
    my $bday = &GetValue($pid_file, $BDAY_HEADER);

    return $bday;
}

sub GetConfirmation(){
    my($pid_file) = @_;
    
    my $conf_time = &GetValue($pid_file, $CONFIRM_HEADER);
    
    return $conf_time;
}

sub GetControl(){
    my($pid_file) = @_;
    
    my $ctl_time = &GetValue($pid_file, $CONTROL_HEADER);

    return $ctl_time;
}

sub GetValue(){
    my( $pid_file, $header ) = @_;

     # Read in the pid file
    open PID_FD, "< $pid_file"
	or die "FAILED: $!";
    my @lines = ();
    while( <PID_FD> ){
	chomp $_;
	push @lines, $_;
    }
    close PID_FD;


    # Scan the data until we get the value
    my $value = -1;
    my $i = 0;
    while( $i < @lines && $value == -1 ){
	if( $lines[$i] =~ m|^$header = (\d+)| ){
	    $value = $1;
	}
	
	$i++;
    }

    # Check for failure
    die "FAILED: Couldn't get the $header out of the $pid_file" if( $value == -1);

    return $value;
}

sub ResetPid(){
    my( $pid_file, $newpid ) = @_;
    
    &ResetValue($pid_file, $newpid, $PID_HEADER);

}

sub ResetBday(){
    my( $pid_file, $new_bday ) = @_; 
    
    &ResetValue($pid_file, $new_bday, $BDAY_HEADER)
}

sub ResetConfirmation(){
    my($pid_file, $new_conf) = @_;
    
    &ResetValue($pid_file, $new_conf, $CONFIRM_HEADER);
}

sub ResetValue(){
    my( $pid_file, $new_value, $header ) = @_;
    
    open PID_FD, "< $pid_file"
	or die "FAILED: $!";
    open NEW_PID_FD, "> $pid_file.tmp"
	or die "FAILED: $!";

    while( <PID_FD> ){
	if( m|^$header = \d+\n| ){
	    print NEW_PID_FD "$header = $new_value\n";
	}
	else{
	    print NEW_PID_FD $_;
	}
    }

    close PID_FD;
    close NEW_PID_FD;

    rename "$pid_file.tmp", $pid_file
	or die "FAILED: $!";
}

sub RemoveLines(){
    my( $pid_file, $amt_to_remove) = @_;

    # Read in the file
    open PID_FILE, "< $pid_file"
	or die "FAILED: $!";
    
    my @lines = ();
    while( <PID_FILE> ){
	push @lines, $_;
    }

    close PID_FILE;

    # Write it back out minus the number of lines to remove
    open PID_FILE, "> $pid_file"
	or die "FAILED: $!";

    my $amt_to_write_back = int(@lines) - $amt_to_remove;
    for(my $i = 0; $i < $amt_to_write_back; $i++){
	print PID_FILE $lines[$i];
    }

    close PID_FILE;
}

sub AddLines(){
    my( $pid_file, @lines) = @_;
    
    open PID_FILE, ">> $pid_file"
	or die "FAILED: $!";
    
    foreach(@lines){
	chomp($_);
	print PID_FILE "$_\n";
    }

    close PID_FILE;
}

sub ShiftTime(){
    my( $shift_minutes ) = @_;  #name the parameters
    
     # Construct time coordinates
    my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
	localtime(time);
    
    # move time forward .5hrs 
    my ($new_hr, $new_min, $new_sec) = &ShiftTimeUnits($hour, $min, $sec, $shift_minutes);
        
    # Enter the TIIIIIMMMMEEE WAAAAARRRRP
    SetTime($new_hr, $new_min, $new_sec);
    
    return( $hour, $min, $sec );
}

sub SetTime(){
    my($hr, $min, $sec) = @_;  #Name the parameters

    # construct the arguments
    my $time_str = sprintf("%02d:%02d:%02d", $hr, $min, $sec);
    my @date_args = ('date', '-s', $time_str);

    #!system 'sudo', @date_args
    #or die "FAILED: Could not set time\n";
    my $returnHashRef = ExecuteAndCapture('sudo', @date_args);
    die "FAILED: Could not set time\n" if( $returnHashRef->{EXIT_VALUE} );
}

sub ShiftTimeUnits(){
    my($hr, $min, $sec, $shift_minutes) = @_;  #Name the parameters

    my $new_hr = $hr;
    my $new_min = $min;
    my $new_sec = $sec;
    
    $new_min += $shift_minutes;
    
    # Shift backward (overflow)
    if( $new_min < 0 ){
	my $shift_hrs = 0;
	while( $new_min < 0 ){
	    $new_min += 60;
	    $shift_hrs++;
	}

	$shift_hrs = $shift_hrs % 24;
	$shift_hrs = 24 - $shift_hrs;
	$new_hr += $shift_hrs;
	$new_hr = $new_hr % 24;
    }
    
    # Shift forward (overflow)
    if( $new_min >= 60 ){
	$new_hr += $new_min / 60;
	$new_hr = $new_hr % 24;
	$new_min = $new_min % 60;;
    }

    return( $new_hr, $new_min, $new_sec );
}


sub GenerateConfirmation(){
    my( $pid_file) = @_;

    my $temp_pid_file = $pid_file.'.gencon';

    &SetupDead($temp_pid_file);
    
    my $conf = &GetConfirmation($temp_pid_file);
    my $ctl = &GetControl($temp_pid_file);

    #clean up
    unlink $temp_pid_file;
    
    return( $conf, $ctl );
}
