/// <summary>
/// Bucketizes tests into execution groups, and creates separate buckets for tests which do not satisfy the minimum execution grouping level.
/// </summary>
public static IEnumerable <List <TestRecord> > Bucketize(ICollection <TestRecord> tests, ExecutionGroupingLevel minGroupingLevel, Func <TestRecord, object> hashMethod)
{
List <TestRecord> groupableTests = new List <TestRecord>(tests.Where(test => Exceeds(test.TestInfo.ExecutionGroupingLevel, minGroupingLevel)));
List <TestRecord> isolatedTests = new List <TestRecord>(tests.Where(test => !Exceeds(test.TestInfo.ExecutionGroupingLevel, minGroupingLevel)));
Dictionary <object, List <TestRecord> > bucketizedTests = ExecutionGrouper.HashTests(groupableTests, hashMethod);
IEnumerable <List <TestRecord> > testGroups = Unify(bucketizedTests, isolatedTests);
return(testGroups);
}
/// <summary>
/// Bucketize and run through groups of matching support file needs.
/// </summary>
private static void ExecuteTestSupportFileGroups(ExecutionSettings settings, List <TestRecord> stateManagementGroup, ExecutionGroupRecord stateGroupRecord, int stateGroupIndex, ExecutionComponents components)
{
int supportFileGroupIndex = 0;
//Bucketize
// NOTE: Hash method for SupportFiles is order sensitive.
IEnumerable <List <TestRecord> > testGroups = ExecutionGrouper.Bucketize(
stateManagementGroup,
ExecutionGroupingLevel.SharedSupportFiles,
x => HashSupportFileGroup(x));
foreach (List <TestRecord> supportFileGroup in testGroups)
{
Stack <ICleanableCommand> filecleanupCommands = new Stack <ICleanableCommand>();
ExecutionGroupRecord supportFileGroupRecord = ExecutionGroupRecord.Begin(ExecutionGroupType.Files, stateGroupRecord.Area);
stateGroupRecord.ExecutionGroupRecords.Add(supportFileGroupRecord);
DirectoryInfo executionDirectory = settings.DetermineTestExecutionDirectory(settings.DetermineGroupPath(stateGroupIndex, supportFileGroupIndex));
DirectoryInfo logDirectory = settings.DetermineTestLogDirectory(settings.DetermineGroupPath(stateGroupIndex, supportFileGroupIndex));
PrepLogDirectory(logDirectory);//HACK: Ideally logging can guarantee this in final configuration.
if (GetCachedExecutionResult(settings, supportFileGroup, logDirectory, components))
{
ExecutionEventLog.RecordStatus("Successfully retrieved previously stored execution result. ");
}
else
{
ExecutionEventLog.RecordStatus("Applying Support Files and Executing.");
ExecutionGroupLogCommand command = ExecutionGroupLogCommand.Apply("SupportFiles", logDirectory, components.LoggingMediator);
filecleanupCommands.Push(command);
filecleanupCommands.Push(DesktopSnapshotCommand.Apply(logDirectory));
// Create temporary directory, but if we are using a fixed test execution directory, don't delete it if it was pre-existing.
filecleanupCommands.Push(TemporaryDirectoryCommand.Apply(executionDirectory, settings.FixedTestExecutionDirectory != null));
filecleanupCommands.Push(SupportFileCommand.Apply(supportFileGroup, settings.TestBinariesDirectory, executionDirectory));
filecleanupCommands.Push(BackupRecordsCommand.Apply(supportFileGroup, logDirectory));
filecleanupCommands.Push(ProcessLogsCommand.Apply(supportFileGroup, components.LoggingMediator));
try
{
ExecuteUniformTestGroup(settings, supportFileGroup, supportFileGroupRecord, stateGroupIndex, supportFileGroupIndex, components);
}
catch (Exception e)
{
ExecutionEventLog.RecordException(e);
}
finally
{
Cleanup(filecleanupCommands);
}
}
supportFileGroupRecord.End();
supportFileGroupIndex++;
}
}
/// <summary>
/// Bucketize collection of tests by matching State management needs, and run through each group.
/// Notion of Execution group + a tracking counter variable seem to be critical elements.
/// This (and supportfiles one) can likely get generalized+factored well to the ExecutionGroup class with some thought.
/// </summary>
private static void ExecuteTestStateGroups(ExecutionSettings settings, ExecutionComponents executionComponents)
{
ExecutionEventLog.RecordStatus("Running Test Sequence.");
List <TestRecord> enabledTests = new List <TestRecord>(settings.Tests.TestCollection.Where(test => (test.ExecutionEnabled == true)));
int stateGroupIndex = 0;
// NOTE: Hash method for Deployments is order sensitive.
IEnumerable <List <TestRecord> > testGroups = ExecutionGrouper.Bucketize(enabledTests,
ExecutionGroupingLevel.SharedStateManagement,
x => x.TestInfo.Area + x.TestInfo.Deployments.ToCommaSeparatedList());
foreach (List <TestRecord> stateManagementGroup in testGroups)
{
ExecutionEventLog.RecordStatus("Running State Group # " + stateGroupIndex + " of " + testGroups.Count());
Stack <ICleanableCommand> stateCleanupCommands = new Stack <ICleanableCommand>();
ExecutionGroupRecord stateGroupRecord = ExecutionGroupRecord.Begin(ExecutionGroupType.State, stateManagementGroup[0].TestInfo.Area);
try
{
settings.Tests.ExecutionGroupRecords.Add(stateGroupRecord);
DirectoryInfo stateLogPath = settings.DetermineTestLogDirectory(settings.DetermineGroupPath(stateGroupIndex));
ExecutionGroupLogCommand command = ExecutionGroupLogCommand.Apply("StateManagement", stateLogPath, executionComponents.LoggingMediator);
stateCleanupCommands.Push(command);
stateCleanupCommands.Push(ExecutionStateCommand.Apply(stateManagementGroup.First(), settings.TestBinariesDirectory));
ExecuteTestSupportFileGroups(settings, stateManagementGroup, stateGroupRecord, stateGroupIndex, executionComponents);
}
catch (Exception e)
{
ExecutionEventLog.RecordException(e);
}
finally
{
Cleanup(stateCleanupCommands);
stateGroupRecord.End();
stateGroupIndex++;
}
}
}
private static void ExecuteUniformTestGroup(ExecutionSettings settings, List <TestRecord> uniformTestGroup, ExecutionGroupRecord fileGroupRecord, int stateGroupIndex, int supportFileGroupIndex, ExecutionComponents components)
{
string groupPath = settings.DetermineGroupPath(stateGroupIndex, supportFileGroupIndex);
DirectoryInfo executionDirectory = settings.DetermineTestExecutionDirectory(groupPath);
string executionLabel = "(" + stateGroupIndex + "," + supportFileGroupIndex + ")";
IEnumerable <List <TestRecord> > testGroups;
if (settings.CodeCoverageEnabled) // When using Code Coverage, we do not group test Appdomains together
{
testGroups = ExecutionGrouper.MakeGroupPerTest(uniformTestGroup);
}
else //Normal logic: Bucketize based on support for Shared App domains and driver
{
testGroups = ExecutionGrouper.Bucketize(
uniformTestGroup,
ExecutionGroupingLevel.SharedAppDomains,
x => x.TestInfo.Driver.Executable + x.TestInfo.DriverParameters["SecurityLevel"]);
}
int testCount = 0;
foreach (List <TestRecord> tests in testGroups)
{
ExecutionGroupRecord appDomainRecord = ExecutionGroupRecord.Begin(ExecutionGroupType.AppDomain, fileGroupRecord.Area);
fileGroupRecord.ExecutionGroupRecords.Add(appDomainRecord);
TestInfo first = tests.First().TestInfo;
//Tests which allow grouping & use STI get to be run as a group during normal runs.
//All tests must run separately with Code coverage runs.
bool runAsGroup = !settings.CodeCoverageEnabled &&
first.Driver.Executable.Equals("Sti.exe", StringComparison.OrdinalIgnoreCase) && first.ExecutionGroupingLevel >= ExecutionGroupingLevel.SharedAppDomains;
TimeSpan processDuration;
if (runAsGroup)//STI gets special treatment
{
ExecutionEventLog.RecordStatus(string.Format(CultureInfo.InvariantCulture, "Starting Shared AppDomain Test Process #{0}-{1}", groupPath, testCount));
DirectoryInfo testLogDirectory = settings.DetermineTestLogDirectory(settings.DetermineGroupPath(stateGroupIndex, supportFileGroupIndex));
processDuration = ExecuteSti(settings, tests, components, executionDirectory, testLogDirectory);
ExecutionEventLog.RecordStatus(string.Format(CultureInfo.InvariantCulture, "Finished Shared AppDomain Test Process #{0}-{1}", groupPath, testCount));
}
else
{
ExecutionEventLog.RecordStatus(string.Format(CultureInfo.InvariantCulture, "Starting Test Process #{0}-{4} Runtests.cmd /Area={1} /Name={3} /Subarea={2}", groupPath, first.Area, first.SubArea, first.Name, testCount));
if (tests.Count > 1)
{
throw new InvalidDataException("[Infra bug]Tests should be hashed into lists of individual tests.");
}
DirectoryInfo testLogDirectory = settings.DetermineTestLogDirectory(settings.DetermineGroupPath(stateGroupIndex, supportFileGroupIndex, testCount));
PrepLogDirectory(testLogDirectory);//HACK: Ideally logging can guarantee this in final configuration.
processDuration = ExecuteTest(settings, tests.First(), components, executionDirectory, testLogDirectory);
ExecutionEventLog.RecordStatus(string.Format(CultureInfo.InvariantCulture, "Finished Test Process #{0}-{4} /Area={1} /Name={3} /Subarea={2}", groupPath, first.Area, first.SubArea, first.Name, testCount));
}
Reporting.ReportingUtilities.ApplyProcessCost(tests, processDuration);
appDomainRecord.End();
testCount++;
}
}