public void StartGroup (TestGroup testGroup)
{
BeginInvokeOnMainThread (() => {
this.testsSection = new Section (testGroup.Name);
Root.Add (this.testsSection);
});
}
/// <summary>
/// Reflect, read and prepare the tags for the group metadata. Performs
/// the work if this is the first time the metadata has been seen.
/// </summary>
/// <param name="group">The test group.</param>
private void CreateClassTags(TestGroup group)
{
// 1. Full class name
//_groupTags.AddTag(group.FullName);
// 2. Class name
_groupTags.AddTag(group.Name);
// 3. All [Tag] attributes on the type
foreach (string tag in group.Tags)
{
_groupTags.AddTag(tag);
}
}
/// <summary>
/// Reflect, read and prepare the tags for the group metadata. Performs
/// the work if this is the first time the metadata has been seen.
/// </summary>
/// <param name="group">The test group.</param>
private void CreateClassTags(TestGroup group)
{
// 1. Full class name
//_groupTags.AddTag(group.FullName);
// 2. Class name
_groupTags.AddTag(group.Name);
// 3. All [Tag] attributes on the type
foreach (string tag in group.Tags)
{
_groupTags.AddTag(tag);
}
}
/// <summary>
/// Initializes a new instance of the TagManager class.
/// </summary>
/// <param name="group">The test group eing filter.</param>
/// <param name="methods">The set of methods to run.</param>
public TagManager(TestGroup group)
{
_group = group;
_groupTags = new Tags();
_methodTags = new Dictionary <TestMethod, Tags>();
_tagsToMethods = new Dictionary <string, List <TestMethod> >();
Universe = new List <TestMethod>(group.Methods);
CreateClassTags(_group);
foreach (TestMethod method in group.Methods)
{
CreateMethodTags(method);
}
}
/// <summary>
/// Initializes a new instance of the TagManager class.
/// </summary>
/// <param name="group">The test group eing filter.</param>
/// <param name="methods">The set of methods to run.</param>
public TagManager(TestGroup group)
{
_group = group;
_groupTags = new Tags();
_methodTags = new Dictionary<TestMethod, Tags>();
_tagsToMethods = new Dictionary<string, List<TestMethod>>();
Universe = new List<TestMethod>(group.Methods);
CreateClassTags(_group);
foreach (TestMethod method in group.Methods)
{
CreateMethodTags(method);
}
}
private static TestGroup CreateGroup(Type type)
{
TestGroup group = new TestGroup();
group.Name = type.Name;
group.Tags.Add(type.Name);
group.Tags.Add(type.FullName);
if (type.GetCustomAttributes(true).Where(a => a.GetType() == typeof(FunctionalTestAttribute)).Any())
{
group.Tags.Add("Functional");
}
foreach (TagAttribute attr in type.GetCustomAttributes(true).Where(a => a.GetType() == typeof(TagAttribute)))
{
group.Tags.Add(attr.Tag);
}
return(group);
}
private static TestGroup CreateGroup(Type type)
{
TestGroup group = new TestGroup();
group.Name = type.Name;
group.Tags.Add(type.Name);
group.Tags.Add(type.FullName);
if (type.GetTypeInfo().GetCustomAttributes <FunctionalTestAttribute>().Any())
{
group.Tags.Add("Functional");
}
foreach (TagAttribute attr in type.GetTypeInfo().GetCustomAttributes <TagAttribute>())
{
group.Tags.Add(attr.Tag);
}
return(group);
}
private static void LoadTestAssembly(TestHarness harness, Assembly testAssembly)
{
Dictionary <Type, TestGroup> groups = new Dictionary <Type, TestGroup>();
Dictionary <TestGroup, object> instances = new Dictionary <TestGroup, object>();
foreach (Type type in testAssembly.GetTypes())
{
foreach (MethodInfo method in type.GetMethods())
{
if (method.GetCustomAttributes(true).Where(a => a.GetType() == typeof(TestMethodAttribute) ||
a.GetType() == typeof(AsyncTestMethodAttribute))
.Any())
{
TestGroup group = null;
object instance = null;
if (!groups.TryGetValue(type, out group))
{
group = CreateGroup(type);
harness.Groups.Add(group);
groups[type] = group;
instance = Activator.CreateInstance(type);
TestBase testBase = instance as TestBase;
if (testBase != null)
{
testBase.SetTestHarness(harness);
}
instances[group] = instance;
}
else
{
instances.TryGetValue(group, out instance);
}
TestMethod test = CreateMethod(type, instance, method);
group.Methods.Add(test);
}
}
}
}
void ITestReporter.StartGroup (TestGroup group)
{
this.currentGroup = new GroupDescription (group);
this.groups.Add (this.currentGroup);
}
/// <summary>
/// Run the unit tests.
/// </summary>
public async void RunAsync()
{
// Ensure there's an interface to display the test results.
if (this.Reporter == null)
{
throw new ArgumentNullException("Reporter");
}
// Setup the progress/failure counters
this.Progress = 0;
this.Failures = 0;
// Filter out any test methods based on the current settings
int filteredTestCount = FilterTests();
// Write out the test status message which may be modified by the
// filters
Reporter.Status(this.Settings.TestRunStatusMessage);
// Enumerators that track the current group and method to execute
// (which allows reentrancy into the test loop below to resume at
// the correct location).
IEnumerator <TestGroup> groups = this.Groups.OrderBy(g => g.Name).GetEnumerator();
IEnumerator <TestMethod> methods = null;
// Keep a reference to the current group so we can pass it to the
// Reporter's EndGroup (we don't need one for the test method,
// however, because we close over in the continuation).
TestGroup currentGroup = null;
// Setup the UI
this.Reporter.StartRun(this);
// The primary test loop is a "recursive" closure that will pass
// itself as the continuation to async tests.
//
// Note: It's really important for performance to note that any
// calls to testLoop only occur in the tail position.
DateTime RunStartTime = DateTime.UtcNow;
Func <Task> testLoop = null;
testLoop =
async() =>
{
if (methods != null && methods.MoveNext())
{
// If we were in the middle of a test group and there
// are more methods to execute, let's move to the next
// test method.
// Update the progress
this.Progress++;
Reporter.Progress(this);
// Start the test method
Reporter.StartTest(methods.Current);
if (methods.Current.Excluded)
{
// Ignore excluded tests and immediately recurse.
Reporter.EndTest(methods.Current);
await testLoop();
}
else
{
// Get the start time for individual tests
DateTime testStartTime = DateTime.UtcNow;
// Record the test result, upload the test log as a blob and clear the
// log for next test
Func <Task> recordTestResult = async() =>
{
if (!Settings.ManualMode)
{
// upload test log to sunlight blob container
string relativeFilePath = this.Platform + "/" + Guid.NewGuid().ToString() + ".txt";
string blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
relativeFilePath);
await UploadToBlobContainerAsync(blobStorageSasUrl, LogDump.ToString());
// record the test result
var testResult = new TestResult()
{
FullName = methods.Current.Name,
StartTime = testStartTime,
EndTime = DateTime.UtcNow,
Outcome = methods.Current.Passed ? "Passed" : "Failed",
Source = currentGroup.Name,
ReferenceUrl = relativeFilePath
};
// Add the test result to the test result collection
testRun.AddTestResult(testResult);
LogDump.Clear();
}
};
// Execute the test method
methods.Current.Test.Start(
new ActionContinuation
{
OnSuccess = async() =>
{
// Mark the test as passing, update the
// UI, and continue with the next test.
methods.Current.Passed = true;
methods.Current.Test = null;
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
},
OnError = async(message) =>
{
// Mark the test as failing, update the
// UI, and continue with the next test.
methods.Current.Passed = false;
methods.Current.Test = null;
methods.Current.ErrorInfo = message;
this.Failures++;
System.Diagnostics.Debug.WriteLine(message);
Reporter.Error(message);
LogDump.AppendLine(message);
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
}
});
}
}
else if (groups.MoveNext())
{
// If we've finished a test group and there are more,
// then move to the next one.
// Finish the UI for the last group.
if (currentGroup != null)
{
Reporter.EndGroup(currentGroup);
currentGroup = null;
}
// Setup the UI for this next group
currentGroup = groups.Current;
Reporter.StartGroup(currentGroup);
// Get the methods and immediately recurse which will
// start executing them.
methods = groups.Current.Methods.OrderBy(m => m.Name).GetEnumerator();
await testLoop();
}
else
{
if (!Settings.ManualMode)
{
// upload test suite result to sunlight blob container
string blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
this.Platform + "-detail.json");
string fileContent = JsonConvert.SerializeObject(testRun.TestResults.ToList(), Formatting.Indented);
await UploadToBlobContainerAsync(blobStorageSasUrl, fileContent);
// upload test result summary to blob container
var masterResult = new MasterTestResult()
{
FullName = this.Platform + "-" + Settings.Custom["RuntimeVersion"],
Outcome = Failures > 0 ? "Failed" : "Passed",
TotalCount = testRun.TestCount,
Passed = filteredTestCount - Failures,
Failed = Failures,
Skipped = testRun.TestCount - filteredTestCount,
StartTime = RunStartTime,
EndTime = DateTime.UtcNow,
ReferenceUrl = this.Platform + "-detail.json"
};
// upload test suite result to sunlight blob container
blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
this.Platform + "-master.json");
fileContent = JsonConvert.SerializeObject(masterResult, Formatting.Indented);
await UploadToBlobContainerAsync(blobStorageSasUrl, fileContent);
}
// Otherwise if we've finished the entire test run
// Finish the UI for the last group and update the
// progress after the very last test method.
Reporter.EndGroup(currentGroup);
Reporter.Progress(this);
// Finish the UI for the test run.
Reporter.EndRun(this);
}
};
// Start running the tests
await testLoop();
}
private static TestGroup CreateGroup(Type type)
{
TestGroup group = new TestGroup();
group.Name = type.Name;
group.Tags.Add(type.Name);
group.Tags.Add(type.FullName);
if (type.GetTypeInfo().GetCustomAttributes<FunctionalTestAttribute>().Any())
{
group.Tags.Add("Functional");
}
foreach (TagAttribute attr in type.GetTypeInfo().GetCustomAttributes<TagAttribute>())
{
group.Tags.Add(attr.Tag);
}
return group;
}
public async void EndGroup(TestGroup group)
{
await Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () =>
{
_currentGroup = null;
});
}
public async void StartGroup(TestGroup group)
{
await Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () =>
{
_currentGroup = new GroupDescription { Name = group.Name };
_groups.Add(_currentGroup);
});
}
public GroupDescription (TestGroup group)
{
Group = group;
this.tests.CollectionChanged += (sender, e) => HasFailures = this.tests.Any (t => !t.Test.Passed);
}
private static TestGroup CreateGroup(TypeInfo type)
{
TestGroup group = new TestGroup();
group.Name = type.Name;
group.Tags.Add(type.Name);
group.Tags.Add(type.FullName);
if (type.GetCustomAttributes(true).Where(a => a.GetType() == typeof(FunctionalTestAttribute)).Any())
{
group.Tags.Add("Functional");
}
foreach (TagAttribute attr in type.GetCustomAttributes(true).Where(a => a.GetType() == typeof(TagAttribute)))
{
group.Tags.Add(attr.Tag);
}
return group;
}
public void StartGroup(TestGroup group)
{
Dispatcher.BeginInvoke(() =>
{
_currentGroup = new GroupDescription { Name = group.Name };
_groups.Add(_currentGroup);
});
}
/// <summary>
/// Run the unit tests.
/// </summary>
public async void RunAsync()
{
// Ensure there's an interface to display the test results.
if (this.Reporter == null)
{
throw new ArgumentNullException("Reporter");
}
/// Record the test metadata to the test run object at the start of the test run
FillTestRunMetaData();
// Setup the progress/failure counters
this.Progress = 0;
this.Failures = 0;
// Filter out any test methods based on the current settings
int filteredTestCount = FilterTests();
// get the actual count of tests going to run
testRun.TestCount = filteredTestCount;
// Write out the test status message which may be modified by the
// filters
Reporter.Status(this.Settings.TestRunStatusMessage);
// Enumerators that track the current group and method to execute
// (which allows reentrancy into the test loop below to resume at
// the correct location).
IEnumerator <TestGroup> groups = this.Groups.OrderBy(g => g.Name).GetEnumerator();
IEnumerator <TestMethod> methods = null;
// Keep a reference to the current group so we can pass it to the
// Reporter's EndGroup (we don't need one for the test method,
// however, because we close over in the continuation).
TestGroup currentGroup = null;
// Create daylight test run and get the run id
string runId = String.Empty;
if (!Settings.ManualMode)
{
runId = await TestLogger.CreateDaylightRun(testRun, Settings);
}
// Setup the UI
this.Reporter.StartRun(this);
// The primary test loop is a "recursive" closure that will pass
// itself as the continuation to async tests.
//
// Note: It's really important for performance to note that any
// calls to testLoop only occur in the tail position.
JObject sasResponseObject = null;
DateTime RunStartTime = DateTime.UtcNow;
Func <Task> testLoop = null;
// Get the SAS token to upload the test logs to upload test logs to blob store
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
sasResponseObject = await TestLogger.GetSaSToken(Settings);
}
testLoop =
async() =>
{
if (methods != null && methods.MoveNext())
{
// If we were in the middle of a test group and there
// are more methods to execute, let's move to the next
// test method.
// Update the progress
this.Progress++;
Reporter.Progress(this);
// Start the test method
Reporter.StartTest(methods.Current);
if (methods.Current.Excluded)
{
// Ignore excluded tests and immediately recurse.
Reporter.EndTest(methods.Current);
await testLoop();
}
else
{
// Get the start time for individual tests
DateTime testStartTime = DateTime.UtcNow;
// Record the test result, upload the test log as a blob and clear the
// log for next test
Func <Task> recordTestResult = async() =>
{
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
var log = new TestLogs()
{
LogLines = LogDump.ToString(),
LogHash = Guid.NewGuid().ToString()
};
// record the test result
var testResult = new TestResult()
{
FullName = methods.Current.Name,
StartTime = testStartTime,
EndTime = DateTime.UtcNow,
Outcome = methods.Current.Passed ? "Passed" : "Failed",
RunId = runId,
Tags = new List <string> {
Platform
},
Source = currentGroup.Name,
Logs = sasResponseObject != null ? log : null
};
// upload test log to the blob store
if (sasResponseObject != null)
{
await TestLogger.UploadTestLog(log, sasResponseObject);
}
// Add the test result to the test result collection, which will eventually
// be uploaded to daylight
testRun.AddTestResult(testResult);
LogDump.Clear();
}
};
// Execute the test method
methods.Current.Test.Start(
new ActionContinuation
{
OnSuccess = async() =>
{
// Mark the test as passing, update the
// UI, and continue with the next test.
methods.Current.Passed = true;
methods.Current.Test = null;
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
},
OnError = async(message) =>
{
// Mark the test as failing, update the
// UI, and continue with the next test.
methods.Current.Passed = false;
methods.Current.Test = null;
methods.Current.ErrorInfo = message;
this.Failures++;
System.Diagnostics.Debug.WriteLine(message);
Reporter.Error(message);
LogDump.AppendLine(message);
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
}
});
}
}
else if (groups.MoveNext())
{
// If we've finished a test group and there are more,
// then move to the next one.
// Finish the UI for the last group.
if (currentGroup != null)
{
Reporter.EndGroup(currentGroup);
currentGroup = null;
}
// Setup the UI for this next group
currentGroup = groups.Current;
Reporter.StartGroup(currentGroup);
// Get the methods and immediately recurse which will
// start executing them.
methods = groups.Current.Methods.OrderBy(m => m.Name).GetEnumerator();
await testLoop();
}
else
{
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
// post all the test results to daylight
await TestLogger.PostTestResults(testRun.TestResults.ToList(), Settings);
// String to store the test result summary of the entire suite
StringBuilder resultLog = new StringBuilder("Total Tests:" + filteredTestCount);
resultLog.AppendLine();
resultLog.AppendLine("Passed Tests:" + (filteredTestCount - Failures).ToString());
resultLog.AppendLine("Failed Tests:" + Failures);
resultLog.AppendLine("Detailed Results:" + Settings.Custom["DayLightUrl"] + "/" + Settings.Custom["DaylightProject"] + "/runs/" + runId);
var logs = new TestLogs()
{
LogLines = resultLog.ToString(),
LogHash = Guid.NewGuid().ToString()
};
// Record the the result of the entire test suite to master test run
var testResult = new TestResult()
{
FullName = Platform + " " + Settings.Custom["RuntimeVersion"],
Name = Platform + " " + Settings.Custom["RuntimeVersion"],
StartTime = RunStartTime,
EndTime = DateTime.UtcNow,
Outcome = Failures > 0 ? "Failed" : "Passed",
RunId = Settings.Custom["MasterRunId"],
Tags = new List <string> {
Platform
},
Source = "Managed",
Logs = sasResponseObject != null ? logs : null
};
// Upload the log of the test result summary for the test suite
if (sasResponseObject != null)
{
await TestLogger.UploadTestLog(logs, sasResponseObject);
}
// Post the test suite result to master run
await TestLogger.PostTestResults(new List <TestResult> {
testResult
}, Settings);
}
// Otherwise if we've finished the entire test run
// Finish the UI for the last group and update the
// progress after the very last test method.
Reporter.EndGroup(currentGroup);
Reporter.Progress(this);
// Finish the UI for the test run.
Reporter.EndRun(this);
}
};
// Start running the tests
await testLoop();
}
public async void EndGroup(TestGroup group)
{
await Dispatcher.InvokeAsync(() =>
{
currentGroup = null;
});
}
public async void StartGroup(TestGroup group)
{
await Dispatcher.InvokeAsync(() =>
{
currentGroup = new GroupDescription { Name = group.Name };
groups.Add(currentGroup);
});
}
public void EndGroup (TestGroup testGroup)
{
}
void ITestReporter.EndGroup (TestGroup group)
{
}
public void EndGroup(TestGroup group)
{
Dispatcher.BeginInvoke(() =>
{
_currentGroup = null;
});
}
/// <summary>
/// Run the unit tests.
/// </summary>
public void RunAsync()
{
// Ensure there's an interface to display the test results.
if (this.Reporter == null)
{
throw new ArgumentNullException("Reporter");
}
// Setup the progress/failure counters
this.Progress = 0;
this.Failures = 0;
// Filter out any test methods based on the current settings
FilterTests();
// Write out the test status message which may be modified by the
// filters
Reporter.Status(this.Settings.TestRunStatusMessage);
// Enumerators that track the current group and method to execute
// (which allows reentrancy into the test loop below to resume at
// the correct location).
IEnumerator <TestGroup> groups = this.Groups.OrderBy(g => g.Name).GetEnumerator();
IEnumerator <TestMethod> methods = null;
// Keep a reference to the current group so we can pass it to the
// Reporter's EndGroup (we don't need one for the test method,
// however, because we close over in the continuation).
TestGroup currentGroup = null;
// Setup the UI
this.Reporter.StartRun(this);
// The primary test loop is a "recursive" closure that will pass
// itself as the continuation to async tests.
//
// Note: It's really important for performance to note that any
// calls to testLoop only occur in the tail position.
Action testLoop = null;
testLoop =
() =>
{
if (methods != null && methods.MoveNext())
{
// If we were in the middle of a test group and there
// are more methods to execute, let's move to the next
// test method.
// Update the progress
this.Progress++;
Reporter.Progress(this);
// Start the test method
Reporter.StartTest(methods.Current);
if (methods.Current.Excluded)
{
// Ignore excluded tests and immediately recurse.
Reporter.EndTest(methods.Current);
testLoop();
}
else
{
// Execute the test method
methods.Current.Test.Start(
new ActionContinuation
{
OnSuccess = () =>
{
// Mark the test as passing, update the
// UI, and continue with the next test.
methods.Current.Passed = true;
methods.Current.Test = null;
Reporter.EndTest(methods.Current);
testLoop();
},
OnError = (message) =>
{
// Mark the test as failing, update the
// UI, and continue with the next test.
methods.Current.Passed = false;
methods.Current.Test = null;
methods.Current.ErrorInfo = message;
this.Failures++;
Reporter.Error(message);
Reporter.EndTest(methods.Current);
testLoop();
}
});
}
}
else if (groups.MoveNext())
{
// If we've finished a test group and there are more,
// then move to the next one.
// Finish the UI for the last group.
if (currentGroup != null)
{
Reporter.EndGroup(currentGroup);
currentGroup = null;
}
// Setup the UI for this next group
currentGroup = groups.Current;
Reporter.StartGroup(currentGroup);
// Get the methods and immediately recurse which will
// start executing them.
methods = groups.Current.Methods.OrderBy(m => m.Name).GetEnumerator();
testLoop();
}
else
{
// Otherwise if we've finished the entire test run
// Finish the UI for the last group and update the
// progress after the very last test method.
Reporter.EndGroup(currentGroup);
Reporter.Progress(this);
// Finish the UI for the test run.
Reporter.EndRun(this);
}
};
// Start running the tests
testLoop();
}