/// <summary>
/// Run the unit tests.
/// </summary>
public async void RunAsync()
{
// Ensure there's an interface to display the test results.
if (this.Reporter == null)
{
throw new ArgumentNullException("Reporter");
}
// Setup the progress/failure counters
this.Progress = 0;
this.Failures = 0;
// Filter out any test methods based on the current settings
int filteredTestCount = FilterTests();
// Write out the test status message which may be modified by the
// filters
Reporter.Status(this.Settings.TestRunStatusMessage);
// Enumerators that track the current group and method to execute
// (which allows reentrancy into the test loop below to resume at
// the correct location).
IEnumerator <TestGroup> groups = this.Groups.OrderBy(g => g.Name).GetEnumerator();
IEnumerator <TestMethod> methods = null;
// Keep a reference to the current group so we can pass it to the
// Reporter's EndGroup (we don't need one for the test method,
// however, because we close over in the continuation).
TestGroup currentGroup = null;
// Setup the UI
this.Reporter.StartRun(this);
// The primary test loop is a "recursive" closure that will pass
// itself as the continuation to async tests.
//
// Note: It's really important for performance to note that any
// calls to testLoop only occur in the tail position.
DateTime RunStartTime = DateTime.UtcNow;
Func <Task> testLoop = null;
testLoop =
async() =>
{
if (methods != null && methods.MoveNext())
{
// If we were in the middle of a test group and there
// are more methods to execute, let's move to the next
// test method.
// Update the progress
this.Progress++;
Reporter.Progress(this);
// Start the test method
Reporter.StartTest(methods.Current);
if (methods.Current.Excluded)
{
// Ignore excluded tests and immediately recurse.
Reporter.EndTest(methods.Current);
await testLoop();
}
else
{
// Get the start time for individual tests
DateTime testStartTime = DateTime.UtcNow;
// Record the test result, upload the test log as a blob and clear the
// log for next test
Func <Task> recordTestResult = async() =>
{
if (!Settings.ManualMode)
{
// upload test log to sunlight blob container
string relativeFilePath = this.Platform + "/" + Guid.NewGuid().ToString() + ".txt";
string blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
relativeFilePath);
await UploadToBlobContainerAsync(blobStorageSasUrl, LogDump.ToString());
// record the test result
var testResult = new TestResult()
{
FullName = methods.Current.Name,
StartTime = testStartTime,
EndTime = DateTime.UtcNow,
Outcome = methods.Current.Passed ? "Passed" : "Failed",
Source = currentGroup.Name,
ReferenceUrl = relativeFilePath
};
// Add the test result to the test result collection
testRun.AddTestResult(testResult);
LogDump.Clear();
}
};
// Execute the test method
methods.Current.Test.Start(
new ActionContinuation
{
OnSuccess = async() =>
{
// Mark the test as passing, update the
// UI, and continue with the next test.
methods.Current.Passed = true;
methods.Current.Test = null;
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
},
OnError = async(message) =>
{
// Mark the test as failing, update the
// UI, and continue with the next test.
methods.Current.Passed = false;
methods.Current.Test = null;
methods.Current.ErrorInfo = message;
this.Failures++;
System.Diagnostics.Debug.WriteLine(message);
Reporter.Error(message);
LogDump.AppendLine(message);
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
}
});
}
}
else if (groups.MoveNext())
{
// If we've finished a test group and there are more,
// then move to the next one.
// Finish the UI for the last group.
if (currentGroup != null)
{
Reporter.EndGroup(currentGroup);
currentGroup = null;
}
// Setup the UI for this next group
currentGroup = groups.Current;
Reporter.StartGroup(currentGroup);
// Get the methods and immediately recurse which will
// start executing them.
methods = groups.Current.Methods.OrderBy(m => m.Name).GetEnumerator();
await testLoop();
}
else
{
if (!Settings.ManualMode)
{
// upload test suite result to sunlight blob container
string blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
this.Platform + "-detail.json");
string fileContent = JsonConvert.SerializeObject(testRun.TestResults.ToList(), Formatting.Indented);
await UploadToBlobContainerAsync(blobStorageSasUrl, fileContent);
// upload test result summary to blob container
var masterResult = new MasterTestResult()
{
FullName = this.Platform + "-" + Settings.Custom["RuntimeVersion"],
Outcome = Failures > 0 ? "Failed" : "Passed",
TotalCount = testRun.TestCount,
Passed = filteredTestCount - Failures,
Failed = Failures,
Skipped = testRun.TestCount - filteredTestCount,
StartTime = RunStartTime,
EndTime = DateTime.UtcNow,
ReferenceUrl = this.Platform + "-detail.json"
};
// upload test suite result to sunlight blob container
blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
this.Platform + "-master.json");
fileContent = JsonConvert.SerializeObject(masterResult, Formatting.Indented);
await UploadToBlobContainerAsync(blobStorageSasUrl, fileContent);
}
// Otherwise if we've finished the entire test run
// Finish the UI for the last group and update the
// progress after the very last test method.
Reporter.EndGroup(currentGroup);
Reporter.Progress(this);
// Finish the UI for the test run.
Reporter.EndRun(this);
}
};
// Start running the tests
await testLoop();
}
/// <summary>
/// Run the unit tests.
/// </summary>
public async void RunAsync()
{
// Ensure there's an interface to display the test results.
if (this.Reporter == null)
{
throw new ArgumentNullException("Reporter");
}
// Setup the progress/failure counters
this.Progress = 0;
this.Failures = 0;
// Filter out any test methods based on the current settings
int filteredTestCount = FilterTests();
// Write out the test status message which may be modified by the
// filters
Reporter.Status(this.Settings.TestRunStatusMessage);
// Enumerators that track the current group and method to execute
// (which allows reentrancy into the test loop below to resume at
// the correct location).
IEnumerator<TestGroup> groups = this.Groups.OrderBy(g => g.Name).GetEnumerator();
IEnumerator<TestMethod> methods = null;
// Keep a reference to the current group so we can pass it to the
// Reporter's EndGroup (we don't need one for the test method,
// however, because we close over in the continuation).
TestGroup currentGroup = null;
// Setup the UI
this.Reporter.StartRun(this);
// The primary test loop is a "recursive" closure that will pass
// itself as the continuation to async tests.
//
// Note: It's really important for performance to note that any
// calls to testLoop only occur in the tail position.
DateTime RunStartTime = DateTime.UtcNow;
Func<Task> testLoop = null;
testLoop =
async () =>
{
if (methods != null && methods.MoveNext())
{
// If we were in the middle of a test group and there
// are more methods to execute, let's move to the next
// test method.
// Update the progress
this.Progress++;
Reporter.Progress(this);
// Start the test method
Reporter.StartTest(methods.Current);
if (methods.Current.Excluded)
{
// Ignore excluded tests and immediately recurse.
Reporter.EndTest(methods.Current);
await testLoop();
}
else
{
// Get the start time for individual tests
DateTime testStartTime = DateTime.UtcNow;
// Record the test result, upload the test log as a blob and clear the
// log for next test
Func<Task> recordTestResult = async () =>
{
if (!Settings.ManualMode)
{
// upload test log to sunlight blob container
string relativeFilePath = this.Platform + "/" + Guid.NewGuid().ToString() + ".txt";
string blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
relativeFilePath);
await UploadToBlobContainerAsync(blobStorageSasUrl, LogDump.ToString());
// record the test result
var testResult = new TestResult()
{
FullName = methods.Current.Name,
StartTime = testStartTime,
EndTime = DateTime.UtcNow,
Outcome = methods.Current.Passed ? "Passed" : "Failed",
Source = currentGroup.Name,
ReferenceUrl = relativeFilePath
};
// Add the test result to the test result collection
testRun.AddTestResult(testResult);
LogDump.Clear();
}
};
// Execute the test method
methods.Current.Test.Start(
new ActionContinuation
{
OnSuccess = async () =>
{
// Mark the test as passing, update the
// UI, and continue with the next test.
methods.Current.Passed = true;
methods.Current.Test = null;
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
},
OnError = async (message) =>
{
// Mark the test as failing, update the
// UI, and continue with the next test.
methods.Current.Passed = false;
methods.Current.Test = null;
methods.Current.ErrorInfo = message;
this.Failures++;
System.Diagnostics.Debug.WriteLine(message);
Reporter.Error(message);
LogDump.AppendLine(message);
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
}
});
}
}
else if (groups.MoveNext())
{
// If we've finished a test group and there are more,
// then move to the next one.
// Finish the UI for the last group.
if (currentGroup != null)
{
Reporter.EndGroup(currentGroup);
currentGroup = null;
}
// Setup the UI for this next group
currentGroup = groups.Current;
Reporter.StartGroup(currentGroup);
// Get the methods and immediately recurse which will
// start executing them.
methods = groups.Current.Methods.OrderBy(m => m.Name).GetEnumerator();
await testLoop();
}
else
{
if (!Settings.ManualMode)
{
// upload test suite result to sunlight blob container
string detailFilePath = string.Format("{0}-{1}-detail.json", this.Settings.Custom["RuntimeVersion"], this.Platform);
string blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
detailFilePath);
string fileContent = JsonConvert.SerializeObject(testRun.TestResults.ToList(), Formatting.Indented);
await UploadToBlobContainerAsync(blobStorageSasUrl, fileContent);
// upload test result summary to blob container
var masterResult = new MasterTestResult()
{
FullName = Settings.Custom["RuntimeVersion"] + "-" + this.Platform,
Backend = Settings.Custom["RuntimeVersion"],
Outcome = Failures > 0 ? "Failed" : "Passed",
TotalCount = testRun.TestCount,
Passed = filteredTestCount - Failures,
Failed = Failures,
Skipped = testRun.TestCount - filteredTestCount,
StartTime = RunStartTime,
EndTime = DateTime.UtcNow,
ReferenceUrl = Settings.Custom["RuntimeVersion"] + "-" + this.Platform + "-detail.json"
};
// upload test suite result to sunlight blob container
string masterFilePath = string.Format("{0}-{1}-master.json", this.Settings.Custom["RuntimeVersion"], this.Platform);
blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
masterFilePath);
fileContent = JsonConvert.SerializeObject(masterResult, Formatting.Indented);
await UploadToBlobContainerAsync(blobStorageSasUrl, fileContent);
}
// Otherwise if we've finished the entire test run
// Finish the UI for the last group and update the
// progress after the very last test method.
Reporter.EndGroup(currentGroup);
Reporter.Progress(this);
// Finish the UI for the test run.
Reporter.EndRun(this);
}
};
// Start running the tests
await testLoop();
}
