/// <summary>
/// Run the unit tests.
/// </summary>
public async void RunAsync()
{
// Ensure there's an interface to display the test results.
if (this.Reporter == null)
{
throw new ArgumentNullException("Reporter");
}
/// Record the test metadata to the test run object at the start of the test run
FillTestRunMetaData();
// Setup the progress/failure counters
this.Progress = 0;
this.Failures = 0;
// Filter out any test methods based on the current settings
int filteredTestCount = FilterTests();
// get the actual count of tests going to run
testRun.TestCount = filteredTestCount;
// Write out the test status message which may be modified by the
// filters
Reporter.Status(this.Settings.TestRunStatusMessage);
// Enumerators that track the current group and method to execute
// (which allows reentrancy into the test loop below to resume at
// the correct location).
IEnumerator<TestGroup> groups = this.Groups.OrderBy(g => g.Name).GetEnumerator();
IEnumerator<TestMethod> methods = null;
// Keep a reference to the current group so we can pass it to the
// Reporter's EndGroup (we don't need one for the test method,
// however, because we close over in the continuation).
TestGroup currentGroup = null;
// Create daylight test run and get the run id
string runId = String.Empty;
if (!Settings.ManualMode)
{
runId = await TestLogger.CreateDaylightRun(testRun, Settings);
}
// Setup the UI
this.Reporter.StartRun(this);
// The primary test loop is a "recursive" closure that will pass
// itself as the continuation to async tests.
//
// Note: It's really important for performance to note that any
// calls to testLoop only occur in the tail position.
JObject sasResponseObject = null;
DateTime RunStartTime = DateTime.UtcNow;
Func<Task> testLoop = null;
// Get the SAS token to upload the test logs to upload test logs to blob store
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
sasResponseObject = await TestLogger.GetSaSToken(Settings);
}
testLoop =
async () =>
{
if (methods != null && methods.MoveNext())
{
// If we were in the middle of a test group and there
// are more methods to execute, let's move to the next
// test method.
// Update the progress
this.Progress++;
Reporter.Progress(this);
// Start the test method
Reporter.StartTest(methods.Current);
if (methods.Current.Excluded)
{
// Ignore excluded tests and immediately recurse.
Reporter.EndTest(methods.Current);
await testLoop();
}
else
{
// Get the start time for individual tests
DateTime testStartTime = DateTime.UtcNow;
// Record the test result, upload the test log as a blob and clear the
// log for next test
Func<Task> recordTestResult = async () =>
{
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
var log = new TestLogs()
{
LogLines = LogDump.ToString(),
LogHash = Guid.NewGuid().ToString()
};
// record the test result
var testResult = new TestResult()
{
FullName = methods.Current.Name,
StartTime = testStartTime,
EndTime = DateTime.UtcNow,
Outcome = methods.Current.Passed ? "Passed" : "Failed",
RunId = runId,
Tags = new List<string> { Platform },
Source = currentGroup.Name,
Logs = sasResponseObject != null ? log : null
};
// upload test log to the blob store
if (sasResponseObject != null)
await TestLogger.UploadTestLog(log, sasResponseObject);
// Add the test result to the test result collection, which will eventually
// be uploaded to daylight
testRun.AddTestResult(testResult);
LogDump.Clear();
}
};
// Execute the test method
methods.Current.Test.Start(
new ActionContinuation
{
OnSuccess = async () =>
{
// Mark the test as passing, update the
// UI, and continue with the next test.
methods.Current.Passed = true;
methods.Current.Test = null;
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
},
OnError = async (message) =>
{
// Mark the test as failing, update the
// UI, and continue with the next test.
methods.Current.Passed = false;
methods.Current.Test = null;
methods.Current.ErrorInfo = message;
this.Failures++;
System.Diagnostics.Debug.WriteLine(message);
Reporter.Error(message);
LogDump.AppendLine(message);
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
}
});
}
}
else if (groups.MoveNext())
{
// If we've finished a test group and there are more,
// then move to the next one.
// Finish the UI for the last group.
if (currentGroup != null)
{
Reporter.EndGroup(currentGroup);
currentGroup = null;
}
// Setup the UI for this next group
currentGroup = groups.Current;
Reporter.StartGroup(currentGroup);
// Get the methods and immediately recurse which will
// start executing them.
methods = groups.Current.Methods.OrderBy(m => m.Name).GetEnumerator();
await testLoop();
}
else
{
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
// post all the test results to daylight
await TestLogger.PostTestResults(testRun.TestResults.ToList(), Settings);
// String to store the test result summary of the entire suite
StringBuilder resultLog = new StringBuilder("Total Tests:" + filteredTestCount);
resultLog.AppendLine();
resultLog.AppendLine("Passed Tests:" + (filteredTestCount - Failures).ToString());
resultLog.AppendLine("Failed Tests:" + Failures);
resultLog.AppendLine("Detailed Results:" + Settings.Custom["DayLightUrl"] + "/" + Settings.Custom["DaylightProject"] + "/runs/" + runId);
var logs = new TestLogs()
{
LogLines = resultLog.ToString(),
LogHash = Guid.NewGuid().ToString()
};
// Record the the result of the entire test suite to master test run
var testResult = new TestResult()
{
FullName = Platform + " " + Settings.Custom["RuntimeVersion"],
Name = Platform + " " + Settings.Custom["RuntimeVersion"],
StartTime = RunStartTime,
EndTime = DateTime.UtcNow,
Outcome = Failures > 0 ? "Failed" : "Passed",
RunId = Settings.Custom["MasterRunId"],
Tags = new List<string> { Platform },
Source = "Managed",
Logs = sasResponseObject != null ? logs : null
};
// Upload the log of the test result summary for the test suite
if (sasResponseObject != null)
await TestLogger.UploadTestLog(logs, sasResponseObject);
// Post the test suite result to master run
await TestLogger.PostTestResults(new List<TestResult> { testResult }, Settings);
}
// Otherwise if we've finished the entire test run
// Finish the UI for the last group and update the
// progress after the very last test method.
Reporter.EndGroup(currentGroup);
Reporter.Progress(this);
// Finish the UI for the test run.
Reporter.EndRun(this);
}
};
// Start running the tests
await testLoop();
}
/// <summary>
/// Run the unit tests.
/// </summary>
public async void RunAsync()
{
// Ensure there's an interface to display the test results.
if (this.Reporter == null)
{
throw new ArgumentNullException("Reporter");
}
// Setup the progress/failure counters
this.Progress = 0;
this.Failures = 0;
// Filter out any test methods based on the current settings
int filteredTestCount = FilterTests();
// Write out the test status message which may be modified by the
// filters
Reporter.Status(this.Settings.TestRunStatusMessage);
// Enumerators that track the current group and method to execute
// (which allows reentrancy into the test loop below to resume at
// the correct location).
IEnumerator<TestGroup> groups = this.Groups.OrderBy(g => g.Name).GetEnumerator();
IEnumerator<TestMethod> methods = null;
// Keep a reference to the current group so we can pass it to the
// Reporter's EndGroup (we don't need one for the test method,
// however, because we close over in the continuation).
TestGroup currentGroup = null;
// Setup the UI
this.Reporter.StartRun(this);
// The primary test loop is a "recursive" closure that will pass
// itself as the continuation to async tests.
//
// Note: It's really important for performance to note that any
// calls to testLoop only occur in the tail position.
DateTime RunStartTime = DateTime.UtcNow;
Func<Task> testLoop = null;
testLoop =
async () =>
{
if (methods != null && methods.MoveNext())
{
// If we were in the middle of a test group and there
// are more methods to execute, let's move to the next
// test method.
// Update the progress
this.Progress++;
Reporter.Progress(this);
// Start the test method
Reporter.StartTest(methods.Current);
if (methods.Current.Excluded)
{
// Ignore excluded tests and immediately recurse.
Reporter.EndTest(methods.Current);
await testLoop();
}
else
{
// Get the start time for individual tests
DateTime testStartTime = DateTime.UtcNow;
// Record the test result, upload the test log as a blob and clear the
// log for next test
Func<Task> recordTestResult = async () =>
{
if (!Settings.ManualMode)
{
// upload test log to sunlight blob container
string relativeFilePath = this.Platform + "/" + Guid.NewGuid().ToString() + ".txt";
string blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
relativeFilePath);
await UploadToBlobContainerAsync(blobStorageSasUrl, LogDump.ToString());
// record the test result
var testResult = new TestResult()
{
FullName = methods.Current.Name,
StartTime = testStartTime,
EndTime = DateTime.UtcNow,
Outcome = methods.Current.Passed ? "Passed" : "Failed",
Source = currentGroup.Name,
ReferenceUrl = relativeFilePath
};
// Add the test result to the test result collection
testRun.AddTestResult(testResult);
LogDump.Clear();
}
};
// Execute the test method
methods.Current.Test.Start(
new ActionContinuation
{
OnSuccess = async () =>
{
// Mark the test as passing, update the
// UI, and continue with the next test.
methods.Current.Passed = true;
methods.Current.Test = null;
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
},
OnError = async (message) =>
{
// Mark the test as failing, update the
// UI, and continue with the next test.
methods.Current.Passed = false;
methods.Current.Test = null;
methods.Current.ErrorInfo = message;
this.Failures++;
System.Diagnostics.Debug.WriteLine(message);
Reporter.Error(message);
LogDump.AppendLine(message);
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
}
});
}
}
else if (groups.MoveNext())
{
// If we've finished a test group and there are more,
// then move to the next one.
// Finish the UI for the last group.
if (currentGroup != null)
{
Reporter.EndGroup(currentGroup);
currentGroup = null;
}
// Setup the UI for this next group
currentGroup = groups.Current;
Reporter.StartGroup(currentGroup);
// Get the methods and immediately recurse which will
// start executing them.
methods = groups.Current.Methods.OrderBy(m => m.Name).GetEnumerator();
await testLoop();
}
else
{
if (!Settings.ManualMode)
{
// upload test suite result to sunlight blob container
string detailFilePath = string.Format("{0}-{1}-detail.json", this.Settings.Custom["RuntimeVersion"], this.Platform);
string blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
detailFilePath);
string fileContent = JsonConvert.SerializeObject(testRun.TestResults.ToList(), Formatting.Indented);
await UploadToBlobContainerAsync(blobStorageSasUrl, fileContent);
// upload test result summary to blob container
var masterResult = new MasterTestResult()
{
FullName = Settings.Custom["RuntimeVersion"] + "-" + this.Platform,
Backend = Settings.Custom["RuntimeVersion"],
Outcome = Failures > 0 ? "Failed" : "Passed",
TotalCount = testRun.TestCount,
Passed = filteredTestCount - Failures,
Failed = Failures,
Skipped = testRun.TestCount - filteredTestCount,
StartTime = RunStartTime,
EndTime = DateTime.UtcNow,
ReferenceUrl = Settings.Custom["RuntimeVersion"] + "-" + this.Platform + "-detail.json"
};
// upload test suite result to sunlight blob container
string masterFilePath = string.Format("{0}-{1}-master.json", this.Settings.Custom["RuntimeVersion"], this.Platform);
blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
masterFilePath);
fileContent = JsonConvert.SerializeObject(masterResult, Formatting.Indented);
await UploadToBlobContainerAsync(blobStorageSasUrl, fileContent);
}
// Otherwise if we've finished the entire test run
// Finish the UI for the last group and update the
// progress after the very last test method.
Reporter.EndGroup(currentGroup);
Reporter.Progress(this);
// Finish the UI for the test run.
Reporter.EndRun(this);
}
};
// Start running the tests
await testLoop();
}
public void AddTestResult(TestResult result)
{
this.testResults.Add(result);
this.TestCount++;
}
