/// <summary>
/// Upload the test log to the test blob store
/// </summary>
/// <param name="testResult">Test result object that contains the test log </param>
/// <param name="sasResponseContent">Sas response object containing access token for blob store</param>
/// <returns>True if log uploaded successfully, otherwise false</returns>
public static async Task <bool> UploadTestLog(TestLogs logs, JObject sasResponseContent)
{
string accessToken = sasResponseContent["access_token"].ToObject <string>();
string containerUri = sasResponseContent["container_uri"].ToObject <string>();
Uri blobHostUri = new Uri(containerUri);
HttpClient c = new HttpClient();
string hash = logs.LogHash;
var blobRequestUri = containerUri + "/" + hash + "?" + accessToken;
c.DefaultRequestHeaders.Add("x-ms-blob-type", "BlockBlob");
c.DefaultRequestHeaders.Add("Host", blobHostUri.Host);
ByteArrayContent byteContent = new ByteArrayContent(Encoding.UTF8.GetBytes(logs.LogLines));
using (var blobResponse = await c.PutAsync(blobRequestUri, byteContent))
{
return(blobResponse.IsSuccessStatusCode);
}
}
/// <summary>
/// Upload the test log to the test blob store
/// </summary>
/// <param name="testResult">Test result object that contains the test log </param>
/// <param name="sasResponseContent">Sas response object containing access token for blob store</param>
/// <returns>True if log uploaded successfully, otherwise false</returns>
public static async Task<bool> UploadTestLog(TestLogs logs, JObject sasResponseContent)
{
string accessToken = sasResponseContent["access_token"].ToObject<string>();
string containerUri = sasResponseContent["container_uri"].ToObject<string>();
Uri blobHostUri = new Uri(containerUri);
HttpClient c = new HttpClient();
string hash = logs.LogHash;
var blobRequestUri = containerUri + "/" + hash + "?" + accessToken;
c.DefaultRequestHeaders.Add("x-ms-blob-type", "BlockBlob");
c.DefaultRequestHeaders.Add("Host", blobHostUri.Host);
ByteArrayContent byteContent = new ByteArrayContent(Encoding.UTF8.GetBytes(logs.LogLines));
using (var blobResponse = await c.PutAsync(blobRequestUri, byteContent))
{
return blobResponse.IsSuccessStatusCode;
}
}
/// <summary>
/// Run the unit tests.
/// </summary>
public async void RunAsync()
{
// Ensure there's an interface to display the test results.
if (this.Reporter == null)
{
throw new ArgumentNullException("Reporter");
}
/// Record the test metadata to the test run object at the start of the test run
FillTestRunMetaData();
// Setup the progress/failure counters
this.Progress = 0;
this.Failures = 0;
// Filter out any test methods based on the current settings
int filteredTestCount = FilterTests();
// get the actual count of tests going to run
testRun.TestCount = filteredTestCount;
// Write out the test status message which may be modified by the
// filters
Reporter.Status(this.Settings.TestRunStatusMessage);
// Enumerators that track the current group and method to execute
// (which allows reentrancy into the test loop below to resume at
// the correct location).
IEnumerator <TestGroup> groups = this.Groups.OrderBy(g => g.Name).GetEnumerator();
IEnumerator <TestMethod> methods = null;
// Keep a reference to the current group so we can pass it to the
// Reporter's EndGroup (we don't need one for the test method,
// however, because we close over in the continuation).
TestGroup currentGroup = null;
// Create daylight test run and get the run id
string runId = String.Empty;
if (!Settings.ManualMode)
{
runId = await TestLogger.CreateDaylightRun(testRun, Settings);
}
// Setup the UI
this.Reporter.StartRun(this);
// The primary test loop is a "recursive" closure that will pass
// itself as the continuation to async tests.
//
// Note: It's really important for performance to note that any
// calls to testLoop only occur in the tail position.
JObject sasResponseObject = null;
DateTime RunStartTime = DateTime.UtcNow;
Func <Task> testLoop = null;
// Get the SAS token to upload the test logs to upload test logs to blob store
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
sasResponseObject = await TestLogger.GetSaSToken(Settings);
}
testLoop =
async() =>
{
if (methods != null && methods.MoveNext())
{
// If we were in the middle of a test group and there
// are more methods to execute, let's move to the next
// test method.
// Update the progress
this.Progress++;
Reporter.Progress(this);
// Start the test method
Reporter.StartTest(methods.Current);
if (methods.Current.Excluded)
{
// Ignore excluded tests and immediately recurse.
Reporter.EndTest(methods.Current);
await testLoop();
}
else
{
// Get the start time for individual tests
DateTime testStartTime = DateTime.UtcNow;
// Record the test result, upload the test log as a blob and clear the
// log for next test
Func <Task> recordTestResult = async() =>
{
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
var log = new TestLogs()
{
LogLines = LogDump.ToString(),
LogHash = Guid.NewGuid().ToString()
};
// record the test result
var testResult = new TestResult()
{
FullName = methods.Current.Name,
StartTime = testStartTime,
EndTime = DateTime.UtcNow,
Outcome = methods.Current.Passed ? "Passed" : "Failed",
RunId = runId,
Tags = new List <string> {
Platform
},
Source = currentGroup.Name,
Logs = sasResponseObject != null ? log : null
};
// upload test log to the blob store
if (sasResponseObject != null)
{
await TestLogger.UploadTestLog(log, sasResponseObject);
}
// Add the test result to the test result collection, which will eventually
// be uploaded to daylight
testRun.AddTestResult(testResult);
LogDump.Clear();
}
};
// Execute the test method
methods.Current.Test.Start(
new ActionContinuation
{
OnSuccess = async() =>
{
// Mark the test as passing, update the
// UI, and continue with the next test.
methods.Current.Passed = true;
methods.Current.Test = null;
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
},
OnError = async(message) =>
{
// Mark the test as failing, update the
// UI, and continue with the next test.
methods.Current.Passed = false;
methods.Current.Test = null;
methods.Current.ErrorInfo = message;
this.Failures++;
System.Diagnostics.Debug.WriteLine(message);
Reporter.Error(message);
LogDump.AppendLine(message);
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
}
});
}
}
else if (groups.MoveNext())
{
// If we've finished a test group and there are more,
// then move to the next one.
// Finish the UI for the last group.
if (currentGroup != null)
{
Reporter.EndGroup(currentGroup);
currentGroup = null;
}
// Setup the UI for this next group
currentGroup = groups.Current;
Reporter.StartGroup(currentGroup);
// Get the methods and immediately recurse which will
// start executing them.
methods = groups.Current.Methods.OrderBy(m => m.Name).GetEnumerator();
await testLoop();
}
else
{
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
// post all the test results to daylight
await TestLogger.PostTestResults(testRun.TestResults.ToList(), Settings);
// String to store the test result summary of the entire suite
StringBuilder resultLog = new StringBuilder("Total Tests:" + filteredTestCount);
resultLog.AppendLine();
resultLog.AppendLine("Passed Tests:" + (filteredTestCount - Failures).ToString());
resultLog.AppendLine("Failed Tests:" + Failures);
resultLog.AppendLine("Detailed Results:" + Settings.Custom["DayLightUrl"] + "/" + Settings.Custom["DaylightProject"] + "/runs/" + runId);
var logs = new TestLogs()
{
LogLines = resultLog.ToString(),
LogHash = Guid.NewGuid().ToString()
};
// Record the the result of the entire test suite to master test run
var testResult = new TestResult()
{
FullName = Platform + " " + Settings.Custom["RuntimeVersion"],
Name = Platform + " " + Settings.Custom["RuntimeVersion"],
StartTime = RunStartTime,
EndTime = DateTime.UtcNow,
Outcome = Failures > 0 ? "Failed" : "Passed",
RunId = Settings.Custom["MasterRunId"],
Tags = new List <string> {
Platform
},
Source = "Managed",
Logs = sasResponseObject != null ? logs : null
};
// Upload the log of the test result summary for the test suite
if (sasResponseObject != null)
{
await TestLogger.UploadTestLog(logs, sasResponseObject);
}
// Post the test suite result to master run
await TestLogger.PostTestResults(new List <TestResult> {
testResult
}, Settings);
}
// Otherwise if we've finished the entire test run
// Finish the UI for the last group and update the
// progress after the very last test method.
Reporter.EndGroup(currentGroup);
Reporter.Progress(this);
// Finish the UI for the test run.
Reporter.EndRun(this);
}
};
// Start running the tests
await testLoop();
}
/// <summary>
/// Run the unit tests.
/// </summary>
public async void RunAsync()
{
// Ensure there's an interface to display the test results.
if (this.Reporter == null)
{
throw new ArgumentNullException("Reporter");
}
/// Record the test metadata to the test run object at the start of the test run
FillTestRunMetaData();
// Setup the progress/failure counters
this.Progress = 0;
this.Failures = 0;
// Filter out any test methods based on the current settings
int filteredTestCount = FilterTests();
// get the actual count of tests going to run
testRun.TestCount = filteredTestCount;
// Write out the test status message which may be modified by the
// filters
Reporter.Status(this.Settings.TestRunStatusMessage);
// Enumerators that track the current group and method to execute
// (which allows reentrancy into the test loop below to resume at
// the correct location).
IEnumerator<TestGroup> groups = this.Groups.OrderBy(g => g.Name).GetEnumerator();
IEnumerator<TestMethod> methods = null;
// Keep a reference to the current group so we can pass it to the
// Reporter's EndGroup (we don't need one for the test method,
// however, because we close over in the continuation).
TestGroup currentGroup = null;
// Create daylight test run and get the run id
string runId = String.Empty;
if (!Settings.ManualMode)
{
runId = await TestLogger.CreateDaylightRun(testRun, Settings);
}
// Setup the UI
this.Reporter.StartRun(this);
// The primary test loop is a "recursive" closure that will pass
// itself as the continuation to async tests.
//
// Note: It's really important for performance to note that any
// calls to testLoop only occur in the tail position.
JObject sasResponseObject = null;
DateTime RunStartTime = DateTime.UtcNow;
Func<Task> testLoop = null;
// Get the SAS token to upload the test logs to upload test logs to blob store
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
sasResponseObject = await TestLogger.GetSaSToken(Settings);
}
testLoop =
async () =>
{
if (methods != null && methods.MoveNext())
{
// If we were in the middle of a test group and there
// are more methods to execute, let's move to the next
// test method.
// Update the progress
this.Progress++;
Reporter.Progress(this);
// Start the test method
Reporter.StartTest(methods.Current);
if (methods.Current.Excluded)
{
// Ignore excluded tests and immediately recurse.
Reporter.EndTest(methods.Current);
await testLoop();
}
else
{
// Get the start time for individual tests
DateTime testStartTime = DateTime.UtcNow;
// Record the test result, upload the test log as a blob and clear the
// log for next test
Func<Task> recordTestResult = async () =>
{
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
var log = new TestLogs()
{
LogLines = LogDump.ToString(),
LogHash = Guid.NewGuid().ToString()
};
// record the test result
var testResult = new TestResult()
{
FullName = methods.Current.Name,
StartTime = testStartTime,
EndTime = DateTime.UtcNow,
Outcome = methods.Current.Passed ? "Passed" : "Failed",
RunId = runId,
Tags = new List<string> { Platform },
Source = currentGroup.Name,
Logs = sasResponseObject != null ? log : null
};
// upload test log to the blob store
if (sasResponseObject != null)
await TestLogger.UploadTestLog(log, sasResponseObject);
// Add the test result to the test result collection, which will eventually
// be uploaded to daylight
testRun.AddTestResult(testResult);
LogDump.Clear();
}
};
// Execute the test method
methods.Current.Test.Start(
new ActionContinuation
{
OnSuccess = async () =>
{
// Mark the test as passing, update the
// UI, and continue with the next test.
methods.Current.Passed = true;
methods.Current.Test = null;
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
},
OnError = async (message) =>
{
// Mark the test as failing, update the
// UI, and continue with the next test.
methods.Current.Passed = false;
methods.Current.Test = null;
methods.Current.ErrorInfo = message;
this.Failures++;
System.Diagnostics.Debug.WriteLine(message);
Reporter.Error(message);
LogDump.AppendLine(message);
Reporter.EndTest(methods.Current);
await recordTestResult();
await testLoop();
}
});
}
}
else if (groups.MoveNext())
{
// If we've finished a test group and there are more,
// then move to the next one.
// Finish the UI for the last group.
if (currentGroup != null)
{
Reporter.EndGroup(currentGroup);
currentGroup = null;
}
// Setup the UI for this next group
currentGroup = groups.Current;
Reporter.StartGroup(currentGroup);
// Get the methods and immediately recurse which will
// start executing them.
methods = groups.Current.Methods.OrderBy(m => m.Name).GetEnumerator();
await testLoop();
}
else
{
if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
{
// post all the test results to daylight
await TestLogger.PostTestResults(testRun.TestResults.ToList(), Settings);
// String to store the test result summary of the entire suite
StringBuilder resultLog = new StringBuilder("Total Tests:" + filteredTestCount);
resultLog.AppendLine();
resultLog.AppendLine("Passed Tests:" + (filteredTestCount - Failures).ToString());
resultLog.AppendLine("Failed Tests:" + Failures);
resultLog.AppendLine("Detailed Results:" + Settings.Custom["DayLightUrl"] + "/" + Settings.Custom["DaylightProject"] + "/runs/" + runId);
var logs = new TestLogs()
{
LogLines = resultLog.ToString(),
LogHash = Guid.NewGuid().ToString()
};
// Record the the result of the entire test suite to master test run
var testResult = new TestResult()
{
FullName = Platform + " " + Settings.Custom["RuntimeVersion"],
Name = Platform + " " + Settings.Custom["RuntimeVersion"],
StartTime = RunStartTime,
EndTime = DateTime.UtcNow,
Outcome = Failures > 0 ? "Failed" : "Passed",
RunId = Settings.Custom["MasterRunId"],
Tags = new List<string> { Platform },
Source = "Managed",
Logs = sasResponseObject != null ? logs : null
};
// Upload the log of the test result summary for the test suite
if (sasResponseObject != null)
await TestLogger.UploadTestLog(logs, sasResponseObject);
// Post the test suite result to master run
await TestLogger.PostTestResults(new List<TestResult> { testResult }, Settings);
}
// Otherwise if we've finished the entire test run
// Finish the UI for the last group and update the
// progress after the very last test method.
Reporter.EndGroup(currentGroup);
Reporter.Progress(this);
// Finish the UI for the test run.
Reporter.EndRun(this);
}
};
// Start running the tests
await testLoop();
}