bool IsActualResultSequenceNumberOlder(TestOperationResult actualResult, TestOperationResult expectedResult)
{
// TODO: The controller for TestResultCoordinator takes in a custom type
// not derived from the original types the test modules send. This
// means we have to rely on string magic like this to get the sequence
// numbers. In order to clean this up we should allow the controller to
// ingest the original type the test modules are sending. Then we
// can cast to MessageTestResult and grab the sequence number attribute.
int actualSequenceNumber = int.Parse(actualResult.Result.Split(";")[2]);
int expectedSequenceNumber = int.Parse(expectedResult.Result.Split(";")[2]);
return(actualSequenceNumber < expectedSequenceNumber);
}
public async Task <StatusCodeResult> PostAsync(TestOperationResult result)
{
try
{
bool success = await this.storage.AddResultAsync(result);
Logger.LogDebug($"Received test result: {result.Source}, {result.Result}, {result.Type}, {success}");
return(success ? this.StatusCode((int)HttpStatusCode.NoContent) : this.StatusCode((int)HttpStatusCode.BadRequest));
}
catch (Exception)
{
return(this.StatusCode((int)HttpStatusCode.InternalServerError));
}
}
public Task <List <TestOperationResult> > Actual2(IAsyncEnumerable <TestOperationResult> enumerable)
{
var actualResult1 = new TestOperationResult("sender", TestOperationResultType.Messages.ToString(), "result1", DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10)));
var actualResult2 = new TestOperationResult("sender", TestOperationResultType.Messages.ToString(), "result2", DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(9)));
var actualResult3 = new TestOperationResult("sender", TestOperationResultType.Messages.ToString(), "result2", DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(9)));
var actualResult4 = new TestOperationResult("sender", TestOperationResultType.Messages.ToString(), "result3", DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(8)));
// 5 Should get added. It's within tolerance, but it's a duplicate
var actualResult5 = new TestOperationResult("sender", TestOperationResultType.Messages.ToString(), "result3", DateTime.UtcNow);
// 6 should get ignored
var actualResult6 = new TestOperationResult("sender", TestOperationResultType.Messages.ToString(), "result4", DateTime.UtcNow);
return(Task.FromResult(new List <TestOperationResult> {
actualResult1, actualResult2, actualResult3, actualResult4, actualResult5, actualResult6
}));
}
void ValidateResult(
TestOperationResult result,
string expectedSource,
string testOperationResultType)
{
if (!result.Source.Equals(expectedSource, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidDataException($"Result source is '{result.Source}' but expected should be '{expectedSource}'.");
}
if (!result.Type.Equals(testOperationResultType, StringComparison.Ordinal))
{
throw new InvalidDataException($"Result type is '{result.Type}' but expected should be '{testOperationResultType}'.");
}
}
UnmatchedResultCounts CheckUnmatchedResult(
TestOperationResult testOperationResult,
NetworkControllerStatus networkControllerStatus,
bool isWithinTolerancePeriod)
{
ulong networkOffSuccess = 0;
ulong networkOnToleratedSuccess = 0;
ulong networkOnFailure = 0;
ulong mismatchSuccess = 0;
// int statusCodeInt = Int32.Parse(testOperationResult.Result.Split(";")[3]);
// HttpStatusCode statusCode = (HttpStatusCode)statusCodeInt;
DirectMethodTestResult dmTestResult = JsonConvert.DeserializeObject <DirectMethodTestResult>(testOperationResult.Result);
HttpStatusCode statusCode = JsonConvert.DeserializeObject <HttpStatusCode>(dmTestResult.Result);
if (HttpStatusCode.InternalServerError.Equals(statusCode))
{
if (NetworkControllerStatus.Enabled.Equals(networkControllerStatus))
{
// If the result is a failure AND network is offline, succeed
networkOffSuccess++;
}
else if (NetworkControllerStatus.Disabled.Equals(networkControllerStatus))
{
if (isWithinTolerancePeriod)
{
// If result is a failure and network is online, but we're within the tolerance period, succeed
networkOnToleratedSuccess++;
}
else
{
networkOnFailure++;
}
}
else
{
throw new InvalidOperationException($"Unexpected Result. NetworkControllerStatus was {networkControllerStatus}");
}
}
else
{
// Success, but no matching report from Actual store, means mismatch
mismatchSuccess++;
}
return(new UnmatchedResultCounts(networkOffSuccess, networkOnToleratedSuccess, networkOnFailure, mismatchSuccess));
}
/// <summary>
/// Compare 2 data stores and counting expect, match, and duplicate results; and return a counting report.
/// It will remove consecutive duplicate results when loading from actual store.
/// It will log fail if actual store has more results than expect store.
/// </summary>
/// <returns>Test Result Report.</returns>
public async Task <ITestResultReport> CreateReportAsync()
{
Logger.LogInformation($"Start to generate report by {nameof(CountingReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]");
TestOperationResult lastLoadedResult = default(TestOperationResult);
ulong totalExpectCount = 0;
ulong totalMatchCount = 0;
ulong totalDuplicateResultCount = 0;
List <TestOperationResult> unmatchedResults = new List <TestOperationResult>();
bool hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
bool hasActualResult = await this.ActualTestResults.MoveNextAsync();
while (hasExpectedResult && hasActualResult)
{
this.ValidateDataSource(this.ExpectedTestResults.Current, this.ExpectedSource);
this.ValidateDataSource(this.ActualTestResults.Current, this.ActualSource);
while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
{
totalDuplicateResultCount++;
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
totalExpectCount++;
if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
{
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
totalMatchCount++;
}
else
{
unmatchedResults.Add(this.ExpectedTestResults.Current);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
}
}
while (hasExpectedResult)
{
if (this.TestResultComparer.Matches(lastLoadedResult, this.ExpectedTestResults.Current))
{
totalDuplicateResultCount++;
lastLoadedResult = this.ExpectedTestResults.Current;
}
else
{
unmatchedResults.Add(this.ExpectedTestResults.Current);
}
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
totalExpectCount++;
}
while (hasActualResult)
{
// Log message for unexpected case.
Logger.LogError($"[{nameof(CountingReportGenerator)}] Actual test result source has unexpected results.");
hasActualResult = await this.ActualTestResults.MoveNextAsync();
// Log actual queue items
Logger.LogError($"Unexpected actual test result: {this.ActualTestResults.Current.Source}, {this.ActualTestResults.Current.Type}, {this.ActualTestResults.Current.Result} at {this.ActualTestResults.Current.CreatedAt}");
}
return(new CountingReport <TestOperationResult>(
this.trackingId,
this.ExpectedSource,
this.ActualSource,
this.ResultType,
totalExpectCount,
totalMatchCount,
totalDuplicateResultCount,
unmatchedResults.AsReadOnly()));
}
static Option <NetworkControllerTestResult> GetNetworkControllerTestOperationResult(TestOperationResult current)
{
if (!current.Type.Equals(TestOperationResultType.Network.ToString(), StringComparison.OrdinalIgnoreCase))
{
return(Option.None <NetworkControllerTestResult>());
}
Logger.LogDebug($"Deserializing for source {current.Source} result: {current.Result} {current.Type}");
NetworkControllerTestResult networkControllerTestResult = JsonConvert.DeserializeObject <NetworkControllerTestResult>(current.Result);
return(Option.Some(networkControllerTestResult));
}
/// <summary>
/// Compare 2 data stores and counting expected, actual, and matched results; and return a deployment test report.
/// Actual deployment results can be less than expected deployment results.
/// An actaul deployment test result is possible to use for verification of more than 1 expected deployment test result.
/// It will log fail if actual store has more results than expect store.
/// </summary>
/// <returns>Test Result Report.</returns>
public async Task <ITestResultReport> CreateReportAsync()
{
Logger.LogInformation($"Start to generate report by {nameof(DeploymentTestReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]");
TestOperationResult lastActualDeploymentTestResult = null;
ulong totalExpectedDeployments = 0;
ulong totalActualDeployments = 0;
ulong totalMatchedDeployments = 0;
var unmatchedResults = new List <TestOperationResult>();
bool hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
if (hasExpectedResult)
{
totalExpectedDeployments++;
}
bool hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
lastActualDeploymentTestResult = this.ActualTestResults.Current;
totalActualDeployments++;
}
while (hasExpectedResult && hasActualResult)
{
this.ValidateDataSource(this.ExpectedTestResults.Current, this.ExpectedSource);
this.ValidateDataSource(this.ActualTestResults.Current, this.ActualSource);
if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
{
totalMatchedDeployments++;
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
if (hasExpectedResult)
{
totalExpectedDeployments++;
}
}
else
{
hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
totalActualDeployments++;
lastActualDeploymentTestResult = this.ActualTestResults.Current;
}
}
}
while (hasExpectedResult)
{
unmatchedResults.Add(this.ExpectedTestResults.Current);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
if (hasExpectedResult)
{
totalExpectedDeployments++;
}
}
hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
totalActualDeployments++;
lastActualDeploymentTestResult = this.ActualTestResults.Current;
while (hasActualResult)
{
// Log message for unexpected case.
Logger.LogError($"[{nameof(DeploymentTestReportGenerator)}] Actual test result source has unexpected results.");
// Log actual queue items
Logger.LogError($"Unexpected actual test result: {this.ActualTestResults.Current.Source}, {this.ActualTestResults.Current.Type}, {this.ActualTestResults.Current.Result} at {this.ActualTestResults.Current.CreatedAt}");
hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
lastActualDeploymentTestResult = this.ActualTestResults.Current;
totalActualDeployments++;
}
}
}
return(new DeploymentTestReport(
this.trackingId,
this.ExpectedSource,
this.ActualSource,
this.ResultType,
totalExpectedDeployments,
totalActualDeployments,
totalMatchedDeployments,
Option.Maybe(lastActualDeploymentTestResult),
unmatchedResults.AsReadOnly()));
}
public async Task <StatusCodeResult> PostAsync(TestOperationResult result)
{
await ReportingCache.Instance.AddResultAsync(result);
return(this.StatusCode((int)HttpStatusCode.NoContent));
}
