public async Task <ITestResultReport> CreateReportAsync()
{
Logger.LogInformation($"Start to generate report by {nameof(TwinCountingReportGenerator)} for Sources [{this.expectedSource}] and [{this.actualSource}]");
ulong totalExpectCount = 0;
ulong totalMatchCount = 0;
ulong totalPatches = 0;
ulong totalDuplicates = 0;
Queue <string> unmatchedResults = new Queue <string>();
Dictionary <string, DateTime> propertiesUpdated = new Dictionary <string, DateTime>();
Dictionary <string, DateTime> propertiesReceived = new Dictionary <string, DateTime>();
while (await this.expectedTestResults.MoveNextAsync())
{
Option <TwinTestResult> twinTestResult = this.GetTwinTestResult(this.expectedTestResults.Current);
Logger.LogDebug($"Expected test results {twinTestResult}");
twinTestResult.ForEach(
r =>
{
foreach (var prop in r.Properties)
{
propertiesUpdated.TryAdd(prop.ToString(), this.expectedTestResults.Current.CreatedAt);
}
});
}
while (await this.actualTestResults.MoveNextAsync())
{
totalPatches++;
Option <TwinTestResult> twinTestResult = this.GetTwinTestResult(this.actualTestResults.Current);
Logger.LogDebug($"Actual test results {twinTestResult}");
twinTestResult.ForEach(
r =>
{
foreach (var prop in r.Properties)
{
bool added = propertiesReceived.TryAdd(prop.ToString(), this.actualTestResults.Current.CreatedAt);
if (!added)
{
Logger.LogDebug($"Duplicate for {this.actualSource} {prop.ToString()}");
totalDuplicates++;
}
}
});
}
foreach (KeyValuePair <string, DateTime> desiredPropertyUpdate in propertiesUpdated)
{
totalExpectCount++;
if (propertiesReceived.ContainsKey(desiredPropertyUpdate.Key))
{
totalMatchCount++;
}
else
{
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, $"{this.expectedSource} {desiredPropertyUpdate.Key}", this.unmatchedResultsMaxSize);
}
}
foreach (KeyValuePair <string, DateTime> desiredPropertyReceived in propertiesReceived)
{
if (!propertiesUpdated.ContainsKey(desiredPropertyReceived.Key))
{
Logger.LogError($"[{nameof(TwinCountingReportGenerator)}] Actual test result source has unexpected results.");
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, $"{this.actualSource} {desiredPropertyReceived.Key}", this.unmatchedResultsMaxSize);
}
}
return(new TwinCountingReport(
this.testDescription,
this.trackingId,
this.expectedSource,
this.actualSource,
this.resultType,
totalExpectCount,
totalMatchCount,
totalPatches,
totalDuplicates,
new List <string>(unmatchedResults).AsReadOnly()));
}
/// <summary>
/// Compare 2 data stores and counting expect, match, and duplicate results; and return a counting report.
/// It will remove consecutive duplicate results when loading from actual store.
/// It will log fail if actual store has more results than expect store.
/// </summary>
/// <returns>Test Result Report.</returns>
public async Task <ITestResultReport> CreateReportAsync()
{
Logger.LogInformation($"Start to generate report by {nameof(CountingReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]");
var lastLoadedResult = default(TestOperationResult);
ulong totalExpectCount = 0;
ulong totalMatchCount = 0;
ulong totalDuplicateResultCount = 0;
var unmatchedResults = new Queue <TestOperationResult>();
bool hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
bool hasActualResult = await this.ActualTestResults.MoveNextAsync();
while (hasExpectedResult && hasActualResult)
{
this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource);
this.ValidateResult(this.ActualTestResults.Current, this.ActualSource);
// Skip any duplicate actual value
while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
{
totalDuplicateResultCount++;
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
totalExpectCount++;
if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
{
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
totalMatchCount++;
}
else
{
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
}
}
// Check duplicates at the end of actual results
while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
{
totalDuplicateResultCount++;
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
while (hasExpectedResult)
{
totalExpectCount++;
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
}
while (hasActualResult)
{
// Log message for unexpected case.
Logger.LogError($"[{nameof(CountingReportGenerator)}] Actual test result source has unexpected results.");
// Log actual queue items
Logger.LogError($"Unexpected actual test result: {this.ActualTestResults.Current.Source}, {this.ActualTestResults.Current.Type}, {this.ActualTestResults.Current.Result} at {this.ActualTestResults.Current.CreatedAt}");
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
return(new CountingReport(
this.TestDescription,
this.trackingId,
this.ExpectedSource,
this.ActualSource,
this.ResultType,
totalExpectCount,
totalMatchCount,
totalDuplicateResultCount,
new List <TestOperationResult>(unmatchedResults).AsReadOnly()));
}
/// <summary>
/// Compare 2 data stores and counting expect, match, and duplicate results; and return a counting report.
/// It will remove consecutive duplicate results when loading from actual store.
/// It will log fail if actual store has more results than expect store.
/// </summary>
/// <returns>Test Result Report.</returns>
public async Task <ITestResultReport> CreateReportAsync()
{
Logger.LogInformation($"Start to generate report by {nameof(CountingReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]");
var lastLoadedResult = default(TestOperationResult);
ulong totalExpectCount = 0;
ulong totalMatchCount = 0;
ulong totalDuplicateResultCount = 0;
var unmatchedResults = new Queue <TestOperationResult>();
bool allActualResultsMatch = false;
Option <EventHubSpecificReportComponents> eventHubSpecificReportComponents = Option.None <EventHubSpecificReportComponents>();
Option <DateTime> lastLoadedResultCreatedAt = Option.None <DateTime>();
bool hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
bool hasActualResult = await this.ActualTestResults.MoveNextAsync();
while (hasExpectedResult && hasActualResult)
{
this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource);
this.ValidateResult(this.ActualTestResults.Current, this.ActualSource);
// Skip any duplicate actual value
while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
{
totalDuplicateResultCount++;
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
totalExpectCount++;
if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
{
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
totalMatchCount++;
}
else
{
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
}
}
// Check duplicates at the end of actual results
while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
{
totalDuplicateResultCount++;
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
allActualResultsMatch = totalExpectCount == totalMatchCount;
while (hasExpectedResult)
{
totalExpectCount++;
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
}
if (this.EventHubLongHaulMode)
{
bool stillReceivingFromEventHub = false;
// If we are are using EventHub to receive messages, we see an issue where EventHub can accrue large delays after
// running for a while. Therefore, if we are using EventHub with this counting report, we do two things.
// 1. Match only actual results. We still report all expected results, but matching actual results only.
// 2. We make sure that the last result we got from EventHub (i.e. the lastLoadedResult) is within our defined tolerance period.
// 'eventHubDelayTolerance' is an arbitrary tolerance period that we have defined, and can be tuned as needed.
// TODO: There is either something wrong with the EventHub service or something wrong with the way we are using it,
// Because we should not be accruing such large delays. If we move off EventHub, we should fix this as well.
TimeSpan eventHubDelayTolerance = Settings.Current.LongHaulSpecificSettings
.Expect <ArgumentException>(
() => throw new ArgumentException("TRC must be in long haul mode to be generating an EventHubLongHaul CountingReport"))
.EventHubDelayTolerance;
if (lastLoadedResult == null || lastLoadedResult.CreatedAt < DateTime.UtcNow - eventHubDelayTolerance)
{
stillReceivingFromEventHub = false;
}
else
{
stillReceivingFromEventHub = true;
}
eventHubSpecificReportComponents = Option.Some(new EventHubSpecificReportComponents
{
StillReceivingFromEventHub = stillReceivingFromEventHub,
AllActualResultsMatch = allActualResultsMatch
});
}
while (hasActualResult)
{
// Log message for unexpected case.
Logger.LogError($"[{nameof(CountingReportGenerator)}] Actual test result source has unexpected results.");
// Log actual queue items
Logger.LogError($"Unexpected actual test result: {this.ActualTestResults.Current.Source}, {this.ActualTestResults.Current.Type}, {this.ActualTestResults.Current.Result} at {this.ActualTestResults.Current.CreatedAt}");
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
if (lastLoadedResult != null)
{
lastLoadedResultCreatedAt = Option.Some(lastLoadedResult.CreatedAt);
}
return(new CountingReport(
this.TestDescription,
this.trackingId,
this.ExpectedSource,
this.ActualSource,
this.ResultType,
totalExpectCount,
totalMatchCount,
totalDuplicateResultCount,
new List <TestOperationResult>(unmatchedResults).AsReadOnly(),
eventHubSpecificReportComponents,
lastLoadedResultCreatedAt));
}
/// <summary>
/// Compare 2 data stores and counting expected, actual, and matched results; and return a deployment test report.
/// Actual deployment results can be less than expected deployment results.
/// An actaul deployment test result is possible to use for verification of more than 1 expected deployment test result.
/// It will log fail if actual store has more results than expect store.
/// </summary>
/// <returns>Test Result Report.</returns>
public async Task <ITestResultReport> CreateReportAsync()
{
Logger.LogInformation($"Start to generate report by {nameof(DeploymentTestReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]");
TestOperationResult lastActualDeploymentTestResult = null;
ulong totalExpectedDeployments = 0;
ulong totalActualDeployments = 0;
ulong totalMatchedDeployments = 0;
var unmatchedResults = new Queue <TestOperationResult>();
bool hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
if (hasExpectedResult)
{
totalExpectedDeployments++;
}
bool hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
lastActualDeploymentTestResult = this.ActualTestResults.Current;
totalActualDeployments++;
}
while (hasExpectedResult && hasActualResult)
{
this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource);
this.ValidateResult(this.ActualTestResults.Current, this.ActualSource);
if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
{
totalMatchedDeployments++;
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
if (hasExpectedResult)
{
totalExpectedDeployments++;
}
}
else
{
hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
totalActualDeployments++;
lastActualDeploymentTestResult = this.ActualTestResults.Current;
}
}
}
while (hasExpectedResult)
{
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
if (hasExpectedResult)
{
totalExpectedDeployments++;
}
}
hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
totalActualDeployments++;
lastActualDeploymentTestResult = this.ActualTestResults.Current;
while (hasActualResult)
{
// Log message for unexpected case.
Logger.LogError($"[{nameof(DeploymentTestReportGenerator)}] Actual test result source has unexpected results.");
// Log actual queue items
Logger.LogError($"Unexpected actual test result: {this.ActualTestResults.Current.Source}, {this.ActualTestResults.Current.Type}, {this.ActualTestResults.Current.Result} at {this.ActualTestResults.Current.CreatedAt}");
hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
lastActualDeploymentTestResult = this.ActualTestResults.Current;
totalActualDeployments++;
}
}
}
return(new DeploymentTestReport(
this.TestDescription,
this.trackingId,
this.ExpectedSource,
this.ActualSource,
this.ResultType,
totalExpectedDeployments,
totalActualDeployments,
totalMatchedDeployments,
Option.Maybe(lastActualDeploymentTestResult),
new List <TestOperationResult>(unmatchedResults).AsReadOnly()));
}