public void ParseReportMetadataList_ParseTwinCountingReportMetadata()
{
const string testDataJson =
@"{
""reportMetadata"": {
""TestDescription"": ""twin | desired property | amqp"",
""TestReportType"": ""TwinCountingReport"",
""TwinTestPropertyType"": ""Desired"",
""ExpectedSource"": ""twinTester1.desiredUpdated"",
""ActualSource"": ""twinTester2.desiredReceived""
}
}";
List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
Assert.Single(results);
var reportMetadata = results[0] as TwinCountingReportMetadata;
Assert.NotNull(reportMetadata);
Assert.Equal("twin | desired property | amqp", reportMetadata.TestDescription);
Assert.Equal(TestOperationResultType.Twin, reportMetadata.TestOperationResultType);
Assert.Equal(TestReportType.TwinCountingReport, reportMetadata.TestReportType);
Assert.Equal(TwinTestPropertyType.Desired, reportMetadata.TwinTestPropertyType);
Assert.Equal("twinTester1.desiredUpdated", reportMetadata.ExpectedSource);
Assert.Equal("twinTester2.desiredReceived", reportMetadata.ActualSource);
}
public void ParseReportMetadataList_ParseDirectMethodConnectivityTestReportMetadataWithoutReceiverSource()
{
const string testDataJson =
@"{
""reportMetadata"": {
""TestDescription"": ""edge agent ping"",
""TestReportType"": ""DirectMethodConnectivityReport"",
""SenderSource"": ""directMethodSender1.send"",
""TolerancePeriod"": ""00:00:00.005""
}
}";
List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
Assert.Single(results);
var reportMetadata = results[0] as DirectMethodConnectivityReportMetadata;
Assert.NotNull(reportMetadata);
Assert.Equal("edge agent ping", reportMetadata.TestDescription);
Assert.Equal(TestOperationResultType.DirectMethod, reportMetadata.TestOperationResultType);
Assert.Equal(TestReportType.DirectMethodConnectivityReport, reportMetadata.TestReportType);
Assert.Equal("directMethodSender1.send", reportMetadata.SenderSource);
Assert.False(reportMetadata.ReceiverSource.HasValue);
Assert.Equal(new TimeSpan(0, 0, 0, 0, 5), reportMetadata.TolerancePeriod);
}
public void ParseReportMetadataList_ParseDirectMethodTestReportMetadata()
{
const string testDataJson =
@"{
""reportMetadata"": {
""TestReportType"": ""DirectMethodReport"",
""SenderSource"": ""directMethodSender1.send"",
""ReceiverSource"": ""directMethodReceiver1.receive"",
""TolerancePeriod"": ""00:00:00.005""
}
}";
List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
Assert.Single(results);
var reportMetadata = results[0] as DirectMethodReportMetadata;
Assert.NotNull(reportMetadata);
Assert.Equal(TestOperationResultType.DirectMethod, reportMetadata.TestOperationResultType);
Assert.Equal(TestReportType.DirectMethodReport, reportMetadata.TestReportType);
Assert.Equal("directMethodSender1.send", reportMetadata.SenderSource);
Assert.True(reportMetadata.ReceiverSource.HasValue);
reportMetadata.ReceiverSource.ForEach(x => Assert.Equal("directMethodReceiver1.receive", x));
Assert.Equal(new TimeSpan(0, 0, 0, 0, 5), reportMetadata.TolerancePeriod);
}
public void ParseReportMetadataList_ParseEdgeHubRestartDirectMethodReportMetadata()
{
const string testDataJson =
@"{
""reportMetadata7"": {
""TestDescription"": ""direct method | cloud | amqp"",
""TestReportType"": ""EdgeHubRestartDirectMethodReport"",
""TestOperationResultType"": ""EdgeHubRestartDirectMethod"",
""SenderSource"": ""edgeHubRestartTester1.EdgeHubRestartDirectMethod"",
""ReceiverSource"": ""directMethodReceiver1.receive""
}
}";
List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
Assert.Single(results);
var reportMetadata = results[0] as EdgeHubRestartDirectMethodReportMetadata;
Assert.NotNull(reportMetadata);
Assert.Equal("direct method | cloud | amqp", reportMetadata.TestDescription);
Assert.Equal(TestOperationResultType.EdgeHubRestartDirectMethod, reportMetadata.TestOperationResultType);
Assert.Equal(TestReportType.EdgeHubRestartDirectMethodReport, reportMetadata.TestReportType);
Assert.Equal("edgeHubRestartTester1.EdgeHubRestartDirectMethod", reportMetadata.SenderSource);
Assert.Equal("directMethodReceiver1.receive", reportMetadata.ReceiverSource);
}
public void ParseReportMetadataList_ThrowExceptionWhenParseInvalidData()
{
const string testDataJson =
@"{
""reportMetadata"": {
""TestDescription"": ""dummy"",
""TestReportType"": ""TypeNotExist"",
""ExpectedSource"": ""deploymentTester1.send"",
""ActualSource"": ""deploymentTester2.receive""
}
}";
bool exceptionThrown = false;
try
{
TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
}
catch (Exception)
{
exceptionThrown = true;
}
Assert.True(exceptionThrown);
}
async void DoWorkAsync(object state)
{
var tesReportGeneratorFactory = new TestReportGeneratorFactory(this.storage);
List <ITestReportMetadata> reportMetadataList = await Settings.Current.GetReportMetadataListAsync(this.logger);
ITestResultReport[] testResultReports = await TestReportUtil.GenerateTestResultReportsAsync(Settings.Current.TrackingId, reportMetadataList, tesReportGeneratorFactory, this.logger);
if (testResultReports.Length == 0)
{
this.logger.LogInformation("No test result report is generated.");
return;
}
var testSummary = new TestSummary(testResultReports);
string reportsContent = JsonConvert.SerializeObject(testSummary, Formatting.Indented);
this.logger.LogInformation($"Test result report{Environment.NewLine}{reportsContent}");
await AzureLogAnalytics.Instance.PostAsync(
Settings.Current.LogAnalyticsWorkspaceId,
Settings.Current.LogAnalyticsSharedKey,
reportsContent,
Settings.Current.LogAnalyticsLogType);
this.logger.LogInformation("Successfully send reports to LogAnalytics");
}
public void ParseReportMetadataList_MultipleReportMetadata()
{
const string testDataJson =
@"{
""reportMetadata1"": {
""TestReportType"": ""CountingReport"",
""TestOperationResultType"": ""Messages"",
""ExpectedSource"": ""loadGen1.send"",
""ActualSource"": ""relayer1.receive""
},
""reportMetadata2"": {
""TestReportType"": ""TwinCountingReport"",
""TwinTestPropertyType"": ""Desired"",
""ExpectedSource"": ""twinTester1.desiredUpdated"",
""ActualSource"": ""twinTester2.desiredReceived""
},
""reportMetadata3"": {
""TestReportType"": ""DeploymentTestReport"",
""ExpectedSource"": ""deploymentTester1.send"",
""ActualSource"": ""deploymentTester2.receive""
}
}";
List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
Assert.Equal(3, results.Count);
}
public void ParseReportMetadataList_ParseCountingReportMetadata()
{
const string testDataJson =
@"{
""reportMetadata"": {
""TestDescription"": ""messages | local | amqp"",
""TestReportType"": ""CountingReport"",
""TestOperationResultType"": ""Messages"",
""ExpectedSource"": ""loadGen1.send"",
""ActualSource"": ""relayer1.receive""
}
}";
List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
Assert.Single(results);
var reportMetadata = results[0] as CountingReportMetadata;
Assert.NotNull(reportMetadata);
Assert.Equal("messages | local | amqp", reportMetadata.TestDescription);
Assert.Equal(TestOperationResultType.Messages, reportMetadata.TestOperationResultType);
Assert.Equal(TestReportType.CountingReport, reportMetadata.TestReportType);
Assert.Equal("loadGen1.send", reportMetadata.ExpectedSource);
Assert.Equal("relayer1.receive", reportMetadata.ActualSource);
}
public void ParseReportMetadataList_ParseDirectMethodLongHaulTestReportMetadata()
{
const string testDataJson =
@"{
""reportMetadata"": {
""TestDescription"": ""direct method | cloud | amqp"",
""TestReportType"": ""DirectMethodLongHaulReport"",
""SenderSource"": ""directMethodSender1.send"",
""ReceiverSource"": ""directMethodReceiver1.receive""
}
}";
List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
Assert.Single(results);
var reportMetadata = results[0] as DirectMethodLongHaulReportMetadata;
Assert.NotNull(reportMetadata);
Assert.Equal("direct method | cloud | amqp", reportMetadata.TestDescription);
Assert.Equal(TestOperationResultType.DirectMethod, reportMetadata.TestOperationResultType);
Assert.Equal(TestReportType.DirectMethodLongHaulReport, reportMetadata.TestReportType);
Assert.Equal("directMethodSender1.send", reportMetadata.SenderSource);
Assert.True(reportMetadata.ReceiverSource.HasValue);
reportMetadata.ReceiverSource.ForEach(x => Assert.Equal("directMethodReceiver1.receive", x));
}
public async Task TestGenerateTestResultReportsAsync_ReportGeneration(
bool throwExceptionForTestReport1,
bool throwExceptionForTestReport2,
bool throwExceptionForTestReport3,
bool throwExceptionForTestReport4,
bool throwExceptionForTestReport5,
int expectedReportCount)
{
var mockLogger = new Mock <ILogger>();
var mockTestReportGeneratorFactory = new Mock <ITestReportGeneratorFactory>();
string trackingId = "fakeTrackingId";
var countingReportMetadata = new CountingReportMetadata("CountingExpectedSource", "CountingAcutalSource", TestOperationResultType.Messages, TestReportType.CountingReport);
var twinCountingReportMetadata = new TwinCountingReportMetadata("TwinExpectedSource", "TwinActualSource", TestReportType.TwinCountingReport, TwinTestPropertyType.Desired);
var deploymentReportMetadata = new DeploymentTestReportMetadata("DeploymentExpectedSource", "DeploymentActualSource");
var directMethodReportMetadata = new DirectMethodReportMetadata("DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource");
var directMethodReportMetadataWithoutReceiverSource = new DirectMethodReportMetadata("DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource");
var mockTestReportGenerator1 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator1.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport1));
var mockTestReportGenerator2 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator2.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport2));
var mockTestReportGenerator3 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator3.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport3));
var mockTestReportGenerator4 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator4.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport4));
var mockTestReportGenerator5 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator5.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport5));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, countingReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator1.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, twinCountingReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator2.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, deploymentReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator3.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator4.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodReportMetadataWithoutReceiverSource)).Returns(Task.FromResult(mockTestReportGenerator5.Object));
ITestResultReport[] reports = await TestReportUtil.GenerateTestResultReportsAsync(
trackingId,
new List <ITestReportMetadata>
{
countingReportMetadata,
twinCountingReportMetadata,
deploymentReportMetadata,
directMethodReportMetadata,
directMethodReportMetadataWithoutReceiverSource
},
mockTestReportGeneratorFactory.Object,
mockLogger.Object);
Assert.Equal(expectedReportCount, reports.Length);
}
public void ParseReportMetadataList_MultipleReportMetadata()
{
const string testDataJson =
@"{
""reportMetadata1"": {
""TestDescription"": ""messages | local | amqp"",
""TestReportType"": ""CountingReport"",
""TestOperationResultType"": ""Messages"",
""ExpectedSource"": ""loadGen1.send"",
""ActualSource"": ""relayer1.receive""
},
""reportMetadata2"": {
""TestDescription"": ""twin | desired property | amqp"",
""TestReportType"": ""TwinCountingReport"",
""TwinTestPropertyType"": ""Desired"",
""ExpectedSource"": ""twinTester1.desiredUpdated"",
""ActualSource"": ""twinTester2.desiredReceived""
},
""reportMetadata3"": {
""TestDescription"": ""deployment"",
""TestReportType"": ""DeploymentTestReport"",
""ExpectedSource"": ""deploymentTester1.send"",
""ActualSource"": ""deploymentTester2.receive""
},
""reportMetadata4"": {
""TestDescription"": ""direct method | cloud | amqp"",
""TestReportType"": ""DirectMethodConnectivityReport"",
""SenderSource"": ""senderSource1.send"",
""ReceiverSource"": ""receiverSource1.receive"",
""TolerancePeriod"": ""00:00:00.005""
},
""reportMetadata5"": {
""TestDescription"": ""edge agent ping"",
""TestReportType"": ""DirectMethodConnectivityReport"",
""SenderSource"": ""senderSource1.send"",
""TolerancePeriod"": ""00:00:00.005""
},
""reportMetadata6"": {
""TestDescription"": ""messages | local | amqp | restart"",
""TestReportType"": ""EdgeHubRestartMessageReport"",
""TestOperationResultType"": ""EdgeHubRestartMessage"",
""SenderSource"": ""edgeHubRestartTester1.EdgeHubRestartMessage"",
""ReceiverSource"": ""relayer1.receive""
},
""reportMetadata7"": {
""TestDescription"": ""direct method | cloud | amqp | restart"",
""TestReportType"": ""EdgeHubRestartDirectMethodReport"",
""TestOperationResultType"": ""EdgeHubRestartDirectMethod"",
""SenderSource"": ""edgeHubRestartTester1.EdgeHubRestartDirectMethod"",
""ReceiverSource"": ""directMethodReceiver1.receive""
}
}";
List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
Assert.Equal(7, results.Count);
}
public async Task TestGenerateTestResultReportsAsync_MissingTrackingId()
{
var mockLogger = new Mock <ILogger>();
var mockTestReportGeneratorFactory = new Mock <ITestReportGeneratorFactory>();
ArgumentException ex = await Assert.ThrowsAsync <ArgumentException>(() =>
TestReportUtil.GenerateTestResultReportsAsync(
string.Empty,
new List <ITestReportMetadata>(),
mockTestReportGeneratorFactory.Object,
mockLogger.Object));
Assert.StartsWith("trackingId", ex.Message, StringComparison.Ordinal);
}
async void DoWorkAsync(object state)
{
var testReportGeneratorFactory = new TestReportGeneratorFactory(this.storage, Settings.Current.NetworkControllerType, Settings.Current.LongHaulSpecificSettings);
List <ITestReportMetadata> reportMetadataList = await Settings.Current.GetReportMetadataListAsync(this.logger);
ITestResultReport[] testResultReports = await TestReportUtil.GenerateTestResultReportsAsync(Settings.Current.TrackingId, reportMetadataList, testReportGeneratorFactory, this.logger);
if (testResultReports.Length == 0)
{
this.logger.LogInformation("No test result report is generated.");
return;
}
string blobContainerUri = string.Empty;
if (this.serviceSpecificSettings.LogUploadEnabled)
{
try
{
Uri blobContainerWriteUriForLog = await TestReportUtil.GetOrCreateBlobContainerSasUriForLogAsync(this.serviceSpecificSettings.StorageAccountConnectionString);
blobContainerUri = $"{blobContainerWriteUriForLog.Scheme}{Uri.SchemeDelimiter}{blobContainerWriteUriForLog.Authority}{blobContainerWriteUriForLog.AbsolutePath}";
await TestReportUtil.UploadLogsAsync(Settings.Current.IoTHubConnectionString, blobContainerWriteUriForLog, this.logUploadDuration, this.logger);
}
catch (Exception ex)
{
this.logger.LogError(ex, "Exception happened when uploading logs");
}
}
var testSummary = new TestSummary(Settings.Current.TestInfo, testResultReports, blobContainerUri);
string reportsContent = JsonConvert.SerializeObject(testSummary, Formatting.Indented);
this.logger.LogInformation($"Test summary{Environment.NewLine}{reportsContent}");
await AzureLogAnalytics.Instance.PostAsync(
this.serviceSpecificSettings.LogAnalyticsWorkspaceId,
this.serviceSpecificSettings.LogAnalyticsSharedKey,
reportsContent,
this.serviceSpecificSettings.LogAnalyticsLogType);
this.logger.LogInformation("Successfully send reports to LogAnalytics");
}
public void ParseReportMetadataList_ParseTestInfoReportMetadata()
{
const string testDataJson =
@"{
""reportMetadata"": {
""TestReportType"": ""TestInfoReport""
}
}";
List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
Assert.Single(results);
var reportMetadata = results[0] as TestInfoReportMetadata;
Assert.NotNull(reportMetadata);
Assert.Equal(TestOperationResultType.TestInfo, reportMetadata.TestOperationResultType);
Assert.Equal(TestReportType.TestInfoReport, reportMetadata.TestReportType);
Assert.Equal(TestConstants.TestInfo.TestResultSource, reportMetadata.Source);
}
public async Task <ContentResult> GetReportsAsync()
{
var testReportGeneratorFactory = new TestReportGeneratorFactory(this.storage);
List <ITestReportMetadata> reportMetadataList = await Settings.Current.GetReportMetadataListAsync(Logger);
ITestResultReport[] testResultReports = await TestReportUtil.GenerateTestResultReportsAsync(Settings.Current.TrackingId, reportMetadataList, testReportGeneratorFactory, Logger);
if (testResultReports.Length == 0)
{
return(new ContentResult {
Content = "No test report generated"
});
}
return(new ContentResult
{
Content = this.AddManualRunReportingHeading(JsonConvert.SerializeObject(testResultReports, Formatting.Indented)) // explicit serialization needed due to the wrapping list
});
}
public void ParseReportMetadataList_ParseNetworkControllerReportMetadata()
{
const string testDataJson =
@"{
""reportMetadata"": {
""TestReportType"": ""NetworkControllerReport"",
""Source"": ""networkController""
}
}";
List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
Assert.Single(results);
var reportMetadata = results[0] as NetworkControllerReportMetadata;
Assert.NotNull(reportMetadata);
Assert.Equal(TestOperationResultType.Network, reportMetadata.TestOperationResultType);
Assert.Equal(TestReportType.NetworkControllerReport, reportMetadata.TestReportType);
Assert.Equal("networkController", reportMetadata.Source);
}
public void ParseReportMetadataList_ParseDeploymentTestReportMetadata()
{
const string testDataJson =
@"{
""reportMetadata"": {
""TestReportType"": ""DeploymentTestReport"",
""ExpectedSource"": ""deploymentTester1.send"",
""ActualSource"": ""deploymentTester2.receive""
}
}";
List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
Assert.Single(results);
var reportMetadata = results[0] as DeploymentTestReportMetadata;
Assert.NotNull(reportMetadata);
Assert.Equal(TestOperationResultType.Deployment, reportMetadata.TestOperationResultType);
Assert.Equal(TestReportType.DeploymentTestReport, reportMetadata.TestReportType);
Assert.Equal("deploymentTester1.send", reportMetadata.ExpectedSource);
Assert.Equal("deploymentTester2.receive", reportMetadata.ActualSource);
}
public async Task <ITestResultReport> CreateReportAsync()
{
Logger.LogInformation($"Start to generate report by {nameof(TwinCountingReportGenerator)} for Sources [{this.expectedSource}] and [{this.actualSource}]");
ulong totalExpectCount = 0;
ulong totalMatchCount = 0;
ulong totalPatches = 0;
ulong totalDuplicates = 0;
Queue <string> unmatchedResults = new Queue <string>();
Dictionary <string, DateTime> propertiesUpdated = new Dictionary <string, DateTime>();
Dictionary <string, DateTime> propertiesReceived = new Dictionary <string, DateTime>();
while (await this.expectedTestResults.MoveNextAsync())
{
Option <TwinTestResult> twinTestResult = this.GetTwinTestResult(this.expectedTestResults.Current);
Logger.LogDebug($"Expected test results {twinTestResult}");
twinTestResult.ForEach(
r =>
{
foreach (var prop in r.Properties)
{
propertiesUpdated.TryAdd(prop.ToString(), this.expectedTestResults.Current.CreatedAt);
}
});
}
while (await this.actualTestResults.MoveNextAsync())
{
totalPatches++;
Option <TwinTestResult> twinTestResult = this.GetTwinTestResult(this.actualTestResults.Current);
Logger.LogDebug($"Actual test results {twinTestResult}");
twinTestResult.ForEach(
r =>
{
foreach (var prop in r.Properties)
{
bool added = propertiesReceived.TryAdd(prop.ToString(), this.actualTestResults.Current.CreatedAt);
if (!added)
{
Logger.LogDebug($"Duplicate for {this.actualSource} {prop.ToString()}");
totalDuplicates++;
}
}
});
}
foreach (KeyValuePair <string, DateTime> desiredPropertyUpdate in propertiesUpdated)
{
totalExpectCount++;
if (propertiesReceived.ContainsKey(desiredPropertyUpdate.Key))
{
totalMatchCount++;
}
else
{
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, $"{this.expectedSource} {desiredPropertyUpdate.Key}", this.unmatchedResultsMaxSize);
}
}
foreach (KeyValuePair <string, DateTime> desiredPropertyReceived in propertiesReceived)
{
if (!propertiesUpdated.ContainsKey(desiredPropertyReceived.Key))
{
Logger.LogError($"[{nameof(TwinCountingReportGenerator)}] Actual test result source has unexpected results.");
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, $"{this.actualSource} {desiredPropertyReceived.Key}", this.unmatchedResultsMaxSize);
}
}
return(new TwinCountingReport(
this.testDescription,
this.trackingId,
this.expectedSource,
this.actualSource,
this.resultType,
totalExpectCount,
totalMatchCount,
totalPatches,
totalDuplicates,
new List <string>(unmatchedResults).AsReadOnly()));
}
/// <summary>
/// Compare 2 data stores and counting expect, match, and duplicate results; and return a counting report.
/// It will remove consecutive duplicate results when loading from actual store.
/// It will log fail if actual store has more results than expect store.
/// </summary>
/// <returns>Test Result Report.</returns>
public async Task <ITestResultReport> CreateReportAsync()
{
Logger.LogInformation($"Start to generate report by {nameof(CountingReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]");
var lastLoadedResult = default(TestOperationResult);
ulong totalExpectCount = 0;
ulong totalMatchCount = 0;
ulong totalDuplicateResultCount = 0;
var unmatchedResults = new Queue <TestOperationResult>();
bool hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
bool hasActualResult = await this.ActualTestResults.MoveNextAsync();
while (hasExpectedResult && hasActualResult)
{
this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource);
this.ValidateResult(this.ActualTestResults.Current, this.ActualSource);
// Skip any duplicate actual value
while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
{
totalDuplicateResultCount++;
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
totalExpectCount++;
if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
{
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
totalMatchCount++;
}
else
{
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
}
}
// Check duplicates at the end of actual results
while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
{
totalDuplicateResultCount++;
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
while (hasExpectedResult)
{
totalExpectCount++;
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
}
while (hasActualResult)
{
// Log message for unexpected case.
Logger.LogError($"[{nameof(CountingReportGenerator)}] Actual test result source has unexpected results.");
// Log actual queue items
Logger.LogError($"Unexpected actual test result: {this.ActualTestResults.Current.Source}, {this.ActualTestResults.Current.Type}, {this.ActualTestResults.Current.Result} at {this.ActualTestResults.Current.CreatedAt}");
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
return(new CountingReport(
this.TestDescription,
this.trackingId,
this.ExpectedSource,
this.ActualSource,
this.ResultType,
totalExpectCount,
totalMatchCount,
totalDuplicateResultCount,
new List <TestOperationResult>(unmatchedResults).AsReadOnly()));
}
/// <summary>
/// Compare 2 data stores and counting expect, match, and duplicate results; and return a counting report.
/// It will remove consecutive duplicate results when loading from actual store.
/// It will log fail if actual store has more results than expect store.
/// </summary>
/// <returns>Test Result Report.</returns>
public async Task <ITestResultReport> CreateReportAsync()
{
Logger.LogInformation($"Start to generate report by {nameof(CountingReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]");
var lastLoadedResult = default(TestOperationResult);
ulong totalExpectCount = 0;
ulong totalMatchCount = 0;
ulong totalDuplicateResultCount = 0;
var unmatchedResults = new Queue <TestOperationResult>();
bool allActualResultsMatch = false;
Option <EventHubSpecificReportComponents> eventHubSpecificReportComponents = Option.None <EventHubSpecificReportComponents>();
Option <DateTime> lastLoadedResultCreatedAt = Option.None <DateTime>();
bool hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
bool hasActualResult = await this.ActualTestResults.MoveNextAsync();
while (hasExpectedResult && hasActualResult)
{
this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource);
this.ValidateResult(this.ActualTestResults.Current, this.ActualSource);
// Skip any duplicate actual value
while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
{
totalDuplicateResultCount++;
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
totalExpectCount++;
if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
{
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
totalMatchCount++;
}
else
{
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
}
}
// Check duplicates at the end of actual results
while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
{
totalDuplicateResultCount++;
lastLoadedResult = this.ActualTestResults.Current;
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
allActualResultsMatch = totalExpectCount == totalMatchCount;
while (hasExpectedResult)
{
totalExpectCount++;
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
}
if (this.EventHubLongHaulMode)
{
bool stillReceivingFromEventHub = false;
// If we are are using EventHub to receive messages, we see an issue where EventHub can accrue large delays after
// running for a while. Therefore, if we are using EventHub with this counting report, we do two things.
// 1. Match only actual results. We still report all expected results, but matching actual results only.
// 2. We make sure that the last result we got from EventHub (i.e. the lastLoadedResult) is within our defined tolerance period.
// 'eventHubDelayTolerance' is an arbitrary tolerance period that we have defined, and can be tuned as needed.
// TODO: There is either something wrong with the EventHub service or something wrong with the way we are using it,
// Because we should not be accruing such large delays. If we move off EventHub, we should fix this as well.
TimeSpan eventHubDelayTolerance = Settings.Current.LongHaulSpecificSettings
.Expect <ArgumentException>(
() => throw new ArgumentException("TRC must be in long haul mode to be generating an EventHubLongHaul CountingReport"))
.EventHubDelayTolerance;
if (lastLoadedResult == null || lastLoadedResult.CreatedAt < DateTime.UtcNow - eventHubDelayTolerance)
{
stillReceivingFromEventHub = false;
}
else
{
stillReceivingFromEventHub = true;
}
eventHubSpecificReportComponents = Option.Some(new EventHubSpecificReportComponents
{
StillReceivingFromEventHub = stillReceivingFromEventHub,
AllActualResultsMatch = allActualResultsMatch
});
}
while (hasActualResult)
{
// Log message for unexpected case.
Logger.LogError($"[{nameof(CountingReportGenerator)}] Actual test result source has unexpected results.");
// Log actual queue items
Logger.LogError($"Unexpected actual test result: {this.ActualTestResults.Current.Source}, {this.ActualTestResults.Current.Type}, {this.ActualTestResults.Current.Result} at {this.ActualTestResults.Current.CreatedAt}");
hasActualResult = await this.ActualTestResults.MoveNextAsync();
}
if (lastLoadedResult != null)
{
lastLoadedResultCreatedAt = Option.Some(lastLoadedResult.CreatedAt);
}
return(new CountingReport(
this.TestDescription,
this.trackingId,
this.ExpectedSource,
this.ActualSource,
this.ResultType,
totalExpectCount,
totalMatchCount,
totalDuplicateResultCount,
new List <TestOperationResult>(unmatchedResults).AsReadOnly(),
eventHubSpecificReportComponents,
lastLoadedResultCreatedAt));
}
public async Task TestGenerateTestResultReportsAsync_NullReportMetadata()
{
var mockLogger = new Mock <ILogger>();
var mockTestReportGeneratorFactory = new Mock <ITestReportGeneratorFactory>();
ArgumentNullException ex = await Assert.ThrowsAsync <ArgumentNullException>(() =>
TestReportUtil.GenerateTestResultReportsAsync(
"fakeTrackingId",
null,
mockTestReportGeneratorFactory.Object,
mockLogger.Object));
Assert.Equal("reportMetadatalist", ex.ParamName);
}
/// <summary>
/// Compare 2 data stores and counting expected, actual, and matched results; and return a deployment test report.
/// Actual deployment results can be less than expected deployment results.
/// An actaul deployment test result is possible to use for verification of more than 1 expected deployment test result.
/// It will log fail if actual store has more results than expect store.
/// </summary>
/// <returns>Test Result Report.</returns>
public async Task <ITestResultReport> CreateReportAsync()
{
Logger.LogInformation($"Start to generate report by {nameof(DeploymentTestReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]");
TestOperationResult lastActualDeploymentTestResult = null;
ulong totalExpectedDeployments = 0;
ulong totalActualDeployments = 0;
ulong totalMatchedDeployments = 0;
var unmatchedResults = new Queue <TestOperationResult>();
bool hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
if (hasExpectedResult)
{
totalExpectedDeployments++;
}
bool hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
lastActualDeploymentTestResult = this.ActualTestResults.Current;
totalActualDeployments++;
}
while (hasExpectedResult && hasActualResult)
{
this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource);
this.ValidateResult(this.ActualTestResults.Current, this.ActualSource);
if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
{
totalMatchedDeployments++;
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
if (hasExpectedResult)
{
totalExpectedDeployments++;
}
}
else
{
hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
totalActualDeployments++;
lastActualDeploymentTestResult = this.ActualTestResults.Current;
}
}
}
while (hasExpectedResult)
{
TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
if (hasExpectedResult)
{
totalExpectedDeployments++;
}
}
hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
totalActualDeployments++;
lastActualDeploymentTestResult = this.ActualTestResults.Current;
while (hasActualResult)
{
// Log message for unexpected case.
Logger.LogError($"[{nameof(DeploymentTestReportGenerator)}] Actual test result source has unexpected results.");
// Log actual queue items
Logger.LogError($"Unexpected actual test result: {this.ActualTestResults.Current.Source}, {this.ActualTestResults.Current.Type}, {this.ActualTestResults.Current.Result} at {this.ActualTestResults.Current.CreatedAt}");
hasActualResult = await this.ActualTestResults.MoveNextAsync();
if (hasActualResult)
{
lastActualDeploymentTestResult = this.ActualTestResults.Current;
totalActualDeployments++;
}
}
}
return(new DeploymentTestReport(
this.TestDescription,
this.trackingId,
this.ExpectedSource,
this.ActualSource,
this.ResultType,
totalExpectedDeployments,
totalActualDeployments,
totalMatchedDeployments,
Option.Maybe(lastActualDeploymentTestResult),
new List <TestOperationResult>(unmatchedResults).AsReadOnly()));
}
public async Task TestGenerateTestResultReportsAsync_ReportGeneration(
bool throwExceptionForTestReport1,
bool throwExceptionForTestReport2,
bool throwExceptionForTestReport3,
bool throwExceptionForTestReport4,
bool throwExceptionForTestReport5,
bool throwExceptionForTestReport6,
bool throwExceptionForTestReport7,
bool throwExceptionForTestReport8,
bool throwExceptionForTestReport9,
int expectedReportCount)
{
var mockLogger = new Mock <ILogger>();
var mockTestReportGeneratorFactory = new Mock <ITestReportGeneratorFactory>();
string trackingId = "fakeTrackingId";
var countingReportMetadata = new CountingReportMetadata(TestDescription, "CountingExpectedSource", "CountingAcutalSource", TestOperationResultType.Messages, TestReportType.CountingReport, false);
var twinCountingReportMetadata = new TwinCountingReportMetadata(TestDescription, "TwinExpectedSource", "TwinActualSource", TestReportType.TwinCountingReport, TwinTestPropertyType.Desired);
var deploymentReportMetadata = new DeploymentTestReportMetadata(TestDescription, "DeploymentExpectedSource", "DeploymentActualSource");
var directMethodConnectivityReportMetadata = new DirectMethodConnectivityReportMetadata(TestDescription, "DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource");
var directMethodConnectivityReportMetadataWithoutReceiverSource = new DirectMethodConnectivityReportMetadata(TestDescription, "DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource");
var directMethodLongHaulReportMetadata = new DirectMethodLongHaulReportMetadata(TestDescription, "DirectMethodSenderSource", "DirectMethodReceiverSource");
var directMethodLongHaulReportMetadataWithoutReceiverSource = new DirectMethodLongHaulReportMetadata(TestDescription, "DirectMethodSenderSource", "DirectMethodReceiverSource");
var edgeHubRestartMessageReportMetadata = new EdgeHubRestartMessageReportMetadata(TestDescription, "edgeHubRestartTester1.EdgeHubRestartMessage", "relayer1.receive");
var edgeHubRestartDirectMethodReportMetadata = new EdgeHubRestartDirectMethodReportMetadata(TestDescription, "edgeHubRestartTester1.EdgeHubRestartDirectMethod", "directMethodReceiver1.receive");
var mockTestReportGenerator1 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator1.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport1));
var mockTestReportGenerator2 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator2.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport2));
var mockTestReportGenerator3 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator3.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport3));
var mockTestReportGenerator4 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator4.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport4));
var mockTestReportGenerator5 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator5.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport5));
var mockTestReportGenerator6 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator6.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport6));
var mockTestReportGenerator7 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator7.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport7));
var mockTestReportGenerator8 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator8.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport8));
var mockTestReportGenerator9 = new Mock <ITestResultReportGenerator>();
mockTestReportGenerator9.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport9));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, countingReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator1.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, twinCountingReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator2.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, deploymentReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator3.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodConnectivityReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator4.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodConnectivityReportMetadataWithoutReceiverSource)).Returns(Task.FromResult(mockTestReportGenerator5.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodLongHaulReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator6.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodLongHaulReportMetadataWithoutReceiverSource)).Returns(Task.FromResult(mockTestReportGenerator7.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, edgeHubRestartMessageReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator8.Object));
mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, edgeHubRestartDirectMethodReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator9.Object));
ITestResultReport[] reports = await TestReportUtil.GenerateTestResultReportsAsync(
trackingId,
new List <ITestReportMetadata>
{
countingReportMetadata,
twinCountingReportMetadata,
deploymentReportMetadata,
directMethodConnectivityReportMetadata,
directMethodConnectivityReportMetadataWithoutReceiverSource,
directMethodLongHaulReportMetadata,
directMethodLongHaulReportMetadataWithoutReceiverSource,
edgeHubRestartMessageReportMetadata,
edgeHubRestartDirectMethodReportMetadata
},
mockTestReportGeneratorFactory.Object,
mockLogger.Object);
Assert.Equal(expectedReportCount, reports.Length);
}