コード例 #1
0
        public void ParseReportMetadataList_ParseTwinCountingReportMetadata()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata"": {
                        ""TestDescription"": ""twin | desired property | amqp"",
                        ""TestReportType"": ""TwinCountingReport"",
                        ""TwinTestPropertyType"": ""Desired"",
                        ""ExpectedSource"": ""twinTester1.desiredUpdated"",
                        ""ActualSource"": ""twinTester2.desiredReceived""
                    }
                }";

            List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);

            Assert.Single(results);
            var reportMetadata = results[0] as TwinCountingReportMetadata;

            Assert.NotNull(reportMetadata);
            Assert.Equal("twin | desired property | amqp", reportMetadata.TestDescription);
            Assert.Equal(TestOperationResultType.Twin, reportMetadata.TestOperationResultType);
            Assert.Equal(TestReportType.TwinCountingReport, reportMetadata.TestReportType);
            Assert.Equal(TwinTestPropertyType.Desired, reportMetadata.TwinTestPropertyType);
            Assert.Equal("twinTester1.desiredUpdated", reportMetadata.ExpectedSource);
            Assert.Equal("twinTester2.desiredReceived", reportMetadata.ActualSource);
        }
コード例 #2
0
        public void ParseReportMetadataList_ParseDirectMethodConnectivityTestReportMetadataWithoutReceiverSource()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata"": {
                        ""TestDescription"": ""edge agent ping"",
                        ""TestReportType"": ""DirectMethodConnectivityReport"",
                        ""SenderSource"": ""directMethodSender1.send"",
                        ""TolerancePeriod"": ""00:00:00.005""
                    }
                }";

            List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);

            Assert.Single(results);
            var reportMetadata = results[0] as DirectMethodConnectivityReportMetadata;

            Assert.NotNull(reportMetadata);
            Assert.Equal("edge agent ping", reportMetadata.TestDescription);
            Assert.Equal(TestOperationResultType.DirectMethod, reportMetadata.TestOperationResultType);
            Assert.Equal(TestReportType.DirectMethodConnectivityReport, reportMetadata.TestReportType);
            Assert.Equal("directMethodSender1.send", reportMetadata.SenderSource);
            Assert.False(reportMetadata.ReceiverSource.HasValue);
            Assert.Equal(new TimeSpan(0, 0, 0, 0, 5), reportMetadata.TolerancePeriod);
        }
コード例 #3
0
ファイル: TestReportUtilTest.cs プロジェクト: rivms/iotedge
        public void ParseReportMetadataList_ParseDirectMethodTestReportMetadata()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata"": {
                        ""TestReportType"": ""DirectMethodReport"",
                        ""SenderSource"": ""directMethodSender1.send"",
                        ""ReceiverSource"": ""directMethodReceiver1.receive"",
                        ""TolerancePeriod"": ""00:00:00.005""
                    }
                }";

            List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);

            Assert.Single(results);
            var reportMetadata = results[0] as DirectMethodReportMetadata;

            Assert.NotNull(reportMetadata);
            Assert.Equal(TestOperationResultType.DirectMethod, reportMetadata.TestOperationResultType);
            Assert.Equal(TestReportType.DirectMethodReport, reportMetadata.TestReportType);
            Assert.Equal("directMethodSender1.send", reportMetadata.SenderSource);
            Assert.True(reportMetadata.ReceiverSource.HasValue);
            reportMetadata.ReceiverSource.ForEach(x => Assert.Equal("directMethodReceiver1.receive", x));
            Assert.Equal(new TimeSpan(0, 0, 0, 0, 5), reportMetadata.TolerancePeriod);
        }
コード例 #4
0
        public void ParseReportMetadataList_ParseEdgeHubRestartDirectMethodReportMetadata()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata7"": {
                        ""TestDescription"": ""direct method | cloud | amqp"",
                        ""TestReportType"": ""EdgeHubRestartDirectMethodReport"",
                        ""TestOperationResultType"": ""EdgeHubRestartDirectMethod"",
                        ""SenderSource"": ""edgeHubRestartTester1.EdgeHubRestartDirectMethod"",
                        ""ReceiverSource"": ""directMethodReceiver1.receive""
                    }
                }";

            List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);

            Assert.Single(results);
            var reportMetadata = results[0] as EdgeHubRestartDirectMethodReportMetadata;

            Assert.NotNull(reportMetadata);
            Assert.Equal("direct method | cloud | amqp", reportMetadata.TestDescription);
            Assert.Equal(TestOperationResultType.EdgeHubRestartDirectMethod, reportMetadata.TestOperationResultType);
            Assert.Equal(TestReportType.EdgeHubRestartDirectMethodReport, reportMetadata.TestReportType);
            Assert.Equal("edgeHubRestartTester1.EdgeHubRestartDirectMethod", reportMetadata.SenderSource);
            Assert.Equal("directMethodReceiver1.receive", reportMetadata.ReceiverSource);
        }
コード例 #5
0
        public void ParseReportMetadataList_ThrowExceptionWhenParseInvalidData()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata"": {
                        ""TestDescription"": ""dummy"",
                        ""TestReportType"": ""TypeNotExist"",
                        ""ExpectedSource"": ""deploymentTester1.send"",
                        ""ActualSource"": ""deploymentTester2.receive""
                    }
                }";

            bool exceptionThrown = false;

            try
            {
                TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);
            }
            catch (Exception)
            {
                exceptionThrown = true;
            }

            Assert.True(exceptionThrown);
        }
コード例 #6
0
        async void DoWorkAsync(object state)
        {
            var tesReportGeneratorFactory = new TestReportGeneratorFactory(this.storage);
            List <ITestReportMetadata> reportMetadataList = await Settings.Current.GetReportMetadataListAsync(this.logger);

            ITestResultReport[] testResultReports = await TestReportUtil.GenerateTestResultReportsAsync(Settings.Current.TrackingId, reportMetadataList, tesReportGeneratorFactory, this.logger);

            if (testResultReports.Length == 0)
            {
                this.logger.LogInformation("No test result report is generated.");
                return;
            }

            var    testSummary    = new TestSummary(testResultReports);
            string reportsContent = JsonConvert.SerializeObject(testSummary, Formatting.Indented);

            this.logger.LogInformation($"Test result report{Environment.NewLine}{reportsContent}");

            await AzureLogAnalytics.Instance.PostAsync(
                Settings.Current.LogAnalyticsWorkspaceId,
                Settings.Current.LogAnalyticsSharedKey,
                reportsContent,
                Settings.Current.LogAnalyticsLogType);

            this.logger.LogInformation("Successfully send reports to LogAnalytics");
        }
コード例 #7
0
        public void ParseReportMetadataList_MultipleReportMetadata()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata1"": {
                        ""TestReportType"": ""CountingReport"",
                        ""TestOperationResultType"": ""Messages"",
                        ""ExpectedSource"": ""loadGen1.send"",
                        ""ActualSource"": ""relayer1.receive""
                    },
                    ""reportMetadata2"": {
                        ""TestReportType"": ""TwinCountingReport"",
                        ""TwinTestPropertyType"": ""Desired"",
                        ""ExpectedSource"": ""twinTester1.desiredUpdated"",
                        ""ActualSource"": ""twinTester2.desiredReceived""
                    },
                    ""reportMetadata3"": {
                        ""TestReportType"": ""DeploymentTestReport"",
                        ""ExpectedSource"": ""deploymentTester1.send"",
                        ""ActualSource"": ""deploymentTester2.receive""
                    }
                }";

            List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);

            Assert.Equal(3, results.Count);
        }
コード例 #8
0
        public void ParseReportMetadataList_ParseCountingReportMetadata()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata"": {
                        ""TestDescription"": ""messages | local | amqp"",
                        ""TestReportType"": ""CountingReport"",
                        ""TestOperationResultType"": ""Messages"",
                        ""ExpectedSource"": ""loadGen1.send"",
                        ""ActualSource"": ""relayer1.receive""
                    }
                }";

            List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);

            Assert.Single(results);
            var reportMetadata = results[0] as CountingReportMetadata;

            Assert.NotNull(reportMetadata);
            Assert.Equal("messages | local | amqp", reportMetadata.TestDescription);
            Assert.Equal(TestOperationResultType.Messages, reportMetadata.TestOperationResultType);
            Assert.Equal(TestReportType.CountingReport, reportMetadata.TestReportType);
            Assert.Equal("loadGen1.send", reportMetadata.ExpectedSource);
            Assert.Equal("relayer1.receive", reportMetadata.ActualSource);
        }
コード例 #9
0
        public void ParseReportMetadataList_ParseDirectMethodLongHaulTestReportMetadata()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata"": {
                        ""TestDescription"": ""direct method | cloud | amqp"",
                        ""TestReportType"": ""DirectMethodLongHaulReport"",
                        ""SenderSource"": ""directMethodSender1.send"",
                        ""ReceiverSource"": ""directMethodReceiver1.receive""
                    }
                }";

            List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);

            Assert.Single(results);
            var reportMetadata = results[0] as DirectMethodLongHaulReportMetadata;

            Assert.NotNull(reportMetadata);
            Assert.Equal("direct method | cloud | amqp", reportMetadata.TestDescription);
            Assert.Equal(TestOperationResultType.DirectMethod, reportMetadata.TestOperationResultType);
            Assert.Equal(TestReportType.DirectMethodLongHaulReport, reportMetadata.TestReportType);
            Assert.Equal("directMethodSender1.send", reportMetadata.SenderSource);
            Assert.True(reportMetadata.ReceiverSource.HasValue);
            reportMetadata.ReceiverSource.ForEach(x => Assert.Equal("directMethodReceiver1.receive", x));
        }
コード例 #10
0
ファイル: TestReportUtilTest.cs プロジェクト: rivms/iotedge
        public async Task TestGenerateTestResultReportsAsync_ReportGeneration(
            bool throwExceptionForTestReport1,
            bool throwExceptionForTestReport2,
            bool throwExceptionForTestReport3,
            bool throwExceptionForTestReport4,
            bool throwExceptionForTestReport5,
            int expectedReportCount)
        {
            var mockLogger = new Mock <ILogger>();
            var mockTestReportGeneratorFactory = new Mock <ITestReportGeneratorFactory>();

            string trackingId                 = "fakeTrackingId";
            var    countingReportMetadata     = new CountingReportMetadata("CountingExpectedSource", "CountingAcutalSource", TestOperationResultType.Messages, TestReportType.CountingReport);
            var    twinCountingReportMetadata = new TwinCountingReportMetadata("TwinExpectedSource", "TwinActualSource", TestReportType.TwinCountingReport, TwinTestPropertyType.Desired);
            var    deploymentReportMetadata   = new DeploymentTestReportMetadata("DeploymentExpectedSource", "DeploymentActualSource");
            var    directMethodReportMetadata = new DirectMethodReportMetadata("DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource");
            var    directMethodReportMetadataWithoutReceiverSource = new DirectMethodReportMetadata("DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource");

            var mockTestReportGenerator1 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator1.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport1));

            var mockTestReportGenerator2 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator2.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport2));

            var mockTestReportGenerator3 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator3.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport3));

            var mockTestReportGenerator4 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator4.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport4));

            var mockTestReportGenerator5 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator5.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport5));

            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, countingReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator1.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, twinCountingReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator2.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, deploymentReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator3.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator4.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodReportMetadataWithoutReceiverSource)).Returns(Task.FromResult(mockTestReportGenerator5.Object));

            ITestResultReport[] reports = await TestReportUtil.GenerateTestResultReportsAsync(
                trackingId,
                new List <ITestReportMetadata>
            {
                countingReportMetadata,
                twinCountingReportMetadata,
                deploymentReportMetadata,
                directMethodReportMetadata,
                directMethodReportMetadataWithoutReceiverSource
            },
                mockTestReportGeneratorFactory.Object,
                mockLogger.Object);

            Assert.Equal(expectedReportCount, reports.Length);
        }
コード例 #11
0
        public void ParseReportMetadataList_MultipleReportMetadata()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata1"": {
                        ""TestDescription"": ""messages | local | amqp"",
                        ""TestReportType"": ""CountingReport"",
                        ""TestOperationResultType"": ""Messages"",
                        ""ExpectedSource"": ""loadGen1.send"",
                        ""ActualSource"": ""relayer1.receive""
                    },
                    ""reportMetadata2"": {
                        ""TestDescription"": ""twin | desired property | amqp"",
                        ""TestReportType"": ""TwinCountingReport"",
                        ""TwinTestPropertyType"": ""Desired"",
                        ""ExpectedSource"": ""twinTester1.desiredUpdated"",
                        ""ActualSource"": ""twinTester2.desiredReceived""
                    },
                    ""reportMetadata3"": {
                        ""TestDescription"": ""deployment"",
                        ""TestReportType"": ""DeploymentTestReport"",
                        ""ExpectedSource"": ""deploymentTester1.send"",
                        ""ActualSource"": ""deploymentTester2.receive""
                    },
                    ""reportMetadata4"": {
                        ""TestDescription"": ""direct method | cloud | amqp"",
                        ""TestReportType"": ""DirectMethodConnectivityReport"",
                        ""SenderSource"": ""senderSource1.send"",
                        ""ReceiverSource"": ""receiverSource1.receive"",
                        ""TolerancePeriod"": ""00:00:00.005""
                    },
                    ""reportMetadata5"": {
                        ""TestDescription"": ""edge agent ping"",
                        ""TestReportType"": ""DirectMethodConnectivityReport"",
                        ""SenderSource"": ""senderSource1.send"",
                        ""TolerancePeriod"": ""00:00:00.005""
                    },
                    ""reportMetadata6"": {
                        ""TestDescription"": ""messages | local | amqp | restart"",
                        ""TestReportType"": ""EdgeHubRestartMessageReport"",
                        ""TestOperationResultType"": ""EdgeHubRestartMessage"",
                        ""SenderSource"": ""edgeHubRestartTester1.EdgeHubRestartMessage"",
                        ""ReceiverSource"": ""relayer1.receive""
                    },
                    ""reportMetadata7"": {
                        ""TestDescription"": ""direct method | cloud | amqp | restart"",
                        ""TestReportType"": ""EdgeHubRestartDirectMethodReport"",
                        ""TestOperationResultType"": ""EdgeHubRestartDirectMethod"",
                        ""SenderSource"": ""edgeHubRestartTester1.EdgeHubRestartDirectMethod"",
                        ""ReceiverSource"": ""directMethodReceiver1.receive""
                    }
                }";

            List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);

            Assert.Equal(7, results.Count);
        }
コード例 #12
0
        public async Task TestGenerateTestResultReportsAsync_MissingTrackingId()
        {
            var mockLogger = new Mock <ILogger>();
            var mockTestReportGeneratorFactory = new Mock <ITestReportGeneratorFactory>();

            ArgumentException ex = await Assert.ThrowsAsync <ArgumentException>(() =>
                                                                                TestReportUtil.GenerateTestResultReportsAsync(
                                                                                    string.Empty,
                                                                                    new List <ITestReportMetadata>(),
                                                                                    mockTestReportGeneratorFactory.Object,
                                                                                    mockLogger.Object));

            Assert.StartsWith("trackingId", ex.Message, StringComparison.Ordinal);
        }
コード例 #13
0
        async void DoWorkAsync(object state)
        {
            var testReportGeneratorFactory = new TestReportGeneratorFactory(this.storage, Settings.Current.NetworkControllerType, Settings.Current.LongHaulSpecificSettings);
            List <ITestReportMetadata> reportMetadataList = await Settings.Current.GetReportMetadataListAsync(this.logger);

            ITestResultReport[] testResultReports = await TestReportUtil.GenerateTestResultReportsAsync(Settings.Current.TrackingId, reportMetadataList, testReportGeneratorFactory, this.logger);

            if (testResultReports.Length == 0)
            {
                this.logger.LogInformation("No test result report is generated.");
                return;
            }

            string blobContainerUri = string.Empty;

            if (this.serviceSpecificSettings.LogUploadEnabled)
            {
                try
                {
                    Uri blobContainerWriteUriForLog = await TestReportUtil.GetOrCreateBlobContainerSasUriForLogAsync(this.serviceSpecificSettings.StorageAccountConnectionString);

                    blobContainerUri = $"{blobContainerWriteUriForLog.Scheme}{Uri.SchemeDelimiter}{blobContainerWriteUriForLog.Authority}{blobContainerWriteUriForLog.AbsolutePath}";

                    await TestReportUtil.UploadLogsAsync(Settings.Current.IoTHubConnectionString, blobContainerWriteUriForLog, this.logUploadDuration, this.logger);
                }
                catch (Exception ex)
                {
                    this.logger.LogError(ex, "Exception happened when uploading logs");
                }
            }

            var    testSummary    = new TestSummary(Settings.Current.TestInfo, testResultReports, blobContainerUri);
            string reportsContent = JsonConvert.SerializeObject(testSummary, Formatting.Indented);

            this.logger.LogInformation($"Test summary{Environment.NewLine}{reportsContent}");

            await AzureLogAnalytics.Instance.PostAsync(
                this.serviceSpecificSettings.LogAnalyticsWorkspaceId,
                this.serviceSpecificSettings.LogAnalyticsSharedKey,
                reportsContent,
                this.serviceSpecificSettings.LogAnalyticsLogType);

            this.logger.LogInformation("Successfully send reports to LogAnalytics");
        }
コード例 #14
0
        public void ParseReportMetadataList_ParseTestInfoReportMetadata()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata"": {
                        ""TestReportType"": ""TestInfoReport""
                    }
                }";

            List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);

            Assert.Single(results);
            var reportMetadata = results[0] as TestInfoReportMetadata;

            Assert.NotNull(reportMetadata);
            Assert.Equal(TestOperationResultType.TestInfo, reportMetadata.TestOperationResultType);
            Assert.Equal(TestReportType.TestInfoReport, reportMetadata.TestReportType);
            Assert.Equal(TestConstants.TestInfo.TestResultSource, reportMetadata.Source);
        }
コード例 #15
0
        public async Task <ContentResult> GetReportsAsync()
        {
            var testReportGeneratorFactory = new TestReportGeneratorFactory(this.storage);
            List <ITestReportMetadata> reportMetadataList = await Settings.Current.GetReportMetadataListAsync(Logger);

            ITestResultReport[] testResultReports = await TestReportUtil.GenerateTestResultReportsAsync(Settings.Current.TrackingId, reportMetadataList, testReportGeneratorFactory, Logger);

            if (testResultReports.Length == 0)
            {
                return(new ContentResult {
                    Content = "No test report generated"
                });
            }

            return(new ContentResult
            {
                Content = this.AddManualRunReportingHeading(JsonConvert.SerializeObject(testResultReports, Formatting.Indented)) // explicit serialization needed due to the wrapping list
            });
        }
コード例 #16
0
ファイル: TestReportUtilTest.cs プロジェクト: rivms/iotedge
        public void ParseReportMetadataList_ParseNetworkControllerReportMetadata()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata"": {
                        ""TestReportType"": ""NetworkControllerReport"",
                        ""Source"": ""networkController""
                    }
                }";

            List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);

            Assert.Single(results);
            var reportMetadata = results[0] as NetworkControllerReportMetadata;

            Assert.NotNull(reportMetadata);
            Assert.Equal(TestOperationResultType.Network, reportMetadata.TestOperationResultType);
            Assert.Equal(TestReportType.NetworkControllerReport, reportMetadata.TestReportType);
            Assert.Equal("networkController", reportMetadata.Source);
        }
コード例 #17
0
ファイル: TestReportUtilTest.cs プロジェクト: rivms/iotedge
        public void ParseReportMetadataList_ParseDeploymentTestReportMetadata()
        {
            const string testDataJson =
                @"{
                    ""reportMetadata"": {
                        ""TestReportType"": ""DeploymentTestReport"",
                        ""ExpectedSource"": ""deploymentTester1.send"",
                        ""ActualSource"": ""deploymentTester2.receive""
                    }
                }";

            List <ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock <ILogger>().Object);

            Assert.Single(results);
            var reportMetadata = results[0] as DeploymentTestReportMetadata;

            Assert.NotNull(reportMetadata);
            Assert.Equal(TestOperationResultType.Deployment, reportMetadata.TestOperationResultType);
            Assert.Equal(TestReportType.DeploymentTestReport, reportMetadata.TestReportType);
            Assert.Equal("deploymentTester1.send", reportMetadata.ExpectedSource);
            Assert.Equal("deploymentTester2.receive", reportMetadata.ActualSource);
        }
コード例 #18
0
        public async Task <ITestResultReport> CreateReportAsync()
        {
            Logger.LogInformation($"Start to generate report by {nameof(TwinCountingReportGenerator)} for Sources [{this.expectedSource}] and [{this.actualSource}]");

            ulong          totalExpectCount = 0;
            ulong          totalMatchCount  = 0;
            ulong          totalPatches     = 0;
            ulong          totalDuplicates  = 0;
            Queue <string> unmatchedResults = new Queue <string>();

            Dictionary <string, DateTime> propertiesUpdated  = new Dictionary <string, DateTime>();
            Dictionary <string, DateTime> propertiesReceived = new Dictionary <string, DateTime>();

            while (await this.expectedTestResults.MoveNextAsync())
            {
                Option <TwinTestResult> twinTestResult = this.GetTwinTestResult(this.expectedTestResults.Current);
                Logger.LogDebug($"Expected test results {twinTestResult}");

                twinTestResult.ForEach(
                    r =>
                {
                    foreach (var prop in r.Properties)
                    {
                        propertiesUpdated.TryAdd(prop.ToString(), this.expectedTestResults.Current.CreatedAt);
                    }
                });
            }

            while (await this.actualTestResults.MoveNextAsync())
            {
                totalPatches++;

                Option <TwinTestResult> twinTestResult = this.GetTwinTestResult(this.actualTestResults.Current);
                Logger.LogDebug($"Actual test results {twinTestResult}");

                twinTestResult.ForEach(
                    r =>
                {
                    foreach (var prop in r.Properties)
                    {
                        bool added = propertiesReceived.TryAdd(prop.ToString(), this.actualTestResults.Current.CreatedAt);
                        if (!added)
                        {
                            Logger.LogDebug($"Duplicate for {this.actualSource} {prop.ToString()}");
                            totalDuplicates++;
                        }
                    }
                });
            }

            foreach (KeyValuePair <string, DateTime> desiredPropertyUpdate in propertiesUpdated)
            {
                totalExpectCount++;

                if (propertiesReceived.ContainsKey(desiredPropertyUpdate.Key))
                {
                    totalMatchCount++;
                }
                else
                {
                    TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, $"{this.expectedSource} {desiredPropertyUpdate.Key}", this.unmatchedResultsMaxSize);
                }
            }

            foreach (KeyValuePair <string, DateTime> desiredPropertyReceived in propertiesReceived)
            {
                if (!propertiesUpdated.ContainsKey(desiredPropertyReceived.Key))
                {
                    Logger.LogError($"[{nameof(TwinCountingReportGenerator)}] Actual test result source has unexpected results.");
                    TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, $"{this.actualSource} {desiredPropertyReceived.Key}", this.unmatchedResultsMaxSize);
                }
            }

            return(new TwinCountingReport(
                       this.testDescription,
                       this.trackingId,
                       this.expectedSource,
                       this.actualSource,
                       this.resultType,
                       totalExpectCount,
                       totalMatchCount,
                       totalPatches,
                       totalDuplicates,
                       new List <string>(unmatchedResults).AsReadOnly()));
        }
コード例 #19
0
        /// <summary>
        /// Compare 2 data stores and counting expect, match, and duplicate results; and return a counting report.
        /// It will remove consecutive duplicate results when loading from actual store.
        /// It will log fail if actual store has more results than expect store.
        /// </summary>
        /// <returns>Test Result Report.</returns>
        public async Task <ITestResultReport> CreateReportAsync()
        {
            Logger.LogInformation($"Start to generate report by {nameof(CountingReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]");

            var   lastLoadedResult          = default(TestOperationResult);
            ulong totalExpectCount          = 0;
            ulong totalMatchCount           = 0;
            ulong totalDuplicateResultCount = 0;
            var   unmatchedResults          = new Queue <TestOperationResult>();

            bool hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();

            bool hasActualResult = await this.ActualTestResults.MoveNextAsync();

            while (hasExpectedResult && hasActualResult)
            {
                this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource);
                this.ValidateResult(this.ActualTestResults.Current, this.ActualSource);

                // Skip any duplicate actual value
                while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
                {
                    totalDuplicateResultCount++;
                    lastLoadedResult = this.ActualTestResults.Current;
                    hasActualResult  = await this.ActualTestResults.MoveNextAsync();
                }

                totalExpectCount++;

                if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
                {
                    lastLoadedResult = this.ActualTestResults.Current;
                    hasActualResult  = await this.ActualTestResults.MoveNextAsync();

                    hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();

                    totalMatchCount++;
                }
                else
                {
                    TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
                    hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
                }
            }

            // Check duplicates at the end of actual results
            while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
            {
                totalDuplicateResultCount++;
                lastLoadedResult = this.ActualTestResults.Current;
                hasActualResult  = await this.ActualTestResults.MoveNextAsync();
            }

            while (hasExpectedResult)
            {
                totalExpectCount++;
                TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
                hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
            }

            while (hasActualResult)
            {
                // Log message for unexpected case.
                Logger.LogError($"[{nameof(CountingReportGenerator)}] Actual test result source has unexpected results.");

                // Log actual queue items
                Logger.LogError($"Unexpected actual test result: {this.ActualTestResults.Current.Source}, {this.ActualTestResults.Current.Type}, {this.ActualTestResults.Current.Result} at {this.ActualTestResults.Current.CreatedAt}");

                hasActualResult = await this.ActualTestResults.MoveNextAsync();
            }

            return(new CountingReport(
                       this.TestDescription,
                       this.trackingId,
                       this.ExpectedSource,
                       this.ActualSource,
                       this.ResultType,
                       totalExpectCount,
                       totalMatchCount,
                       totalDuplicateResultCount,
                       new List <TestOperationResult>(unmatchedResults).AsReadOnly()));
        }
コード例 #20
0
        /// <summary>
        /// Compare 2 data stores and counting expect, match, and duplicate results; and return a counting report.
        /// It will remove consecutive duplicate results when loading from actual store.
        /// It will log fail if actual store has more results than expect store.
        /// </summary>
        /// <returns>Test Result Report.</returns>
        public async Task <ITestResultReport> CreateReportAsync()
        {
            Logger.LogInformation($"Start to generate report by {nameof(CountingReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]");

            var   lastLoadedResult          = default(TestOperationResult);
            ulong totalExpectCount          = 0;
            ulong totalMatchCount           = 0;
            ulong totalDuplicateResultCount = 0;
            var   unmatchedResults          = new Queue <TestOperationResult>();
            bool  allActualResultsMatch     = false;
            Option <EventHubSpecificReportComponents> eventHubSpecificReportComponents = Option.None <EventHubSpecificReportComponents>();
            Option <DateTime> lastLoadedResultCreatedAt = Option.None <DateTime>();

            bool hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();

            bool hasActualResult = await this.ActualTestResults.MoveNextAsync();

            while (hasExpectedResult && hasActualResult)
            {
                this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource);
                this.ValidateResult(this.ActualTestResults.Current, this.ActualSource);

                // Skip any duplicate actual value
                while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
                {
                    totalDuplicateResultCount++;
                    lastLoadedResult = this.ActualTestResults.Current;
                    hasActualResult  = await this.ActualTestResults.MoveNextAsync();
                }

                totalExpectCount++;

                if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
                {
                    lastLoadedResult = this.ActualTestResults.Current;
                    hasActualResult  = await this.ActualTestResults.MoveNextAsync();

                    hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();

                    totalMatchCount++;
                }
                else
                {
                    TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
                    hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
                }
            }

            // Check duplicates at the end of actual results
            while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
            {
                totalDuplicateResultCount++;
                lastLoadedResult = this.ActualTestResults.Current;
                hasActualResult  = await this.ActualTestResults.MoveNextAsync();
            }

            allActualResultsMatch = totalExpectCount == totalMatchCount;

            while (hasExpectedResult)
            {
                totalExpectCount++;
                TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
                hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();
            }

            if (this.EventHubLongHaulMode)
            {
                bool stillReceivingFromEventHub = false;
                // If we are are using EventHub to receive messages, we see an issue where EventHub can accrue large delays after
                // running for a while. Therefore, if we are using EventHub with this counting report, we do two things.
                // 1. Match only actual results. We still report all expected results, but matching actual results only.
                // 2. We make sure that the last result we got from EventHub (i.e. the lastLoadedResult) is within our defined tolerance period.
                //    'eventHubDelayTolerance' is an arbitrary tolerance period that we have defined, and can be tuned as needed.
                // TODO: There is either something wrong with the EventHub service or something wrong with the way we are using it,
                // Because we should not be accruing such large delays. If we move off EventHub, we should fix this as well.
                TimeSpan eventHubDelayTolerance = Settings.Current.LongHaulSpecificSettings
                                                  .Expect <ArgumentException>(
                    () => throw new ArgumentException("TRC must be in long haul mode to be generating an EventHubLongHaul CountingReport"))
                                                  .EventHubDelayTolerance;
                if (lastLoadedResult == null || lastLoadedResult.CreatedAt < DateTime.UtcNow - eventHubDelayTolerance)
                {
                    stillReceivingFromEventHub = false;
                }
                else
                {
                    stillReceivingFromEventHub = true;
                }

                eventHubSpecificReportComponents = Option.Some(new EventHubSpecificReportComponents
                {
                    StillReceivingFromEventHub = stillReceivingFromEventHub,
                    AllActualResultsMatch      = allActualResultsMatch
                });
            }

            while (hasActualResult)
            {
                // Log message for unexpected case.
                Logger.LogError($"[{nameof(CountingReportGenerator)}] Actual test result source has unexpected results.");

                // Log actual queue items
                Logger.LogError($"Unexpected actual test result: {this.ActualTestResults.Current.Source}, {this.ActualTestResults.Current.Type}, {this.ActualTestResults.Current.Result} at {this.ActualTestResults.Current.CreatedAt}");

                hasActualResult = await this.ActualTestResults.MoveNextAsync();
            }

            if (lastLoadedResult != null)
            {
                lastLoadedResultCreatedAt = Option.Some(lastLoadedResult.CreatedAt);
            }

            return(new CountingReport(
                       this.TestDescription,
                       this.trackingId,
                       this.ExpectedSource,
                       this.ActualSource,
                       this.ResultType,
                       totalExpectCount,
                       totalMatchCount,
                       totalDuplicateResultCount,
                       new List <TestOperationResult>(unmatchedResults).AsReadOnly(),
                       eventHubSpecificReportComponents,
                       lastLoadedResultCreatedAt));
        }
コード例 #21
0
        public async Task TestGenerateTestResultReportsAsync_NullReportMetadata()
        {
            var mockLogger = new Mock <ILogger>();
            var mockTestReportGeneratorFactory = new Mock <ITestReportGeneratorFactory>();

            ArgumentNullException ex = await Assert.ThrowsAsync <ArgumentNullException>(() =>
                                                                                        TestReportUtil.GenerateTestResultReportsAsync(
                                                                                            "fakeTrackingId",
                                                                                            null,
                                                                                            mockTestReportGeneratorFactory.Object,
                                                                                            mockLogger.Object));

            Assert.Equal("reportMetadatalist", ex.ParamName);
        }
コード例 #22
0
        /// <summary>
        /// Compare 2 data stores and counting expected, actual, and matched results; and return a deployment test report.
        /// Actual deployment results can be less than expected deployment results.
        /// An actaul deployment test result is possible to use for verification of more than 1 expected deployment test result.
        /// It will log fail if actual store has more results than expect store.
        /// </summary>
        /// <returns>Test Result Report.</returns>
        public async Task <ITestResultReport> CreateReportAsync()
        {
            Logger.LogInformation($"Start to generate report by {nameof(DeploymentTestReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]");

            TestOperationResult lastActualDeploymentTestResult = null;
            ulong totalExpectedDeployments = 0;
            ulong totalActualDeployments   = 0;
            ulong totalMatchedDeployments  = 0;
            var   unmatchedResults         = new Queue <TestOperationResult>();

            bool hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();

            if (hasExpectedResult)
            {
                totalExpectedDeployments++;
            }

            bool hasActualResult = await this.ActualTestResults.MoveNextAsync();

            if (hasActualResult)
            {
                lastActualDeploymentTestResult = this.ActualTestResults.Current;
                totalActualDeployments++;
            }

            while (hasExpectedResult && hasActualResult)
            {
                this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource);
                this.ValidateResult(this.ActualTestResults.Current, this.ActualSource);

                if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
                {
                    totalMatchedDeployments++;

                    hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();

                    if (hasExpectedResult)
                    {
                        totalExpectedDeployments++;
                    }
                }
                else
                {
                    hasActualResult = await this.ActualTestResults.MoveNextAsync();

                    if (hasActualResult)
                    {
                        totalActualDeployments++;
                        lastActualDeploymentTestResult = this.ActualTestResults.Current;
                    }
                }
            }

            while (hasExpectedResult)
            {
                TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize);
                hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync();

                if (hasExpectedResult)
                {
                    totalExpectedDeployments++;
                }
            }

            hasActualResult = await this.ActualTestResults.MoveNextAsync();

            if (hasActualResult)
            {
                totalActualDeployments++;
                lastActualDeploymentTestResult = this.ActualTestResults.Current;

                while (hasActualResult)
                {
                    // Log message for unexpected case.
                    Logger.LogError($"[{nameof(DeploymentTestReportGenerator)}] Actual test result source has unexpected results.");

                    // Log actual queue items
                    Logger.LogError($"Unexpected actual test result: {this.ActualTestResults.Current.Source}, {this.ActualTestResults.Current.Type}, {this.ActualTestResults.Current.Result} at {this.ActualTestResults.Current.CreatedAt}");

                    hasActualResult = await this.ActualTestResults.MoveNextAsync();

                    if (hasActualResult)
                    {
                        lastActualDeploymentTestResult = this.ActualTestResults.Current;
                        totalActualDeployments++;
                    }
                }
            }

            return(new DeploymentTestReport(
                       this.TestDescription,
                       this.trackingId,
                       this.ExpectedSource,
                       this.ActualSource,
                       this.ResultType,
                       totalExpectedDeployments,
                       totalActualDeployments,
                       totalMatchedDeployments,
                       Option.Maybe(lastActualDeploymentTestResult),
                       new List <TestOperationResult>(unmatchedResults).AsReadOnly()));
        }
コード例 #23
0
        public async Task TestGenerateTestResultReportsAsync_ReportGeneration(
            bool throwExceptionForTestReport1,
            bool throwExceptionForTestReport2,
            bool throwExceptionForTestReport3,
            bool throwExceptionForTestReport4,
            bool throwExceptionForTestReport5,
            bool throwExceptionForTestReport6,
            bool throwExceptionForTestReport7,
            bool throwExceptionForTestReport8,
            bool throwExceptionForTestReport9,
            int expectedReportCount)
        {
            var mockLogger = new Mock <ILogger>();
            var mockTestReportGeneratorFactory = new Mock <ITestReportGeneratorFactory>();

            string trackingId                             = "fakeTrackingId";
            var    countingReportMetadata                 = new CountingReportMetadata(TestDescription, "CountingExpectedSource", "CountingAcutalSource", TestOperationResultType.Messages, TestReportType.CountingReport, false);
            var    twinCountingReportMetadata             = new TwinCountingReportMetadata(TestDescription, "TwinExpectedSource", "TwinActualSource", TestReportType.TwinCountingReport, TwinTestPropertyType.Desired);
            var    deploymentReportMetadata               = new DeploymentTestReportMetadata(TestDescription, "DeploymentExpectedSource", "DeploymentActualSource");
            var    directMethodConnectivityReportMetadata = new DirectMethodConnectivityReportMetadata(TestDescription, "DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource");
            var    directMethodConnectivityReportMetadataWithoutReceiverSource = new DirectMethodConnectivityReportMetadata(TestDescription, "DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource");
            var    directMethodLongHaulReportMetadata = new DirectMethodLongHaulReportMetadata(TestDescription, "DirectMethodSenderSource", "DirectMethodReceiverSource");
            var    directMethodLongHaulReportMetadataWithoutReceiverSource = new DirectMethodLongHaulReportMetadata(TestDescription, "DirectMethodSenderSource", "DirectMethodReceiverSource");
            var    edgeHubRestartMessageReportMetadata      = new EdgeHubRestartMessageReportMetadata(TestDescription, "edgeHubRestartTester1.EdgeHubRestartMessage", "relayer1.receive");
            var    edgeHubRestartDirectMethodReportMetadata = new EdgeHubRestartDirectMethodReportMetadata(TestDescription, "edgeHubRestartTester1.EdgeHubRestartDirectMethod", "directMethodReceiver1.receive");

            var mockTestReportGenerator1 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator1.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport1));

            var mockTestReportGenerator2 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator2.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport2));

            var mockTestReportGenerator3 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator3.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport3));

            var mockTestReportGenerator4 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator4.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport4));

            var mockTestReportGenerator5 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator5.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport5));

            var mockTestReportGenerator6 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator6.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport6));

            var mockTestReportGenerator7 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator7.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport7));

            var mockTestReportGenerator8 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator8.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport8));

            var mockTestReportGenerator9 = new Mock <ITestResultReportGenerator>();

            mockTestReportGenerator9.Setup(g => g.CreateReportAsync()).Returns(this.MockTestResultReport(throwExceptionForTestReport9));

            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, countingReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator1.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, twinCountingReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator2.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, deploymentReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator3.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodConnectivityReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator4.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodConnectivityReportMetadataWithoutReceiverSource)).Returns(Task.FromResult(mockTestReportGenerator5.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodLongHaulReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator6.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, directMethodLongHaulReportMetadataWithoutReceiverSource)).Returns(Task.FromResult(mockTestReportGenerator7.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, edgeHubRestartMessageReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator8.Object));
            mockTestReportGeneratorFactory.Setup(f => f.CreateAsync(trackingId, edgeHubRestartDirectMethodReportMetadata)).Returns(Task.FromResult(mockTestReportGenerator9.Object));

            ITestResultReport[] reports = await TestReportUtil.GenerateTestResultReportsAsync(
                trackingId,
                new List <ITestReportMetadata>
            {
                countingReportMetadata,
                twinCountingReportMetadata,
                deploymentReportMetadata,
                directMethodConnectivityReportMetadata,
                directMethodConnectivityReportMetadataWithoutReceiverSource,
                directMethodLongHaulReportMetadata,
                directMethodLongHaulReportMetadataWithoutReceiverSource,
                edgeHubRestartMessageReportMetadata,
                edgeHubRestartDirectMethodReportMetadata
            },
                mockTestReportGeneratorFactory.Object,
                mockLogger.Object);

            Assert.Equal(expectedReportCount, reports.Length);
        }