private static void RunTest(Options o, string subTest, Action <string> logger, ITest test, int i) { TestResultBase result = null; try { result = test.Test(o); result.StepNr = i; result.SubTest = subTest; } catch (Exception ex) { if (result == null) { result = new GenericTestResult(); } result.StepNr = i; result.SubTest = subTest; result.Status = TestResult.FAIL; result.CauseOfFailure = ex.Message; result.ExtraInformation = ex.StackTrace; } if (logger != null) { logger(result.ToString()); } }
public static async Task <int> MainAsync() { Logger.LogInformation($"Starting DirectMethodSender with the following settings:\r\n{Settings.Current}"); (CancellationTokenSource cts, ManualResetEventSlim completed, Option <object> handler) = ShutdownHandler.Init(TimeSpan.FromSeconds(5), Logger); DirectMethodSenderBase directMethodClient = null; ReporterClientBase reportClient = null; try { Guid batchId = Guid.NewGuid(); Logger.LogInformation($"Batch Id={batchId}"); Logger.LogInformation($"Load gen delay start for {Settings.Current.TestStartDelay}."); await Task.Delay(Settings.Current.TestStartDelay, cts.Token); DateTime testStartAt = DateTime.UtcNow; directMethodClient = await CreateClientAsync(Settings.Current.InvocationSource); reportClient = ReporterClientBase.Create( Logger, Settings.Current.ReportingEndpointUrl, Settings.Current.TransportType); while (!cts.Token.IsCancellationRequested && IsTestTimeUp(testStartAt)) { (HttpStatusCode result, ulong dmCounter) = await directMethodClient.InvokeDirectMethodAsync(Settings.Current.DirectMethodName, cts); // Generate a testResult type depending on the reporting endpoint TestResultBase testResult = ConstructTestResult( Settings.Current.DirectMethodResultType, batchId, dmCounter, result); await reportClient.SendTestResultAsync(testResult); await Task.Delay(Settings.Current.DirectMethodDelay, cts.Token); } await cts.Token.WhenCanceled(); } catch (Exception e) { Logger.LogError(e, "Error occurred during direct method sender test setup"); } finally { // Implicit CloseAsync() directMethodClient?.Dispose(); reportClient?.Dispose(); } completed.Set(); handler.ForEach(h => GC.KeepAlive(h)); Logger.LogInformation("DirectMethodSender Main() finished."); return(0); }
public TestResultBase Test(Options o) { var testResult = new GenericTestResult { ShortDescription = "Performance (uptime) " + RelativeUrl, Status = TestResult.INCONCLUSIVE }; DateTime current = DateTime.Now; // to check if we elapsed total seconds. var results = new List <Tuple <long, TestResult> >(); //long lastMessageTime = 0; while (true) { var test = new CheckCompletenessResponseTest { RelativeUrl = RelativeUrl, //string.Format(RelativeUrl, lastMessageTime) FieldsThatShouldBePresent = FieldsThatShouldBePresent }; TestResultBase res = test.Test(o); results.Add(new Tuple <long, TestResult>(test.RequestDuration, res.Status)); //if (test.ResponseData != null) //{ // // now store the last updated time from the previous // string data = test.ResponseData; //} Thread.Sleep(IntervalTime); if (DateTime.Now.Subtract(current).TotalSeconds >= TestDuration) { break; } } IEnumerable <Tuple <long, TestResult> > succesCount = results.Where(t => t.Item2 == TestResult.OK).ToList(); var totalResultCount = results.Count; double percentageSuccess = totalResultCount == 0 ? 0 : succesCount.Count() / (double)totalResultCount; double avgTime = succesCount.Any() ? succesCount.Select(t => t.Item1).Average() : -1; testResult.ExtraInformation = Math.Round(avgTime, 0) + " ms average time"; if (percentageSuccess >= 0.95) { testResult.Status = TestResult.OK; } else { testResult.Status = TestResult.FAIL; testResult.CauseOfFailure = "Percentage success: " + Math.Round(percentageSuccess * 100) + "%"; } return(testResult); }
internal async Task SendTestResultAsync(TestResultBase testResult) { Preconditions.CheckNotNull(testResult, nameof(testResult)); try { await this.ReportStatusAsync(testResult); } catch (Exception e) { this.logger.LogError(e, "Failed to send testResult"); } }
internal static void RunContinuityTests(Options options) { var test = new KvGPerformanceTest { RelativeUrl = FcdRetrieveRequest(), IntervalTime = Options.PerformanceTestInterval, TestDuration = Options.PerformanceTestDuration }; TestResultBase result1 = test.Test(options); result1.StepNr = 1; result1.SubTest = "Continuity"; Out.Info(result1.ToString()); }
internal override async Task ReportStatusAsync(TestResultBase testResult) { switch (testResult.GetType().Name) { case nameof(LegacyDirectMethodTestResult): LegacyDirectMethodTestResult shadowReport = testResult as LegacyDirectMethodTestResult; // The Old End-to-End test framework checks if there is an event in the eventHub for a particular module // to determine if the test case is passed/failed. Hence, this function need to check if the testResult status is Http.OK // before sending an event. if (shadowReport.Result == HttpStatusCode.OK.ToString()) { await this.ModuleClient.SendEventAsync("AnyOutput", new Message(Encoding.UTF8.GetBytes($"Source:{shadowReport.Source} CreatedAt:{shadowReport.CreatedAt}."))); } break; default: throw new NotSupportedException($"{this.GetType().Name} does not have an implementation for {testResult.GetType().Name} type"); } }
internal static void RunPerformanceTests(Options options) { var test = new KvGPerformanceTest { RelativeUrl = FcdRetrieveRequestWithTime(), IntervalTime = Options.PerformanceTestInterval, TestDuration = Options.PerformanceTestDuration, }; TestResultBase result1 = test.Test(options); result1.StepNr = 1; result1.SubTest = "Performance"; Out.Info(result1.ToString()); // test 2 if (test.GenerationTimeValid.Value) { Out.Info(new GenericTestResult { SubTest = "Performance", StepNr = 2, ShortDescription = "Update interval " + FcdRetrieveRequest(), Status = TestResult.OK }.ToString()); } else { Out.Info(new GenericTestResult { SubTest = "Performance", StepNr = 2, ShortDescription = "Update interval " + FcdRetrieveRequest(), Status = TestResult.FAIL, CauseOfFailure = "Generation time did not update properly." }.ToString()); } }
public static async Task ReportTestResultAsync(TestResultReportingClient apiClient, ILogger logger, TestResultBase testResult) { try { logger.LogInformation($"Sending test result: Source={testResult.Source}, Type={testResult.ResultType}, CreatedAt={testResult.CreatedAt}, Result={testResult.GetFormattedResult()}"); await apiClient.ReportResultAsync(testResult.ToTestOperationResultDto()); } catch (Exception e) { logger.LogError(e, "Failed call to report status to TestResultCoordinator"); } }
internal override async Task ReportStatusAsync(TestResultBase testResult) { Preconditions.CheckNotNull(testResult, nameof(testResult)); await this.testResultReportingClient.ReportResultAsync(testResult.ToTestOperationResultDto()); }
public TestResultBase Test(Options o) { var testResult = new GenericTestResult { ShortDescription = "Performance (uptime) " + RelativeUrl, Status = TestResult.INCONCLUSIVE }; DateTime current = DateTime.Now; // to check if we elapsed total seconds. var results = new List <Tuple <long, TestResult> >(); long lastMessageTime = 0; while (true) { var test = new CheckCompletenessResponseTest { RelativeUrl = string.Format("/fcd?last_message_time={0}", lastMessageTime), FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInFCD(), TestResultWhenNoData = TestResult.OK }; TestResultBase res = test.Test(o); results.Add(new Tuple <long, TestResult>(test.RequestDuration, res.Status)); if (test.ResponseData != null) { // now store the last updated time from the previous string data = test.ResponseData; // deserialize var fcdMsg = JsonConvert.DeserializeObject <FcdMessage>(data); if (lastMessageTime == fcdMsg.message_time) { GenerationTimeValid = false; } else { if (!GenerationTimeValid.HasValue) { GenerationTimeValid = true; } } lastMessageTime = fcdMsg.message_time; } else { GenerationTimeValid = false; } Thread.Sleep(IntervalTime); if (DateTime.Now.Subtract(current).TotalSeconds >= TestDuration) { break; } } IEnumerable <Tuple <long, TestResult> > succesCount = results.Where(t => t.Item2 == TestResult.OK); double percentageSuccess = succesCount.Count() / (double)results.Count; double avgTime = results.Select(t => t.Item1).Average(); testResult.ExtraInformation = Math.Round(avgTime, 0) + " ms average time"; if (percentageSuccess >= 0.95) { testResult.Status = TestResult.OK; } else { testResult.Status = TestResult.FAIL; testResult.CauseOfFailure = "Percentage success: " + Math.Round(percentageSuccess * 100, 0) + " %"; } return(testResult); }
private static void Main(string[] args) { #if DEBUG //args = new[] //{ // "--all", // "--url", "http://yourtesturl", // "--apikey", "MyFamousApiKey", // "--user", "TheOneAndOnlyKvAUser", // "--pass", "MyVeryHardToGuesspassword" //}; #endif var options = new Options(); if (Parser.Default.ParseArguments(args, options)) { var stuffToTest = new Queue <Action <Options> >(5); if (options.All) { stuffToTest.Enqueue(Tester.RunConnectivityTests); stuffToTest.Enqueue(Tester.RunFunctionalityTests); stuffToTest.Enqueue(Tester.RunSecurityTests); stuffToTest.Enqueue(Tester.RunPerformanceTests); stuffToTest.Enqueue(Tester.RunContinuityTests); } else { // check for each option individually if (options.Connectivity) { stuffToTest.Enqueue(Tester.RunConnectivityTests); } if (options.Functionality) { stuffToTest.Enqueue(Tester.RunFunctionalityTests); } if (options.Security) { stuffToTest.Enqueue(Tester.RunSecurityTests); } if (options.Performance) { stuffToTest.Enqueue(Tester.RunPerformanceTests); } if (options.Continuity) { stuffToTest.Enqueue(Tester.RunContinuityTests); } } if (stuffToTest.Count > 0) { Out.Info(TestResultBase.WriteHeader()); foreach (var methodPointer in stuffToTest) { methodPointer(options); } } } else { HelpText txt = HelpText.AutoBuild(options); Out.Info(txt.ToString()); } }
internal abstract Task ReportStatusAsync(TestResultBase testResult);
// TODO: Remove this function once the TRC support the two new endpoint properly. public static async Task ReportTestResultUntilSuccessAsync(TestResultReportingClient apiClient, ILogger logger, TestResultBase testResult, CancellationToken cancellationToken) { bool isSuccessful = false; while (!isSuccessful && !cancellationToken.IsCancellationRequested) { try { logger.LogInformation($"Sending test result: Source={testResult.Source}, Type={testResult.ResultType}, CreatedAt={testResult.CreatedAt}, Result={testResult.GetFormattedResult()}"); await apiClient.ReportResultAsync(testResult.ToTestOperationResultDto(), cancellationToken); isSuccessful = true; } catch (Exception ex) { logger.LogDebug(ex, "Exception caught in ReportTestResultAsync()"); } } }
public static async Task ReportTestResultAsync(TestResultReportingClient apiClient, ILogger logger, TestResultBase testResult, CancellationToken cancellationToken = default(CancellationToken)) { logger.LogInformation($"Sending test result: Source={testResult.Source}, Type={testResult.ResultType}, CreatedAt={testResult.CreatedAt}, Result={testResult.GetFormattedResult()}"); await apiClient.ReportResultAsync(testResult.ToTestOperationResultDto(), cancellationToken); }