private static void RunTest(Options o, string subTest, Action <string> logger, ITest test, int i)
{
TestResultBase result = null;
try
{
result = test.Test(o);
result.StepNr = i;
result.SubTest = subTest;
}
catch (Exception ex)
{
if (result == null)
{
result = new GenericTestResult();
}
result.StepNr = i;
result.SubTest = subTest;
result.Status = TestResult.FAIL;
result.CauseOfFailure = ex.Message;
result.ExtraInformation = ex.StackTrace;
}
if (logger != null)
{
logger(result.ToString());
}
}
public static async Task <int> MainAsync()
{
Logger.LogInformation($"Starting DirectMethodSender with the following settings:\r\n{Settings.Current}");
(CancellationTokenSource cts, ManualResetEventSlim completed, Option <object> handler) = ShutdownHandler.Init(TimeSpan.FromSeconds(5), Logger);
DirectMethodSenderBase directMethodClient = null;
ReporterClientBase reportClient = null;
try
{
Guid batchId = Guid.NewGuid();
Logger.LogInformation($"Batch Id={batchId}");
Logger.LogInformation($"Load gen delay start for {Settings.Current.TestStartDelay}.");
await Task.Delay(Settings.Current.TestStartDelay, cts.Token);
DateTime testStartAt = DateTime.UtcNow;
directMethodClient = await CreateClientAsync(Settings.Current.InvocationSource);
reportClient = ReporterClientBase.Create(
Logger,
Settings.Current.ReportingEndpointUrl,
Settings.Current.TransportType);
while (!cts.Token.IsCancellationRequested && IsTestTimeUp(testStartAt))
{
(HttpStatusCode result, ulong dmCounter) = await directMethodClient.InvokeDirectMethodAsync(Settings.Current.DirectMethodName, cts);
// Generate a testResult type depending on the reporting endpoint
TestResultBase testResult = ConstructTestResult(
Settings.Current.DirectMethodResultType,
batchId,
dmCounter,
result);
await reportClient.SendTestResultAsync(testResult);
await Task.Delay(Settings.Current.DirectMethodDelay, cts.Token);
}
await cts.Token.WhenCanceled();
}
catch (Exception e)
{
Logger.LogError(e, "Error occurred during direct method sender test setup");
}
finally
{
// Implicit CloseAsync()
directMethodClient?.Dispose();
reportClient?.Dispose();
}
completed.Set();
handler.ForEach(h => GC.KeepAlive(h));
Logger.LogInformation("DirectMethodSender Main() finished.");
return(0);
}
public TestResultBase Test(Options o)
{
var testResult = new GenericTestResult
{
ShortDescription = "Performance (uptime) " + RelativeUrl,
Status = TestResult.INCONCLUSIVE
};
DateTime current = DateTime.Now; // to check if we elapsed total seconds.
var results = new List <Tuple <long, TestResult> >();
//long lastMessageTime = 0;
while (true)
{
var test = new CheckCompletenessResponseTest
{
RelativeUrl = RelativeUrl, //string.Format(RelativeUrl, lastMessageTime)
FieldsThatShouldBePresent = FieldsThatShouldBePresent
};
TestResultBase res = test.Test(o);
results.Add(new Tuple <long, TestResult>(test.RequestDuration, res.Status));
//if (test.ResponseData != null)
//{
// // now store the last updated time from the previous
// string data = test.ResponseData;
//}
Thread.Sleep(IntervalTime);
if (DateTime.Now.Subtract(current).TotalSeconds >= TestDuration)
{
break;
}
}
IEnumerable <Tuple <long, TestResult> > succesCount = results.Where(t => t.Item2 == TestResult.OK).ToList();
var totalResultCount = results.Count;
double percentageSuccess = totalResultCount == 0 ? 0 : succesCount.Count() / (double)totalResultCount;
double avgTime = succesCount.Any() ? succesCount.Select(t => t.Item1).Average() : -1;
testResult.ExtraInformation = Math.Round(avgTime, 0) + " ms average time";
if (percentageSuccess >= 0.95)
{
testResult.Status = TestResult.OK;
}
else
{
testResult.Status = TestResult.FAIL;
testResult.CauseOfFailure = "Percentage success: " + Math.Round(percentageSuccess * 100) + "%";
}
return(testResult);
}
internal async Task SendTestResultAsync(TestResultBase testResult)
{
Preconditions.CheckNotNull(testResult, nameof(testResult));
try
{
await this.ReportStatusAsync(testResult);
}
catch (Exception e)
{
this.logger.LogError(e, "Failed to send testResult");
}
}
internal static void RunContinuityTests(Options options)
{
var test = new KvGPerformanceTest
{
RelativeUrl = FcdRetrieveRequest(),
IntervalTime = Options.PerformanceTestInterval,
TestDuration = Options.PerformanceTestDuration
};
TestResultBase result1 = test.Test(options);
result1.StepNr = 1;
result1.SubTest = "Continuity";
Out.Info(result1.ToString());
}
internal override async Task ReportStatusAsync(TestResultBase testResult)
{
switch (testResult.GetType().Name)
{
case nameof(LegacyDirectMethodTestResult):
LegacyDirectMethodTestResult shadowReport = testResult as LegacyDirectMethodTestResult;
// The Old End-to-End test framework checks if there is an event in the eventHub for a particular module
// to determine if the test case is passed/failed. Hence, this function need to check if the testResult status is Http.OK
// before sending an event.
if (shadowReport.Result == HttpStatusCode.OK.ToString())
{
await this.ModuleClient.SendEventAsync("AnyOutput", new Message(Encoding.UTF8.GetBytes($"Source:{shadowReport.Source} CreatedAt:{shadowReport.CreatedAt}.")));
}
break;
default:
throw new NotSupportedException($"{this.GetType().Name} does not have an implementation for {testResult.GetType().Name} type");
}
}
internal static void RunPerformanceTests(Options options)
{
var test = new KvGPerformanceTest
{
RelativeUrl = FcdRetrieveRequestWithTime(),
IntervalTime = Options.PerformanceTestInterval,
TestDuration = Options.PerformanceTestDuration,
};
TestResultBase result1 = test.Test(options);
result1.StepNr = 1;
result1.SubTest = "Performance";
Out.Info(result1.ToString());
// test 2
if (test.GenerationTimeValid.Value)
{
Out.Info(new GenericTestResult
{
SubTest = "Performance",
StepNr = 2,
ShortDescription = "Update interval " + FcdRetrieveRequest(),
Status = TestResult.OK
}.ToString());
}
else
{
Out.Info(new GenericTestResult
{
SubTest = "Performance",
StepNr = 2,
ShortDescription = "Update interval " + FcdRetrieveRequest(),
Status = TestResult.FAIL,
CauseOfFailure = "Generation time did not update properly."
}.ToString());
}
}
public static async Task ReportTestResultAsync(TestResultReportingClient apiClient, ILogger logger, TestResultBase testResult)
{
try
{
logger.LogInformation($"Sending test result: Source={testResult.Source}, Type={testResult.ResultType}, CreatedAt={testResult.CreatedAt}, Result={testResult.GetFormattedResult()}");
await apiClient.ReportResultAsync(testResult.ToTestOperationResultDto());
}
catch (Exception e)
{
logger.LogError(e, "Failed call to report status to TestResultCoordinator");
}
}
internal override async Task ReportStatusAsync(TestResultBase testResult)
{
Preconditions.CheckNotNull(testResult, nameof(testResult));
await this.testResultReportingClient.ReportResultAsync(testResult.ToTestOperationResultDto());
}
public TestResultBase Test(Options o)
{
var testResult = new GenericTestResult
{
ShortDescription = "Performance (uptime) " + RelativeUrl,
Status = TestResult.INCONCLUSIVE
};
DateTime current = DateTime.Now; // to check if we elapsed total seconds.
var results = new List <Tuple <long, TestResult> >();
long lastMessageTime = 0;
while (true)
{
var test = new CheckCompletenessResponseTest
{
RelativeUrl = string.Format("/fcd?last_message_time={0}", lastMessageTime),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInFCD(),
TestResultWhenNoData = TestResult.OK
};
TestResultBase res = test.Test(o);
results.Add(new Tuple <long, TestResult>(test.RequestDuration, res.Status));
if (test.ResponseData != null)
{
// now store the last updated time from the previous
string data = test.ResponseData;
// deserialize
var fcdMsg = JsonConvert.DeserializeObject <FcdMessage>(data);
if (lastMessageTime == fcdMsg.message_time)
{
GenerationTimeValid = false;
}
else
{
if (!GenerationTimeValid.HasValue)
{
GenerationTimeValid = true;
}
}
lastMessageTime = fcdMsg.message_time;
}
else
{
GenerationTimeValid = false;
}
Thread.Sleep(IntervalTime);
if (DateTime.Now.Subtract(current).TotalSeconds >= TestDuration)
{
break;
}
}
IEnumerable <Tuple <long, TestResult> > succesCount = results.Where(t => t.Item2 == TestResult.OK);
double percentageSuccess = succesCount.Count() / (double)results.Count;
double avgTime = results.Select(t => t.Item1).Average();
testResult.ExtraInformation = Math.Round(avgTime, 0) + " ms average time";
if (percentageSuccess >= 0.95)
{
testResult.Status = TestResult.OK;
}
else
{
testResult.Status = TestResult.FAIL;
testResult.CauseOfFailure = "Percentage success: " + Math.Round(percentageSuccess * 100, 0) + " %";
}
return(testResult);
}
private static void Main(string[] args)
{
#if DEBUG
//args = new[]
//{
// "--all",
// "--url", "http://yourtesturl",
// "--apikey", "MyFamousApiKey",
// "--user", "TheOneAndOnlyKvAUser",
// "--pass", "MyVeryHardToGuesspassword"
//};
#endif
var options = new Options();
if (Parser.Default.ParseArguments(args, options))
{
var stuffToTest = new Queue <Action <Options> >(5);
if (options.All)
{
stuffToTest.Enqueue(Tester.RunConnectivityTests);
stuffToTest.Enqueue(Tester.RunFunctionalityTests);
stuffToTest.Enqueue(Tester.RunSecurityTests);
stuffToTest.Enqueue(Tester.RunPerformanceTests);
stuffToTest.Enqueue(Tester.RunContinuityTests);
}
else
{
// check for each option individually
if (options.Connectivity)
{
stuffToTest.Enqueue(Tester.RunConnectivityTests);
}
if (options.Functionality)
{
stuffToTest.Enqueue(Tester.RunFunctionalityTests);
}
if (options.Security)
{
stuffToTest.Enqueue(Tester.RunSecurityTests);
}
if (options.Performance)
{
stuffToTest.Enqueue(Tester.RunPerformanceTests);
}
if (options.Continuity)
{
stuffToTest.Enqueue(Tester.RunContinuityTests);
}
}
if (stuffToTest.Count > 0)
{
Out.Info(TestResultBase.WriteHeader());
foreach (var methodPointer in stuffToTest)
{
methodPointer(options);
}
}
}
else
{
HelpText txt = HelpText.AutoBuild(options);
Out.Info(txt.ToString());
}
}
internal abstract Task ReportStatusAsync(TestResultBase testResult);
// TODO: Remove this function once the TRC support the two new endpoint properly.
public static async Task ReportTestResultUntilSuccessAsync(TestResultReportingClient apiClient, ILogger logger, TestResultBase testResult, CancellationToken cancellationToken)
{
bool isSuccessful = false;
while (!isSuccessful && !cancellationToken.IsCancellationRequested)
{
try
{
logger.LogInformation($"Sending test result: Source={testResult.Source}, Type={testResult.ResultType}, CreatedAt={testResult.CreatedAt}, Result={testResult.GetFormattedResult()}");
await apiClient.ReportResultAsync(testResult.ToTestOperationResultDto(), cancellationToken);
isSuccessful = true;
}
catch (Exception ex)
{
logger.LogDebug(ex, "Exception caught in ReportTestResultAsync()");
}
}
}
public static async Task ReportTestResultAsync(TestResultReportingClient apiClient, ILogger logger, TestResultBase testResult, CancellationToken cancellationToken = default(CancellationToken))
{
logger.LogInformation($"Sending test result: Source={testResult.Source}, Type={testResult.ResultType}, CreatedAt={testResult.CreatedAt}, Result={testResult.GetFormattedResult()}");
await apiClient.ReportResultAsync(testResult.ToTestOperationResultDto(), cancellationToken);
}
