public TestResultBase Test(Options o)
{
var res = new GenericTestResult
{
ShortDescription = "HTTP response valid",
Status = TestResult.INCONCLUSIVE
};
try
{
var c = new WebClient();
byte[] data = c.DownloadData(o.Url + RelativeUrl);
if (HttpValidationTests.IsValidJson(c.ResponseHeaders) && HttpValidationTests.IsValidCharsetUtf8(c.ResponseHeaders))
res.Status = TestResult.OK;
else
{
res.Status = TestResult.FAIL;
res.ExtraInformation = "No response header Content-Type. Invalid HTTP";
}
}
catch (Exception ex)
{
res.Status = TestResult.FAIL;
res.CauseOfFailure = ex.Message;
}
return res;
}
public TestResultBase Test(Options o)
{
var res = new GenericTestResult
{
ShortDescription = "Sanity light check",
Status = TestResult.INCONCLUSIVE
};
try
{
WebClientEx webClient = SetupWebClient(o);
string responseData = Encoding.UTF8.GetString(webClient.DownloadData(o.Url + RelativeUrl));
if (responseData == "[]" || webClient.StatusCode == HttpStatusCode.NoContent)
{
res.Status = TestResult.INCONCLUSIVE;
res.ExtraInformation = "NO DATA at " + RelativeUrl;
return res;
}
var responseObj = ServiceStack.Text.JsonSerializer.DeserializeFromString(responseData, TypeToDeserialize);
// check if response is ok according to the handler
res.Status = CheckValidDataInsideFunctionHandler(responseObj) ? TestResult.OK : TestResult.FAIL;
}
catch (Exception ex)
{
res.Status = TestResult.FAIL;
res.CauseOfFailure = ex.Message;
}
finally
{
res.ExtraInformation += " Url: " + RelativeUrl;
}
return res;
}
public TestResultBase Test(Options o)
{
var p = new Uri(o.Url);
var res = new GenericTestResult
{
ShortDescription = "TCP connection port " + p.Port,
Status = TestResult.OK
};
try
{
var client = new TcpClient();
client.Connect(p.DnsSafeHost, p.Port);
res.Status = TestResult.OK;
client.Close();
}
catch (Exception ex)
{
res.Status = TestResult.FAIL;
res.CauseOfFailure = ex.Message;
}
return res;
}
public TestResultBase Test(Options o)
{
var res = new GenericTestResult
{
ShortDescription =
string.Format("Calling {0} with {1} credentials", RelativeUrl,
UseCredentials == AuthenticationMode.UseInvalidCredentials ? "invalid" : "no"),
Status = TestResult.INCONCLUSIVE
};
try
{
WebClient c = SetupWebClient(o, UseCredentials);
string data = Encoding.UTF8.GetString(c.DownloadData(o.Url + RelativeUrl));
res.Status = TestResult.FAIL;
res.CauseOfFailure = "HTTP OK where it should be 401.";
}
catch (WebException wex)
{
var response = (HttpWebResponse) wex.Response;
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
res.Status = TestResult.OK;
res.ExtraInformation = "Ok - received " + response.StatusCode + " (expected behaviour)";
}
}
catch (Exception ex)
{
res.Status = TestResult.FAIL;
res.CauseOfFailure = ex.Message;
}
return res;
}
internal static void RunConnectivityTests(Options options)
{
RunTests(options, "Connectivity", Out.Info,
new PingTest(),
new TcpConnectionTest(),
new HttpServiceOnlineTest(),
new HttpResponseValidTest { RelativeUrl = GetTrafficStateLatest() });
}
/// <summary>
/// General todo: refactor the url getters to a static method for flexibility purposes.
/// </summary>
/// <param name="options"></param>
internal static void RunConnectivityTests(Options options)
{
RunTests(options, "Connectivity", Out.Info,
new PingTest(),
new TcpConnectionTest(),
new HttpServiceOnlineTest(),
new HttpResponseValidTest { RelativeUrl = FcdRetrieveRequestWithTime() });
}
/// <summary>
/// Run these tests for the category 'subTest', given Options o.
/// </summary>
/// <param name="o"></param>
/// <param name="subTest"></param>
/// <param name="tests"></param>
protected static void RunTests(Options o, string subTest, Action<string> logger, params ITest[] tests)
{
for (int i = 1; i <= tests.Length; i++)
{
var test = tests[i - 1];
RunTest(o, subTest, logger, test, i);
}
}
internal static void RunFunctionalityTests(Options options)
{
RunTests(options, "Functionality", Out.Info,
new HttpResponseValidTest { RelativeUrl = FcdRetrieveRequestWithTime() },
new CheckCompletenessResponseTest
{
RelativeUrl = FcdRetrieveRequestWithTime(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInFCD()
},
new SanityCheck()
{
RelativeUrl = FcdRetrieveRequestWithTime(),
TypeToDeserialize = typeof(FcdMessage),
CheckValidDataInsideFunctionHandler = o =>
{
var fcd = (FcdMessage)o;
if (fcd.provider_id == null) return false;
if (fcd.fcd_records.Count == 0) return false;
foreach (var fcdrecord in fcd.fcd_records)
{
if (fcdrecord.user_id_anonymous == 0) return false;
foreach (var trail in fcdrecord.trail)
{
if (trail.generation_time.ToUniversalTime() > DateTime.UtcNow)
{
return false;
}
if (trail.speed < 0 || trail.speed > 150)
{
return false;
}
if (trail.ref_position_lat < 51.482733 || trail.ref_position_lat > 51.555198)
{
return false;
}
if (trail.ref_position_lon < 5.116459 || trail.ref_position_lon > 5.393983)
{
return false;
}
if (trail.heading < 0 || trail.heading > 360)
{
return false;
}
// Check Not implimented no valid time to countermeasure with
//if (trail.generation_time.ToUniversalTime() < DateTime.UtcNow.AddMinutes(-2))
//{
// return false;
//}
}
}
return true;
}
}
);
}
public TestResultBase Test(Options o)
{
var res = new GenericTestResult
{
ShortDescription = "Check completeness response",
Status = TestResult.INCONCLUSIVE
};
try
{
var c = SetupWebClient(o);
if (OnSetupWebClient != null)
OnSetupWebClient(c);
var watch = new Stopwatch();
watch.Start();
string data = Encoding.UTF8.GetString(c.DownloadData(o.Url + RelativeUrl));
if (data == "[]" || c.StatusCode == HttpStatusCode.NoContent)
{
res.Status = TestResultWhenNoData.HasValue ? TestResultWhenNoData.Value : TestResult.INCONCLUSIVE;
res.ExtraInformation = "NO DATA at " + RelativeUrl;
return res;
}
watch.Stop();
ResponseData = data;
RequestDuration = watch.ElapsedMilliseconds;
if (HttpValidationTests.IsValidJson(c.ResponseHeaders) && HttpValidationTests.IsValidCharsetUtf8(c.ResponseHeaders))
{
if (FieldsThatShouldBePresent.All(data.Contains))
res.Status = TestResult.OK;
else
{
res.Status = TestResult.FAIL;
res.ExtraInformation = "Response DTO did not contain all the required fields";
}
}
else
{
res.Status = TestResult.FAIL;
res.ExtraInformation = "No response header Content-Type. Invalid HTTP";
}
}
catch (Exception ex)
{
res.Status = TestResult.FAIL;
res.CauseOfFailure = ex.Message;
}
finally
{
res.ExtraInformation += " Url: " + RelativeUrl;
}
return res;
}
public TestResultBase Test(Options o)
{
var res = new GenericTestResult
{
ShortDescription = "Calling HTTP (no SSL)",
Status = TestResult.INCONCLUSIVE
};
try
{
WebClient c = SetupWebClient(o);
string data = Encoding.UTF8.GetString(c.DownloadData((o.Url + RelativeUrl).Replace("https", "http")));
if (c.ResponseHeaders["Location"] != null && c.ResponseHeaders["Location"].Contains("https"))
{
res.Status = TestResult.OK;
res.ExtraInformation = "Using redirect";
}
if (HttpValidationTests.IsValidJson(c.ResponseHeaders) && HttpValidationTests.IsValidCharsetUtf8(c.ResponseHeaders))
{
if (data.Contains("generation_time") && data.Contains("ref_position_lon") &&
data.Contains("ref_position_lat") && data.Contains("speed")
&& data.Contains("heading") && data.Contains("provider_id") &&
data.Contains("user_id_anonymous"))
{
res.Status = TestResult.FAIL;
res.ExtraInformation = "Received http 200 ok with all data fields present.";
}
else
{
res.Status = TestResult.FAIL;
res.ExtraInformation = "Received http 200 ok, but required data fields were not present.";
}
}
else
{
res.Status = TestResult.OK;
res.ExtraInformation = "No json data received. ";
}
}
catch (WebException wex)
{
var response = (HttpWebResponse) wex.Response;
if (response.StatusCode != HttpStatusCode.Accepted && response.StatusCode != HttpStatusCode.NotModified)
{
res.Status = TestResult.OK;
res.ExtraInformation = "Ok - received " + response.StatusCode + " (expected behaviour)";
}
}
catch (Exception ex)
{
res.Status = TestResult.FAIL;
res.CauseOfFailure = ex.Message;
}
return res;
}
/// <summary>
/// Run these tests for the category 'subTest', given Options o.
/// </summary>
/// <param name="o"></param>
/// <param name="subTest"></param>
/// <param name="tests"></param>
protected static void RunTests(Options o, string subTest, Action<string> logger, params ITest[] tests)
{
for (int i = 1; i <= tests.Length; i++)
{
TestResultBase result = tests[i - 1].Test(o);
result.StepNr = i;
result.SubTest = subTest;
if (logger != null)
logger(result.ToString());
}
}
public TestResultBase Test(Options o)
{
var testResult = new GenericTestResult
{
ShortDescription = "Performance (uptime) " + RelativeUrl,
Status = TestResult.INCONCLUSIVE
};
DateTime current = DateTime.Now; // to check if we elapsed total seconds.
var results = new List<Tuple<long, TestResult>>();
//long lastMessageTime = 0;
while (true)
{
var test = new CheckCompletenessResponseTest
{
RelativeUrl = RelativeUrl, //string.Format(RelativeUrl, lastMessageTime)
FieldsThatShouldBePresent = FieldsThatShouldBePresent
};
TestResultBase res = test.Test(o);
results.Add(new Tuple<long, TestResult>(test.RequestDuration, res.Status));
//if (test.ResponseData != null)
//{
// // now store the last updated time from the previous
// string data = test.ResponseData;
//}
Thread.Sleep(IntervalTime);
if (DateTime.Now.Subtract(current).TotalSeconds >= TestDuration)
break;
}
IEnumerable<Tuple<long, TestResult>> succesCount = results.Where(t => t.Item2 == TestResult.OK).ToList();
var totalResultCount = results.Count;
double percentageSuccess = totalResultCount == 0 ? 0 : succesCount.Count() / (double)totalResultCount;
double avgTime = succesCount.Any() ? succesCount.Select(t => t.Item1).Average() : -1;
testResult.ExtraInformation = Math.Round(avgTime, 0) + " ms average time";
if (percentageSuccess >= 0.95)
{
testResult.Status = TestResult.OK;
}
else
{
testResult.Status = TestResult.FAIL;
testResult.CauseOfFailure = "Percentage success: " + Math.Round(percentageSuccess * 100) + "%";
}
return testResult;
}
/// <summary>
/// Run these tests for the category 'subTest', given Options o.
/// </summary>
/// <param name="o"></param>
/// <param name="subTest"></param>
/// <param name="tests"></param>
protected static void RunTestsParallel(Options o, string subTest, Action<string> logger, params ITest[] tests)
{
var tasks = new List<Task>();
for (int i = 1; i <= tests.Length; i++)
{
var test = tests[i - 1];
int i1 = i;
var task = Task.Factory.StartNew(() => RunTest(o, subTest, logger, test, i1));
tasks.Add(task);
// We sleep 30 seconds after every test. This gives us a slow and sequential start up
Thread.Sleep(TimeSpan.FromSeconds(30));
}
Task.WaitAll(tasks.ToArray());
}
private static void Main(string[] args)
{
#if DEBUG
//args = new[]
//{
// "--all",
// "--url", "http://yourtesturl",
// "--apikey", "MyFamousApiKey",
// "--user", "TheOneAndOnlyKvAUser",
// "--pass", "MyVeryHardToGuesspassword"
//};
#endif
var options = new Options();
if (Parser.Default.ParseArguments(args, options))
{
var stuffToTest = new Queue<Action<Options>>(5);
if (options.All)
{
stuffToTest.Enqueue(Tester.RunConnectivityTests);
stuffToTest.Enqueue(Tester.RunFunctionalityTests);
stuffToTest.Enqueue(Tester.RunSecurityTests);
stuffToTest.Enqueue(Tester.RunPerformanceTests);
stuffToTest.Enqueue(Tester.RunContinuityTests);
}
else
{
// check for each option individually
if (options.Connectivity) stuffToTest.Enqueue(Tester.RunConnectivityTests);
if (options.Functionality) stuffToTest.Enqueue(Tester.RunFunctionalityTests);
if (options.Security) stuffToTest.Enqueue(Tester.RunSecurityTests);
if (options.Performance) stuffToTest.Enqueue(Tester.RunPerformanceTests);
if (options.Continuity) stuffToTest.Enqueue(Tester.RunContinuityTests);
}
if (stuffToTest.Count > 0)
{
Out.Info(TestResultBase.WriteHeader());
foreach (var methodPointer in stuffToTest)
methodPointer(options);
}
}
else
{
HelpText txt = HelpText.AutoBuild(options);
Out.Info(txt.ToString());
}
}
/// <summary>
/// This will check the input parameters and provide the options to the webclient instance.
/// It supports api key and basic auth at the moment.
/// </summary>
/// <param name="c"></param>
/// <param name="o"></param>
public static void ConfigureAuth(WebClient c, Options o)
{
if (!String.IsNullOrEmpty(o.ApiKey))
{
if (!String.IsNullOrEmpty(o.ApiKeyHeader))
{
c.Headers.Add(o.ApiKeyHeader, o.ApiKey);
}
else
{
c.Headers.Add(Options.DefaultApiKeyHeaderName, o.ApiKey);
}
}
if (!String.IsNullOrEmpty(o.User))
{
c.Credentials = new NetworkCredential(o.User, o.Pass);
}
}
/// <summary>
/// Test ICMP/ PING Connectivity
/// </summary>
/// <param name="o"></param>
public TestResultBase Test(Options o)
{
string domain = new Uri(o.Url).DnsSafeHost;
var res = new GenericTestResult
{
ShortDescription = "Ping",
Status = TestResult.OK
};
long rtt = 0;
int nrSuccess = 0;
try
{
for (int i = 0; i < NrOfPingChecks; i++)
{
PingReply pingReply = new Ping().Send(domain);
if (pingReply.Status == IPStatus.Success)
{
long replyTimes = pingReply.RoundtripTime;
// successd
rtt += pingReply.RoundtripTime;
nrSuccess++;
}
else
{
// fail
res.Status = TestResult.FAIL;
res.CauseOfFailure = pingReply.Status.ToString();
}
}
}
catch (Exception ex)
{
res.Status = TestResult.FAIL;
res.CauseOfFailure = ex.Message;
}
res.ExtraInformation = rtt != 0 ? (rtt/nrSuccess) + " ms avg RTT" : "0";
return res;
}
protected HttpWebRequest SetupWebRequest(Options o, string url,
AuthenticationMode validCredentials = AuthenticationMode.UseValidCredentials)
{
var request = (HttpWebRequest) WebRequest.Create(url);
request.Method = "GET";
request.Accept = "application/json";
if (!String.IsNullOrEmpty(o.ApiKey) && validCredentials != AuthenticationMode.UseNoCredentials)
{
string headerName = !String.IsNullOrEmpty(o.ApiKeyHeader)
? o.ApiKeyHeader
: Options.DefaultApiKeyHeaderName;
request.Headers.Add(headerName,
validCredentials == AuthenticationMode.UseValidCredentials ? o.ApiKey : Options.InvalidApiKey);
}
if (!String.IsNullOrEmpty(o.User) && validCredentials != AuthenticationMode.UseNoCredentials)
{
request.Credentials = validCredentials == AuthenticationMode.UseValidCredentials
? new NetworkCredential(o.User, o.Pass)
: new NetworkCredential(Options.InvalidUserName, Options.InvalidPassword);
}
return request;
}
protected WebClientEx SetupWebClient(Options o,
AuthenticationMode validCredentials = AuthenticationMode.UseValidCredentials)
{
var c = new WebClientEx();
c.Headers.Add(HttpRequestHeader.Accept, "application/json");
if (!String.IsNullOrEmpty(o.ApiKey) && validCredentials != AuthenticationMode.UseNoCredentials)
{
string headerName = !String.IsNullOrEmpty(o.ApiKeyHeader)
? o.ApiKeyHeader
: Options.DefaultApiKeyHeaderName;
c.Headers.Add(headerName,
validCredentials == AuthenticationMode.UseValidCredentials ? o.ApiKey : Options.InvalidApiKey);
}
if (!String.IsNullOrEmpty(o.User) && validCredentials != AuthenticationMode.UseNoCredentials)
{
c.Credentials = validCredentials == AuthenticationMode.UseValidCredentials
? new NetworkCredential(o.User, o.Pass)
: new NetworkCredential(Options.InvalidUserName, Options.InvalidPassword);
}
return c;
}
internal static void RunSecurityTests(Options options)
{
RunTests(options, "Security", Out.Info,
new CheckCompletenessResponseTest
{
RelativeUrl = FcdRetrieveRequestWithTime(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInFCD()
},
// 1 // note: this is the same test as runfunctionality - the last one.
new CheckCertificateTest { RelativeUrl = FcdRetrieveRequestWithTime() }, // 2
new CallingWithInvalidCredentials
{
RelativeUrl = FcdRetrieveRequestWithTime(),
UseCredentials = HttpTestBase.AuthenticationMode.UseInvalidCredentials
}, // 3
new CallingWithInvalidCredentials
{
RelativeUrl = FcdRetrieveRequestWithTime(),
UseCredentials = HttpTestBase.AuthenticationMode.UseNoCredentials
}, // 4
new CheckHttpAvailableTest { RelativeUrl = FcdRetrieveRequestWithTime() } // 5
);
}
public TestResultBase Test(Options o)
{
var res = new GenericTestResult
{
ShortDescription = "HTTP service running",
Status = TestResult.INCONCLUSIVE
};
try
{
var c = new WebClient();
byte[] data = c.DownloadData(o.Url);
if (c.ResponseHeaders.AllKeys.Contains("Content-Type"))
res.Status = TestResult.OK;
else
{
res.Status = TestResult.FAIL;
res.ExtraInformation = "No response header Content-Type. Invalid HTTP";
}
}
catch (WebException wex)
{
var response = (HttpWebResponse) wex.Response;
//if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.ServiceUnavailable || response.StatusCode == HttpStatusCode.NotFound)
//{
res.Status = TestResult.OK;
res.ExtraInformation = "Ok - HTTP confirmed. Received " + response.StatusCode;
//}
}
catch (Exception ex)
{
res.Status = TestResult.FAIL;
res.CauseOfFailure = ex.Message;
}
return res;
}
private static void RunTest(Options o, string subTest, Action<string> logger, ITest test, int i)
{
TestResultBase result = null;
try
{
result = test.Test(o);
result.StepNr = i;
result.SubTest = subTest;
}
catch (Exception ex)
{
if (result == null)
{
result = new GenericTestResult();
}
result.StepNr = i;
result.SubTest = subTest;
result.Status = TestResult.FAIL;
result.CauseOfFailure = ex.Message;
result.ExtraInformation = ex.StackTrace;
}
if (logger != null)
logger(result.ToString());
}
internal static void RunFunctionalityTests(Options options)
{
RunTests(options, "Functionality", Out.Info,
new HttpResponseValidTest { RelativeUrl = GetTrafficStateLatest() }, // traffic state / latest
new CheckCompletenessResponseTest
{
RelativeUrl = GetTrafficStateLatest(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInTrafficState()
},
new SanityCheck()
{
RelativeUrl = GetTrafficStateLatest(),
TypeToDeserialize = typeof(List<Trafficstate>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Trafficstate>)o)
{
if (i.publication_time.ToUniversalTime() > DateTime.UtcNow)
return false;
if (i.speed >= 150 || i.speed < 0)
return false;
if (i.max_speed > 130 || i.max_speed < 0)
return false;
if (i.freeflow_speed > 130 || i.freeflow_speed < 0)
return false;
}
return true;
}
},
new HttpResponseValidTest { RelativeUrl = GetTrafficStateHistoric(options.SegmentCsv) }, // traffic state historic
new CheckCompletenessResponseTest
{
RelativeUrl = GetTrafficStateHistoric(options.SegmentCsv),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInTrafficState()
},
new SanityCheck()
{
RelativeUrl = GetTrafficStateHistoric(options.SegmentCsv),
TypeToDeserialize = typeof(List<Trafficstate>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Trafficstate>)o)
{
if (i.publication_time.ToUniversalTime() > DateTime.UtcNow)
return false;
if (i.speed >= 150 || i.speed < 0)
return false;
if (i.max_speed > 130 || i.max_speed < 0)
return false;
if (i.freeflow_speed > 130 || i.freeflow_speed < 0)
return false;
}
return true;
}
},
new HttpResponseValidTest { RelativeUrl = GetEventsLatest() }, // Events/latest
new CheckCompletenessResponseTest
{
RelativeUrl = GetEventsLatest(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInEvents()
},
new SanityCheck()
{
RelativeUrl = GetEventsLatest(),
TypeToDeserialize = typeof(List<TrafficEvent>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<TrafficEvent>)o)
{
if (i.start_time.Value.ToUniversalTime() >= DateTime.UtcNow)
return false;
if (i.event_code.Any(eventCode => eventCode < 1 || eventCode > 255))
return false;
if (i.start_time > i.expected_end_time)
// // note: if traffic events in the future are allowed, this should also be removed.
return false;
if (i.expected_end_time < DateTime.UtcNow)
return false;
}
return true;
}
},
// mandatory: start_time, end_time
new HttpResponseValidTest { RelativeUrl = GetEventsHistoric() },
new CheckCompletenessResponseTest
{
RelativeUrl = GetEventsHistoric(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInEvents()
},
new SanityCheck()
{
RelativeUrl = GetEventsHistoric(),
TypeToDeserialize = typeof(List<TrafficEvent>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<TrafficEvent>)o)
{
if (i.start_time.Value.ToUniversalTime() >= DateTime.UtcNow)
return false;
if (i.event_code.Any(eventCode => eventCode < 1 || eventCode > 255))
return false;
if (i.start_time > i.expected_end_time)
// // note: if traffic events in the future are allowed, this should also be removed.
return false;
//if (i.expected_end_time < DateTime.UtcNow) -> should be ok for historic
// return false;
}
return true;
}
},
new HttpResponseValidTest { RelativeUrl = GetWeatherStateLatest() },
new CheckCompletenessResponseTest
{
RelativeUrl = GetWeatherStateLatest(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInWeatherState()
},
new SanityCheck()
{
RelativeUrl = GetWeatherStateLatest(),
TypeToDeserialize = typeof(List<Weatherstate>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Weatherstate>)o)
{
if (i.measurement_time.ToUniversalTime() > DateTime.UtcNow)
return false;
if (i.segment_id.Count == 0)
return false;
}
return true;
}
},
// mandatory: start_time, end_time
new HttpResponseValidTest { RelativeUrl = GetWeatherStateHistoric() },
new CheckCompletenessResponseTest
{
RelativeUrl = GetWeatherStateHistoric(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInWeatherState()
},
new SanityCheck()
{
RelativeUrl = GetWeatherStateHistoric(),
TypeToDeserialize = typeof(List<Weatherstate>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Weatherstate>)o)
{
if (i.measurement_time.ToUniversalTime() > DateTime.UtcNow)
return false;
if (i.segment_id.Count == 0)
return false;
}
return true;
}
},
new HttpResponseValidTest { RelativeUrl = GetWeatherForecastLatest() },
new CheckCompletenessResponseTest
{
RelativeUrl = GetWeatherForecastLatest(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInWeatherForecast()
},
new SanityCheck()
{
RelativeUrl = GetWeatherForecastLatest(),
TypeToDeserialize = typeof(List<Weatherforecast>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Weatherforecast>)o)
{
//if (i.measurement_time.ToUniversalTime() > DateTime.UtcNow)
// return false;
if (i.segment_id.Count == 0)
return false;
//if (i.measurement_time.ToUniversalTime() < DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(15)))
// return false;
}
return true;
}
},
// mandatory: start_time, end_time
new HttpResponseValidTest { RelativeUrl = GetWeatherForecastHistoric() },
new CheckCompletenessResponseTest
{
RelativeUrl = GetWeatherForecastHistoric(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInWeatherForecast()
},
new SanityCheck()
{
RelativeUrl = GetWeatherForecastHistoric(),
TypeToDeserialize = typeof(List<Weatherforecast>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Weatherforecast>)o)
{
//if (i.measurement_time.ToUniversalTime() > DateTime.UtcNow)
// return false;
if (i.segment_id.Count == 0)
return false;
//if (i.measurement_time.ToUniversalTime() < DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(15)))
// return false;
}
return true;
}
},
new HttpResponseValidTest { RelativeUrl = GetSegmentState() },
new CheckCompletenessResponseTest
{
RelativeUrl = GetSegmentState(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInSegmentState()
},
new SanityCheck()
{
RelativeUrl = GetSegmentState(),
TypeToDeserialize = typeof(List<Segmentstate>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Segmentstate>)o)
{
if (i.update_time.ToUniversalTime() > DateTime.UtcNow)
return false;
// todo: add more sanity checks here.
}
return true;
}
},
// mandatory: start_time, end_time
new HttpResponseValidTest { RelativeUrl = GetSegmentStateHistoric() },
new CheckCompletenessResponseTest
{
RelativeUrl = GetSegmentStateHistoric(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInSegmentState()
},
new SanityCheck()
{
RelativeUrl = GetSegmentStateHistoric(),
TypeToDeserialize = typeof(List<Segmentstate>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Segmentstate>)o)
{
if (i.update_time.ToUniversalTime() > DateTime.UtcNow)
return false;
// todo: add more sanity checks here.
}
return true;
}
},
new HttpResponseValidTest { RelativeUrl = GetRoadSegments() },
new CheckCompletenessResponseTest
{
RelativeUrl = GetRoadSegments(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInRoadSegments()
},
new SanityCheck()
{
RelativeUrl = GetRoadSegments(),
TypeToDeserialize = typeof(List<Roadsegments>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Roadsegments>)o)
{
if (i.max_speed > 130)
return false;
if (i.length > 100)
return false;
}
return true;
}
},
new HttpResponseValidTest { RelativeUrl = GetVehicleStateLatest() },
new CheckCompletenessResponseTest
{
RelativeUrl = GetVehicleStateLatest(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInVehicleState()
},
new SanityCheck()
{
RelativeUrl = GetVehicleStateLatest(),
TypeToDeserialize = typeof(List<Vehiclestate>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Vehiclestate>)o)
{
if (i.measurement_time.ToUniversalTime() > DateTime.UtcNow)
return false;
}
return true;
}
},
new HttpResponseValidTest { RelativeUrl = GetVehicleStateHistoric() },
new CheckCompletenessResponseTest
{
RelativeUrl = GetVehicleStateHistoric(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInVehicleState()
},
new SanityCheck()
{
RelativeUrl = GetVehicleStateHistoric(),
TypeToDeserialize = typeof(List<Vehiclestate>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Vehiclestate>)o)
{
if (i.measurement_time.ToUniversalTime() > DateTime.UtcNow)
return false;
}
return true;
}
},
new HttpResponseValidTest { RelativeUrl = GetWeatherForecastHistoric() },
new CheckCompletenessResponseTest
{
RelativeUrl = GetWeatherForecastHistoric(),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInVehicleState()
},
new SanityCheck()
{
RelativeUrl = GetVehicleStateHistoric(),
TypeToDeserialize = typeof(List<Vehiclestate>),
CheckValidDataInsideFunctionHandler = o =>
{
foreach (var i in (List<Vehiclestate>)o)
{
if (i.measurement_time.ToUniversalTime() > DateTime.UtcNow)
return false;
}
return true;
}
}
);
}
public TestResultBase Test(Options o)
{
var testResult = new GenericTestResult
{
ShortDescription = "Performance (uptime) " + RelativeUrl,
Status = TestResult.INCONCLUSIVE
};
DateTime current = DateTime.Now; // to check if we elapsed total seconds.
var results = new List<Tuple<long, TestResult>>();
long lastMessageTime = 0;
while (true)
{
var test = new CheckCompletenessResponseTest
{
RelativeUrl = string.Format("/fcd?last_message_time={0}", lastMessageTime),
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInFCD(),
TestResultWhenNoData = TestResult.OK
};
TestResultBase res = test.Test(o);
results.Add(new Tuple<long, TestResult>(test.RequestDuration, res.Status));
if (test.ResponseData != null)
{
// now store the last updated time from the previous
string data = test.ResponseData;
// deserialize
var fcdMsg = JsonConvert.DeserializeObject<FcdMessage>(data);
if (lastMessageTime == fcdMsg.message_time)
{
GenerationTimeValid = false;
}
else
{
if (!GenerationTimeValid.HasValue)
GenerationTimeValid = true;
}
lastMessageTime = fcdMsg.message_time;
}
else
{
GenerationTimeValid = false;
}
Thread.Sleep(IntervalTime);
if (DateTime.Now.Subtract(current).TotalSeconds >= TestDuration)
break;
}
IEnumerable<Tuple<long, TestResult>> succesCount = results.Where(t => t.Item2 == TestResult.OK);
double percentageSuccess = succesCount.Count()/(double) results.Count;
double avgTime = results.Select(t => t.Item1).Average();
testResult.ExtraInformation = Math.Round(avgTime, 0) + " ms average time";
if (percentageSuccess >= 0.95)
{
testResult.Status = TestResult.OK;
}
else
{
testResult.Status = TestResult.FAIL;
testResult.CauseOfFailure = "Percentage success: " + Math.Round(percentageSuccess * 100, 0) + " %";
}
return testResult;
}
internal static void RunPerformanceTests(Options options)
{
var test = new KvGPerformanceTest
{
RelativeUrl = FcdRetrieveRequestWithTime(),
IntervalTime = Options.PerformanceTestInterval,
TestDuration = Options.PerformanceTestDuration,
};
TestResultBase result1 = test.Test(options);
result1.StepNr = 1;
result1.SubTest = "Performance";
Out.Info(result1.ToString());
// test 2
if (test.GenerationTimeValid.Value)
{
Out.Info(new GenericTestResult
{
SubTest = "Performance",
StepNr = 2,
ShortDescription = "Update interval " + FcdRetrieveRequest(),
Status = TestResult.OK
}.ToString());
}
else
{
Out.Info(new GenericTestResult
{
SubTest = "Performance",
StepNr = 2,
ShortDescription = "Update interval " + FcdRetrieveRequest(),
Status = TestResult.FAIL,
CauseOfFailure = "Generation time did not update properly."
}.ToString());
}
}
internal static void RunContinuityTests(Options options)
{
var test = new KvGPerformanceTest
{
RelativeUrl = FcdRetrieveRequest(),
IntervalTime = Options.PerformanceTestInterval,
TestDuration = Options.PerformanceTestDuration
};
TestResultBase result1 = test.Test(options);
result1.StepNr = 1;
result1.SubTest = "Continuity";
Out.Info(result1.ToString());
}
public TestResultBase Test(Options o)
{
var res = new GenericTestResult
{
ShortDescription = "Validity server-side certificate",
Status = TestResult.OK
};
try
{
ServicePointManager.MaxServicePoints = 0;
ServicePointManager.MaxServicePointIdleTime = 0;
ServicePointManager.ServerCertificateValidationCallback =
(sender, certificate, chain, sslPolicyErrors) =>
{
res.ExtraInformation = certificate.ToString();
// CHECK CERTIFICATE
// If the certificate is a valid, signed certificate, return true.
if (sslPolicyErrors == SslPolicyErrors.None)
{
res.Status = TestResult.OK;
return true;
}
// If there are errors in the certificate chain, look at each error to determine the cause.
if ((sslPolicyErrors & SslPolicyErrors.RemoteCertificateChainErrors) != 0)
{
if (chain != null && chain.ChainStatus != null)
{
foreach (
X509ChainStatus status in
chain.ChainStatus)
{
if ((certificate.Subject == certificate.Issuer) &&
(status.Status ==
X509ChainStatusFlags
.UntrustedRoot))
{
// Self-signed certificates with an untrusted root are valid.
if (Options.AllowValidSelfSignedCertificates)
continue;
res.Status = TestResult.FAIL;
res.CauseOfFailure =
"Self signed certificate. While valid - is not approved.";
return false;
}
if (status.Status !=
X509ChainStatusFlags.NoError)
{
// If there are any other errors in the certificate chain, the certificate is invalid,
// so the method returns false.
res.Status = TestResult.FAIL;
res.CauseOfFailure = status.StatusInformation;
return false;
}
}
}
// When processing reaches this line, the only errors in the certificate chain are
// untrusted root errors for self-signed certificates. These certificates are valid
// for default Exchange server installations, so return true.
if (Options.AllowValidSelfSignedCertificates)
{
res.Status = TestResult.OK;
return true;
}
res.Status = TestResult.FAIL;
res.CauseOfFailure =
"Self signed certificate. While valid - is not approved.";
return false;
}
res.Status = TestResult.FAIL;
res.CauseOfFailure =
"Invalid certificate";
return false;
};
HttpWebRequest req = SetupWebRequest(o, o.Url + RelativeUrl);
var response = (HttpWebResponse) req.GetResponse();
}
catch (Exception ex)
{
res.Status = TestResult.FAIL;
res.CauseOfFailure = ex.Message;
}
return res;
}
internal static void RunContinuityTests(Options options)
{
RunTestsParallel(options, "Continuity", Out.Info,
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInTrafficState(),
RelativeUrl = GetTrafficStateLatest(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInTrafficState(),
RelativeUrl = GetTrafficStateHistoric(options.SegmentCsv),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInEvents(),
RelativeUrl = GetEventsLatest(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInEvents(),
RelativeUrl = GetEventsHistoric(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInWeatherState(),
RelativeUrl = GetWeatherStateLatest(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInWeatherState(),
RelativeUrl = GetWeatherStateHistoric(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInWeatherForecast(),
RelativeUrl = GetWeatherForecastLatest(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInWeatherForecast(),
RelativeUrl = GetWeatherForecastHistoric(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInSegmentState(),
RelativeUrl = GetSegmentState(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInSegmentState(),
RelativeUrl = GetSegmentStateHistoric(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInRoadSegments(),
RelativeUrl = GetRoadSegments(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInVehicleState(),
RelativeUrl = GetVehicleStateLatest(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
},
new KvAPerformanceTest
{
FieldsThatShouldBePresent = FieldTester.FieldsThatShouldBePresentInVehicleState(),
RelativeUrl = GetVehicleStateHistoric(),
IntervalTime = Options.ContinuityTestInterval,
TestDuration = Options.ContinuityTestDuration
}
);
}