public virtual void AddEndpointCheckResult(int endpointId, EndpointCheckResult result, bool isFinal)
{
var status = _endpointStates.AddOrUpdate(endpointId, (id) =>
{
var es = new EndpointStatus();
es.InProgressResults = new List <EndpointCheckResult> {
result
};
return(es);
},
(id, oldStatus) =>
{
var es = oldStatus.DeepClone();
if (es.InProgressResults == null)
{
es.InProgressResults = new List <EndpointCheckResult>();
}
es.InProgressResults.Add(result);
return(es);
});
PostProcessStatus(endpointId, status, isFinal);
}
private void SetFailure(EndpointCheckResult result, Exception ex)
{
result.Error = ex;
result.Success = false;
result.Stop = DateTimeOffset.UtcNow;
result.ResponseTime = null;
}
private void SetSuccess(EndpointCheckResult result)
{
result.Error = null;
result.Success = true;
result.Stop = DateTimeOffset.UtcNow;
result.ResponseTime = result.Stop - result.Start;
}
public void ShouldNotifyWhenEndpointGoesDownFromUpOrUnknownStates()
{
var result = new EndpointCheckResult {
Start = DateTimeOffset.UtcNow, Stop = DateTimeOffset.UtcNow, Success = false, Error = new Exception("ERROR")
};
_service.AddEndpointCheckResult(_ep1.Id, result, true);
AssertResultNotification(_ep1.Id, result);
AssertDownNotification(_ep1.Id, "ERROR");
AssertNoUpNotifications();
_notifier.ClearReceivedCalls();
_service.AddEndpointCheckResult(_ep2.Id, result, true);
AssertResultNotification(_ep2.Id, result);
AssertDownNotification(_ep2.Id, "ERROR");
AssertNoUpNotifications();
_notifier.ClearReceivedCalls();
_service.AddEndpointCheckResult(_ep3.Id, result, true);
AssertResultNotification(_ep3.Id, result);
AssertNoDownNotifications();
AssertNoUpNotifications();
}
private void SetEndpointSuccess(Endpoint endpoint, EndpointCheckResult result)
{
if (!IsEndpointCurrent(endpoint))
{
return;
}
_stateService.AddEndpointCheckResult(endpoint.Id, result, true);
}
public void ShouldNotifySingleResult()
{
var ep = new Endpoint();
var ecr = new EndpointCheckResult();
_service.NotifySingleResult(ep, ecr);
_bgExec.Received(1).Execute(Arg.Any <Action>());
_notifier1.Received().NotifySingleResult(ep, ecr);
_notifier2.Received().NotifySingleResult(ep, ecr);
}
public void ShouldNotifyWhenEndpointIsStillDownButWithDifferentErrorMessage()
{
var result = new EndpointCheckResult {
Start = DateTimeOffset.UtcNow, Stop = DateTimeOffset.UtcNow, Success = false, Error = new Exception("err1")
};
_service.AddEndpointCheckResult(_ep3.Id, result, true);
AssertResultNotification(_ep3.Id, result);
AssertNoUpNotifications();
AssertDownNotification(_ep3.Id, "err1");
}
public void ShouldNotifySingleResultSuccessWithResponseTime()
{
var ep = new Endpoint {
Name = "ep3"
};
var result = new EndpointCheckResult {
Start = DateTimeOffset.Parse("2018-01-01T12:00:08.000Z"), Success = true, ResponseTime = TimeSpan.FromMilliseconds(54321)
};
_service.NotifySingleResult(ep, result);
Assert.AreEqual("sama.ep3.response.success 1 1514808008\nsama.ep3.response.timeMsec 54321 1514808008\n", _sentData);
}
public void ShouldNotifySingleResultSuccessWithoutResponseTime()
{
var ep = new Endpoint {
Name = "ep2"
};
var result = new EndpointCheckResult {
Start = DateTimeOffset.Parse("2018-01-01T12:00:01.000Z"), Success = true
};
_service.NotifySingleResult(ep, result);
Assert.AreEqual("sama.ep2.response.success 1 1514808001\n", _sentData);
}
public void ShouldNotifySingleResultFailure()
{
var ep = new Endpoint {
Name = "ep1"
};
var result = new EndpointCheckResult {
Start = DateTimeOffset.Parse("2018-01-01T12:00:00.000Z"), Success = false
};
_service.NotifySingleResult(ep, result);
Assert.AreEqual("sama.ep1.response.success 0 1514808000\n", _sentData);
}
public void InProgressUpdateShouldNotChangeReportedState()
{
AssertStateEquality(true, null, _service.GetStatus(_ep2.Id));
var result = new EndpointCheckResult {
Start = DateTimeOffset.UtcNow, Stop = DateTimeOffset.UtcNow, Success = false, Error = new Exception("ERR")
};
_service.AddEndpointCheckResult(_ep2.Id, result, false);
Assert.IsTrue(_service.GetStatus(_ep2.Id).IsInProgress);
AssertStateEquality(true, null, _service.GetStatus(_ep2.Id));
AssertResultNotification(_ep2.Id, result);
AssertNoUpNotifications();
AssertNoDownNotifications();
}
public void ShouldNotifyWhenEndpointGoesUpFromDownStateOnly()
{
var result = new EndpointCheckResult {
Start = DateTimeOffset.UtcNow, Stop = DateTimeOffset.UtcNow, Success = true
};
_service.AddEndpointCheckResult(_ep2.Id, result, true);
AssertResultNotification(_ep2.Id, result);
AssertNoUpNotifications();
AssertNoDownNotifications();
_service.AddEndpointCheckResult(_ep3.Id, result, true);
AssertResultNotification(_ep3.Id, result);
AssertUpNotification(_ep3.Id);
AssertNoDownNotifications();
}
public virtual void ProcessEndpoint(Endpoint endpoint, int retryCount)
{
if (!IsEndpointCurrent(endpoint))
{
return;
}
var service = GetCheckService(endpoint);
if (service == null)
{
var result = new EndpointCheckResult {
Start = DateTimeOffset.UtcNow,
Stop = DateTimeOffset.UtcNow,
Success = false,
Error = new Exception("There is no registered handler for this kind of endpoint.")
};
SetEndpointFailure(endpoint, result, retryCount);
return;
}
try
{
SetEndpointInProgress(endpoint);
var result = service.Check(endpoint);
if (result.Success)
{
SetEndpointSuccess(endpoint, result);
}
else
{
SetEndpointFailure(endpoint, result, retryCount);
}
}
catch (Exception ex)
{
var result = new EndpointCheckResult
{
Start = DateTimeOffset.UtcNow,
Stop = DateTimeOffset.UtcNow,
Success = false,
Error = new Exception($"Unexpected check failure: {ex.Message}", ex)
};
SetEndpointFailure(endpoint, result, retryCount);
return;
}
}
private void SetEndpointFailure(Endpoint endpoint, EndpointCheckResult result, int retryCount)
{
if (!IsEndpointCurrent(endpoint))
{
return;
}
if (retryCount < MaxRetries)
{
_stateService.AddEndpointCheckResult(endpoint.Id, result, false);
Thread.Sleep(RetrySleep);
ProcessEndpoint(endpoint, retryCount + 1);
return;
}
_stateService.AddEndpointCheckResult(endpoint.Id, result, true);
}
public void NotifySingleResult(Endpoint endpoint, EndpointCheckResult result)
{
var host = _settings.Notifications_Graphite_Host;
var port = _settings.Notifications_Graphite_Port;
if (string.IsNullOrWhiteSpace(host) || port < 1)
{
return;
}
var filteredEndpointName = Regex.Replace(endpoint.Name, @"[^a-zA-Z0-9-]+", "-").Trim('-');
if (string.IsNullOrWhiteSpace(filteredEndpointName))
{
filteredEndpointName = "none";
}
var epoch = result.Start.ToUnixTimeSeconds();
var successMetric = (result.Success ? 1 : 0);
var graphiteMessage = new StringBuilder();
graphiteMessage.Append($"sama.{filteredEndpointName}.response.success {successMetric} {epoch}\n");
if (result.Success && result.ResponseTime.HasValue)
{
var responseTime = (int)Math.Round(result.ResponseTime.Value.TotalMilliseconds);
graphiteMessage.Append($"sama.{filteredEndpointName}.response.timeMsec {responseTime} {epoch}\n");
}
var messageBytes = Encoding.ASCII.GetBytes(graphiteMessage.ToString());
try
{
_tcpWrapper.SendData(host, port, messageBytes);
}
catch (Exception ex)
{
_logger.LogError(0, "Unable to send Graphite notification", ex);
}
}
public EndpointCheckResult Check(Endpoint endpoint)
{
var checkResult = new EndpointCheckResult {
Start = DateTimeOffset.UtcNow
};
try
{
var(status, roundtripTime) = _pingWrapper.SendPing(endpoint.GetIcmpAddress());
if (status == IPStatus.Success)
{
checkResult.Success = true;
checkResult.Stop = DateTimeOffset.UtcNow;
checkResult.ResponseTime = roundtripTime;
return(checkResult);
}
else
{
checkResult.Error = new Exception(GetFriendlyError(status));
checkResult.Success = false;
checkResult.Stop = DateTimeOffset.UtcNow;
checkResult.ResponseTime = null;
return(checkResult);
}
}
catch (Exception ex)
{
if (ex.GetType() == typeof(PingException) && ex.InnerException != null)
{
ex = ex.InnerException;
}
checkResult.Error = new Exception($"Unable to ping: {ex.Message}.", ex);
checkResult.Success = false;
checkResult.Stop = DateTimeOffset.UtcNow;
checkResult.ResponseTime = null;
return(checkResult);
}
}
public void ShouldNotNotifyWhenUnconfigured()
{
var ep = new Endpoint {
Name = "epU"
};
var result = new EndpointCheckResult {
Start = DateTimeOffset.Parse("2018-01-01T12:08:20.000"), Success = false
};
_settings.Notifications_Graphite_Host.Returns("");
_settings.Notifications_Graphite_Port.Returns(1234);
_service.NotifySingleResult(ep, result);
_tcpWrapper.DidNotReceiveWithAnyArgs().SendData("", 0, Arg.Any <byte[]>());
_settings.Notifications_Graphite_Host.Returns("asdf");
_settings.Notifications_Graphite_Port.Returns(0);
_service.NotifySingleResult(ep, result);
_tcpWrapper.DidNotReceiveWithAnyArgs().SendData("", 0, Arg.Any <byte[]>());
}
private void NotifyNewestCheckResult(Endpoint endpoint, EndpointCheckResult endpointCheckResult)
{
_notifier.NotifySingleResult(endpoint, endpointCheckResult);
}
public virtual void NotifySingleResult(Endpoint endpoint, EndpointCheckResult result)
{
// Ignore this notification type.
}
public void NotifySingleResult(Endpoint endpoint, EndpointCheckResult result)
{
// Ignore.
}
public EndpointCheckResult Check(Endpoint endpoint)
{
var result = new EndpointCheckResult {
Start = DateTimeOffset.UtcNow
};
using (var httpHandler = _serviceProvider.GetRequiredService <HttpClientHandler>())
using (var client = new HttpClient(httpHandler, false))
using (var message = new HttpRequestMessage(HttpMethod.Get, endpoint.GetHttpLocation()))
{
httpHandler.ServerCertificateCustomValidationCallback = (msg, certificate, chain, sslPolicyErrors) =>
{
_certService.ValidateHttpEndpoint(endpoint, chain, sslPolicyErrors);
return(true);
};
var statusCodes = endpoint.GetHttpStatusCodes() ?? new List <int>();
if (statusCodes.Count > 0)
{
httpHandler.AllowAutoRedirect = false;
}
message.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko");
message.Headers.Add("Accept", "text/html, application/xhtml+xml, */*");
client.Timeout = ClientTimeout;
var task = client.SendAsync(message);
try
{
task.Wait();
}
catch (Exception ex)
{
if (ex is AggregateException)
{
ex = ex.InnerException;
}
if (ex is HttpRequestException && ex.InnerException != null)
{
ex = ex.InnerException;
}
if (ex is TaskCanceledException)
{
ex = new Exception($"The request timed out after {ClientTimeout.TotalSeconds} sec.");
}
SetFailure(result, ex);
return(result);
}
var response = task.Result;
if (!IsExpectedStatusCode(endpoint, response))
{
SetFailure(result, new Exception($"HTTP status code is {(int)response.StatusCode}."));
return(result);
}
if (string.IsNullOrWhiteSpace(endpoint.GetHttpResponseMatch()))
{
SetSuccess(result);
return(result);
}
var contentTask = response.Content.ReadAsStringAsync();
try
{
contentTask.Wait();
}
catch (Exception ex)
{
SetFailure(result, new Exception($"Failed to read HTTP content: {ex.Message}.", ex));
return(result);
}
var index = contentTask.Result.IndexOf(endpoint.GetHttpResponseMatch());
if (index < 0)
{
SetFailure(result, new Exception("The keyword match was not found."));
return(result);
}
else
{
SetSuccess(result);
return(result);
}
}
}
private void AssertResultNotification(int endpointId, EndpointCheckResult result)
{
_notifier.Received(1).NotifySingleResult(Arg.Is <Endpoint>(ep => ep.Id == endpointId), Arg.Any <EndpointCheckResult>());
}
public virtual void NotifySingleResult(Endpoint endpoint, EndpointCheckResult result)
{
_bgExec.Execute(() => _notificationServices
.ForEach(ns => ns.NotifySingleResult(endpoint, result)));
}