internal static object ToSeqObject(this IWardenCheckResult checkResult, string wardenName = null)
{
string messageTemplate = "Watcher {WatcherName} ran with result {Description}";
if (checkResult.Exception != null)
{
messageTemplate = "Watcher {WatcherName} ran but encountered an exception {Exception}";
}
var @event = new
{
Timestamp = checkResult.CompletedAt,
Level = checkResult.IsValid ? "Debug" : "Error",
MessageTemplate = messageTemplate,
Properties = new
{
WatcherName = checkResult.WatcherCheckResult.WatcherName,
Warden = wardenName,
WatcherGroup = checkResult.WatcherCheckResult.WatcherGroup,
WatcherType = checkResult.WatcherCheckResult.WatcherType,
Description = checkResult.WatcherCheckResult.Description,
Exception = checkResult.Exception,
StartedAt = checkResult.StartedAt,
CompletedAt = checkResult.CompletedAt,
ExecutionTime = checkResult.ExecutionTime
}
};
return(@event);
}
private static async Task WebsiteHookOnCompletedAsync(IWardenCheckResult check)
{
System.Console.WriteLine("Invoking the hook OnCompletedAsync() " +
$"by watcher: '{check.WatcherCheckResult.WatcherName}' " +
$"in group '{check.WatcherCheckResult.WatcherGroup}'.");
await Task.FromResult(true);
}
private static async Task WebsiteHookOnSuccessAsync(IWardenCheckResult check)
{
var webWatcherCheckResult = (WebWatcherCheckResult)check.WatcherCheckResult;
LogMessage("Invoking the hook OnSuccessAsync() " +
$"by watcher: '{webWatcherCheckResult.WatcherName}'.", MessageType.Info);
await Task.FromResult(true);
}
private static async Task WebsiteHookOnSuccessAsync(IWardenCheckResult check)
{
var webWatcherCheckResult = (WebWatcherCheckResult)check.WatcherCheckResult;
System.Console.WriteLine("Invoking the hook OnSuccessAsync() " +
$"by watcher: '{webWatcherCheckResult.WatcherName}'.");
await Task.FromResult(true);
}
private async Task InvokeOnCompletedHooksAsync(WatcherConfiguration watcherConfiguration,
IWardenCheckResult checkResult)
{
watcherConfiguration.Hooks.OnCompleted.Execute(checkResult);
await watcherConfiguration.Hooks.OnCompletedAsync.ExecuteAsync(checkResult);
_configuration.GlobalWatcherHooks.OnCompleted.Execute(checkResult);
await _configuration.GlobalWatcherHooks.OnCompletedAsync.ExecuteAsync(checkResult);
}
private async Task InvokeOnFirstFailureHooksAsync(WatcherConfiguration watcherConfiguration,
IWardenCheckResult checkResult)
{
watcherConfiguration.Hooks.OnFirstFailure.Execute(checkResult);
await watcherConfiguration.Hooks.OnFirstFailureAsync.ExecuteAsync(checkResult);
_configuration.GlobalWatcherHooks.OnFirstFailure.Execute(checkResult);
await _configuration.GlobalWatcherHooks.OnFirstFailureAsync.ExecuteAsync(checkResult);
}
public WatcherExecutionResult(IWatcher watcher, WatcherResultState currentState,
WatcherResultState previousState, IWardenCheckResult wardenCheckResult,
Exception exception = null)
{
Watcher = watcher;
CurrentState = currentState;
PreviousState = previousState;
WardenCheckResult = wardenCheckResult;
Exception = exception;
}
public async Task PostCheckToSeqAsync(IWardenCheckResult checkResult)
{
if (checkResult == null)
{
throw new ArgumentNullException(nameof(checkResult), "Check result can not be null.");
}
await _configuration.HttpServiceProvider().PostAsync(_configuration.Uri.ToString(),
checkResult.ToSeqJson(_configuration.JsonSerializerSettings), _configuration.Headers,
_configuration.FailFast);
}
internal static string ToSeqJson(this IWardenCheckResult checkResult, JsonSerializerSettings serializerSettings)
{
var rawData = new
{
Events = new List <object>()
};
rawData.Events.Add(checkResult.ToSeqObject());
return(JsonConvert.SerializeObject(rawData, serializerSettings));
}
private async Task InvokeOnCompletedHooksAsync(WatcherConfiguration watcherConfiguration,
IWardenCheckResult checkResult)
{
_logger.Trace($"Executing Watcher: {checkResult.WatcherCheckResult.WatcherName} hooks OnCompleted.");
watcherConfiguration.Hooks.OnCompleted.Execute(checkResult);
_logger.Trace($"Executing Watcher: {checkResult.WatcherCheckResult.WatcherName} hooks OnCompletedAsync.");
await watcherConfiguration.Hooks.OnCompletedAsync.ExecuteAsync(checkResult);
_logger.Trace("Executing Global Watcher hooks OnCompleted.");
_configuration.GlobalWatcherHooks.OnCompleted.Execute(checkResult);
_logger.Trace("Executing Global Watcher hooks OnCompletedAsync.");
await _configuration.GlobalWatcherHooks.OnCompletedAsync.ExecuteAsync(checkResult);
}
private async Task <IList <WatcherExecutionResult> > TryExecuteWatcherCheckAndHooksAsync(
WatcherConfiguration watcherConfiguration)
{
var startedAt = _configuration.DateTimeProvider();
var watcher = watcherConfiguration.Watcher;
IWardenCheckResult wardenCheckResult = null;
var results = new List <WatcherExecutionResult>();
try
{
await InvokeOnStartHooksAsync(watcherConfiguration, WatcherCheck.Create(watcher));
_logger.Info($"Executing Watcher: {watcher.Name}.");
var watcherCheckResult = await watcher.ExecuteAsync();
_logger.Info($"Completed executing Watcher: {watcher.Name}. " +
$"Is valid: {watcherCheckResult.IsValid}. " +
$"Description: {watcherCheckResult.Description}");
var completedAt = _configuration.DateTimeProvider();
wardenCheckResult = WardenCheckResult.Create(watcherCheckResult, startedAt, completedAt);
var watcherResults = await ExecuteWatcherCheckAsync(watcher, watcherCheckResult, wardenCheckResult,
watcherConfiguration);
results.AddRange(watcherResults);
}
catch (Exception exception)
{
_logger.Error($"There was an error while executing Watcher: {watcher.Name}.", exception);
var completedAt = _configuration.DateTimeProvider();
wardenCheckResult = WardenCheckResult.Create(WatcherCheckResult.Create(watcher, false, exception.Message),
startedAt, completedAt, exception);
results.Add(new WatcherExecutionResult(watcher, WatcherResultState.Error,
GetPreviousWatcherState(watcher), wardenCheckResult, exception));
await UpdateWatcherResultStateAndExecuteHooksPossibleAsync(watcher, WatcherResultState.Error,
() => InvokeOnFirstErrorHooksAsync(watcherConfiguration, exception));
var wardenException = new WardenException("There was an error while executing Warden " +
$"caused by watcher: '{watcher.Name}'.", exception);
await InvokeOnErrorHooksAsync(watcherConfiguration, wardenException);
}
finally
{
await InvokeOnCompletedHooksAsync(watcherConfiguration, wardenCheckResult);
}
return(results);
}
/// <summary>
/// Saves the IWardenCheckResult using Cachet API.
/// </summary>
/// <param name="result">Result object that will be saved using Cachet API.</param>
/// <param name="notify">Flag determining whether to notify the system administrator(s).</param>
/// <returns></returns>
public async Task SaveCheckResultAsync(IWardenCheckResult result, bool notify = false)
{
var groupId = _configuration.GroupId;
var checkResult = result.WatcherCheckResult;
var name = checkResult.WatcherName;
var status = checkResult.IsValid ? ComponentStatus.Operational : ComponentStatus.MajorOutage;
var component = await _cachetService.GetComponentAsync(checkResult.WatcherName);
if (component == null)
{
component = await _cachetService.CreateComponentAsync(name,
status, groupId : groupId);
}
else
{
component = await _cachetService.UpdateComponentAsync(component.Id,
name, status, groupId : groupId);
}
await SaveIncidentAsync(component.Id, checkResult, notify);
}
/// <summary>
/// Sends a POST request to the new Warden Panel API endpoint.
/// </summary>
/// <param name="checkResult">Warden check result object that will be serialized to the JSON.</param>
/// <returns></returns>
public async Task PostCheckResultToWardenPanelAsync(IWardenCheckResult checkResult)
{
if (checkResult == null)
{
throw new ArgumentNullException(nameof(checkResult), "Warden check result can not be null.");
}
if (string.IsNullOrWhiteSpace(checkResult.WatcherCheckResult.WatcherName))
{
throw new ArgumentException("Watcher name can not be empty.",
nameof(checkResult.WatcherCheckResult.WatcherName));
}
var baseUrl = _configuration.Uri.ToString();
var endpoint = GetWardenApiChecksEndpoint(_configuration.OrganizationId, _configuration.WardenId);
var fullUrl = baseUrl.GetFullUrl(endpoint);
var data = new { check = checkResult };
await _configuration.HttpServiceProvider().PostAsync(fullUrl,
data.ToJson(_configuration.JsonSerializerSettings),
_configuration.Headers, _configuration.Timeout, _configuration.FailFast);
}
private async Task TryExecuteWatcherCheckAndHooksAsync(WatcherConfiguration watcherConfiguration,
IList <WatcherExecutionResult> results)
{
var startedAt = _configuration.DateTimeProvider();
var watcher = watcherConfiguration.Watcher;
IWardenCheckResult wardenCheckResult = null;
try
{
await InvokeOnStartHooksAsync(watcherConfiguration, WatcherCheck.Create(watcher));
var watcherCheckResult = await watcher.ExecuteAsync();
var completedAt = _configuration.DateTimeProvider();
wardenCheckResult = WardenCheckResult.Create(watcherCheckResult, startedAt, completedAt);
await ExecuteWatcherCheckAsync(watcher, watcherCheckResult, wardenCheckResult,
watcherConfiguration, results);
}
catch (Exception exception)
{
var completedAt = _configuration.DateTimeProvider();
wardenCheckResult = WardenCheckResult.Create(WatcherCheckResult.Create(watcher, false),
startedAt, completedAt, exception);
results.Add(new WatcherExecutionResult(watcher, WatcherResultState.Error,
GetPreviousWatcherState(watcher), wardenCheckResult, exception));
await UpdateWatcherResultStateAndExecuteHooksPossibleAsync(watcher, WatcherResultState.Error,
() => InvokeOnFirstErrorHooksAsync(watcherConfiguration, exception));
var wardenException = new WardenException("There was an error while executing Warden " +
$"caused by watcher: '{watcher.Name}'.", exception);
await InvokeOnErrorHooksAsync(watcherConfiguration, wardenException);
}
finally
{
await InvokeOnCompletedHooksAsync(watcherConfiguration, wardenCheckResult);
}
}
/// <summary>
/// Saves the IWardenCheckResult using Cachet API.
/// </summary>
/// <param name="result">Result object that will be saved using Cachet API.</param>
/// <param name="notify">Flag determining whether to notify the system administrator(s).</param>
/// <param name="saveValidIncidents">Flag determining whether to save the valid incidents even if there were no errors previously reported (false by default).</param>
/// <param name="updateIfStatusIsTheSame">Flag determining whether to update the component and/or incident even if the previous status is the same (false by default).</param>
/// <returns></returns>
public async Task SaveCheckResultAsync(IWardenCheckResult result, bool notify = false,
bool saveValidIncidents = false, bool updateIfStatusIsTheSame = false)
{
var groupKey = result.WatcherCheckResult.WatcherGroup ?? string.Empty;
var groupId = _configuration.WatcherGroups.ContainsKey(groupKey)
? _configuration.WatcherGroups[groupKey]
: _configuration.GroupId;
var checkResult = result.WatcherCheckResult;
var name = checkResult.WatcherName;
var status = checkResult.IsValid ? ComponentStatus.Operational : ComponentStatus.MajorOutage;
var component = await _cachetService.GetComponentAsync(checkResult.WatcherName, groupId);
if (component == null)
{
component = await _cachetService.CreateComponentAsync(name,
status, groupId : groupId);
}
else
{
component = await _cachetService.UpdateComponentAsync(component.Id,
name, status, groupId : groupId, updateIfStatusIsTheSame : updateIfStatusIsTheSame);
}
await SaveIncidentAsync(component.Id, checkResult, notify, saveValidIncidents, updateIfStatusIsTheSame);
}
private async Task ExecuteWatcherCheckAsync(IWatcher watcher, IWatcherCheckResult watcherCheckResult,
IWardenCheckResult wardenCheckResult, WatcherConfiguration watcherConfiguration,
IList <WatcherExecutionResult> results)
{
if (watcherCheckResult.IsValid)
{
var result = wardenCheckResult;
results.Add(new WatcherExecutionResult(watcher, WatcherResultState.Success,
GetPreviousWatcherState(watcher), result));
await UpdateWatcherResultStateAndExecuteHooksPossibleAsync(watcher, WatcherResultState.Success,
() => InvokeOnFirstSuccessHooksAsync(watcherConfiguration, result),
executeIfLatestStateIsNotSet : false);
await InvokeOnSuccessHooksAsync(watcherConfiguration, wardenCheckResult);
}
else
{
var result = wardenCheckResult;
results.Add(new WatcherExecutionResult(watcher, WatcherResultState.Failure,
GetPreviousWatcherState(watcher), result));
await UpdateWatcherResultStateAndExecuteHooksPossibleAsync(watcher, WatcherResultState.Failure,
() => InvokeOnFirstFailureHooksAsync(watcherConfiguration, result));
await InvokeOnFailureHooksAsync(watcherConfiguration, wardenCheckResult);
}
}
private static void GlobalHookOnFailure(IWardenCheckResult check)
{
System.Console.WriteLine("Invoking the global hook OnFailure() " +
$"by watcher: '{check.WatcherCheckResult.WatcherName}'.");
}
static void PrintResult(IWardenCheckResult result)
{
Console.WriteLine($"{result.WatcherCheckResult.WatcherName} is {result.IsValid}");
}
private async Task InvokeOnCompletedHooksAsync(WatcherConfiguration watcherConfiguration,
IWardenCheckResult checkResult)
{
watcherConfiguration.Hooks.OnCompleted.Execute(checkResult);
await watcherConfiguration.Hooks.OnCompletedAsync.ExecuteAsync(checkResult);
_configuration.GlobalWatcherHooks.OnCompleted.Execute(checkResult);
await _configuration.GlobalWatcherHooks.OnCompletedAsync.ExecuteAsync(checkResult);
}
private async Task InvokeOnFirstFailureHooksAsync(WatcherConfiguration watcherConfiguration,
IWardenCheckResult checkResult)
{
watcherConfiguration.Hooks.OnFirstFailure.Execute(checkResult);
await watcherConfiguration.Hooks.OnFirstFailureAsync.ExecuteAsync(checkResult);
_configuration.GlobalWatcherHooks.OnFirstFailure.Execute(checkResult);
await _configuration.GlobalWatcherHooks.OnFirstFailureAsync.ExecuteAsync(checkResult);
}
public WatcherExecutionResult(IWatcher watcher, WatcherResultState currentState,
WatcherResultState previousState, IWardenCheckResult wardenCheckResult,
Exception exception = null)
{
Watcher = watcher;
CurrentState = currentState;
PreviousState = previousState;
WardenCheckResult = wardenCheckResult;
Exception = exception;
}
private async Task<IList<WatcherExecutionResult>> ExecuteWatcherCheckAsync(IWatcher watcher, IWatcherCheckResult watcherCheckResult,
IWardenCheckResult wardenCheckResult, WatcherConfiguration watcherConfiguration)
{
var results = new List<WatcherExecutionResult>();
if (watcherCheckResult.IsValid)
{
var result = wardenCheckResult;
results.Add(new WatcherExecutionResult(watcher, WatcherResultState.Success,
GetPreviousWatcherState(watcher), result));
await UpdateWatcherResultStateAndExecuteHooksPossibleAsync(watcher, WatcherResultState.Success,
() => InvokeOnFirstSuccessHooksAsync(watcherConfiguration, result),
executeIfLatestStateIsNotSet: false);
await InvokeOnSuccessHooksAsync(watcherConfiguration, wardenCheckResult);
}
else
{
var result = wardenCheckResult;
results.Add(new WatcherExecutionResult(watcher, WatcherResultState.Failure,
GetPreviousWatcherState(watcher), result));
await UpdateWatcherResultStateAndExecuteHooksPossibleAsync(watcher, WatcherResultState.Failure,
() => InvokeOnFirstFailureHooksAsync(watcherConfiguration, result));
await InvokeOnFailureHooksAsync(watcherConfiguration, wardenCheckResult);
}
return results;
}
private static async Task WebsiteHookOnFailureAsync(IWardenCheckResult check)
{
Console.WriteLine("Invoking the hook OnFailureAsync() " +
$"by watcher: '{check.WatcherCheckResult.WatcherName}'.");
await Task.FromResult(true);
}
private static void GlobalHookOnCompleted(IWardenCheckResult check)
{
Console.WriteLine("Invoking the global hook OnCompleted() " +
$"by watcher: '{check.WatcherCheckResult.WatcherName}'.");
}
private static async Task WebsiteHookOnFailureAsync(IWardenCheckResult check)
{
LogMessage("Invoking the hook OnFailureAsync() " +
$"by watcher: '{check.WatcherCheckResult.WatcherName}'.", MessageType.Info);
await Task.FromResult(true);
}
/// <summary>
/// Run a single iteration (cycle) that will execute all of the watchers and theirs hooks.
/// </summary>
/// <param name="wardenName">Name of the Warden that will execute the iteration.</param>
/// <param name="ordinal">Number (ordinal) of executed iteration</param>
/// <returns>Single iteration containing its ordinal and results of all executed watcher checks</returns>
public async Task <IWardenIteration> ExecuteAsync(string wardenName, long ordinal)
{
var iterationStartedAt = _configuration.DateTimeProvider();
var results = new List <WatcherExecutionResult>();
var iterationTasks = _configuration.Watchers.Select(async watcherConfiguration =>
{
var startedAt = _configuration.DateTimeProvider();
var watcher = watcherConfiguration.Watcher;
IWardenCheckResult wardenCheckResult = null;
try
{
await InvokeOnStartHooksAsync(watcherConfiguration, WatcherCheck.Create(watcher));
var watcherCheckResult = await watcher.ExecuteAsync();
var completedAt = _configuration.DateTimeProvider();
wardenCheckResult = WardenCheckResult.Create(watcherCheckResult, startedAt, completedAt);
if (watcherCheckResult.IsValid)
{
var result = wardenCheckResult;
results.Add(new WatcherExecutionResult(watcher, WatcherResultState.Success,
GetPreviousWatcherState(watcher), result));
await UpdateWatcherResultStateAndExecuteHooksPossibleAsync(watcher, WatcherResultState.Success,
() => InvokeOnFirstSuccessHooksAsync(watcherConfiguration, result),
executeIfLatestStateIsNotSet: false);
await InvokeOnSuccessHooksAsync(watcherConfiguration, wardenCheckResult);
}
else
{
var result = wardenCheckResult;
results.Add(new WatcherExecutionResult(watcher, WatcherResultState.Failure,
GetPreviousWatcherState(watcher), result));
await UpdateWatcherResultStateAndExecuteHooksPossibleAsync(watcher, WatcherResultState.Failure,
() => InvokeOnFirstFailureHooksAsync(watcherConfiguration, result));
await InvokeOnFailureHooksAsync(watcherConfiguration, wardenCheckResult);
}
}
catch (Exception exception)
{
var completedAt = _configuration.DateTimeProvider();
wardenCheckResult = WardenCheckResult.Create(WatcherCheckResult.Create(watcher, false),
startedAt, completedAt, exception);
results.Add(new WatcherExecutionResult(watcher, WatcherResultState.Error,
GetPreviousWatcherState(watcher), wardenCheckResult, exception));
await UpdateWatcherResultStateAndExecuteHooksPossibleAsync(watcher, WatcherResultState.Error,
() => InvokeOnFirstErrorHooksAsync(watcherConfiguration, exception));
var wardenException = new WardenException("There was an error while executing Warden " +
$"caused by watcher: '{watcher.Name}'.", exception);
await InvokeOnErrorHooksAsync(watcherConfiguration, wardenException);
}
finally
{
await InvokeOnCompletedHooksAsync(watcherConfiguration, wardenCheckResult);
}
});
await Task.WhenAll(iterationTasks);
await ExecuteAggregatedHooksAsync(results);
var iterationCompletedAt = _configuration.DateTimeProvider();
var iteration = WardenIteration.Create(wardenName, ordinal, results.Select(x => x.WardenCheckResult),
iterationStartedAt, iterationCompletedAt);
return(iteration);
}
private static void GlobalHookOnFailure(IWardenCheckResult check)
{
LogMessage("Invoking the global hook OnFailure() " +
$"by watcher: '{check.WatcherCheckResult.WatcherName}'.", MessageType.Info);
}
private static async Task GlobalHookOnCompletedPostToNewApiAsync(IWardenCheckResult check,
HttpApiIntegration httpApiIntegration)
{
await httpApiIntegration.PostCheckResultToWardenPanelAsync(check);
}
static void Print(IWardenCheckResult result)
{
//var resultObj = (MsSqlWatcherCheckResult) result;
Console.WriteLine($"{result.IsValid}");
}
