private long _frequencyTicks; // interlocked -- the current audit frequency, adjusted based on how long the audit takes (slower audits should run less frequently) and what the resulting rating status is.
/// <summary>
/// Constructs an <see cref="StatusAuditor"/> with the specified values.
/// </summary>
/// <param name="targetSystem">The name of the target system (if any).</param>
/// <param name="baselineAuditFrequency">
/// The baseline frequency with which audits should be run.
/// The system will start running tests this frequently but will automatically tune the frequency to between one quarter this frequency and ten times this frequency based on the status rating (worse rating means do audits more frequently) and the duration of the audit (fast audits cause more frequent audits).
/// This means that systems that process status audits faster will automatically be more responsive in reporting issues.
/// If audits start to timeout, audits will happen less frequently even if they are failing so that the status tests don't contribute to system congestion.
/// <see cref="TimeSpan.Zero"/> and negative time spans are treated as if they were <see cref="TimeSpan.MaxValue"/>.
/// </param>
/// <param name="status">The <see cref="Status"/> this auditor belongs to, or null if this should be a standalone auditor owned by the caller.</param>
protected internal StatusAuditor(string targetSystem, TimeSpan baselineAuditFrequency, Status?status)
: base(targetSystem)
{
_status = status;
_baselineAuditFrequency = baselineAuditFrequency;
_frequencyTicks = baselineAuditFrequency.Ticks;
_nextAuditTime = AmbientClock.UtcNow.AddMilliseconds(10).Ticks;
// create a timer for the initial audit (we'll dispose of this one immediately as soon as that audit finishes)
_initialAuditTimer = new System.Timers.Timer(10);
_initialAuditTimer.Elapsed += InitialAuditTimer_Elapsed;
_initialAuditTimer.AutoReset = false;
// should we update periodically?
if (baselineAuditFrequency < TimeSpan.MaxValue && baselineAuditFrequency > TimeSpan.FromTicks(0))
{
_auditTimer = new AmbientEventTimer(TimeSpan.FromTicks(_frequencyTicks).TotalMilliseconds);
_auditTimer.Elapsed += AuditTimer_Elapsed;
}
else // other parts of the code assume that _auditTimer is not null, so we will create one here that we don't hook up to any event handler
{
_auditTimer = new AmbientEventTimer(Int32.MaxValue - 1);
}
_auditTimer.AutoReset = false;
// note that the audit timer should remain stopped until we start it after the first audit happens
_shutdownInProgress = StatusResults.GetPendingResults(null, targetSystem);
}
public StatusResultsOrganizer(StatusResults results, string?source = null, IStatusThresholdsRegistry?thresholds = null)
{
_thresholds = thresholds;
Target = "/";
NatureOfSystem = StatusNatureOfSystem.ChildrenHeterogenous;
Add(results, source);
}
private StatusResultsOrganizer(StatusResults results, string?source, string localTarget)
{
MostRecentTime = results.Time;
Source = results.SourceSystem ?? source;
Target = localTarget;
NatureOfSystem = results.NatureOfSystem;
OverallReport = results.Report;
}
private static float?Rating(StatusResults results)
{
if (results == null || results.Report == null || results.Report.Alert == null)
{
return(null);
}
return(results.Report.Alert.Rating);
}
internal static int RatingCompare(StatusResults a, StatusResults b)
{
float?fa = Rating(a);
float?fb = Rating(b);
if (fa == null)
{
return((fb == null) ? 0 : -1);
}
return((fb == null) ? 1 : fa.Value.CompareTo(fb.Value));
}
/// <summary>
/// Adds the specified results as the latest results, moving the previous results to the history.
/// </summary>
/// <param name="newResults">The new latest results.</param>
/// <remarks>
/// Note that the new results will replace the old results and the old results will briefly disappear before being put into the history.
/// </remarks>
public void SetLatestResults(StatusResults newResults)
{
StatusResults oldResults = Interlocked.Exchange(ref _statusResults, newResults);
if (oldResults != null)
{
ConcurrentQueue <StatusResults> history = _statusResultsHistory;
history.Enqueue(oldResults);
// NOTE that there is a race here that might remove too many previous entries--this should be rare and not catastrophic
TruncateQueue(_statusResultsHistory);
}
}
/// <summary>
/// Sets the latest results.
/// </summary>
/// <param name="newResults">The new <see cref="StatusResults"/>. Note that null results will not be stored.</param>
protected internal void SetLatestResults(StatusResults newResults)
{
if (newResults != null)
{
if (!String.Equals(_targetSystem, newResults.TargetSystem, StringComparison.Ordinal))
{
throw new ArgumentException("The target system for the specified status results and must match the this StatusChecker's target system!", nameof(newResults));
}
Status.Logger.Log($"{newResults.TargetSystemDisplayName}: {newResults.Report?.Alert}", "Results");
_resultsTracker.SetLatestResults(newResults);
}
}
/// <summary>
/// Refreshes the status audits immediately.
/// Normally audits will be refreshed automatically in the background, but in some circumstances, users may want to force an immediate update.
/// </summary>
public async Task <StatusResults> RefreshAsync(CancellationToken cancel = default(CancellationToken))
{
Logger.Log("Explicit Refreshing", "Check");
// asynchronously get the status of each system
IEnumerable <Task <StatusResults> > checkerTasks = _checkers.Select(checker => checker.GetStatus(cancel));
await Task.WhenAll(checkerTasks).ConfigureAwait(false);
// sort the results
List <StatusResults> results = new List <StatusResults>(checkerTasks.Select(t => t.Result));
results.Sort((a, b) => RatingCompare(a, b));
StatusResults overallResults = new StatusResults(null, "/", results);
return(overallResults);
}
private async Task <StatusResults> InternalAuditAsync(bool foreground = false, CancellationToken cancel = default(CancellationToken))
{
StatusResultsBuilder builder = new StatusResultsBuilder(this);
try
{
try
{
// in case the timer went off more than once due to test (or overall system) slowness, disable the timer until we're done here
_auditTimer.Stop();
// have we already shut down? bail out now!
if (foreground)
{
Interlocked.Increment(ref _foregroundAuditCount);
}
else
{
Interlocked.Increment(ref _backgroundAuditCount);
}
// call the derived object to get the status
await Audit(builder, cancel).ConfigureAwait(false);
// schedule the next audit
builder.NextAuditTime = ScheduleNextAudit(builder.WorstAlert == null ? (float?)null : builder.WorstAlert.Rating, builder.Elapsed);
}
#pragma warning disable CA1031 // we really DO want to catch ALL exceptions here--this is a status test, and the exception will be reported through the status system. if we rethrew it, it would crash the program
catch (Exception ex)
#pragma warning restore CA1031
{
builder.AddException(ex);
}
finally
{
_auditTimer.Start();
}
}
catch (ObjectDisposedException)
{
// ignore this exception--given the design of System.Timers.Timer, it's impossible to prevent
// it happens when an audit happens to get triggered just before shutdown/disposal
return(_shutdownInProgress);
}
// get the results
StatusResults newStatusResults = builder.FinalResults;
// save the results AND return them
SetLatestResults(newStatusResults);
return(newStatusResults);
}
/// <summary>
/// Constructs a StatusResultsBuilder from the specified <see cref="StatusResults"/>.
/// </summary>
/// <param name="results">The <see cref="StatusResults"/> to copy from.</param>
public StatusResultsBuilder(StatusResults results)
{
if (results == null)
{
throw new ArgumentNullException(nameof(results));
}
_sourceSystem = results.SourceSystem;
_targetSystem = results.TargetSystem;
_auditStartTime = results.Time;
_relativeDetailLevel = results.RelativeDetailLevel;
_natureOfSystem = results.NatureOfSystem;
_worstAlert = (results.Report == null) ? null : results.Report.Alert;
_nextAuditTime = (results.Report == null) ? null : results.Report.NextAuditTime;
_properties = new List <StatusProperty>(results.Properties);
_children = new List <StatusResultsBuilder>(results.Children.Select(c => new StatusResultsBuilder(c)));
}
private void MergeProperties(StatusResults results)
{
// merge any attibutes into the matching node
foreach (StatusProperty property in results.Properties)
{
StatusPropertyRange?range = _propertyRanges.Where(ar => ar.Name == property.Name).FirstOrDefault();
if (range != null)
{
range.Merge(property.Value);
}
else
{
_propertyRanges.Add(new StatusPropertyRange(property));
}
}
}
public AggregatedAlert(StatusResults initialResults)
{
StatusAuditReport report = initialResults.Report ?? new StatusAuditReport(initialResults.Time, TimeSpan.Zero, null, StatusAuditAlert.None);
CommonAlert = report.Alert;
RatingSum = report.Alert?.Rating ?? StatusRating.Okay;
Sources = new List <string>();
Sources.Add(RenderSource(initialResults.SourceSystem));
Target = RenderTarget(initialResults.TargetSystem);
TimeRange = new DateTimeRange(initialResults.Time);
Report = report;
AuditStartRange = new DateTimeRange(report.AuditStartTime);
AuditDurationRange = new TimeSpanRange(report.AuditDuration);
NextAuditTime = report.NextAuditTime;
PropertyRanges = new List <StatusPropertyRange>();
}
public void Aggregate(StatusResults additionalResults)
{
StatusAuditReport report = additionalResults.Report ?? new StatusAuditReport(additionalResults.Time, TimeSpan.Zero, null, StatusAuditAlert.None);
if (CommonAlert != report.Alert)
{
throw new InvalidOperationException("Only results with the same alert can be aggregated!");
}
if (Target != RenderTarget(additionalResults.TargetSystem))
{
throw new InvalidOperationException("Only results with the same target can be aggregated!");
}
RatingSum += report.Alert?.Rating ?? StatusRating.Okay;
Sources.Add(RenderSource(additionalResults.SourceSystem));
TimeRange.AddSample(additionalResults.Time);
AuditStartRange.AddSample(report.AuditStartTime);
AuditDurationRange.AddSample(report.AuditDuration);
NextAuditTime = (NextAuditTime == null) ? report.NextAuditTime : new DateTime(Math.Min(NextAuditTime.Value.Ticks, (report.NextAuditTime ?? DateTime.MaxValue).Ticks));
PropertyRanges = new List <StatusPropertyRange>();
}
/// <summary>
/// Adds the specified <see cref="StatusResults"/> as a child to the node we're building.
/// </summary>
/// <param name="child">The child <see cref="StatusResults"/>.</param>
public void AddChild(StatusResults child)
{
_children.Add(new StatusResultsBuilder(child));
}
/// <summary>
/// Adds the specified <see cref="StatusResults"/> as a descendant of the specified root.
/// The position in the tree where the results will be put is determined by the <see cref="StatusResults.TargetDisplayName"/> property,
/// with nodes that have the same target ending up as siblings to a common parent.
/// </summary>
/// <param name="root">The root <see cref="StatusResultsOrganizer"/> being added to.</param>
/// <param name="results">The <see cref="StatusResults"/> being added.</param>
/// <param name="source">A source, if one needs to be assigned.</param>
/// <param name="target">A parent target for the specified node.</param>
public void Add(StatusResultsOrganizer root, StatusResults results, string?source = null, string target = "")
{
// a leaf node?
if (results.NatureOfSystem == StatusNatureOfSystem.Leaf)
{
// rate this leaf node if we can (we need the ratings here in order to collate and sort results in ComputeOverallRatingAndSort)
string localTarget = results.TargetSystem;
StatusResultsOrganizer organizedResults = new StatusResultsOrganizer(results, source, localTarget);
StatusResultsOrganizer parent = this;
string?localtarget = results.TargetSystem;
// a child of the root?
if (localtarget?.StartsWith("/", StringComparison.Ordinal) ?? false)
{
parent = root;
localtarget = localtarget.Substring(1);
}
parent._children.Add(organizedResults);
// merge in the properties
organizedResults.MergeProperties(results);
}
else // the results instance we're adding is an inner node
{
StatusResultsOrganizer?match;
// do the results specify a target?
if (!string.IsNullOrEmpty(results.TargetSystem))
{
if (results.TargetSystem == "/")
{
match = root;
}
else
{
StatusResultsOrganizer parent = this;
string?localTarget = results.TargetSystem;
// a child of the root?
if (localTarget.StartsWith("/", StringComparison.Ordinal))
{
parent = root;
localTarget = localTarget.Substring(1);
}
// is there NOT a matching node that's a child of the parent?
match = parent._children.Where(c => c.Target == localTarget && c.Source == source && c.NatureOfSystem == results.NatureOfSystem).FirstOrDefault(); // note that the matching node cannot be a leaf in this case because the NatureOfSystem had to match the results and we already checked results for that
if (match == null)
{ // we don't have a node to merge into, so build a new node now with the properties of these results
match = new StatusResultsOrganizer(results, source, localTarget); // note that this new matching node cannot be a leaf in this case because the NatureOfSystem had to match the results and we already checked results for that
parent._children.Add(match);
}
System.Diagnostics.Debug.Assert(match.NatureOfSystem != StatusNatureOfSystem.Leaf);
}
}
else // no target is specified, so we should merge these results directly into this node
{
match = this;
}
// does the matching node have children?
if (match.NatureOfSystem != StatusNatureOfSystem.Leaf)
{
DateTime?time = null;
// merge the children of the specified results
foreach (StatusResults child in results.Children)
{
match.Add(root, child, results.SourceSystem ?? source, ComputeTarget(target, results.TargetSystem));
if (time == null || child.Time > time)
{
time = child.Time;
}
}
MostRecentTime = (time > results.Time) ? time.Value : results.Time;
}
else // the matching node is a leaf node (this is strange because the node we're adding is NOT a leaf node)
{
match.MostRecentTime = results.Time;
// the node we're adding is NOT a leaf node, so it CANNOT have a report (StatusResults cannot have both children and a report, though we can here once we rate parent nodes based on their children)
System.Diagnostics.Debug.Assert(results.Report == null);
}
// merge in the properties
match.MergeProperties(results);
}
}
/// <summary>
/// Constructs a <see cref="StatusChecker"/>.
/// </summary>
/// <param name="targetSystem">The name of the target system (if any).</param>
/// <remarks>
/// Target system names are concatenated with ancestor and descendant nodes and used to aggregate errors from the same system reported by multiple sources so that they can be summarized rather than listed individually.
/// Targets with a leading slash character indicate that the system is a shared system and may have status results measured by other source systems which should be combined.
/// Shared targets are not concatenated to the targets indicated by ancestor nodes, and their parents are ignored during summarization, treating shared systems as top-level nodes.
/// Defaults to null, but should almost always be set to a non-empty string.
/// Null should only be used to indicate that this node is not related to any specific target system, which would probably only happen if <see cref="StatusResults.NatureOfSystem"/> this, parent, and child nodes is such that some kind of special grouping is needed to make the overall status rating computation work correctly and the target system identifier for child nodes makes more sense without any identifier at this level.
/// </remarks>
protected internal StatusChecker(string targetSystem)
{
_targetSystem = targetSystem;
_resultsTracker = new StatusResultsTracker(StatusResults.GetPendingResults(null, targetSystem));
}
private ConcurrentQueue <StatusResults> _statusResultsHistory; // interlocked
public StatusResultsTracker(StatusResults pendingStatusResults)
{
_statusResults = pendingStatusResults;
_statusResultsHistory = new ConcurrentQueue <StatusResults>();
}
/// <summary>
/// Adds the specified <see cref="StatusResults"/> to this root node.
/// </summary>
/// <param name="results">The <see cref="StatusResults"/> for this node.</param>
/// <param name="source">The source to assign for nodes that don't have a source assigned.</param>
public void Add(StatusResults results, string?source = null)
{
Add(this, results, source);
}
