public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <GraphiteLine> target,
string rootNamespace,
IIntervalService intervalService,
ILog log,
int maxItemsPerBucket = 1000)
{
var latencies = new ConcurrentDictionary <string, LatencyBucket>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var latency = p as Timing;
latencies.AddOrUpdate(latency.Name,
(key) =>
{
return(new LatencyBucket(maxItemsPerBucket, latency.ValueMS));
},
(key, bag) =>
{
bag.Add(latency.ValueMS);
return(bag);
});
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (latencies.Count == 0)
{
return;
}
var buckets = latencies.ToArray();
latencies.Clear();
foreach (var bucket in buckets)
{
target.Post(new GraphiteLine(ns + bucket.Key + ".count", bucket.Value.Count, e.Epoch));
target.Post(new GraphiteLine(ns + bucket.Key + ".min", bucket.Value.Min, e.Epoch));
target.Post(new GraphiteLine(ns + bucket.Key + ".max", bucket.Value.Max, e.Epoch));
target.Post(new GraphiteLine(ns + bucket.Key + ".mean", bucket.Value.Mean, e.Epoch));
target.Post(new GraphiteLine(ns + bucket.Key + ".sum", bucket.Value.Sum, e.Epoch));
}
log.InfoFormat("TimedLatencyAggregatorBlock - Posted {0} buckets and {1} lines.", buckets.Length, buckets.Length * 5);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public CronJobService(IAppCallerService appCallerService, IAppService appService, INotificationService notificationService, INotificationSender notificationSender, IIntervalService intervalService, ILogService logService)
{
_appCallerService = appCallerService;
_appService = appService;
_notificationService = notificationService;
_notificationSender = notificationSender;
_intervalService = intervalService;
_logService = logService;
}
public UserFLashcardMemoryService(IFlashcardUnit unit, ISessionService sessionService, IQuestionService questionService, IStrengthService strengthService,
IIntervalService intervalService)
{
this.unit = unit;
this.sessionService = sessionService;
this.questionService = questionService;
this.strengthService = strengthService;
this.intervalService = intervalService;
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <Bucket> target,
string rootNamespace,
IIntervalService intervalService,
int percentile,
string percentileName,
ILog log,
int maxItemsPerBucket = 1000)
{
var latencies = new ConcurrentDictionary <MetricInfo, DatapointBox>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var random = new Random();
percentileName = "." + (percentileName ?? ("p" + percentile));
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var latency = p as Timing;
var metricInfo = new MetricInfo(latency.Name, latency.Tags);
latencies.AddOrUpdate(metricInfo,
(key) =>
{
return(new DatapointBox(maxItemsPerBucket, latency.ValueMS));
},
(key, bag) =>
{
bag.Add(latency.ValueMS);
return(bag);
});
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (latencies.Count == 0)
{
return;
}
var bucket = new PercentileBucket(latencies.ToArray(),
e.Epoch,
ns,
percentileName,
percentile);
latencies.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <GraphiteLine> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary <string, ConcurrentDictionary <int, bool> >();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var set = p as Set;
sets.AddOrUpdate(set.Name,
(key) =>
{
var setDict = new ConcurrentDictionary <int, bool>();
setDict.AddOrUpdate(set.Value, (key2) => true, (key2, oldValue) => true);
return(setDict);
},
(key, setDict) =>
{
setDict.AddOrUpdate(set.Value, (key2) => true, (key2, oldValue) => true);
return(setDict);
});
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var bucketOfSets = sets.ToArray();
sets.Clear();
int numLinesPosted = 0;
foreach (var bucket in bucketOfSets)
{
foreach (var set in bucket.Value)
{
target.Post(new GraphiteLine(ns + bucket.Key + "." + set.Key.ToString(), 1, e.Epoch));
numLinesPosted++;
}
}
log.InfoFormat("TimedSetAggregatorBlock - Posted {0} buckets and {1} lines.", bucketOfSets.Length, numLinesPosted);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
bool removeZeroGauges,
IIntervalService intervalService,
ILog log)
{
var gauges = new ConcurrentDictionary<string, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var gauge = p as Gauge;
gauges.AddOrUpdate(gauge.Name, gauge.Value, (key, oldValue) => gauge.Value);
},
Utility.UnboundedExecution());
intervalService.Elapsed += (sender, e) =>
{
if (gauges.Count == 0)
{
return;
}
var items = gauges.ToArray();
var bucket = new GaugesBucket(items, e.Epoch, ns);
if (removeZeroGauges)
{
// Get all zero-value gauges
double placeholder;
var zeroGauges = 0;
for (int index = 0; index < items.Length; index++)
{
if (items[index].Value == 0)
{
gauges.TryRemove(items[index].Key, out placeholder);
zeroGauges += 1;
}
}
if (zeroGauges > 0)
{
log.InfoFormat("Removed {0} empty gauges.", zeroGauges);
}
}
gauges.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <Bucket> target,
string rootNamespace,
bool removeZeroGauges,
IIntervalService intervalService,
ILog log)
{
var gauges = new ConcurrentDictionary <string, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var gauge = p as Gauge;
gauges.AddOrUpdate(gauge.Name, gauge.Value, (key, oldValue) => gauge.Value);
},
Utility.UnboundedExecution());
intervalService.Elapsed += (sender, e) =>
{
if (gauges.Count == 0)
{
return;
}
var items = gauges.ToArray();
var bucket = new GaugesBucket(items, e.Epoch, ns);
if (removeZeroGauges)
{
// Get all zero-value gauges
double placeholder;
var zeroGauges = 0;
for (int index = 0; index < items.Length; index++)
{
if (items[index].Value == 0)
{
gauges.TryRemove(items[index].Key, out placeholder);
zeroGauges += 1;
}
}
if (zeroGauges > 0)
{
log.InfoFormat("Removed {0} empty gauges.", zeroGauges);
}
}
gauges.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public SystemMetricsService(string serviceName, string prefix = null, IIntervalService intervalService = null)
{
if (intervalService == null)
{
intervalService = new IntervalService(10);
}
_prefix = serviceName + "." + (String.IsNullOrEmpty(prefix) ? String.Empty : (prefix + "."));
_metrics = new ConcurrentDictionary <string, int>();
intervalService.Elapsed += SendMetrics;
intervalService.Start();
}
public RelayMetricsService(string serviceName,
CancellationToken cancellationToken,
string prefix = null)
{
_prefix = serviceName + "." + (String.IsNullOrEmpty(prefix) ? String.Empty : (prefix + "."));
_counts = new ConcurrentDictionary <string, int>();
_gauges = new ConcurrentDictionary <string, int>();
_intervalService = new IntervalService(60, cancellationToken);
_intervalService.Elapsed += SendPendingMetrics;
_intervalService.Start();
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <SetsBucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary <string, ConcurrentDictionary <int, bool> >();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var set = p as Set;
sets.AddOrUpdate(set.Name,
(key) =>
{
var setDict = new ConcurrentDictionary <int, bool>();
setDict.AddOrUpdate(set.Value, (key2) => true, (key2, oldValue) => true);
return(setDict);
},
(key, setDict) =>
{
setDict.AddOrUpdate(set.Value, (key2) => true, (key2, oldValue) => true);
return(setDict);
});
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var rawData = sets.ToArray();
sets.Clear();
var bucket = new SetsBucket(
rawData.Select(p =>
new KeyValuePair <string, List <KeyValuePair <int, bool> > >(p.Key, p.Value.ToList())
).ToList(),
e.Epoch,
ns);
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public SystemMetricsService(string serviceName, string prefix = null, IIntervalService intervalService = null, bool hideSystemStats = false)
{
if (intervalService == null)
{
intervalService = new IntervalService(10);
}
_prefix = serviceName + "." + (String.IsNullOrEmpty(prefix) ? String.Empty : (prefix + "."));
_metrics = new ConcurrentDictionary<string, double>();
HideSystemStats = hideSystemStats;
intervalService.Elapsed += SendMetrics;
intervalService.Start();
}
public SystemMetricsService(string serviceName, string prefix = null, IIntervalService intervalService = null, bool hideSystemStats = false)
{
if (intervalService == null)
{
intervalService = new IntervalService(10);
}
_prefix = serviceName + "." + (String.IsNullOrEmpty(prefix) ? String.Empty : (prefix + "."));
_metrics = new ConcurrentDictionary <MetricInfo, double>();
HideSystemStats = hideSystemStats;
intervalService.Elapsed += SendMetrics;
intervalService.Start();
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
IIntervalService intervalService,
int percentile,
string percentileName,
ILog log,
int maxItemsPerBucket = 1000)
{
var latencies = new ConcurrentDictionary<string, DatapointBox>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var random = new Random();
percentileName = "." + ( percentileName ?? ( "p" + percentile ) );
var incoming = new ActionBlock<StatsdMessage>( p =>
{
var latency = p as Timing;
latencies.AddOrUpdate(latency.Name,
(key) =>
{
return new DatapointBox(maxItemsPerBucket, latency.ValueMS);
},
(key, bag) =>
{
bag.Add(latency.ValueMS);
return bag;
});
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (latencies.Count == 0)
{
return;
}
var bucket = new PercentileBucket(latencies.ToArray(),
e.Epoch,
ns,
percentileName,
percentile);
latencies.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<SetsBucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary<string, ConcurrentDictionary<int, bool>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var set = p as Set;
sets.AddOrUpdate(set.Name,
(key) =>
{
var setDict = new ConcurrentDictionary<int, bool>();
setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true );
return setDict;
},
(key, setDict) =>
{
setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true );
return setDict;
});
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var rawData = sets.ToArray();
sets.Clear();
var bucket = new SetsBucket(
rawData.Select(p =>
new KeyValuePair<string, List<KeyValuePair<int, bool>>>(p.Key, p.Value.ToList())
).ToList(),
e.Epoch,
ns);
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <Bucket> target,
string rootNamespace,
IIntervalService intervalService,
bool calculateSumSquares,
ILog log,
int maxItemsPerBucket = 1000)
{
var latencies = new ConcurrentDictionary <string, LatencyDatapointBox>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var latency = p as Timing;
latencies.AddOrUpdate(latency.Name,
(key) =>
{
return(new LatencyDatapointBox(maxItemsPerBucket, latency.ValueMS));
},
(key, bag) =>
{
bag.Add(latency.ValueMS);
return(bag);
});
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (latencies.Count == 0)
{
return;
}
var latencyBucket = new LatencyBucket(latencies.ToArray(), e.Epoch, ns, calculateSumSquares);
latencies.Clear();
target.Post(latencyBucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <GraphiteLine> target,
string rootNamespace,
bool deleteGaugesOnFlush,
IIntervalService intervalService,
ILog log)
{
var gauges = new ConcurrentDictionary <string, int>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var gauge = p as Gauge;
gauges.AddOrUpdate(gauge.Name, gauge.Value, (key, oldValue) => gauge.Value);
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (gauges.Count == 0)
{
return;
}
var bucket = gauges.ToArray();
if (deleteGaugesOnFlush)
{
}
var lines = bucket.Select(q => new GraphiteLine(ns + q.Key, q.Value, e.Epoch)).ToArray();
for (int i = 0; i < lines.Length; i++)
{
target.Post(lines[i]);
}
log.InfoFormat("TimedGaugeAggregatorBlock - Posted {0} lines.", lines.Length);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <Bucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var counters = new ConcurrentDictionary <MetricInfo, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : (rootNamespace + ".");
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var counter = p as Counter;
var metricInfo = new MetricInfo(counter.Name, counter.Tags);
counters.AddOrUpdate(metricInfo, counter.Value, (key, oldValue) => oldValue + counter.Value);
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (counters.Count == 0)
{
return;
}
var bucket = new CounterBucket(counters.ToArray(), e.Epoch, ns);
counters.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
IIntervalService intervalService)
{
var rawLines = new ConcurrentStack<Raw>();
var incoming = new ActionBlock<StatsdMessage>(p =>
{
rawLines.Push(p as Raw);
},
Utility.UnboundedExecution());
intervalService.Elapsed += (sender, e) =>
{
if (rawLines.Count == 0)
{
return;
}
var lines = rawLines.ToArray();
rawLines.Clear();
var bucket = new RawBucket(lines, e.Epoch);
target.Post(bucket);
};
return incoming;
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <Bucket> target,
IIntervalService intervalService)
{
var rawLines = new ConcurrentStack <Raw>();
var incoming = new ActionBlock <StatsdMessage>(p =>
{
rawLines.Push(p as Raw);
},
Utility.UnboundedExecution());
intervalService.Elapsed += (sender, e) =>
{
if (rawLines.Count == 0)
{
return;
}
var lines = rawLines.ToArray();
rawLines.Clear();
var bucket = new RawBucket(lines, e.Epoch);
target.Post(bucket);
};
return(incoming);
}
public TimedBufferBlock(TimeSpan flushPeriod,
Action <T[]> flushAction,
IIntervalService interval = null,
CancellationToken?cancellationToken = null)
{
_buffer = new ConcurrentStack <T>();
_completionTask = new Task(() =>
{
_isActive = false;
});
_flushAction = flushAction;
if (interval == null)
{
_intervalService = new IntervalService(flushPeriod, cancellationToken);
}
_intervalService.Elapsed += (sender, e) =>
{
Flush();
};
_intervalService.Start();
_isActive = true;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var counters = new ConcurrentDictionary<string, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : (rootNamespace + ".");
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var counter = p as Counter;
counters.AddOrUpdate(counter.Name, counter.Value, (key, oldValue) => oldValue + counter.Value);
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (counters.Count == 0)
{
return;
}
var bucket = new CounterBucket(counters.ToArray(), e.Epoch, ns);
counters.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public AppMonitoringController(IIntervalService intervalService, ICronJobService cronJobService, IAppService applicationService)
{
this.intervalService = intervalService;
this.cronJobService = cronJobService;
this.applicationService = applicationService;
}
public NoteService(IKeyService keyService, IIntervalService intervalService)
{
_keyService = keyService;
_intervalService = intervalService;
}
public NoteIntervalService(INoteService noteService, IIntervalService intervalService)
{
NoteService = noteService;
IntervalService = intervalService;
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <CounterBucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary <string, ConcurrentDictionary <string, int> >();
var windows = new ConcurrentDictionary <string, ConcurrentDictionary <string, bool> >();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var timeWindow = new TimeWindow();
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var set = p as Set;
var metricName = set.Name + METRIC_IDENTIFIER_SEPARATOR + set.Value;
foreach (var period in timeWindow.AllPeriods)
{
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary <string, bool>();
window.AddOrUpdate(metricName, (key2) => true, (key2, oldValue) => true);
return(window);
},
(key, window) =>
{
window.AddOrUpdate(metricName, (key2) => true, (key2, oldValue) => true);
return(window);
}
);
}
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var currentTimeWindow = new TimeWindow();
var periodsNotPresent = currentTimeWindow.GetDifferences(timeWindow);
// Make the current time window the one we use to manage the collections
timeWindow = currentTimeWindow;
CounterBucket bucket;
// (Parallel) For each period that was measured (Day, Week, Month etc)
// for every unique metric + value
// Count the number of entries that start with the metric name
// Add that metric to the list
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary <String, bool> window;
// Take this window out of the dictionary
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary <String, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Split(METRIC_IDENTIFIER_SEPARATOR_SPLITTER, StringSplitOptions.RemoveEmptyEntries)[0] + qualifier;
if (metrics.ContainsKey(metricName))
{
metrics[metricName] += 1;
}
else
{
metrics[metricName] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return(new KeyValuePair <string, double>(
metric.Key,
metric.Value
));
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
break;
}
}
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="interval"></param>
public IntervalViewModel(Interval interval)
{
intervalService = CommonServiceLocator.ServiceLocator.Current.GetInstance <IIntervalService>();
model = interval;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<CounterBucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary<string, ConcurrentDictionary<string, int>>();
var windows = new ConcurrentDictionary<string, ConcurrentDictionary<string, bool>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var timeWindow = new TimeWindow();
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var set = p as Set;
var metricName = set.Name + METRIC_IDENTIFIER_SEPARATOR + set.Value;
foreach (var period in timeWindow.AllPeriods)
{
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary<string, bool>();
window.AddOrUpdate(metricName, (key2) => true, (key2, oldValue) => true);
return window;
},
(key, window) =>
{
window.AddOrUpdate(metricName, (key2) => true, (key2, oldValue) => true);
return window;
}
);
}
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var currentTimeWindow = new TimeWindow();
var periodsNotPresent = currentTimeWindow.GetDifferences(timeWindow);
// Make the current time window the one we use to manage the collections
timeWindow = currentTimeWindow;
CounterBucket bucket;
// (Parallel) For each period that was measured (Day, Week, Month etc)
// for every unique metric + value
// Count the number of entries that start with the metric name
// Add that metric to the list
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary<String, bool> window;
// Take this window out of the dictionary
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary<String, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Split(METRIC_IDENTIFIER_SEPARATOR_SPLITTER, StringSplitOptions.RemoveEmptyEntries)[0] + qualifier;
if (metrics.ContainsKey(metricName))
{
metrics[metricName] += 1;
}
else
{
metrics[metricName] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return new KeyValuePair<string, double>(
metric.Key,
metric.Value
);
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
break;
}
}
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <GraphiteLine> target,
string rootNamespace,
IIntervalService intervalService,
int percentile,
string percentileName,
ILog log,
int maxItemsPerBucket = 1000)
{
var latencies = new ConcurrentDictionary <string, DatapointBox>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var random = new Random();
percentileName = "." + (percentileName ?? ("p" + percentile));
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var latency = p as Timing;
latencies.AddOrUpdate(latency.Name,
(key) =>
{
return(new DatapointBox(maxItemsPerBucket, latency.ValueMS));
},
(key, bag) =>
{
bag.Add(latency.ValueMS);
return(bag);
});
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (latencies.Count == 0)
{
return;
}
var buckets = latencies.ToArray();
latencies.Clear();
int percentileValue;
int numLinesPosted = 0;
foreach (var measurements in buckets)
{
if (Percentile.TryCompute(measurements.Value.ToArray().ToList(), percentile, out percentileValue))
{
target.Post(new GraphiteLine(ns + measurements.Key + percentileName, percentileValue, e.Epoch));
numLinesPosted++;
}
}
log.InfoFormat("TimedLatencyPercentileAggregatorBlock - Posted {0} buckets and {1} lines.", buckets.Length, numLinesPosted);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <CounterBucket> target,
string rootNamespace,
IIntervalService intervalService,
ITimeWindowService timeWindowService,
ILog log)
{
var windows = new ConcurrentDictionary <string, ConcurrentDictionary <string, double> >();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var calendargram = p as Calendargram;
var metricName = calendargram.Name + METRIC_IDENTIFIER_SEPARATOR + calendargram.Value;
var period = timeWindowService.GetTimeWindow().GetTimePeriod(calendargram.Period);
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary <string, double>();
window.AddOrUpdate(metricName, (key2) => 1, (key2, oldValue) => 1);
return(window);
},
(key, window) =>
{
window.AddOrUpdate(metricName, (key2) => 1, (key2, oldValue) => 1);
return(window);
}
);
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (windows.Count == 0)
{
return;
}
var currentTimeWindow = timeWindowService.GetTimeWindow();
var periodsNotPresent = windows
.ToArray()
.Where(p => !currentTimeWindow.AllPeriods.Contains(p.Key))
.Select(p => p.Key);
CounterBucket bucket;
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary <String, double> window;
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary <String, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Split(
METRIC_IDENTIFIER_SEPARATOR_SPLITTER,
StringSplitOptions.RemoveEmptyEntries
)[0] + qualifier;
if (metrics.ContainsKey(metricName))
{
metrics[metricName] += 1;
}
else
{
metrics[metricName] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return(new KeyValuePair <string, double>(
metric.Key,
metric.Value
));
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
}
}
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public IEnumerable <IInterval> GetIntervals([FromServices] IIntervalService intervalService) => intervalService.GetIntervals();
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<CounterBucket> target,
string rootNamespace,
IIntervalService intervalService,
ITimeWindowService timeWindowService,
ILog log)
{
var windows = new ConcurrentDictionary<string, ConcurrentDictionary<string, double>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var calendargram = p as Calendargram;
var metricName = calendargram.Name + METRIC_IDENTIFIER_SEPARATOR + calendargram.Value;
var period = timeWindowService.GetTimeWindow().GetTimePeriod(calendargram.Period);
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary<string, double>();
window.AddOrUpdate(metricName, (key2) => 1, (key2, oldValue) => 1);
return window;
},
(key, window) =>
{
window.AddOrUpdate(metricName, (key2) => 1, (key2, oldValue) => 1);
return window;
}
);
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (windows.Count == 0)
{
return;
}
var currentTimeWindow = timeWindowService.GetTimeWindow();
var periodsNotPresent = windows
.ToArray()
.Where(p => !currentTimeWindow.AllPeriods.Contains(p.Key))
.Select(p => p.Key);
CounterBucket bucket;
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary<String, double> window;
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary<String, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Split(
METRIC_IDENTIFIER_SEPARATOR_SPLITTER,
StringSplitOptions.RemoveEmptyEntries
)[0] + qualifier;
if (metrics.ContainsKey(metricName))
{
metrics[metricName] += 1;
}
else
{
metrics[metricName] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return new KeyValuePair<string, double>(
metric.Key,
metric.Value
);
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
}
}
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public IntervalBusiness(IIntervalService intervalService)
{
_intervalService = intervalService;
}