private static async Task RunConsumer(ConnectionMultiplexer conn, string consumerName)
{
var redisDb = conn.GetDatabase();
while (true)
{
var currentSeconds = TimeStampProvider.GetCurrentTimeStamp(TimePrecisionUnit.Seconds, 10);
var sortedSetEntries = await redisDb.SortedSetRangeByScoreWithScoresAsync("recentlyset:" + consumerName, double.NegativeInfinity, currentSeconds, take : 1);
if (sortedSetEntries.Length == 1)
{
var metricId = sortedSetEntries[0].Element.ToString();
var metricKey = metricId.Split(':')[1];
var timestamp = long.Parse(metricId.Split(':')[2]);
// Aggreagate this metric
var metricValues = await redisDb.SortedSetRangeByRankWithScoresAsync(metricId, 0, -1);
var values = new List <long>();
foreach (var metricValue in metricValues)
{
var temp = metricValue.Element.ToString();
values.Add(long.Parse(temp));
}
var aggregatedMetric = new AggregatedMetric()
{
Count = values.Count,
Max = values.Max(),
Min = values.Min(),
Sum = values.Sum(),
Timestamp = timestamp
};
var isInserted = await redisDb.SortedSetAddAsync(metricKey + ":values", aggregatedMetric.ToString(), timestamp);
// delete the recentry set metric
var deletedCount = await redisDb.SortedSetRemoveRangeByScoreAsync("recentlyset:" + consumerName, sortedSetEntries[0].Score, sortedSetEntries[0].Score);
// delete the metric's values in the specific time range
var metricValuesDeleted = await redisDb.KeyDeleteAsync(metricId);
// Remove the metric assigned to consumer key
var assignedKeyRemoved = await redisDb.KeyDeleteAsync("assigned.consumer:" + metricId);
}
else
{
await Task.Delay(100);
}
}
}
public void ClearCache(string authenticationCookie)
{
AuthHandler.Authorize(authenticationCookie, PermissionsTable.Instance.CanDeleteCache);
try
{
TimeStampProvider.RefreshTimeStamp();
if (null != _storage)
{
_storage.Value.RunCleanup(null);
}
}
catch { }
}
private async Task HandleRequest(string metricName, long value)
{
var redisDb = conn.GetDatabase(DBNbr);
var currentTimeStampInMicroseconds = TimeStampProvider.GetCurrentTimeStamp(TimePrecisionUnit.Microseconds);
// Decide in which consumer we should put this metric
var metricConsumerName = await GetMetricConsumer(conn, metricName, currentTimeStampInMicroseconds);
// Put the metric value into the consumer's stream
var nameValueEntry = new NameValueEntry[] {
new NameValueEntry("metricName", metricName),
new NameValueEntry("value", value),
new NameValueEntry("time", currentTimeStampInMicroseconds),
};
await redisDb.StreamAddAsync("stream:" + metricConsumerName, nameValueEntry);
}
public static IRuleBuilderOptions <T, long?> TimestampValidation <T>(this IRuleBuilder <T, long?> rule)
{
bool BeWithinRange(long?timestamp)
{
var currentTimestamp = TimeStampProvider.GetCurrentTimeStamp(TimePrecisionUnit.Seconds);
var allowedRangeInSeconds = TimeSpan.FromHours(6).TotalSeconds;
if (timestamp <= currentTimestamp - allowedRangeInSeconds || timestamp >= currentTimestamp + allowedRangeInSeconds)
{
return(false);
}
return(true);
}
return(rule.Must(BeWithinRange)
.When(x => x != null)
.WithMessage("Timestamp should be unix timestamp in seconds that falls between the last and next six hours."));
}
private static async Task RedisRequestHandler(ConnectionMultiplexer conn, string metricName, long value)
{
var redisDb = conn.GetDatabase();
var currentTimeStampInMicroseconds = TimeStampProvider.GetCurrentTimeStamp(TimePrecisionUnit.Microseconds);
var currentTimeStampInSeconds = (long)(Math.Floor((currentTimeStampInMicroseconds / 1_000_000) / 10.0) * 10);
var redisMetricKey = "m:" + metricName + ":" + currentTimeStampInSeconds;
// Insert the value of this metric
var added = await redisDb.SortedSetAddAsync(redisMetricKey, value, currentTimeStampInMicroseconds);
// Check if the metric at this timestamp was already assigned into some consumer
var consumer = await GetMetricConsumer(conn, redisMetricKey);
// Add this metric to the consumer's
var added2 = await redisDb.SortedSetAddAsync("recentlyset:" + consumer, redisMetricKey, currentTimeStampInSeconds + 10);
}
public async Task CollectAggregatedMetricAsync(Guid applicationId, string metricName, string metricNamespace, MetricValue metricValue)
{
if (metricValue.TimeStamp <= 0)
{
metricValue.TimeStamp = TimeStampProvider.GetCurrentTimeStamp(TimePrecisionUnit.Seconds);
}
var redisDb = connectionMultiplexer.GetDatabase();
var metricKey = GetMetricKey(applicationId, metricName, metricNamespace);
await loadedLuaScript.EvaluateAsync(redisDb, new
{
metricKey = (RedisKey)metricKey,
timestamp = metricValue.TimeStamp,
count = metricValue.Count,
min = metricValue.Min,
max = metricValue.Max,
sum = metricValue.Sum,
});;
}
public async Task CollectSingleMetricAsync(Guid applicationId, string metricName, string metricNamespace, double value, long?timestamp)
{
if (timestamp == null || timestamp <= 0)
{
timestamp = TimeStampProvider.GetCurrentTimeStamp(TimePrecisionUnit.Seconds);
}
var redisDb = connectionMultiplexer.GetDatabase();
var metricKey = GetMetricKey(applicationId, metricName, metricNamespace);
await loadedLuaScript.EvaluateAsync(redisDb, new
{
metricKey = (RedisKey)metricKey,
timestamp,
count = 1,
min = value,
max = value,
sum = value,
});
}
public void ShouldReturnTimestampInMinutesPrecision()
{
Assert.DoesNotThrow(() => TimeStampProvider.GetCurrentTimeStamp(TimePrecisionUnit.Minutes));
}
public void ShouldReturnTimestampWithSecondsGranularity(double granularity)
{
var timestamp = TimeStampProvider.GetCurrentTimeStamp(TimePrecisionUnit.Seconds, granularity);
(timestamp / granularity).Should().Equals(0);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var redisDb = connectionMultiplexer.GetDatabase();
ISubscriber sub = connectionMultiplexer.GetSubscriber();
try
{
redisDb.StreamCreateConsumerGroup("metrics_stream", "metrics_consumer_group", "$");
}
catch (Exception)
{ }
while (!stoppingToken.IsCancellationRequested)
{
Stopwatch stopwatch = Stopwatch.StartNew();
Stopwatch stopwatchRead = null;
Stopwatch stopwatchStreamAdding = null;
Stopwatch stopwatchPublish = null;
Stopwatch stopwatchRemove = null;
int takeCount = 100;
int count = -1;
try
{
var currentSeconds = TimeStampProvider.GetCurrentTimeStamp(TimePrecisionUnit.Seconds, 10);
stopwatchRead = Stopwatch.StartNew();
var sortedSetEntries = await redisDb.SortedSetRangeByScoreWithScoresAsync("recentlyset", double.NegativeInfinity, currentSeconds, take : takeCount);
stopwatchRead.Stop();
var itemsToAdd = sortedSetEntries.Select(x => x.Element).ToArray();
count = itemsToAdd.Length;
if (itemsToAdd.Length > 0)
{
stopwatchStreamAdding = Stopwatch.StartNew();
foreach (var item in itemsToAdd)
{
await redisDb.StreamAddAsync("metrics_stream", "key", item);
}
stopwatchStreamAdding.Stop();
//stopwatchPublish = Stopwatch.StartNew();
//foreach (var item in itemsToAdd)
//{
// await sub.PublishAsync("metrics", item);
//}
//stopwatchPublish.Stop();
stopwatchRemove = Stopwatch.StartNew();
await redisDb.SortedSetRemoveRangeByRankAsync("recentlyset", 0, itemsToAdd.Length - 1);
stopwatchRemove.Stop();
}
}
catch (Exception ex)
{
logger.LogError(ex, "Exception in QueueBackgroundService");
}
stopwatch.Stop();
if (count != 0)
{
logger.LogInformation("Adding to Queue took:" + stopwatch.ElapsedMilliseconds + "ms with " + count
+ $" items | Read: {stopwatchRead?.ElapsedMilliseconds}, Stream: {stopwatchStreamAdding?.ElapsedMilliseconds}, "
+ $" Publish: {stopwatchPublish?.ElapsedMilliseconds}, Remove: {stopwatchRemove?.ElapsedMilliseconds}.");
}
int waitTime = 100;
if (count >= takeCount)
{
waitTime = 0;
}
else if (count == 0)
{
waitTime = 200;
}
if (waitTime > 0)
{
await Task.Delay(waitTime, stoppingToken);
}
}
}
