protected override long ExecuteCmd(DocumentsOperationContext context)
{
var logger = LoggingSource.Instance.GetLogger <TimeSeriesPolicyRunner>(context.DocumentDatabase.Name);
var request = new TimeSeriesStorage.DeletionRangeRequest
{
From = DateTime.MinValue,
To = _to,
Collection = _collection
};
var retained = 0;
foreach (var key in _keys)
{
SplitKey(key, out var docId, out var name);
request.Name = name;
request.DocumentId = docId;
var done = context.DocumentDatabase.DocumentsStorage.TimeSeriesStorage.DeleteTimestampRange(context, request) != null;
if (done)
{
retained++;
}
if (logger.IsInfoEnabled)
{
logger.Info($"{request} was executed (successfully: {done})");
}
}
return(retained);
}
protected override long ExecuteCmd(DocumentsOperationContext context)
{
string docCollection = GetDocumentCollection(_database, context, _documentId, _fromEtl);
if (docCollection == null)
{
return(0L);
}
var changes = 0L;
var tss = _database.DocumentsStorage.TimeSeriesStorage;
if (_operation.Deletes?.Count > 0)
{
foreach (var removal in _operation.Deletes)
{
var deletionRange = new TimeSeriesStorage.DeletionRangeRequest
{
DocumentId = _documentId,
Collection = docCollection,
Name = _operation.Name,
From = removal.From ?? DateTime.MinValue,
To = removal.To ?? DateTime.MaxValue
};
LastChangeVector = tss.DeleteTimestampRange(context, deletionRange);
changes++;
}
}
if (_operation.Appends?.Count > 0 == false)
{
return(changes);
}
LastChangeVector = tss.AppendTimestamp(context,
_documentId,
docCollection,
_operation.Name,
_operation.Appends
);
changes += _operation.Appends.Count;
return(changes);
}
private void RollupOne(DocumentsOperationContext context, Table table, RollupState item, TimeSeriesPolicy policy, TimeSeriesCollectionConfiguration config)
{
var tss = context.DocumentDatabase.DocumentsStorage.TimeSeriesStorage;
var rawTimeSeries = item.Name.Split(TimeSeriesConfiguration.TimeSeriesRollupSeparator)[0];
var intoTimeSeries = policy.GetTimeSeriesName(rawTimeSeries);
var rollupStart = item.NextRollup.Add(-policy.AggregationTime);
if (config.MaxRetention < TimeValue.MaxValue)
{
var next = new DateTime(NextRollup(_now.Add(-config.MaxRetention), policy)).Add(-policy.AggregationTime);
var rollupStartTicks = Math.Max(rollupStart.Ticks, next.Ticks);
rollupStart = new DateTime(rollupStartTicks);
}
var intoReader = tss.GetReader(context, item.DocId, intoTimeSeries, rollupStart, DateTime.MaxValue);
var previouslyAggregated = intoReader.AllValues().Any();
if (previouslyAggregated)
{
var changeVector = intoReader.GetCurrentSegmentChangeVector();
if (ChangeVectorUtils.GetConflictStatus(item.ChangeVector, changeVector) == ConflictStatus.AlreadyMerged)
{
// this rollup is already done
table.DeleteByKey(item.Key);
return;
}
}
if (_isFirstInTopology == false)
{
return;
}
var rollupEnd = new DateTime(NextRollup(_now, policy)).Add(-policy.AggregationTime).AddMilliseconds(-1);
var reader = tss.GetReader(context, item.DocId, item.Name, rollupStart, rollupEnd);
if (previouslyAggregated)
{
var hasPriorValues = tss.GetReader(context, item.DocId, item.Name, DateTime.MinValue, rollupStart).AllValues().Any();
if (hasPriorValues == false)
{
table.DeleteByKey(item.Key);
var first = tss.GetReader(context, item.DocId, item.Name, rollupStart, DateTime.MaxValue).First();
if (first == default)
{
return;
}
if (first.Timestamp > item.NextRollup)
{
// if the 'source' time-series doesn't have any values it is retained.
// so we need to aggregate only from the next time frame
using (var slicer = new TimeSeriesSliceHolder(context, item.DocId, item.Name, item.Collection))
{
tss.Rollups.MarkForPolicy(context, slicer, policy, first.Timestamp);
}
return;
}
}
}
// rollup from the the raw data will generate 6-value roll up of (first, last, min, max, sum, count)
// other rollups will aggregate each of those values by the type
var mode = item.Name.Contains(TimeSeriesConfiguration.TimeSeriesRollupSeparator) ? AggregationMode.FromAggregated : AggregationMode.FromRaw;
var rangeSpec = new RangeGroup();
switch (policy.AggregationTime.Unit)
{
case TimeValueUnit.Second:
rangeSpec.Ticks = TimeSpan.FromSeconds(policy.AggregationTime.Value).Ticks;
rangeSpec.TicksAlignment = RangeGroup.Alignment.Second;
break;
case TimeValueUnit.Month:
rangeSpec.Months = policy.AggregationTime.Value;
break;
default:
throw new ArgumentOutOfRangeException(nameof(policy.AggregationTime.Unit), $"Not supported time value unit '{policy.AggregationTime.Unit}'");
}
rangeSpec.InitializeRange(rollupStart);
var values = GetAggregatedValues(reader, rangeSpec, mode);
if (previouslyAggregated)
{
// if we need to re-aggregate we need to delete everything we have from that point on.
var removeRequest = new TimeSeriesStorage.DeletionRangeRequest
{
Collection = item.Collection,
DocumentId = item.DocId,
Name = intoTimeSeries,
From = rollupStart,
To = DateTime.MaxValue,
};
tss.DeleteTimestampRange(context, removeRequest);
}
var before = context.LastDatabaseChangeVector;
var after = tss.AppendTimestamp(context, item.DocId, item.Collection, intoTimeSeries, values, verifyName: false);
if (before != after)
{
RolledUp++;
}
table.DeleteByKey(item.Key);
var stats = tss.Stats.GetStats(context, item.DocId, item.Name);
if (stats.End > rollupEnd)
{
// we know that we have values after the current rollup and we need to mark them
var nextRollup = rollupEnd.AddMilliseconds(1);
intoReader = tss.GetReader(context, item.DocId, item.Name, nextRollup, DateTime.MaxValue);
if (intoReader.Init() == false)
{
Debug.Assert(false, "We have values but no segment?");
return;
}
using (var slicer = new TimeSeriesSliceHolder(context, item.DocId, item.Name, item.Collection))
{
tss.Rollups.MarkForPolicy(context, slicer, policy, intoReader.First().Timestamp);
}
}
}
protected override long ExecuteCmd(DocumentsOperationContext context)
{
var tss = context.DocumentDatabase.DocumentsStorage.TimeSeriesStorage;
RollupSchema.Create(context.Transaction.InnerTransaction, TimeSeriesRollupTable, 16);
var table = context.Transaction.InnerTransaction.OpenTable(RollupSchema, TimeSeriesRollupTable);
foreach (var item in _states)
{
if (_configuration == null)
{
return(RolledUp);
}
if (_configuration.Collections.TryGetValue(item.Collection, out var config) == false)
{
continue;
}
if (config.Disabled)
{
continue;
}
if (table.ReadByKey(item.Key, out var current) == false)
{
continue;
}
var policy = config.GetPolicyByName(item.RollupPolicy, out _);
if (policy == null)
{
table.DeleteByKey(item.Key);
continue;
}
if (item.Etag != DocumentsStorage.TableValueToLong((int)RollupColumns.Etag, ref current))
{
continue; // concurrency check
}
var rawTimeSeries = item.Name.Split(TimeSeriesConfiguration.TimeSeriesRollupSeparator)[0];
var intoTimeSeries = policy.GetTimeSeriesName(rawTimeSeries);
var rollupStart = item.NextRollup.Add(-policy.AggregationTime);
if (config.MaxRetention < TimeValue.MaxValue)
{
var next = new DateTime(NextRollup(_now.Add(-config.MaxRetention), policy)).Add(-policy.AggregationTime);
var rollupStartTicks = Math.Max(rollupStart.Ticks, next.Ticks);
rollupStart = new DateTime(rollupStartTicks);
}
var intoReader = tss.GetReader(context, item.DocId, intoTimeSeries, rollupStart, DateTime.MaxValue);
var previouslyAggregated = intoReader.AllValues().Any();
if (previouslyAggregated)
{
var changeVector = intoReader.GetCurrentSegmentChangeVector();
if (ChangeVectorUtils.GetConflictStatus(item.ChangeVector, changeVector) == ConflictStatus.AlreadyMerged)
{
// this rollup is already done
table.DeleteByKey(item.Key);
continue;
}
}
if (_isFirstInTopology == false)
{
continue; // we execute the actual rollup only on the primary node to avoid conflicts
}
var rollupEnd = new DateTime(NextRollup(_now, policy)).Add(-policy.AggregationTime).AddMilliseconds(-1);
var reader = tss.GetReader(context, item.DocId, item.Name, rollupStart, rollupEnd);
if (previouslyAggregated)
{
var hasPriorValues = tss.GetReader(context, item.DocId, item.Name, DateTime.MinValue, rollupStart).AllValues().Any();
if (hasPriorValues == false)
{
table.DeleteByKey(item.Key);
var first = tss.GetReader(context, item.DocId, item.Name, rollupStart, DateTime.MaxValue).First();
if (first == default)
{
continue; // nothing we can do here
}
if (first.Timestamp > item.NextRollup)
{
// if the 'source' time-series doesn't have any values it is retained.
// so we need to aggregate only from the next time frame
using (var slicer = new TimeSeriesSliceHolder(context, item.DocId, item.Name, item.Collection))
{
tss.Rollups.MarkForPolicy(context, slicer, policy, first.Timestamp);
}
continue;
}
}
}
// rollup from the the raw data will generate 6-value roll up of (first, last, min, max, sum, count)
// other rollups will aggregate each of those values by the type
var mode = item.Name.Contains(TimeSeriesConfiguration.TimeSeriesRollupSeparator) ? AggregationMode.FromAggregated : AggregationMode.FromRaw;
var rangeSpec = new RangeGroup();
switch (policy.AggregationTime.Unit)
{
case TimeValueUnit.Second:
rangeSpec.Ticks = TimeSpan.FromSeconds(policy.AggregationTime.Value).Ticks;
rangeSpec.TicksAlignment = RangeGroup.Alignment.Second;
break;
case TimeValueUnit.Month:
rangeSpec.Months = policy.AggregationTime.Value;
break;
default:
throw new ArgumentOutOfRangeException(nameof(policy.AggregationTime.Unit), $"Not supported time value unit '{policy.AggregationTime.Unit}'");
}
rangeSpec.InitializeRange(rollupStart);
List <SingleResult> values = null;
try
{
values = GetAggregatedValues(reader, rangeSpec, mode);
}
catch (RollupExceedNumberOfValuesException e)
{
var name = item.Name;
var docId = item.DocId;
try
{
var document = context.DocumentDatabase.DocumentsStorage.Get(context, item.DocId, throwOnConflict: false);
docId = document?.Id ?? docId;
name = tss.GetOriginalName(context, docId, name);
}
catch
{
// ignore
}
var msg = $"Rollup '{item.RollupPolicy}' for time-series '{name}' in document '{docId}' failed.";
if (_logger.IsInfoEnabled)
{
_logger.Info(msg, e);
}
var alert = AlertRaised.Create(context.DocumentDatabase.Name, "Failed to perform rollup because the time-series has more than 5 values", msg,
AlertType.RollupExceedNumberOfValues, NotificationSeverity.Warning, $"{item.DocId}/{item.Name}", new ExceptionDetails(e));
context.DocumentDatabase.NotificationCenter.Add(alert);
continue;
}
if (previouslyAggregated)
{
// if we need to re-aggregate we need to delete everything we have from that point on.
var removeRequest = new TimeSeriesStorage.DeletionRangeRequest
{
Collection = item.Collection,
DocumentId = item.DocId,
Name = intoTimeSeries,
From = rollupStart,
To = DateTime.MaxValue,
};
tss.DeleteTimestampRange(context, removeRequest);
}
var before = context.LastDatabaseChangeVector;
var after = tss.AppendTimestamp(context, item.DocId, item.Collection, intoTimeSeries, values, verifyName: false);
if (before != after)
{
RolledUp++;
}
table.DeleteByKey(item.Key);
var stats = tss.Stats.GetStats(context, item.DocId, item.Name);
if (stats.End > rollupEnd)
{
// we know that we have values after the current rollup and we need to mark them
var nextRollup = rollupEnd.AddMilliseconds(1);
intoReader = tss.GetReader(context, item.DocId, item.Name, nextRollup, DateTime.MaxValue);
if (intoReader.Init() == false)
{
Debug.Assert(false, "We have values but no segment?");
continue;
}
using (var slicer = new TimeSeriesSliceHolder(context, item.DocId, item.Name, item.Collection))
{
tss.Rollups.MarkForPolicy(context, slicer, policy, intoReader.First().Timestamp);
}
}
}
return(RolledUp);
}
