public async Task RapidRetentionAndRollup()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromSeconds(15));
var p1 = new TimeSeriesPolicy("By1", TimeValue.FromSeconds(1), raw.RetentionTime * 2);
var p2 = new TimeSeriesPolicy("By2", TimeValue.FromSeconds(2), raw.RetentionTime * 3);
var p3 = new TimeSeriesPolicy("By4", TimeValue.FromSeconds(4), raw.RetentionTime * 4);
var p4 = new TimeSeriesPolicy("By8", TimeValue.FromSeconds(8), raw.RetentionTime * 5);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var now = DateTime.UtcNow;
var baseline = now.AddSeconds(-15 * 3);
var total = ((TimeSpan)TimeValue.FromSeconds(15 * 3)).TotalMilliseconds;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMilliseconds(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
}
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
WaitForUserToContinueTheTest(store);
await Task.Delay((TimeSpan)(p4.RetentionTime + TimeValue.FromSeconds(10)));
// nothing should be left
using (var session = store.OpenSession())
{
var user = session.Load <User>("users/karmel");
Assert.Equal(0, session.Advanced.GetTimeSeriesFor(user)?.Count ?? 0);
}
}
}
public AddedNewRollupPoliciesCommand(CollectionName collection, TimeSeriesPolicy from, TimeSeriesPolicy to, int skip)
{
_collection = collection;
_from = from;
_to = to;
_skip = skip;
}
public AddedNewRollupPoliciesCommandDto(CollectionName name, TimeSeriesPolicy from, TimeSeriesPolicy to, int skip)
{
_name = name;
_from = @from;
_to = to;
_skip = skip;
}
public static long NextRollup(DateTime time, TimeSeriesPolicy nextPolicy)
{
if (time == DateTime.MinValue)
{
return(time.Add(nextPolicy.AggregationTime).Ticks);
}
switch (nextPolicy.AggregationTime.Unit)
{
case TimeValueUnit.Second:
// align by seconds
var timespan = TimeSpan.FromSeconds(nextPolicy.AggregationTime.Value);
var integerPart = time.Ticks / timespan.Ticks;
var nextRollup = timespan.Ticks * (integerPart + 1);
return(nextRollup);
case TimeValueUnit.Month:
// align by months
var totalMonths = time.Year * 12 + time.Month - 1;
var integerAggPart = totalMonths / nextPolicy.AggregationTime.Value;
var nextInMonths = nextPolicy.AggregationTime.Value * (integerAggPart + 1);
var years = nextInMonths / 12;
var months = nextInMonths % 12;
return(new DateTime(years, months + 1, 1).Ticks);
default:
throw new ArgumentOutOfRangeException(nameof(nextPolicy.AggregationTime.Unit), $"Not supported time value unit '{nextPolicy.AggregationTime.Unit}'");
}
}
private async Task ApplyRetention(
DocumentsOperationContext context,
TimeSeriesCollectionConfiguration config,
CollectionName collectionName,
TimeSeriesPolicy policy,
DateTime now)
{
var tss = context.DocumentDatabase.DocumentsStorage.TimeSeriesStorage;
if (policy.RetentionTime == TimeValue.MaxValue)
{
return;
}
var to = now.Add(-policy.RetentionTime);
var list = new List <Slice>();
while (true)
{
Cts.Token.ThrowIfCancellationRequested();
context.Reset();
context.Renew();
list.Clear();
using (context.OpenReadTransaction())
{
foreach (var item in tss.Stats.GetTimeSeriesByPolicyFromStartDate(context, collectionName, policy.Name, to, TimeSeriesRollups.TimeSeriesRetentionCommand.BatchSize))
{
if (RequiredForNextPolicy(context, config, policy, item, to))
{
continue;
}
if (tss.Rollups.HasPendingRollupFrom(context, item, to) == false)
{
list.Add(item);
}
}
if (list.Count == 0)
{
return;
}
if (Logger.IsInfoEnabled)
{
Logger.Info($"Found {list.Count} time-series for retention in policy {policy.Name} with collection '{collectionName.Name}' up-to {to}"
#if DEBUG
+ $"{Environment.NewLine}{string.Join(Environment.NewLine, list)}"
#endif
);
}
var cmd = new TimeSeriesRollups.TimeSeriesRetentionCommand(list, collectionName.Name, to);
await _database.TxMerger.Enqueue(cmd);
}
}
}
public async Task TimeSeriesLinqQuery_CanUseSimpleCallExpressionInName()
{
using (var store = GetDocumentStore())
{
var timeSeries = "HeartRate";
var retention = TimeValue.FromHours(48);
var p = new TimeSeriesPolicy("ByHour", TimeValue.FromHours(1), TimeValue.FromHours(24));
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = new RawTimeSeriesPolicy(TimeValue.FromHours(96)),
Policies = new List <TimeSeriesPolicy> {
p
}
}
}
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var baseline = DateTime.UtcNow.Add(-retention * 2);
using (var session = store.OpenSession())
{
session.Store(new User(), DocId);
var timeSeriesFor = session.TimeSeriesFor(DocId, timeSeries);
for (int i = 0; i < 100; i++)
{
timeSeriesFor.Append(baseline.AddHours(i), 29 * i, "watches/fitbit");
}
session.SaveChanges();
}
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
await TimeSeries.WaitForPolicyRunnerAsync(database);
using (var session = store.OpenSession())
{
// todo aviv : remove the single quotes from name when RavenDB-15792 is fixed
var q = session.Query <User>()
.Where(u => u.Id == DocId)
.Select(u => RavenQuery.TimeSeries(u, $"'{p.GetTimeSeriesName(timeSeries)}'")
.ToList());
var result = q.First();
Assert.Equal(24, result.Count);
}
}
}
public async Task ValidateCorrectRetentionAndGet(int minutesOffset)
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var database = await GetDocumentDatabaseInstanceFor(store);
var now = new DateTime(2020, 4, 2).AddMinutes(minutesOffset);
database.Time.UtcDateTime = () => now.AddMilliseconds(1);
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new Core.Utils.Entities.User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMinutes(i), i, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await QueryFromMultipleTimeSeries.VerifyFullPolicyExecution(store, config.Collections["Users"]);
}
}
private async Task AddNewPolicy(CollectionName collectionName, TimeSeriesPolicy prev, TimeSeriesPolicy policy)
{
var skip = 0;
while (true)
{
Cts.Token.ThrowIfCancellationRequested();
var cmd = new TimeSeriesRollups.AddedNewRollupPoliciesCommand(collectionName, prev, policy, skip);
await _database.TxMerger.Enqueue(cmd);
if (Logger.IsInfoEnabled)
Logger.Info($"New policy '{policy.Name}' marked {cmd.Marked} time-series");
if (cmd.Marked < TimeSeriesRollups.AddedNewRollupPoliciesCommand.BatchSize)
break;
skip += cmd.Marked;
}
}
private bool ShouldMarkForPolicy(
DocumentsOperationContext context,
TimeSeriesSliceHolder slicerHolder,
DateTime timestamp,
ulong status,
out TimeSeriesPolicy nextPolicy)
{
nextPolicy = default;
if (Configuration.Collections.TryGetValue(slicerHolder.Collection, out var config) == false)
return false;
var currentIndex = config.GetPolicyIndexByTimeSeries(slicerHolder.Name);
if (currentIndex == -1) // policy not found
return false;
nextPolicy = config.GetNextPolicy(currentIndex);
if (nextPolicy == null)
return false;
if (ReferenceEquals(nextPolicy, TimeSeriesPolicy.AfterAllPolices))
return false;
if (status == TimeSeriesValuesSegment.Dead)
{
var currentPolicy = config.GetPolicy(currentIndex);
if (currentPolicy.RetentionTime < TimeValue.MaxValue)
{
var now = context.DocumentDatabase.Time.GetUtcNow();
var startRollup = new DateTime(TimeSeriesRollups.NextRollup(timestamp, nextPolicy)).Add(-currentPolicy.RetentionTime);
if (startRollup.Add(currentPolicy.RetentionTime) < now)
return false; // ignore this since it is outside our retention frame
}
}
return true;
}
public async Task RapidRetentionAndRollupInACluster()
{
var cluster = await CreateRaftCluster(3, watcherCluster : true);
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ReplicationFactor = 3
}))
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromSeconds(15));
var p1 = new TimeSeriesPolicy("By1", TimeValue.FromSeconds(1), raw.RetentionTime * 2);
var p2 = new TimeSeriesPolicy("By2", TimeValue.FromSeconds(2), raw.RetentionTime * 3);
var p3 = new TimeSeriesPolicy("By4", TimeValue.FromSeconds(4), raw.RetentionTime * 4);
var p4 = new TimeSeriesPolicy("By8", TimeValue.FromSeconds(8), raw.RetentionTime * 5);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var now = DateTime.UtcNow;
var baseline = now.AddSeconds(-15 * 3);
var total = ((TimeSpan)TimeValue.FromSeconds(15 * 3)).TotalMilliseconds;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMilliseconds(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
session.Store(new User {
Name = "Karmel"
}, "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>((DocumentSession)session, "marker", null, TimeSpan.FromSeconds(15)));
}
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await Task.Delay((TimeSpan)(p4.RetentionTime * 2));
// nothing should be left
foreach (var node in cluster.Nodes)
{
using (var nodeStore = GetDocumentStore(new Options
{
Server = node,
CreateDatabase = false,
DeleteDatabaseOnDispose = false,
ModifyDocumentStore = s => s.Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
},
ModifyDatabaseName = _ => store.Database
}))
{
using (var session = nodeStore.OpenSession())
{
var user = session.Load <User>("users/karmel");
Assert.Equal(0, session.Advanced.GetTimeSeriesFor(user)?.Count ?? 0);
var db = await node.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
await TimeSeriesReplicationTests.AssertNoLeftOvers(db);
}
}
}
}
}
public async Task CanWorkWithRollupTimeSeries2()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await store.TimeSeries.RegisterAsync <User, StockPrice>();
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
var ts = session.TimeSeriesFor <StockPrice>("users/karmel");
var entry = new StockPrice();
for (int i = 0; i <= total; i++)
{
entry.Open = i;
entry.Close = i + 100_000;
entry.High = i + 200_000;
entry.Low = i + 300_000;
entry.Volume = i + 400_000;
ts.Append(baseline.AddMinutes(i), entry, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 12, rawName: "StockPrices");
using (var session = store.OpenSession())
{
var query = session.Query <User>()
.Select(u =>
RavenQuery.TimeSeries <StockPrice>(u, "StockPrices").Select(x =>
new {
First = x.First(),
Last = x.Last(),
Min = x.Min(),
Max = x.Max(),
Sum = x.Sum(),
Count = x.Count(),
Avg = x.Average()
})
.ToList());
var result = query.Single();
Assert.Equal(1, result.Results.Length);
var r = result.Results[0];
Assert.NotNull(r.First);
Assert.NotNull(r.Last);
Assert.NotNull(r.Min);
Assert.NotNull(r.Max);
Assert.NotNull(r.Sum);
Assert.NotNull(r.Count);
Assert.NotNull(r.Average);
}
using (var session = store.OpenSession())
{
var query = session.Query <User>()
.Select(u =>
RavenQuery.TimeSeries <StockPrice>(u, "StockPrices")
.GroupBy(x => x.Hours(3))
.Select(x =>
new
{
First = x.First(),
Last = x.Last(),
Min = x.Min(),
Max = x.Max(),
Sum = x.Sum(),
Count = x.Count(),
Avg = x.Average()
})
.ToList());
var result = query.Single();
var r = result.Results[0];
Assert.NotNull(r.First);
Assert.NotNull(r.Last);
Assert.NotNull(r.Min);
Assert.NotNull(r.Max);
Assert.NotNull(r.Sum);
Assert.NotNull(r.Count);
Assert.NotNull(r.Average);
}
using (var session = store.OpenSession())
{
var ts1 = session.TimeSeriesRollupFor <StockPrice>("users/karmel", p1.Name);
var r = ts1.Get().First();
Assert.NotNull(r.First);
Assert.NotNull(r.Last);
Assert.NotNull(r.Min);
Assert.NotNull(r.Max);
Assert.NotNull(r.Sum);
Assert.NotNull(r.Count);
Assert.NotNull(r.Average);
}
}
}
/// <summary>
/// Set rollup and retention policy
/// </summary>
/// <param name="name">Policy name</param>
/// <param name="aggregation">Aggregation time</param>
/// <param name="retention">Retention time</param>
public Task SetPolicyAsync(string collection, string name, TimeValue aggregation, TimeValue retention)
{
var p = new TimeSeriesPolicy(name, aggregation, retention);
return(_executor.SendAsync(new ConfigureTimeSeriesPolicyOperation(collection, p)));
}
public async Task CanIncludeTypedTimeSeries_Rollup()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await store.TimeSeries.RegisterAsync <User, StockPrice>();
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var nowMinutes = now.Minute;
now = now.AddMinutes(-nowMinutes);
database.Time.UtcDateTime = () => DateTime.UtcNow.AddMinutes(-nowMinutes);
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
var ts = session.TimeSeriesFor <StockPrice>("users/karmel");
var entry = new StockPrice();
for (int i = 0; i <= total; i++)
{
entry.Open = i;
entry.Close = i + 100_000;
entry.High = i + 200_000;
entry.Low = i + 300_000;
entry.Volume = i + 400_000;
ts.Append(baseline.AddMinutes(i), entry, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 12, rawName : "StockPrices");
using (var session = store.OpenSession())
{
var user = session.Query <User>()
.Include(x => x.IncludeTimeSeries($"StockPrices@{p1.Name}"))
.First();
// should not go to server
var ts = session.TimeSeriesRollupFor <StockPrice>(user.Id, p1.Name);
var res = ts.Get().ToList();
Assert.Equal(16, res.Count);
Assert.Equal(1, session.Advanced.NumberOfRequests);
}
}
}
public unsafe void MarkForPolicy(DocumentsOperationContext context, TimeSeriesSliceHolder slicerHolder, TimeSeriesPolicy nextPolicy, DateTime timestamp)
{
var nextRollup = NextRollup(timestamp, nextPolicy);
// mark for rollup
RollupSchema.Create(context.Transaction.InnerTransaction, TimeSeriesRollupTable, 16);
var table = context.Transaction.InnerTransaction.OpenTable(RollupSchema, TimeSeriesRollupTable);
using (table.Allocate(out var tvb))
using (Slice.From(context.Allocator, nextPolicy.Name, ByteStringType.Immutable, out var policyToApply))
{
if (table.ReadByKey(slicerHolder.StatsKey, out var tvr))
{
// check if we need to update this
var existingRollup = Bits.SwapBytes(*(long *)tvr.Read((int)RollupColumns.NextRollup, out _));
if (existingRollup <= nextRollup)
{
return; // we have an earlier date to roll up from
}
}
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Marking {slicerHolder.Name} in document {slicerHolder.DocId} for policy {nextPolicy.Name} to rollup at {new DateTime(nextRollup)} (ticks:{nextRollup})");
}
var etag = context.DocumentDatabase.DocumentsStorage.GenerateNextEtag();
var changeVector = context.DocumentDatabase.DocumentsStorage.GetNewChangeVector(context, etag);
using (Slice.From(context.Allocator, changeVector, ByteStringType.Immutable, out var changeVectorSlice))
{
tvb.Add(slicerHolder.StatsKey);
tvb.Add(slicerHolder.CollectionSlice);
tvb.Add(Bits.SwapBytes(nextRollup));
tvb.Add(policyToApply);
tvb.Add(etag);
tvb.Add(changeVectorSlice);
table.Set(tvb);
}
}
}
private void RollupOne(DocumentsOperationContext context, Table table, RollupState item, TimeSeriesPolicy policy, TimeSeriesCollectionConfiguration config)
{
var tss = context.DocumentDatabase.DocumentsStorage.TimeSeriesStorage;
var rawTimeSeries = item.Name.Split(TimeSeriesConfiguration.TimeSeriesRollupSeparator)[0];
var intoTimeSeries = policy.GetTimeSeriesName(rawTimeSeries);
var rollupStart = item.NextRollup.Add(-policy.AggregationTime);
if (config.MaxRetention < TimeValue.MaxValue)
{
var next = new DateTime(NextRollup(_now.Add(-config.MaxRetention), policy)).Add(-policy.AggregationTime);
var rollupStartTicks = Math.Max(rollupStart.Ticks, next.Ticks);
rollupStart = new DateTime(rollupStartTicks);
}
var intoReader = tss.GetReader(context, item.DocId, intoTimeSeries, rollupStart, DateTime.MaxValue);
var previouslyAggregated = intoReader.AllValues().Any();
if (previouslyAggregated)
{
var changeVector = intoReader.GetCurrentSegmentChangeVector();
if (ChangeVectorUtils.GetConflictStatus(item.ChangeVector, changeVector) == ConflictStatus.AlreadyMerged)
{
// this rollup is already done
table.DeleteByKey(item.Key);
return;
}
}
if (_isFirstInTopology == false)
{
return;
}
var rollupEnd = new DateTime(NextRollup(_now, policy)).Add(-policy.AggregationTime).AddMilliseconds(-1);
var reader = tss.GetReader(context, item.DocId, item.Name, rollupStart, rollupEnd);
if (previouslyAggregated)
{
var hasPriorValues = tss.GetReader(context, item.DocId, item.Name, DateTime.MinValue, rollupStart).AllValues().Any();
if (hasPriorValues == false)
{
table.DeleteByKey(item.Key);
var first = tss.GetReader(context, item.DocId, item.Name, rollupStart, DateTime.MaxValue).First();
if (first == default)
{
return;
}
if (first.Timestamp > item.NextRollup)
{
// if the 'source' time-series doesn't have any values it is retained.
// so we need to aggregate only from the next time frame
using (var slicer = new TimeSeriesSliceHolder(context, item.DocId, item.Name, item.Collection))
{
tss.Rollups.MarkForPolicy(context, slicer, policy, first.Timestamp);
}
return;
}
}
}
// rollup from the the raw data will generate 6-value roll up of (first, last, min, max, sum, count)
// other rollups will aggregate each of those values by the type
var mode = item.Name.Contains(TimeSeriesConfiguration.TimeSeriesRollupSeparator) ? AggregationMode.FromAggregated : AggregationMode.FromRaw;
var rangeSpec = new RangeGroup();
switch (policy.AggregationTime.Unit)
{
case TimeValueUnit.Second:
rangeSpec.Ticks = TimeSpan.FromSeconds(policy.AggregationTime.Value).Ticks;
rangeSpec.TicksAlignment = RangeGroup.Alignment.Second;
break;
case TimeValueUnit.Month:
rangeSpec.Months = policy.AggregationTime.Value;
break;
default:
throw new ArgumentOutOfRangeException(nameof(policy.AggregationTime.Unit), $"Not supported time value unit '{policy.AggregationTime.Unit}'");
}
rangeSpec.InitializeRange(rollupStart);
var values = GetAggregatedValues(reader, rangeSpec, mode);
if (previouslyAggregated)
{
// if we need to re-aggregate we need to delete everything we have from that point on.
var removeRequest = new TimeSeriesStorage.DeletionRangeRequest
{
Collection = item.Collection,
DocumentId = item.DocId,
Name = intoTimeSeries,
From = rollupStart,
To = DateTime.MaxValue,
};
tss.DeleteTimestampRange(context, removeRequest);
}
var before = context.LastDatabaseChangeVector;
var after = tss.AppendTimestamp(context, item.DocId, item.Collection, intoTimeSeries, values, verifyName: false);
if (before != after)
{
RolledUp++;
}
table.DeleteByKey(item.Key);
var stats = tss.Stats.GetStats(context, item.DocId, item.Name);
if (stats.End > rollupEnd)
{
// we know that we have values after the current rollup and we need to mark them
var nextRollup = rollupEnd.AddMilliseconds(1);
intoReader = tss.GetReader(context, item.DocId, item.Name, nextRollup, DateTime.MaxValue);
if (intoReader.Init() == false)
{
Debug.Assert(false, "We have values but no segment?");
return;
}
using (var slicer = new TimeSeriesSliceHolder(context, item.DocId, item.Name, item.Collection))
{
tss.Rollups.MarkForPolicy(context, slicer, policy, intoReader.First().Timestamp);
}
}
}
public void Examples()
{
var store = new DocumentStore
{
Urls = new[] { "http://*****:*****@DailyRollupForOneYear");
//Create local instance of the rollup time-series - second method:
//using the rollup policy itself and the raw time-series' name
var rollupTimeSeries2 = session.TimeSeriesFor("sales/1",
dailyRollup.GetTimeSeriesName("rawSales"));
//Retrieve all the data from both time-series
var rawData = rawTS.Get(DateTime.MinValue, DateTime.MaxValue).ToList();
var rollupData = dailyRollupTS.Get(DateTime.MinValue, DateTime.MaxValue).ToList();
#endregion
}
}
private static bool RequiredForNextPolicy(DocumentsOperationContext context, TimeSeriesCollectionConfiguration config, TimeSeriesPolicy policy, Slice item, DateTime to)
{
var tss = context.DocumentDatabase.DocumentsStorage.TimeSeriesStorage;
var next = config.GetNextPolicy(policy);
if (ReferenceEquals(next, TimeSeriesPolicy.AfterAllPolices) == false)
{
TimeSeriesRollups.SplitKey(item, out var docId, out var name);
var raw = name.Split(TimeSeriesConfiguration.TimeSeriesRollupSeparator)[0];
var currentStats = tss.Stats.GetStats(context, docId, policy.GetTimeSeriesName(raw));
var nextStats = tss.Stats.GetStats(context, docId, next.GetTimeSeriesName(raw));
var nextEnd = nextStats.End.Add(next.AggregationTime).AddMilliseconds(-1);
if (nextEnd >= currentStats.End)
return false;
if (nextEnd < to)
return true;
}
return false;
}
public async Task CanWorkWithRollupTimeSeries()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await store.TimeSeries.RegisterAsync <User, StockPrice>();
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var nowMinutes = now.Minute;
now = now.AddMinutes(-nowMinutes);
database.Time.UtcDateTime = () => DateTime.UtcNow.AddMinutes(-nowMinutes);
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
var ts = session.TimeSeriesFor <StockPrice>("users/karmel");
var entry = new StockPrice();
for (int i = 0; i <= total; i++)
{
entry.Open = i;
entry.Close = i + 100_000;
entry.High = i + 200_000;
entry.Low = i + 300_000;
entry.Volume = i + 400_000;
ts.Append(baseline.AddMinutes(i), entry, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 12, rawName: "StockPrices");
using (var session = store.OpenSession())
{
var query = session.Advanced.RawQuery <TimeSeriesRawResult <StockPrice> >(@"
declare timeseries out()
{
from StockPrices
between $start and $end
}
from Users as u
select out()
")
.AddParameter("start", baseline.AddDays(-1))
.AddParameter("end", now.AddDays(1));
var result = query.Single();
var count = result.Results.Length;
Assert.Equal(5, result.Results[count - 1440].Values.Length);
foreach (var res in result.Results)
{
Assert.Equal(5, res.Values.Length);
}
}
using (var session = store.OpenSession())
{
// test the same query using linq
var query = session.Query <User>()
.Select(u =>
RavenQuery.TimeSeries <StockPrice>(u, "StockPrices", baseline.AddDays(-1), now.AddDays(1))
.ToList());
var result = query.Single();
var count = result.Results.Length;
Assert.Equal(5, result.Results[count - 1440].Values.Length);
foreach (var res in result.Results)
{
Assert.Equal(5, res.Values.Length);
}
}
now = DateTime.UtcNow;
using (var session = store.OpenSession())
{
var ts = session.TimeSeriesRollupFor <StockPrice>("users/karmel", p1.Name);
var a = new TimeSeriesRollupEntry <StockPrice>(DateTime.Now)
{
Max = new StockPrice
{
Close = 1
}
};
ts.Append(a);
session.SaveChanges();
}
using (var session = store.OpenSession())
{
var ts = session.TimeSeriesRollupFor <StockPrice>("users/karmel", p1.Name);
var res = ts.Get(from: now.AddMilliseconds(-1)).ToList();
Assert.Equal(1, res.Count);
Assert.Equal(1, res[0].Max.Close);
}
}
}
public async Task CanIndexRollups()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeValue.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeValue.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeValue.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeValue.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var nowSeconds = now.Second;
now = now.AddSeconds(-nowSeconds);
database.Time.UtcDateTime = () => DateTime.UtcNow.AddSeconds(-nowSeconds);
var baseline = now.AddDays(-12);
var total = ((TimeSpan)TimeValue.FromDays(12)).TotalMinutes;
await new TimeSeriesIndex().ExecuteAsync(store);
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMinutes(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
}
await TimeSeries.WaitForPolicyRunnerAsync(database);
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 4, policies : new List <TimeSeriesPolicy> {
p1
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 5, policies : new List <TimeSeriesPolicy> {
p2
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 2, policies : new List <TimeSeriesPolicy> {
p3
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 3, policies : new List <TimeSeriesPolicy> {
p4
});
Indexes.WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
using (var session = store.OpenSession())
{
var user = session.Load <User>("users/karmel");
var count = session
.TimeSeriesFor(user, "Heartrate")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count(entry => entry.IsRollup == false);
count += session
.TimeSeriesFor(user, "Heartrate@By6Hours")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By1Day")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By30Minutes")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By1Hour")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
var results = session.Query <TimeSeriesIndex.Result, TimeSeriesIndex>()
.ToList();
Assert.True(count == results.Count, $"Test time = {now}");
}
}
}
private static void MarkForNextPolicyAfterRollup(DocumentsOperationContext context, Table table, RollupState item, TimeSeriesPolicy policy, TimeSeriesStorage tss,
DateTime rollupEnd)
{
table.DeleteByKey(item.Key);
(long Count, DateTime Start, DateTime End)stats = tss.Stats.GetStats(context, item.DocId, item.Name);
if (stats.End > rollupEnd)
{
// we know that we have values after the current rollup and we need to mark them
var nextRollup = rollupEnd.AddMilliseconds(1);
TimeSeriesReader intoReader = tss.GetReader(context, item.DocId, item.Name, nextRollup, DateTime.MaxValue);
if (intoReader.Init() == false)
{
Debug.Assert(false, "We have values but no segment?");
return;
}
using (var slicer = new TimeSeriesSliceHolder(context, item.DocId, item.Name, item.Collection))
{
tss.Rollups.MarkForPolicy(context, slicer, policy, intoReader.First().Timestamp);
}
}
}
