public async Task RapidRetentionAndRollup()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromSeconds(15));
var p1 = new TimeSeriesPolicy("By1", TimeValue.FromSeconds(1), raw.RetentionTime * 2);
var p2 = new TimeSeriesPolicy("By2", TimeValue.FromSeconds(2), raw.RetentionTime * 3);
var p3 = new TimeSeriesPolicy("By4", TimeValue.FromSeconds(4), raw.RetentionTime * 4);
var p4 = new TimeSeriesPolicy("By8", TimeValue.FromSeconds(8), raw.RetentionTime * 5);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var now = DateTime.UtcNow;
var baseline = now.AddSeconds(-15 * 3);
var total = ((TimeSpan)TimeValue.FromSeconds(15 * 3)).TotalMilliseconds;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMilliseconds(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
}
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
WaitForUserToContinueTheTest(store);
await Task.Delay((TimeSpan)(p4.RetentionTime + TimeValue.FromSeconds(10)));
// nothing should be left
using (var session = store.OpenSession())
{
var user = session.Load <User>("users/karmel");
Assert.Equal(0, session.Advanced.GetTimeSeriesFor(user)?.Count ?? 0);
}
}
}
public async Task ValidateCorrectRetentionAndGet(int minutesOffset)
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var database = await GetDocumentDatabaseInstanceFor(store);
var now = new DateTime(2020, 4, 2).AddMinutes(minutesOffset);
database.Time.UtcDateTime = () => now.AddMilliseconds(1);
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new Core.Utils.Entities.User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMinutes(i), i, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await QueryFromMultipleTimeSeries.VerifyFullPolicyExecution(store, config.Collections["Users"]);
}
}
public async Task AddTimeSeriesPolicy()
{
await ServerStore.EnsureNotPassiveAsync();
var collection = GetStringQueryString("collection", required: true);
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var json = context.ReadForDisk(RequestBodyStream(), "time-series policy config"))
{
var policy = JsonDeserializationCluster.TimeSeriesPolicy(json);
TimeSeriesConfiguration current;
using (context.OpenReadTransaction())
{
current = ServerStore.Cluster.ReadRawDatabaseRecord(context, Database.Name).TimeSeriesConfiguration ?? new TimeSeriesConfiguration();
}
current.Collections ??= new Dictionary <string, TimeSeriesCollectionConfiguration>(StringComparer.OrdinalIgnoreCase);
if (current.Collections.ContainsKey(collection) == false)
{
current.Collections[collection] = new TimeSeriesCollectionConfiguration();
}
if (RawTimeSeriesPolicy.IsRaw(policy))
{
current.Collections[collection].RawPolicy = new RawTimeSeriesPolicy(policy.RetentionTime);
}
else
{
current.Collections[collection].Policies ??= new List <TimeSeriesPolicy>();
var existing = current.Collections[collection].GetPolicyByName(policy.Name, out _);
if (existing != null)
{
current.Collections[collection].Policies.Remove(existing);
}
current.Collections[collection].Policies.Add(policy);
}
current.InitializeRollupAndRetention();
ServerStore.LicenseManager.AssertCanAddTimeSeriesRollupsAndRetention(current);
var editTimeSeries = new EditTimeSeriesConfigurationCommand(current, Database.Name, GetRaftRequestIdFromQuery());
var(index, _) = await ServerStore.SendToLeaderAsync(editTimeSeries);
await WaitForIndexToBeApplied(context, index);
SendConfigurationResponse(context, index);
}
}
public async Task CanIndexRollups()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeValue.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeValue.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeValue.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeValue.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var nowSeconds = now.Second;
now = now.AddSeconds(-nowSeconds);
database.Time.UtcDateTime = () => DateTime.UtcNow.AddSeconds(-nowSeconds);
var baseline = now.AddDays(-12);
var total = ((TimeSpan)TimeValue.FromDays(12)).TotalMinutes;
await new TimeSeriesIndex().ExecuteAsync(store);
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMinutes(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
}
await TimeSeries.WaitForPolicyRunnerAsync(database);
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 4, policies : new List <TimeSeriesPolicy> {
p1
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 5, policies : new List <TimeSeriesPolicy> {
p2
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 2, policies : new List <TimeSeriesPolicy> {
p3
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 3, policies : new List <TimeSeriesPolicy> {
p4
});
Indexes.WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
using (var session = store.OpenSession())
{
var user = session.Load <User>("users/karmel");
var count = session
.TimeSeriesFor(user, "Heartrate")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count(entry => entry.IsRollup == false);
count += session
.TimeSeriesFor(user, "Heartrate@By6Hours")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By1Day")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By30Minutes")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By1Hour")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
var results = session.Query <TimeSeriesIndex.Result, TimeSeriesIndex>()
.ToList();
Assert.True(count == results.Count, $"Test time = {now}");
}
}
}
public async Task CanIncludeTypedTimeSeries_Rollup()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await store.TimeSeries.RegisterAsync <User, StockPrice>();
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var nowMinutes = now.Minute;
now = now.AddMinutes(-nowMinutes);
database.Time.UtcDateTime = () => DateTime.UtcNow.AddMinutes(-nowMinutes);
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
var ts = session.TimeSeriesFor <StockPrice>("users/karmel");
var entry = new StockPrice();
for (int i = 0; i <= total; i++)
{
entry.Open = i;
entry.Close = i + 100_000;
entry.High = i + 200_000;
entry.Low = i + 300_000;
entry.Volume = i + 400_000;
ts.Append(baseline.AddMinutes(i), entry, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 12, rawName : "StockPrices");
using (var session = store.OpenSession())
{
var user = session.Query <User>()
.Include(x => x.IncludeTimeSeries($"StockPrices@{p1.Name}"))
.First();
// should not go to server
var ts = session.TimeSeriesRollupFor <StockPrice>(user.Id, p1.Name);
var res = ts.Get().ToList();
Assert.Equal(16, res.Count);
Assert.Equal(1, session.Advanced.NumberOfRequests);
}
}
}
public async Task RapidRetentionAndRollupInACluster()
{
var cluster = await CreateRaftCluster(3, watcherCluster : true);
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ReplicationFactor = 3
}))
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromSeconds(15));
var p1 = new TimeSeriesPolicy("By1", TimeValue.FromSeconds(1), raw.RetentionTime * 2);
var p2 = new TimeSeriesPolicy("By2", TimeValue.FromSeconds(2), raw.RetentionTime * 3);
var p3 = new TimeSeriesPolicy("By4", TimeValue.FromSeconds(4), raw.RetentionTime * 4);
var p4 = new TimeSeriesPolicy("By8", TimeValue.FromSeconds(8), raw.RetentionTime * 5);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var now = DateTime.UtcNow;
var baseline = now.AddSeconds(-15 * 3);
var total = ((TimeSpan)TimeValue.FromSeconds(15 * 3)).TotalMilliseconds;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMilliseconds(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
session.Store(new User {
Name = "Karmel"
}, "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>((DocumentSession)session, "marker", null, TimeSpan.FromSeconds(15)));
}
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await Task.Delay((TimeSpan)(p4.RetentionTime * 2));
// nothing should be left
foreach (var node in cluster.Nodes)
{
using (var nodeStore = GetDocumentStore(new Options
{
Server = node,
CreateDatabase = false,
DeleteDatabaseOnDispose = false,
ModifyDocumentStore = s => s.Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
},
ModifyDatabaseName = _ => store.Database
}))
{
using (var session = nodeStore.OpenSession())
{
var user = session.Load <User>("users/karmel");
Assert.Equal(0, session.Advanced.GetTimeSeriesFor(user)?.Count ?? 0);
var db = await node.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
await TimeSeriesReplicationTests.AssertNoLeftOvers(db);
}
}
}
}
}
public async Task CanWorkWithRollupTimeSeries2()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await store.TimeSeries.RegisterAsync <User, StockPrice>();
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
var ts = session.TimeSeriesFor <StockPrice>("users/karmel");
var entry = new StockPrice();
for (int i = 0; i <= total; i++)
{
entry.Open = i;
entry.Close = i + 100_000;
entry.High = i + 200_000;
entry.Low = i + 300_000;
entry.Volume = i + 400_000;
ts.Append(baseline.AddMinutes(i), entry, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 12, rawName: "StockPrices");
using (var session = store.OpenSession())
{
var query = session.Query <User>()
.Select(u =>
RavenQuery.TimeSeries <StockPrice>(u, "StockPrices").Select(x =>
new {
First = x.First(),
Last = x.Last(),
Min = x.Min(),
Max = x.Max(),
Sum = x.Sum(),
Count = x.Count(),
Avg = x.Average()
})
.ToList());
var result = query.Single();
Assert.Equal(1, result.Results.Length);
var r = result.Results[0];
Assert.NotNull(r.First);
Assert.NotNull(r.Last);
Assert.NotNull(r.Min);
Assert.NotNull(r.Max);
Assert.NotNull(r.Sum);
Assert.NotNull(r.Count);
Assert.NotNull(r.Average);
}
using (var session = store.OpenSession())
{
var query = session.Query <User>()
.Select(u =>
RavenQuery.TimeSeries <StockPrice>(u, "StockPrices")
.GroupBy(x => x.Hours(3))
.Select(x =>
new
{
First = x.First(),
Last = x.Last(),
Min = x.Min(),
Max = x.Max(),
Sum = x.Sum(),
Count = x.Count(),
Avg = x.Average()
})
.ToList());
var result = query.Single();
var r = result.Results[0];
Assert.NotNull(r.First);
Assert.NotNull(r.Last);
Assert.NotNull(r.Min);
Assert.NotNull(r.Max);
Assert.NotNull(r.Sum);
Assert.NotNull(r.Count);
Assert.NotNull(r.Average);
}
using (var session = store.OpenSession())
{
var ts1 = session.TimeSeriesRollupFor <StockPrice>("users/karmel", p1.Name);
var r = ts1.Get().First();
Assert.NotNull(r.First);
Assert.NotNull(r.Last);
Assert.NotNull(r.Min);
Assert.NotNull(r.Max);
Assert.NotNull(r.Sum);
Assert.NotNull(r.Count);
Assert.NotNull(r.Average);
}
}
}
public async Task CanWorkWithRollupTimeSeries()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await store.TimeSeries.RegisterAsync <User, StockPrice>();
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var nowMinutes = now.Minute;
now = now.AddMinutes(-nowMinutes);
database.Time.UtcDateTime = () => DateTime.UtcNow.AddMinutes(-nowMinutes);
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
var ts = session.TimeSeriesFor <StockPrice>("users/karmel");
var entry = new StockPrice();
for (int i = 0; i <= total; i++)
{
entry.Open = i;
entry.Close = i + 100_000;
entry.High = i + 200_000;
entry.Low = i + 300_000;
entry.Volume = i + 400_000;
ts.Append(baseline.AddMinutes(i), entry, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 12, rawName: "StockPrices");
using (var session = store.OpenSession())
{
var query = session.Advanced.RawQuery <TimeSeriesRawResult <StockPrice> >(@"
declare timeseries out()
{
from StockPrices
between $start and $end
}
from Users as u
select out()
")
.AddParameter("start", baseline.AddDays(-1))
.AddParameter("end", now.AddDays(1));
var result = query.Single();
var count = result.Results.Length;
Assert.Equal(5, result.Results[count - 1440].Values.Length);
foreach (var res in result.Results)
{
Assert.Equal(5, res.Values.Length);
}
}
using (var session = store.OpenSession())
{
// test the same query using linq
var query = session.Query <User>()
.Select(u =>
RavenQuery.TimeSeries <StockPrice>(u, "StockPrices", baseline.AddDays(-1), now.AddDays(1))
.ToList());
var result = query.Single();
var count = result.Results.Length;
Assert.Equal(5, result.Results[count - 1440].Values.Length);
foreach (var res in result.Results)
{
Assert.Equal(5, res.Values.Length);
}
}
now = DateTime.UtcNow;
using (var session = store.OpenSession())
{
var ts = session.TimeSeriesRollupFor <StockPrice>("users/karmel", p1.Name);
var a = new TimeSeriesRollupEntry <StockPrice>(DateTime.Now)
{
Max = new StockPrice
{
Close = 1
}
};
ts.Append(a);
session.SaveChanges();
}
using (var session = store.OpenSession())
{
var ts = session.TimeSeriesRollupFor <StockPrice>("users/karmel", p1.Name);
var res = ts.Get(from: now.AddMilliseconds(-1)).ToList();
Assert.Equal(1, res.Count);
Assert.Equal(1, res[0].Max.Close);
}
}
}
public async Task RapidRetention()
{
var cluster = await CreateRaftCluster(3);
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ReplicationFactor = 3
}))
{
var retention = TimeValue.FromSeconds(120);
var raw = new RawTimeSeriesPolicy(retention);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
}
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var now = DateTime.UtcNow;
var baseline = now.Add(-retention * 3);
var total = (int)((TimeSpan)retention).TotalMilliseconds * 3;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i < total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMilliseconds(i), new[] { 29d * i, i, i * 0.01, i * 0.1 }, "watches/fitbit");
}
session.SaveChanges();
session.Store(new User(), "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>(cluster.Nodes, store.Database, "marker", null, TimeSpan.FromSeconds(15)));
}
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var sp = Stopwatch.StartNew();
await Task.Delay((TimeSpan)retention / 2);
var debug = new Dictionary <string, (long Count, DateTime Start, DateTime End)>();
var check = true;
while (check)
{
Assert.True(sp.Elapsed < ((TimeSpan)retention).Add((TimeSpan)retention),
$"too long has passed {sp.Elapsed}, retention is {retention} {Environment.NewLine}" +
$"debug: {string.Join(',', debug.Select(kvp => $"{kvp.Key}: ({kvp.Value.Count},{kvp.Value.Start},{kvp.Value.End})"))}");
await Task.Delay(100);
check = false;
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var tss = database.DocumentsStorage.TimeSeriesStorage;
var stats = tss.Stats.GetStats(ctx, "users/karmel", "Heartrate");
var reader = tss.GetReader(ctx, "users/karmel", "Heartrate", DateTime.MinValue, DateTime.MaxValue);
if (stats.Count == 0)
{
debug.Remove(server.ServerStore.NodeTag);
continue;
}
check = true;
Assert.Equal(stats.Start, reader.First().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(stats.End, reader.Last().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
debug[server.ServerStore.NodeTag] = stats;
}
}
}
Assert.Empty(debug);
Assert.True(sp.Elapsed < (TimeSpan)retention + (TimeSpan)retention);
await Task.Delay(3000); // let the dust to settle
await EnsureNoReplicationLoop(Servers[0], store.Database);
await EnsureNoReplicationLoop(Servers[1], store.Database);
await EnsureNoReplicationLoop(Servers[2], store.Database);
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
await TimeSeriesReplicationTests.AssertNoLeftOvers(database);
}
}
}
public async Task ManyTimeSeriesRetention()
{
DefaultClusterSettings[RavenConfiguration.GetKey(x => x.Tombstones.CleanupInterval)] = 1.ToString();
var cluster = await CreateRaftCluster(3, watcherCluster : true);
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ReplicationFactor = 3,
RunInMemory = false
}))
{
var retention = TimeSpan.FromSeconds(60);
var raw = new RawTimeSeriesPolicy(retention);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
new TimeSeriesPolicy("ByMinute", TimeSpan.FromSeconds(60))
}
}
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
for (int j = 0; j < 1024; j++)
{
var now = DateTime.UtcNow;
var baseline = now.Add(-retention * 2);
var total = (int)retention.TotalSeconds * 2;
using (var session = store.OpenSession())
{
var id = "users/karmel/" + j;
session.Store(new User {
Name = "Karmel"
}, id);
for (int i = 0; i < total; i++)
{
session.TimeSeriesFor(id, "Heartrate")
.Append(baseline.AddSeconds(i), i);
}
session.SaveChanges();
}
}
using (var session = store.OpenSession())
{
session.Store(new User(), "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>(cluster.Nodes, store.Database, "marker", null, TimeSpan.FromSeconds(15)));
}
var sp = Stopwatch.StartNew();
await Task.Delay(retention / 2);
var check = true;
while (check)
{
Assert.True(sp.Elapsed < retention.Add(retention * 5), $"too long has passed {sp.Elapsed}, retention is {retention}");
await Task.Delay(200);
check = false;
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var tss = database.DocumentsStorage.TimeSeriesStorage;
for (int j = 0; j < 1024; j++)
{
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var id = $"users/karmel/{j}";
var stats = tss.Stats.GetStats(ctx, id, "Heartrate");
TimeSeriesReader reader;
if (stats == default || stats.Count == 0)
{
var name = config.Collections["Users"].Policies[0].GetTimeSeriesName("Heartrate");
stats = tss.Stats.GetStats(ctx, id, name);
reader = tss.GetReader(ctx, id, name, DateTime.MinValue, DateTime.MaxValue);
Assert.True(stats.Count > 0);
Assert.Equal(stats.Start, reader.First().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(stats.End, reader.Last().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
continue;
}
check = true;
reader = tss.GetReader(ctx, id, "Heartrate", DateTime.MinValue, DateTime.MaxValue);
Assert.Equal(stats.Start, reader.First().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(stats.End, reader.Last().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
}
}
}
}
await EnsureNoReplicationLoop(Servers[0], store.Database);
await EnsureNoReplicationLoop(Servers[1], store.Database);
await EnsureNoReplicationLoop(Servers[2], store.Database);
}
}
public void Examples()
{
var store = new DocumentStore
{
Urls = new[] { "http://*****:*****@DailyRollupForOneYear");
//Create local instance of the rollup time-series - second method:
//using the rollup policy itself and the raw time-series' name
var rollupTimeSeries2 = session.TimeSeriesFor("sales/1",
dailyRollup.GetTimeSeriesName("rawSales"));
//Retrieve all the data from both time-series
var rawData = rawTS.Get(DateTime.MinValue, DateTime.MaxValue).ToList();
var rollupData = dailyRollupTS.Get(DateTime.MinValue, DateTime.MaxValue).ToList();
#endregion
}
}
public async Task SingleResultTest()
{
DefaultClusterSettings[RavenConfiguration.GetKey(x => x.Tombstones.CleanupInterval)] = 1.ToString();
var cluster = await CreateRaftCluster(3, watcherCluster : true);
using (var store = GetDocumentStore(new Options {
Server = cluster.Leader, ReplicationFactor = 3, RunInMemory = false
}))
{
var raw = new RawTimeSeriesPolicy();
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy> {
new TimeSeriesPolicy("ByMinute", TimeSpan.FromSeconds(60))
}
}
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var now = DateTime.UtcNow;
var baseline = now.AddSeconds(-120);
using (var session = store.OpenSession())
{
var id = "users/karmel/0";
session.Store(new User {
Name = "Karmel"
}, id);
for (int i = 0; i < 120; i++)
{
session.TimeSeriesFor(id, "Heartrate")
.Append(baseline.AddSeconds(i), i);
}
session.SaveChanges();
}
using (var session = store.OpenSession())
{
session.Store(new User(), "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>(cluster.Nodes, store.Database, "marker", null, TimeSpan.FromSeconds(15)));
}
var database = await Servers[0].ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
{
using (var tx = ctx.OpenReadTransaction())
{
var name = config.Collections["Users"].Policies[0].GetTimeSeriesName("Heartrate");
var reader = database.DocumentsStorage.TimeSeriesStorage.GetReader(ctx, "users/karmel/0", "Heartrate", baseline.AddSeconds(120), DateTime.MaxValue);
Assert.True(reader.Last() == null);
}
}
}
}
public async Task MarkPolicyAfterRollup()
{
DefaultClusterSettings[RavenConfiguration.GetKey(x => x.Tombstones.CleanupInterval)] = 1.ToString();
var cluster = await CreateRaftCluster(3, watcherCluster : true);
using (var store = GetDocumentStore(new Options {
Server = cluster.Leader, ReplicationFactor = 3, RunInMemory = false
}))
{
var raw = new RawTimeSeriesPolicy();
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy> {
new TimeSeriesPolicy("ByMinute", TimeSpan.FromMinutes(10))
}
}
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var firstNode = record.Topology.Members[0];
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var now = new DateTime(2021, 6, 1, 10, 7, 29, DateTimeKind.Utc);
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
database.Time.UtcDateTime = () => now;
}
var baseline = now.Add(-TimeSpan.FromMinutes(15));
using (var session = store.OpenSession())
{
var id = "users/karmel/0";
session.Store(new User {
Name = "Karmel"
}, id);
for (int i = 0; i < 15; i++)
{
session.TimeSeriesFor(id, "Heartrate")
.Append(baseline.AddMinutes(i), i);
}
session.SaveChanges();
}
using (var session = store.OpenSession())
{
session.Store(new User(), "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>(cluster.Nodes, store.Database, "marker", null, TimeSpan.FromSeconds(15)));
}
var res = new Dictionary <string, int>();
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var tss = database.DocumentsStorage.TimeSeriesStorage;
await database.TimeSeriesPolicyRunner.RunRollups();
var name = config.Collections["Users"].Policies[0].GetTimeSeriesName("Heartrate");
WaitForValue(() =>
{
using (var session = store.OpenSession())
{
var val = session.TimeSeriesFor("users/karmel/0", name)
.Get(DateTime.MinValue, DateTime.MaxValue);
return(val != null);
}
}, true);
using (var session = store.OpenSession())
{
var val = session.TimeSeriesFor("users/karmel/0", name)
.Get(DateTime.MinValue, DateTime.MaxValue).Length;
res.Add(server.ServerStore.NodeTag, val);
Assert.True(val > 0);
}
}
record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var firstNode2 = record.Topology.Members[0];
Assert.Equal(firstNode2, firstNode);
record = store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)).Result;
var list = record.Topology.Members;
list.Reverse();
await store.Maintenance.Server.SendAsync(new ReorderDatabaseMembersOperation(store.Database, list));
record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
firstNode2 = record.Topology.Members[0];
Assert.NotEqual(firstNode2, firstNode);
await Task.Delay(1000);
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
database.Time.UtcDateTime = () => now.AddMinutes(10);
await database.TimeSeriesPolicyRunner.RunRollups();
}
foreach (var server in Servers)
{
WaitForValue(() =>
{
using (var session = store.OpenSession())
{
var name = config.Collections["Users"].Policies[0].GetTimeSeriesName("Heartrate");
var val = session.TimeSeriesFor("users/karmel/0", name)
.Get(DateTime.MinValue, DateTime.MaxValue);
return(val.Length > res[server.ServerStore.NodeTag]);
}
}, true);
}
using (var session = store.OpenSession())
{
var name = config.Collections["Users"].Policies[0].GetTimeSeriesName("Heartrate");
var val = session.TimeSeriesFor("users/karmel/0", name)
.Get(DateTime.MinValue, DateTime.MaxValue);
Assert.True(val.Length > res[Servers[0].ServerStore.NodeTag]);
}
record = store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)).Result;
firstNode2 = record.Topology.Members[0];
Assert.NotEqual(firstNode2, firstNode);
}
}
/// <summary>
/// Set raw retention policy
/// </summary>
/// <param name="retention">Retention time</param>
/// <returns></returns>
public Task SetRawPolicyAsync(string collection, TimeValue retention)
{
var p = new RawTimeSeriesPolicy(retention);
return(_executor.SendAsync(new ConfigureRawTimeSeriesPolicyOperation(collection, p)));
}
