public async Task RapidRetentionAndRollup()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromSeconds(15));
var p1 = new TimeSeriesPolicy("By1", TimeValue.FromSeconds(1), raw.RetentionTime * 2);
var p2 = new TimeSeriesPolicy("By2", TimeValue.FromSeconds(2), raw.RetentionTime * 3);
var p3 = new TimeSeriesPolicy("By4", TimeValue.FromSeconds(4), raw.RetentionTime * 4);
var p4 = new TimeSeriesPolicy("By8", TimeValue.FromSeconds(8), raw.RetentionTime * 5);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var now = DateTime.UtcNow;
var baseline = now.AddSeconds(-15 * 3);
var total = ((TimeSpan)TimeValue.FromSeconds(15 * 3)).TotalMilliseconds;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMilliseconds(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
}
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
WaitForUserToContinueTheTest(store);
await Task.Delay((TimeSpan)(p4.RetentionTime + TimeValue.FromSeconds(10)));
// nothing should be left
using (var session = store.OpenSession())
{
var user = session.Load <User>("users/karmel");
Assert.Equal(0, session.Advanced.GetTimeSeriesFor(user)?.Count ?? 0);
}
}
}
public RollupTimeSeriesCommandDto(TimeSeriesConfiguration configuration, DateTime now, List <RollupState> states, bool isFirstInTopology)
{
_configuration = configuration;
_now = now;
_states = states;
_isFirstInTopology = isFirstInTopology;
}
internal RollupTimeSeriesCommand(TimeSeriesConfiguration configuration, DateTime now, List <RollupState> states, bool isFirstInTopology)
{
_configuration = configuration;
_now = now;
_states = states;
_isFirstInTopology = isFirstInTopology;
_logger = LoggingSource.Instance.GetLogger <TimeSeriesRollups>(nameof(RollupTimeSeriesCommand));
}
public async Task TimeSeriesLinqQuery_CanUseSimpleCallExpressionInName()
{
using (var store = GetDocumentStore())
{
var timeSeries = "HeartRate";
var retention = TimeValue.FromHours(48);
var p = new TimeSeriesPolicy("ByHour", TimeValue.FromHours(1), TimeValue.FromHours(24));
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = new RawTimeSeriesPolicy(TimeValue.FromHours(96)),
Policies = new List <TimeSeriesPolicy> {
p
}
}
}
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var baseline = DateTime.UtcNow.Add(-retention * 2);
using (var session = store.OpenSession())
{
session.Store(new User(), DocId);
var timeSeriesFor = session.TimeSeriesFor(DocId, timeSeries);
for (int i = 0; i < 100; i++)
{
timeSeriesFor.Append(baseline.AddHours(i), 29 * i, "watches/fitbit");
}
session.SaveChanges();
}
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
await TimeSeries.WaitForPolicyRunnerAsync(database);
using (var session = store.OpenSession())
{
// todo aviv : remove the single quotes from name when RavenDB-15792 is fixed
var q = session.Query <User>()
.Where(u => u.Id == DocId)
.Select(u => RavenQuery.TimeSeries(u, $"'{p.GetTimeSeriesName(timeSeries)}'")
.ToList());
var result = q.First();
Assert.Equal(24, result.Count);
}
}
}
private void WriteTimeSeries(TimeSeriesConfiguration timeSeries)
{
if (timeSeries == null)
{
_writer.WriteNull();
return;
}
_context.Write(_writer, timeSeries.ToJson());
}
public async Task ValidateCorrectRetentionAndGet(int minutesOffset)
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var database = await GetDocumentDatabaseInstanceFor(store);
var now = new DateTime(2020, 4, 2).AddMinutes(minutesOffset);
database.Time.UtcDateTime = () => now.AddMilliseconds(1);
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new Core.Utils.Entities.User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMinutes(i), i, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await QueryFromMultipleTimeSeries.VerifyFullPolicyExecution(store, config.Collections["Users"]);
}
}
public TimeSeriesPolicyRunner(DocumentDatabase database, TimeSeriesConfiguration configuration) : base(database.Name, database.DatabaseShutdown)
{
_database = database;
Configuration = configuration;
if (configuration.Collections != null)
Configuration.Collections =
new Dictionary<string, TimeSeriesCollectionConfiguration>(Configuration.Collections, StringComparer.OrdinalIgnoreCase);
Configuration.InitializeRollupAndRetention();
_checkFrequency = Configuration.PolicyCheckFrequency ?? TimeSeriesConfiguration.DefaultPolicyCheckFrequency;
}
public StronglyTypedRavenSettings(NameValueCollection settings)
{
Replication = new ReplicationConfiguration();
Voron = new VoronConfiguration();
Esent = new EsentConfiguration();
Prefetcher = new PrefetcherConfiguration();
FileSystem = new FileSystemConfiguration();
Counter = new CounterConfiguration();
TimeSeries = new TimeSeriesConfiguration();
Encryption = new EncryptionConfiguration();
Indexing = new IndexingConfiguration();
WebSockets = new WebSocketsConfiguration();
Cluster = new ClusterConfiguration();
Monitoring = new MonitoringConfiguration();
Studio = new StudioConfiguration();
this.settings = settings;
}
public RavenConfiguration()
{
Settings = new NameValueCollection(StringComparer.OrdinalIgnoreCase);
Core = new CoreConfiguration(this);
FileSystem = new FileSystemConfiguration(Core);
Counter = new CounterConfiguration(Core);
TimeSeries = new TimeSeriesConfiguration(Core);
Replication = new ReplicationConfiguration();
Prefetcher = new PrefetcherConfiguration();
Storage = new StorageConfiguration();
Encryption = new EncryptionConfiguration();
Indexing = new IndexingConfiguration();
WebSockets = new WebSocketsConfiguration();
Cluster = new ClusterConfiguration();
Monitoring = new MonitoringConfiguration();
Queries = new QueryConfiguration();
Patching = new PatchingConfiguration();
BulkInsert = new BulkInsertConfiguration();
Server = new ServerConfiguration();
Memory = new MemoryConfiguration();
Expiration = new ExpirationBundleConfiguration();
Versioning = new VersioningBundleConfiguration();
Studio = new StudioConfiguration();
Tenants = new TenantConfiguration();
Licensing = new LicenseConfiguration();
Quotas = new QuotasBundleConfiguration();
IndexingClassifier = new DefaultIndexingClassifier();
Catalog = new AggregateCatalog(CurrentAssemblyCatalog);
Catalog.Changed += (sender, args) => ResetContainer();
}
public async Task CanIndexRollups()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeValue.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeValue.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeValue.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeValue.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var nowSeconds = now.Second;
now = now.AddSeconds(-nowSeconds);
database.Time.UtcDateTime = () => DateTime.UtcNow.AddSeconds(-nowSeconds);
var baseline = now.AddDays(-12);
var total = ((TimeSpan)TimeValue.FromDays(12)).TotalMinutes;
await new TimeSeriesIndex().ExecuteAsync(store);
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMinutes(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
}
await TimeSeries.WaitForPolicyRunnerAsync(database);
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 4, policies : new List <TimeSeriesPolicy> {
p1
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 5, policies : new List <TimeSeriesPolicy> {
p2
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 2, policies : new List <TimeSeriesPolicy> {
p3
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 3, policies : new List <TimeSeriesPolicy> {
p4
});
Indexes.WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
using (var session = store.OpenSession())
{
var user = session.Load <User>("users/karmel");
var count = session
.TimeSeriesFor(user, "Heartrate")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count(entry => entry.IsRollup == false);
count += session
.TimeSeriesFor(user, "Heartrate@By6Hours")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By1Day")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By30Minutes")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By1Hour")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
var results = session.Query <TimeSeriesIndex.Result, TimeSeriesIndex>()
.ToList();
Assert.True(count == results.Count, $"Test time = {now}");
}
}
}
public async Task CanIncludeTypedTimeSeries_Rollup()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await store.TimeSeries.RegisterAsync <User, StockPrice>();
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var nowMinutes = now.Minute;
now = now.AddMinutes(-nowMinutes);
database.Time.UtcDateTime = () => DateTime.UtcNow.AddMinutes(-nowMinutes);
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
var ts = session.TimeSeriesFor <StockPrice>("users/karmel");
var entry = new StockPrice();
for (int i = 0; i <= total; i++)
{
entry.Open = i;
entry.Close = i + 100_000;
entry.High = i + 200_000;
entry.Low = i + 300_000;
entry.Volume = i + 400_000;
ts.Append(baseline.AddMinutes(i), entry, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 12, rawName : "StockPrices");
using (var session = store.OpenSession())
{
var user = session.Query <User>()
.Include(x => x.IncludeTimeSeries($"StockPrices@{p1.Name}"))
.First();
// should not go to server
var ts = session.TimeSeriesRollupFor <StockPrice>(user.Id, p1.Name);
var res = ts.Get().ToList();
Assert.Equal(16, res.Count);
Assert.Equal(1, session.Advanced.NumberOfRequests);
}
}
}
public async Task MarkPolicyAfterRollup()
{
DefaultClusterSettings[RavenConfiguration.GetKey(x => x.Tombstones.CleanupInterval)] = 1.ToString();
var cluster = await CreateRaftCluster(3, watcherCluster : true);
using (var store = GetDocumentStore(new Options {
Server = cluster.Leader, ReplicationFactor = 3, RunInMemory = false
}))
{
var raw = new RawTimeSeriesPolicy();
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy> {
new TimeSeriesPolicy("ByMinute", TimeSpan.FromMinutes(10))
}
}
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var firstNode = record.Topology.Members[0];
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var now = new DateTime(2021, 6, 1, 10, 7, 29, DateTimeKind.Utc);
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
database.Time.UtcDateTime = () => now;
}
var baseline = now.Add(-TimeSpan.FromMinutes(15));
using (var session = store.OpenSession())
{
var id = "users/karmel/0";
session.Store(new User {
Name = "Karmel"
}, id);
for (int i = 0; i < 15; i++)
{
session.TimeSeriesFor(id, "Heartrate")
.Append(baseline.AddMinutes(i), i);
}
session.SaveChanges();
}
using (var session = store.OpenSession())
{
session.Store(new User(), "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>(cluster.Nodes, store.Database, "marker", null, TimeSpan.FromSeconds(15)));
}
var res = new Dictionary <string, int>();
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var tss = database.DocumentsStorage.TimeSeriesStorage;
await database.TimeSeriesPolicyRunner.RunRollups();
var name = config.Collections["Users"].Policies[0].GetTimeSeriesName("Heartrate");
WaitForValue(() =>
{
using (var session = store.OpenSession())
{
var val = session.TimeSeriesFor("users/karmel/0", name)
.Get(DateTime.MinValue, DateTime.MaxValue);
return(val != null);
}
}, true);
using (var session = store.OpenSession())
{
var val = session.TimeSeriesFor("users/karmel/0", name)
.Get(DateTime.MinValue, DateTime.MaxValue).Length;
res.Add(server.ServerStore.NodeTag, val);
Assert.True(val > 0);
}
}
record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
var firstNode2 = record.Topology.Members[0];
Assert.Equal(firstNode2, firstNode);
record = store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)).Result;
var list = record.Topology.Members;
list.Reverse();
await store.Maintenance.Server.SendAsync(new ReorderDatabaseMembersOperation(store.Database, list));
record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database));
firstNode2 = record.Topology.Members[0];
Assert.NotEqual(firstNode2, firstNode);
await Task.Delay(1000);
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
database.Time.UtcDateTime = () => now.AddMinutes(10);
await database.TimeSeriesPolicyRunner.RunRollups();
}
foreach (var server in Servers)
{
WaitForValue(() =>
{
using (var session = store.OpenSession())
{
var name = config.Collections["Users"].Policies[0].GetTimeSeriesName("Heartrate");
var val = session.TimeSeriesFor("users/karmel/0", name)
.Get(DateTime.MinValue, DateTime.MaxValue);
return(val.Length > res[server.ServerStore.NodeTag]);
}
}, true);
}
using (var session = store.OpenSession())
{
var name = config.Collections["Users"].Policies[0].GetTimeSeriesName("Heartrate");
var val = session.TimeSeriesFor("users/karmel/0", name)
.Get(DateTime.MinValue, DateTime.MaxValue);
Assert.True(val.Length > res[Servers[0].ServerStore.NodeTag]);
}
record = store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)).Result;
firstNode2 = record.Topology.Members[0];
Assert.NotEqual(firstNode2, firstNode);
}
}
public async Task RapidRetentionAndRollupInACluster()
{
var cluster = await CreateRaftCluster(3, watcherCluster : true);
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ReplicationFactor = 3
}))
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromSeconds(15));
var p1 = new TimeSeriesPolicy("By1", TimeValue.FromSeconds(1), raw.RetentionTime * 2);
var p2 = new TimeSeriesPolicy("By2", TimeValue.FromSeconds(2), raw.RetentionTime * 3);
var p3 = new TimeSeriesPolicy("By4", TimeValue.FromSeconds(4), raw.RetentionTime * 4);
var p4 = new TimeSeriesPolicy("By8", TimeValue.FromSeconds(8), raw.RetentionTime * 5);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var now = DateTime.UtcNow;
var baseline = now.AddSeconds(-15 * 3);
var total = ((TimeSpan)TimeValue.FromSeconds(15 * 3)).TotalMilliseconds;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMilliseconds(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
session.Store(new User {
Name = "Karmel"
}, "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>((DocumentSession)session, "marker", null, TimeSpan.FromSeconds(15)));
}
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await Task.Delay((TimeSpan)(p4.RetentionTime * 2));
// nothing should be left
foreach (var node in cluster.Nodes)
{
using (var nodeStore = GetDocumentStore(new Options
{
Server = node,
CreateDatabase = false,
DeleteDatabaseOnDispose = false,
ModifyDocumentStore = s => s.Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
},
ModifyDatabaseName = _ => store.Database
}))
{
using (var session = nodeStore.OpenSession())
{
var user = session.Load <User>("users/karmel");
Assert.Equal(0, session.Advanced.GetTimeSeriesFor(user)?.Count ?? 0);
var db = await node.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
await TimeSeriesReplicationTests.AssertNoLeftOvers(db);
}
}
}
}
}
public async Task SingleResultTest()
{
DefaultClusterSettings[RavenConfiguration.GetKey(x => x.Tombstones.CleanupInterval)] = 1.ToString();
var cluster = await CreateRaftCluster(3, watcherCluster : true);
using (var store = GetDocumentStore(new Options {
Server = cluster.Leader, ReplicationFactor = 3, RunInMemory = false
}))
{
var raw = new RawTimeSeriesPolicy();
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy> {
new TimeSeriesPolicy("ByMinute", TimeSpan.FromSeconds(60))
}
}
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var now = DateTime.UtcNow;
var baseline = now.AddSeconds(-120);
using (var session = store.OpenSession())
{
var id = "users/karmel/0";
session.Store(new User {
Name = "Karmel"
}, id);
for (int i = 0; i < 120; i++)
{
session.TimeSeriesFor(id, "Heartrate")
.Append(baseline.AddSeconds(i), i);
}
session.SaveChanges();
}
using (var session = store.OpenSession())
{
session.Store(new User(), "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>(cluster.Nodes, store.Database, "marker", null, TimeSpan.FromSeconds(15)));
}
var database = await Servers[0].ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
{
using (var tx = ctx.OpenReadTransaction())
{
var name = config.Collections["Users"].Policies[0].GetTimeSeriesName("Heartrate");
var reader = database.DocumentsStorage.TimeSeriesStorage.GetReader(ctx, "users/karmel/0", "Heartrate", baseline.AddSeconds(120), DateTime.MaxValue);
Assert.True(reader.Last() == null);
}
}
}
}
public async Task PostCreateSampleData()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
using (context.OpenReadTransaction())
{
foreach (var collection in Database.DocumentsStorage.GetCollections(context))
{
if (collection.Count > 0)
{
throw new InvalidOperationException("You cannot create sample data in a database that already contains documents");
}
}
}
var operateOnTypesAsString = GetStringValuesQueryString("operateOnTypes", required: false);
var operateOnTypes = GetOperateOnTypes(operateOnTypesAsString);
if (operateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments))
{
var editRevisions = new EditRevisionsConfigurationCommand(new RevisionsConfiguration
{
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Orders"] = new RevisionsCollectionConfiguration
{
Disabled = false
}
}
}, Database.Name, GetRaftRequestIdFromQuery() + "/revisions");
var(index, _) = await ServerStore.SendToLeaderAsync(editRevisions);
await Database.RachisLogIndexNotifications.WaitForIndexNotification(index, Database.ServerStore.Engine.OperationTimeout);
}
if (operateOnTypes.HasFlag(DatabaseItemType.TimeSeries))
{
var tsConfig = new TimeSeriesConfiguration
{
NamedValues = new Dictionary <string, Dictionary <string, string[]> >
{
["Companies"] = new Dictionary <string, string[]>
{
["StockPrices"] = new[] { "Open", "Close", "High", "Low", "Volume" }
},
["Employees"] = new Dictionary <string, string[]>
{
["HeartRates"] = new[] { "BPM" }
}
}
};
var editTimeSeries = new EditTimeSeriesConfigurationCommand(tsConfig, Database.Name, GetRaftRequestIdFromQuery() + "/time-series");
var(index, _) = await ServerStore.SendToLeaderAsync(editTimeSeries);
await Database.RachisLogIndexNotifications.WaitForIndexNotification(index, Database.ServerStore.Engine.OperationTimeout);
}
await using (var sampleData = typeof(SampleDataHandler).Assembly
.GetManifestResourceStream("Raven.Server.Web.Studio.EmbeddedData.Northwind.ravendbdump"))
{
await using (var stream = new GZipStream(sampleData, CompressionMode.Decompress))
using (var source = new StreamSource(stream, context, Database))
{
var destination = new DatabaseDestination(Database);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time,
options: new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = operateOnTypes,
SkipRevisionCreation = true
});
await smuggler.ExecuteAsync();
}
}
await NoContent();
}
public void Examples()
{
var store = new DocumentStore
{
Urls = new[] { "http://*****:*****@DailyRollupForOneYear");
//Create local instance of the rollup time-series - second method:
//using the rollup policy itself and the raw time-series' name
var rollupTimeSeries2 = session.TimeSeriesFor("sales/1",
dailyRollup.GetTimeSeriesName("rawSales"));
//Retrieve all the data from both time-series
var rawData = rawTS.Get(DateTime.MinValue, DateTime.MaxValue).ToList();
var rollupData = dailyRollupTS.Get(DateTime.MinValue, DateTime.MaxValue).ToList();
#endregion
}
}
public async Task CanWorkWithRollupTimeSeries2()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await store.TimeSeries.RegisterAsync <User, StockPrice>();
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
var ts = session.TimeSeriesFor <StockPrice>("users/karmel");
var entry = new StockPrice();
for (int i = 0; i <= total; i++)
{
entry.Open = i;
entry.Close = i + 100_000;
entry.High = i + 200_000;
entry.Low = i + 300_000;
entry.Volume = i + 400_000;
ts.Append(baseline.AddMinutes(i), entry, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 12, rawName: "StockPrices");
using (var session = store.OpenSession())
{
var query = session.Query <User>()
.Select(u =>
RavenQuery.TimeSeries <StockPrice>(u, "StockPrices").Select(x =>
new {
First = x.First(),
Last = x.Last(),
Min = x.Min(),
Max = x.Max(),
Sum = x.Sum(),
Count = x.Count(),
Avg = x.Average()
})
.ToList());
var result = query.Single();
Assert.Equal(1, result.Results.Length);
var r = result.Results[0];
Assert.NotNull(r.First);
Assert.NotNull(r.Last);
Assert.NotNull(r.Min);
Assert.NotNull(r.Max);
Assert.NotNull(r.Sum);
Assert.NotNull(r.Count);
Assert.NotNull(r.Average);
}
using (var session = store.OpenSession())
{
var query = session.Query <User>()
.Select(u =>
RavenQuery.TimeSeries <StockPrice>(u, "StockPrices")
.GroupBy(x => x.Hours(3))
.Select(x =>
new
{
First = x.First(),
Last = x.Last(),
Min = x.Min(),
Max = x.Max(),
Sum = x.Sum(),
Count = x.Count(),
Avg = x.Average()
})
.ToList());
var result = query.Single();
var r = result.Results[0];
Assert.NotNull(r.First);
Assert.NotNull(r.Last);
Assert.NotNull(r.Min);
Assert.NotNull(r.Max);
Assert.NotNull(r.Sum);
Assert.NotNull(r.Count);
Assert.NotNull(r.Average);
}
using (var session = store.OpenSession())
{
var ts1 = session.TimeSeriesRollupFor <StockPrice>("users/karmel", p1.Name);
var r = ts1.Get().First();
Assert.NotNull(r.First);
Assert.NotNull(r.Last);
Assert.NotNull(r.Min);
Assert.NotNull(r.Max);
Assert.NotNull(r.Sum);
Assert.NotNull(r.Count);
Assert.NotNull(r.Average);
}
}
}
public async Task CanWorkWithRollupTimeSeries()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeSpan.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeSpan.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeSpan.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeSpan.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeSpan.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await store.TimeSeries.RegisterAsync <User, StockPrice>();
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var nowMinutes = now.Minute;
now = now.AddMinutes(-nowMinutes);
database.Time.UtcDateTime = () => DateTime.UtcNow.AddMinutes(-nowMinutes);
var baseline = now.AddDays(-12);
var total = TimeSpan.FromDays(12).TotalMinutes;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
var ts = session.TimeSeriesFor <StockPrice>("users/karmel");
var entry = new StockPrice();
for (int i = 0; i <= total; i++)
{
entry.Open = i;
entry.Close = i + 100_000;
entry.High = i + 200_000;
entry.Low = i + 300_000;
entry.Volume = i + 400_000;
ts.Append(baseline.AddMinutes(i), entry, "watches/fitbit");
}
session.SaveChanges();
}
await database.TimeSeriesPolicyRunner.RunRollups();
await database.TimeSeriesPolicyRunner.DoRetention();
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 12, rawName: "StockPrices");
using (var session = store.OpenSession())
{
var query = session.Advanced.RawQuery <TimeSeriesRawResult <StockPrice> >(@"
declare timeseries out()
{
from StockPrices
between $start and $end
}
from Users as u
select out()
")
.AddParameter("start", baseline.AddDays(-1))
.AddParameter("end", now.AddDays(1));
var result = query.Single();
var count = result.Results.Length;
Assert.Equal(5, result.Results[count - 1440].Values.Length);
foreach (var res in result.Results)
{
Assert.Equal(5, res.Values.Length);
}
}
using (var session = store.OpenSession())
{
// test the same query using linq
var query = session.Query <User>()
.Select(u =>
RavenQuery.TimeSeries <StockPrice>(u, "StockPrices", baseline.AddDays(-1), now.AddDays(1))
.ToList());
var result = query.Single();
var count = result.Results.Length;
Assert.Equal(5, result.Results[count - 1440].Values.Length);
foreach (var res in result.Results)
{
Assert.Equal(5, res.Values.Length);
}
}
now = DateTime.UtcNow;
using (var session = store.OpenSession())
{
var ts = session.TimeSeriesRollupFor <StockPrice>("users/karmel", p1.Name);
var a = new TimeSeriesRollupEntry <StockPrice>(DateTime.Now)
{
Max = new StockPrice
{
Close = 1
}
};
ts.Append(a);
session.SaveChanges();
}
using (var session = store.OpenSession())
{
var ts = session.TimeSeriesRollupFor <StockPrice>("users/karmel", p1.Name);
var res = ts.Get(from: now.AddMilliseconds(-1)).ToList();
Assert.Equal(1, res.Count);
Assert.Equal(1, res[0].Max.Close);
}
}
}
public EditTimeSeriesConfigurationCommand(TimeSeriesConfiguration configuration, string databaseName, string uniqueRequestId) : base(databaseName, uniqueRequestId)
{
Configuration = configuration;
}
public async Task ManyTimeSeriesRetention()
{
DefaultClusterSettings[RavenConfiguration.GetKey(x => x.Tombstones.CleanupInterval)] = 1.ToString();
var cluster = await CreateRaftCluster(3, watcherCluster : true);
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ReplicationFactor = 3,
RunInMemory = false
}))
{
var retention = TimeSpan.FromSeconds(60);
var raw = new RawTimeSeriesPolicy(retention);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
new TimeSeriesPolicy("ByMinute", TimeSpan.FromSeconds(60))
}
}
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
for (int j = 0; j < 1024; j++)
{
var now = DateTime.UtcNow;
var baseline = now.Add(-retention * 2);
var total = (int)retention.TotalSeconds * 2;
using (var session = store.OpenSession())
{
var id = "users/karmel/" + j;
session.Store(new User {
Name = "Karmel"
}, id);
for (int i = 0; i < total; i++)
{
session.TimeSeriesFor(id, "Heartrate")
.Append(baseline.AddSeconds(i), i);
}
session.SaveChanges();
}
}
using (var session = store.OpenSession())
{
session.Store(new User(), "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>(cluster.Nodes, store.Database, "marker", null, TimeSpan.FromSeconds(15)));
}
var sp = Stopwatch.StartNew();
await Task.Delay(retention / 2);
var check = true;
while (check)
{
Assert.True(sp.Elapsed < retention.Add(retention * 5), $"too long has passed {sp.Elapsed}, retention is {retention}");
await Task.Delay(200);
check = false;
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var tss = database.DocumentsStorage.TimeSeriesStorage;
for (int j = 0; j < 1024; j++)
{
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var id = $"users/karmel/{j}";
var stats = tss.Stats.GetStats(ctx, id, "Heartrate");
TimeSeriesReader reader;
if (stats == default || stats.Count == 0)
{
var name = config.Collections["Users"].Policies[0].GetTimeSeriesName("Heartrate");
stats = tss.Stats.GetStats(ctx, id, name);
reader = tss.GetReader(ctx, id, name, DateTime.MinValue, DateTime.MaxValue);
Assert.True(stats.Count > 0);
Assert.Equal(stats.Start, reader.First().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(stats.End, reader.Last().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
continue;
}
check = true;
reader = tss.GetReader(ctx, id, "Heartrate", DateTime.MinValue, DateTime.MaxValue);
Assert.Equal(stats.Start, reader.First().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(stats.End, reader.Last().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
}
}
}
}
await EnsureNoReplicationLoop(Servers[0], store.Database);
await EnsureNoReplicationLoop(Servers[1], store.Database);
await EnsureNoReplicationLoop(Servers[2], store.Database);
}
}
public async Task RapidRetention()
{
var cluster = await CreateRaftCluster(3);
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ReplicationFactor = 3
}))
{
var retention = TimeValue.FromSeconds(120);
var raw = new RawTimeSeriesPolicy(retention);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
}
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var now = DateTime.UtcNow;
var baseline = now.Add(-retention * 3);
var total = (int)((TimeSpan)retention).TotalMilliseconds * 3;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i < total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMilliseconds(i), new[] { 29d * i, i, i * 0.01, i * 0.1 }, "watches/fitbit");
}
session.SaveChanges();
session.Store(new User(), "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>(cluster.Nodes, store.Database, "marker", null, TimeSpan.FromSeconds(15)));
}
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var sp = Stopwatch.StartNew();
await Task.Delay((TimeSpan)retention / 2);
var debug = new Dictionary <string, (long Count, DateTime Start, DateTime End)>();
var check = true;
while (check)
{
Assert.True(sp.Elapsed < ((TimeSpan)retention).Add((TimeSpan)retention),
$"too long has passed {sp.Elapsed}, retention is {retention} {Environment.NewLine}" +
$"debug: {string.Join(',', debug.Select(kvp => $"{kvp.Key}: ({kvp.Value.Count},{kvp.Value.Start},{kvp.Value.End})"))}");
await Task.Delay(100);
check = false;
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var tss = database.DocumentsStorage.TimeSeriesStorage;
var stats = tss.Stats.GetStats(ctx, "users/karmel", "Heartrate");
var reader = tss.GetReader(ctx, "users/karmel", "Heartrate", DateTime.MinValue, DateTime.MaxValue);
if (stats.Count == 0)
{
debug.Remove(server.ServerStore.NodeTag);
continue;
}
check = true;
Assert.Equal(stats.Start, reader.First().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(stats.End, reader.Last().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
debug[server.ServerStore.NodeTag] = stats;
}
}
}
Assert.Empty(debug);
Assert.True(sp.Elapsed < (TimeSpan)retention + (TimeSpan)retention);
await Task.Delay(3000); // let the dust to settle
await EnsureNoReplicationLoop(Servers[0], store.Database);
await EnsureNoReplicationLoop(Servers[1], store.Database);
await EnsureNoReplicationLoop(Servers[2], store.Database);
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
await TimeSeriesReplicationTests.AssertNoLeftOvers(database);
}
}
}
