public async Task RapidRetentionAndRollup()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromSeconds(15));
var p1 = new TimeSeriesPolicy("By1", TimeValue.FromSeconds(1), raw.RetentionTime * 2);
var p2 = new TimeSeriesPolicy("By2", TimeValue.FromSeconds(2), raw.RetentionTime * 3);
var p3 = new TimeSeriesPolicy("By4", TimeValue.FromSeconds(4), raw.RetentionTime * 4);
var p4 = new TimeSeriesPolicy("By8", TimeValue.FromSeconds(8), raw.RetentionTime * 5);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var now = DateTime.UtcNow;
var baseline = now.AddSeconds(-15 * 3);
var total = ((TimeSpan)TimeValue.FromSeconds(15 * 3)).TotalMilliseconds;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMilliseconds(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
}
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
WaitForUserToContinueTheTest(store);
await Task.Delay((TimeSpan)(p4.RetentionTime + TimeValue.FromSeconds(10)));
// nothing should be left
using (var session = store.OpenSession())
{
var user = session.Load <User>("users/karmel");
Assert.Equal(0, session.Advanced.GetTimeSeriesFor(user)?.Count ?? 0);
}
}
}
private void GetTimeSeriesQueryString(DocumentDatabase database, DocumentsOperationContext context, out IncludeTimeSeriesCommand includeTimeSeries)
{
includeTimeSeries = null;
var timeSeriesNames = GetStringValuesQueryString("timeseries", required: false);
var timeSeriesTimeNames = GetStringValuesQueryString("timeseriestime", required: false);
var timeSeriesCountNames = GetStringValuesQueryString("timeseriescount", required: false);
if (timeSeriesNames.Count == 0 && timeSeriesTimeNames.Count == 0 && timeSeriesCountNames.Count == 0)
{
return;
}
if (timeSeriesNames.Count > 1 && timeSeriesNames.Contains(Constants.TimeSeries.All))
{
throw new InvalidOperationException($"Cannot have more than one include on '{Constants.TimeSeries.All}'.");
}
if (timeSeriesTimeNames.Count > 1 && timeSeriesTimeNames.Contains(Constants.TimeSeries.All))
{
throw new InvalidOperationException($"Cannot have more than one include on '{Constants.TimeSeries.All}'.");
}
if (timeSeriesCountNames.Count > 1 && timeSeriesCountNames.Contains(Constants.TimeSeries.All))
{
throw new InvalidOperationException($"Cannot have more than one include on '{Constants.TimeSeries.All}'.");
}
var fromList = GetStringValuesQueryString("from", required: false);
var toList = GetStringValuesQueryString("to", required: false);
if (timeSeriesNames.Count != fromList.Count || fromList.Count != toList.Count)
{
throw new InvalidOperationException("Parameters 'timeseriesNames', 'fromList' and 'toList' must be of equal length. " +
$"Got : timeseriesNames.Count = {timeSeriesNames.Count}, fromList.Count = {fromList.Count}, toList.Count = {toList.Count}.");
}
var timeTypeList = GetStringValuesQueryString("timeType", required: false);
var timeValueList = GetStringValuesQueryString("timeValue", required: false);
var timeUnitList = GetStringValuesQueryString("timeUnit", required: false);
if (timeSeriesTimeNames.Count != timeTypeList.Count || timeTypeList.Count != timeValueList.Count || timeValueList.Count != timeUnitList.Count)
{
throw new InvalidOperationException($"Parameters '{nameof(timeSeriesTimeNames)}', '{nameof(timeTypeList)}', '{nameof(timeValueList)}' and '{nameof(timeUnitList)}' must be of equal length. " +
$"Got : {nameof(timeSeriesTimeNames)}.Count = {timeSeriesTimeNames.Count}, {nameof(timeTypeList)}.Count = {timeTypeList.Count}, {nameof(timeValueList)}.Count = {timeValueList.Count}, {nameof(timeUnitList)}.Count = {timeUnitList.Count}.");
}
var countTypeList = GetStringValuesQueryString("countType", required: false);
var countValueList = GetStringValuesQueryString("countValue", required: false);
if (timeSeriesCountNames.Count != countTypeList.Count || countTypeList.Count != countValueList.Count)
{
throw new InvalidOperationException($"Parameters '{nameof(timeSeriesCountNames)}', '{nameof(countTypeList)}', '{nameof(countValueList)}' must be of equal length. " +
$"Got : {nameof(timeSeriesCountNames)}.Count = {timeSeriesCountNames.Count}, {nameof(countTypeList)}.Count = {countTypeList.Count}, {nameof(countValueList)}.Count = {countValueList.Count}.");
}
var hs = new HashSet <AbstractTimeSeriesRange>(AbstractTimeSeriesRangeComparer.Instance);
for (int i = 0; i < timeSeriesNames.Count; i++)
{
hs.Add(new TimeSeriesRange
{
Name = timeSeriesNames[i],
From = string.IsNullOrEmpty(fromList[i])
? DateTime.MinValue
: TimeSeriesHandler.ParseDate(fromList[i], "from"),
To = string.IsNullOrEmpty(toList[i])
? DateTime.MaxValue
: TimeSeriesHandler.ParseDate(toList[i], "to")
});
}
for (int i = 0; i < timeSeriesTimeNames.Count; i++)
{
var timeValueUnit = (TimeValueUnit)Enum.Parse(typeof(TimeValueUnit), timeUnitList[i]);
if (timeValueUnit == TimeValueUnit.None)
{
throw new InvalidOperationException($"Got unexpected {nameof(TimeValueUnit)} '{nameof(TimeValueUnit.None)}'. Only the following are supported: '{nameof(TimeValueUnit.Second)}' or '{nameof(TimeValueUnit.Month)}'.");
}
if (int.TryParse(timeValueList[i], out int res) == false)
{
throw new InvalidOperationException($"Could not parse timeseries time range value.");
}
hs.Add(new TimeSeriesTimeRange
{
Name = timeSeriesTimeNames[i],
Type = (TimeSeriesRangeType)Enum.Parse(typeof(TimeSeriesRangeType), timeTypeList[i]),
Time = timeValueUnit == TimeValueUnit.Second ? TimeValue.FromSeconds(res) : TimeValue.FromMonths(res)
});
}
for (int i = 0; i < timeSeriesCountNames.Count; i++)
{
if (int.TryParse(countValueList[i], out int res) == false)
{
throw new InvalidOperationException($"Could not parse timeseries count value.");
}
hs.Add(new TimeSeriesCountRange
{
Name = timeSeriesCountNames[i],
Type = (TimeSeriesRangeType)Enum.Parse(typeof(TimeSeriesRangeType), countTypeList[i]),
Count = res
});
}
includeTimeSeries = new IncludeTimeSeriesCommand(context, new Dictionary <string, HashSet <AbstractTimeSeriesRange> > {
{ string.Empty, hs }
});
}
public async Task RapidRetentionAndRollupInACluster()
{
var cluster = await CreateRaftCluster(3, watcherCluster : true);
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ReplicationFactor = 3
}))
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromSeconds(15));
var p1 = new TimeSeriesPolicy("By1", TimeValue.FromSeconds(1), raw.RetentionTime * 2);
var p2 = new TimeSeriesPolicy("By2", TimeValue.FromSeconds(2), raw.RetentionTime * 3);
var p3 = new TimeSeriesPolicy("By4", TimeValue.FromSeconds(4), raw.RetentionTime * 4);
var p4 = new TimeSeriesPolicy("By8", TimeValue.FromSeconds(8), raw.RetentionTime * 5);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var now = DateTime.UtcNow;
var baseline = now.AddSeconds(-15 * 3);
var total = ((TimeSpan)TimeValue.FromSeconds(15 * 3)).TotalMilliseconds;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMilliseconds(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
session.Store(new User {
Name = "Karmel"
}, "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>((DocumentSession)session, "marker", null, TimeSpan.FromSeconds(15)));
}
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
await Task.Delay((TimeSpan)(p4.RetentionTime * 2));
// nothing should be left
foreach (var node in cluster.Nodes)
{
using (var nodeStore = GetDocumentStore(new Options
{
Server = node,
CreateDatabase = false,
DeleteDatabaseOnDispose = false,
ModifyDocumentStore = s => s.Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
},
ModifyDatabaseName = _ => store.Database
}))
{
using (var session = nodeStore.OpenSession())
{
var user = session.Load <User>("users/karmel");
Assert.Equal(0, session.Advanced.GetTimeSeriesFor(user)?.Count ?? 0);
var db = await node.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
await TimeSeriesReplicationTests.AssertNoLeftOvers(db);
}
}
}
}
}
public static TimeValue ParseTimePeriodFromString(string source)
{
TimeValue result;
var offset = 0;
var duration = ParseNumber(source, ref offset);
if (offset >= source.Length)
{
throw new ArgumentException("Unable to find range specification in: " + source);
}
while (char.IsWhiteSpace(source[offset]) && offset < source.Length)
{
offset++;
}
if (offset >= source.Length)
{
throw new ArgumentException("Unable to find range specification in: " + source);
}
switch (char.ToLower(source[offset++]))
{
case 's':
if (TryConsumeMatch(source, ref offset, "seconds") == false)
{
TryConsumeMatch(source, ref offset, "second");
}
result = TimeValue.FromSeconds((int)duration);
break;
case 'm':
if (TryConsumeMatch(source, ref offset, "minutes") ||
TryConsumeMatch(source, ref offset, "minute") ||
TryConsumeMatch(source, ref offset, "min"))
{
result = TimeValue.FromMinutes((int)duration);
break;
}
if (TryConsumeMatch(source, ref offset, "ms") ||
TryConsumeMatch(source, ref offset, "milliseconds") ||
TryConsumeMatch(source, ref offset, "milli"))
{
// TODO use TimeValue.FromMilliseconds when RavenDB-14988 is fixed
throw new NotSupportedException("Unsupported time period. Using milliseconds in Last/First is not supported : " + source);
}
if (TryConsumeMatch(source, ref offset, "months") ||
TryConsumeMatch(source, ref offset, "month") ||
TryConsumeMatch(source, ref offset, "mon") ||
TryConsumeMatch(source, ref offset, "mo"))
{
result = TimeValue.FromMonths((int)duration);
break;
}
goto default;
case 'h':
if (TryConsumeMatch(source, ref offset, "hours") == false)
{
TryConsumeMatch(source, ref offset, "hour");
}
result = TimeValue.FromHours((int)duration);
break;
case 'd':
if (TryConsumeMatch(source, ref offset, "days") == false)
{
TryConsumeMatch(source, ref offset, "day");
}
result = TimeValue.FromDays((int)duration);
break;
case 'q':
if (TryConsumeMatch(source, ref offset, "quarters") == false)
{
TryConsumeMatch(source, ref offset, "quarter");
}
duration *= 3;
AssertValidDurationInMonths(duration);
result = TimeValue.FromMonths((int)duration);
break;
case 'y':
if (TryConsumeMatch(source, ref offset, "years") == false)
{
TryConsumeMatch(source, ref offset, "year");
}
duration *= 12;
AssertValidDurationInMonths(duration);
result = TimeValue.FromMonths((int)duration);
break;
default:
throw new ArgumentException($"Unable to understand time range: '{source}'");
}
while (offset < source.Length && char.IsWhiteSpace(source[offset]))
{
offset++;
}
if (offset != source.Length)
{
throw new ArgumentException("After range specification, found additional unknown data: " + source);
}
return(result);
}
public async Task RapidRetention()
{
var cluster = await CreateRaftCluster(3);
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ReplicationFactor = 3
}))
{
var retention = TimeValue.FromSeconds(120);
var raw = new RawTimeSeriesPolicy(retention);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
}
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
var now = DateTime.UtcNow;
var baseline = now.Add(-retention * 3);
var total = (int)((TimeSpan)retention).TotalMilliseconds * 3;
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i < total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMilliseconds(i), new[] { 29d * i, i, i * 0.01, i * 0.1 }, "watches/fitbit");
}
session.SaveChanges();
session.Store(new User(), "marker");
session.SaveChanges();
Assert.True(await WaitForDocumentInClusterAsync <User>(cluster.Nodes, store.Database, "marker", null, TimeSpan.FromSeconds(15)));
}
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var sp = Stopwatch.StartNew();
await Task.Delay((TimeSpan)retention / 2);
var debug = new Dictionary <string, (long Count, DateTime Start, DateTime End)>();
var check = true;
while (check)
{
Assert.True(sp.Elapsed < ((TimeSpan)retention).Add((TimeSpan)retention),
$"too long has passed {sp.Elapsed}, retention is {retention} {Environment.NewLine}" +
$"debug: {string.Join(',', debug.Select(kvp => $"{kvp.Key}: ({kvp.Value.Count},{kvp.Value.Start},{kvp.Value.End})"))}");
await Task.Delay(100);
check = false;
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var tss = database.DocumentsStorage.TimeSeriesStorage;
var stats = tss.Stats.GetStats(ctx, "users/karmel", "Heartrate");
var reader = tss.GetReader(ctx, "users/karmel", "Heartrate", DateTime.MinValue, DateTime.MaxValue);
if (stats.Count == 0)
{
debug.Remove(server.ServerStore.NodeTag);
continue;
}
check = true;
Assert.Equal(stats.Start, reader.First().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(stats.End, reader.Last().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
debug[server.ServerStore.NodeTag] = stats;
}
}
}
Assert.Empty(debug);
Assert.True(sp.Elapsed < (TimeSpan)retention + (TimeSpan)retention);
await Task.Delay(3000); // let the dust to settle
await EnsureNoReplicationLoop(Servers[0], store.Database);
await EnsureNoReplicationLoop(Servers[1], store.Database);
await EnsureNoReplicationLoop(Servers[2], store.Database);
foreach (var server in Servers)
{
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
await TimeSeriesReplicationTests.AssertNoLeftOvers(database);
}
}
}