public void IncludeTimeSeriesAndMergeWithExistingRangesInCache_Typed()
{
using (var store = GetDocumentStore())
{
var baseline = RavenTestHelper.UtcToday;
var documentId = "users/ayende";
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Oren"
}, documentId);
session.SaveChanges();
}
using (var session = store.OpenSession())
{
var tsf = session.TimeSeriesFor <HeartRateMeasure>(documentId);
for (int i = 0; i < 360; i++)
{
var typedMeasure = new HeartRateMeasure
{
HeartRate = 6
};
tsf.Append(baseline.AddSeconds(i * 10), typedMeasure, "watches/fitbit");
}
session.SaveChanges();
}
using (var session = store.OpenSession())
{
var vals = session.TimeSeriesFor <HeartRateMeasure>(documentId)
.Get(baseline.AddMinutes(2), baseline.AddMinutes(10))
.ToList();
Assert.Equal(1, session.Advanced.NumberOfRequests);
Assert.Equal(49, vals.Count);
Assert.Equal(baseline.AddMinutes(2), vals[0].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(10), vals[48].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
var user = session.Load <User>(
documentId,
i => i.IncludeTimeSeries("HeartRateMeasures", baseline.AddMinutes(40), baseline.AddMinutes(50)));
Assert.Equal(2, session.Advanced.NumberOfRequests);
// should not go to server
vals = session.TimeSeriesFor <HeartRateMeasure>(documentId)
.Get(baseline.AddMinutes(40), baseline.AddMinutes(50))
.ToList();
Assert.Equal(2, session.Advanced.NumberOfRequests);
Assert.Equal(61, vals.Count);
Assert.Equal(baseline.AddMinutes(40), vals[0].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(50), vals[60].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
var sessionOperations = (InMemoryDocumentSessionOperations)session;
Assert.True(sessionOperations.TimeSeriesByDocId.TryGetValue(documentId, out var cache));
Assert.True(cache.TryGetValue("HeartRateMeasures", out var ranges));
Assert.Equal(2, ranges.Count);
Assert.Equal(baseline.AddMinutes(2), ranges[0].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(10), ranges[0].To, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(40), ranges[1].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(50), ranges[1].To, RavenTestHelper.DateTimeComparer.Instance);
// we intentionally evict just the document (without it's TS data),
// so that Load request will go to server
sessionOperations.DocumentsByEntity.Evict(user);
sessionOperations.DocumentsById.Remove(documentId);
// should go to server to get [0, 2] and merge it into existing [2, 10]
user = session.Load <User>(
documentId,
i => i.IncludeTimeSeries("HeartRateMeasures", baseline, baseline.AddMinutes(2)));
Assert.Equal(3, session.Advanced.NumberOfRequests);
// should not go to server
vals = session.TimeSeriesFor <HeartRateMeasure>(documentId)
.Get(baseline, baseline.AddMinutes(2))
.ToList();
Assert.Equal(3, session.Advanced.NumberOfRequests);
Assert.Equal(13, vals.Count);
Assert.Equal(baseline, vals[0].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(2), vals[12].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(2, ranges.Count);
Assert.Equal(baseline.AddMinutes(0), ranges[0].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(10), ranges[0].To, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(40), ranges[1].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(50), ranges[1].To, RavenTestHelper.DateTimeComparer.Instance);
// evict just the document
sessionOperations.DocumentsByEntity.Evict(user);
sessionOperations.DocumentsById.Remove(documentId);
// should go to server to get [10, 16] and merge it into existing [0, 10]
user = session.Load <User>(
documentId,
i => i.IncludeTimeSeries("HeartRateMeasures", baseline.AddMinutes(10), baseline.AddMinutes(16)));
Assert.Equal(4, session.Advanced.NumberOfRequests);
// should not go to server
vals = session.TimeSeriesFor <HeartRateMeasure>(documentId)
.Get(baseline.AddMinutes(10), baseline.AddMinutes(16))
.ToList();
Assert.Equal(4, session.Advanced.NumberOfRequests);
Assert.Equal(37, vals.Count);
Assert.Equal(baseline.AddMinutes(10), vals[0].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(16), vals[36].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(2, ranges.Count);
Assert.Equal(baseline.AddMinutes(0), ranges[0].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(16), ranges[0].To, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(40), ranges[1].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(50), ranges[1].To, RavenTestHelper.DateTimeComparer.Instance);
// evict just the document
sessionOperations.DocumentsByEntity.Evict(user);
sessionOperations.DocumentsById.Remove(documentId);
// should go to server to get range [17, 19]
// and add it to cache in between [10, 16] and [40, 50]
user = session.Load <User>(
documentId,
i => i.IncludeTimeSeries("HeartRateMeasures", baseline.AddMinutes(17), baseline.AddMinutes(19)));
Assert.Equal(5, session.Advanced.NumberOfRequests);
// should not go to server
vals = session.TimeSeriesFor <HeartRateMeasure>(documentId)
.Get(baseline.AddMinutes(17), baseline.AddMinutes(19))
.ToList();
Assert.Equal(5, session.Advanced.NumberOfRequests);
Assert.Equal(13, vals.Count);
Assert.Equal(baseline.AddMinutes(17), vals[0].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(19), vals[12].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(3, ranges.Count);
Assert.Equal(baseline.AddMinutes(0), ranges[0].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(16), ranges[0].To, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(17), ranges[1].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(19), ranges[1].To, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(40), ranges[2].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(50), ranges[2].To, RavenTestHelper.DateTimeComparer.Instance);
// evict just the document
sessionOperations.DocumentsByEntity.Evict(user);
sessionOperations.DocumentsById.Remove(documentId);
// should go to server to get range [19, 40]
// and merge the result with existing ranges [17, 19] and [40, 50]
// into single range [17, 50]
user = session.Load <User>(
documentId,
i => i.IncludeTimeSeries("HeartRateMeasures", baseline.AddMinutes(18), baseline.AddMinutes(48)));
Assert.Equal(6, session.Advanced.NumberOfRequests);
// should not go to server
vals = session.TimeSeriesFor <HeartRateMeasure>(documentId)
.Get(baseline.AddMinutes(18), baseline.AddMinutes(48))
.ToList();
Assert.Equal(6, session.Advanced.NumberOfRequests);
Assert.Equal(181, vals.Count);
Assert.Equal(baseline.AddMinutes(18), vals[0].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(48), vals[180].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(2, ranges.Count);
Assert.Equal(baseline.AddMinutes(0), ranges[0].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(16), ranges[0].To, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(17), ranges[1].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(50), ranges[1].To, RavenTestHelper.DateTimeComparer.Instance);
// evict just the document
sessionOperations.DocumentsByEntity.Evict(user);
sessionOperations.DocumentsById.Remove(documentId);
// should go to server to get range [12, 22]
// and merge the result with existing ranges [0, 16] and [17, 50]
// into single range [0, 50]
user = session.Load <User>(
documentId,
i => i.IncludeTimeSeries("HeartRateMeasures", baseline.AddMinutes(12), baseline.AddMinutes(22)));
Assert.Equal(7, session.Advanced.NumberOfRequests);
// should not go to server
vals = session.TimeSeriesFor <HeartRateMeasure>(documentId)
.Get(baseline.AddMinutes(12), baseline.AddMinutes(22))
.ToList();
Assert.Equal(7, session.Advanced.NumberOfRequests);
Assert.Equal(61, vals.Count);
Assert.Equal(baseline.AddMinutes(12), vals[0].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(22), vals[60].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(1, ranges.Count);
Assert.Equal(baseline.AddMinutes(0), ranges[0].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(50), ranges[0].To, RavenTestHelper.DateTimeComparer.Instance);
// evict just the document
sessionOperations.DocumentsByEntity.Evict(user);
sessionOperations.DocumentsById.Remove(documentId);
// should go to server to get range [50, ∞]
// and merge the result with existing range [0, 50] into single range [0, ∞]
user = session.Load <User>(
documentId,
i => i.IncludeTimeSeries("HeartRateMeasures", TimeSeriesRangeType.Last, TimeValue.FromMinutes(10)));
Assert.Equal(8, session.Advanced.NumberOfRequests);
// should not go to server
vals = session.TimeSeriesFor <HeartRateMeasure>(documentId)
.Get(baseline.AddMinutes(50))
.ToList();
Assert.Equal(8, session.Advanced.NumberOfRequests);
Assert.Equal(60, vals.Count);
Assert.Equal(baseline.AddMinutes(50), vals[0].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(baseline.AddMinutes(59).AddSeconds(50), vals[59].Timestamp, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(1, ranges.Count);
Assert.Equal(baseline.AddMinutes(0), ranges[0].From, RavenTestHelper.DateTimeComparer.Instance);
Assert.Equal(DateTime.MaxValue, ranges[0].To, RavenTestHelper.DateTimeComparer.Instance);
}
}
}
public async Task CanIndexRollups()
{
using (var store = GetDocumentStore())
{
var raw = new RawTimeSeriesPolicy(TimeValue.FromHours(24));
var p1 = new TimeSeriesPolicy("By6Hours", TimeValue.FromHours(6), raw.RetentionTime * 4);
var p2 = new TimeSeriesPolicy("By1Day", TimeValue.FromDays(1), raw.RetentionTime * 5);
var p3 = new TimeSeriesPolicy("By30Minutes", TimeValue.FromMinutes(30), raw.RetentionTime * 2);
var p4 = new TimeSeriesPolicy("By1Hour", TimeValue.FromMinutes(60), raw.RetentionTime * 3);
var config = new TimeSeriesConfiguration
{
Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
{
["Users"] = new TimeSeriesCollectionConfiguration
{
RawPolicy = raw,
Policies = new List <TimeSeriesPolicy>
{
p1, p2, p3, p4
}
},
},
PolicyCheckFrequency = TimeSpan.FromSeconds(1)
};
await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
var now = DateTime.UtcNow;
var nowSeconds = now.Second;
now = now.AddSeconds(-nowSeconds);
database.Time.UtcDateTime = () => DateTime.UtcNow.AddSeconds(-nowSeconds);
var baseline = now.AddDays(-12);
var total = ((TimeSpan)TimeValue.FromDays(12)).TotalMinutes;
await new TimeSeriesIndex().ExecuteAsync(store);
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Karmel"
}, "users/karmel");
for (int i = 0; i <= total; i++)
{
session.TimeSeriesFor("users/karmel", "Heartrate")
.Append(baseline.AddMinutes(i), new[] { 29d * i, i }, "watches/fitbit");
}
session.SaveChanges();
}
await TimeSeries.WaitForPolicyRunnerAsync(database);
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 4, policies : new List <TimeSeriesPolicy> {
p1
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 5, policies : new List <TimeSeriesPolicy> {
p2
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 2, policies : new List <TimeSeriesPolicy> {
p3
});
await TimeSeries.VerifyPolicyExecutionAsync(store, config.Collections["Users"], 3, policies : new List <TimeSeriesPolicy> {
p4
});
Indexes.WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
using (var session = store.OpenSession())
{
var user = session.Load <User>("users/karmel");
var count = session
.TimeSeriesFor(user, "Heartrate")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count(entry => entry.IsRollup == false);
count += session
.TimeSeriesFor(user, "Heartrate@By6Hours")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By1Day")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By30Minutes")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
count += session
.TimeSeriesFor(user, "Heartrate@By1Hour")
.Get(DateTime.MinValue, DateTime.MaxValue)
.Count();
var results = session.Query <TimeSeriesIndex.Result, TimeSeriesIndex>()
.ToList();
Assert.True(count == results.Count, $"Test time = {now}");
}
}
}
public static TimeValue ParseTimePeriodFromString(string source)
{
TimeValue result;
var offset = 0;
var duration = ParseNumber(source, ref offset);
if (offset >= source.Length)
{
throw new ArgumentException("Unable to find range specification in: " + source);
}
while (char.IsWhiteSpace(source[offset]) && offset < source.Length)
{
offset++;
}
if (offset >= source.Length)
{
throw new ArgumentException("Unable to find range specification in: " + source);
}
switch (char.ToLower(source[offset++]))
{
case 's':
if (TryConsumeMatch(source, ref offset, "seconds") == false)
{
TryConsumeMatch(source, ref offset, "second");
}
result = TimeValue.FromSeconds((int)duration);
break;
case 'm':
if (TryConsumeMatch(source, ref offset, "minutes") ||
TryConsumeMatch(source, ref offset, "minute") ||
TryConsumeMatch(source, ref offset, "min"))
{
result = TimeValue.FromMinutes((int)duration);
break;
}
if (TryConsumeMatch(source, ref offset, "ms") ||
TryConsumeMatch(source, ref offset, "milliseconds") ||
TryConsumeMatch(source, ref offset, "milli"))
{
// TODO use TimeValue.FromMilliseconds when RavenDB-14988 is fixed
throw new NotSupportedException("Unsupported time period. Using milliseconds in Last/First is not supported : " + source);
}
if (TryConsumeMatch(source, ref offset, "months") ||
TryConsumeMatch(source, ref offset, "month") ||
TryConsumeMatch(source, ref offset, "mon") ||
TryConsumeMatch(source, ref offset, "mo"))
{
result = TimeValue.FromMonths((int)duration);
break;
}
goto default;
case 'h':
if (TryConsumeMatch(source, ref offset, "hours") == false)
{
TryConsumeMatch(source, ref offset, "hour");
}
result = TimeValue.FromHours((int)duration);
break;
case 'd':
if (TryConsumeMatch(source, ref offset, "days") == false)
{
TryConsumeMatch(source, ref offset, "day");
}
result = TimeValue.FromDays((int)duration);
break;
case 'q':
if (TryConsumeMatch(source, ref offset, "quarters") == false)
{
TryConsumeMatch(source, ref offset, "quarter");
}
duration *= 3;
AssertValidDurationInMonths(duration);
result = TimeValue.FromMonths((int)duration);
break;
case 'y':
if (TryConsumeMatch(source, ref offset, "years") == false)
{
TryConsumeMatch(source, ref offset, "year");
}
duration *= 12;
AssertValidDurationInMonths(duration);
result = TimeValue.FromMonths((int)duration);
break;
default:
throw new ArgumentException($"Unable to understand time range: '{source}'");
}
while (offset < source.Length && char.IsWhiteSpace(source[offset]))
{
offset++;
}
if (offset != source.Length)
{
throw new ArgumentException("After range specification, found additional unknown data: " + source);
}
return(result);
}
