public async Task RapidRetentionAndRollup()
        {
            using (var store = GetDocumentStore())
            {
                var raw = new RawTimeSeriesPolicy(TimeValue.FromSeconds(15));

                var p1 = new TimeSeriesPolicy("By1", TimeValue.FromSeconds(1), raw.RetentionTime * 2);
                var p2 = new TimeSeriesPolicy("By2", TimeValue.FromSeconds(2), raw.RetentionTime * 3);
                var p3 = new TimeSeriesPolicy("By4", TimeValue.FromSeconds(4), raw.RetentionTime * 4);
                var p4 = new TimeSeriesPolicy("By8", TimeValue.FromSeconds(8), raw.RetentionTime * 5);

                var config = new TimeSeriesConfiguration
                {
                    Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
                    {
                        ["Users"] = new TimeSeriesCollectionConfiguration
                        {
                            RawPolicy = raw,
                            Policies  = new List <TimeSeriesPolicy>
                            {
                                p1, p2, p3, p4
                            }
                        },
                    },
                    PolicyCheckFrequency = TimeSpan.FromSeconds(1)
                };

                var now      = DateTime.UtcNow;
                var baseline = now.AddSeconds(-15 * 3);
                var total    = ((TimeSpan)TimeValue.FromSeconds(15 * 3)).TotalMilliseconds;

                using (var session = store.OpenSession())
                {
                    session.Store(new User {
                        Name = "Karmel"
                    }, "users/karmel");

                    for (int i = 0; i <= total; i++)
                    {
                        session.TimeSeriesFor("users/karmel", "Heartrate")
                        .Append(baseline.AddMilliseconds(i), new[] { 29d * i, i }, "watches/fitbit");
                    }
                    session.SaveChanges();
                }

                await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));

                WaitForUserToContinueTheTest(store);

                await Task.Delay((TimeSpan)(p4.RetentionTime + TimeValue.FromSeconds(10)));

                // nothing should be left

                using (var session = store.OpenSession())
                {
                    var user = session.Load <User>("users/karmel");
                    Assert.Equal(0, session.Advanced.GetTimeSeriesFor(user)?.Count ?? 0);
                }
            }
        }
Esempio n. 2
0
        private void GetTimeSeriesQueryString(DocumentDatabase database, DocumentsOperationContext context, out IncludeTimeSeriesCommand includeTimeSeries)
        {
            includeTimeSeries = null;

            var timeSeriesNames      = GetStringValuesQueryString("timeseries", required: false);
            var timeSeriesTimeNames  = GetStringValuesQueryString("timeseriestime", required: false);
            var timeSeriesCountNames = GetStringValuesQueryString("timeseriescount", required: false);

            if (timeSeriesNames.Count == 0 && timeSeriesTimeNames.Count == 0 && timeSeriesCountNames.Count == 0)
            {
                return;
            }

            if (timeSeriesNames.Count > 1 && timeSeriesNames.Contains(Constants.TimeSeries.All))
            {
                throw new InvalidOperationException($"Cannot have more than one include on '{Constants.TimeSeries.All}'.");
            }
            if (timeSeriesTimeNames.Count > 1 && timeSeriesTimeNames.Contains(Constants.TimeSeries.All))
            {
                throw new InvalidOperationException($"Cannot have more than one include on '{Constants.TimeSeries.All}'.");
            }
            if (timeSeriesCountNames.Count > 1 && timeSeriesCountNames.Contains(Constants.TimeSeries.All))
            {
                throw new InvalidOperationException($"Cannot have more than one include on '{Constants.TimeSeries.All}'.");
            }

            var fromList = GetStringValuesQueryString("from", required: false);
            var toList   = GetStringValuesQueryString("to", required: false);

            if (timeSeriesNames.Count != fromList.Count || fromList.Count != toList.Count)
            {
                throw new InvalidOperationException("Parameters 'timeseriesNames', 'fromList' and 'toList' must be of equal length. " +
                                                    $"Got : timeseriesNames.Count = {timeSeriesNames.Count}, fromList.Count = {fromList.Count}, toList.Count = {toList.Count}.");
            }

            var timeTypeList  = GetStringValuesQueryString("timeType", required: false);
            var timeValueList = GetStringValuesQueryString("timeValue", required: false);
            var timeUnitList  = GetStringValuesQueryString("timeUnit", required: false);

            if (timeSeriesTimeNames.Count != timeTypeList.Count || timeTypeList.Count != timeValueList.Count || timeValueList.Count != timeUnitList.Count)
            {
                throw new InvalidOperationException($"Parameters '{nameof(timeSeriesTimeNames)}', '{nameof(timeTypeList)}', '{nameof(timeValueList)}' and '{nameof(timeUnitList)}' must be of equal length. " +
                                                    $"Got : {nameof(timeSeriesTimeNames)}.Count = {timeSeriesTimeNames.Count}, {nameof(timeTypeList)}.Count = {timeTypeList.Count}, {nameof(timeValueList)}.Count = {timeValueList.Count}, {nameof(timeUnitList)}.Count = {timeUnitList.Count}.");
            }

            var countTypeList  = GetStringValuesQueryString("countType", required: false);
            var countValueList = GetStringValuesQueryString("countValue", required: false);

            if (timeSeriesCountNames.Count != countTypeList.Count || countTypeList.Count != countValueList.Count)
            {
                throw new InvalidOperationException($"Parameters '{nameof(timeSeriesCountNames)}', '{nameof(countTypeList)}', '{nameof(countValueList)}' must be of equal length. " +
                                                    $"Got : {nameof(timeSeriesCountNames)}.Count = {timeSeriesCountNames.Count}, {nameof(countTypeList)}.Count = {countTypeList.Count}, {nameof(countValueList)}.Count = {countValueList.Count}.");
            }

            var hs = new HashSet <AbstractTimeSeriesRange>(AbstractTimeSeriesRangeComparer.Instance);

            for (int i = 0; i < timeSeriesNames.Count; i++)
            {
                hs.Add(new TimeSeriesRange
                {
                    Name = timeSeriesNames[i],
                    From = string.IsNullOrEmpty(fromList[i])
                        ? DateTime.MinValue
                        : TimeSeriesHandler.ParseDate(fromList[i], "from"),
                    To = string.IsNullOrEmpty(toList[i])
                        ? DateTime.MaxValue
                        : TimeSeriesHandler.ParseDate(toList[i], "to")
                });
            }

            for (int i = 0; i < timeSeriesTimeNames.Count; i++)
            {
                var timeValueUnit = (TimeValueUnit)Enum.Parse(typeof(TimeValueUnit), timeUnitList[i]);
                if (timeValueUnit == TimeValueUnit.None)
                {
                    throw new InvalidOperationException($"Got unexpected {nameof(TimeValueUnit)} '{nameof(TimeValueUnit.None)}'. Only the following are supported: '{nameof(TimeValueUnit.Second)}' or '{nameof(TimeValueUnit.Month)}'.");
                }

                if (int.TryParse(timeValueList[i], out int res) == false)
                {
                    throw new InvalidOperationException($"Could not parse timeseries time range value.");
                }

                hs.Add(new TimeSeriesTimeRange
                {
                    Name = timeSeriesTimeNames[i],
                    Type = (TimeSeriesRangeType)Enum.Parse(typeof(TimeSeriesRangeType), timeTypeList[i]),
                    Time = timeValueUnit == TimeValueUnit.Second ? TimeValue.FromSeconds(res) : TimeValue.FromMonths(res)
                });
            }

            for (int i = 0; i < timeSeriesCountNames.Count; i++)
            {
                if (int.TryParse(countValueList[i], out int res) == false)
                {
                    throw new InvalidOperationException($"Could not parse timeseries count value.");
                }

                hs.Add(new TimeSeriesCountRange
                {
                    Name  = timeSeriesCountNames[i],
                    Type  = (TimeSeriesRangeType)Enum.Parse(typeof(TimeSeriesRangeType), countTypeList[i]),
                    Count = res
                });
            }

            includeTimeSeries = new IncludeTimeSeriesCommand(context, new Dictionary <string, HashSet <AbstractTimeSeriesRange> > {
                { string.Empty, hs }
            });
        }
        public async Task RapidRetentionAndRollupInACluster()
        {
            var cluster = await CreateRaftCluster(3, watcherCluster : true);

            using (var store = GetDocumentStore(new Options
            {
                Server = cluster.Leader,
                ReplicationFactor = 3
            }))
            {
                var raw = new RawTimeSeriesPolicy(TimeValue.FromSeconds(15));

                var p1 = new TimeSeriesPolicy("By1", TimeValue.FromSeconds(1), raw.RetentionTime * 2);
                var p2 = new TimeSeriesPolicy("By2", TimeValue.FromSeconds(2), raw.RetentionTime * 3);
                var p3 = new TimeSeriesPolicy("By4", TimeValue.FromSeconds(4), raw.RetentionTime * 4);
                var p4 = new TimeSeriesPolicy("By8", TimeValue.FromSeconds(8), raw.RetentionTime * 5);

                var config = new TimeSeriesConfiguration
                {
                    Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
                    {
                        ["Users"] = new TimeSeriesCollectionConfiguration
                        {
                            RawPolicy = raw,
                            Policies  = new List <TimeSeriesPolicy>
                            {
                                p1, p2, p3, p4
                            }
                        },
                    },
                    PolicyCheckFrequency = TimeSpan.FromSeconds(1)
                };

                var now      = DateTime.UtcNow;
                var baseline = now.AddSeconds(-15 * 3);
                var total    = ((TimeSpan)TimeValue.FromSeconds(15 * 3)).TotalMilliseconds;

                using (var session = store.OpenSession())
                {
                    session.Store(new User {
                        Name = "Karmel"
                    }, "users/karmel");

                    for (int i = 0; i <= total; i++)
                    {
                        session.TimeSeriesFor("users/karmel", "Heartrate")
                        .Append(baseline.AddMilliseconds(i), new[] { 29d * i, i }, "watches/fitbit");
                    }
                    session.SaveChanges();

                    session.Store(new User {
                        Name = "Karmel"
                    }, "marker");
                    session.SaveChanges();

                    Assert.True(await WaitForDocumentInClusterAsync <User>((DocumentSession)session, "marker", null, TimeSpan.FromSeconds(15)));
                }

                await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));

                await Task.Delay((TimeSpan)(p4.RetentionTime * 2));

                // nothing should be left

                foreach (var node in cluster.Nodes)
                {
                    using (var nodeStore = GetDocumentStore(new Options
                    {
                        Server = node,
                        CreateDatabase = false,
                        DeleteDatabaseOnDispose = false,
                        ModifyDocumentStore = s => s.Conventions = new DocumentConventions
                        {
                            DisableTopologyUpdates = true
                        },
                        ModifyDatabaseName = _ => store.Database
                    }))
                    {
                        using (var session = nodeStore.OpenSession())
                        {
                            var user = session.Load <User>("users/karmel");
                            Assert.Equal(0, session.Advanced.GetTimeSeriesFor(user)?.Count ?? 0);
                            var db = await node.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);

                            await TimeSeriesReplicationTests.AssertNoLeftOvers(db);
                        }
                    }
                }
            }
        }
Esempio n. 4
0
        public static TimeValue ParseTimePeriodFromString(string source)
        {
            TimeValue result;
            var       offset = 0;

            var duration = ParseNumber(source, ref offset);

            if (offset >= source.Length)
            {
                throw new ArgumentException("Unable to find range specification in: " + source);
            }

            while (char.IsWhiteSpace(source[offset]) && offset < source.Length)
            {
                offset++;
            }

            if (offset >= source.Length)
            {
                throw new ArgumentException("Unable to find range specification in: " + source);
            }

            switch (char.ToLower(source[offset++]))
            {
            case 's':
                if (TryConsumeMatch(source, ref offset, "seconds") == false)
                {
                    TryConsumeMatch(source, ref offset, "second");
                }

                result = TimeValue.FromSeconds((int)duration);
                break;

            case 'm':
                if (TryConsumeMatch(source, ref offset, "minutes") ||
                    TryConsumeMatch(source, ref offset, "minute") ||
                    TryConsumeMatch(source, ref offset, "min"))
                {
                    result = TimeValue.FromMinutes((int)duration);
                    break;
                }

                if (TryConsumeMatch(source, ref offset, "ms") ||
                    TryConsumeMatch(source, ref offset, "milliseconds") ||
                    TryConsumeMatch(source, ref offset, "milli"))
                {
                    // TODO use TimeValue.FromMilliseconds when RavenDB-14988 is fixed
                    throw new NotSupportedException("Unsupported time period. Using milliseconds in Last/First is not supported : " + source);
                }

                if (TryConsumeMatch(source, ref offset, "months") ||
                    TryConsumeMatch(source, ref offset, "month") ||
                    TryConsumeMatch(source, ref offset, "mon") ||
                    TryConsumeMatch(source, ref offset, "mo"))
                {
                    result = TimeValue.FromMonths((int)duration);
                    break;
                }
                goto default;

            case 'h':
                if (TryConsumeMatch(source, ref offset, "hours") == false)
                {
                    TryConsumeMatch(source, ref offset, "hour");
                }

                result = TimeValue.FromHours((int)duration);
                break;

            case 'd':
                if (TryConsumeMatch(source, ref offset, "days") == false)
                {
                    TryConsumeMatch(source, ref offset, "day");
                }
                result = TimeValue.FromDays((int)duration);
                break;

            case 'q':
                if (TryConsumeMatch(source, ref offset, "quarters") == false)
                {
                    TryConsumeMatch(source, ref offset, "quarter");
                }

                duration *= 3;
                AssertValidDurationInMonths(duration);
                result = TimeValue.FromMonths((int)duration);
                break;

            case 'y':
                if (TryConsumeMatch(source, ref offset, "years") == false)
                {
                    TryConsumeMatch(source, ref offset, "year");
                }
                duration *= 12;
                AssertValidDurationInMonths(duration);
                result = TimeValue.FromMonths((int)duration);
                break;

            default:
                throw new ArgumentException($"Unable to understand time range: '{source}'");
            }

            while (offset < source.Length && char.IsWhiteSpace(source[offset]))
            {
                offset++;
            }

            if (offset != source.Length)
            {
                throw new ArgumentException("After range specification, found additional unknown data: " + source);
            }

            return(result);
        }
Esempio n. 5
0
        public async Task RapidRetention()
        {
            var cluster = await CreateRaftCluster(3);

            using (var store = GetDocumentStore(new Options
            {
                Server = cluster.Leader,
                ReplicationFactor = 3
            }))
            {
                var retention = TimeValue.FromSeconds(120);
                var raw       = new RawTimeSeriesPolicy(retention);
                var config    = new TimeSeriesConfiguration
                {
                    Collections = new Dictionary <string, TimeSeriesCollectionConfiguration>
                    {
                        ["Users"] = new TimeSeriesCollectionConfiguration
                        {
                            RawPolicy = raw,
                        }
                    },
                    PolicyCheckFrequency = TimeSpan.FromSeconds(1)
                };

                var now      = DateTime.UtcNow;
                var baseline = now.Add(-retention * 3);
                var total    = (int)((TimeSpan)retention).TotalMilliseconds * 3;

                using (var session = store.OpenSession())
                {
                    session.Store(new User {
                        Name = "Karmel"
                    }, "users/karmel");

                    for (int i = 0; i < total; i++)
                    {
                        session.TimeSeriesFor("users/karmel", "Heartrate")
                        .Append(baseline.AddMilliseconds(i), new[] { 29d * i, i, i * 0.01, i * 0.1 }, "watches/fitbit");
                    }
                    session.SaveChanges();

                    session.Store(new User(), "marker");
                    session.SaveChanges();

                    Assert.True(await WaitForDocumentInClusterAsync <User>(cluster.Nodes, store.Database, "marker", null, TimeSpan.FromSeconds(15)));
                }

                await store.Maintenance.SendAsync(new ConfigureTimeSeriesOperation(config));

                var sp = Stopwatch.StartNew();
                await Task.Delay((TimeSpan)retention / 2);

                var debug = new Dictionary <string, (long Count, DateTime Start, DateTime End)>();
                var check = true;
                while (check)
                {
                    Assert.True(sp.Elapsed < ((TimeSpan)retention).Add((TimeSpan)retention),
                                $"too long has passed {sp.Elapsed}, retention is {retention} {Environment.NewLine}" +
                                $"debug: {string.Join(',', debug.Select(kvp => $"{kvp.Key}: ({kvp.Value.Count},{kvp.Value.Start},{kvp.Value.End})"))}");

                    await Task.Delay(100);

                    check = false;
                    foreach (var server in Servers)
                    {
                        var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);

                        using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
                            using (ctx.OpenReadTransaction())
                            {
                                var tss    = database.DocumentsStorage.TimeSeriesStorage;
                                var stats  = tss.Stats.GetStats(ctx, "users/karmel", "Heartrate");
                                var reader = tss.GetReader(ctx, "users/karmel", "Heartrate", DateTime.MinValue, DateTime.MaxValue);

                                if (stats.Count == 0)
                                {
                                    debug.Remove(server.ServerStore.NodeTag);
                                    continue;
                                }

                                check = true;
                                Assert.Equal(stats.Start, reader.First().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
                                Assert.Equal(stats.End, reader.Last().Timestamp, RavenTestHelper.DateTimeComparer.Instance);
                                debug[server.ServerStore.NodeTag] = stats;
                            }
                    }
                }
                Assert.Empty(debug);
                Assert.True(sp.Elapsed < (TimeSpan)retention + (TimeSpan)retention);
                await Task.Delay(3000); // let the dust to settle

                await EnsureNoReplicationLoop(Servers[0], store.Database);
                await EnsureNoReplicationLoop(Servers[1], store.Database);
                await EnsureNoReplicationLoop(Servers[2], store.Database);

                foreach (var server in Servers)
                {
                    var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);

                    await TimeSeriesReplicationTests.AssertNoLeftOvers(database);
                }
            }
        }