Exemple #1
0
        /// <summary>
        /// Initiate log-only checkpoint
        /// </summary>
        /// <param name="token">Checkpoint token</param>
        /// <param name="checkpointType">Checkpoint type</param>
        /// <param name="tryIncremental">For snapshot, try to store as incremental delta over last snapshot</param>
        /// <param name="targetVersion">
        /// intended version number of the next version. Checkpoint will not execute if supplied version is not larger
        /// than current version. Actual new version may have version number greater than supplied number. If the supplied
        /// number is -1, checkpoint will unconditionally create a new version.
        /// </param>
        /// <returns>Whether we could initiate the checkpoint. Use CompleteCheckpointAsync to wait completion.</returns>
        public bool TakeHybridLogCheckpoint(out Guid token, CheckpointType checkpointType, bool tryIncremental = false,
                                            long targetVersion = -1)
        {
            ISynchronizationTask backend;

            if (checkpointType == CheckpointType.FoldOver)
            {
                backend = new FoldOverCheckpointTask();
            }
            else if (checkpointType == CheckpointType.Snapshot)
            {
                if (tryIncremental && _lastSnapshotCheckpoint.info.guid != default && _lastSnapshotCheckpoint.info.finalLogicalAddress > hlog.FlushedUntilAddress)
                {
                    backend = new IncrementalSnapshotCheckpointTask();
                }
                else
                {
                    backend = new SnapshotCheckpointTask();
                }
            }
            else
            {
                throw new FasterException("Unsupported checkpoint type");
            }

            var result = StartStateMachine(new HybridLogCheckpointStateMachine(backend, targetVersion));

            token = _hybridLogCheckpointToken;
            return(result);
        }
Exemple #2
0
 private void Prepare(CheckpointType checkpointType, out string logPath, out string objPath, out IDevice log, out IDevice objlog, out FasterKV <MyKey, MyValue> h, out MyContext context)
 {
     logPath = Path.Combine(FasterFolderPath, $"FasterRecoverTests.log");
     objPath = Path.Combine(FasterFolderPath, $"FasterRecoverTests_HEAP.log");
     log     = Devices.CreateLogDevice(logPath);
     objlog  = Devices.CreateLogDevice(objPath);
     h       = new FasterKV
               <MyKey, MyValue>
                   (1L << 20,
             new LogSettings
             {
             LogDevice       = log,
             ObjectLogDevice = objlog,
             SegmentSizeBits = 12,
             MemorySizeBits  = 12,
             PageSizeBits    = 9
         },
             new CheckpointSettings()
             {
             CheckpointDir  = Path.Combine(FasterFolderPath, "check-points"),
             CheckPointType = checkpointType
         },
             new SerializerSettings <MyKey, MyValue> {
         keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer()
         }
                   );
     context = new MyContext();
 }
Exemple #3
0
        public void ObjectRecoveryTest1([Values] CheckpointType checkpointType)
        {
            Prepare(checkpointType, out string logPath, out string objPath, out IDevice log, out IDevice objlog, out FasterKV <MyKey, MyValue, MyInput, MyOutput, MyContext, MyFunctions> h, out MyContext context);

            h.StartSession();

            Write(h, context);

            h.Refresh();

            Read(h, context, false);

            h.TakeFullCheckpoint(out Guid CheckPointID);
            h.CompleteCheckpoint(true);

            Destroy(log, objlog, h);

            Prepare(checkpointType, out logPath, out objPath, out log, out objlog, out h, out context);

            h.Recover();

            h.StartSession();

            Read(h, context, true);

            Destroy(log, objlog, h);
        }
Exemple #4
0
        public void ObjectRecoveryTest1(
            [Values] CheckpointType checkpointType,
            [Range(100, 1500, 600)] int iterations)
        {
            this.iterations = iterations;
            Prepare(checkpointType, out _, out _, out IDevice log, out IDevice objlog, out FasterKV <MyKey, MyValue> h, out MyContext context);

            var session1 = h.For(new MyFunctions()).NewSession <MyFunctions>();

            Write(session1, context, h);
            Read(session1, context, false);
            session1.Dispose();

            h.TakeFullCheckpoint(out _);
            h.CompleteCheckpointAsync().GetAwaiter().GetResult();

            Destroy(log, objlog, h);

            Prepare(checkpointType, out _, out _, out log, out objlog, out h, out context);

            h.Recover();

            var session2 = h.For(new MyFunctions()).NewSession <MyFunctions>();

            Read(session2, context, true);
            session2.Dispose();

            Destroy(log, objlog, h);
        }
Exemple #5
0
        public void AddCheckpoint(
            string happeningId,
            string id,
            CheckpointType checkpointType,
            int order,
            string name,
            double latitude,
            double longitude,
            decimal distanceFromPrevious,
            decimal distanceFromStart)
        {
            // Only one can be start
            this.ValidateUniqueCheckpointType(checkpointType, CheckpointType.Start);

            // Only one can be end
            this.ValidateUniqueCheckpointType(checkpointType, CheckpointType.Start);

            // All ok, add
            this.Apply(
                new CheckpointValidated(
                    happeningId,
                    id,
                    order,
                    checkpointType,
                    name,
                    latitude,
                    longitude,
                    distanceFromPrevious,
                    distanceFromStart));
        }
Exemple #6
0
        public void ObjectRecoveryTest1([Values] CheckpointType checkpointType)
        {
            Prepare(checkpointType, out _, out _, out IDevice log, out IDevice objlog, out FasterKV <MyKey, MyValue, MyInput, MyOutput, MyContext, MyFunctions> h, out MyContext context);

            var session1 = h.NewSession();

            Write(session1, context);
            Read(session1, context, false);
            session1.Dispose();

            h.TakeFullCheckpoint(out _);
            h.CompleteCheckpointAsync().GetAwaiter().GetResult();

            Destroy(log, objlog, h);

            Prepare(checkpointType, out _, out _, out log, out objlog, out h, out context);

            h.Recover();

            var session2 = h.NewSession();

            Read(session2, context, true);
            session2.Dispose();

            Destroy(log, objlog, h);
        }
Exemple #7
0
        /// <summary>
        /// Initiate full checkpoint
        /// </summary>
        /// <param name="token">Checkpoint token</param>
        /// <param name="checkpointType">Checkpoint type</param>
        /// <param name="targetVersion">
        /// intended version number of the next version. Checkpoint will not execute if supplied version is not larger
        /// than current version. Actual new version may have version number greater than supplied number. If the supplied
        /// number is -1, checkpoint will unconditionally create a new version.
        /// </param>
        /// <returns>
        /// Whether we successfully initiated the checkpoint (initiation may
        /// fail if we are already taking a checkpoint or performing some other
        /// operation such as growing the index). Use CompleteCheckpointAsync to wait completion.
        /// </returns>
        public bool TakeFullCheckpoint(out Guid token, CheckpointType checkpointType, long targetVersion = -1)
        {
            ISynchronizationTask backend;

            if (checkpointType == CheckpointType.FoldOver)
            {
                backend = new FoldOverCheckpointTask();
            }
            else if (checkpointType == CheckpointType.Snapshot)
            {
                backend = new SnapshotCheckpointTask();
            }
            else
            {
                throw new FasterException("Unsupported full checkpoint type");
            }

            var result = StartStateMachine(new FullCheckpointStateMachine(backend, targetVersion));

            if (result)
            {
                token = _hybridLogCheckpointToken;
            }
            else
            {
                token = default;
            }
            return(result);
        }
Exemple #8
0
        private bool IsCheckpointSafe(Guid token, CheckpointType checkpointType)
        {
            switch (checkpointType)
            {
            case CheckpointType.INDEX_ONLY:
            {
                var dir = new DirectoryInfo(directoryConfiguration.GetIndexCheckpointFolder(token));
                return(File.Exists(dir.FullName + Path.DirectorySeparatorChar + "completed.dat"));
            }

            case CheckpointType.HYBRID_LOG_ONLY:
            {
                var dir = new DirectoryInfo(directoryConfiguration.GetHybridLogCheckpointFolder(token));
                return(File.Exists(dir.FullName + Path.DirectorySeparatorChar + "completed.dat"));
            }

            case CheckpointType.FULL:
            {
                return(IsCheckpointSafe(token, CheckpointType.INDEX_ONLY) &&
                       IsCheckpointSafe(token, CheckpointType.HYBRID_LOG_ONLY));
            }

            default:
                return(false);
            }
        }
Exemple #9
0
        public async ValueTask ObjectRecoveryTest3(
            [Values] CheckpointType checkpointType,
            [Values(2000)] int iterations,
            [Values] bool isAsync)
        {
            this.iterations = iterations;
            Prepare(checkpointType, out _, out _, out IDevice log, out IDevice objlog, out FasterKV <MyKey, MyValue> h, out MyContext context);

            var session1 = h.For(new MyFunctions()).NewSession <MyFunctions>();
            var tokens   = Write(session1, context, h);

            Read(session1, context, false, iterations);
            session1.Dispose();

            h.TakeHybridLogCheckpoint(out Guid token);
            h.CompleteCheckpointAsync().GetAwaiter().GetResult();
            tokens.Add((iterations, token));
            Destroy(log, objlog, h);

            foreach (var item in tokens)
            {
                Prepare(checkpointType, out _, out _, out log, out objlog, out h, out context);

                if (isAsync)
                {
                    await h.RecoverAsync(default, item.Item2);
Exemple #10
0
        public void RecoveryCheck2([Values] CheckpointType checkpointType)
        {
            using var s1 = fht1.NewSession(new SimpleFunctions <long, long>());

            using var fht2 = new FasterKV <long, long>
                                 (1L << 10,
                                 logSettings: new LogSettings { LogDevice = log, MutableFraction = 1, PageSizeBits = 10, MemorySizeBits = 20 },
                                 checkpointSettings: new CheckpointSettings { CheckpointDir = path }
                                 );

            for (int i = 0; i < 5; i++)
            {
                for (long key = 1000 * i; key < 1000 * i + 1000; key++)
                {
                    s1.Upsert(ref key, ref key);
                }
                fht1.TakeHybridLogCheckpointAsync(checkpointType).GetAwaiter().GetResult();

                fht2.Recover();

                Assert.IsTrue(fht1.Log.HeadAddress == fht2.Log.HeadAddress);
                Assert.IsTrue(fht1.Log.ReadOnlyAddress == fht2.Log.ReadOnlyAddress);
                Assert.IsTrue(fht1.Log.TailAddress == fht2.Log.TailAddress);

                using var s2 = fht2.NewSession(new SimpleFunctions <long, long>());
                for (long key = 0; key < 1000 * i + 1000; key++)
                {
                    long output = default;
                    var  status = s2.Read(ref key, ref output);
                    Assert.IsTrue(status == Status.OK && output == key);
                }
            }
        }
Exemple #11
0
 public DynamicRaceCheckpoint(CheckpointType type, Vector3 position, Vector3 nextPosition,
                              float size           = 3.0f, int worldid = -1, int interiorid = -1, BasePlayer player = null,
                              float streamdistance = 100.0f, DynamicArea area = null, int priority = 0)
 {
     Id = Internal.CreateDynamicRaceCP((int)type, position.X, position.Y, position.Z, nextPosition.X,
                                       nextPosition.Y, nextPosition.Z, size, worldid, interiorid, player?.Id ?? -1,
                                       streamdistance, area?.Id ?? -1, priority);
 }
Exemple #12
0
        public async ValueTask LocalDeviceSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync, [Values] bool testCommitCookie)
        {
            checkpointManager = new DeviceLogCommitCheckpointManager(
                new LocalStorageNamedDeviceFactory(),
                new DefaultCheckpointNamingScheme($"{TestUtils.MethodTestDir}/{TestUtils.AzureTestDirectory}"));
            await SimpleRecoveryTest1_Worker(checkpointType, isAsync, testCommitCookie);

            checkpointManager.PurgeAll();
        }
Exemple #13
0
        public async ValueTask PageBlobSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync, [Values] bool testCommitCookie)
        {
            TestUtils.IgnoreIfNotRunningAzureTests();
            checkpointManager = new DeviceLogCommitCheckpointManager(
                new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
                new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
            await SimpleRecoveryTest1_Worker(checkpointType, isAsync, testCommitCookie);

            checkpointManager.PurgeAll();
        }
Exemple #14
0
        public void LocalDeviceSimpleRecoveryTest(CheckpointType checkpointType)
        {
            ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
                new LocalStorageNamedDeviceFactory(),
                new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));

            SimpleRecoveryTest1(checkpointType, checkpointManager);
            checkpointManager.PurgeAll();
            checkpointManager.Dispose();
        }
Exemple #15
0
        public void ChangeType(string id, CheckpointType newType)
        {
            // Only one can be start
            this.ValidateUniqueCheckpointType(newType, CheckpointType.Start);

            // Only one can be end
            this.ValidateUniqueCheckpointType(newType, CheckpointType.Start);

            // All ok, change
            this.Apply(new CheckpointTypeChangeValidated(id, newType));
        }
Exemple #16
0
        /// <summary>
        /// Take full (index + log) checkpoint
        /// </summary>
        /// <param name="checkpointType">Checkpoint type</param>
        /// <param name="cancellationToken">Cancellation token</param>
        /// <returns>
        /// (bool success, Guid token)
        /// success: Whether we successfully initiated the checkpoint (initiation may
        /// fail if we are already taking a checkpoint or performing some other
        /// operation such as growing the index).
        /// token: Token for taken checkpoint
        /// Await task to complete checkpoint, if initiated successfully
        /// </returns>
        public async ValueTask <(bool success, Guid token)> TakeFullCheckpointAsync(CheckpointType checkpointType, CancellationToken cancellationToken = default)
        {
            var success = TakeFullCheckpoint(out Guid token, checkpointType);

            if (success)
            {
                await CompleteCheckpointAsync(cancellationToken);
            }

            return(success, token);
        }
        public async ValueTask LocalDeviceSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync)
        {
            ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
                new LocalStorageNamedDeviceFactory(),
                new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));

            await SimpleRecoveryTest1_Worker(checkpointType, checkpointManager, isAsync);

            checkpointManager.PurgeAll();
            checkpointManager.Dispose();
        }
Exemple #18
0
        /// <summary>
        /// Take full (index + log) checkpoint
        /// </summary>
        /// <param name="checkpointType">Checkpoint type</param>
        /// <param name="cancellationToken">Cancellation token</param>
        /// <param name="targetVersion">
        /// intended version number of the next version. Checkpoint will not execute if supplied version is not larger
        /// than current version. Actual new version may have version number greater than supplied number. If the supplied
        /// number is -1, checkpoint will unconditionally create a new version.
        /// </param>
        /// <returns>
        /// (bool success, Guid token)
        /// success: Whether we successfully initiated the checkpoint (initiation may
        /// fail if we are already taking a checkpoint or performing some other
        /// operation such as growing the index).
        /// token: Token for taken checkpoint
        /// Await task to complete checkpoint, if initiated successfully
        /// </returns>
        public async ValueTask <(bool success, Guid token)> TakeFullCheckpointAsync(CheckpointType checkpointType,
                                                                                    CancellationToken cancellationToken = default, long targetVersion = -1)
        {
            var success = TakeFullCheckpoint(out Guid token, checkpointType, targetVersion);

            if (success)
            {
                await CompleteCheckpointAsync(cancellationToken).ConfigureAwait(false);
            }

            return(success, token);
        }
Exemple #19
0
        private void ValidateUniqueCheckpointType(CheckpointType checkpointType, CheckpointType uniqueType)
        {
            if (checkpointType != uniqueType)
            {
                return;
            }

            if (this.State.Checkpoints.Any(m => m.CheckpointType == uniqueType))
            {
                throw new InvalidOperationException();
            }
        }
Exemple #20
0
 public void PageBlobSimpleRecoveryTest(CheckpointType checkpointType)
 {
     if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
     {
         ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
             new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
             new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
         SimpleRecoveryTest1(checkpointType, checkpointManager);
         checkpointManager.PurgeAll();
         checkpointManager.Dispose();
     }
 }
        public async ValueTask PageBlobSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync)
        {
            if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
            {
                ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
                    new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
                    new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
                await SimpleRecoveryTest1_Worker(checkpointType, checkpointManager, isAsync);

                checkpointManager.PurgeAll();
                checkpointManager.Dispose();
            }
        }
        [InlineData(50_000, 1, CheckpointType.Snapshot)] //OK
        public async Task AddRestartColdRead(int loops, int step, CheckpointType checkpointType)
        {
            var options = GetOptions($"{nameof(AddRestartGetValues)}-{loops}");

            options.CheckPointType = checkpointType;
            options.DeleteOnClose  = false;

            Stopwatch watch = new Stopwatch();

            watch.Reset();

            FasterDictionary <int, string> .ReadResult result;
            using (var dictionary = new FasterDictionary <int, string>(TestHelper.GetKeyComparer <int>(), options))
            {
                for (var i = 0; i < loops; i++)
                {
                    var guid = GetGuid(i);
                    dictionary.Upsert(i, guid).Dismiss();
                }

                await dictionary.Ping();

                for (var i = 0; i < loops; i += step)
                {
                    var guid = GetGuid(loops - 1);
                    result = await dictionary.TryGet(loops - 1);

                    Assert.True(result.Found);
                    Assert.Equal(guid.ToString(), result.Value);
                }

                await dictionary.Save();
            }

            for (var k = 0; k < 3; k++)
            {
                using (var dictionary = new FasterDictionary <int, string>(TestHelper.GetKeyComparer <int>(), options))
                {
                    await dictionary.Ping();

                    for (var i = 0; i < loops; i += step)
                    {
                        var guid = GetGuid(loops - 1);
                        result = await dictionary.TryGet(loops - 1);

                        Assert.True(result.Found);
                        Assert.Equal(guid.ToString(), result.Value);
                    }
                }
            }
        }
Exemple #23
0
 private bool InternalTakeCheckpoint(CheckpointType type)
 {
     if (_systemState.phase == Phase.REST)
     {
         var context      = (long)type;
         var currentState = SystemState.Make(Phase.REST, _systemState.version);
         var nextState    = GetNextState(currentState, type);
         return(GlobalMoveToNextState(currentState, nextState, ref context));
     }
     else
     {
         return(false);
     }
 }
        [InlineData(1_000_000, CheckpointType.FoldOver)]    //FAIL

        public async Task AddUpdateGet(int loops, CheckpointType checkpointType)
        {
            var options = GetOptions($"{nameof(AddIterateUpdateIterate)}-{loops}");

            options.CheckPointType = checkpointType;
            options.DeleteOnClose  = false;

            FasterDictionary <int, string> .ReadResult result;
            using (var dictionary = new FasterDictionary <int, string>(TestHelper.GetKeyComparer <int>(), options))
            {
                for (var i = 0; i < loops; i++)
                {
                    dictionary.Upsert(i, (i + 1).ToString()).Dismiss();
                }


                await dictionary.Save();
            }


            using (var dictionary = new FasterDictionary <int, string>(TestHelper.GetKeyComparer <int>(), options))
            {
                for (var i = 0; i < loops; i++)
                {
                    dictionary.Upsert(i, (i + 2).ToString()).Dismiss();
                }


                await dictionary.Save();
            }

            options.DeleteOnClose = true;

            using (var dictionary = new FasterDictionary <int, string>(TestHelper.GetKeyComparer <int>(), options))
            {
                for (var i = 0; i < 100; i++)
                {
                    var guid = GetGuid(i);
                    result = await dictionary.TryGet(i);

                    Assert.True(result.Found);
                    Assert.Equal((result.Key + 2).ToString(), result.Value);
                }

                result = await dictionary.TryGet(loops);

                Assert.False(result.Found);
            }
        }
Exemple #25
0
        /// <summary>
        /// Create FASTER instance
        /// </summary>
        /// <param name="size">Size of core index (#cache lines)</param>
        /// <param name="comparer">FASTER equality comparer for key</param>
        /// <param name="functions">Callback functions</param>
        /// <param name="logSettings">Log settings</param>
        /// <param name="checkpointSettings">Checkpoint settings</param>
        /// <param name="serializerSettings">Serializer settings</param>
        public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null)
        {
            if (comparer != null)
            {
                this.comparer = comparer;
            }
            else
            {
                if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
                {
                    this.comparer = new Key() as IFasterEqualityComparer <Key>;
                }
                else
                {
                    Console.WriteLine("***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
                    this.comparer = FasterEqualityComparer <Key> .Default;
                }
            }

            if (checkpointSettings == null)
            {
                checkpointSettings = new CheckpointSettings();
            }

            directoryConfiguration = new DirectoryConfiguration(checkpointSettings.CheckpointDir);

            FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
            CopyReadsToTail  = logSettings.CopyReadsToTail;
            this.functions   = functions;

            if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
            {
                hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer);
            }
            else
            {
                hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer);
            }

            hlog.Initialize();

            sectorSize = (int)logSettings.LogDevice.SectorSize;
            Initialize(size, sectorSize);

            _systemState         = default(SystemState);
            _systemState.phase   = Phase.REST;
            _systemState.version = 1;
            _checkpointType      = CheckpointType.HYBRID_LOG_ONLY;
        }
Exemple #26
0
        public FasterKV(long size, IDevice logDevice, IDevice objectLogDevice, string checkpointDir = null)
        {
            if (checkpointDir != null)
            {
                Config.CheckpointDirectory = checkpointDir;
            }

            hlog = new PersistentMemoryMalloc(logDevice, objectLogDevice);
            var recordSize = Layout.EstimatePhysicalSize(null, null);

            Initialize(size, hlog.GetSectorSize());

            _systemState         = default(SystemState);
            _systemState.phase   = Phase.REST;
            _systemState.version = 1;
            _checkpointType      = CheckpointType.HYBRID_LOG_ONLY;
        }
Exemple #27
0
        public RecoveryStatus(int capacity,
                              long startPage,
                              long endPage, long untilAddress, CheckpointType checkpointType)
        {
            this.capacity       = capacity;
            this.startPage      = startPage;
            this.endPage        = endPage;
            this.untilAddress   = untilAddress;
            this.checkpointType = checkpointType;

            readStatus  = new ReadStatus[capacity];
            flushStatus = new FlushStatus[capacity];
            for (int i = 0; i < capacity; i++)
            {
                flushStatus[i] = FlushStatus.Done;
                readStatus[i]  = ReadStatus.Pending;
            }
        }
Exemple #28
0
        public async ValueTask RecoveryCheck1([Values] CheckpointType checkpointType, [Values] bool isAsync, [Values] bool useReadCache, [Values(128, 1 << 10)] int size)
        {
            using var fht1 = new FasterKV <long, long>
                                 (size,
                                 logSettings: new LogSettings { LogDevice = log, MutableFraction = 1, PageSizeBits = 10, MemorySizeBits = 20, ReadCacheSettings = useReadCache ? new ReadCacheSettings() : null },
                                 checkpointSettings: new CheckpointSettings { CheckpointDir = path }
                                 );

            using var s1 = fht1.NewSession(new MyFunctions());
            for (long key = 0; key < 1000; key++)
            {
                s1.Upsert(ref key, ref key);
            }

            if (useReadCache)
            {
                fht1.Log.FlushAndEvict(true);
                for (long key = 0; key < 1000; key++)
                {
                    long output = default;
                    var  status = s1.Read(ref key, ref output);
                    if (status != Status.PENDING)
                    {
                        Assert.AreEqual(Status.OK, status, $"status = {status}");
                        Assert.AreEqual(key, output, $"output = {output}");
                    }
                }
                s1.CompletePending(true);
            }

            var task = fht1.TakeFullCheckpointAsync(checkpointType);

            using var fht2 = new FasterKV <long, long>
                                 (size,
                                 logSettings: new LogSettings { LogDevice = log, MutableFraction = 1, PageSizeBits = 10, MemorySizeBits = 20, ReadCacheSettings = useReadCache ? new ReadCacheSettings() : null },
                                 checkpointSettings: new CheckpointSettings { CheckpointDir = path }
                                 );

            if (isAsync)
            {
                var(status, token) = await task;
                await fht2.RecoverAsync(default, token);
Exemple #29
0
        /// <summary>
        /// Create FASTER instance
        /// </summary>
        /// <param name="size"></param>
        /// <param name="logSettings"></param>
        /// <param name="checkpointSettings"></param>
        public FasterKV(long size, LogSettings logSettings, CheckpointSettings checkpointSettings = null)
        {
            if (checkpointSettings == null)
            {
                checkpointSettings = new CheckpointSettings();
            }

            Config.CheckpointDirectory = checkpointSettings.CheckpointDir;
            FoldOverSnapshot           = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;

            hlog = new PersistentMemoryMalloc(logSettings, this);
            var recordSize = Layout.EstimatePhysicalSize(null, null);

            Initialize(size, hlog.GetSectorSize());

            _systemState         = default(SystemState);
            _systemState.phase   = Phase.REST;
            _systemState.version = 1;
            _checkpointType      = CheckpointType.HYBRID_LOG_ONLY;
        }
Exemple #30
0
        private bool InternalTakeCheckpoint(CheckpointType type)
        {
            if (_systemState.phase == Phase.GC)
            {
                Debug.WriteLine("Forcing completion of GC");
                GarbageCollectBuckets(0, true);
            }

            if (_systemState.phase == Phase.REST)
            {
                var context      = (long)type;
                var currentState = SystemState.Make(Phase.REST, _systemState.version);
                var nextState    = GetNextState(currentState, type);
                return(GlobalMoveToNextState(currentState, nextState, ref context));
            }
            else
            {
                return(false);
            }
        }
 /// <summary>
 /// Constructor
 /// </summary>
 /// <param name="number"></param>
 /// <param name="id"></param>
 public ArbiterCheckpoint(int number, IAreaSubtypeWaypointId id)
 {
     this.CheckpointNumber = number;
     this.WaypointId = id;
     this.Type = CheckpointType.Normal;
 }
 /// <summary>
 /// Constructor
 /// </summary>
 /// <param name="number"></param>
 /// <param name="id"></param>
 public ArbiterCheckpoint(int number, IAreaSubtypeWaypointId id, CheckpointType type)
 {
     this.CheckpointNumber = number;
     this.WaypointId = id;
     this.Type = type;
 }