Example #1
0
        public void CompactorShouldNotBePessimistDespiteRunningTransactions()
        {
            using var fileCollection = new InMemoryFileCollection();
            var options = new KeyValueDBOptions
            {
                Compression        = new NoCompressionStrategy(),
                FileCollection     = fileCollection,
                FileSplitSize      = 8096,
                CompactorScheduler = CompactorScheduler.Instance,
            };

            using var kvDb = new KeyValueDB(options);
            for (var i = 0; i < 100; i++)
            {
                using var tr = kvDb.StartWritingTransaction().Result;
                var key = new byte[4];
                BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
                tr.CreateOrUpdateKeyValue(key, new byte[2000]);
                tr.Commit();
            }
            kvDb.Compact(new CancellationToken());
            var fileCountAfterFirstCompaction = fileCollection.GetCount();

            using (kvDb.StartReadOnlyTransaction())
            {
                for (var i = 0; i < 50; i++)
                {
                    using var tr = kvDb.StartWritingTransaction().Result;
                    var key = new byte[4];
                    BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i * 2);
                    tr.FindExactKey(key);
                    tr.EraseCurrent();
                    tr.Commit();
                }
                while (kvDb.Compact(new CancellationToken()))
                {
                }

                Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction + 2, fileCountAfterFirstCompaction + 50);
            }
            for (var i = 0; i < 4; i++)
            {
                using var tr = kvDb.StartWritingTransaction().Result;
                var key = new byte[4];
                BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
                tr.CreateOrUpdateKeyValue(key, new byte[2000]);
                tr.Commit();
            }
            while (kvDb.Compact(new CancellationToken()))
            {
            }

            Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
        }
Example #2
0
        void Create2TrlFiles(KeyValueDBOptions options)
        {
            using (var db = NewKeyValueDB(options))
            {
                using (var tr = db.StartTransaction())
                {
                    tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
                    tr.Commit();
                }

                using (var tr = db.StartTransaction())
                {
                    tr.CreateOrUpdateKeyValue(_key3, new byte[50]);
                    tr.Commit();
                }
            }
        }
Example #3
0
        public void OpeningDbWithLenientOpenWithMissingFirstTrlAndKviWarnsAndRecoversData()
        {
            using var fileCollection = new InMemoryFileCollection();
            var options = new KeyValueDBOptions
            {
                FileCollection = fileCollection, Compression = new NoCompressionStrategy(), FileSplitSize = 1024,
                LenientOpen    = true,
                Logger         = new LoggerMock()
            };

            Create2TrlFiles(options);
            fileCollection.GetFile(1) !.Remove();
            using var db = NewKeyValueDB(options);
            Assert.Equal("No valid Kvi and lowest Trl in chain is not first. Missing 1. LenientOpen is true, recovering data.", ((LoggerMock)options.Logger).LastWarning);
            using var tr = db.StartTransaction();
            Assert.Equal(1, tr.GetKeyValueCount());
            Assert.Equal(1u, fileCollection.GetCount());
        }
Example #4
0
        public void CanOpenDbAfterDeletingAndCompacting()
        {
            using (var fileCollection = new InMemoryFileCollection())
            {
                var options = new KeyValueDBOptions
                {
                    Compression                    = new NoCompressionStrategy(),
                    FileCollection                 = fileCollection,
                    FileSplitSize                  = 4096,
                    OpenUpToCommitUlong            = null,
                    PreserveHistoryUpToCommitUlong = null,
                    CompactorScheduler             = null,
                };

                using (var kvDb = new KeyValueDB(options))
                {
                    using (var tr = kvDb.StartWritingTransaction().Result)
                    {
                        tr.CreateOrUpdateKeyValue(new byte[5], new byte[3000]);
                        tr.CreateOrUpdateKeyValue(new byte[6], new byte[2000]);
                        tr.Commit();
                    }

                    kvDb.Compact(CancellationToken.None);
                }
                using (var kvDb = new KeyValueDB(options))
                {
                    using (var tr = kvDb.StartWritingTransaction().Result)
                    {
                        tr.FindFirstKey();
                        tr.EraseCurrent();
                        tr.Commit();
                    }

                    kvDb.Compact(CancellationToken.None);
                }

                using (var kvDb = new KeyValueDB(options))
                {
                    // If there is error in KVI 3 it will create new KVI 4, but there is no problem in KVI 3
                    Assert.Null(kvDb.FileCollection.FileInfoByIdx(4));
                }
            }
        }
Example #5
0
        public void OpeningDbWithMissingFirstTrlAndKviWarnsAndOpenEmptyDb()
        {
            using var fileCollection = new InMemoryFileCollection();
            var options = new KeyValueDBOptions
            {
                FileCollection = fileCollection, Compression = new NoCompressionStrategy(), FileSplitSize = 1024,
                Logger         = new LoggerMock()
            };

            Create2TrlFiles(options);
            fileCollection.GetFile(1) !.Remove();
            using var db = NewKeyValueDB(options);
            Assert.Equal("No valid Kvi and lowest Trl in chain is not first. Missing 1", ((LoggerMock)options.Logger).LastWarning);
            using var tr = db.StartTransaction();
            Assert.Equal(0, tr.GetKeyValueCount());
            Assert.Equal(0u, fileCollection.GetCount());
            tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
            tr.Commit();
        }
Example #6
0
        public void CompatorShouldNotBePesimist()
        {
            using (var fileCollection = new InMemoryFileCollection())
            {
                var options = new KeyValueDBOptions
                {
                    Compression                    = new NoCompressionStrategy(),
                    FileCollection                 = fileCollection,
                    FileSplitSize                  = 8096,
                    OpenUpToCommitUlong            = null,
                    PreserveHistoryUpToCommitUlong = null,
                    CompactorScheduler             = CompactorScheduler.Instance,
                };

                using (var kvDb = new KeyValueDB(options))
                {
                    for (var i = 0; i < 100; i++)
                    {
                        using (var tr = kvDb.StartWritingTransaction().Result)
                        {
                            var key = new byte[4];
                            BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
                            tr.CreateOrUpdateKeyValueUnsafe(key, new byte[2000]);
                            tr.SetCommitUlong((ulong)i);
                            tr.Commit();
                        }
                    }
                    kvDb.PreserveHistoryUpToCommitUlong = 100;
                    kvDb.Compact(new System.Threading.CancellationToken());
                    var fileCountAfterFirstCompaction = fileCollection.GetCount();
                    for (var i = 0; i < 50; i++)
                    {
                        using (var tr = kvDb.StartWritingTransaction().Result)
                        {
                            var key = new byte[4];
                            BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
                            tr.FindExactKey(key);
                            tr.EraseCurrent();
                            tr.SetCommitUlong(100 + (ulong)i);
                            tr.Commit();
                        }
                    }
                    kvDb.PreserveHistoryUpToCommitUlong = 150;
                    kvDb.Compact(new System.Threading.CancellationToken());
                    Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction + 1, fileCountAfterFirstCompaction + 3);
                    using (var trOlder = kvDb.StartReadOnlyTransaction())
                    {
                        for (var i = 50; i < 100; i++)
                        {
                            using (var tr = kvDb.StartWritingTransaction().Result)
                            {
                                var key = new byte[4];
                                BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
                                tr.FindExactKey(key);
                                tr.EraseCurrent();
                                tr.SetCommitUlong(100 + (ulong)i);
                                tr.Commit();
                            }
                        }
                        kvDb.Compact(new System.Threading.CancellationToken());
                        Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
                        kvDb.PreserveHistoryUpToCommitUlong = 200;
                        kvDb.Compact(new System.Threading.CancellationToken());
                        Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
                    }
                    kvDb.Compact(new System.Threading.CancellationToken());
                    Assert.InRange <uint>(fileCollection.GetCount(), 1, 4);
                }
            }
        }
Example #7
0
        public void ComplexTrlRollbackWhenKviLost()
        {
            using (var fileCollection = new InMemoryFileCollection())
            {
                var options = new KeyValueDBOptions
                {
                    Compression                    = new SnappyCompressionStrategy(),
                    FileCollection                 = fileCollection,
                    FileSplitSize                  = 100 * 1024 * 1024,
                    OpenUpToCommitUlong            = null,
                    PreserveHistoryUpToCommitUlong = null,
                    CompactorScheduler             = CompactorScheduler.Instance,
                };

                using (var kvDb = new KeyValueDB(options))
                    using (var objDb = new ObjectDB())
                    {
                        objDb.Open(kvDb, false);

                        for (ulong i = 0; i < 100; i += 3)
                        {
                            using (var tr = objDb.StartWritingTransaction().Result)
                            {
                                var person = tr.Singleton <Person>();
                                person.Age = (uint)i;
                                tr.Store(person);
                                tr.SetCommitUlong(i);
                                tr.Commit();
                            }
                        }
                    }

                options = new KeyValueDBOptions
                {
                    Compression                    = new SnappyCompressionStrategy(),
                    FileCollection                 = fileCollection,
                    FileSplitSize                  = 100 * 1024 * 1024,
                    OpenUpToCommitUlong            = 9UL,
                    PreserveHistoryUpToCommitUlong = 9UL,
                    CompactorScheduler             = CompactorScheduler.Instance,
                };
                using (var kvDb = new KeyValueDB(options))
                    using (var objDb = new ObjectDB())
                    {
                        objDb.Open(kvDb, false);

                        using (var tr = objDb.StartReadOnlyTransaction())
                        {
                            Assert.Equal(9UL, tr.GetCommitUlong());
                        }
                    }

                // Delete KVI file
                fileCollection.GetFile(3).Remove();

                options = new KeyValueDBOptions
                {
                    Compression                    = new SnappyCompressionStrategy(),
                    FileCollection                 = fileCollection,
                    OpenUpToCommitUlong            = 9UL,
                    PreserveHistoryUpToCommitUlong = 9UL,
                    FileSplitSize                  = 100 * 1024 * 1024,
                    CompactorScheduler             = CompactorScheduler.Instance,
                };
                using (var kvDb = new KeyValueDB(options))
                    using (var objDb = new ObjectDB())
                    {
                        objDb.Open(kvDb, false);

                        using (var tr = objDb.StartReadOnlyTransaction())
                        {
                            Assert.Equal(9UL, tr.GetCommitUlong());
                        }
                        for (ulong i = 10; i < 200; i += 5)
                        {
                            using (var tr = objDb.StartWritingTransaction().Result)
                            {
                                var person = tr.Singleton <Person>();
                                person.Age = (uint)i;
                                tr.Store(person);
                                tr.SetCommitUlong(i);
                                tr.Commit();
                            }
                        }
                    }

                options = new KeyValueDBOptions
                {
                    Compression                    = new SnappyCompressionStrategy(),
                    FileCollection                 = fileCollection,
                    FileSplitSize                  = 100 * 1024 * 1024,
                    OpenUpToCommitUlong            = 50UL,
                    PreserveHistoryUpToCommitUlong = 50UL,
                    CompactorScheduler             = CompactorScheduler.Instance,
                };
                using (var kvDb = new KeyValueDB(options))
                    using (var objDb = new ObjectDB())
                    {
                        objDb.Open(kvDb, false);

                        using (var tr = objDb.StartReadOnlyTransaction())
                        {
                            Assert.Equal(50UL, tr.GetCommitUlong());
                        }
                    }
            }
        }
Example #8
0
 public override IKeyValueDB NewKeyValueDB(KeyValueDBOptions options)
 {
     return(new KeyValueDB(options));
 }
Example #9
0
 public override IKeyValueDB NewKeyValueDB(KeyValueDBOptions options)
 {
     options.Allocator = _allocator;
     return(new BTreeKeyValueDB(options));
 }
Example #10
0
 protected abstract IKeyValueDB NewKeyValueDB(KeyValueDBOptions options);
Example #11
0
 protected override IKeyValueDB NewKeyValueDB(KeyValueDBOptions options)
 {
     return(new KeyValueDB(options));
 }