public void TestGetOrAddRange() { using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.CreateNew, planeDBOptions)) { foreach (var keyValuePair in db.GetOrAddRange(Enumerable.Range(0, COUNT).Where(i => (i % 2) == 0).Select( i => new KeyValuePair <byte[], byte[]>(Encoding.UTF8.GetBytes(i.ToString()), Encoding.UTF8.GetBytes(i.ToString() + i))))) { var key = Encoding.UTF8.GetString(keyValuePair.Key); var val = Encoding.UTF8.GetString(keyValuePair.Value); Assert.AreEqual(key + key, val); } } using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.OpenOrCreate, planeDBOptions)) { foreach (var keyValuePair in db.GetOrAddRange(Enumerable.Range(0, COUNT).Select(i => Encoding.UTF8.GetBytes(i.ToString())), bytes => Encoding.UTF8.GetBytes(Encoding.UTF8.GetString(bytes) + "i"))) { var key = Encoding.UTF8.GetString(keyValuePair.Key); var i = int.Parse(key); var val = Encoding.UTF8.GetString(keyValuePair.Value); if ((i % 2) == 0) { Assert.AreEqual(key + key, val); } else { Assert.AreEqual(key + "i", val); } } } }
public void TestSetConcurrentSharedLock(int concurrency) { var options = planeDBOptions.UsingDefaultLock(); int j = 0; using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.CreateNew, options.UsingTableSpace(concurrency.ToString()))) { void Adder() { int i; while ((i = Interlocked.Increment(ref j)) < COUNT) { var k = Encoding.UTF8.GetBytes(i.ToString()); var v = Encoding.UTF8.GetBytes(i.ToString() + i + i + i + i); db[k] = v; } } var threads = Enumerable.Range(0, concurrency).Select(_ => new Thread(Adder)).ToArray(); foreach (var thread in threads) { thread.Start(); } foreach (var thread in threads) { thread.Join(); } } j = 0; using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, options.UsingTableSpace(concurrency.ToString()))) { void Reader() { int i; while ((i = Interlocked.Increment(ref j)) < COUNT) { var k = Encoding.UTF8.GetBytes(i.ToString()); var v = Encoding.UTF8.GetString(db[k]); Assert.AreEqual(v, i.ToString() + i + i + i + i); } } var threads = Enumerable.Range(0, concurrency).Select(_ => new Thread(Reader)).ToArray(); foreach (var thread in threads) { thread.Start(); } foreach (var thread in threads) { thread.Join(); } } }
public void TestBasicIterators() { var di = new DirectoryInfo("testdb"); using (var db = new PlaneDB(di, FileMode.CreateNew, planeDBOptions.DisableJournal())) { var value = new byte[100]; for (var i = 0; i < 10000; ++i) { Assert.IsTrue(db.TryAdd(BitConverter.GetBytes(i), value)); } } using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) { db.Compact(); // XXX Verify table count and sequence are correct } int read; using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) { Assert.AreEqual(db.Count, 10000); Assert.IsTrue(db.TryRemove(BitConverter.GetBytes(1000), out _)); Assert.AreEqual(db.Count, 9999); Assert.IsFalse(db.TryRemove(BitConverter.GetBytes(1000), out _)); read = db.Select((e, i) => new KeyValuePair <byte[], int>(e.Key, i)).Count(); Assert.AreEqual(db.Count, read); } using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) { Assert.AreEqual(db.Count, 9999); Assert.IsFalse(db.TryRemove(BitConverter.GetBytes(1000), out _)); Assert.AreEqual(db.Count, 9999); read = db.KeysIterator.Select((e, i) => new KeyValuePair <byte[], int>(e, i)).Count(); Assert.AreEqual(db.Count, read); } using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) { Assert.AreEqual(db.Count, 9999); Assert.IsFalse(db.TryRemove(BitConverter.GetBytes(1000), out _)); Assert.AreEqual(db.Count, 9999); read = db.Keys.Select((e, i) => new KeyValuePair <byte[], int>(e, i)).Count(); Assert.AreEqual(db.Count, read); } using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) { Assert.AreEqual(db.Count, read); db.Clear(); Assert.AreEqual(db.Count, 0); } using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) { Assert.AreEqual(db.Count, 0); } }
private static void VerifyDB(DirectoryInfo di, PlaneDBOptions tableoptions) { using var db = new PlaneDB(di, FileMode.Open, tableoptions); for (var i = 0; i < 10000; ++i) { if (!db.TryGetValue(BitConverter.GetBytes(i), out _)) { throw new KeyNotFoundException(); } } }
private static void CopyFromRocks(PlaneDB plane, Iterator iter) { for (iter.SeekToFirst(); iter.Valid(); iter.Next()) { plane.Set(iter.Key(), iter.Value()); copyCount++; if (copyCount > 0 && copyCount % 10_000 == 0) { Console.WriteLine($"{copyCount:N0} entries copied..."); } } }
private static void FillDB(DirectoryInfo di, PlaneDBOptions tableoptions) { using var db = new PlaneDB(di, FileMode.CreateNew, tableoptions); var value = Enumerable.Repeat(Encoding.UTF8.GetBytes("hello world"), 512).SelectMany(i => i).ToArray(); Console.WriteLine($"Writing len={value.Length:N0}"); for (var i = 0; i < 10000; ++i) { if (!db.TryAdd(BitConverter.GetBytes(i), value)) { throw new IOException("oops"); } } }
public void TestRemoveOrphans() { var di = new DirectoryInfo("testdb"); using (var db = new PlaneDB(di, FileMode.CreateNew, planeDBOptions.DisableJournal())) { var value = new byte[100]; for (var i = 0; i < 10; ++i) { Assert.IsTrue(db.TryAdd(BitConverter.GetBytes(i), value)); } } var junk = Path.Combine(di.FullName, "default-119191919191.planedb"); File.WriteAllText(junk, "test"); using (new PlaneDB(di, FileMode.OpenOrCreate, planeDBOptions)) { Assert.IsFalse(File.Exists(junk)); } }
private static void CompactOne(DirectoryInfo db, PlaneDBOptions popts) { var id = $"{db.FullName}:{popts.TableSpace}"; try { Options.Write($"{id} - Opening"); using var plane = new PlaneDB(db, FileMode.Open, popts); plane.OnFlushMemoryTable += (_, __) => Options.Error($"{id} - Flushed memory table"); plane.OnMergedTables += (_, __) => Options.Error($"{id} - Merged tables"); Options.Write($"{id} - Starting compaction"); plane.Compact(); Options.Write($"{id} - Finished compaction"); } catch (BadMagicException ex) { Options.Error($"{id} - {ex.Message}"); } catch (AlreadyLockedException ex) { Options.Error($"{id} - {ex.Message}"); } }
public override void Execute() { var popts = new PlaneDBOptions().DisableJournal(); if (!string.IsNullOrEmpty(Owner.Passphrase)) { popts = popts.EnableEncryption(Owner.Passphrase); } else if (Owner.Compressed) { popts = popts.EnableCompression(); } if (!string.IsNullOrEmpty(Owner.Tablespace)) { popts = popts.UsingTableSpace(Owner.Tablespace); } if (From == null) { throw new GetOptException("No from"); } if (To == null) { throw new GetOptException("No to"); } using var rocks = RocksDb.OpenReadOnly(new DbOptions(), From.FullName, false); using var plane = new PlaneDB(To, FileMode.OpenOrCreate, popts); plane.OnFlushMemoryTable += (_, __) => Console.WriteLine("Flushed memory table"); plane.OnMergedTables += (_, __) => Console.WriteLine("Merged tables"); plane.Clear(); var iter = rocks.NewIterator(); plane.MassInsert(() => CopyFromRocks(plane, iter)); Console.WriteLine($"{copyCount:N0} entries copied in total"); }
public void TestSetMostlySync() { var options = planeDBOptions.MakeMostlySync(); using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.CreateNew, options)) { for (var i = 0; i < 100; ++i) { var k = Encoding.UTF8.GetBytes(i.ToString()); var v = Encoding.UTF8.GetBytes(i.ToString() + i + i + i + i); db[k] = v; } } using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, options)) { for (var i = 0; i < 100; ++i) { var k = Encoding.UTF8.GetBytes(i.ToString()); var v = Encoding.UTF8.GetString(db[k]); Assert.AreEqual(v, i.ToString() + i + i + i + i); } } }
public void TestLargeish() { var di = new DirectoryInfo("testdb"); var value = new byte[4096]; value[0] = 1; value[4095] = 0xff; using (var db = new PlaneDB(di, FileMode.CreateNew, planeDBOptions)) { for (var i = 0; i < 10000; ++i) { Assert.IsTrue(db.TryAdd(BitConverter.GetBytes(i), value)); } } using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions)) { for (var i = 0; i < 10000; ++i) { Assert.IsTrue(db.TryGetValue(BitConverter.GetBytes(i), out var val)); Assert.IsTrue(value.AsSpan().SequenceEqual(val)); } } using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions)) { Assert.AreEqual(db.Count, 10000); Assert.IsTrue(db.TryRemove(BitConverter.GetBytes(1000), out _)); Assert.AreEqual(db.Count, 9999); Assert.IsFalse(db.TryRemove(BitConverter.GetBytes(1000), out _)); var read = db.Select((e, i) => new KeyValuePair <byte[], int>(e.Key, i)).Count(); Assert.AreEqual(db.Count, read); db.Clear(); Assert.AreEqual(db.Count, 0); } using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions)) { Assert.AreEqual(db.Count, 0); } }
private void InfoOne(DirectoryInfo db, PlaneDBOptions popts) { infos.Clear(); try { var stop = new Stopwatch(); stop.Start(); Options.Write($"{db.FullName}:{popts.TableSpace} - Querying"); using var plane = new PlaneDB(db, FileMode.Open, popts); stop.Stop(); Add("General Information"); Add("Time To Open", stop.Elapsed.ToString()); Add("Location", plane.Location.FullName); Add("Tablespace", plane.TableSpace); if (Verbose) { Add("Table Files", plane.CurrentTableCount); Add("Index Blocks", plane.CurrentIndexBlockCount); foreach (var(key, value) in plane.AllLevels) { Add($"Level {key}", string.Join(", ", value)); } } var currentDiskSize = plane.CurrentDiskSize; var currentRealSize = plane.CurrentRealSize; Add("Sizes"); AddByte("Disk Size", currentDiskSize); AddByte("Raw Size", currentRealSize); if (Verbose) { Add("Bloom Bits", plane.CurrentBloomBits); AddByte("Bloom Bytes (In Memory)", (long)Math.Ceiling(plane.CurrentBloomBits / 8.0)); } stop.Reset(); stop.Start(); Add("Item Information"); var count = 0; var minK = int.MaxValue; var maxK = 0; var minV = int.MaxValue; var maxV = 0; long allK = 0; long allV = 0; long inlined = 0; if (!Verbose) { count = plane.Count; } else { foreach (var(key, value) in plane) { ++count; var kl = key.Length; var vl = value.Length; if (vl <= 9) { ++inlined; } minK = Math.Min(kl, minK); minV = Math.Min(vl, minV); maxK = Math.Max(kl, maxK); maxV = Math.Max(vl, maxV); allK += kl; allV += vl; } } stop.Stop(); Add("Time To Gather Information", stop.Elapsed.ToString()); Add("Item count", count); if (Verbose && count > 0) { if (minK != maxK) { AddByte("Min Key Length", minK); AddByte("Max Key Length", maxK); Add("Average Key Length", allK / (double)count); } else { AddByte("Key Length", minK); } if (minV != maxV) { AddByte("Min Value Length", minV); AddByte("Max Value Length", maxV); Add("Average Value Length", allV / (double)count); } else { AddByte("Value Length", minV); } AddByte("Keys Length", allK); AddByte("Values Length", allV); Add("Inlined Values", inlined); } if (Verbose) { Add("Overheads (bookkeeping, bloom, etc)"); var datalen = allV + allK; AddByte("Raw Data Length", datalen); AddByte("Disk Overhead", currentDiskSize - datalen); AddByte("Raw Overhead", currentRealSize - datalen); Add("Disk factor", (double)currentDiskSize / datalen); Add("Raw factor", (double)currentRealSize / datalen); } var maxlen = infos.Where(i => !string.IsNullOrEmpty(i.Value)).Max(i => i.Key.Length) + 4; var maxvlen = infos.Max(i => i.Value.Length) + 2; foreach (var(key, value) in infos) { if (string.IsNullOrEmpty(value)) { var tlen = Math.Min(Console.WindowWidth - 2, maxlen + maxvlen); var half = (int)Math.Ceiling((tlen - key.Length) / 2.0); Console.WriteLine(); Console.WriteLine( $"{key.PadLeft(key.Length + 1).PadRight(key.Length + 2).PadLeft(half + key.Length, '-').PadRight(tlen, '-')}"); Console.WriteLine(); continue; } Console.WriteLine($"{key.PadLeft(maxlen)}: {value}"); } Console.WriteLine(); } catch (BadMagicException ex) { Options.Error($"{db.FullName}:{popts.TableSpace} - {ex.Message}"); } catch (AlreadyLockedException ex) { Options.Error($"{db.FullName}:{popts.TableSpace} - {ex.Message}"); } }
public void TestSetThreadUnsafe() { var opts = planeDBOptions.DisableThreadSafety(); using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.CreateNew, opts)) { for (var i = 0; i < COUNT; ++i) { var k = Encoding.UTF8.GetBytes(i.ToString()); var v = Encoding.UTF8.GetBytes(i.ToString() + i + i + i + i); db[k] = v; } } using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) { for (var i = 0; i < COUNT; ++i) { var k = Encoding.UTF8.GetBytes(i.ToString()); var v = Encoding.UTF8.GetString(db[k]); Assert.AreEqual(v, i.ToString() + i + i + i + i); } Assert.IsTrue(db.TryRemove(Encoding.UTF8.GetBytes("0"), out var removed)); Assert.AreEqual("00000", Encoding.UTF8.GetString(removed)); } using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) { for (var i = 1; i < COUNT; ++i) { var k = Encoding.UTF8.GetBytes(i.ToString()); var v = Encoding.UTF8.GetString(db[k]); Assert.IsTrue(db.ContainsKey(k)); Assert.AreEqual(v, i.ToString() + i + i + i + i); } } using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) { for (var i = 0; i < COUNT; ++i) { var k = Encoding.UTF8.GetBytes(i.ToString()); if (i == 0) { Assert.IsFalse(db.ContainsKey(k)); Assert.IsFalse(db.TryGetValue(k, out _)); } else { Assert.IsTrue(db.ContainsKey(k)); Assert.IsTrue(db.TryGetValue(k, out var v)); Assert.AreEqual(Encoding.UTF8.GetString(v), i.ToString() + i + i + i + i); } } } using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) { db.Add(Encoding.UTF8.GetBytes("test1"), Encoding.UTF8.GetBytes("test1")); } using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) { db.Add(Encoding.UTF8.GetBytes("test2"), Encoding.UTF8.GetBytes("test2")); } using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) { db.AddOrUpdate(Encoding.UTF8.GetBytes("test2"), Encoding.UTF8.GetBytes("test3"), (_, __) => Encoding.UTF8.GetBytes("test3")); } using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) { db.Add(Encoding.UTF8.GetBytes("test3"), Encoding.UTF8.GetBytes("test4")); } using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) { foreach (var i in new[] { new[] { "test1", "test1" }, new[] { "test2", "test3" }, new[] { "test3", "test4" } }) { Assert.IsTrue(db.ContainsKey(Encoding.UTF8.GetBytes(i[0]))); Assert.IsTrue(db.TryGetValue(Encoding.UTF8.GetBytes(i[0]), out var v)); Assert.AreEqual(i[1], Encoding.UTF8.GetString(v)); } } }