public void TestGetOrAddRange()
{
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.CreateNew, planeDBOptions)) {
foreach (var keyValuePair in db.GetOrAddRange(Enumerable.Range(0, COUNT).Where(i => (i % 2) == 0).Select(
i => new KeyValuePair <byte[], byte[]>(Encoding.UTF8.GetBytes(i.ToString()),
Encoding.UTF8.GetBytes(i.ToString() + i)))))
{
var key = Encoding.UTF8.GetString(keyValuePair.Key);
var val = Encoding.UTF8.GetString(keyValuePair.Value);
Assert.AreEqual(key + key, val);
}
}
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.OpenOrCreate, planeDBOptions)) {
foreach (var keyValuePair in db.GetOrAddRange(Enumerable.Range(0, COUNT).Select(i => Encoding.UTF8.GetBytes(i.ToString())), bytes => Encoding.UTF8.GetBytes(Encoding.UTF8.GetString(bytes) + "i")))
{
var key = Encoding.UTF8.GetString(keyValuePair.Key);
var i = int.Parse(key);
var val = Encoding.UTF8.GetString(keyValuePair.Value);
if ((i % 2) == 0)
{
Assert.AreEqual(key + key, val);
}
else
{
Assert.AreEqual(key + "i", val);
}
}
}
}
public void TestSetConcurrentSharedLock(int concurrency)
{
var options = planeDBOptions.UsingDefaultLock();
int j = 0;
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.CreateNew,
options.UsingTableSpace(concurrency.ToString()))) {
void Adder()
{
int i;
while ((i = Interlocked.Increment(ref j)) < COUNT)
{
var k = Encoding.UTF8.GetBytes(i.ToString());
var v = Encoding.UTF8.GetBytes(i.ToString() + i + i + i + i);
db[k] = v;
}
}
var threads = Enumerable.Range(0, concurrency).Select(_ => new Thread(Adder)).ToArray();
foreach (var thread in threads)
{
thread.Start();
}
foreach (var thread in threads)
{
thread.Join();
}
}
j = 0;
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open,
options.UsingTableSpace(concurrency.ToString()))) {
void Reader()
{
int i;
while ((i = Interlocked.Increment(ref j)) < COUNT)
{
var k = Encoding.UTF8.GetBytes(i.ToString());
var v = Encoding.UTF8.GetString(db[k]);
Assert.AreEqual(v, i.ToString() + i + i + i + i);
}
}
var threads = Enumerable.Range(0, concurrency).Select(_ => new Thread(Reader)).ToArray();
foreach (var thread in threads)
{
thread.Start();
}
foreach (var thread in threads)
{
thread.Join();
}
}
}
public void TestBasicIterators()
{
var di = new DirectoryInfo("testdb");
using (var db = new PlaneDB(di, FileMode.CreateNew, planeDBOptions.DisableJournal())) {
var value = new byte[100];
for (var i = 0; i < 10000; ++i)
{
Assert.IsTrue(db.TryAdd(BitConverter.GetBytes(i), value));
}
}
using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) {
db.Compact();
// XXX Verify table count and sequence are correct
}
int read;
using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) {
Assert.AreEqual(db.Count, 10000);
Assert.IsTrue(db.TryRemove(BitConverter.GetBytes(1000), out _));
Assert.AreEqual(db.Count, 9999);
Assert.IsFalse(db.TryRemove(BitConverter.GetBytes(1000), out _));
read = db.Select((e, i) => new KeyValuePair <byte[], int>(e.Key, i)).Count();
Assert.AreEqual(db.Count, read);
}
using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) {
Assert.AreEqual(db.Count, 9999);
Assert.IsFalse(db.TryRemove(BitConverter.GetBytes(1000), out _));
Assert.AreEqual(db.Count, 9999);
read = db.KeysIterator.Select((e, i) => new KeyValuePair <byte[], int>(e, i)).Count();
Assert.AreEqual(db.Count, read);
}
using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) {
Assert.AreEqual(db.Count, 9999);
Assert.IsFalse(db.TryRemove(BitConverter.GetBytes(1000), out _));
Assert.AreEqual(db.Count, 9999);
read = db.Keys.Select((e, i) => new KeyValuePair <byte[], int>(e, i)).Count();
Assert.AreEqual(db.Count, read);
}
using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) {
Assert.AreEqual(db.Count, read);
db.Clear();
Assert.AreEqual(db.Count, 0);
}
using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions.DisableJournal())) {
Assert.AreEqual(db.Count, 0);
}
}
private static void VerifyDB(DirectoryInfo di, PlaneDBOptions tableoptions)
{
using var db = new PlaneDB(di, FileMode.Open, tableoptions);
for (var i = 0; i < 10000; ++i)
{
if (!db.TryGetValue(BitConverter.GetBytes(i), out _))
{
throw new KeyNotFoundException();
}
}
}
private static void CopyFromRocks(PlaneDB plane, Iterator iter)
{
for (iter.SeekToFirst(); iter.Valid(); iter.Next())
{
plane.Set(iter.Key(), iter.Value());
copyCount++;
if (copyCount > 0 && copyCount % 10_000 == 0)
{
Console.WriteLine($"{copyCount:N0} entries copied...");
}
}
}
private static void FillDB(DirectoryInfo di, PlaneDBOptions tableoptions)
{
using var db = new PlaneDB(di, FileMode.CreateNew, tableoptions);
var value = Enumerable.Repeat(Encoding.UTF8.GetBytes("hello world"), 512).SelectMany(i => i).ToArray();
Console.WriteLine($"Writing len={value.Length:N0}");
for (var i = 0; i < 10000; ++i)
{
if (!db.TryAdd(BitConverter.GetBytes(i), value))
{
throw new IOException("oops");
}
}
}
public void TestRemoveOrphans()
{
var di = new DirectoryInfo("testdb");
using (var db = new PlaneDB(di, FileMode.CreateNew, planeDBOptions.DisableJournal())) {
var value = new byte[100];
for (var i = 0; i < 10; ++i)
{
Assert.IsTrue(db.TryAdd(BitConverter.GetBytes(i), value));
}
}
var junk = Path.Combine(di.FullName, "default-119191919191.planedb");
File.WriteAllText(junk, "test");
using (new PlaneDB(di, FileMode.OpenOrCreate, planeDBOptions)) {
Assert.IsFalse(File.Exists(junk));
}
}
private static void CompactOne(DirectoryInfo db, PlaneDBOptions popts)
{
var id = $"{db.FullName}:{popts.TableSpace}";
try {
Options.Write($"{id} - Opening");
using var plane = new PlaneDB(db, FileMode.Open, popts);
plane.OnFlushMemoryTable += (_, __) => Options.Error($"{id} - Flushed memory table");
plane.OnMergedTables += (_, __) => Options.Error($"{id} - Merged tables");
Options.Write($"{id} - Starting compaction");
plane.Compact();
Options.Write($"{id} - Finished compaction");
}
catch (BadMagicException ex) {
Options.Error($"{id} - {ex.Message}");
}
catch (AlreadyLockedException ex) {
Options.Error($"{id} - {ex.Message}");
}
}
public override void Execute()
{
var popts = new PlaneDBOptions().DisableJournal();
if (!string.IsNullOrEmpty(Owner.Passphrase))
{
popts = popts.EnableEncryption(Owner.Passphrase);
}
else if (Owner.Compressed)
{
popts = popts.EnableCompression();
}
if (!string.IsNullOrEmpty(Owner.Tablespace))
{
popts = popts.UsingTableSpace(Owner.Tablespace);
}
if (From == null)
{
throw new GetOptException("No from");
}
if (To == null)
{
throw new GetOptException("No to");
}
using var rocks = RocksDb.OpenReadOnly(new DbOptions(), From.FullName, false);
using var plane = new PlaneDB(To, FileMode.OpenOrCreate, popts);
plane.OnFlushMemoryTable += (_, __) => Console.WriteLine("Flushed memory table");
plane.OnMergedTables += (_, __) => Console.WriteLine("Merged tables");
plane.Clear();
var iter = rocks.NewIterator();
plane.MassInsert(() => CopyFromRocks(plane, iter));
Console.WriteLine($"{copyCount:N0} entries copied in total");
}
public void TestSetMostlySync()
{
var options = planeDBOptions.MakeMostlySync();
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.CreateNew, options)) {
for (var i = 0; i < 100; ++i)
{
var k = Encoding.UTF8.GetBytes(i.ToString());
var v = Encoding.UTF8.GetBytes(i.ToString() + i + i + i + i);
db[k] = v;
}
}
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, options)) {
for (var i = 0; i < 100; ++i)
{
var k = Encoding.UTF8.GetBytes(i.ToString());
var v = Encoding.UTF8.GetString(db[k]);
Assert.AreEqual(v, i.ToString() + i + i + i + i);
}
}
}
public void TestLargeish()
{
var di = new DirectoryInfo("testdb");
var value = new byte[4096];
value[0] = 1;
value[4095] = 0xff;
using (var db = new PlaneDB(di, FileMode.CreateNew, planeDBOptions)) {
for (var i = 0; i < 10000; ++i)
{
Assert.IsTrue(db.TryAdd(BitConverter.GetBytes(i), value));
}
}
using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions)) {
for (var i = 0; i < 10000; ++i)
{
Assert.IsTrue(db.TryGetValue(BitConverter.GetBytes(i), out var val));
Assert.IsTrue(value.AsSpan().SequenceEqual(val));
}
}
using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions)) {
Assert.AreEqual(db.Count, 10000);
Assert.IsTrue(db.TryRemove(BitConverter.GetBytes(1000), out _));
Assert.AreEqual(db.Count, 9999);
Assert.IsFalse(db.TryRemove(BitConverter.GetBytes(1000), out _));
var read = db.Select((e, i) => new KeyValuePair <byte[], int>(e.Key, i)).Count();
Assert.AreEqual(db.Count, read);
db.Clear();
Assert.AreEqual(db.Count, 0);
}
using (var db = new PlaneDB(di, FileMode.Open, planeDBOptions)) {
Assert.AreEqual(db.Count, 0);
}
}
private void InfoOne(DirectoryInfo db, PlaneDBOptions popts)
{
infos.Clear();
try {
var stop = new Stopwatch();
stop.Start();
Options.Write($"{db.FullName}:{popts.TableSpace} - Querying");
using var plane = new PlaneDB(db, FileMode.Open, popts);
stop.Stop();
Add("General Information");
Add("Time To Open", stop.Elapsed.ToString());
Add("Location", plane.Location.FullName);
Add("Tablespace", plane.TableSpace);
if (Verbose)
{
Add("Table Files", plane.CurrentTableCount);
Add("Index Blocks", plane.CurrentIndexBlockCount);
foreach (var(key, value) in plane.AllLevels)
{
Add($"Level {key}", string.Join(", ", value));
}
}
var currentDiskSize = plane.CurrentDiskSize;
var currentRealSize = plane.CurrentRealSize;
Add("Sizes");
AddByte("Disk Size", currentDiskSize);
AddByte("Raw Size", currentRealSize);
if (Verbose)
{
Add("Bloom Bits", plane.CurrentBloomBits);
AddByte("Bloom Bytes (In Memory)", (long)Math.Ceiling(plane.CurrentBloomBits / 8.0));
}
stop.Reset();
stop.Start();
Add("Item Information");
var count = 0;
var minK = int.MaxValue;
var maxK = 0;
var minV = int.MaxValue;
var maxV = 0;
long allK = 0;
long allV = 0;
long inlined = 0;
if (!Verbose)
{
count = plane.Count;
}
else
{
foreach (var(key, value) in plane)
{
++count;
var kl = key.Length;
var vl = value.Length;
if (vl <= 9)
{
++inlined;
}
minK = Math.Min(kl, minK);
minV = Math.Min(vl, minV);
maxK = Math.Max(kl, maxK);
maxV = Math.Max(vl, maxV);
allK += kl;
allV += vl;
}
}
stop.Stop();
Add("Time To Gather Information", stop.Elapsed.ToString());
Add("Item count", count);
if (Verbose && count > 0)
{
if (minK != maxK)
{
AddByte("Min Key Length", minK);
AddByte("Max Key Length", maxK);
Add("Average Key Length", allK / (double)count);
}
else
{
AddByte("Key Length", minK);
}
if (minV != maxV)
{
AddByte("Min Value Length", minV);
AddByte("Max Value Length", maxV);
Add("Average Value Length", allV / (double)count);
}
else
{
AddByte("Value Length", minV);
}
AddByte("Keys Length", allK);
AddByte("Values Length", allV);
Add("Inlined Values", inlined);
}
if (Verbose)
{
Add("Overheads (bookkeeping, bloom, etc)");
var datalen = allV + allK;
AddByte("Raw Data Length", datalen);
AddByte("Disk Overhead", currentDiskSize - datalen);
AddByte("Raw Overhead", currentRealSize - datalen);
Add("Disk factor", (double)currentDiskSize / datalen);
Add("Raw factor", (double)currentRealSize / datalen);
}
var maxlen = infos.Where(i => !string.IsNullOrEmpty(i.Value)).Max(i => i.Key.Length) + 4;
var maxvlen = infos.Max(i => i.Value.Length) + 2;
foreach (var(key, value) in infos)
{
if (string.IsNullOrEmpty(value))
{
var tlen = Math.Min(Console.WindowWidth - 2, maxlen + maxvlen);
var half = (int)Math.Ceiling((tlen - key.Length) / 2.0);
Console.WriteLine();
Console.WriteLine(
$"{key.PadLeft(key.Length + 1).PadRight(key.Length + 2).PadLeft(half + key.Length, '-').PadRight(tlen, '-')}");
Console.WriteLine();
continue;
}
Console.WriteLine($"{key.PadLeft(maxlen)}: {value}");
}
Console.WriteLine();
}
catch (BadMagicException ex) {
Options.Error($"{db.FullName}:{popts.TableSpace} - {ex.Message}");
}
catch (AlreadyLockedException ex) {
Options.Error($"{db.FullName}:{popts.TableSpace} - {ex.Message}");
}
}
public void TestSetThreadUnsafe()
{
var opts = planeDBOptions.DisableThreadSafety();
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.CreateNew, opts)) {
for (var i = 0; i < COUNT; ++i)
{
var k = Encoding.UTF8.GetBytes(i.ToString());
var v = Encoding.UTF8.GetBytes(i.ToString() + i + i + i + i);
db[k] = v;
}
}
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) {
for (var i = 0; i < COUNT; ++i)
{
var k = Encoding.UTF8.GetBytes(i.ToString());
var v = Encoding.UTF8.GetString(db[k]);
Assert.AreEqual(v, i.ToString() + i + i + i + i);
}
Assert.IsTrue(db.TryRemove(Encoding.UTF8.GetBytes("0"), out var removed));
Assert.AreEqual("00000", Encoding.UTF8.GetString(removed));
}
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) {
for (var i = 1; i < COUNT; ++i)
{
var k = Encoding.UTF8.GetBytes(i.ToString());
var v = Encoding.UTF8.GetString(db[k]);
Assert.IsTrue(db.ContainsKey(k));
Assert.AreEqual(v, i.ToString() + i + i + i + i);
}
}
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) {
for (var i = 0; i < COUNT; ++i)
{
var k = Encoding.UTF8.GetBytes(i.ToString());
if (i == 0)
{
Assert.IsFalse(db.ContainsKey(k));
Assert.IsFalse(db.TryGetValue(k, out _));
}
else
{
Assert.IsTrue(db.ContainsKey(k));
Assert.IsTrue(db.TryGetValue(k, out var v));
Assert.AreEqual(Encoding.UTF8.GetString(v), i.ToString() + i + i + i + i);
}
}
}
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) {
db.Add(Encoding.UTF8.GetBytes("test1"), Encoding.UTF8.GetBytes("test1"));
}
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) {
db.Add(Encoding.UTF8.GetBytes("test2"), Encoding.UTF8.GetBytes("test2"));
}
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) {
db.AddOrUpdate(Encoding.UTF8.GetBytes("test2"), Encoding.UTF8.GetBytes("test3"),
(_, __) => Encoding.UTF8.GetBytes("test3"));
}
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) {
db.Add(Encoding.UTF8.GetBytes("test3"), Encoding.UTF8.GetBytes("test4"));
}
using (var db = new PlaneDB(new DirectoryInfo("testdb"), FileMode.Open, opts)) {
foreach (var i in new[] {
new[] { "test1", "test1" }, new[] { "test2", "test3" }, new[] { "test3", "test4" }
})
{
Assert.IsTrue(db.ContainsKey(Encoding.UTF8.GetBytes(i[0])));
Assert.IsTrue(db.TryGetValue(Encoding.UTF8.GetBytes(i[0]), out var v));
Assert.AreEqual(i[1], Encoding.UTF8.GetString(v));
}
}
}
