private static async Task CheckGetKeys(string dbFolder)
{
using var blockStore = new Core.Storage.Blocks.CachePrototype.ProtoSharded(2_560);
await using var db = new TrimDatabase(() => new SkipList32(new NativeAllocator32(4096 * 1024, 25)), blockStore, 5, dbFolder, 0);
await db.LoadAsync(startWithoutMerges : true);
var numberOfThreads = Environment.ProcessorCount;
var tasks = new Task[numberOfThreads];
var seed = 7722;
var generator = new KeyValueGenerator(numberOfThreads, seed);
Console.WriteLine("Starting the get test");
var sw = Stopwatch.StartNew();
for (var i = 0; i < numberOfThreads; i++)
{
tasks[i] = ReadFromDB((short)i, generator, _keysPerThread, db);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine($"Total time taken {sw.ElapsedMilliseconds}ms");
var timePerKey = (double)sw.ElapsedMilliseconds / (_keysPerThread * numberOfThreads);
Console.WriteLine($"Time taken per key {timePerKey * 1000.0}µs");
Console.WriteLine($"Total misses {_numberOfMisses}");
}
private static async Task CheckGetKeysSingleThread(string dbFolder)
{
var dbOptions = new TrimDatabaseOptions()
{
DatabaseFolder = dbFolder, OpenReadOnly = true
};
await using var db = new TrimDatabase(dbOptions);
await db.LoadAsync();
var numberOfThreads = Environment.ProcessorCount;
var seed = 7722;
var generator = new KeyValueGenerator(numberOfThreads, seed);
var key = new byte[10];
var value = new byte[100];
for (var t = 0; t < numberOfThreads; t++)
{
for (var i = 0; i < _keysPerThread; i++)
{
//if (i == 7) Debugger.Break();
generator.GetKeyValue(key, value, (short)t, i);
Console.WriteLine($"Thread Id {t} iteration {i}");
await db.GetAsync(key);
}
}
}
private static async Task CheckGetKeysSingleThread(string dbFolder)
{
using var blockStore = new Core.Storage.Blocks.CachePrototype.ProtoSharded(2_560);
await using var db = new TrimDatabase(() => new SkipList32(new NativeAllocator32(4096 * 1024, 25)), blockStore, 5, dbFolder, 0);
await db.LoadAsync(startWithoutMerges : true);
var numberOfThreads = Environment.ProcessorCount;
var seed = 7722;
var generator = new KeyValueGenerator(numberOfThreads, seed);
var key = new byte[10];
var value = new byte[100];
for (var t = 0; t < numberOfThreads; t++)
{
for (var i = 0; i < _keysPerThread; i++)
{
//if (i == 7) Debugger.Break();
generator.GetKeyValue(key, value, (short)t, i);
Console.WriteLine($"Thread Id {t} iteration {i}");
await db.GetAsync(key);
}
}
}
private static async Task WriteToDB(short threadId, KeyValueGenerator generator, int numberOfIterations, TrimDatabase trimDB)
{
await Task.Yield();
var keyMemory = new byte[_keySize];
var valueMemory = new byte[_valueSize];
for (var i = 0; i < numberOfIterations; i++)
{
generator.GetKeyValue(keyMemory.AsSpan(), valueMemory.AsSpan(), threadId, numberOfIterations);
await trimDB.PutAsync(keyMemory, valueMemory);
}
}
private static async Task WriteDB(string dbFolder)
{
System.IO.Directory.Delete(dbFolder, true);
System.IO.Directory.CreateDirectory(dbFolder);
using var blockStore = new Core.Storage.Blocks.CachePrototype.ProtoSharded(2_560);
await using var db = new TrimDatabase(() => new SkipList32(new NativeAllocator32(4096 * 1024, 25)), blockStore, 5, dbFolder, 64 * 1024 * 1024);
await db.LoadAsync();
var numberOfThreads = Environment.ProcessorCount;
var tasks = new Task[numberOfThreads];
var seed = 7722;
var generator = new KeyValueGenerator(numberOfThreads, seed);
var sw = Stopwatch.StartNew();
for (var i = 0; i < numberOfThreads; i++)
{
tasks[i] = WriteToDB((short)i, generator, _keysPerThread, db);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine($"Total time taken {sw.ElapsedMilliseconds}ms");
Console.WriteLine($"Total number of keys written {_keysPerThread * numberOfThreads}");
var totalDatasize = _keysPerThread * numberOfThreads * (_keySize + _valueSize);
Console.WriteLine($"Total data set {totalDatasize / 1024 / 1204 }mb");
Console.WriteLine("Waiting for db to shutdown");
for (var i = 0; i < 10; i++)
{
Console.WriteLine("Waiting for the merges to finish BRB");
await Task.Delay(TimeSpan.FromSeconds(10));
}
}
private static async Task WriteDB(string dbFolder, bool disableMerging)
{
var dbOptions = new TrimDatabaseOptions()
{
DatabaseFolder = dbFolder, DisableMerging = disableMerging
};
await using var db = new TrimDatabase(dbOptions);
await db.LoadAsync();
var numberOfThreads = Environment.ProcessorCount;
var tasks = new Task[numberOfThreads];
var seed = 7722;
var generator = new KeyValueGenerator(numberOfThreads, seed);
var sw = Stopwatch.StartNew();
for (var i = 0; i < numberOfThreads; i++)
{
tasks[i] = WriteToDB((short)i, generator, _keysPerThread, db);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine($"Total time taken {sw.ElapsedMilliseconds}ms");
Console.WriteLine($"Total number of keys written {_keysPerThread * numberOfThreads}");
var totalDatasize = _keysPerThread * numberOfThreads * (_keySize + _valueSize);
Console.WriteLine($"Total data set {totalDatasize / 1024 / 1024 }mb");
Console.WriteLine("Waiting for db to shutdown");
//for (var i = 0; i < 10; i++)
//{
// Console.WriteLine("Waiting for the merges to finish BRB");
// await Task.Delay(TimeSpan.FromSeconds(10));
//}
}
private static async Task ReadFromDB(short threadId, KeyValueGenerator generator, int numberOfIterations, TrimDatabase trimDB)
{
await Task.Yield();
var keyMemory = new byte[_keySize];
var valueMemory = new byte[_valueSize];
for (var i = 0; i < numberOfIterations; i++)
{
generator.GetKeyValue(keyMemory.AsSpan(), valueMemory.AsSpan(), threadId, numberOfIterations);
var result = await trimDB.GetAsync(keyMemory);
if (result.IsEmpty)
{
Interlocked.Increment(ref _numberOfMisses);
}
//if ((i + 1) % 10 == 0) Console.WriteLine($"Thread {threadId} has read {i + 1} keys");
//if (threadId == 3 && i > 228) Debugger.Break(); // Console.WriteLine($"ERROR Thread {threadId} has read {i + 1} keys");
}
}
private static async Task CheckGetKeys(string dbFolder)
{
var dbOptions = new TrimDatabaseOptions()
{
DatabaseFolder = dbFolder, OpenReadOnly = true
};
await using var db = new TrimDatabase(dbOptions);
await db.LoadAsync();
var numberOfThreads = Environment.ProcessorCount;
var tasks = new Task[numberOfThreads];
var seed = 7722;
var generator = new KeyValueGenerator(numberOfThreads, seed);
Console.WriteLine("Starting the get test");
var sw = Stopwatch.StartNew();
for (var i = 0; i < numberOfThreads; i++)
{
tasks[i] = ReadFromDB((short)i, generator, _keysPerThread, db);
}
await Task.WhenAll(tasks);
sw.Stop();
Console.WriteLine($"Total time taken {sw.ElapsedMilliseconds}ms");
var timePerKey = (double)sw.ElapsedMilliseconds / (_keysPerThread * numberOfThreads);
Console.WriteLine($"Time taken per key {timePerKey * 1000.0}µs");
Console.WriteLine($"Total misses {_numberOfMisses}");
}
