public async Task Test_FdbKey_Batched()
{
// we want numbers from 0 to 999 split between 5 workers that will consume batches of 20 items at a time
// > we get 5 enumerables that all take ranges from the same pool and all complete where there is no more values
const int N = 1000;
const int B = 20;
const int W = 5;
var query = FdbKey.Batched(0, N, W, B);
Assert.That(query, Is.Not.Null);
var batches = query.ToArray();
Assert.That(batches, Is.Not.Null);
Assert.That(batches.Length, Is.EqualTo(W));
Assert.That(batches, Is.All.Not.Null);
var used = new bool[N];
var signal = new TaskCompletionSource <object>();
// each batch should return new numbers
var tasks = batches.Select(async(iterator, id) =>
{
// force async
await signal.Task.ConfigureAwait(false);
foreach (var chunk in iterator)
{
// kvp = (offset, count)
// > count should always be 20
// > offset should always be a multiple of 20
// > there should never be any overlap between workers
Assert.That(chunk.Value, Is.EqualTo(B), "{0}:{1}", chunk.Key, chunk.Value);
Assert.That(chunk.Key % B, Is.EqualTo(0), "{0}:{1}", chunk.Key, chunk.Value);
lock (used)
{
for (int i = chunk.Key; i < chunk.Key + chunk.Value; i++)
{
if (used[i])
{
Assert.Fail("Duplicate index {0} chunk {1}:{2} for worker {3}", i, chunk.Key, chunk.Value, id);
}
else
{
used[i] = true;
}
}
}
await Task.Delay(1).ConfigureAwait(false);
}
}).ToArray();
ThreadPool.UnsafeQueueUserWorkItem((_) => signal.TrySetResult(null), null);
await Task.WhenAll(tasks);
Assert.That(used, Is.All.True);
}
private static async Task BenchBulkInsertThenBulkReadAsync(IFdbDatabase db, int N, int K, int B, CancellationToken ct, bool instrumented = false)
{
// test that we can bulk write / bulk read
var timings = instrumented ? new List <KeyValuePair <double, double> >() : null;
// put test values inside a namespace
var subspace = db.Partition("BulkInsert");
// cleanup everything
using (var tr = db.BeginTransaction(ct))
{
tr.ClearRange(subspace);
await tr.CommitAsync();
}
// insert all values (batched)
Console.WriteLine("Inserting " + N.ToString("N0", CultureInfo.InvariantCulture) + " keys: ");
var insert = Stopwatch.StartNew();
int batches = 0;
long bytes = 0;
var start = Stopwatch.StartNew();
var tasks = new List <Task>();
foreach (var worker in FdbKey.Batched(0, N, K, B))
{
//hack
tasks.Add(Task.Run(async() =>
{
foreach (var chunk in worker)
{
using (var tr = db.BeginTransaction(ct))
{
int z = 0;
foreach (int i in Enumerable.Range(chunk.Key, chunk.Value))
{
tr.Set(subspace.Pack(i), Slice.Create(new byte[256]));
z++;
}
//Console.Write("#");
//Console.WriteLine(" Commiting batch (" + tr.Size.ToString("N0", CultureInfo.InvariantCulture) + " bytes) " + z + " keys");
var localStart = start.Elapsed.TotalSeconds;
await tr.CommitAsync();
var localDuration = start.Elapsed.TotalSeconds - localStart;
if (instrumented)
{
lock (timings) { timings.Add(new KeyValuePair <double, double>(localStart, localDuration)); }
}
Interlocked.Increment(ref batches);
Interlocked.Add(ref bytes, tr.Size);
}
}
}, ct));
}
await Task.WhenAll(tasks);
insert.Stop();
Console.WriteLine("Committed " + batches + " batches in " + FormatTimeMilli(insert.Elapsed.TotalMilliseconds) + " (" + FormatTimeMilli(insert.Elapsed.TotalMilliseconds / batches) + " / batch, " + FormatTimeMicro(insert.Elapsed.TotalMilliseconds / N) + " / item");
Console.WriteLine("Throughput " + FormatThroughput(bytes, insert.Elapsed.TotalSeconds));
if (instrumented)
{
var sb = new StringBuilder();
foreach (var kvp in timings)
{
sb.Append(kvp.Key.ToString()).Append(';').Append((kvp.Key + kvp.Value).ToString()).Append(';').Append(kvp.Value.ToString()).AppendLine();
}
#if DEBUG
System.IO.File.WriteAllText(@"c:\temp\fdb\timings_" + N + "_" + K + "_" + B + ".csv", sb.ToString());
#else
Console.WriteLine(sb.ToString());
#endif
}
// Read values
using (var tr = db.BeginTransaction(ct))
{
Console.WriteLine("Reading all keys...");
var sw = Stopwatch.StartNew();
var items = await tr.GetRangeStartsWith(subspace).ToListAsync();
sw.Stop();
Console.WriteLine("Took " + FormatTimeMilli(sw.Elapsed.TotalMilliseconds) + " to get " + items.Count.ToString("N0", CultureInfo.InvariantCulture) + " results");
}
}