/// <summary>
/// Signup a student to a class
/// </summary>
public async Task Signup(IFdbTransaction tr, string s, string c)
{
var rec = AttendsKey(s, c);
if ((await tr.GetAsync(rec)).IsPresent)
{ // already signed up
return;
}
int seatsLeft = Int32.Parse((await tr.GetAsync(ClassKey(c))).ToStringAscii());
if (seatsLeft <= 0)
{
throw new InvalidOperationException("No remaining seats");
}
var classes = await tr.GetRange(AttendsKeys(s)).ToListAsync();
if (classes.Count >= 5)
{
throw new InvalidOperationException("Too many classes");
}
tr.Set(ClassKey(c), Slice.FromStringAscii((seatsLeft - 1).ToString()));
tr.Set(rec, Slice.Empty);
}
/// <summary>Insert a new document in the collection</summary>
public async Task InsertAsync(IFdbTransaction trans, TDocument document)
{
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
if (document == null)
{
throw new ArgumentNullException(nameof(document));
}
var id = this.IdSelector(document);
if (id == null)
{
throw new InvalidOperationException("Cannot insert a document with a null identifier");
}
// encode the document
var packed = this.ValueEncoder.EncodeValue(document);
var subspace = await this.Location.Resolve(trans);
if (subspace == null)
{
throw new InvalidOperationException($"Location '{this.Location}' referenced by Document Collection Layer was not found.");
}
// Key Prefix = ...(id,)
var key = subspace.EncodePartial(id);
// clear previous value
trans.ClearRange(KeyRange.StartsWith(key));
int remaining = packed.Count;
if (remaining <= this.ChunkSize)
{ // stored as a single element
// Key = ...(id,)
trans.Set(key, packed);
}
else
{ // splits in as many chunks as necessary
// Key = ...(id, N) where N is the chunk index (0-based)
int p = 0;
int index = 0;
while (remaining > 0)
{
int sz = Math.Max(remaining, this.ChunkSize);
trans.Set(subspace[id, index], packed.Substring(p, sz));
++index;
p += sz;
remaining -= sz;
}
}
}
private Task Scenario3(IFdbTransaction tr)
{
var location = FdbSubspace.Create(Slice.FromAscii("TEST"));
tr.Set(location.Key + (byte)'a', Slice.FromAscii("A"));
tr.AtomicAdd(location.Key + (byte)'k', Slice.FromFixed32(1));
tr.Set(location.Key + (byte)'z', Slice.FromAscii("C"));
tr.ClearRange(location.Key + (byte)'a', location.Key + (byte)'k');
tr.ClearRange(location.Key + (byte)'k', location.Key + (byte)'z');
return(Task.FromResult <object>(null));
}
private Task Scenario3(IFdbTransaction tr)
{
var location = FdbSubspace.Create(Slice.FromAscii("TEST"));
tr.Set(location.Key + (byte)'a', Slice.FromAscii("A"));
tr.AtomicAdd(location.Key + (byte)'k', Slice.FromFixed32(1));
tr.Set(location.Key + (byte)'z', Slice.FromAscii("C"));
tr.ClearRange(location.Key + (byte)'a', location.Key + (byte)'k');
tr.ClearRange(location.Key + (byte)'k', location.Key + (byte)'z');
return Task.FromResult<object>(null);
}
private void StoreTask(IFdbTransaction tr, Slice taskId, DateTime scheduledUtc, Slice taskBody)
{
tr.Annotate("Writing task {0:P}", taskId);
var prefix = this.TaskStore.Partition.ByKey(taskId);
// store task body and timestamp
tr.Set(prefix.GetPrefix(), taskBody);
tr.Set(prefix.Keys.Encode(TASK_META_SCHEDULED), Slice.FromInt64(scheduledUtc.Ticks));
// increment total and pending number of tasks
this.Counters.Increment(tr, COUNTER_TOTAL_TASKS);
this.Counters.Increment(tr, COUNTER_PENDING_TASKS);
}
/// <summary>Insert a new document in the collection</summary>
public void Insert(IFdbTransaction trans, TDocument document)
{
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
if (document == null)
{
throw new ArgumentNullException(nameof(document));
}
var id = this.IdSelector(document);
if (id == null)
{
throw new InvalidOperationException("Cannot insert a document with a null identifier");
}
// encode the document
var packed = this.ValueEncoder.EncodeValue(document);
// Key Prefix = ...(id,)
var key = this.Location.Partial.Keys.Encode(id);
// clear previous value
trans.ClearRange(KeyRange.StartsWith(key));
int remaining = packed.Count;
if (remaining <= this.ChunkSize)
{ // stored as a single element
// Key = ...(id,)
trans.Set(key, packed);
}
else
{ // splits in as many chunks as necessary
// Key = ...(id, N) where N is the chunk index (0-based)
int p = 0;
int index = 0;
while (remaining > 0)
{
int sz = Math.Max(remaining, this.ChunkSize);
trans.Set(this.Location.Keys.Encode(id, index), packed.Substring(p, sz));
++index;
p += sz;
remaining -= sz;
}
}
}
private async Task <bool> TryRemoteSplitPointAsync([NotNull] IFdbTransaction trans, long offset)
{
Contract.Requires(trans != null && offset >= 0);
var b = await GetChunkAtAsync(trans, offset).ConfigureAwait(false);
if (b.Offset == 0 || b.Key == Slice.Nil)
{
return(false); // in sparse region, or at beginning
}
var a = await GetChunkAtAsync(trans, b.Offset - 1).ConfigureAwait(false);
if (a.Key == Slice.Nil)
{
return(false); // no previous chunk
}
if (a.Offset + a.Data.Count != b.Offset)
{
return(false); // chunks can't be joined
}
if (a.Data.Count + b.Data.Count > CHUNK_SMALL)
{
return(false); // chunks shouldn't be joined
}
// yay--merge chunks
trans.Clear(b.Key);
trans.Set(a.Key, a.Data + b.Data);
return(true);
}
/// <summary>Add the value x to the counter.</summary>
public async ValueTask Add(IFdbTransaction trans, long x)
{
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
var subspace = await this.Location.Resolve(trans);
if (subspace == null)
{
throw new InvalidOperationException($"Location '{this.Location} referenced by High Contention Counter Layer was not found.");
}
trans.Set(subspace.Encode(RandomId()), this.Encoder.EncodeValue(x));
// decide if we must coalesce
//note: Random() is not thread-safe so we must lock
bool coalesce;
lock (this.Rng) { coalesce = this.Rng.NextDouble() < 0.1; }
if (coalesce)
{
//REVIEW: 20 is too small if there is a lot of activity on the counter !
BackgroundCoalesce(trans.Context.Database, 20, CancellationToken.None);
}
}
private void SetSize([NotNull] IFdbTransaction trans, long size)
{
Contract.Requires(trans != null && size >= 0);
//note: python code converts the size into a string
trans.Set(SizeKey(), Slice.FromString(size.ToString()));
}
/// <summary>Swap the items at positions i1 and i2.</summary>
public async Task SwapAsync([NotNull] IFdbTransaction tr, long index1, long index2)
{
if (tr == null)
{
throw new ArgumentNullException("tr");
}
if (index1 < 0 || index2 < 0)
{
throw new IndexOutOfRangeException(String.Format("Indices ({0}, {1}) must be positive", index1, index2));
}
var k1 = GetKeyAt(index1);
var k2 = GetKeyAt(index2);
long currentSize = await ComputeSizeAsync(tr).ConfigureAwait(false);
if (index1 >= currentSize || index2 >= currentSize)
{
throw new IndexOutOfRangeException(String.Format("Indices ({0}, {1}) are out of range", index1, index2));
}
var vs = await tr.GetValuesAsync(new[] { k1, k2 }).ConfigureAwait(false);
var v1 = vs[0];
var v2 = vs[1];
if (!v2.IsNullOrEmpty)
{
tr.Set(k1, v2);
}
else if (v1.IsPresent && index1 < currentSize - 1)
{
tr.Clear(k1);
}
if (!v1.IsNullOrEmpty)
{
tr.Set(k2, v1);
}
else if (v2.IsPresent && index2 < currentSize - 1)
{
tr.Clear(k2);
}
}
/// <summary>Set the value at a particular index in the Vector.</summary>
public void Set([NotNull] IFdbTransaction tr, long index, T value)
{
if (tr == null)
{
throw new ArgumentNullException("tr");
}
tr.Set(GetKeyAt(index), this.Encoder.EncodeValue(value));
}
/// <summary>Insert a newly created entity to the index</summary>
/// <param name="trans">Transaction to use</param>
/// <param name="id">Id of the new entity (that was never indexed before)</param>
/// <param name="value">Value of this entity in the index</param>
/// <returns>True if a value was inserted into the index; otherwise false (if value is null and <see cref="IndexNullValues"/> is false)</returns>
public bool Add([NotNull] IFdbTransaction trans, TId id, TValue value)
{
if (this.IndexNullValues || value != null)
{
trans.Set(this.Location.Keys.Encode(value, id), Slice.Empty);
return(true);
}
return(false);
}
/// <summary>Insert a newly created entity to the index</summary>
/// <param name="trans">Transaction to use</param>
/// <param name="id">Id of the new entity (that was never indexed before)</param>
/// <param name="value">Value of this entity in the index</param>
/// <returns>True if a value was inserted into the index; otherwise false (if value is null and <see cref="IndexNullValues"/> is false)</returns>
public bool Add([NotNull] IFdbTransaction trans, TId id, TValue value)
{
if (this.IndexNullValues || value != null)
{
trans.Set(this.Subspace.Keys[value, id], Slice.Empty);
return(true);
}
return(false);
}
private Task Scenario2(IFdbTransaction tr)
{
var location = FdbSubspace.Create(Slice.FromAscii("TEST"));
tr.ClearRange(FdbKeyRange.StartsWith(location.Key));
for (int i = 0; i < 10; i++)
{
tr.Set(location.Pack(i), Slice.FromString("value of " + i));
}
return Task.FromResult<object>(null);
}
public async Task InsertAsync(IFdbTransaction trans, Slice key)
{
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
if (await ContainsAsync(trans, key).ConfigureAwait(false))
{
return;
}
int keyHash = key.GetHashCode(); //TODO: proper hash function?
//Console.WriteLine("Inserting " + key + " with hash " + keyHash.ToString("x"));
for (int level = 0; level < MAX_LEVELS; level++)
{
var prevKey = await GetPreviousNodeAsync(trans, level, key);
if ((keyHash & ((1 << (level * LEVEL_FAN_POW)) - 1)) != 0)
{
//Console.WriteLine("> [" + level + "] Incrementing previous key: " + FdbKey.Dump(prevKey));
trans.AtomicIncrement64(this.Subspace.Encode(level, prevKey));
}
else
{
//Console.WriteLine("> [" + level + "] inserting and updating previous key: " + FdbKey.Dump(prevKey));
// Insert into this level by looking at the count of the previous
// key in the level and recounting the next lower level to correct
// the counts
var prevCount = DecodeCount(await trans.GetAsync(this.Subspace.Encode(level, prevKey)).ConfigureAwait(false));
var newPrevCount = await SlowCountAsync(trans, level - 1, prevKey, key);
var count = checked ((prevCount - newPrevCount) + 1);
// print "insert", key, "level", level, "count", count,
// "splits", prevKey, "oldC", prevCount, "newC", newPrevCount
trans.Set(this.Subspace.Encode(level, prevKey), EncodeCount(newPrevCount));
trans.Set(this.Subspace.Encode(level, key), EncodeCount(count));
}
}
}
private Task Scenario2(IFdbTransaction tr)
{
var location = FdbSubspace.Create(Slice.FromAscii("TEST"));
tr.ClearRange(FdbKeyRange.StartsWith(location.Key));
for (int i = 0; i < 10; i++)
{
tr.Set(location.Pack(i), Slice.FromString("value of " + i));
}
return(Task.FromResult <object>(null));
}
/// <summary>Push a single item onto the end of the Vector.</summary>
public async Task PushAsync(IFdbTransaction tr, T value)
{
if (tr == null)
{
throw new ArgumentNullException(nameof(tr));
}
var size = await ComputeSizeAsync(tr).ConfigureAwait(false);
tr.Set(GetKeyAt(size), this.Encoder.EncodeValue(value));
}
private async Task MakeSplitPointAsync([NotNull] IFdbTransaction trans, long offset)
{
Contract.Requires(trans != null && offset >= 0);
var chunk = await GetChunkAtAsync(trans, offset).ConfigureAwait(false);
if (chunk.Key == Slice.Nil)
{
return; // already sparse
}
if (chunk.Offset == offset)
{
return; // already a split point
}
int splitPoint;
checked { splitPoint = (int)(offset - chunk.Offset); }
trans.Set(DataKey(chunk.Offset), chunk.Data.Substring(0, splitPoint));
trans.Set(DataKey(offset), chunk.Data.Substring(splitPoint));
}
private async Task PushAtAsync([NotNull] IFdbTransaction tr, T value, long index)
{
// Items are pushed on the queue at an (index, randomID) pair. Items pushed at the
// same time will have the same index, and so their ordering will be random.
// This makes pushes fast and usually conflict free (unless the queue becomes empty
// during the push)
Slice key = this.QueueItem.Keys.Encode(index, this.RandId());
await tr.GetAsync(key).ConfigureAwait(false);
tr.Set(key, this.Encoder.EncodeValue(value));
}
/// <summary>Add or update an entry in the map</summary>
/// <param name="trans">Transaction used for the operation</param>
/// <param name="id">Key of the entry to add or update</param>
/// <param name="value">New value of the entry</param>
/// <remarks>If the entry did not exist, it will be created. If not, its value will be replace with <paramref name="value"/>.</remarks>
public void Set([NotNull] IFdbTransaction trans, TKey id, TValue value)
{
if (trans == null)
{
throw new ArgumentNullException("trans");
}
if (id == null)
{
throw new ArgumentNullException("id");
}
trans.Set(this.Location.Keys.Encode(id), this.ValueEncoder.EncodeValue(value));
}
/// <summary>Update the indexed values of an entity</summary>
/// <param name="trans">Transaction to use</param>
/// <param name="id">Id of the entity that has changed</param>
/// <param name="newValue">Previous value of this entity in the index</param>
/// <param name="previousValue">New value of this entity in the index</param>
/// <returns>True if a change was performed in the index; otherwise false (if <paramref name="previousValue"/> and <paramref name="newValue"/>)</returns>
/// <remarks>If <paramref name="newValue"/> and <paramref name="previousValue"/> are identical, then nothing will be done. Otherwise, the old index value will be deleted and the new value will be added</remarks>
public async Task <bool> UpdateAsync([NotNull] IFdbTransaction trans, long id, TValue newValue, TValue previousValue)
{
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
if (!this.ValueComparer.Equals(newValue, previousValue))
{
// remove previous value
if (this.IndexNullValues || previousValue != null)
{
var key = this.Location.Keys.Encode(previousValue);
var data = await trans.GetAsync(key).ConfigureAwait(false);
if (data.HasValue)
{
var builder = new CompressedBitmapBuilder(data);
builder.Clear((int)id); //BUGBUG: 64 bit id!
trans.Set(key, builder.ToSlice());
}
}
// add new value
if (this.IndexNullValues || newValue != null)
{
var key = this.Location.Keys.Encode(newValue);
var data = await trans.GetAsync(key).ConfigureAwait(false);
var builder = data.HasValue ? new CompressedBitmapBuilder(data) : CompressedBitmapBuilder.Empty;
builder.Set((int)id); //BUGBUG: 64 bit id!
trans.Set(key, builder.ToSlice());
}
// cannot be both null, so we did at least something)
return(true);
}
return(false);
}
/// <summary>Add or update an entry in the map</summary>
/// <param name="trans">Transaction used for the operation</param>
/// <param name="id">Key of the entry to add or update</param>
/// <param name="value">New value of the entry</param>
/// <remarks>If the entry did not exist, it will be created. If not, its value will be replace with <paramref name="value"/>.</remarks>
public void Set([NotNull] IFdbTransaction trans, TKey id, TValue value)
{
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
if (id == null)
{
throw new ArgumentNullException(nameof(id));
}
trans.Set(this.Subspace.Keys[id], this.ValueEncoder.EncodeValue(value));
}
private async Task Scenario5(IFdbTransaction tr)
{
var location = FdbSubspace.Create(Slice.FromAscii("TEST"));
//tr.Set(location.Pack(42), Slice.FromString("42"));
//tr.Set(location.Pack(50), Slice.FromString("50"));
//tr.Set(location.Pack(60), Slice.FromString("60"));
var x = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(location.Pack(49)));
Console.WriteLine(x);
tr.Set(location.Pack("FOO"), Slice.FromString("BAR"));
}
/// <summary>
/// Drop a student from a class
/// </summary>
public async Task Drop(IFdbTransaction tr, string s, string c)
{
var rec = AttendsKey(s, c);
if ((await tr.GetAsync(rec)).IsNullOrEmpty)
{ // not taking this class
return;
}
var students = Int32.Parse((await tr.GetAsync(ClassKey(c))).ToStringAscii());
tr.Set(ClassKey(c), Slice.FromStringAscii((students + 1).ToString()));
tr.Clear(rec);
}
private async Task Scenario5(IFdbTransaction tr)
{
var location = FdbSubspace.Create(Slice.FromAscii("TEST"));
//tr.Set(location.Pack(42), Slice.FromString("42"));
//tr.Set(location.Pack(50), Slice.FromString("50"));
//tr.Set(location.Pack(60), Slice.FromString("60"));
var x = await tr.GetKeyAsync(FdbKeySelector.LastLessThan(location.Pack(49)));
Console.WriteLine(x);
tr.Set(location.Pack("FOO"), Slice.FromString("BAR"));
}
private async Task <Slice> InternSlowAsync(IFdbTransaction trans, string value)
{
var stringKey = StringKey(value);
var uid = await trans.GetAsync(stringKey).ConfigureAwait(false);
if (uid.IsNull)
{
#if DEBUG_STRING_INTERNING
Debug.WriteLine("_ not found in db, will create...");
#endif
uid = await FindUidAsync(trans).ConfigureAwait(false);
if (uid.IsNull)
{
throw new InvalidOperationException("Failed to allocate a new uid while attempting to intern a string");
}
#if DEBUG_STRING_INTERNING
Debug.WriteLine("> using new uid " + uid.ToBase64());
#endif
trans.Set(UidKey(uid), Slice.FromString(value));
trans.Set(stringKey, uid);
//BUGBUG: if the transaction fails to commit, we will inserted a bad value in the cache!
this.Layer.AddToCache(value, uid);
}
else
{
#if DEBUG_STRING_INTERNING
Debug.WriteLine("> found in db with uid " + uid.ToBase64());
#endif
}
return(uid);
}
private async Task <Slice> AddConflictedPopAsync([NotNull] IFdbTransaction tr, bool forced)
{
long index = await GetNextIndexAsync(tr.Snapshot, this.ConflictedPop).ConfigureAwait(false);
if (index == 0 && !forced)
{
return(Slice.Nil);
}
Slice waitKey = this.ConflictedPop.Keys.Encode(index, this.RandId());
await tr.GetAsync(waitKey).ConfigureAwait(false);
tr.Set(waitKey, Slice.Empty);
return(waitKey);
}
/// <summary>Grow or shrink the size of the Vector.</summary>
public async Task ResizeAsync([NotNull] IFdbTransaction tr, long length)
{
if (tr == null)
{
throw new ArgumentNullException("tr");
}
long currentSize = await ComputeSizeAsync(tr).ConfigureAwait(false);
if (length < currentSize)
{
tr.ClearRange(GetKeyAt(length), this.Subspace.Keys.ToRange().End);
// Check if the new end of the vector was being sparsely represented
if (await ComputeSizeAsync(tr).ConfigureAwait(false) < length)
{
tr.Set(GetKeyAt(length - 1), this.Encoder.EncodeValue(this.DefaultValue));
}
}
else if (length > currentSize)
{
tr.Set(GetKeyAt(length - 1), this.Encoder.EncodeValue(this.DefaultValue));
}
}
private async Task PushQueueAsync(IFdbTransaction tr, IDynamicKeySubspace queue, Slice taskId)
{
//TODO: use a high contention algo ?
// - must support Push and Pop
// - an empty queue must correspond to an empty subspace
// get the current size of the queue
var range = queue.Keys.ToRange();
var lastKey = await tr.Snapshot.GetKeyAsync(KeySelector.LastLessThan(range.End)).ConfigureAwait(false);
int count = lastKey < range.Begin ? 0 : queue.Keys.DecodeFirst <int>(lastKey) + 1;
// set the value
tr.Set(queue.Keys.Encode(count, GetRandomId()), taskId);
}
internal async Task SetupLevelsAsync(IFdbTransaction trans)
{
var ks = Enumerable.Range(0, MAX_LEVELS)
.Select((l) => this.Subspace.Encode(l, Slice.Empty))
.ToList();
var res = await trans.GetValuesAsync(ks).ConfigureAwait(false);
for (int l = 0; l < res.Length; l++)
{
if (res[l].IsNull)
{
trans.Set(ks[l], EncodeCount(0));
}
}
}
private async Task <List <PerformanceRecord> > WriteInternalAsync(string operation,
IEnumerator <TestData> enumerator,
long itemsPerTransaction, long numberOfTransactions, PerfTracker perfTracker, FdbDatabase db)
{
var sw = new Stopwatch();
byte[] valueToWrite = null;
var records = new List <PerformanceRecord>();
var location = db.GlobalSpace;
sw.Restart();
for (int transactions = 0; transactions < numberOfTransactions; transactions++)
{
sw.Restart();
using (IFdbTransaction tx = db.BeginTransaction())
{
for (int i = 0; i < itemsPerTransaction; i++)
{
enumerator.MoveNext();
valueToWrite = GetValueToWrite(valueToWrite, enumerator.Current.ValueSize);
tx.Set(location.Pack(enumerator.Current.Id), Slice.Create(valueToWrite));
}
await tx.CommitAsync();
perfTracker.Record(sw.ElapsedMilliseconds);
}
sw.Stop();
records.Add(new PerformanceRecord
{
Operation = operation,
Time = DateTime.Now,
Duration = sw.ElapsedMilliseconds,
ProcessedItems = itemsPerTransaction
});
}
sw.Stop();
return(records);
}
public void SetValue(IFdbTransaction trans, IVarTuple id, string field, Slice value)
{
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
if (id == null)
{
throw new ArgumentNullException(nameof(id));
}
if (string.IsNullOrEmpty(field))
{
throw new ArgumentNullException(nameof(field));
}
trans.Set(GetFieldKey(id, field), value);
}
private void WriteToSparse([NotNull] IFdbTransaction trans, long offset, Slice data)
{
Contract.Requires(trans != null && offset >= 0);
if (data.IsNullOrEmpty)
{
return;
}
int chunks = (int)((data.Count + CHUNK_LARGE - 1) / CHUNK_LARGE);
int chunkSize = (data.Count + chunks) / chunks;
for (int n = 0; n < data.Count; n += chunkSize)
{
trans.Set(DataKey(offset + n), data[n, n + chunkSize]);
}
}
private void StoreTask(IFdbTransaction tr, Slice taskId, DateTime scheduledUtc, Slice taskBody)
{
tr.Annotate("Writing task {0}", taskId.ToAsciiOrHexaString());
var prefix = this.TaskStore.Partition(taskId);
// store task body and timestamp
tr.Set(prefix.Key, taskBody);
tr.Set(prefix.Pack(TASK_META_SCHEDULED), Slice.FromInt64(scheduledUtc.Ticks));
// increment total and pending number of tasks
this.Counters.Increment(tr, COUNTER_TOTAL_TASKS);
this.Counters.Increment(tr, COUNTER_PENDING_TASKS);
}
private async Task PushQueueAsync(IFdbTransaction tr, FdbSubspace queue, Slice taskId)
{
//TODO: use a high contention algo ?
// - must support Push and Pop
// - an empty queue must correspond to an empty subspace
// get the current size of the queue
var range = queue.ToRange();
var lastKey = await tr.Snapshot.GetKeyAsync(FdbKeySelector.LastLessThan(range.End)).ConfigureAwait(false);
int count = lastKey < range.Begin ? 0 : queue.Unpack(lastKey).Get<int>(0) + 1;
// set the value
tr.Set(queue.Pack(count, GetRandomId()), taskId);
}
private void WriteToSparse(IFdbTransaction trans, long offset, Slice data)
{
Contract.Requires(trans != null && offset >= 0);
if (data.IsNullOrEmpty) return;
int chunks = (int)((data.Count + CHUNK_LARGE - 1) / CHUNK_LARGE);
int chunkSize = (data.Count + chunks) / chunks;
for (int n = 0; n < data.Count; n += chunkSize)
{
trans.Set(DataKey(offset + n), data[n, n + chunkSize]);
}
}
private void SetSize(IFdbTransaction trans, long size)
{
Contract.Requires(trans != null && size >= 0);
//note: python code converts the size into a string
trans.Set(SizeKey(), Slice.FromString(size.ToString()));
}
/// <summary>Add the value x to the counter.</summary>
public void Add(IFdbTransaction trans, long x)
{
if (trans == null) throw new ArgumentNullException("trans");
trans.Set(this.Subspace.Pack(RandomId()), this.Encoder.EncodeValue(x));
// decide if we must coalesce
//note: Random() is not thread-safe so we must lock
bool coalesce;
lock (this.Rng) { coalesce = this.Rng.NextDouble() < 0.1; }
if (coalesce)
{
//REVIEW: 20 is too small if there is a lot of activity on the counter !
BackgroundCoalesce(20, CancellationToken.None);
}
}
internal async Task<FdbDirectorySubspace> CreateOrOpenInternalAsync(IFdbReadOnlyTransaction readTrans, IFdbTransaction trans, [NotNull] IFdbTuple path, Slice layer, Slice prefix, bool allowCreate, bool allowOpen, bool throwOnError)
{
Contract.Requires(readTrans != null || trans != null, "Need at least one transaction");
Contract.Requires(path != null, "Path must be specified");
Contract.Requires(readTrans == null || trans == null || object.ReferenceEquals(readTrans, trans), "The write transaction should be the same as the read transaction");
if (path.Count == 0)
{ // Root directory contains node metadata and so may not be opened.
throw new InvalidOperationException("The root directory may not be opened.");
}
// to open an existing directory, we only need the read transaction
// if none was specified, we can use the writeable transaction
if (readTrans == null) readTrans = trans;
await CheckReadVersionAsync(readTrans).ConfigureAwait(false);
if (prefix.HasValue && this.Path.Count > 0)
throw new InvalidOperationException("Cannot specify a prefix in a partition.");
var existingNode = await FindAsync(readTrans, path).ConfigureAwait(false);
if (existingNode.Exists)
{
if (existingNode.IsInPartition(false))
{
var subpath = existingNode.PartitionSubPath;
var dl = GetPartitionForNode(existingNode).DirectoryLayer;
return await dl.CreateOrOpenInternalAsync(readTrans, trans, subpath, layer, prefix, allowCreate, allowOpen, throwOnError).ConfigureAwait(false);
}
if (!allowOpen)
{
if (throwOnError) throw new InvalidOperationException(string.Format("The directory {0} already exists.", path));
return null;
}
if (layer.IsPresent && layer != existingNode.Layer)
{
throw new InvalidOperationException(String.Format("The directory {0} was created with incompatible layer {1} instead of expected {2}.", path, layer.ToAsciiOrHexaString(), existingNode.Layer.ToAsciiOrHexaString()));
}
return ContentsOfNode(existingNode.Subspace, path, existingNode.Layer);
}
if (!allowCreate)
{
if (throwOnError) throw new InvalidOperationException(string.Format("The directory {0} does not exist.", path));
return null;
}
// from there, we actually do need a wrtieable transaction
if (trans == null) throw new InvalidOperationException("A writeable transaction is needed to create a new directory");
await CheckWriteVersionAsync(trans).ConfigureAwait(false);
if (prefix == null)
{ // automatically allocate a new prefix inside the ContentSubspace
long id = await this.Allocator.AllocateAsync(trans).ConfigureAwait(false);
prefix = this.ContentSubspace.Pack(id);
// ensure that there is no data already present under this prefix
if (FdbDirectoryLayer.AnnotateTransactions) trans.Annotate("Ensure that there is no data already present under prefix {0}", prefix);
if (await trans.GetRange(FdbKeyRange.StartsWith(prefix)).AnyAsync().ConfigureAwait(false))
{
throw new InvalidOperationException(String.Format("The database has keys stored at the prefix chosen by the automatic prefix allocator: {0}", prefix.ToAsciiOrHexaString()));
}
// ensure that the prefix has not already been allocated
if (FdbDirectoryLayer.AnnotateTransactions) trans.Annotate("Ensure that the prefix {0} has not already been allocated", prefix);
if (!(await IsPrefixFree(trans.Snapshot, prefix).ConfigureAwait(false)))
{
throw new InvalidOperationException("The directory layer has manually allocated prefixes that conflict with the automatic prefix allocator.");
}
}
else
{
if (FdbDirectoryLayer.AnnotateTransactions) trans.Annotate("Ensure that the prefix {0} hasn't already been allocated", prefix);
// ensure that the prefix has not already been allocated
if (!(await IsPrefixFree(trans, prefix).ConfigureAwait(false)))
{
throw new InvalidOperationException("The given prefix is already in use.");
}
}
// we need to recursively create any missing parents
FdbSubspace parentNode;
if (path.Count > 1)
{
var parentSubspace = await CreateOrOpenInternalAsync(readTrans, trans, path.Substring(0, path.Count - 1), Slice.Nil, Slice.Nil, true, true, true).ConfigureAwait(false);
parentNode = NodeWithPrefix(parentSubspace.Key);
}
else
{
parentNode = this.RootNode;
}
if (parentNode == null) throw new InvalidOperationException(string.Format("The parent directory of {0} doesn't exist.", path));
// initialize the metadata for this new directory
var node = NodeWithPrefix(prefix);
if (FdbDirectoryLayer.AnnotateTransactions) trans.Annotate("Registering the new prefix {0} into the folder sub-tree", prefix);
trans.Set(GetSubDirKey(parentNode, path.Get<string>(-1)), prefix);
SetLayer(trans, node, layer);
return ContentsOfNode(node, path, layer);
}
private static void SetLayer(IFdbTransaction trans, FdbSubspace subspace, Slice layer)
{
if (layer.IsNull) layer = Slice.Empty;
trans.Set(subspace.Pack(LayerSuffix), layer);
}
private async Task<bool> TryRemoteSplitPointAsync(IFdbTransaction trans, long offset)
{
Contract.Requires(trans != null && offset >= 0);
var b = await GetChunkAtAsync(trans, offset).ConfigureAwait(false);
if (b.Offset == 0 || b.Key == Slice.Nil) return false; // in sparse region, or at beginning
var a = await GetChunkAtAsync(trans, b.Offset - 1).ConfigureAwait(false);
if (a.Key == Slice.Nil) return false; // no previous chunk
if (a.Offset + a.Data.Count != b.Offset) return false; // chunks can't be joined
if (a.Data.Count + b.Data.Count > CHUNK_SMALL) return false; // chunks shouldn't be joined
// yay--merge chunks
trans.Clear(b.Key);
trans.Set(a.Key, a.Data + b.Data);
return true;
}
public void Set(IFdbTransaction trans, IFdbTuple id, IEnumerable<KeyValuePair<string, Slice>> fields)
{
if (trans == null) throw new ArgumentNullException("trans");
if (id == null) throw new ArgumentNullException("id");
if (fields == null) throw new ArgumentNullException("fields");
foreach (var field in fields)
{
if (string.IsNullOrEmpty(field.Key)) throw new ArgumentException("Field cannot have an empty name", "fields");
trans.Set(GetFieldKey(id, field.Key), field.Value);
}
}
/// <summary>
/// Drop a student from a class
/// </summary>
public async Task Drop(IFdbTransaction tr, string s, string c)
{
var rec = AttendsKey(s, c);
if ((await tr.GetAsync(rec)).IsNullOrEmpty)
{ // not taking this class
return;
}
var students = Int32.Parse((await tr.GetAsync(ClassKey(c))).ToAscii());
tr.Set(ClassKey(c), Slice.FromAscii((students + 1).ToString()));
tr.Clear(rec);
}
/// <summary>
/// Signup a student to a class
/// </summary>
public async Task Signup(IFdbTransaction tr, string s, string c)
{
var rec = AttendsKey(s, c);
if ((await tr.GetAsync(rec)).IsPresent)
{ // already signed up
return;
}
int seatsLeft = Int32.Parse((await tr.GetAsync(ClassKey(c))).ToAscii());
if (seatsLeft <= 0)
{
throw new InvalidOperationException("No remaining seats");
}
var classes = await tr.GetRange(AttendsKeys(s)).ToListAsync();
if (classes.Count >= 5) throw new InvalidOperationException("Too many classes");
tr.Set(ClassKey(c), Slice.FromAscii((seatsLeft - 1).ToString()));
tr.Set(rec, Slice.Empty);
}
private async Task SetupLevelsAsync(IFdbTransaction trans)
{
var ks = Enumerable.Range(0, MAX_LEVELS)
.Select((l) => this.Subspace.Pack(l, Slice.Empty))
.ToList();
var res = await trans.GetValuesAsync(ks).ConfigureAwait(false);
for (int l = 0; l < res.Length; l++)
{
//Console.WriteLine(ks[l]);
if (res[l].IsNull) trans.Set(ks[l], EncodeCount(0));
}
}
internal async Task<FdbDirectorySubspace> MoveInternalAsync(IFdbTransaction trans, IFdbTuple oldPath, IFdbTuple newPath, bool throwOnError)
{
Contract.Requires(trans != null && oldPath != null && newPath != null);
if (oldPath.Count == 0)
{
throw new InvalidOperationException("The root directory may not be moved.");
}
if (newPath.Count == 0)
{
throw new InvalidOperationException("The root directory cannot be overwritten.");
}
if (newPath.StartsWith(oldPath))
{
throw new InvalidOperationException(string.Format("The destination directory({0}) cannot be a subdirectory of the source directory({1}).", newPath, oldPath));
}
await CheckWriteVersionAsync(trans).ConfigureAwait(false);
var oldNode = await FindAsync(trans, oldPath).ConfigureAwait(false);
if (!oldNode.Exists)
{
if (throwOnError) throw new InvalidOperationException(string.Format("The source directory '{0}' does not exist.", oldPath));
return null;
}
var newNode = await FindAsync(trans, newPath).ConfigureAwait(false);
// we have already checked that old and new are under this partition path, but one of them (or both?) could be under a sub-partition..
if (oldNode.IsInPartition(false) || newNode.IsInPartition(false))
{
if (!oldNode.IsInPartition(false) || !newNode.IsInPartition(false) || !FdbTuple.Equals(oldNode.Path, newNode.Path))
{
throw new InvalidOperationException("Cannot move between partitions.");
}
// both nodes are in the same sub-partition, delegate to it
return await GetPartitionForNode(newNode).DirectoryLayer.MoveInternalAsync(trans, oldNode.PartitionSubPath, newNode.PartitionSubPath, throwOnError).ConfigureAwait(false);
}
if (newNode.Exists)
{
if (throwOnError) throw new InvalidOperationException(string.Format("The destination directory '{0}' already exists. Remove it first.", newPath));
return null;
}
var parentNode = await FindAsync(trans, newPath.Substring(0, newPath.Count - 1)).ConfigureAwait(false);
if (!parentNode.Exists)
{
if (throwOnError) throw new InvalidOperationException(string.Format("The parent of the destination directory '{0}' does not exist. Create it first.", newPath));
return null;
}
trans.Set(GetSubDirKey(parentNode.Subspace, newPath.Get<string>(-1)), this.NodeSubspace.UnpackSingle<Slice>(oldNode.Subspace.Key));
await RemoveFromParent(trans, oldPath).ConfigureAwait(false);
return ContentsOfNode(oldNode.Subspace, newPath, oldNode.Layer);
}
// Compare the behavior of the MemoryDB against a FoundationDB database
private async Task Scenario1(IFdbTransaction tr)
{
tr.Set(Slice.FromAscii("hello"), Slice.FromAscii("world!"));
tr.Clear(Slice.FromAscii("removed"));
var result = await tr.GetAsync(Slice.FromAscii("narf"));
}
public void SetValue(IFdbTransaction trans, IFdbTuple id, string field, Slice value)
{
if (trans == null) throw new ArgumentNullException("trans");
if (id == null) throw new ArgumentNullException("id");
if (string.IsNullOrEmpty(field)) throw new ArgumentNullException("field");
trans.Set(GetFieldKey(id, field), value);
}
private void InitializeDirectory(IFdbTransaction trans)
{
// Set the version key
var writer = new SliceWriter(3 * 4);
writer.WriteFixed32((uint)LayerVersion.Major);
writer.WriteFixed32((uint)LayerVersion.Minor);
writer.WriteFixed32((uint)LayerVersion.Build);
trans.Set(this.RootNode.Pack(VersionKey), writer.ToSlice());
}
/// <summary>Returns a 64-bit integer that
/// 1) has never and will never be returned by another call to this
/// method on the same subspace
/// 2) is nearly as short as possible given the above
/// </summary>
public async Task<long> AllocateAsync(IFdbTransaction trans)
{
// find the current window size, by reading the last entry in the 'counters' subspace
long start = 0, count = 0;
var kv = await trans
.Snapshot
.GetRange(this.Counters.ToRange())
.LastOrDefaultAsync();
if (kv.Key.IsPresent)
{
start = this.Counters.UnpackSingle<long>(kv.Key);
count = kv.Value.ToInt64();
}
// check if the window is full
int window = GetWindowSize(start);
if ((count + 1) * 2 >= window)
{ // advance the window
trans.ClearRange(this.Counters.Key, this.Counters.Pack(start) + FdbKey.MinValue);
start += window;
trans.ClearRange(this.Recent.Key, this.Recent.Pack(start));
}
// Increment the allocation count for the current window
trans.AtomicAdd(this.Counters.Pack(start), Slice.FromFixed64(1));
// As of the snapshot being read from, the window is less than half
// full, so this should be expected to take 2 tries. Under high
// contention (and when the window advances), there is an additional
// subsequent risk of conflict for this transaction.
while (true)
{
// Find a random free slot in the current window...
long candidate;
lock (m_rnd)
{
candidate = start + m_rnd.Next(window);
}
// test if the key is used
var key = this.Recent.Pack(candidate);
var value = await trans.GetAsync(key).ConfigureAwait(false);
if (value.IsNull)
{ // free slot
// mark as used
trans.Set(key, Slice.Empty);
return candidate;
}
// no luck this time, try again...
}
}
private async Task MakeSplitPointAsync(IFdbTransaction trans, long offset)
{
Contract.Requires(trans != null && offset >= 0);
var chunk = await GetChunkAtAsync(trans, offset).ConfigureAwait(false);
if (chunk.Key == Slice.Nil) return; // already sparse
if (chunk.Offset == offset) return; // already a split point
int splitPoint;
checked { splitPoint = (int)(offset - chunk.Offset); }
trans.Set(DataKey(chunk.Offset), chunk.Data.Substring(0, splitPoint));
trans.Set(DataKey(offset), chunk.Data.Substring(splitPoint));
}
