/// <summary>Pop the next item from the queue. Cannot be composed with other functions in a single transaction.</summary>
public async Task <(T Value, bool HasValue)> PopAsync(IFdbTransaction tr)
{
Contract.NotNull(tr, nameof(tr));
#if DEBUG
tr.Annotate("Pop()");
#endif
var first = await tr.GetRangeAsync(this.Subspace.ToRange(), SingleOptions);
if (first.IsEmpty)
{
#if DEBUG
tr.Annotate("Got nothing");
#endif
return(default);
private void ClearTask(IFdbTransaction tr, Slice taskId)
{
tr.Annotate("Deleting task {0:P}", taskId);
// clear all metadata about the task
tr.ClearRange(KeyRange.StartsWith(this.TaskStore.Keys.Encode(taskId)));
// decrement pending number of tasks
this.Counters.Decrement(tr, COUNTER_PENDING_TASKS);
}
private void ClearTask(IFdbTransaction tr, Slice taskId)
{
tr.Annotate("Deleting task {0}", taskId.ToAsciiOrHexaString());
// clear all metadata about the task
tr.ClearRange(FdbKeyRange.StartsWith(this.TaskStore.Pack(taskId)));
// decrement pending number of tasks
this.Counters.Decrement(tr, COUNTER_PENDING_TASKS);
}
/// <summary>Push a single item onto the queue.</summary>
public async Task PushAsync([NotNull] IFdbTransaction trans, T value)
{
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
#if DEBUG
trans.Annotate("Push({0})", value);
#endif
long index = await GetNextIndexAsync(trans.Snapshot, this.QueueItem).ConfigureAwait(false);
#if DEBUG
trans.Annotate("Index = {0}", index);
#endif
await PushAtAsync(trans, value, index).ConfigureAwait(false);
}
/// <summary>Push a single item onto the queue.</summary>
public void Push(IFdbTransaction tr, T value)
{
Contract.NotNull(tr, nameof(tr));
#if DEBUG
tr.Annotate("Push({0})", value);
#endif
//BUGBUG: can be called multiple times per transaction, so need a unique stamp _per_ transaction!!
tr.SetVersionStampedKey(this.Subspace[tr.CreateUniqueVersionStamp()], this.Encoder.EncodeValue(value));
}
private void StoreTask(IFdbTransaction tr, Slice taskId, DateTime scheduledUtc, Slice taskBody)
{
tr.Annotate("Writing task {0:P}", taskId);
var prefix = this.TaskStore.Partition.ByKey(taskId);
// store task body and timestamp
tr.Set(prefix.GetPrefix(), taskBody);
tr.Set(prefix.Keys.Encode(TASK_META_SCHEDULED), Slice.FromInt64(scheduledUtc.Ticks));
// increment total and pending number of tasks
this.Counters.Increment(tr, COUNTER_TOTAL_TASKS);
this.Counters.Increment(tr, COUNTER_PENDING_TASKS);
}
private async Task <Optional <T> > PopSimpleAsync([NotNull] IFdbTransaction tr)
{
#if DEBUG
tr.Annotate("PopSimple()");
#endif
var firstItem = await GetFirstItemAsync(tr).ConfigureAwait(false);
if (firstItem.Key.IsNull)
{
return(default(Optional <T>));
}
tr.Clear(firstItem.Key);
return(this.Encoder.DecodeValue(firstItem.Value));
}
private void ClearTask(IFdbTransaction tr, Slice taskId)
{
tr.Annotate("Deleting task {0}", taskId.ToAsciiOrHexaString());
// clear all metadata about the task
tr.ClearRange(FdbKeyRange.StartsWith(this.TaskStore.Pack(taskId)));
// decrement pending number of tasks
this.Counters.Decrement(tr, COUNTER_PENDING_TASKS);
}
private void StoreTask(IFdbTransaction tr, Slice taskId, DateTime scheduledUtc, Slice taskBody)
{
tr.Annotate("Writing task {0}", taskId.ToAsciiOrHexaString());
var prefix = this.TaskStore.Partition(taskId);
// store task body and timestamp
tr.Set(prefix.Key, taskBody);
tr.Set(prefix.Pack(TASK_META_SCHEDULED), Slice.FromInt64(scheduledUtc.Ticks));
// increment total and pending number of tasks
this.Counters.Increment(tr, COUNTER_TOTAL_TASKS);
this.Counters.Increment(tr, COUNTER_PENDING_TASKS);
}
/// <summary>Returns a 64-bit integer that
/// 1) has never and will never be returned by another call to this
/// method on the same subspace
/// 2) is nearly as short as possible given the above
/// </summary>
public async Task <long> AllocateAsync([NotNull] IFdbTransaction trans)
{
if (trans == null)
{
throw new ArgumentNullException(nameof(trans));
}
// find the current window size, by reading the last entry in the 'counters' subspace
long start = 0, count = 0;
var kv = await trans
.Snapshot
.GetRange(this.Counters.Keys.ToRange())
.LastOrDefaultAsync();
if (kv.Key.IsPresent)
{
start = this.Counters.Keys.Decode <long>(kv.Key);
count = kv.Value.ToInt64();
}
// check if the window is full
int window = GetWindowSize(start);
if ((count + 1) * 2 >= window)
{ // advance the window
if (FdbDirectoryLayer.AnnotateTransactions)
{
trans.Annotate("Advance allocator window size to {0} starting at {1}", window, start + window);
}
trans.ClearRange(this.Counters.GetPrefix(), this.Counters.Keys.Encode(start) + FdbKey.MinValue);
start += window;
count = 0;
trans.ClearRange(this.Recent.GetPrefix(), this.Recent.Keys.Encode(start));
}
// Increment the allocation count for the current window
trans.AtomicAdd64(this.Counters.Keys.Encode(start), 1);
// As of the snapshot being read from, the window is less than half
// full, so this should be expected to take 2 tries. Under high
// contention (and when the window advances), there is an additional
// subsequent risk of conflict for this transaction.
while (true)
{
// Find a random free slot in the current window...
long candidate;
lock (m_rnd)
{
candidate = start + m_rnd.Next(window);
}
// test if the key is used
var key = this.Recent.Keys.Encode(candidate);
var value = await trans.GetAsync(key).ConfigureAwait(false);
if (value.IsNull)
{ // free slot
// mark as used
trans.Set(key, Slice.Empty);
if (FdbDirectoryLayer.AnnotateTransactions)
{
trans.Annotate("Allocated prefix {0} from window [{1}..{2}] ({3} used)", candidate, start, start + window - 1, count + 1);
}
return(candidate);
}
// no luck this time, try again...
}
}
public static async Task <long> AllocateAsync(IFdbTransaction trans, ITypedKeySubspace <int, long> subspace, Random rng)
{
Contract.NotNull(trans);
// find the current window size, by reading the last entry in the 'counters' subspace
long start = 0, count = 0;
var kv = await trans
.Snapshot
.GetRange(subspace.EncodePartialRange(COUNTERS))
.LastOrDefaultAsync();
if (kv.Key.Count != 0)
{
start = subspace.DecodeLast(kv.Key);
count = kv.Value.ToInt64();
}
// check if the window is full
int window = GetWindowSize(start);
if ((count + 1) * 2 >= window)
{ // advance the window
if (FdbDirectoryLayer.AnnotateTransactions)
{
trans.Annotate("Advance allocator window size to {0} starting at {1}", window, start + window);
}
trans.ClearRange(subspace[COUNTERS, 0], subspace[COUNTERS, start + 1]);
start += window;
count = 0;
trans.ClearRange(subspace[RECENT, 0], subspace[RECENT, start]);
}
// Increment the allocation count for the current window
trans.AtomicAdd64(subspace[COUNTERS, start], 1);
// As of the snapshot being read from, the window is less than half
// full, so this should be expected to take 2 tries. Under high
// contention (and when the window advances), there is an additional
// subsequent risk of conflict for this transaction.
while (true)
{
// Find a random free slot in the current window...
long candidate;
lock (rng)
{
candidate = start + rng.Next(window);
}
// test if the key is used
var key = subspace[RECENT, candidate];
var value = await trans.GetAsync(key).ConfigureAwait(false);
if (value.IsNull)
{ // free slot
// mark as used
trans.Set(key, Slice.Empty);
if (FdbDirectoryLayer.AnnotateTransactions)
{
trans.Annotate("Allocated prefix {0} from window [{1}..{2}] ({3} used)", candidate, start, start + window - 1, count + 1);
}
return(candidate);
}
// no luck this time, try again...
}
}
/// <summary>Resursively remove a node (including the content), all its children</summary>
private async Task RemoveRecursive(IFdbTransaction tr, FdbSubspace node)
{
Contract.Requires(tr != null && node != null);
//note: we could use Task.WhenAll to remove the children, but there is a risk of task explosion if the subtree is very large...
await SubdirNamesAndNodes(tr, node).ForEachAsync((kvp) => RemoveRecursive(tr, kvp.Value)).ConfigureAwait(false);
// remove ALL the contents
if (FdbDirectoryLayer.AnnotateTransactions) tr.Annotate("Removing all content located under {0}", node.Key);
tr.ClearRange(FdbKeyRange.StartsWith(ContentsOfNode(node, FdbTuple.Empty, Slice.Empty).Key));
// and all the metadata for this folder
if (FdbDirectoryLayer.AnnotateTransactions) tr.Annotate("Removing all metadata for folder under {0}", node.Key);
tr.ClearRange(node.ToRange());
}
/// <summary>Remove an existing node from its parents</summary>
/// <returns>True if the parent node was found, otherwise false</returns>
private async Task<bool> RemoveFromParent(IFdbTransaction tr, IFdbTuple path)
{
Contract.Requires(tr != null && path != null);
var parent = await FindAsync(tr, path.Substring(0, path.Count - 1)).ConfigureAwait(false);
if (parent.Exists)
{
if (FdbDirectoryLayer.AnnotateTransactions) tr.Annotate("Removing path {0} from its parent folder at {1}", path, parent.Subspace.Key);
tr.Clear(GetSubDirKey(parent.Subspace, path.Get<string>(-1)));
return true;
}
return false;
}
internal async Task<FdbDirectorySubspace> CreateOrOpenInternalAsync(IFdbReadOnlyTransaction readTrans, IFdbTransaction trans, [NotNull] IFdbTuple path, Slice layer, Slice prefix, bool allowCreate, bool allowOpen, bool throwOnError)
{
Contract.Requires(readTrans != null || trans != null, "Need at least one transaction");
Contract.Requires(path != null, "Path must be specified");
Contract.Requires(readTrans == null || trans == null || object.ReferenceEquals(readTrans, trans), "The write transaction should be the same as the read transaction");
if (path.Count == 0)
{ // Root directory contains node metadata and so may not be opened.
throw new InvalidOperationException("The root directory may not be opened.");
}
// to open an existing directory, we only need the read transaction
// if none was specified, we can use the writeable transaction
if (readTrans == null) readTrans = trans;
await CheckReadVersionAsync(readTrans).ConfigureAwait(false);
if (prefix.HasValue && this.Path.Count > 0)
throw new InvalidOperationException("Cannot specify a prefix in a partition.");
var existingNode = await FindAsync(readTrans, path).ConfigureAwait(false);
if (existingNode.Exists)
{
if (existingNode.IsInPartition(false))
{
var subpath = existingNode.PartitionSubPath;
var dl = GetPartitionForNode(existingNode).DirectoryLayer;
return await dl.CreateOrOpenInternalAsync(readTrans, trans, subpath, layer, prefix, allowCreate, allowOpen, throwOnError).ConfigureAwait(false);
}
if (!allowOpen)
{
if (throwOnError) throw new InvalidOperationException(string.Format("The directory {0} already exists.", path));
return null;
}
if (layer.IsPresent && layer != existingNode.Layer)
{
throw new InvalidOperationException(String.Format("The directory {0} was created with incompatible layer {1} instead of expected {2}.", path, layer.ToAsciiOrHexaString(), existingNode.Layer.ToAsciiOrHexaString()));
}
return ContentsOfNode(existingNode.Subspace, path, existingNode.Layer);
}
if (!allowCreate)
{
if (throwOnError) throw new InvalidOperationException(string.Format("The directory {0} does not exist.", path));
return null;
}
// from there, we actually do need a wrtieable transaction
if (trans == null) throw new InvalidOperationException("A writeable transaction is needed to create a new directory");
await CheckWriteVersionAsync(trans).ConfigureAwait(false);
if (prefix == null)
{ // automatically allocate a new prefix inside the ContentSubspace
long id = await this.Allocator.AllocateAsync(trans).ConfigureAwait(false);
prefix = this.ContentSubspace.Pack(id);
// ensure that there is no data already present under this prefix
if (FdbDirectoryLayer.AnnotateTransactions) trans.Annotate("Ensure that there is no data already present under prefix {0}", prefix);
if (await trans.GetRange(FdbKeyRange.StartsWith(prefix)).AnyAsync().ConfigureAwait(false))
{
throw new InvalidOperationException(String.Format("The database has keys stored at the prefix chosen by the automatic prefix allocator: {0}", prefix.ToAsciiOrHexaString()));
}
// ensure that the prefix has not already been allocated
if (FdbDirectoryLayer.AnnotateTransactions) trans.Annotate("Ensure that the prefix {0} has not already been allocated", prefix);
if (!(await IsPrefixFree(trans.Snapshot, prefix).ConfigureAwait(false)))
{
throw new InvalidOperationException("The directory layer has manually allocated prefixes that conflict with the automatic prefix allocator.");
}
}
else
{
if (FdbDirectoryLayer.AnnotateTransactions) trans.Annotate("Ensure that the prefix {0} hasn't already been allocated", prefix);
// ensure that the prefix has not already been allocated
if (!(await IsPrefixFree(trans, prefix).ConfigureAwait(false)))
{
throw new InvalidOperationException("The given prefix is already in use.");
}
}
// we need to recursively create any missing parents
FdbSubspace parentNode;
if (path.Count > 1)
{
var parentSubspace = await CreateOrOpenInternalAsync(readTrans, trans, path.Substring(0, path.Count - 1), Slice.Nil, Slice.Nil, true, true, true).ConfigureAwait(false);
parentNode = NodeWithPrefix(parentSubspace.Key);
}
else
{
parentNode = this.RootNode;
}
if (parentNode == null) throw new InvalidOperationException(string.Format("The parent directory of {0} doesn't exist.", path));
// initialize the metadata for this new directory
var node = NodeWithPrefix(prefix);
if (FdbDirectoryLayer.AnnotateTransactions) trans.Annotate("Registering the new prefix {0} into the folder sub-tree", prefix);
trans.Set(GetSubDirKey(parentNode, path.Get<string>(-1)), prefix);
SetLayer(trans, node, layer);
return ContentsOfNode(node, path, layer);
}
