/// <summary>
/// Verify that ringmaster correctly handles a batch sent to the Transaction Manager execution queue with just book keeping requests.
/// </summary>
/// <returns>A <see cref="Task"/> that tracks completion of this test</returns>
/// <remarks>
/// RDBug 6351033: When a batch with just book keeping requests was sent to RingMaster,
/// it did not update an internal variable that tracked last applied batch id and refused
/// the subsequent batches. This verifies the fix for that issue.
/// </remarks>
public async Task TestTransactionManagerExecutionQueue_EmptyBatch()
{
using (var ringMaster = this.ConnectToRingMaster())
{
IRingMasterRequestHandler ringMasterRequestHandler = (IRingMasterRequestHandler)ringMaster;
ulong lastAppliedBatchId = await QueryRingMasterLastAppliedBatchId(ringMaster);
ulong batchId = lastAppliedBatchId + 1;
// First send a regular batch to initialize the lastAppliedBatchId tracker in RM
RequestBatch initializationBatch = CreateBatch(batchId, CreateTransaction(1));
RequestResponse batchResponse = await ringMasterRequestHandler.Request(initializationBatch);
Assert.AreEqual((int)RingMasterException.Code.Ok, batchResponse.ResultCode);
lastAppliedBatchId = await QueryRingMasterLastAppliedBatchId(ringMaster);
batchId = lastAppliedBatchId + 1;
// Create a batch with only book keeping requests.
RequestBatch firstBatch = CreateBatch(batchId);
Trace.TraceInformation("Sending first batch. batchId={0}, executionQueueId={1}", batchId, firstBatch.ExecutionQueueId);
batchResponse = await ringMasterRequestHandler.Request(firstBatch);
Assert.AreEqual((int)RingMasterException.Code.Ok, batchResponse.ResultCode);
// The last applied batch id value must have been updated.
lastAppliedBatchId = await QueryRingMasterLastAppliedBatchId(ringMaster);
Trace.TraceInformation("After first batch. lastAppliedBatchId={0}", lastAppliedBatchId);
Assert.AreEqual(batchId, lastAppliedBatchId);
// Verify that the internal last applied batch id variable has been updated and subsequent
// batches will be accepted.
batchId = lastAppliedBatchId + 1;
RequestBatch secondBatch = CreateBatch(batchId);
Trace.TraceInformation("Sending second batch. batchId={0} executionQueueId={1}", batchId, secondBatch.ExecutionQueueId);
batchResponse = await ringMasterRequestHandler.Request(secondBatch);
Assert.AreEqual((int)RingMasterException.Code.Ok, batchResponse.ResultCode);
lastAppliedBatchId = await QueryRingMasterLastAppliedBatchId(ringMaster);
Trace.TraceInformation("After second batch. lastAppliedBatchId={0}", lastAppliedBatchId);
Assert.AreEqual(batchId, lastAppliedBatchId);
}
}
private async Task IssueRequest(IRingMasterRequestHandler ringMaster, ulong id, string nodePath, CountdownEvent operationsCompletedEvent)
{
try
{
string bulkWatcherPath = $"bulkwatcher:{nodePath}";
operationsCompletedEvent.AddCount();
var timer = Stopwatch.StartNew();
var watcher = new Watcher(id, this.instrumentation);
var installBulkWatcherRequest = new RequestExists(bulkWatcherPath, watcher);
RequestResponse response = await ringMaster.Request(installBulkWatcherRequest);
timer.Stop();
if (response.ResultCode == (int)RingMasterException.Code.Ok)
{
this.instrumentation?.SetWatcherSucceeded(timer.Elapsed);
}
else
{
this.instrumentation?.SetWatcherFailed();
}
}
catch (Exception ex)
{
Trace.TraceError($"SetBulkWatcher failed id={id}, path={nodePath}, exception={ex.ToString()}");
this.instrumentation?.SetWatcherFailed();
}
finally
{
this.semaphore.Release();
operationsCompletedEvent.Signal();
}
}
private void IssueMultiRequest(IRingMasterRequestHandler ringMaster, IReadOnlyList <Op> operations, CountdownEvent operationsCompletedEvent)
{
var multiRequest = new RequestMulti(operations, completeSynchronously: true, uid: 0);
this.semaphore.Wait();
var timer = Stopwatch.StartNew();
int operationsCount = operations.Count;
operationsCompletedEvent.AddCount(operationsCount);
ringMaster.Request(multiRequest).ContinueWith(responseTask =>
{
this.semaphore.Release();
timer.Stop();
try
{
RequestResponse response = responseTask.Result;
if (response.ResultCode == (int)RingMasterException.Code.Ok)
{
this.instrumentation?.DeleteMultiSucceeded(operationsCount, timer.Elapsed);
}
else
{
this.instrumentation?.DeleteMultiFailed(operationsCount);
}
}
catch (Exception)
{
this.instrumentation?.DeleteMultiFailed(operationsCount);
}
finally
{
operationsCompletedEvent.Signal(operationsCount);
}
});
}
/// <summary>
/// Gets the Access Control List associated with a node.
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="path">Node path</param>
/// <param name="stat"><see cref="Stat"/> associated with the node</param>
/// <returns>Task that will resolve on success to a List of <see cref="Acl"/>s associated
/// with the node</returns>
public static async Task <IReadOnlyList <Acl> > GetACL(this IRingMasterRequestHandler ringMaster, string path, IStat stat)
{
RequestResponse response = await ringMaster.Request(
new RequestGetAcl(path, stat));
ThrowIfError(response);
return((IReadOnlyList <Acl>)response.Content);
}
/// <summary>
/// Synchronizes with the given path.
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="path">Node path</param>
/// <returns>Task that tracks the execution of this method</returns>
public static async Task Sync(this IRingMasterRequestHandler ringMaster, string path)
{
RequestResponse response = await ringMaster.Request(
new RequestSync(
path));
ThrowIfError(response);
}
/// <summary>
/// Gets the subtree under a given path.
/// </summary>
/// <param name="ringMaster">Interface to RingMaster.</param>
/// <param name="path">The path to get the subtree of.</param>
/// <param name="retrievalCondition">Request retrieval condition of the form >:[Top]:[ContinuationPath]</param>
/// <param name="options">Request options.</param>
/// <returns>Task that will resolve on success to the root of the subtree under the given path.</returns>
public static async Task <ResponseGetSubtree> GetSubtree(this IRingMasterRequestHandler ringMaster, string path, string retrievalCondition, RequestGetSubtree.GetSubtreeOptions options = RequestGetSubtree.GetSubtreeOptions.None)
{
RequestResponse response = await ringMaster.Request(new RequestGetSubtree(path, retrievalCondition, options));
ThrowIfError(response);
var treeNodeData = TreeNode.Deserialize((byte[])response.Content);
return(new ResponseGetSubtree(treeNodeData, response.ResponsePath));
}
/// <summary>
/// Executes multiple operations in a sequence at the server. No atomicity guarantees are provided.
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="operations">List of operations</param>
/// <param name="mustCompleteSynchronously">If <c>true</c> the server does not complete the operation
/// until all successful operations are guaranteed to be durable (and are applied locally).</param>
/// <returns>Task that will resolve on success to a list of
/// <see cref="OpResult"/>s</returns>
public static async Task <IReadOnlyList <OpResult> > Batch(this IRingMasterRequestHandler ringMaster, IReadOnlyList <Op> operations, bool mustCompleteSynchronously = false)
{
RequestResponse response = await ringMaster.Request(
new RequestBatch(
operations,
mustCompleteSynchronously));
ThrowIfError(response);
return((IReadOnlyList <OpResult>)response.Content);
}
/// <summary>
/// Gets the list of children of the node at the given path.
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="path">Node path</param>
/// <param name="watcher">Watcher interface that receives notifications for changes to this path or null</param>
/// <param name="retrievalCondition">If not null, the retrieval condition in the form >:[top]:[startingChildName].
/// <c>
/// ">:[Top]:[ChildName]" ... returns the elements greater than the [ChildName] limited to Top count
/// so ">:1000:contoso" means give me first 1000 childrens greater than contoso
/// so ">:1000:" means give me first 1000 elements
/// </c>
/// </param>
/// <returns>Task that will resolve on success to the list of names of children of the node</returns>
public static async Task <IReadOnlyList <string> > GetChildren(this IRingMasterRequestHandler ringMaster, string path, IWatcher watcher, string retrievalCondition = null)
{
RequestResponse response = await ringMaster.Request(
new RequestGetChildren(
path,
watcher,
retrievalCondition : retrievalCondition));
ThrowIfError(response);
return((IReadOnlyList <string>)response.Content);
}
/// <summary>
/// Sets the client authentication digest
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="clientId">The client id</param>
/// <returns>Task that tracks the execution of this method</returns>
public static async Task SetAuth(this IRingMasterRequestHandler ringMaster, Id clientId)
{
string clientIdentity = string.Format(
CultureInfo.InvariantCulture,
"{0}:{1}",
clientId.Scheme,
clientId.Identifier);
RequestResponse response = await ringMaster.Request(new RequestSetAuth(clientIdentity));
ThrowIfError(response);
}
/// <summary>
/// Sets the access control list for the node at the given path if
/// the given version matches the current version of the node.
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="path">Node path</param>
/// <param name="acl">Access control list to associate with the node</param>
/// <param name="version">Version to compare with the current version of the node</param>
/// <returns>Task that will resolve on success to the <see cref="Stat"/> associated with
/// the node</returns>
public static async Task <IStat> SetACL(this IRingMasterRequestHandler ringMaster, string path, IReadOnlyList <Acl> acl, int version)
{
RequestResponse response = await ringMaster.Request(
new RequestSetAcl(
path,
acl,
version));
ThrowIfError(response);
return(response.Stat);
}
/// <summary>
/// Moves a node with the given path.
/// </summary>
/// <param name="ringMaster">The ringmaster handler to operate on</param>
/// <param name="pathSrc">Node Path to move</param>
/// <param name="version">Version of the source node</param>
/// <param name="pathDst">Node Path to be parent of the moved node</param>
/// <param name="moveMode">Modifiers for the move operation</param>
/// <returns>Task that will resolve on success to the path to the newly created node</returns>
public static async Task <string> Move(this IRingMasterRequestHandler ringMaster, string pathSrc, int version, string pathDst, MoveMode moveMode)
{
RequestResponse response = await ringMaster.Request(
new RequestMove(
pathSrc,
version,
pathDst,
moveMode));
ThrowIfError(response);
return((string)response.Content);
}
/// <summary>
/// Creates a node with the given path.
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="path">Node Path</param>
/// <param name="data">Data to associate with the node</param>
/// <param name="acl">Access Control List</param>
/// <param name="createMode">Specifies the node will be created</param>
/// <returns>Task that will resolve on success to the path to the newly created node</returns>
public static async Task <string> Create(this IRingMasterRequestHandler ringMaster, string path, byte[] data, IReadOnlyList <Acl> acl, CreateMode createMode)
{
RequestResponse response = await ringMaster.Request(
new RequestCreate(
path,
data,
acl,
createMode));
ThrowIfError(response);
return((string)response.Content);
}
/// <summary>
/// Gets the data and stat associated with the node at the given path.
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="path">Node path</param>
/// <param name="watcher">Watcher interface that receives notifications for changes to this path or null</param>
/// <returns>Task that will resolve on success to the data associated with the node</returns>
public static async Task <Tuple <IStat, byte[]> > GetDataWithStat(this IRingMasterRequestHandler ringMaster, string path, IWatcher watcher)
{
RequestResponse response = await ringMaster.Request(
new RequestGetData(
path,
RequestGetData.GetDataOptions.None,
watcher,
0));
ThrowIfError(response);
return(new Tuple <IStat, byte[]>(response.Stat, (byte[])response.Content));
}
/// <summary>
/// Sets the data for the node at the given path if the given version matches
/// the current version of the node (If the given version is -1, it matches any version).
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="path">Node path</param>
/// <param name="data">Data to associate with the node</param>
/// <param name="version">Version to compare with the current version of the node</param>
/// <returns>Task that will resolve on success to the <see cref="Stat"/> associated with the node</returns>
public static async Task <IStat> SetData(this IRingMasterRequestHandler ringMaster, string path, byte[] data, int version)
{
RequestResponse response = await ringMaster.Request(
new RequestSetData(
path,
data,
version,
dataCommand : false));
ThrowIfError(response);
return(response.Stat);
}
public void SetWatchers(IRingMasterRequestHandler ringMaster, int maxWatchers)
{
if (ringMaster == null)
{
throw new ArgumentNullException(nameof(ringMaster));
}
// Semaphore that limits how many simultaneous active watchers are allowed.
using (var watcherSemaphore = new SemaphoreSlim(maxWatchers, maxWatchers))
{
long lastAssignedUniqueId = 0;
var random = new RandomGenerator();
Trace.TraceInformation($"Set watchers");
while (!this.cancellationToken.IsCancellationRequested)
{
var index = random.GetRandomInt(0, this.nodeList.Count);
var nodePath = this.nodeList[index];
ulong id = (ulong)Interlocked.Increment(ref lastAssignedUniqueId);
var watcher = new Watcher(id, WatcherKind.OneUse, this.instrumentation, watcherSemaphore);
watcherSemaphore.Wait(this.cancellationToken);
var existsRequest = new RequestExists(nodePath, watcher, id);
var timer = Stopwatch.StartNew();
ringMaster.Request(existsRequest).ContinueWith(responseTask =>
{
try
{
timer.Stop();
RequestResponse response = responseTask.Result;
if (response.ResultCode == (int)RingMasterException.Code.Ok)
{
this.instrumentation?.SetWatcherSucceeded(timer.Elapsed);
}
else
{
this.instrumentation?.SetWatcherFailed();
}
}
catch (Exception)
{
this.instrumentation?.SetWatcherFailed();
}
});
}
}
}
/// <summary>
/// Queries the <see cref="Stat"/> of the node with the given path.
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="path">Node path</param>
/// <param name="watcher">Watcher interface that receives notifications for changes to this path or null</param>
/// <param name="ignoreNonodeError">If set to <c>true</c> an exception is not thrown if no node is found at the given path</param>
/// <returns>Task that will resolve on success to the <see cref="Stat"/> associated with the node</returns>
public static async Task <IStat> Exists(this IRingMasterRequestHandler ringMaster, string path, IWatcher watcher, bool ignoreNonodeError)
{
RequestResponse response = await ringMaster.Request(
new RequestExists(
path,
watcher));
if (ignoreNonodeError &&
(RingMasterException.GetCode(response.ResultCode) == RingMasterException.Code.Nonode))
{
return(response.Stat);
}
ThrowIfError(response);
return((IStat)response.Content);
}
/// <summary>
/// Apply a set of delete operations as a multi.
/// </summary>
/// <param name="ringMaster">Interface to RingMaster</param>
/// <param name="deleteOperations">Delete operations to apply as a batch</param>
/// <returns>Async task to indicate the completion of the request</returns>
private async Task DeleteMulti(IRingMasterRequestHandler ringMaster, IReadOnlyList <Op> deleteOperations)
{
var timer = Stopwatch.StartNew();
var multiRequest = new RequestMulti(deleteOperations, completeSynchronously: true);
var response = await ringMaster.Request(multiRequest);
if (response.ResultCode == (int)RingMasterException.Code.Ok)
{
this.deletedCount += deleteOperations.Count;
this.instrumentation?.DeleteMultiSucceeded(this.deletedCount, deleteOperations.Count, timer.Elapsed);
}
else
{
this.instrumentation?.DeleteMultiFailed(this.deletedCount, deleteOperations.Count, timer.Elapsed);
throw RingMasterException.GetException(response);
}
}
/// <summary>
/// Gets the data associated with the node at the given path.
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="path">Node path</param>
/// <param name="options">Options for this request</param>
/// <param name="optionArgument">Argument for options</param>
/// <param name="watcher">Watcher interface that receives notifications for changes to this path or null</param>
/// <returns>Task that will resolve on success to the data associated with the node</returns>
public static async Task <byte[]> GetData(
this IRingMasterRequestHandler ringMaster,
string path,
RequestGetData.GetDataOptions options,
RequestGetData.IGetDataOptionArgument optionArgument,
IWatcher watcher)
{
RequestResponse response = await ringMaster.Request(
new RequestGetData(
path,
options,
optionArgument,
watcher : watcher));
ThrowIfError(response);
return((byte[])response.Content);
}
/// <summary>
/// Queue GetChildren requests.
/// </summary>
/// <param name="ringMaster">RingMaster client</param>
/// <param name="rootPath">Path to enumerate</param>
/// <param name="maxChildren">Maximum number of children to retrieve</param>
public void QueueRequests(IRingMasterRequestHandler ringMaster, string rootPath, int maxChildren)
{
if (ringMaster == null)
{
throw new ArgumentNullException(nameof(ringMaster));
}
var random = new RandomGenerator();
ulong requestId = 0;
Trace.TraceInformation($"Queue GetChildren: path={rootPath}");
while (!this.cancellationToken.IsCancellationRequested)
{
string startingChildName = random.GetRandomName(this.MinNodeNameLength, this.MaxNodeNameLength);
var getChildrenRequest = new RequestGetChildren(rootPath, watcher: null, retrievalCondition: $">:{maxChildren}:{startingChildName}", uid: requestId++);
this.semaphore.Wait();
var timer = Stopwatch.StartNew();
ringMaster.Request(getChildrenRequest).ContinueWith(responseTask =>
{
this.semaphore.Release();
timer.Stop();
try
{
RequestResponse response = responseTask.Result;
if (response.ResultCode == (int)RingMasterException.Code.Ok)
{
var children = (IReadOnlyList <string>)response.Content;
this.instrumentation?.GetChildrenSucceeded(rootPath, children.Count, timer.Elapsed);
}
else
{
this.instrumentation?.GetChildrenFailed(rootPath);
}
}
catch (Exception)
{
this.instrumentation?.GetChildrenFailed(rootPath);
}
});
}
}
public static async Task <bool> Delete(this IRingMasterRequestHandler ringMaster, string path, int version, DeleteMode deletemode = DeleteMode.None)
{
RequestResponse response = await ringMaster.Request(
new RequestDelete(
path,
version,
deletemode));
switch (RingMasterException.GetCode(response.ResultCode))
{
case RingMasterException.Code.Ok: return(true);
case RingMasterException.Code.Nonode: return(false);
default:
break;
}
ThrowIfError(response);
return(false);
}
private void IssueRequest(IRingMasterRequestHandler ringMaster, string nodePath, byte[] nodeData, CountdownEvent operationsCompletedEvent)
{
var setDataRequest = new RequestSetData(nodePath, nodeData, -1);
this.semaphore.Wait();
operationsCompletedEvent.AddCount();
var timer = Stopwatch.StartNew();
ringMaster.Request(setDataRequest).ContinueWith(responseTask =>
{
try
{
this.semaphore.Release();
timer.Stop();
RequestResponse response = responseTask.Result;
if (response.ResultCode == (int)RingMasterException.Code.Ok)
{
this.instrumentation?.SetDataMultiSucceeded(1, timer.Elapsed);
}
else
{
this.instrumentation?.SetDataMultiFailed(1);
}
}
catch (Exception)
{
this.instrumentation?.SetDataMultiFailed(1);
}
finally
{
operationsCompletedEvent.Signal();
}
});
}
public void QueueBatches(IRingMasterRequestHandler ringMaster, int batchLength)
{
if (ringMaster == null)
{
throw new ArgumentNullException(nameof(ringMaster));
}
ulong batchId = 0;
var random = new RandomGenerator();
Trace.TraceInformation($"Queue Exists Batches");
while (!this.cancellationToken.IsCancellationRequested)
{
var operations = new Op[batchLength];
for (int i = 0; i < batchLength; i++)
{
var index = random.GetRandomInt(0, this.nodeList.Count);
var nodePath = this.nodeList[index];
operations[i] = Op.Check(nodePath, -1);
}
var batchRequest = new RequestBatch(operations, completeSynchronously: false, uid: batchId++);
this.semaphore.Wait();
var timer = Stopwatch.StartNew();
ringMaster.Request(batchRequest).ContinueWith(responseTask =>
{
try
{
this.semaphore.Release();
timer.Stop();
int successCount = 0;
int failureCount = 0;
RequestResponse response = responseTask.Result;
if (response.ResultCode == (int)RingMasterException.Code.Ok)
{
var results = (IReadOnlyList <OpResult>)response.Content;
if (results != null)
{
foreach (var result in results)
{
if (result.ErrCode == RingMasterException.Code.Ok)
{
successCount++;
}
else
{
failureCount++;
}
}
}
this.instrumentation?.BatchProcessed(timer.Elapsed, batchRequest.Requests.Count, successCount, failureCount);
}
else
{
this.instrumentation?.BatchFailed(batchRequest.Requests.Count);
}
}
catch (Exception)
{
this.instrumentation?.BatchFailed(batchRequest.Requests.Count);
}
});
}
}