public void TestRequestUid()
{
var request1 = new RequestCreate(path: "/", data: null, acl: null, createMode: CreateMode.Persistent);
var request2 = new RequestExists(path: "/", watcher: null);
var batch1 = new RequestMulti(
new List <Op>
{
Op.Check("/", 1),
Op.Delete("/a", -1)
},
completeSynchronously: false);
var batch2 = new RequestMulti(
new List <Op>
{
Op.GetChildren("/"),
Op.Delete("/a", -1)
},
completeSynchronously: true);
Assert.IsTrue(request2.Uid > request1.Uid);
Assert.IsTrue(batch1.Requests[0].Uid > request2.Uid);
Assert.IsTrue(batch1.Requests[1].Uid > batch1.Requests[0].Uid);
Assert.IsTrue(batch1.Uid > batch1.Requests[1].Uid);
Assert.IsTrue(batch2.Uid > batch1.Uid);
}
/// <summary>
/// Schedule delete for nodes under the given path
/// </summary>
/// <param name="ringMaster">RingMaster client</param>
/// <param name="rootPath">Root path to the nodes</param>
public void ScheduledDelete(IRingMasterRequestHandler ringMaster, string rootPath)
{
if (ringMaster == null)
{
throw new ArgumentNullException(nameof(ringMaster));
}
using (var operationsCompletedEvent = new CountdownEvent(1))
{
this.instrumentation?.NodeQueuedForDelete(this.nodeList.Count);
string scheduleName = Guid.NewGuid().ToString();
string stagingLocation = $"/$ScheduledDelete/{scheduleName}";
var operations = new List <Op>();
operations.Add(Op.Move(rootPath, -1, stagingLocation, MoveMode.AllowPathCreationFlag));
var scheduledOperations = new Op[1];
scheduledOperations[0] = Op.Delete(stagingLocation, -1, DeleteMode.FastDelete | DeleteMode.CascadeDelete);
operations.Add(Op.Run(new RequestMulti(scheduledOperations, completeSynchronously: true, scheduledName: scheduleName)));
var timer = Stopwatch.StartNew();
Trace.TraceInformation($"DeletePerformance.ScheduledDelete: rootPath={rootPath}, stagingLocation={stagingLocation}, scheduleName={scheduleName}");
this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);
operationsCompletedEvent.Signal();
operationsCompletedEvent.Wait();
timer.Stop();
Trace.TraceInformation($"DeletePerformance.ScheduledDelete: scheduleName={scheduleName}, elapsedMilliseconds={timer.ElapsedMilliseconds}");
}
}
/// <summary>
/// Recursively delete nodes under the given path
/// </summary>
/// <param name="ringMaster">RingMaster client</param>
/// <param name="rootPath">Root path to the nodes</param>
public void CascadeDelete(IRingMasterRequestHandler ringMaster, string rootPath)
{
if (ringMaster == null)
{
throw new ArgumentNullException(nameof(ringMaster));
}
using (var operationsCompletedEvent = new CountdownEvent(1))
{
this.instrumentation?.NodeQueuedForDelete(this.nodeList.Count);
var operations = new Op[1];
operations[0] = Op.Delete(rootPath, version: -1, deletemode: DeleteMode.FastDelete | DeleteMode.CascadeDelete);
this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);
operationsCompletedEvent.Signal();
operationsCompletedEvent.Wait();
}
}
public void TestBatch()
{
using (var requestHandler = new TestRequestHandler())
using (var client = new RingMasterClient(requestHandler))
{
string path = this.GetRandomString();
var version = this.GetRandomInt();
var operations = new List <Op>();
operations.Add(Op.Delete(path, version, recursive: false));
var expectedResults = new List <OpResult>();
expectedResults.Add(new OpResult.DeleteResult());
requestHandler.Implementation = request =>
{
Assert.IsTrue(request is RequestBatch);
var batchRequest = (RequestBatch)request;
Assert.IsTrue(batchRequest.CompleteSynchronously);
Assert.AreEqual(operations.Count, batchRequest.Requests.Count);
Assert.IsTrue(batchRequest.Requests[0] is RequestDelete);
return(new RequestResponse()
{
ResultCode = (int)RingMasterException.Code.Ok,
Content = expectedResults
});
};
var results = client.Batch(operations, mustCompleteSynchronously: true).Result;
Assert.AreEqual(results.Count, expectedResults.Count);
Assert.IsTrue(results[0] is OpResult.DeleteResult);
try
{
client.Batch(null, mustCompleteSynchronously: false).Wait();
Assert.Fail("Batch call must have thrown ArgumentNullException");
}
catch (ArgumentException)
{
}
}
}
/// <inheritdoc />
protected override void ProcessRecord()
{
var deleteMode = DeleteMode.None;
if (this.FastDelete.IsPresent)
{
deleteMode |= DeleteMode.FastDelete;
}
if (this.CascadeDelete.IsPresent)
{
deleteMode |= DeleteMode.CascadeDelete;
}
if (this.SuccessEvenIfNodeDoesntExist.IsPresent)
{
deleteMode |= DeleteMode.SuccessEvenIfNodeDoesntExist;
}
this.WriteObject(Op.Delete(this.Path, this.Version, deleteMode));
}
private IReadOnlyList <Op> TranslateZkprOpsListToRmOpsList(IReadOnlyList <IZooKeeperRequest> zkprOps)
{
List <Op> rmOps = new List <Op>();
foreach (IZooKeeperRequest zkReq in zkprOps)
{
switch (zkReq.RequestType)
{
case ZooKeeperRequestType.Create:
ZkprProtocolMessages.Create zkCreate = zkReq as ZkprProtocolMessages.Create;
IReadOnlyList <Acl> acls = this.TranslateZkprAclListToRMAclList(zkCreate.Acls);
CreateMode cm = this.TranslateZkprCreatFlagsToRmCreateMode(zkCreate.Flags);
rmOps.Add(Op.Create(zkCreate.Path, zkCreate.Data, acls, cm));
break;
case ZooKeeperRequestType.Delete:
ZkprProtocolMessages.Delete zkDelete = zkReq as ZkprProtocolMessages.Delete;
rmOps.Add(Op.Delete(zkDelete.Path, zkDelete.Version, false));
break;
case ZooKeeperRequestType.SetData:
ZkprProtocolMessages.SetData zkSetData = zkReq as ZkprProtocolMessages.SetData;
rmOps.Add(Op.SetData(zkSetData.Path, zkSetData.Data, zkSetData.Version));
break;
case ZooKeeperRequestType.Check:
ZkprProtocolMessages.Check zkCheck = zkReq as ZkprProtocolMessages.Check;
rmOps.Add(Op.Check(zkCheck.Path, zkCheck.Version));
break;
}
}
return(rmOps);
}
/// <summary>
/// Queue Delete requests.
/// </summary>
/// <param name="ringMaster">RingMaster client</param>
/// <param name="batchLength">Number of deletes per multi</param>
public void QueueDeletes(IRingMasterRequestHandler ringMaster, int batchLength)
{
if (ringMaster == null)
{
throw new ArgumentNullException(nameof(ringMaster));
}
string[] nodePaths = this.nodeList.ToArray();
Trace.TraceInformation($"DeletePerformance.QueueDeletes: nodePathsLength={nodePaths.Length}, batchLength={batchLength}");
using (var operationsCompletedEvent = new CountdownEvent(1))
{
int nodeCount = 0;
var operations = new List <Op>();
while (!this.cancellationToken.IsCancellationRequested && nodeCount < nodePaths.Length)
{
string nodePath = nodePaths[nodeCount];
operations.Add(Op.Delete(nodePath, version: -1, recursive: true));
nodeCount++;
this.instrumentation?.NodeQueuedForDelete(nodeCount);
if (operations.Count >= batchLength)
{
this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);
operations.Clear();
}
}
if (operations.Count > 0)
{
this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);
}
operationsCompletedEvent.Signal();
operationsCompletedEvent.Wait(this.cancellationToken);
}
}
/// <summary>
/// updates the health pesudo-nodes
/// </summary>
private void UpdateHealthNodes()
{
try
{
string basepath = "/$metadata/health";
Dictionary <string, HealthDefinition> health = this.backend.Factory.GetHealth();
List <Op> ops = new List <Op>();
foreach (string child in this.self.GetChildren(basepath, false))
{
ops.Add(Op.Delete(basepath + "/" + child, -1, DeleteMode.SuccessEvenIfNodeDoesntExist));
}
foreach (KeyValuePair <string, HealthDefinition> line in health)
{
ops.Add(Op.Delete(basepath + "/" + line.Key + " : " + line.Value.Description, -1, DeleteMode.SuccessEvenIfNodeDoesntExist));
ops.Add(Op.Create(basepath + "/" + line.Key + " : " + line.Value.Description, null, null, CreateMode.Ephemeral));
}
string clusterpath = "/$metadata/clusterreplicaset";
foreach (string child in this.self.GetChildren(clusterpath, false))
{
ops.Add(Op.Delete(clusterpath + "/" + child, -1));
}
ops.Add(Op.Create(clusterpath + "/" + ServiceHealingManager.ToString(this.GetClusterMemberset()), null, null, CreateMode.Ephemeral | CreateMode.SuccessEvenIfNodeExistsFlag));
this.self.Multi(ops.AsReadOnly(), true, null, 0);
}
catch (Exception e)
{
Trace.TraceWarning("While UpdateHealthNodes: {0}", e);
}
}
public void TestWrongChildrenCountAfterFailedMulti()
{
TestWrongNumChildrenInStastAsync().GetAwaiter().GetResult();
async Task TestWrongNumChildrenInStastAsync()
{
const string path = "/$rmbvt/test";
var stop = false;
// Create a parent node with 3 children. During the test, the number of children is not expected
// to change.
using (var rm = new RingMasterClient(serverAddress, null, null, 10000))
{
var ops = new List <Op>
{
Op.Create($"{path}/parent/child1", null, null, CreateMode.PersistentAllowPathCreation),
Op.Create($"{path}/parent/child2", null, null, CreateMode.PersistentAllowPathCreation),
Op.Create($"{path}/parent/child3", null, null, CreateMode.PersistentAllowPathCreation),
};
await rm.Multi(ops);
}
// Start multiple threads to stress the backend
var tasks = Enumerable.Range(0, 2).Select(_ => Task.Run(async() =>
{
using (var rm = new RingMasterClient(serverAddress, null, null, 10000))
{
var ops = new List <Op>();
while (!stop)
{
// Randomly add or delete children in Multi
ops.Clear();
ops.AddRange(
Enumerable.Range(1, 3).Select(
x => Op.Delete($"{path}/parent/child{x}", -1, false)));
// Add one more operation to fail the multi, so nothing get committed, in other words the
// locklist will always abort.
ops.Add(Op.GetData(
$"{path}/parent/nonexisting/node",
Azure.Networking.Infrastructure.RingMaster.Requests.RequestGetData.GetDataOptions.None,
null));
var result = (await rm.Multi(ops)).Last();
Assert.AreEqual(OpCode.Error, result.ResultType);
Assert.AreEqual(RingMasterException.Code.Nonode, result.ErrCode);
var children = await rm.GetChildren($"{path}/parent", null);
var stat = await rm.Exists($"{path}/parent", null);
Assert.AreEqual(
children.Count,
stat.NumChildren,
$"Children count {children.Count} should be consistent with Stat {stat.NumChildren}");
Assert.AreEqual(
3,
stat.NumChildren,
"Number of children returned by Exists should not change");
}
}
})).ToArray();
var clock = Stopwatch.StartNew();
while (clock.Elapsed.TotalMinutes < 60)
{
await Task.Delay(1000);
if (tasks.Any(t => t.IsCompleted))
{
break;
}
}
stop = true;
await Task.WhenAll(tasks);
}
}
/// <summary>
/// Recursively deletes all the nodes under the given path.
/// </summary>
/// <param name="ringMaster">Interface to RingMaster</param>
/// <param name="path">Path to recursively delete</param>
/// <param name="cancellationToken">Token to be observed for cancellation signal</param>
/// <returns>A <see cref="Task"/> that resolves to the number of nodes deleted</returns>
public async Task <int> Delete(IRingMasterRequestHandler ringMaster, string path, CancellationToken cancellationToken)
{
var recursiveDeleteTimer = Stopwatch.StartNew();
var pendingNodes = new Stack <NodeState>();
var deleteOperations = new List <Op>();
this.deletedCount = 0;
try
{
pendingNodes.Push(new NodeState(path));
while (pendingNodes.Count > 0)
{
cancellationToken.ThrowIfCancellationRequested();
var currentNode = pendingNodes.Pop();
if (!currentNode.AllChildrenProcessed)
{
IReadOnlyList <string> children = await ringMaster.GetChildren(
currentNode.Path,
watcher : null,
retrievalCondition : string.Format(">:{0}:{1}", this.MaxChildrenEnumerationCount, currentNode.StartingChildName));
pendingNodes.Push(new NodeState
{
Path = currentNode.Path,
StartingChildName = children.Count > 0 ? children[children.Count - 1] : string.Empty,
AllChildrenProcessed = children.Count < this.MaxChildrenEnumerationCount,
});
foreach (var child in children)
{
string childFullPath = (currentNode.Path == "/") ? $"/{child}" : $"{currentNode.Path}/{child}";
pendingNodes.Push(new NodeState(childFullPath));
}
}
else
{
this.instrumentation?.DeleteQueued(this.deletedCount, currentNode.Path);
deleteOperations.Add(Op.Delete(currentNode.Path, version: -1));
}
if (deleteOperations.Count >= this.MaxDeleteBatchLength)
{
await this.DeleteMulti(ringMaster, deleteOperations);
deleteOperations.Clear();
}
}
if (deleteOperations.Count > 0)
{
await this.DeleteMulti(ringMaster, deleteOperations);
}
this.instrumentation?.RecursiveDeleteSucceeded(this.deletedCount, recursiveDeleteTimer.Elapsed);
return(this.deletedCount);
}
catch
{
this.instrumentation?.RecursiveDeleteFailed(this.deletedCount, recursiveDeleteTimer.Elapsed);
throw;
}
}
/// <summary>
/// Work load for adding / deleting and checking the number of children in VNET nodes
/// </summary>
/// <param name="id">Task sequence number to avoid write conflict</param>
/// <param name="cancellationToken">Cancellation token to stop the operation</param>
/// <returns>Async task to indicate the completion of operation</returns>
private static async Task CheckNodeThread(int id, CancellationToken cancellationToken)
{
var lastMzxids = new long[VnetCount];
RingMasterClient client = null;
var createMode = CreateMode.PersistentAllowPathCreation | CreateMode.SuccessEvenIfNodeExistsFlag;
while (!cancellationToken.IsCancellationRequested)
{
if (client == null)
{
client = new RingMasterClient(
connectionString: serverAddress,
clientCerts: null,
serverCerts: null,
requestTimeout: requestTimeout,
watcher: null);
}
for (int vnetId = 0; vnetId < VnetCount; vnetId++)
{
try
{
var parent = $"/vnets-{vnetId}/lnms";
// Create some children
await client.Multi(
Enumerable.Range(0, ChildrenCount).Select(n => Op.Create($"{parent}/node-{id}-{n}", null, null, createMode)).ToList(),
true);
var result = await client.Multi(new Op[] { Op.Check(parent, -1), Op.GetChildren(parent), }, true);
// Check number of children is correct -- it must be more than the number of children being created
var stat = ((OpResult.CheckResult)result[0]).Stat;
var children = ((OpResult.GetChildrenResult)result[1]).Children;
if (stat.NumChildren < MinNodeCount + ChildrenCount)
{
log($"Task {id}: wrong stat {stat.NumChildren} < {MinNodeCount + ChildrenCount}");
totalFailures++;
}
if (children.Count < MinNodeCount + ChildrenCount)
{
log($"Task {id}: wrong children {children.Count} < {MinNodeCount + ChildrenCount}");
totalFailures++;
}
if (stat.NumChildren != children.Count)
{
log($"Task {id}: stat {stat.NumChildren} inconsistent with children {children.Count}");
totalFailures++;
}
if (stat.NumChildren <= 0)
{
log($"Task {id}: Stat at {parent} is wrong: {stat}");
totalFailures++;
}
// Delete children being added -- the minimal number of children should be still there
await client.Multi(
Enumerable.Range(0, ChildrenCount).Select(n => Op.Delete($"{parent}/node-{id}-{n}", -1, false)).ToList(),
true);
result = await client.Multi(new Op[] { Op.Check(parent, -1), Op.GetChildren(parent), }, true);
stat = ((OpResult.CheckResult)result[0]).Stat;
children = ((OpResult.GetChildrenResult)result[1]).Children;
if (stat.NumChildren < MinNodeCount)
{
log($"Task {id}: wrong stat {stat.NumChildren} < {MinNodeCount}");
totalFailures++;
}
if (children.Count < MinNodeCount)
{
log($"Task {id}: wrong children {children.Count} < {MinNodeCount}");
totalFailures++;
}
if (stat.NumChildren != children.Count)
{
log($"Task {id}: stat {stat.NumChildren} inconsistent with children {children.Count}");
totalFailures++;
}
if (stat.NumChildren <= 0)
{
log($"Task {id}: Stat at {parent} is wrong: {stat}");
totalFailures++;
}
totalOperationCount++;
}
catch (Exception ex)
{
client = null;
log($"Task {id}: Exception: {ex.Message}");
break;
}
}
}
}
