/// <summary>
/// Create the specified number of nodes nodes directly under the given path.
/// </summary>
/// <param name="ringMaster">RingMaster client</param>
/// <param name="rootPath">Path where the hierarchy must be created</param>
/// <param name="batchLength">Number create requests per multi</param>
/// <param name="maxNodes">Number of nodes to create</param>
public void CreateFlat(IRingMasterRequestHandler ringMaster, string rootPath, int batchLength, int maxNodes)
{
var random = new RandomGenerator();
int nodesCreated = 0;
using (var operationsCompletedEvent = new CountdownEvent(1))
{
var operations = new List <Op>();
while (!this.cancellationToken.IsCancellationRequested && nodesCreated < maxNodes)
{
string childName = random.GetRandomName(this.MinNodeNameLength, this.MaxNodeNameLength);
string childPath = (rootPath == "/") ? $"/{childName}" : $"{rootPath}/{childName}";
byte[] childData = random.GetRandomData(this.MinDataSizePerNode, this.MaxDataSizePerNode);
operations.Add(Op.Create(childPath, childData, null, CreateMode.Persistent | CreateMode.AllowPathCreationFlag));
nodesCreated++;
this.instrumentation?.NodeQueuedForCreate(nodesCreated);
if (operations.Count >= batchLength)
{
this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);
operations.Clear();
}
}
if (operations.Count > 0)
{
this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);
}
operationsCompletedEvent.Signal();
operationsCompletedEvent.Wait(this.cancellationToken);
}
}
/// <summary>
/// Thread to mock a NSM / LNM which is publishing VNET data
/// </summary>
/// <param name="cancel">Cancellation token</param>
/// <param name="threadId">Thread sequence number</param>
/// <param name="operationCount">Object to store operation statistics</param>
private void MockLnmThread(CancellationToken cancel, int threadId, OperationCount operationCount)
{
using (var client = new RingMasterClient(serverAddress, null, null, 10000))
{
var rnd = new Random();
while (!cancel.IsCancellationRequested)
{
Task.Run(async() =>
{
try
{
var vnet = $"/mud/vnets/{CreateSpanningVnetId(threadId)}";
var stat = await client.Exists(vnet, null, true);
var ops = new List <Op>();
if (stat == null)
{
ops.Add(Op.Create($"{vnet}/mappings/v4ca", null, null, CreateMode.PersistentAllowPathCreation));
ops.Add(Op.Create($"{vnet}/lnms/thread-{threadId}", null, null, CreateMode.PersistentAllowPathCreation));
await client.Multi(ops, true);
ops.Clear();
operationCount.AddCreate(2);
}
var mappingCount = rnd.Next(1, 1024 * 8);
for (int i = 0; i < mappingCount; i++)
{
ops.Add(Op.Create($"{vnet}/mappings/v4ca/{i}", null, null, CreateMode.PersistentAllowPathCreation));
operationCount.AddCreate(1);
}
await client.Multi(ops, true);
ops.Clear();
for (int i = 0; i < mappingCount; i++)
{
var data = new byte[rnd.Next(minPayloadSize, maxPayloadSize)];
ops.Add(Op.SetData($"{vnet}/mappings/v4ca/{i}", data, -1));
operationCount.AddSet(1);
operationCount.AddData(data.Length);
}
await client.Multi(ops, true);
ops.Clear();
}
catch (Exception ex)
{
operationCount.IncrementFailure();
// Ignore and keep going
log($"FAIL in {threadId}: {ex.Message}");
}
}).GetAwaiter().GetResult();
}
}
}
/// <summary>
/// Create a hierarchy of nodes under the given path.
/// </summary>
/// <param name="ringMaster">RingMaster client</param>
/// <param name="rootPath">Path where the hierarchy must be created</param>
/// <param name="batchLength">Number create requests per multi</param>
/// <param name="maxNodes">Number of nodes to create</param>
public void CreateHierarchy(IRingMasterRequestHandler ringMaster, string rootPath, int batchLength, int maxNodes)
{
int nodesCreated = 0;
int numNodesToCreate = maxNodes;
var paths = new Queue <string>();
paths.Enqueue(rootPath);
using (var operationsCompletedEvent = new CountdownEvent(1))
{
var random = new RandomGenerator();
var operations = new List <Op>();
while (!this.cancellationToken.IsCancellationRequested && numNodesToCreate > 0)
{
string currentNode = paths.Dequeue();
int numChildrenToCreate = random.GetRandomInt(this.MinChildrenCountPerNode, this.MaxChildrenCountPerNode);
numChildrenToCreate = Math.Min(numChildrenToCreate, numNodesToCreate);
numNodesToCreate -= numChildrenToCreate;
for (int i = 0; i < numChildrenToCreate; i++)
{
string childName = random.GetRandomName(this.MinNodeNameLength, this.MaxNodeNameLength, this.MaxAllowedCodePoint);
string childPath = (currentNode == "/") ? $"/{childName}" : $"{currentNode}/{childName}";
byte[] childData = random.GetRandomData(this.MinDataSizePerNode, this.MaxDataSizePerNode);
paths.Enqueue(childPath);
operations.Add(Op.Create(childPath, childData, null, CreateMode.Persistent | CreateMode.AllowPathCreationFlag));
nodesCreated++;
this.instrumentation?.NodeQueuedForCreate(nodesCreated);
if (operations.Count >= batchLength)
{
this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);
operations.Clear();
}
if (this.cancellationToken.IsCancellationRequested)
{
break;
}
}
}
if (operations.Count > 0)
{
this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);
}
operationsCompletedEvent.Signal();
operationsCompletedEvent.Wait(this.cancellationToken);
}
}
/// <inheritdoc />
protected override void ProcessRecord()
{
CreateMode mode = this.Ephemeral ? CreateMode.Ephemeral : CreateMode.Persistent;
if (this.Sequential.IsPresent)
{
mode |= CreateMode.Sequential;
}
if (this.SucceedEvenIfNodeExists.IsPresent)
{
mode |= CreateMode.SuccessEvenIfNodeExistsFlag;
}
if (this.AllowPathCreation.IsPresent)
{
mode |= CreateMode.AllowPathCreationFlag;
}
this.WriteObject(Op.Create(this.Path, this.Data, this.Acls, mode));
}
/// <summary>
/// Creates the VNET base data before multi-thread operation is started.
/// </summary>
private static async Task CreateBaseData()
{
var client = new RingMasterClient(
connectionString: serverAddress,
clientCerts: null,
serverCerts: null,
requestTimeout: requestTimeout,
watcher: null);
var createMode = CreateMode.PersistentAllowPathCreation | CreateMode.SuccessEvenIfNodeExistsFlag;
for (int vnetId = 0; vnetId < VnetCount; vnetId++)
{
await client.Delete($"/vnets-{vnetId}", -1, true);
await client.Multi(
Enumerable.Range(0, MinNodeCount)
.Select(n => Op.Create($"/vnets-{vnetId}/lnms/dn-{vnetId}-{n}", null, null, createMode))
.ToList(),
true);
}
}
private IReadOnlyList <Op> TranslateZkprOpsListToRmOpsList(IReadOnlyList <IZooKeeperRequest> zkprOps)
{
List <Op> rmOps = new List <Op>();
foreach (IZooKeeperRequest zkReq in zkprOps)
{
switch (zkReq.RequestType)
{
case ZooKeeperRequestType.Create:
ZkprProtocolMessages.Create zkCreate = zkReq as ZkprProtocolMessages.Create;
IReadOnlyList <Acl> acls = this.TranslateZkprAclListToRMAclList(zkCreate.Acls);
CreateMode cm = this.TranslateZkprCreatFlagsToRmCreateMode(zkCreate.Flags);
rmOps.Add(Op.Create(zkCreate.Path, zkCreate.Data, acls, cm));
break;
case ZooKeeperRequestType.Delete:
ZkprProtocolMessages.Delete zkDelete = zkReq as ZkprProtocolMessages.Delete;
rmOps.Add(Op.Delete(zkDelete.Path, zkDelete.Version, false));
break;
case ZooKeeperRequestType.SetData:
ZkprProtocolMessages.SetData zkSetData = zkReq as ZkprProtocolMessages.SetData;
rmOps.Add(Op.SetData(zkSetData.Path, zkSetData.Data, zkSetData.Version));
break;
case ZooKeeperRequestType.Check:
ZkprProtocolMessages.Check zkCheck = zkReq as ZkprProtocolMessages.Check;
rmOps.Add(Op.Check(zkCheck.Path, zkCheck.Version));
break;
}
}
return(rmOps);
}
public void TestConcurrentDelete()
{
TestAsync().GetAwaiter().GetResult();
async Task TestAsync()
{
const int ChildrenCount = 1000;
const string RootName = nameof(this.TestConcurrentDelete);
const int threadCount = 64;
using (var client = new RingMasterClient(serverAddress, null, null, 10000))
{
await client.Create($"/{RootName}", null, null, CreateMode.PersistentAllowPathCreation).ConfigureAwait(false);
var ops = new List <Op>(ChildrenCount);
for (int count = 0; count < ChildrenCount; count++)
{
ops.Add(Op.Create($"/{RootName}/{count}", null, null, CreateMode.PersistentAllowPathCreation));
}
await client.Batch(ops).ConfigureAwait(false);
}
for (int i = 0; i < threadCount; i++)
{
var deleteChildTask = this.DeleteChild(RootName, ChildrenCount);
var deleteParent = Task.Run(async() =>
{
using (var client = new RingMasterClient(serverAddress, null, null, 10000))
{
await client.Delete($"/{RootName}", -1, DeleteMode.None);
}
});
}
}
}
/// <summary>
/// updates the health pesudo-nodes
/// </summary>
private void UpdateHealthNodes()
{
try
{
string basepath = "/$metadata/health";
Dictionary <string, HealthDefinition> health = this.backend.Factory.GetHealth();
List <Op> ops = new List <Op>();
foreach (string child in this.self.GetChildren(basepath, false))
{
ops.Add(Op.Delete(basepath + "/" + child, -1, DeleteMode.SuccessEvenIfNodeDoesntExist));
}
foreach (KeyValuePair <string, HealthDefinition> line in health)
{
ops.Add(Op.Delete(basepath + "/" + line.Key + " : " + line.Value.Description, -1, DeleteMode.SuccessEvenIfNodeDoesntExist));
ops.Add(Op.Create(basepath + "/" + line.Key + " : " + line.Value.Description, null, null, CreateMode.Ephemeral));
}
string clusterpath = "/$metadata/clusterreplicaset";
foreach (string child in this.self.GetChildren(clusterpath, false))
{
ops.Add(Op.Delete(clusterpath + "/" + child, -1));
}
ops.Add(Op.Create(clusterpath + "/" + ServiceHealingManager.ToString(this.GetClusterMemberset()), null, null, CreateMode.Ephemeral | CreateMode.SuccessEvenIfNodeExistsFlag));
this.self.Multi(ops.AsReadOnly(), true, null, 0);
}
catch (Exception e)
{
Trace.TraceWarning("While UpdateHealthNodes: {0}", e);
}
}
/// <summary>
/// Tests create scenario using either batch or multi
/// </summary>
/// <param name="batch">true if using batch, false if using multi</param>
/// <param name="operationType">Type of the operation.</param>
/// <param name="jobState">the job state</param>
/// <returns>
/// Request per second
/// </returns>
protected async Task <double> TestBatchOrMultiCreate(bool batch, Test.Helpers.OperationType operationType, JobState jobState)
{
var name = batch ? "Batch" : "Multi";
Random rnd = new Random();
// number of large trees will be a random number between (20, 50)
this.LargeTreeRoots = Enumerable.Range(0, rnd.Next(20, 50)).Select(x => Guid.NewGuid()).ToList();
return(await this.TestFlowAsync(
$"{name}(Create) node perf test",
operationType,
async (client, cancellationToken, threadId) =>
{
var clock = Stopwatch.StartNew();
var rootName = $"{this.RootNodeName}_{name}";
while (!cancellationToken.IsCancellationRequested)
{
var ops = new List <Op>(this.BatchOpCount);
var totalSize = 0;
int smallTreeNodeCount = rnd.Next(this.BatchOpCount / 2);
int bigTreeNodeCount = this.BatchOpCount - smallTreeNodeCount;
var smallSubtreeId = Guid.NewGuid();
while (smallTreeNodeCount-- > 0)
{
var data = Helpers.MakeRandomData(rnd, rnd.Next(this.MinDataSize, this.MaxDataSize));
totalSize += data.Length;
ops.Add(Op.Create($"/{rootName}/vnet{smallSubtreeId}/mappings/v4ca/{Guid.NewGuid()}", data, null, CreateMode.PersistentAllowPathCreation));
}
while (bigTreeNodeCount-- > 0)
{
var data = Helpers.MakeRandomData(rnd, rnd.Next(this.MinDataSize, this.MaxDataSize));
totalSize += data.Length;
int idx = rnd.Next(this.LargeTreeRoots.Count);
ops.Add(Op.Create($"/{rootName}/vnet{this.LargeTreeRoots[idx]}/mappings/v4ca/{Guid.NewGuid()}", data, null, CreateMode.PersistentAllowPathCreation));
}
try
{
var startTime = clock.Elapsed;
if (batch)
{
await client.Batch(ops).ConfigureAwait(false);
}
else
{
await client.Multi(ops).ConfigureAwait(false);
}
var duration = clock.Elapsed - startTime;
MdmHelper.LogOperationDuration((long)duration.TotalMilliseconds, operationType);
this.AddTotalDataSize(totalSize);
this.IncrementTotalDataCount(ops.Count);
}
catch (Exception ex)
{
this.IncrementTotalFailures();
this.Log($"Failed to call {name}: {ex.Message}");
}
}
},
jobState,
this.TestCaseSeconds));
}
// [TestMethod]
public void ValidateScheduler()
{
bool ok = false;
TestEvents ev = new TestEvents();
TestMarshaller marshaller = new TestMarshaller(ev);
TestRingMaster testRM = new TestRingMaster(ev, marshaller);
ScheduledCommand scheduler = null;
try
{
foreach (string path in ScheduledCommand.GetPaths())
{
testRM.Create(path, null, null, CreateMode.PersistentAllowPathCreation);
}
scheduler = new ScheduledCommand(() => { return(testRM.IsPrimary); }, testRM, marshaller);
scheduler.InternalOnAbandon = new Action <Exception>(ex =>
{
Assert.Fail("Scheduler Abandoned: " + ex);
});
scheduler.Start();
this.VerifySetup(ev, TimeSpan.FromSeconds(15));
string res = testRM.Create("/$metadata/scheduler/commands/c1", null, null, CreateMode.Persistent);
this.VerifyCommandTaken(ev, TimeSpan.FromSeconds(10), "c1");
this.VerifyCommandFailed(ev, TimeSpan.FromSeconds(10), "c1");
List <Op> ops = new List <Op>();
ops.Add(Op.Create("/$metadata/scheduler/commands/c2", null, null, CreateMode.Persistent));
ops.Add(Op.Create("/$metadata/scheduler/commands/c4", null, null, CreateMode.Persistent));
ops.Add(Op.Create("/$metadata/scheduler/commands/c3", null, null, CreateMode.Persistent));
IReadOnlyList <OpResult> resL = testRM.Multi(ops.AsReadOnly(), completeSynchronously: true, scheduledName: null);
this.VerifyCommandTaken(ev, TimeSpan.FromSeconds(10), "c2");
this.VerifyCommandTaken(ev, TimeSpan.FromSeconds(10), "c3");
this.VerifyCommandTaken(ev, TimeSpan.FromSeconds(10), "c4");
this.VerifyCommandFailed(ev, TimeSpan.FromSeconds(10), "c2", "c3", "c4");
ops.Clear();
ops.Add(Op.Create("/test1", null, null, CreateMode.Persistent));
resL = testRM.Multi(ops.AsReadOnly(), completeSynchronously: true, scheduledName: "c5");
this.VerifyCommandTaken(ev, TimeSpan.FromSeconds(10), "c5");
this.VerifyCommandSucceeded(ev, TimeSpan.FromSeconds(10), "c5");
this.VerifyCreated(ev, TimeSpan.FromSeconds(10), "/test1");
Assert.IsTrue(testRM.Exists("/test1", false, false) != null);
Assert.IsTrue(testRM.Exists("/$metadata/scheduler/commands/c5", false, false) == null);
ok = true;
}
finally
{
if (!ok)
{
testRM.DumpNodes();
}
ThreadPool.QueueUserWorkItem(_ =>
{
if (scheduler != null)
{
scheduler.Close();
}
testRM.Close();
});
}
}
public void TestWrongChildrenCountAfterFailedMulti()
{
TestWrongNumChildrenInStastAsync().GetAwaiter().GetResult();
async Task TestWrongNumChildrenInStastAsync()
{
const string path = "/$rmbvt/test";
var stop = false;
// Create a parent node with 3 children. During the test, the number of children is not expected
// to change.
using (var rm = new RingMasterClient(serverAddress, null, null, 10000))
{
var ops = new List <Op>
{
Op.Create($"{path}/parent/child1", null, null, CreateMode.PersistentAllowPathCreation),
Op.Create($"{path}/parent/child2", null, null, CreateMode.PersistentAllowPathCreation),
Op.Create($"{path}/parent/child3", null, null, CreateMode.PersistentAllowPathCreation),
};
await rm.Multi(ops);
}
// Start multiple threads to stress the backend
var tasks = Enumerable.Range(0, 2).Select(_ => Task.Run(async() =>
{
using (var rm = new RingMasterClient(serverAddress, null, null, 10000))
{
var ops = new List <Op>();
while (!stop)
{
// Randomly add or delete children in Multi
ops.Clear();
ops.AddRange(
Enumerable.Range(1, 3).Select(
x => Op.Delete($"{path}/parent/child{x}", -1, false)));
// Add one more operation to fail the multi, so nothing get committed, in other words the
// locklist will always abort.
ops.Add(Op.GetData(
$"{path}/parent/nonexisting/node",
Azure.Networking.Infrastructure.RingMaster.Requests.RequestGetData.GetDataOptions.None,
null));
var result = (await rm.Multi(ops)).Last();
Assert.AreEqual(OpCode.Error, result.ResultType);
Assert.AreEqual(RingMasterException.Code.Nonode, result.ErrCode);
var children = await rm.GetChildren($"{path}/parent", null);
var stat = await rm.Exists($"{path}/parent", null);
Assert.AreEqual(
children.Count,
stat.NumChildren,
$"Children count {children.Count} should be consistent with Stat {stat.NumChildren}");
Assert.AreEqual(
3,
stat.NumChildren,
"Number of children returned by Exists should not change");
}
}
})).ToArray();
var clock = Stopwatch.StartNew();
while (clock.Elapsed.TotalMinutes < 60)
{
await Task.Delay(1000);
if (tasks.Any(t => t.IsCompleted))
{
break;
}
}
stop = true;
await Task.WhenAll(tasks);
}
}
public void TestGetFullSubtreeWhileUpdating()
{
TestAsync().GetAwaiter().GetResult();
async Task TestAsync()
{
const int InitialNodeData = 1;
const int NewNodeData = 2;
const int ChildrenCount = 50000;
const string RootName = nameof(this.TestGetFullSubtreeWhileUpdating);
using (var client = new RingMasterClient(serverAddress, null, null, 100000))
{
byte[] data = BitConverter.GetBytes(InitialNodeData);
await client.Create($"/{RootName}/node1", data, null, CreateMode.PersistentAllowPathCreation).ConfigureAwait(false);
await client.Create($"/{RootName}/node2", data, null, CreateMode.PersistentAllowPathCreation).ConfigureAwait(false);
await client.Create($"/{RootName}/node3", data, null, CreateMode.PersistentAllowPathCreation).ConfigureAwait(false);
var ops = new List <Op>(ChildrenCount);
for (int count = 0; count < ChildrenCount; count++)
{
ops.Add(Op.Create($"/{RootName}/node2/{count}", data, null, CreateMode.PersistentAllowPathCreation));
}
await client.Batch(ops).ConfigureAwait(false);
}
ManualResetEvent manualResetEvent = new ManualResetEvent(false);
Task <TreeNode> getSubtreeTask = new Task <TreeNode>(() =>
{
using (var client = new RingMasterClient(serverAddress, null, null, 10000))
{
return(client.GetFullSubtree($"/{RootName}").Result);
}
});
Task updateDataTask = Task.Run(async() =>
{
using (var client = new RingMasterClient(serverAddress, null, null, 10000))
{
var ops = new List <Op>(2);
byte[] newData = BitConverter.GetBytes(NewNodeData);
ops.Add(Op.SetData($"/{RootName}/node1", newData, -1));
ops.Add(Op.SetData($"/{RootName}/node3", newData, -1));
manualResetEvent.WaitOne();
// this is to make sure the set data occurs after get full substree started.
Thread.Sleep(20);
await client.Batch(ops).ConfigureAwait(false);
}
});
getSubtreeTask.Start();
manualResetEvent.Set();
await Task.WhenAll(getSubtreeTask, updateDataTask);
var tree = getSubtreeTask.Result;
int node1Data = BitConverter.ToInt32(tree.Children[0].Data, 0);
int node3Data = BitConverter.ToInt32(tree.Children[2].Data, 0);
Assert.IsTrue(node1Data >= node3Data);
}
}
private async Task LnmPublishingThread(IRingMasterRequestHandler client, CancellationToken token, int threadId)
{
var rnd = new Random();
while (!token.IsCancellationRequested)
{
try
{
var vnet = $"/Instance{this.ServiceContext.ReplicaOrInstanceId}/vnets/{CreateSpanningVnetId(threadId)}";
var stat = await client.Exists(vnet, null, true);
var ops = new List <Op>();
if (stat == null)
{
ops.Add(Op.Create($"{vnet}/mappings/v4ca", null, null, CreateMode.PersistentAllowPathCreation));
ops.Add(Op.Create($"{vnet}/lnms/thread-{threadId}", null, null, CreateMode.PersistentAllowPathCreation));
await client.Multi(ops, true);
ops.Clear();
this.IncrementTotalDataCount(2);
}
var mappingCount = rnd.Next(1, 1024 * 8);
for (int i = 0; i < mappingCount; i++)
{
ops.Add(Op.Create($"{vnet}/mappings/v4ca/{i}", null, null, CreateMode.PersistentAllowPathCreation));
}
this.IncrementTotalDataCount(ops.Count);
if (token.IsCancellationRequested)
{
return;
}
await client.Multi(ops, true);
ops.Clear();
for (int i = 0; i < mappingCount; i++)
{
var data = new byte[rnd.Next(this.MinDataSize, this.MaxDataSize)];
ops.Add(Op.SetData($"{vnet}/mappings/v4ca/{i}", data, -1));
this.AddTotalDataSize(data.Length);
}
this.IncrementTotalDataCount(mappingCount);
if (token.IsCancellationRequested)
{
return;
}
await client.Multi(ops, true);
ops.Clear();
}
catch (Exception ex)
{
this.IncrementTotalFailures();
// Ignore and keep going
this.Log($"FAIL in {threadId}: {ex.Message}");
}
}
}
/// <summary>
/// Work load for adding / deleting and checking the number of children in VNET nodes
/// </summary>
/// <param name="id">Task sequence number to avoid write conflict</param>
/// <param name="cancellationToken">Cancellation token to stop the operation</param>
/// <returns>Async task to indicate the completion of operation</returns>
private static async Task CheckNodeThread(int id, CancellationToken cancellationToken)
{
var lastMzxids = new long[VnetCount];
RingMasterClient client = null;
var createMode = CreateMode.PersistentAllowPathCreation | CreateMode.SuccessEvenIfNodeExistsFlag;
while (!cancellationToken.IsCancellationRequested)
{
if (client == null)
{
client = new RingMasterClient(
connectionString: serverAddress,
clientCerts: null,
serverCerts: null,
requestTimeout: requestTimeout,
watcher: null);
}
for (int vnetId = 0; vnetId < VnetCount; vnetId++)
{
try
{
var parent = $"/vnets-{vnetId}/lnms";
// Create some children
await client.Multi(
Enumerable.Range(0, ChildrenCount).Select(n => Op.Create($"{parent}/node-{id}-{n}", null, null, createMode)).ToList(),
true);
var result = await client.Multi(new Op[] { Op.Check(parent, -1), Op.GetChildren(parent), }, true);
// Check number of children is correct -- it must be more than the number of children being created
var stat = ((OpResult.CheckResult)result[0]).Stat;
var children = ((OpResult.GetChildrenResult)result[1]).Children;
if (stat.NumChildren < MinNodeCount + ChildrenCount)
{
log($"Task {id}: wrong stat {stat.NumChildren} < {MinNodeCount + ChildrenCount}");
totalFailures++;
}
if (children.Count < MinNodeCount + ChildrenCount)
{
log($"Task {id}: wrong children {children.Count} < {MinNodeCount + ChildrenCount}");
totalFailures++;
}
if (stat.NumChildren != children.Count)
{
log($"Task {id}: stat {stat.NumChildren} inconsistent with children {children.Count}");
totalFailures++;
}
if (stat.NumChildren <= 0)
{
log($"Task {id}: Stat at {parent} is wrong: {stat}");
totalFailures++;
}
// Delete children being added -- the minimal number of children should be still there
await client.Multi(
Enumerable.Range(0, ChildrenCount).Select(n => Op.Delete($"{parent}/node-{id}-{n}", -1, false)).ToList(),
true);
result = await client.Multi(new Op[] { Op.Check(parent, -1), Op.GetChildren(parent), }, true);
stat = ((OpResult.CheckResult)result[0]).Stat;
children = ((OpResult.GetChildrenResult)result[1]).Children;
if (stat.NumChildren < MinNodeCount)
{
log($"Task {id}: wrong stat {stat.NumChildren} < {MinNodeCount}");
totalFailures++;
}
if (children.Count < MinNodeCount)
{
log($"Task {id}: wrong children {children.Count} < {MinNodeCount}");
totalFailures++;
}
if (stat.NumChildren != children.Count)
{
log($"Task {id}: stat {stat.NumChildren} inconsistent with children {children.Count}");
totalFailures++;
}
if (stat.NumChildren <= 0)
{
log($"Task {id}: Stat at {parent} is wrong: {stat}");
totalFailures++;
}
totalOperationCount++;
}
catch (Exception ex)
{
client = null;
log($"Task {id}: Exception: {ex.Message}");
break;
}
}
}
}
public void TestIsReadOnly()
{
Assert.IsTrue(new RequestCheck(path: "/", version: 1).IsReadOnly());
Assert.IsFalse(new RequestCreate(path: "/", data: null, acl: null, createMode: CreateMode.Persistent).IsReadOnly());
Assert.IsFalse(new RequestDelete(path: "/", version: 1, cascade: false).IsReadOnly());
Assert.IsTrue(new RequestExists(path: "/", watcher: null).IsReadOnly());
Assert.IsTrue(new RequestGetAcl(path: "/", stat: null).IsReadOnly());
Assert.IsTrue(new RequestGetChildren(path: "/", watcher: null, retrievalCondition: null).IsReadOnly());
Assert.IsTrue(new RequestGetData(path: "/", options: RequestGetData.GetDataOptions.None, watcher: null).IsReadOnly());
Assert.IsFalse(new RequestInit(sessionId: 0, sessionPwd: "abc", readOnlyInterfaceRequiresLocks: true, redirection: RequestInit.RedirectionPolicy.ServerDefault).IsReadOnly());
Assert.IsFalse(new RequestSetAcl(path: "/", acl: null, version: -1).IsReadOnly());
Assert.IsFalse(new RequestSetAuth(clientId: "abc").IsReadOnly());
Assert.IsFalse(new RequestSetData(path: "/", data: null, version: 1, dataCommand: false).IsReadOnly());
Assert.IsTrue(new RequestSync(path: "/").IsReadOnly());
Assert.IsTrue(new RequestGetSubtree(path: "/", retrievalCondition: null).IsReadOnly());
Assert.IsTrue(new RequestMulti(new IRingMasterRequest[0], completeSynchronously: false).IsReadOnly());
var readMulti = new RequestMulti(
new List <Op>
{
Op.Check("/", 1),
Op.GetData("/a", RequestGetData.GetDataOptions.None, Op.Check("/a", 1))
},
completeSynchronously: false);
Assert.IsTrue(readMulti.IsReadOnly());
var readWriteMulti = new RequestMulti(
new List <Op>
{
Op.Check("/a", 1),
Op.Create("/a/b", null, null, CreateMode.Ephemeral)
},
completeSynchronously: true);
Assert.IsFalse(readWriteMulti.IsReadOnly());
var nestedReadMulti = new RequestMulti(
new IRingMasterRequest[]
{
readMulti
},
completeSynchronously: false);
Assert.IsTrue(nestedReadMulti.IsReadOnly());
var nestedReadWriteMulti = new RequestMulti(
new IRingMasterRequest[]
{
readMulti,
readWriteMulti
},
completeSynchronously: true);
Assert.IsFalse(nestedReadWriteMulti.IsReadOnly());
Assert.IsTrue(new RequestBatch(new IRingMasterRequest[0], completeSynchronously: false).IsReadOnly());
var readBatch = new RequestBatch(
new List <Op>
{
Op.Check("/", 1),
Op.GetData("/a", RequestGetData.GetDataOptions.None, Op.Check("/a", 1))
},
completeSynchronously: false);
Assert.IsTrue(readBatch.IsReadOnly());
var readWriteBatch = new RequestBatch(
new List <Op>
{
Op.Check("/a", 1),
Op.Create("/a/b", null, null, CreateMode.Ephemeral)
},
completeSynchronously: false);
Assert.IsFalse(readWriteBatch.IsReadOnly());
var nestedReadBatch = new RequestBatch(
new IRingMasterRequest[]
{
readMulti,
readBatch
},
completeSynchronously: false);
Assert.IsTrue(nestedReadBatch.IsReadOnly());
var nestedReadWriteBatch = new RequestMulti(
new IRingMasterRequest[]
{
readBatch,
readWriteMulti
},
completeSynchronously: false);
Assert.IsFalse(nestedReadWriteBatch.IsReadOnly());
}