Example #1
0
        public void TestRequestUid()
        {
            var request1 = new RequestCreate(path: "/", data: null, acl: null, createMode: CreateMode.Persistent);
            var request2 = new RequestExists(path: "/", watcher: null);

            var batch1 = new RequestMulti(
                new List <Op>
            {
                Op.Check("/", 1),
                Op.Delete("/a", -1)
            },
                completeSynchronously: false);

            var batch2 = new RequestMulti(
                new List <Op>
            {
                Op.GetChildren("/"),
                Op.Delete("/a", -1)
            },
                completeSynchronously: true);

            Assert.IsTrue(request2.Uid > request1.Uid);
            Assert.IsTrue(batch1.Requests[0].Uid > request2.Uid);
            Assert.IsTrue(batch1.Requests[1].Uid > batch1.Requests[0].Uid);
            Assert.IsTrue(batch1.Uid > batch1.Requests[1].Uid);
            Assert.IsTrue(batch2.Uid > batch1.Uid);
        }
Example #2
0
        /// <summary>
        /// Schedule delete for nodes under the given path
        /// </summary>
        /// <param name="ringMaster">RingMaster client</param>
        /// <param name="rootPath">Root path to the nodes</param>
        public void ScheduledDelete(IRingMasterRequestHandler ringMaster, string rootPath)
        {
            if (ringMaster == null)
            {
                throw new ArgumentNullException(nameof(ringMaster));
            }

            using (var operationsCompletedEvent = new CountdownEvent(1))
            {
                this.instrumentation?.NodeQueuedForDelete(this.nodeList.Count);
                string scheduleName    = Guid.NewGuid().ToString();
                string stagingLocation = $"/$ScheduledDelete/{scheduleName}";
                var    operations      = new List <Op>();
                operations.Add(Op.Move(rootPath, -1, stagingLocation, MoveMode.AllowPathCreationFlag));

                var scheduledOperations = new Op[1];
                scheduledOperations[0] = Op.Delete(stagingLocation, -1, DeleteMode.FastDelete | DeleteMode.CascadeDelete);
                operations.Add(Op.Run(new RequestMulti(scheduledOperations, completeSynchronously: true, scheduledName: scheduleName)));

                var timer = Stopwatch.StartNew();
                Trace.TraceInformation($"DeletePerformance.ScheduledDelete: rootPath={rootPath}, stagingLocation={stagingLocation}, scheduleName={scheduleName}");
                this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);

                operationsCompletedEvent.Signal();
                operationsCompletedEvent.Wait();
                timer.Stop();
                Trace.TraceInformation($"DeletePerformance.ScheduledDelete: scheduleName={scheduleName}, elapsedMilliseconds={timer.ElapsedMilliseconds}");
            }
        }
Example #3
0
        /// <summary>
        /// Recursively delete nodes under the given path
        /// </summary>
        /// <param name="ringMaster">RingMaster client</param>
        /// <param name="rootPath">Root path to the nodes</param>
        public void CascadeDelete(IRingMasterRequestHandler ringMaster, string rootPath)
        {
            if (ringMaster == null)
            {
                throw new ArgumentNullException(nameof(ringMaster));
            }

            using (var operationsCompletedEvent = new CountdownEvent(1))
            {
                this.instrumentation?.NodeQueuedForDelete(this.nodeList.Count);
                var operations = new Op[1];
                operations[0] = Op.Delete(rootPath, version: -1, deletemode: DeleteMode.FastDelete | DeleteMode.CascadeDelete);
                this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);

                operationsCompletedEvent.Signal();
                operationsCompletedEvent.Wait();
            }
        }
Example #4
0
        public void TestBatch()
        {
            using (var requestHandler = new TestRequestHandler())
                using (var client = new RingMasterClient(requestHandler))
                {
                    string path       = this.GetRandomString();
                    var    version    = this.GetRandomInt();
                    var    operations = new List <Op>();
                    operations.Add(Op.Delete(path, version, recursive: false));

                    var expectedResults = new List <OpResult>();
                    expectedResults.Add(new OpResult.DeleteResult());

                    requestHandler.Implementation = request =>
                    {
                        Assert.IsTrue(request is RequestBatch);
                        var batchRequest = (RequestBatch)request;
                        Assert.IsTrue(batchRequest.CompleteSynchronously);
                        Assert.AreEqual(operations.Count, batchRequest.Requests.Count);
                        Assert.IsTrue(batchRequest.Requests[0] is RequestDelete);

                        return(new RequestResponse()
                        {
                            ResultCode = (int)RingMasterException.Code.Ok,
                            Content = expectedResults
                        });
                    };

                    var results = client.Batch(operations, mustCompleteSynchronously: true).Result;
                    Assert.AreEqual(results.Count, expectedResults.Count);
                    Assert.IsTrue(results[0] is OpResult.DeleteResult);

                    try
                    {
                        client.Batch(null, mustCompleteSynchronously: false).Wait();
                        Assert.Fail("Batch call must have thrown ArgumentNullException");
                    }
                    catch (ArgumentException)
                    {
                    }
                }
        }
        /// <inheritdoc />
        protected override void ProcessRecord()
        {
            var deleteMode = DeleteMode.None;

            if (this.FastDelete.IsPresent)
            {
                deleteMode |= DeleteMode.FastDelete;
            }

            if (this.CascadeDelete.IsPresent)
            {
                deleteMode |= DeleteMode.CascadeDelete;
            }

            if (this.SuccessEvenIfNodeDoesntExist.IsPresent)
            {
                deleteMode |= DeleteMode.SuccessEvenIfNodeDoesntExist;
            }

            this.WriteObject(Op.Delete(this.Path, this.Version, deleteMode));
        }
Example #6
0
        private IReadOnlyList <Op> TranslateZkprOpsListToRmOpsList(IReadOnlyList <IZooKeeperRequest> zkprOps)
        {
            List <Op> rmOps = new List <Op>();

            foreach (IZooKeeperRequest zkReq in zkprOps)
            {
                switch (zkReq.RequestType)
                {
                case ZooKeeperRequestType.Create:
                    ZkprProtocolMessages.Create zkCreate = zkReq as ZkprProtocolMessages.Create;
                    IReadOnlyList <Acl>         acls     = this.TranslateZkprAclListToRMAclList(zkCreate.Acls);
                    CreateMode cm = this.TranslateZkprCreatFlagsToRmCreateMode(zkCreate.Flags);

                    rmOps.Add(Op.Create(zkCreate.Path, zkCreate.Data, acls, cm));
                    break;

                case ZooKeeperRequestType.Delete:
                    ZkprProtocolMessages.Delete zkDelete = zkReq as ZkprProtocolMessages.Delete;

                    rmOps.Add(Op.Delete(zkDelete.Path, zkDelete.Version, false));
                    break;

                case ZooKeeperRequestType.SetData:
                    ZkprProtocolMessages.SetData zkSetData = zkReq as ZkprProtocolMessages.SetData;

                    rmOps.Add(Op.SetData(zkSetData.Path, zkSetData.Data, zkSetData.Version));
                    break;

                case ZooKeeperRequestType.Check:
                    ZkprProtocolMessages.Check zkCheck = zkReq as ZkprProtocolMessages.Check;

                    rmOps.Add(Op.Check(zkCheck.Path, zkCheck.Version));
                    break;
                }
            }

            return(rmOps);
        }
Example #7
0
        /// <summary>
        /// Queue Delete requests.
        /// </summary>
        /// <param name="ringMaster">RingMaster client</param>
        /// <param name="batchLength">Number of deletes per multi</param>
        public void QueueDeletes(IRingMasterRequestHandler ringMaster, int batchLength)
        {
            if (ringMaster == null)
            {
                throw new ArgumentNullException(nameof(ringMaster));
            }

            string[] nodePaths = this.nodeList.ToArray();
            Trace.TraceInformation($"DeletePerformance.QueueDeletes: nodePathsLength={nodePaths.Length}, batchLength={batchLength}");

            using (var operationsCompletedEvent = new CountdownEvent(1))
            {
                int nodeCount  = 0;
                var operations = new List <Op>();
                while (!this.cancellationToken.IsCancellationRequested && nodeCount < nodePaths.Length)
                {
                    string nodePath = nodePaths[nodeCount];
                    operations.Add(Op.Delete(nodePath, version: -1, recursive: true));
                    nodeCount++;
                    this.instrumentation?.NodeQueuedForDelete(nodeCount);
                    if (operations.Count >= batchLength)
                    {
                        this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);
                        operations.Clear();
                    }
                }

                if (operations.Count > 0)
                {
                    this.IssueMultiRequest(ringMaster, operations, operationsCompletedEvent);
                }

                operationsCompletedEvent.Signal();
                operationsCompletedEvent.Wait(this.cancellationToken);
            }
        }
Example #8
0
        /// <summary>
        /// updates the health pesudo-nodes
        /// </summary>
        private void UpdateHealthNodes()
        {
            try
            {
                string basepath = "/$metadata/health";
                Dictionary <string, HealthDefinition> health = this.backend.Factory.GetHealth();

                List <Op> ops = new List <Op>();

                foreach (string child in this.self.GetChildren(basepath, false))
                {
                    ops.Add(Op.Delete(basepath + "/" + child, -1, DeleteMode.SuccessEvenIfNodeDoesntExist));
                }

                foreach (KeyValuePair <string, HealthDefinition> line in health)
                {
                    ops.Add(Op.Delete(basepath + "/" + line.Key + " : " + line.Value.Description, -1, DeleteMode.SuccessEvenIfNodeDoesntExist));
                    ops.Add(Op.Create(basepath + "/" + line.Key + " : " + line.Value.Description, null, null, CreateMode.Ephemeral));
                }

                string clusterpath = "/$metadata/clusterreplicaset";

                foreach (string child in this.self.GetChildren(clusterpath, false))
                {
                    ops.Add(Op.Delete(clusterpath + "/" + child, -1));
                }

                ops.Add(Op.Create(clusterpath + "/" + ServiceHealingManager.ToString(this.GetClusterMemberset()), null, null, CreateMode.Ephemeral | CreateMode.SuccessEvenIfNodeExistsFlag));

                this.self.Multi(ops.AsReadOnly(), true, null, 0);
            }
            catch (Exception e)
            {
                Trace.TraceWarning("While UpdateHealthNodes: {0}", e);
            }
        }
Example #9
0
        public void TestWrongChildrenCountAfterFailedMulti()
        {
            TestWrongNumChildrenInStastAsync().GetAwaiter().GetResult();

            async Task TestWrongNumChildrenInStastAsync()
            {
                const string path = "/$rmbvt/test";
                var          stop = false;

                // Create a parent node with 3 children. During the test, the number of children is not expected
                // to change.
                using (var rm = new RingMasterClient(serverAddress, null, null, 10000))
                {
                    var ops = new List <Op>
                    {
                        Op.Create($"{path}/parent/child1", null, null, CreateMode.PersistentAllowPathCreation),
                        Op.Create($"{path}/parent/child2", null, null, CreateMode.PersistentAllowPathCreation),
                        Op.Create($"{path}/parent/child3", null, null, CreateMode.PersistentAllowPathCreation),
                    };

                    await rm.Multi(ops);
                }

                // Start multiple threads to stress the backend
                var tasks = Enumerable.Range(0, 2).Select(_ => Task.Run(async() =>
                {
                    using (var rm = new RingMasterClient(serverAddress, null, null, 10000))
                    {
                        var ops = new List <Op>();

                        while (!stop)
                        {
                            // Randomly add or delete children in Multi
                            ops.Clear();
                            ops.AddRange(
                                Enumerable.Range(1, 3).Select(
                                    x => Op.Delete($"{path}/parent/child{x}", -1, false)));

                            // Add one more operation to fail the multi, so nothing get committed, in other words the
                            // locklist will always abort.
                            ops.Add(Op.GetData(
                                        $"{path}/parent/nonexisting/node",
                                        Azure.Networking.Infrastructure.RingMaster.Requests.RequestGetData.GetDataOptions.None,
                                        null));
                            var result = (await rm.Multi(ops)).Last();
                            Assert.AreEqual(OpCode.Error, result.ResultType);
                            Assert.AreEqual(RingMasterException.Code.Nonode, result.ErrCode);

                            var children = await rm.GetChildren($"{path}/parent", null);
                            var stat     = await rm.Exists($"{path}/parent", null);

                            Assert.AreEqual(
                                children.Count,
                                stat.NumChildren,
                                $"Children count {children.Count} should be consistent with Stat {stat.NumChildren}");
                            Assert.AreEqual(
                                3,
                                stat.NumChildren,
                                "Number of children returned by Exists should not change");
                        }
                    }
                })).ToArray();

                var clock = Stopwatch.StartNew();

                while (clock.Elapsed.TotalMinutes < 60)
                {
                    await Task.Delay(1000);

                    if (tasks.Any(t => t.IsCompleted))
                    {
                        break;
                    }
                }

                stop = true;
                await Task.WhenAll(tasks);
            }
        }
Example #10
0
        /// <summary>
        /// Recursively deletes all the nodes under the given path.
        /// </summary>
        /// <param name="ringMaster">Interface to RingMaster</param>
        /// <param name="path">Path to recursively delete</param>
        /// <param name="cancellationToken">Token to be observed for cancellation signal</param>
        /// <returns>A <see cref="Task"/> that resolves to the number of nodes deleted</returns>
        public async Task <int> Delete(IRingMasterRequestHandler ringMaster, string path, CancellationToken cancellationToken)
        {
            var recursiveDeleteTimer = Stopwatch.StartNew();
            var pendingNodes         = new Stack <NodeState>();
            var deleteOperations     = new List <Op>();

            this.deletedCount = 0;

            try
            {
                pendingNodes.Push(new NodeState(path));

                while (pendingNodes.Count > 0)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    var currentNode = pendingNodes.Pop();

                    if (!currentNode.AllChildrenProcessed)
                    {
                        IReadOnlyList <string> children = await ringMaster.GetChildren(
                            currentNode.Path,
                            watcher : null,
                            retrievalCondition : string.Format(">:{0}:{1}", this.MaxChildrenEnumerationCount, currentNode.StartingChildName));

                        pendingNodes.Push(new NodeState
                        {
                            Path = currentNode.Path,
                            StartingChildName    = children.Count > 0 ? children[children.Count - 1] : string.Empty,
                            AllChildrenProcessed = children.Count < this.MaxChildrenEnumerationCount,
                        });

                        foreach (var child in children)
                        {
                            string childFullPath = (currentNode.Path == "/") ? $"/{child}" : $"{currentNode.Path}/{child}";
                            pendingNodes.Push(new NodeState(childFullPath));
                        }
                    }
                    else
                    {
                        this.instrumentation?.DeleteQueued(this.deletedCount, currentNode.Path);
                        deleteOperations.Add(Op.Delete(currentNode.Path, version: -1));
                    }

                    if (deleteOperations.Count >= this.MaxDeleteBatchLength)
                    {
                        await this.DeleteMulti(ringMaster, deleteOperations);

                        deleteOperations.Clear();
                    }
                }

                if (deleteOperations.Count > 0)
                {
                    await this.DeleteMulti(ringMaster, deleteOperations);
                }

                this.instrumentation?.RecursiveDeleteSucceeded(this.deletedCount, recursiveDeleteTimer.Elapsed);
                return(this.deletedCount);
            }
            catch
            {
                this.instrumentation?.RecursiveDeleteFailed(this.deletedCount, recursiveDeleteTimer.Elapsed);
                throw;
            }
        }
Example #11
0
        /// <summary>
        /// Work load for adding / deleting and checking the number of children in VNET nodes
        /// </summary>
        /// <param name="id">Task sequence number to avoid write conflict</param>
        /// <param name="cancellationToken">Cancellation token to stop the operation</param>
        /// <returns>Async task to indicate the completion of operation</returns>
        private static async Task CheckNodeThread(int id, CancellationToken cancellationToken)
        {
            var lastMzxids          = new long[VnetCount];
            RingMasterClient client = null;
            var createMode          = CreateMode.PersistentAllowPathCreation | CreateMode.SuccessEvenIfNodeExistsFlag;

            while (!cancellationToken.IsCancellationRequested)
            {
                if (client == null)
                {
                    client = new RingMasterClient(
                        connectionString: serverAddress,
                        clientCerts: null,
                        serverCerts: null,
                        requestTimeout: requestTimeout,
                        watcher: null);
                }

                for (int vnetId = 0; vnetId < VnetCount; vnetId++)
                {
                    try
                    {
                        var parent = $"/vnets-{vnetId}/lnms";

                        // Create some children
                        await client.Multi(
                            Enumerable.Range(0, ChildrenCount).Select(n => Op.Create($"{parent}/node-{id}-{n}", null, null, createMode)).ToList(),
                            true);

                        var result = await client.Multi(new Op[] { Op.Check(parent, -1), Op.GetChildren(parent), }, true);

                        // Check number of children is correct -- it must be more than the number of children being created
                        var stat     = ((OpResult.CheckResult)result[0]).Stat;
                        var children = ((OpResult.GetChildrenResult)result[1]).Children;

                        if (stat.NumChildren < MinNodeCount + ChildrenCount)
                        {
                            log($"Task {id}: wrong stat {stat.NumChildren} < {MinNodeCount + ChildrenCount}");
                            totalFailures++;
                        }

                        if (children.Count < MinNodeCount + ChildrenCount)
                        {
                            log($"Task {id}: wrong children {children.Count} < {MinNodeCount + ChildrenCount}");
                            totalFailures++;
                        }

                        if (stat.NumChildren != children.Count)
                        {
                            log($"Task {id}: stat {stat.NumChildren} inconsistent with children {children.Count}");
                            totalFailures++;
                        }

                        if (stat.NumChildren <= 0)
                        {
                            log($"Task {id}: Stat at {parent} is wrong: {stat}");
                            totalFailures++;
                        }

                        // Delete children being added -- the minimal number of children should be still there
                        await client.Multi(
                            Enumerable.Range(0, ChildrenCount).Select(n => Op.Delete($"{parent}/node-{id}-{n}", -1, false)).ToList(),
                            true);

                        result = await client.Multi(new Op[] { Op.Check(parent, -1), Op.GetChildren(parent), }, true);

                        stat     = ((OpResult.CheckResult)result[0]).Stat;
                        children = ((OpResult.GetChildrenResult)result[1]).Children;

                        if (stat.NumChildren < MinNodeCount)
                        {
                            log($"Task {id}: wrong stat {stat.NumChildren} < {MinNodeCount}");
                            totalFailures++;
                        }

                        if (children.Count < MinNodeCount)
                        {
                            log($"Task {id}: wrong children {children.Count} < {MinNodeCount}");
                            totalFailures++;
                        }

                        if (stat.NumChildren != children.Count)
                        {
                            log($"Task {id}: stat {stat.NumChildren} inconsistent with children {children.Count}");
                            totalFailures++;
                        }

                        if (stat.NumChildren <= 0)
                        {
                            log($"Task {id}: Stat at {parent} is wrong: {stat}");
                            totalFailures++;
                        }

                        totalOperationCount++;
                    }
                    catch (Exception ex)
                    {
                        client = null;

                        log($"Task {id}: Exception: {ex.Message}");

                        break;
                    }
                }
            }
        }