Ejemplo n.º 1
0
        private async Task <bool> CreateNodeTree(CancellationToken cancellation)
        {
            this.totalDataCount = 0;
            this.totalDataSize  = 0;
            var startTime = this.stopwatch.Elapsed;

            var createTask = Helpers.ForEachAsync(
                Enumerable.Range(0, this.partitionCount),
                async(partitionIndex) =>
            {
                var rnd = new Random();

                for (int i = 0; i < this.nodeCountPerPartition && !cancellation.IsCancellationRequested; i++)
                {
                    var path = $"{this.partitionKeyPrefix}{partitionIndex}/{RelativePathPrefix}/{i}";
                    var data = Helpers.MakeRandomData(rnd, rnd.Next(this.minDataSize, this.maxDataSize));

                    try
                    {
                        var operationStartTime = this.stopwatch.Elapsed;
                        await this.clients[this.partitionCount % this.channelCount].Create(path, data, null, CreateMode.PersistentAllowPathCreation, false).ConfigureAwait(false);
                        var operationDuration = this.stopwatch.Elapsed - operationStartTime;
                        MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, Test.Helpers.OperationType.BulkWatcherCreateNode);
                        this.CreateLatency.Add(operationDuration.TotalMilliseconds);

                        Interlocked.Add(ref this.totalDataSize, data.Length);
                        Interlocked.Increment(ref this.totalDataCount);
                    }
                    catch (Exception ex)
                    {
                        this.log($"Failed to create {path}: {ex.Message}");
                    }
                }
            },
                this.threadCount);

            await Task.WhenAny(createTask, Task.Delay(this.requestTimeout)).ConfigureAwait(false);

            if (!createTask.IsCompleted && this.totalDataCount == 0)
            {
                // If no data is created successfully within timeout time, don't bother to read any more.
                return(false);
            }
            else
            {
                // It is making progress. Wait until it's completed.
                await createTask.ConfigureAwait(false);
            }

            var duration = (this.stopwatch.Elapsed - startTime).TotalSeconds;
            var bps      = this.totalDataSize / duration;
            var qps      = this.totalDataCount / duration;

            this.log($"{nameof(this.CreateNodeTree)}: {this.totalDataCount} nodes created, total data size is {this.totalDataSize}. Rate: {bps:G4} byte/sec {qps:G4} /sec");
            MdmHelper.LogBytesProcessed(this.totalDataSize, Test.Helpers.OperationType.BulkWatcherCreateNode);

            return(true);
        }
Ejemplo n.º 2
0
        private async Task <bool> ReadNodeTree(CancellationToken cancellation)
        {
            totalDataCount = 0;
            totalDataSize  = 0;
            var startTime = stopwatch.Elapsed;

            var readTask = Helpers.ForEachAsync(
                Enumerable.Range(0, partitionCount),
                async(partitionIndex) =>
            {
                for (int i = 0; i < nodeCountPerPartition && !cancellation.IsCancellationRequested; i++)
                {
                    var path = $"{PartitionKeyPrefix}{partitionIndex}/{RelativePathPrefix}/{i}";

                    try
                    {
                        var operationStartTime = stopwatch.Elapsed;
                        var data = await clients[partitionCount % channelCount].GetData(path, false).ConfigureAwait(false);
                        var operationDuration = stopwatch.Elapsed - operationStartTime;
                        MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, OperationType.BulkWatcherReadNode);

                        Interlocked.Add(ref totalDataSize, data.Length);
                        Interlocked.Increment(ref totalDataCount);
                    }
                    catch (Exception ex)
                    {
                        log($"Failed to read {path}: {ex.Message}");
                    }
                }
            },
                threadCount);

            await Task.WhenAny(readTask, Task.Delay(requestTimeout)).ConfigureAwait(false);

            if (!readTask.IsCompleted && totalDataCount == 0)
            {
                // If no data is read successfully within timeout time, don't bother to read any more.
                return(false);
            }
            else
            {
                // It is making progress. Wait until it's completed.
                await readTask.ConfigureAwait(false);
            }

            var duration = (stopwatch.Elapsed - startTime).TotalSeconds;
            var bps      = totalDataSize / duration;
            var qps      = totalDataCount / duration;

            log($"{nameof(this.ReadNodeTree)}: {totalDataCount} nodes read, total data size is {totalDataSize}. Rate: {bps:G4} byte/sec {qps:G4} /sec");
            MdmHelper.LogBytesProcessed(totalDataSize, OperationType.BulkWatcherReadNode);

            return(true);
        }
Ejemplo n.º 3
0
        private async Task ChangeRandomNodeInTree()
        {
            this.totalDataCount = 0;
            this.totalDataSize  = 0;
            var startTime = this.stopwatch.Elapsed;

            await Helpers.ForEachAsync(
                Enumerable.Range(0, this.partitionCount),
                async (partitionIndex) =>
            {
                var rnd = new Random();

                var index = rnd.Next(this.nodeCountPerPartition);
                var path  = $"{this.partitionKeyPrefix}{partitionIndex}/{RelativePathPrefix}/{index}";
                var data  = Helpers.MakeRandomData(rnd, rnd.Next(this.minDataSize, this.maxDataSize));

                try
                {
                    var operationStartTime = this.stopwatch.Elapsed;
                    await this.clients[partitionIndex % this.channelCount].SetData(path, data, -1).ConfigureAwait(false);
                    var operationDuration = this.stopwatch.Elapsed - operationStartTime;
                    MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, Test.Helpers.OperationType.BulkWatcherChangeNode);
                    this.SetLatency.Add(operationDuration.TotalMilliseconds);

                    Interlocked.Add(ref this.totalDataSize, data.Length);
                    Interlocked.Increment(ref this.totalDataCount);
                }
                catch (Exception ex)
                {
                    this.log($"Failed to set {path}: {ex.Message}");
                }
            },
                this.threadCount)
            .ConfigureAwait(false);

            var duration = (this.stopwatch.Elapsed - startTime).TotalSeconds;
            var bps      = this.totalDataSize / duration;
            var qps      = this.totalDataCount / duration;

            this.log($"{nameof(this.ChangeRandomNodeInTree)}: {this.totalDataCount} nodes updated, total data size is {this.totalDataSize}. Rate: {bps:G4} byte/sec {qps:G4} /sec");
            MdmHelper.LogBytesProcessed(this.totalDataSize, Test.Helpers.OperationType.BulkWatcherChangeNode);
        }
Ejemplo n.º 4
0
        /// <summary>
        /// Main test workflow
        /// </summary>
        /// <param name="testTitle">Title of the test case</param>
        /// <param name="operationType">the operation type</param>
        /// <param name="workload">Workload in each thread</param>
        /// <param name="jobState">the job state</param>
        /// <param name="durationInSeconds">How long the test should run</param>
        /// <returns>Number of operations per second</returns>
        protected async Task <double> TestFlowAsync(
            string testTitle,
            Test.Helpers.OperationType operationType,
            Func <IRingMasterRequestHandler, CancellationToken, int, Task> workload,
            JobState jobState,
            int durationInSeconds)
        {
            this.ResetCounts();

            var cancellation = new CancellationTokenSource();

            this.Log($"Starting test {testTitle} in {this.threadCount} threads");

            var lastCount = Interlocked.Read(ref this.totalDataCount);
            var lastSize  = Interlocked.Read(ref this.totalDataSize);

            var threads = Helpers.StartMultipleThreads(
                this.threadCount,
                (object n) => workload(this.clients[(int)n], cancellation.Token, (int)n).GetAwaiter().GetResult());

            var initialCount = lastCount;
            var initialSize  = lastSize;
            var stopwatch    = Stopwatch.StartNew();

            for (int i = 0; i < durationInSeconds; i++)
            {
                await Task.Delay(TimeSpan.FromSeconds(1));

                long size  = Interlocked.Read(ref this.totalDataSize);
                long delta = size - lastSize;

                long count      = Interlocked.Read(ref this.totalDataCount);
                long deltaCount = count - lastCount;

                this.Log($"{DateTime.Now} - {deltaCount} - {delta}");
                jobState.Status = $"processed data count: {count}, failures: {Interlocked.Read(ref this.totalFailures)}. Queued node count: {this.QueuedNodes.Count()}";
                this.processedDataCounts.Add(deltaCount);

                lastSize  = size;
                lastCount = count;
            }

            stopwatch.Stop();
            var processedCount = Interlocked.Read(ref this.totalDataCount) - initialCount;
            var processedSize  = Interlocked.Read(ref this.totalDataSize) - initialSize;
            var rate           = processedCount / stopwatch.Elapsed.TotalSeconds;

            Helper.LogAndSetJobStatus(this.Log, jobState, $"Stopping test {testTitle}. Data processed {processedSize} bytes in {processedCount} ops. rate: {rate:G4} /sec, Failures = {this.totalFailures}");
            MdmHelper.LogBytesProcessed(processedSize, operationType);

            cancellation.Cancel();

            foreach (var thread in threads)
            {
                thread.Join();
            }

            this.Log($"Stopped {testTitle}.");

            return(rate);
        }
Ejemplo n.º 5
0
        private async Task TestPublishSubscribeAsync(JobState jobState, CancellationToken cancellation)
        {
            var totalSizeMB = 0.5 * (this.maxDataSize + this.minDataSize) * this.partitionCount * this.nodeCountPerPartition / 1024 / 1024;

            this.log($"Creating {this.partitionCount} partitions, {this.nodeCountPerPartition} nodes in each partition, total amount of data {totalSizeMB} MB");

            var cancelShowProgress = new CancellationTokenSource();

            _ = this.ShowProgress(jobState, cancelShowProgress.Token);

            if (!await this.CreateNodeTree(cancellation).ConfigureAwait(false))
            {
                cancelShowProgress.Cancel();
                Assert.Fail($"No progress in CreateNodeTree after {this.requestTimeout} ms");
            }

            this.log("Reading all nodes...");
            if (!await this.ReadNodeTree(cancellation).ConfigureAwait(false))
            {
                cancelShowProgress.Cancel();
                Assert.Fail($"No progress in ReadNodeTree after {this.requestTimeout} ms");
            }

            cancelShowProgress.Cancel();

            var  watchers             = new ConcurrentBag <IWatcher>();
            int  watcherTriggerCount  = 0;
            long watcherDataDelivered = 0;
            var  startTime            = this.stopwatch.Elapsed;

            this.log($"Installing bulk watchers...");

            long watcherId   = 0;
            var  installTask = Helpers.ForEachAsync(
                Enumerable.Range(0, this.partitionCount),
                async(partitionIndex) =>
            {
                foreach (var client in this.clients)
                {
                    if (cancellation.IsCancellationRequested)
                    {
                        break;
                    }

                    var path    = $"{this.partitionKeyPrefix}{partitionIndex}";
                    var watcher = new CallbackWatcher
                    {
                        OnProcess = (watchedEvent) =>
                        {
                            if (watchedEvent.EventType == WatchedEvent.WatchedEventType.NodeDataChanged)
                            {
                                Interlocked.Add(ref watcherDataDelivered, watchedEvent.Data.Length);
                                Interlocked.Increment(ref watcherTriggerCount);
                            }
                            else if (watchedEvent.EventType != WatchedEvent.WatchedEventType.WatcherRemoved)
                            {
                                this.log($" -- {watchedEvent.EventType} / {watchedEvent.KeeperState} - {watchedEvent.Path}");
                            }
                        },
                        Id = (ulong)Interlocked.Increment(ref watcherId),
                    };
                    try
                    {
                        var operationStartTime = this.stopwatch.Elapsed;
                        await client.RegisterBulkWatcher(path, watcher).ConfigureAwait(false);
                        var operationDuration = this.stopwatch.Elapsed - operationStartTime;
                        MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, Test.Helpers.OperationType.InstallBulkWatcher);
                        this.InstallWatcherLatency.Add(operationDuration.TotalMilliseconds);

                        watchers.Add(watcher);
                    }
                    catch (Exception ex)
                    {
                        this.log($"  Watcher at path {path} failed to install: {ex.Message}");
                    }
                }
            },
                this.threadCount);

            await Task.WhenAny(installTask, Task.Delay(this.requestTimeout)).ConfigureAwait(false);

            if (!installTask.IsCompleted && watcherTriggerCount == 0)
            {
                Assert.Fail($"No watcher event received after {this.requestTimeout} ms");
            }

            var duration    = (this.stopwatch.Elapsed - startTime).TotalSeconds;
            var installRate = watchers.Count / duration;

            this.log($"Finished installing bulk watchers in {duration:F3} sec. Rate: {installRate:G4} /sec");
            MdmHelper.LogWatcherCountProcessed(watchers.Count, Test.Helpers.OperationType.InstallBulkWatcher);

            // Make some random change, one node in each partition, and wait for watcher being triggered
            for (int i = 0; i < this.testRepetitions && !cancellation.IsCancellationRequested; i++)
            {
                startTime            = this.stopwatch.Elapsed;
                this.totalDataSize   = 0;
                watcherTriggerCount  = 0;
                watcherDataDelivered = 0;

                var unused1 = Task.Run(() => this.ChangeRandomNodeInTree());

                var timeoutClock = Stopwatch.StartNew();
                while (watcherTriggerCount < this.partitionCount * this.clients.Length &&
                       timeoutClock.ElapsedMilliseconds < 30 * 1000 &&
                       !cancellation.IsCancellationRequested)
                {
                    await Task.Delay(1000).ConfigureAwait(false);

                    Helper.LogAndSetJobStatus(this.log, jobState, $"Iteration {i} -- watcher event received: {watcherTriggerCount}, data received: {watcherDataDelivered}");
                }

                duration = (this.stopwatch.Elapsed - startTime).TotalSeconds;
                this.log($"Iteration {i} - {watcherTriggerCount} events / {watcherDataDelivered} bytes received in {duration} seconds. Read {this.totalDataSize} bytes.");
                MdmHelper.LogWatcherCountProcessed(watcherTriggerCount, Test.Helpers.OperationType.BulkWatcherTrigger);
                MdmHelper.LogOperationDuration((long)(duration * 1000), Test.Helpers.OperationType.BulkWatcherTrigger);
                MdmHelper.LogBytesProcessed(watcherDataDelivered, Test.Helpers.OperationType.BulkWatcherTrigger);
            }

            Assert.IsTrue(await this.DeleteNodeTree(cancellation).ConfigureAwait(false));
        }