private async Task <bool> CreateNodeTree(CancellationToken cancellation)
{
this.totalDataCount = 0;
this.totalDataSize = 0;
var startTime = this.stopwatch.Elapsed;
var createTask = Helpers.ForEachAsync(
Enumerable.Range(0, this.partitionCount),
async(partitionIndex) =>
{
var rnd = new Random();
for (int i = 0; i < this.nodeCountPerPartition && !cancellation.IsCancellationRequested; i++)
{
var path = $"{this.partitionKeyPrefix}{partitionIndex}/{RelativePathPrefix}/{i}";
var data = Helpers.MakeRandomData(rnd, rnd.Next(this.minDataSize, this.maxDataSize));
try
{
var operationStartTime = this.stopwatch.Elapsed;
await this.clients[this.partitionCount % this.channelCount].Create(path, data, null, CreateMode.PersistentAllowPathCreation, false).ConfigureAwait(false);
var operationDuration = this.stopwatch.Elapsed - operationStartTime;
MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, Test.Helpers.OperationType.BulkWatcherCreateNode);
this.CreateLatency.Add(operationDuration.TotalMilliseconds);
Interlocked.Add(ref this.totalDataSize, data.Length);
Interlocked.Increment(ref this.totalDataCount);
}
catch (Exception ex)
{
this.log($"Failed to create {path}: {ex.Message}");
}
}
},
this.threadCount);
await Task.WhenAny(createTask, Task.Delay(this.requestTimeout)).ConfigureAwait(false);
if (!createTask.IsCompleted && this.totalDataCount == 0)
{
// If no data is created successfully within timeout time, don't bother to read any more.
return(false);
}
else
{
// It is making progress. Wait until it's completed.
await createTask.ConfigureAwait(false);
}
var duration = (this.stopwatch.Elapsed - startTime).TotalSeconds;
var bps = this.totalDataSize / duration;
var qps = this.totalDataCount / duration;
this.log($"{nameof(this.CreateNodeTree)}: {this.totalDataCount} nodes created, total data size is {this.totalDataSize}. Rate: {bps:G4} byte/sec {qps:G4} /sec");
MdmHelper.LogBytesProcessed(this.totalDataSize, Test.Helpers.OperationType.BulkWatcherCreateNode);
return(true);
}
private async Task <bool> ReadNodeTree(CancellationToken cancellation)
{
totalDataCount = 0;
totalDataSize = 0;
var startTime = stopwatch.Elapsed;
var readTask = Helpers.ForEachAsync(
Enumerable.Range(0, partitionCount),
async(partitionIndex) =>
{
for (int i = 0; i < nodeCountPerPartition && !cancellation.IsCancellationRequested; i++)
{
var path = $"{PartitionKeyPrefix}{partitionIndex}/{RelativePathPrefix}/{i}";
try
{
var operationStartTime = stopwatch.Elapsed;
var data = await clients[partitionCount % channelCount].GetData(path, false).ConfigureAwait(false);
var operationDuration = stopwatch.Elapsed - operationStartTime;
MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, OperationType.BulkWatcherReadNode);
Interlocked.Add(ref totalDataSize, data.Length);
Interlocked.Increment(ref totalDataCount);
}
catch (Exception ex)
{
log($"Failed to read {path}: {ex.Message}");
}
}
},
threadCount);
await Task.WhenAny(readTask, Task.Delay(requestTimeout)).ConfigureAwait(false);
if (!readTask.IsCompleted && totalDataCount == 0)
{
// If no data is read successfully within timeout time, don't bother to read any more.
return(false);
}
else
{
// It is making progress. Wait until it's completed.
await readTask.ConfigureAwait(false);
}
var duration = (stopwatch.Elapsed - startTime).TotalSeconds;
var bps = totalDataSize / duration;
var qps = totalDataCount / duration;
log($"{nameof(this.ReadNodeTree)}: {totalDataCount} nodes read, total data size is {totalDataSize}. Rate: {bps:G4} byte/sec {qps:G4} /sec");
MdmHelper.LogBytesProcessed(totalDataSize, OperationType.BulkWatcherReadNode);
return(true);
}
private async Task ChangeRandomNodeInTree()
{
this.totalDataCount = 0;
this.totalDataSize = 0;
var startTime = this.stopwatch.Elapsed;
await Helpers.ForEachAsync(
Enumerable.Range(0, this.partitionCount),
async (partitionIndex) =>
{
var rnd = new Random();
var index = rnd.Next(this.nodeCountPerPartition);
var path = $"{this.partitionKeyPrefix}{partitionIndex}/{RelativePathPrefix}/{index}";
var data = Helpers.MakeRandomData(rnd, rnd.Next(this.minDataSize, this.maxDataSize));
try
{
var operationStartTime = this.stopwatch.Elapsed;
await this.clients[partitionIndex % this.channelCount].SetData(path, data, -1).ConfigureAwait(false);
var operationDuration = this.stopwatch.Elapsed - operationStartTime;
MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, Test.Helpers.OperationType.BulkWatcherChangeNode);
this.SetLatency.Add(operationDuration.TotalMilliseconds);
Interlocked.Add(ref this.totalDataSize, data.Length);
Interlocked.Increment(ref this.totalDataCount);
}
catch (Exception ex)
{
this.log($"Failed to set {path}: {ex.Message}");
}
},
this.threadCount)
.ConfigureAwait(false);
var duration = (this.stopwatch.Elapsed - startTime).TotalSeconds;
var bps = this.totalDataSize / duration;
var qps = this.totalDataCount / duration;
this.log($"{nameof(this.ChangeRandomNodeInTree)}: {this.totalDataCount} nodes updated, total data size is {this.totalDataSize}. Rate: {bps:G4} byte/sec {qps:G4} /sec");
MdmHelper.LogBytesProcessed(this.totalDataSize, Test.Helpers.OperationType.BulkWatcherChangeNode);
}
/// <summary>
/// Main test workflow
/// </summary>
/// <param name="testTitle">Title of the test case</param>
/// <param name="operationType">the operation type</param>
/// <param name="workload">Workload in each thread</param>
/// <param name="jobState">the job state</param>
/// <param name="durationInSeconds">How long the test should run</param>
/// <returns>Number of operations per second</returns>
protected async Task <double> TestFlowAsync(
string testTitle,
Test.Helpers.OperationType operationType,
Func <IRingMasterRequestHandler, CancellationToken, int, Task> workload,
JobState jobState,
int durationInSeconds)
{
this.ResetCounts();
var cancellation = new CancellationTokenSource();
this.Log($"Starting test {testTitle} in {this.threadCount} threads");
var lastCount = Interlocked.Read(ref this.totalDataCount);
var lastSize = Interlocked.Read(ref this.totalDataSize);
var threads = Helpers.StartMultipleThreads(
this.threadCount,
(object n) => workload(this.clients[(int)n], cancellation.Token, (int)n).GetAwaiter().GetResult());
var initialCount = lastCount;
var initialSize = lastSize;
var stopwatch = Stopwatch.StartNew();
for (int i = 0; i < durationInSeconds; i++)
{
await Task.Delay(TimeSpan.FromSeconds(1));
long size = Interlocked.Read(ref this.totalDataSize);
long delta = size - lastSize;
long count = Interlocked.Read(ref this.totalDataCount);
long deltaCount = count - lastCount;
this.Log($"{DateTime.Now} - {deltaCount} - {delta}");
jobState.Status = $"processed data count: {count}, failures: {Interlocked.Read(ref this.totalFailures)}. Queued node count: {this.QueuedNodes.Count()}";
this.processedDataCounts.Add(deltaCount);
lastSize = size;
lastCount = count;
}
stopwatch.Stop();
var processedCount = Interlocked.Read(ref this.totalDataCount) - initialCount;
var processedSize = Interlocked.Read(ref this.totalDataSize) - initialSize;
var rate = processedCount / stopwatch.Elapsed.TotalSeconds;
Helper.LogAndSetJobStatus(this.Log, jobState, $"Stopping test {testTitle}. Data processed {processedSize} bytes in {processedCount} ops. rate: {rate:G4} /sec, Failures = {this.totalFailures}");
MdmHelper.LogBytesProcessed(processedSize, operationType);
cancellation.Cancel();
foreach (var thread in threads)
{
thread.Join();
}
this.Log($"Stopped {testTitle}.");
return(rate);
}
private async Task TestPublishSubscribeAsync(JobState jobState, CancellationToken cancellation)
{
var totalSizeMB = 0.5 * (this.maxDataSize + this.minDataSize) * this.partitionCount * this.nodeCountPerPartition / 1024 / 1024;
this.log($"Creating {this.partitionCount} partitions, {this.nodeCountPerPartition} nodes in each partition, total amount of data {totalSizeMB} MB");
var cancelShowProgress = new CancellationTokenSource();
_ = this.ShowProgress(jobState, cancelShowProgress.Token);
if (!await this.CreateNodeTree(cancellation).ConfigureAwait(false))
{
cancelShowProgress.Cancel();
Assert.Fail($"No progress in CreateNodeTree after {this.requestTimeout} ms");
}
this.log("Reading all nodes...");
if (!await this.ReadNodeTree(cancellation).ConfigureAwait(false))
{
cancelShowProgress.Cancel();
Assert.Fail($"No progress in ReadNodeTree after {this.requestTimeout} ms");
}
cancelShowProgress.Cancel();
var watchers = new ConcurrentBag <IWatcher>();
int watcherTriggerCount = 0;
long watcherDataDelivered = 0;
var startTime = this.stopwatch.Elapsed;
this.log($"Installing bulk watchers...");
long watcherId = 0;
var installTask = Helpers.ForEachAsync(
Enumerable.Range(0, this.partitionCount),
async(partitionIndex) =>
{
foreach (var client in this.clients)
{
if (cancellation.IsCancellationRequested)
{
break;
}
var path = $"{this.partitionKeyPrefix}{partitionIndex}";
var watcher = new CallbackWatcher
{
OnProcess = (watchedEvent) =>
{
if (watchedEvent.EventType == WatchedEvent.WatchedEventType.NodeDataChanged)
{
Interlocked.Add(ref watcherDataDelivered, watchedEvent.Data.Length);
Interlocked.Increment(ref watcherTriggerCount);
}
else if (watchedEvent.EventType != WatchedEvent.WatchedEventType.WatcherRemoved)
{
this.log($" -- {watchedEvent.EventType} / {watchedEvent.KeeperState} - {watchedEvent.Path}");
}
},
Id = (ulong)Interlocked.Increment(ref watcherId),
};
try
{
var operationStartTime = this.stopwatch.Elapsed;
await client.RegisterBulkWatcher(path, watcher).ConfigureAwait(false);
var operationDuration = this.stopwatch.Elapsed - operationStartTime;
MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, Test.Helpers.OperationType.InstallBulkWatcher);
this.InstallWatcherLatency.Add(operationDuration.TotalMilliseconds);
watchers.Add(watcher);
}
catch (Exception ex)
{
this.log($" Watcher at path {path} failed to install: {ex.Message}");
}
}
},
this.threadCount);
await Task.WhenAny(installTask, Task.Delay(this.requestTimeout)).ConfigureAwait(false);
if (!installTask.IsCompleted && watcherTriggerCount == 0)
{
Assert.Fail($"No watcher event received after {this.requestTimeout} ms");
}
var duration = (this.stopwatch.Elapsed - startTime).TotalSeconds;
var installRate = watchers.Count / duration;
this.log($"Finished installing bulk watchers in {duration:F3} sec. Rate: {installRate:G4} /sec");
MdmHelper.LogWatcherCountProcessed(watchers.Count, Test.Helpers.OperationType.InstallBulkWatcher);
// Make some random change, one node in each partition, and wait for watcher being triggered
for (int i = 0; i < this.testRepetitions && !cancellation.IsCancellationRequested; i++)
{
startTime = this.stopwatch.Elapsed;
this.totalDataSize = 0;
watcherTriggerCount = 0;
watcherDataDelivered = 0;
var unused1 = Task.Run(() => this.ChangeRandomNodeInTree());
var timeoutClock = Stopwatch.StartNew();
while (watcherTriggerCount < this.partitionCount * this.clients.Length &&
timeoutClock.ElapsedMilliseconds < 30 * 1000 &&
!cancellation.IsCancellationRequested)
{
await Task.Delay(1000).ConfigureAwait(false);
Helper.LogAndSetJobStatus(this.log, jobState, $"Iteration {i} -- watcher event received: {watcherTriggerCount}, data received: {watcherDataDelivered}");
}
duration = (this.stopwatch.Elapsed - startTime).TotalSeconds;
this.log($"Iteration {i} - {watcherTriggerCount} events / {watcherDataDelivered} bytes received in {duration} seconds. Read {this.totalDataSize} bytes.");
MdmHelper.LogWatcherCountProcessed(watcherTriggerCount, Test.Helpers.OperationType.BulkWatcherTrigger);
MdmHelper.LogOperationDuration((long)(duration * 1000), Test.Helpers.OperationType.BulkWatcherTrigger);
MdmHelper.LogBytesProcessed(watcherDataDelivered, Test.Helpers.OperationType.BulkWatcherTrigger);
}
Assert.IsTrue(await this.DeleteNodeTree(cancellation).ConfigureAwait(false));
}