/// <summary>
/// Work load for ping pong test
/// </summary>
/// <param name="client">RingMasterClient object</param>
/// <param name="token">Cancellation token</param>
/// <param name="threadId">Thread sequence number</param>
/// <returns>Async task</returns>
private async Task PingPongThread(IRingMasterRequestHandler client, CancellationToken token, int threadId)
{
var clock = Stopwatch.StartNew();
while (!token.IsCancellationRequested)
{
try
{
var startTime = clock.Elapsed;
var tasks = Enumerable.Range(0, this.AsyncTaskCount)
.Select(task => client.Exists(string.Empty, null, true)
.ContinueWith(t =>
{
var duration = clock.Elapsed - startTime;
MdmHelper.LogOperationDuration((long)duration.TotalMilliseconds, OperationType.PingPong);
}))
.ToArray();
await Task.WhenAll(tasks);
this.IncrementTotalDataCount(tasks.Length);
}
catch (Exception ex)
{
this.IncrementTotalFailures();
this.Log($"Failed to call Batch: {ex.Message}");
}
}
}
private async Task <bool> CreateNodeTree(CancellationToken cancellation)
{
this.totalDataCount = 0;
this.totalDataSize = 0;
var startTime = this.stopwatch.Elapsed;
var createTask = Helpers.ForEachAsync(
Enumerable.Range(0, this.partitionCount),
async(partitionIndex) =>
{
var rnd = new Random();
for (int i = 0; i < this.nodeCountPerPartition && !cancellation.IsCancellationRequested; i++)
{
var path = $"{this.partitionKeyPrefix}{partitionIndex}/{RelativePathPrefix}/{i}";
var data = Helpers.MakeRandomData(rnd, rnd.Next(this.minDataSize, this.maxDataSize));
try
{
var operationStartTime = this.stopwatch.Elapsed;
await this.clients[this.partitionCount % this.channelCount].Create(path, data, null, CreateMode.PersistentAllowPathCreation, false).ConfigureAwait(false);
var operationDuration = this.stopwatch.Elapsed - operationStartTime;
MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, Test.Helpers.OperationType.BulkWatcherCreateNode);
this.CreateLatency.Add(operationDuration.TotalMilliseconds);
Interlocked.Add(ref this.totalDataSize, data.Length);
Interlocked.Increment(ref this.totalDataCount);
}
catch (Exception ex)
{
this.log($"Failed to create {path}: {ex.Message}");
}
}
},
this.threadCount);
await Task.WhenAny(createTask, Task.Delay(this.requestTimeout)).ConfigureAwait(false);
if (!createTask.IsCompleted && this.totalDataCount == 0)
{
// If no data is created successfully within timeout time, don't bother to read any more.
return(false);
}
else
{
// It is making progress. Wait until it's completed.
await createTask.ConfigureAwait(false);
}
var duration = (this.stopwatch.Elapsed - startTime).TotalSeconds;
var bps = this.totalDataSize / duration;
var qps = this.totalDataCount / duration;
this.log($"{nameof(this.CreateNodeTree)}: {this.totalDataCount} nodes created, total data size is {this.totalDataSize}. Rate: {bps:G4} byte/sec {qps:G4} /sec");
MdmHelper.LogBytesProcessed(this.totalDataSize, Test.Helpers.OperationType.BulkWatcherCreateNode);
return(true);
}
public void TestRingMasterDelete()
{
ResetCounts();
queuedNodes = new ConcurrentQueue <string>();
var cancellation = new CancellationTokenSource();
Task.Run(() => TraverseTree(new List <string>()
{
rootNodeName, $"{rootNodeName}_Batch", $"{rootNodeName}_Multi"
}, cancellation.Token));
Task.Run(() => CheckQueueCount(cancellation.Token));
Thread.Sleep(2 * 1000);
var rate = TestFlowAsync(
"Delete node perf test",
OperationType.Delete,
(client, cancellationToken, threadId) =>
{
int taskCount = 0;
var clock = Stopwatch.StartNew();
while (!cancellationToken.IsCancellationRequested)
{
while (queuedNodes.TryDequeue(out string path) && !cancellationToken.IsCancellationRequested)
{
SpinWait.SpinUntil(() => taskCount < asyncTaskCount || cancellationToken.IsCancellationRequested);
var startTime = clock.Elapsed;
var task = client.Delete(path, -1, DeleteMode.None)
.ContinueWith(
t =>
{
Interlocked.Decrement(ref taskCount);
if (!t.Result)
{
log($"Failed to delete {path}.");
if (t.Exception != null)
{
if (Interlocked.Increment(ref totalFailures) < PrintFailureThreshold)
{
log($"Exception: {t.Exception.Message}");
}
}
}
else
{
Interlocked.Increment(ref totalDataCount);
var duration = clock.Elapsed - startTime;
MdmHelper.LogOperationDuration((long)duration.TotalMilliseconds, OperationType.Delete);
}
});
Interlocked.Increment(ref taskCount);
}
}
private async Task <bool> ReadNodeTree(CancellationToken cancellation)
{
totalDataCount = 0;
totalDataSize = 0;
var startTime = stopwatch.Elapsed;
var readTask = Helpers.ForEachAsync(
Enumerable.Range(0, partitionCount),
async(partitionIndex) =>
{
for (int i = 0; i < nodeCountPerPartition && !cancellation.IsCancellationRequested; i++)
{
var path = $"{PartitionKeyPrefix}{partitionIndex}/{RelativePathPrefix}/{i}";
try
{
var operationStartTime = stopwatch.Elapsed;
var data = await clients[partitionCount % channelCount].GetData(path, false).ConfigureAwait(false);
var operationDuration = stopwatch.Elapsed - operationStartTime;
MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, OperationType.BulkWatcherReadNode);
Interlocked.Add(ref totalDataSize, data.Length);
Interlocked.Increment(ref totalDataCount);
}
catch (Exception ex)
{
log($"Failed to read {path}: {ex.Message}");
}
}
},
threadCount);
await Task.WhenAny(readTask, Task.Delay(requestTimeout)).ConfigureAwait(false);
if (!readTask.IsCompleted && totalDataCount == 0)
{
// If no data is read successfully within timeout time, don't bother to read any more.
return(false);
}
else
{
// It is making progress. Wait until it's completed.
await readTask.ConfigureAwait(false);
}
var duration = (stopwatch.Elapsed - startTime).TotalSeconds;
var bps = totalDataSize / duration;
var qps = totalDataCount / duration;
log($"{nameof(this.ReadNodeTree)}: {totalDataCount} nodes read, total data size is {totalDataSize}. Rate: {bps:G4} byte/sec {qps:G4} /sec");
MdmHelper.LogBytesProcessed(totalDataSize, OperationType.BulkWatcherReadNode);
return(true);
}
/// <summary>
/// Work load for testing SetNode method
/// </summary>
/// <param name="client">RingMasterClient object</param>
/// <param name="token">Cancellation token</param>
/// <param name="threadId">Thread sequence number</param>
/// <returns>Async task</returns>
private Task SetNodeThread(IRingMasterRequestHandler client, CancellationToken token, int threadId)
{
int taskCount = 0;
var rnd = new Random();
var clock = Stopwatch.StartNew();
while (!token.IsCancellationRequested)
{
while (this.QueuedNodes.TryDequeue(out string path) && !token.IsCancellationRequested)
{
var data = Helpers.MakeRandomData(rnd, rnd.Next(this.MinDataSize, this.MaxDataSize));
SpinWait.SpinUntil(() => taskCount < this.AsyncTaskCount || token.IsCancellationRequested);
var startTime = clock.Elapsed;
var task = client.SetData(path, data, -1)
.ContinueWith(
t =>
{
Interlocked.Decrement(ref taskCount);
if (t.Exception != null)
{
this.IncrementTotalFailures();
this.Log($"Failed to set {path}: {t.Exception.Message}");
}
else
{
this.AddTotalDataSize(data.Length);
this.IncrementTotalDataCount();
var duration = clock.Elapsed - startTime;
MdmHelper.LogOperationDuration((long)duration.TotalMilliseconds, OperationType.Set);
}
});
Interlocked.Increment(ref taskCount);
this.QueuedNodes.Enqueue(path);
}
}
SpinWait.SpinUntil(() => taskCount == 0);
return(Task.FromResult(0));
}
/// <summary>
/// Work load for testing Create method
/// </summary>
/// <param name="client">RingMasterClient object</param>
/// <param name="token">Cancellation token</param>
/// <param name="threadId">Thread sequence number</param>
/// <returns>Async task</returns>
private Task DeleteNodeThread(IRingMasterRequestHandler client, CancellationToken token, int threadId)
{
int taskCount = 0;
var clock = Stopwatch.StartNew();
while (!token.IsCancellationRequested)
{
while (this.QueuedNodes.TryDequeue(out string path) && !token.IsCancellationRequested)
{
SpinWait.SpinUntil(() => taskCount < this.AsyncTaskCount || token.IsCancellationRequested);
var startTime = clock.Elapsed;
var task = client.Delete(path, -1, DeleteMode.None)
.ContinueWith(
t =>
{
Interlocked.Decrement(ref taskCount);
if (!t.Result)
{
this.Log($"Failed to delete {path}.");
if (t.Exception != null)
{
this.Log($"Exception: {t.Exception.Message}");
this.IncrementTotalFailures();
}
}
else
{
this.IncrementTotalDataCount();
var duration = clock.Elapsed - startTime;
MdmHelper.LogOperationDuration((long)duration.TotalMilliseconds, OperationType.Delete);
}
});
Interlocked.Increment(ref taskCount);
}
}
SpinWait.SpinUntil(() => taskCount == 0);
return(Task.FromResult(0));
}
private async Task ChangeRandomNodeInTree()
{
this.totalDataCount = 0;
this.totalDataSize = 0;
var startTime = this.stopwatch.Elapsed;
await Helpers.ForEachAsync(
Enumerable.Range(0, this.partitionCount),
async (partitionIndex) =>
{
var rnd = new Random();
var index = rnd.Next(this.nodeCountPerPartition);
var path = $"{this.partitionKeyPrefix}{partitionIndex}/{RelativePathPrefix}/{index}";
var data = Helpers.MakeRandomData(rnd, rnd.Next(this.minDataSize, this.maxDataSize));
try
{
var operationStartTime = this.stopwatch.Elapsed;
await this.clients[partitionIndex % this.channelCount].SetData(path, data, -1).ConfigureAwait(false);
var operationDuration = this.stopwatch.Elapsed - operationStartTime;
MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, Test.Helpers.OperationType.BulkWatcherChangeNode);
this.SetLatency.Add(operationDuration.TotalMilliseconds);
Interlocked.Add(ref this.totalDataSize, data.Length);
Interlocked.Increment(ref this.totalDataCount);
}
catch (Exception ex)
{
this.log($"Failed to set {path}: {ex.Message}");
}
},
this.threadCount)
.ConfigureAwait(false);
var duration = (this.stopwatch.Elapsed - startTime).TotalSeconds;
var bps = this.totalDataSize / duration;
var qps = this.totalDataCount / duration;
this.log($"{nameof(this.ChangeRandomNodeInTree)}: {this.totalDataCount} nodes updated, total data size is {this.totalDataSize}. Rate: {bps:G4} byte/sec {qps:G4} /sec");
MdmHelper.LogBytesProcessed(this.totalDataSize, Test.Helpers.OperationType.BulkWatcherChangeNode);
}
/// <summary>
/// Work load for getting full sub-tree
/// </summary>
/// <param name="client">RingMasterClient object</param>
/// <param name="token">Cancellation token</param>
/// <param name="threadId">Thread sequence number</param>
/// <returns>Async task</returns>
private Task GetFullSubtreeThread(IRingMasterRequestHandler client, CancellationToken token, int threadId)
{
int taskCount = 0;
var clock = Stopwatch.StartNew();
while (!token.IsCancellationRequested)
{
while (this.QueuedNodes.TryDequeue(out string path) && !token.IsCancellationRequested)
{
SpinWait.SpinUntil(() => taskCount < this.AsyncTaskCount || token.IsCancellationRequested);
var startTime = clock.Elapsed;
var task = client.GetFullSubtree(path, true)
.ContinueWith(
t =>
{
Interlocked.Decrement(ref taskCount);
if (t.Exception != null)
{
this.IncrementTotalFailures();
this.Log($"Failed to get full subtree on path {path}: {t.Exception.Message}");
}
else
{
var children = t.Result.Children;
this.AddTotalDataSize(children.Sum(c => c.Data.Length));
this.IncrementTotalDataCount(children.Count);
var duration = (clock.Elapsed - startTime).TotalMilliseconds;
MdmHelper.LogOperationDuration((long)duration, OperationType.GetFullSubtree);
}
});
Interlocked.Increment(ref taskCount);
this.QueuedNodes.Enqueue(path);
}
}
SpinWait.SpinUntil(() => taskCount == 0);
return(Task.FromResult(0));
}
public void TestRingMasterPingPong()
{
ResetCounts();
var rate = TestFlowAsync(
"Ping-Pong Test",
OperationType.PingPong,
async(client, cancellationToken, threadId) =>
{
var clock = Stopwatch.StartNew();
while (!cancellationToken.IsCancellationRequested)
{
try
{
var startTime = clock.Elapsed;
var tasks = Enumerable.Range(0, asyncTaskCount)
.Select(task => client.Exists(string.Empty, null, true)
.ContinueWith(t =>
{
var duration = clock.Elapsed - startTime;
MdmHelper.LogOperationDuration((long)duration.TotalMilliseconds, OperationType.PingPong);
}))
.ToArray();
await Task.WhenAll(tasks);
Interlocked.Add(ref totalDataCount, tasks.Length);
}
catch (Exception ex)
{
if (Interlocked.Increment(ref totalFailures) < PrintFailureThreshold)
{
log($"Failed to call Batch: {ex.Message}");
}
}
}
},
testCaseSeconds)
.GetAwaiter().GetResult();
log($"Ping-Pong test rate: {rate:G4} /sec");
}
/// <summary>
/// Work load for testing Create method
/// </summary>
/// <param name="client">RingMasterClient object</param>
/// <param name="token">Cancellation token</param>
/// <param name="threadId">Thread sequence number</param>
/// <returns>Async task</returns>
private Task CreateNodeThread(IRingMasterRequestHandler client, CancellationToken token, int threadId)
{
var rnd = new Random();
int taskCount = 0;
var clock = Stopwatch.StartNew();
bool createSmallTree = true;
var rootName = $"{this.RootNodeName}/Instance{this.ServiceContext.ReplicaOrInstanceId}";
while (!token.IsCancellationRequested)
{
var dataSize = 0;
var dataCount = 0;
int numToCreate = 0;
var subtree = string.Empty;
if (createSmallTree)
{
numToCreate = rnd.Next(0, 20);
subtree = $"/{rootName}/vnet{Guid.NewGuid()}/mappings/v4ca";
}
else
{
// create big tree children;
numToCreate = this.LargeTreeRatio;
int idx = rnd.Next(this.LargeTreeRoots.Count);
subtree = $"/{rootName}/vnet{this.LargeTreeRoots[idx]}/mappings/v4ca";
}
// flip the flag so that the thread switches between creating small trees and large trees.
createSmallTree = !createSmallTree;
while (numToCreate-- > 0 && !token.IsCancellationRequested)
{
SpinWait.SpinUntil(() => taskCount < this.AsyncTaskCount || token.IsCancellationRequested);
var path = $"{subtree}/{Guid.NewGuid()}";
var data = Helpers.MakeRandomData(rnd, rnd.Next(this.MinDataSize, this.MaxDataSize));
var startTime = clock.Elapsed;
var unused = client.Create(path, data, null, CreateMode.PersistentAllowPathCreation | CreateMode.SuccessEvenIfNodeExistsFlag)
.ContinueWith(t =>
{
Interlocked.Decrement(ref taskCount);
if (t.Exception != null)
{
this.Log($"Failed path: {path}");
this.IncrementTotalFailures();
}
else
{
this.AddTotalDataSize(data.Length);
this.IncrementTotalDataCount();
var duration = clock.Elapsed - startTime;
MdmHelper.LogOperationDuration((long)duration.TotalMilliseconds, OperationType.Create);
}
});
Interlocked.Increment(ref taskCount);
dataSize += data.Length;
dataCount++;
}
}
SpinWait.SpinUntil(() => taskCount == 0);
return(Task.FromResult(0));
}
/// <summary>
/// Tests create scenario using either batch or multi
/// </summary>
/// <param name="batch">true if using batch, false if using multi</param>
/// <param name="operationType">Type of the operation.</param>
/// <param name="jobState">the job state</param>
/// <returns>
/// Request per second
/// </returns>
protected async Task <double> TestBatchOrMultiCreate(bool batch, Test.Helpers.OperationType operationType, JobState jobState)
{
var name = batch ? "Batch" : "Multi";
Random rnd = new Random();
// number of large trees will be a random number between (20, 50)
this.LargeTreeRoots = Enumerable.Range(0, rnd.Next(20, 50)).Select(x => Guid.NewGuid()).ToList();
return(await this.TestFlowAsync(
$"{name}(Create) node perf test",
operationType,
async (client, cancellationToken, threadId) =>
{
var clock = Stopwatch.StartNew();
var rootName = $"{this.RootNodeName}_{name}";
while (!cancellationToken.IsCancellationRequested)
{
var ops = new List <Op>(this.BatchOpCount);
var totalSize = 0;
int smallTreeNodeCount = rnd.Next(this.BatchOpCount / 2);
int bigTreeNodeCount = this.BatchOpCount - smallTreeNodeCount;
var smallSubtreeId = Guid.NewGuid();
while (smallTreeNodeCount-- > 0)
{
var data = Helpers.MakeRandomData(rnd, rnd.Next(this.MinDataSize, this.MaxDataSize));
totalSize += data.Length;
ops.Add(Op.Create($"/{rootName}/vnet{smallSubtreeId}/mappings/v4ca/{Guid.NewGuid()}", data, null, CreateMode.PersistentAllowPathCreation));
}
while (bigTreeNodeCount-- > 0)
{
var data = Helpers.MakeRandomData(rnd, rnd.Next(this.MinDataSize, this.MaxDataSize));
totalSize += data.Length;
int idx = rnd.Next(this.LargeTreeRoots.Count);
ops.Add(Op.Create($"/{rootName}/vnet{this.LargeTreeRoots[idx]}/mappings/v4ca/{Guid.NewGuid()}", data, null, CreateMode.PersistentAllowPathCreation));
}
try
{
var startTime = clock.Elapsed;
if (batch)
{
await client.Batch(ops).ConfigureAwait(false);
}
else
{
await client.Multi(ops).ConfigureAwait(false);
}
var duration = clock.Elapsed - startTime;
MdmHelper.LogOperationDuration((long)duration.TotalMilliseconds, operationType);
this.AddTotalDataSize(totalSize);
this.IncrementTotalDataCount(ops.Count);
}
catch (Exception ex)
{
this.IncrementTotalFailures();
this.Log($"Failed to call {name}: {ex.Message}");
}
}
},
jobState,
this.TestCaseSeconds));
}
private async Task TestPublishSubscribeAsync(JobState jobState, CancellationToken cancellation)
{
var totalSizeMB = 0.5 * (this.maxDataSize + this.minDataSize) * this.partitionCount * this.nodeCountPerPartition / 1024 / 1024;
this.log($"Creating {this.partitionCount} partitions, {this.nodeCountPerPartition} nodes in each partition, total amount of data {totalSizeMB} MB");
var cancelShowProgress = new CancellationTokenSource();
_ = this.ShowProgress(jobState, cancelShowProgress.Token);
if (!await this.CreateNodeTree(cancellation).ConfigureAwait(false))
{
cancelShowProgress.Cancel();
Assert.Fail($"No progress in CreateNodeTree after {this.requestTimeout} ms");
}
this.log("Reading all nodes...");
if (!await this.ReadNodeTree(cancellation).ConfigureAwait(false))
{
cancelShowProgress.Cancel();
Assert.Fail($"No progress in ReadNodeTree after {this.requestTimeout} ms");
}
cancelShowProgress.Cancel();
var watchers = new ConcurrentBag <IWatcher>();
int watcherTriggerCount = 0;
long watcherDataDelivered = 0;
var startTime = this.stopwatch.Elapsed;
this.log($"Installing bulk watchers...");
long watcherId = 0;
var installTask = Helpers.ForEachAsync(
Enumerable.Range(0, this.partitionCount),
async(partitionIndex) =>
{
foreach (var client in this.clients)
{
if (cancellation.IsCancellationRequested)
{
break;
}
var path = $"{this.partitionKeyPrefix}{partitionIndex}";
var watcher = new CallbackWatcher
{
OnProcess = (watchedEvent) =>
{
if (watchedEvent.EventType == WatchedEvent.WatchedEventType.NodeDataChanged)
{
Interlocked.Add(ref watcherDataDelivered, watchedEvent.Data.Length);
Interlocked.Increment(ref watcherTriggerCount);
}
else if (watchedEvent.EventType != WatchedEvent.WatchedEventType.WatcherRemoved)
{
this.log($" -- {watchedEvent.EventType} / {watchedEvent.KeeperState} - {watchedEvent.Path}");
}
},
Id = (ulong)Interlocked.Increment(ref watcherId),
};
try
{
var operationStartTime = this.stopwatch.Elapsed;
await client.RegisterBulkWatcher(path, watcher).ConfigureAwait(false);
var operationDuration = this.stopwatch.Elapsed - operationStartTime;
MdmHelper.LogOperationDuration((long)operationDuration.TotalMilliseconds, Test.Helpers.OperationType.InstallBulkWatcher);
this.InstallWatcherLatency.Add(operationDuration.TotalMilliseconds);
watchers.Add(watcher);
}
catch (Exception ex)
{
this.log($" Watcher at path {path} failed to install: {ex.Message}");
}
}
},
this.threadCount);
await Task.WhenAny(installTask, Task.Delay(this.requestTimeout)).ConfigureAwait(false);
if (!installTask.IsCompleted && watcherTriggerCount == 0)
{
Assert.Fail($"No watcher event received after {this.requestTimeout} ms");
}
var duration = (this.stopwatch.Elapsed - startTime).TotalSeconds;
var installRate = watchers.Count / duration;
this.log($"Finished installing bulk watchers in {duration:F3} sec. Rate: {installRate:G4} /sec");
MdmHelper.LogWatcherCountProcessed(watchers.Count, Test.Helpers.OperationType.InstallBulkWatcher);
// Make some random change, one node in each partition, and wait for watcher being triggered
for (int i = 0; i < this.testRepetitions && !cancellation.IsCancellationRequested; i++)
{
startTime = this.stopwatch.Elapsed;
this.totalDataSize = 0;
watcherTriggerCount = 0;
watcherDataDelivered = 0;
var unused1 = Task.Run(() => this.ChangeRandomNodeInTree());
var timeoutClock = Stopwatch.StartNew();
while (watcherTriggerCount < this.partitionCount * this.clients.Length &&
timeoutClock.ElapsedMilliseconds < 30 * 1000 &&
!cancellation.IsCancellationRequested)
{
await Task.Delay(1000).ConfigureAwait(false);
Helper.LogAndSetJobStatus(this.log, jobState, $"Iteration {i} -- watcher event received: {watcherTriggerCount}, data received: {watcherDataDelivered}");
}
duration = (this.stopwatch.Elapsed - startTime).TotalSeconds;
this.log($"Iteration {i} - {watcherTriggerCount} events / {watcherDataDelivered} bytes received in {duration} seconds. Read {this.totalDataSize} bytes.");
MdmHelper.LogWatcherCountProcessed(watcherTriggerCount, Test.Helpers.OperationType.BulkWatcherTrigger);
MdmHelper.LogOperationDuration((long)(duration * 1000), Test.Helpers.OperationType.BulkWatcherTrigger);
MdmHelper.LogBytesProcessed(watcherDataDelivered, Test.Helpers.OperationType.BulkWatcherTrigger);
}
Assert.IsTrue(await this.DeleteNodeTree(cancellation).ConfigureAwait(false));
}