private static async Task RegisterBulkWatcherLegacyMethod(IRingMasterRequestHandler ringMaster, string pathPrefix, IWatcher watcher)
{
string watcherBody = string.Format("$startswith:{0}", pathPrefix);
string name = await ringMaster.Create("/$bulkwatcher/watcher", Encoding.UTF8.GetBytes(watcherBody), null, CreateMode.EphemeralSequential);
string bulkWatcherPath = string.Format("/$bulkwatcher/{0}", name);
await ringMaster.Exists(bulkWatcherPath, watcher);
}
/// <summary>
/// Tests basic ringmaster functionality
/// </summary>
/// <param name="ringMaster">RingMaster client</param>
/// <param name="instanceRootPath">Root path that must be used by this instance for creating nodes</param>
/// <param name="iteration">Current iteration</param>
/// <returns><c>true</c> if the functionality test passed, <c>false</c> otherwise</returns>
private async Task <bool> TestRingMasterFunctionality(IRingMasterRequestHandler ringMaster, string instanceRootPath, long iteration)
{
var timer = Stopwatch.StartNew();
try
{
var random = new Random();
string nodePath = string.Format($"{instanceRootPath}/Node");
RingMasterWatchdogEventSource.Log.Create(iteration, nodePath);
await ringMaster.Create(nodePath, null, null, CreateMode.PersistentAllowPathCreation, throwIfNodeExists : false);
RingMasterWatchdogEventSource.Log.Exists(iteration, nodePath);
var nodeStat = await ringMaster.Exists(nodePath, watcher : null);
int nodeDataLength = random.Next(RingMasterWatchdog.DefaultMaxNodeDataLength);
byte[] nodeData = new byte[nodeDataLength];
random.NextBytes(nodeData);
RingMasterWatchdogEventSource.Log.SetData(iteration, nodePath, nodeData.Length);
await ringMaster.SetData(nodePath, nodeData, nodeStat.Version);
RingMasterWatchdogEventSource.Log.GetData(iteration, nodePath);
var retrievedData = await ringMaster.GetData(nodePath, watcher : null);
if (retrievedData == null)
{
RingMasterWatchdogEventSource.Log.GetDataFailed_RetrievedDataIsNull(iteration, nodePath, nodeData.Length);
throw new InvalidOperationException($"Node {nodePath}: Retrieved data is null. expectedDataLength={nodeData.Length}");
}
if (retrievedData.Length != nodeData.Length)
{
RingMasterWatchdogEventSource.Log.GetDataFailed_RetrievedDataLengthMismatch(iteration, nodePath, nodeData.Length, retrievedData.Length);
throw new InvalidOperationException($"Node {nodePath}: Retrieved data length mismatch retrievedDataLength={retrievedData.Length} expectedDataLength={nodeData.Length}");
}
if (!retrievedData.SequenceEqual(nodeData))
{
RingMasterWatchdogEventSource.Log.GetDataFailed_RetrievedDataIsDifferent(iteration, nodePath, nodeData.Length);
throw new InvalidOperationException($"Node {nodePath}: Retrieved data is different");
}
RingMasterWatchdogEventSource.Log.Delete(iteration, nodePath, nodeStat.Version);
await ringMaster.Delete(nodePath, -1);
RingMasterWatchdogEventSource.Log.TestRingMasterFunctionalitySucceeded(iteration, timer.ElapsedMilliseconds);
return(true);
}
catch (System.Exception ex)
{
RingMasterWatchdogEventSource.Log.TestRingMasterFunctionalityFailed(iteration, timer.ElapsedMilliseconds, ex.ToString());
}
return(false);
}
private async Task CreateAndDeleteHierarchyTest(IRingMasterRequestHandler ringMaster, ConfigurationSection config, CancellationToken cancellationToken)
{
try
{
string testPath = config.GetStringValue("CreateAndDeleteHierarchy.TestPath");
int maxNodes = config.GetIntValue("CreateAndDeleteHierarchy.MaxNodes");
bool useScheduledDelete = config.GetBoolValue("CreateAndDeleteHierarchy.UseScheduledDelete");
await ringMaster.Create(testPath, null, null, CreateMode.PersistentAllowPathCreation, throwIfNodeExists : false);
while (!cancellationToken.IsCancellationRequested)
{
var createInstrumentation = new CreatePerformanceInstrumentation(this.MetricsFactory);
int maxConcurrentCreateBatches = config.GetIntValue("Create.MaxConcurrentBatches");
int createBatchLength = config.GetIntValue("Create.BatchLength");
var createPerformanceTest = new CreatePerformance(createInstrumentation, maxConcurrentCreateBatches, cancellationToken);
createPerformanceTest.MinChildrenCountPerNode = config.GetIntValue("Create.MinChildrenCountPerNode");
createPerformanceTest.MaxChildrenCountPerNode = config.GetIntValue("Create.MaxChildrenCountPerNode");
createPerformanceTest.MinDataSizePerNode = config.GetIntValue("Create.MinDataSizePerNode");
createPerformanceTest.MaxDataSizePerNode = config.GetIntValue("Create.MaxDataSizePerNode");
createPerformanceTest.MaxNodeNameLength = config.GetIntValue("Create.MaxNodeNameLength");
PopulationStressServiceEventSource.Log.CreateAndDeleteHierarchyCreateStarted(testPath, maxNodes, createBatchLength);
createPerformanceTest.CreateHierarchy(ringMaster, testPath, createBatchLength, maxNodes);
int maxConcurrentDeleteBatches = config.GetIntValue("Delete.MaxConcurrentBatches");
int deleteBatchLength = config.GetIntValue("Delete.BatchLength");
var deleteInstrumentation = new DeletePerformanceInstrumentation(this.MetricsFactory);
var deletePerformanceTest = new DeletePerformance(deleteInstrumentation, maxConcurrentDeleteBatches, cancellationToken);
PopulationStressServiceEventSource.Log.CreateAndDeleteHierarchyDeleteStarted(testPath, maxNodes, deleteBatchLength, useScheduledDelete);
if (useScheduledDelete)
{
deletePerformanceTest.ScheduledDelete(ringMaster, testPath);
}
else
{
await deletePerformanceTest.LoadNodes(ringMaster, testPath, maxNodes);
deletePerformanceTest.QueueDeletes(ringMaster, deleteBatchLength);
}
}
PopulationStressServiceEventSource.Log.CreateAndDeleteHierarchyTestCompleted();
}
catch (Exception ex)
{
PopulationStressServiceEventSource.Log.CreateAndDeleteHierarchyTestFailed(ex.ToString());
}
}
/// <summary>
/// Work load for testing Create method
/// </summary>
/// <param name="client">RingMasterClient object</param>
/// <param name="token">Cancellation token</param>
/// <param name="threadId">Thread sequence number</param>
/// <returns>Async task</returns>
private Task CreateNodeThread(IRingMasterRequestHandler client, CancellationToken token, int threadId)
{
var rnd = new Random();
int taskCount = 0;
var clock = Stopwatch.StartNew();
bool createSmallTree = true;
var rootName = $"{this.RootNodeName}/Instance{this.ServiceContext.ReplicaOrInstanceId}";
while (!token.IsCancellationRequested)
{
var dataSize = 0;
var dataCount = 0;
int numToCreate = 0;
var subtree = string.Empty;
if (createSmallTree)
{
numToCreate = rnd.Next(0, 20);
subtree = $"/{rootName}/vnet{Guid.NewGuid()}/mappings/v4ca";
}
else
{
// create big tree children;
numToCreate = this.LargeTreeRatio;
int idx = rnd.Next(this.LargeTreeRoots.Count);
subtree = $"/{rootName}/vnet{this.LargeTreeRoots[idx]}/mappings/v4ca";
}
// flip the flag so that the thread switches between creating small trees and large trees.
createSmallTree = !createSmallTree;
while (numToCreate-- > 0 && !token.IsCancellationRequested)
{
SpinWait.SpinUntil(() => taskCount < this.AsyncTaskCount || token.IsCancellationRequested);
var path = $"{subtree}/{Guid.NewGuid()}";
var data = Helpers.MakeRandomData(rnd, rnd.Next(this.MinDataSize, this.MaxDataSize));
var startTime = clock.Elapsed;
var unused = client.Create(path, data, null, CreateMode.PersistentAllowPathCreation | CreateMode.SuccessEvenIfNodeExistsFlag)
.ContinueWith(t =>
{
Interlocked.Decrement(ref taskCount);
if (t.Exception != null)
{
this.Log($"Failed path: {path}");
this.IncrementTotalFailures();
}
else
{
this.AddTotalDataSize(data.Length);
this.IncrementTotalDataCount();
var duration = clock.Elapsed - startTime;
MdmHelper.LogOperationDuration((long)duration.TotalMilliseconds, OperationType.Create);
}
});
Interlocked.Increment(ref taskCount);
dataSize += data.Length;
dataCount++;
}
}
SpinWait.SpinUntil(() => taskCount == 0);
return(Task.FromResult(0));
}
/// <summary>
/// Registers to notifications for any change under the given path.
/// </summary>
/// <param name="ringMaster">The ringmaster handler to use.</param>
/// <param name="timeout">The timeout for retries on setting the watcher.</param>
/// <param name="pathToWatch">The path to watch.</param>
/// <param name="oneuse">if set to <c>true</c> there will be just one notification triggered, and the watcher will be removed then.</param>
/// <param name="sessionlocal">if set to <c>true</c> we will use a local session for this on the server.</param>
/// <param name="onChange">The notification callback.</param>
/// <returns>an async task indicating a boolean where true means the callback will be invoked</returns>
public static async Task <bool> RegisterOnAnySubPathChange(
this IRingMasterRequestHandler ringMaster,
int timeout,
string pathToWatch,
bool oneuse,
bool sessionlocal,
RegisterOnAnySubPathChangeDelegate onChange)
{
if (ringMaster == null)
{
throw new ArgumentNullException("rm");
}
if (pathToWatch == null)
{
throw new ArgumentNullException("pathToWatch");
}
if (onChange == null)
{
throw new ArgumentNullException("onChange");
}
string path = GetBulkWatcherName(pathToWatch.Replace('/', '_')) + "_" + Guid.NewGuid().ToString();
byte[] data = Encoding.UTF8.GetBytes("$startswith:" + pathToWatch + ",$sessionlocal:" + sessionlocal);
DelegateWatcher watcher = new DelegateWatcher(
ev =>
{
// if the event was signaled because the bulkwatcher node was deleted, this means the watcher is removed as well.
if (ev.EventType == WatchedEvent.WatchedEventType.NodeDeleted && string.Equals(ev.Path, path))
{
return;
}
onChange(RingMasterException.Code.Ok, ev);
if (ev.EventType == WatchedEvent.WatchedEventType.WatcherRemoved && ev.KeeperState == WatchedEvent.WatchedEventKeeperState.SyncConnected)
{
ringMaster.Delete(path, -1, DeleteMode.None).Wait();
}
},
oneuse ? WatcherKind.OneUse : default(WatcherKind));
DateTime maxTime = DateTime.UtcNow + TimeSpan.FromMilliseconds(timeout);
while (true)
{
try
{
await ringMaster.Create(GetBulkWatcherName(null), null, null, CreateMode.Persistent);
break;
}
catch (RingMasterException ex)
{
if (ex.ErrorCode == RingMasterException.Code.Connectionloss || ex.ErrorCode == RingMasterException.Code.Operationtimeout)
{
if (DateTime.UtcNow > maxTime)
{
return(false);
}
continue;
}
if (ex.ErrorCode == RingMasterException.Code.Nodeexists)
{
break;
}
onChange(ex.ErrorCode, null);
return(true);
}
}
while (true)
{
try
{
await ringMaster.Create(path, data, null, CreateMode.Ephemeral);
break;
}
catch (RingMasterException ex)
{
if (ex.ErrorCode == RingMasterException.Code.Connectionloss || ex.ErrorCode == RingMasterException.Code.Operationtimeout)
{
if (DateTime.UtcNow > maxTime)
{
return(false);
}
continue;
}
if (ex.ErrorCode == RingMasterException.Code.Nodeexists)
{
break;
}
onChange(ex.ErrorCode, null);
return(true);
}
}
while (true)
{
try
{
await ringMaster.Exists(path, watcher, false);
break;
}
catch (RingMasterException ex)
{
if (ex.ErrorCode == RingMasterException.Code.Connectionloss)
{
if (DateTime.UtcNow > maxTime)
{
return(false);
}
continue;
}
onChange(ex.ErrorCode, null);
return(true);
}
}
return(true);
}