/// <summary>
/// Work load for ping pong test
/// </summary>
/// <param name="client">RingMasterClient object</param>
/// <param name="token">Cancellation token</param>
/// <param name="threadId">Thread sequence number</param>
/// <returns>Async task</returns>
private async Task PingPongThread(IRingMasterRequestHandler client, CancellationToken token, int threadId)
{
var clock = Stopwatch.StartNew();
while (!token.IsCancellationRequested)
{
try
{
var startTime = clock.Elapsed;
var tasks = Enumerable.Range(0, this.AsyncTaskCount)
.Select(task => client.Exists(string.Empty, null, true)
.ContinueWith(t =>
{
var duration = clock.Elapsed - startTime;
MdmHelper.LogOperationDuration((long)duration.TotalMilliseconds, OperationType.PingPong);
}))
.ToArray();
await Task.WhenAll(tasks);
this.IncrementTotalDataCount(tasks.Length);
}
catch (Exception ex)
{
this.IncrementTotalFailures();
this.Log($"Failed to call Batch: {ex.Message}");
}
}
}
private static async Task RegisterBulkWatcherLegacyMethod(IRingMasterRequestHandler ringMaster, string pathPrefix, IWatcher watcher)
{
string watcherBody = string.Format("$startswith:{0}", pathPrefix);
string name = await ringMaster.Create("/$bulkwatcher/watcher", Encoding.UTF8.GetBytes(watcherBody), null, CreateMode.EphemeralSequential);
string bulkWatcherPath = string.Format("/$bulkwatcher/{0}", name);
await ringMaster.Exists(bulkWatcherPath, watcher);
}
/// <summary>
/// Tests basic ringmaster functionality
/// </summary>
/// <param name="ringMaster">RingMaster client</param>
/// <param name="instanceRootPath">Root path that must be used by this instance for creating nodes</param>
/// <param name="iteration">Current iteration</param>
/// <returns><c>true</c> if the functionality test passed, <c>false</c> otherwise</returns>
private async Task <bool> TestRingMasterFunctionality(IRingMasterRequestHandler ringMaster, string instanceRootPath, long iteration)
{
var timer = Stopwatch.StartNew();
try
{
var random = new Random();
string nodePath = string.Format($"{instanceRootPath}/Node");
RingMasterWatchdogEventSource.Log.Create(iteration, nodePath);
await ringMaster.Create(nodePath, null, null, CreateMode.PersistentAllowPathCreation, throwIfNodeExists : false);
RingMasterWatchdogEventSource.Log.Exists(iteration, nodePath);
var nodeStat = await ringMaster.Exists(nodePath, watcher : null);
int nodeDataLength = random.Next(RingMasterWatchdog.DefaultMaxNodeDataLength);
byte[] nodeData = new byte[nodeDataLength];
random.NextBytes(nodeData);
RingMasterWatchdogEventSource.Log.SetData(iteration, nodePath, nodeData.Length);
await ringMaster.SetData(nodePath, nodeData, nodeStat.Version);
RingMasterWatchdogEventSource.Log.GetData(iteration, nodePath);
var retrievedData = await ringMaster.GetData(nodePath, watcher : null);
if (retrievedData == null)
{
RingMasterWatchdogEventSource.Log.GetDataFailed_RetrievedDataIsNull(iteration, nodePath, nodeData.Length);
throw new InvalidOperationException($"Node {nodePath}: Retrieved data is null. expectedDataLength={nodeData.Length}");
}
if (retrievedData.Length != nodeData.Length)
{
RingMasterWatchdogEventSource.Log.GetDataFailed_RetrievedDataLengthMismatch(iteration, nodePath, nodeData.Length, retrievedData.Length);
throw new InvalidOperationException($"Node {nodePath}: Retrieved data length mismatch retrievedDataLength={retrievedData.Length} expectedDataLength={nodeData.Length}");
}
if (!retrievedData.SequenceEqual(nodeData))
{
RingMasterWatchdogEventSource.Log.GetDataFailed_RetrievedDataIsDifferent(iteration, nodePath, nodeData.Length);
throw new InvalidOperationException($"Node {nodePath}: Retrieved data is different");
}
RingMasterWatchdogEventSource.Log.Delete(iteration, nodePath, nodeStat.Version);
await ringMaster.Delete(nodePath, -1);
RingMasterWatchdogEventSource.Log.TestRingMasterFunctionalitySucceeded(iteration, timer.ElapsedMilliseconds);
return(true);
}
catch (System.Exception ex)
{
RingMasterWatchdogEventSource.Log.TestRingMasterFunctionalityFailed(iteration, timer.ElapsedMilliseconds, ex.ToString());
}
return(false);
}
private async Task LnmPublishingThread(IRingMasterRequestHandler client, CancellationToken token, int threadId)
{
var rnd = new Random();
while (!token.IsCancellationRequested)
{
try
{
var vnet = $"/Instance{this.ServiceContext.ReplicaOrInstanceId}/vnets/{CreateSpanningVnetId(threadId)}";
var stat = await client.Exists(vnet, null, true);
var ops = new List <Op>();
if (stat == null)
{
ops.Add(Op.Create($"{vnet}/mappings/v4ca", null, null, CreateMode.PersistentAllowPathCreation));
ops.Add(Op.Create($"{vnet}/lnms/thread-{threadId}", null, null, CreateMode.PersistentAllowPathCreation));
await client.Multi(ops, true);
ops.Clear();
this.IncrementTotalDataCount(2);
}
var mappingCount = rnd.Next(1, 1024 * 8);
for (int i = 0; i < mappingCount; i++)
{
ops.Add(Op.Create($"{vnet}/mappings/v4ca/{i}", null, null, CreateMode.PersistentAllowPathCreation));
}
this.IncrementTotalDataCount(ops.Count);
if (token.IsCancellationRequested)
{
return;
}
await client.Multi(ops, true);
ops.Clear();
for (int i = 0; i < mappingCount; i++)
{
var data = new byte[rnd.Next(this.MinDataSize, this.MaxDataSize)];
ops.Add(Op.SetData($"{vnet}/mappings/v4ca/{i}", data, -1));
this.AddTotalDataSize(data.Length);
}
this.IncrementTotalDataCount(mappingCount);
if (token.IsCancellationRequested)
{
return;
}
await client.Multi(ops, true);
ops.Clear();
}
catch (Exception ex)
{
this.IncrementTotalFailures();
// Ignore and keep going
this.Log($"FAIL in {threadId}: {ex.Message}");
}
}
}
/// <summary>
/// Registers a bulk watcher for the given path prefix.
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="pathPrefix">Path prefix to watch</param>
/// <param name="watcher">The watcher that will be notified of changes that happen under the given path.</param>
/// <returns>A <see cref="Task"/> that tracks execution of this method</returns>
public static async Task RegisterBulkWatcher(this IRingMasterRequestHandler ringMaster, string pathPrefix, IWatcher watcher)
{
await ringMaster.Exists(string.Format("bulkwatcher:{0}", pathPrefix), watcher);
}
/// <summary>
/// Registers to notifications for any change under the given path.
/// </summary>
/// <param name="ringMaster">The ringmaster handler to use.</param>
/// <param name="timeout">The timeout for retries on setting the watcher.</param>
/// <param name="pathToWatch">The path to watch.</param>
/// <param name="oneuse">if set to <c>true</c> there will be just one notification triggered, and the watcher will be removed then.</param>
/// <param name="sessionlocal">if set to <c>true</c> we will use a local session for this on the server.</param>
/// <param name="onChange">The notification callback.</param>
/// <returns>an async task indicating a boolean where true means the callback will be invoked</returns>
public static async Task <bool> RegisterOnAnySubPathChange(
this IRingMasterRequestHandler ringMaster,
int timeout,
string pathToWatch,
bool oneuse,
bool sessionlocal,
RegisterOnAnySubPathChangeDelegate onChange)
{
if (ringMaster == null)
{
throw new ArgumentNullException("rm");
}
if (pathToWatch == null)
{
throw new ArgumentNullException("pathToWatch");
}
if (onChange == null)
{
throw new ArgumentNullException("onChange");
}
string path = GetBulkWatcherName(pathToWatch.Replace('/', '_')) + "_" + Guid.NewGuid().ToString();
byte[] data = Encoding.UTF8.GetBytes("$startswith:" + pathToWatch + ",$sessionlocal:" + sessionlocal);
DelegateWatcher watcher = new DelegateWatcher(
ev =>
{
// if the event was signaled because the bulkwatcher node was deleted, this means the watcher is removed as well.
if (ev.EventType == WatchedEvent.WatchedEventType.NodeDeleted && string.Equals(ev.Path, path))
{
return;
}
onChange(RingMasterException.Code.Ok, ev);
if (ev.EventType == WatchedEvent.WatchedEventType.WatcherRemoved && ev.KeeperState == WatchedEvent.WatchedEventKeeperState.SyncConnected)
{
ringMaster.Delete(path, -1, DeleteMode.None).Wait();
}
},
oneuse ? WatcherKind.OneUse : default(WatcherKind));
DateTime maxTime = DateTime.UtcNow + TimeSpan.FromMilliseconds(timeout);
while (true)
{
try
{
await ringMaster.Create(GetBulkWatcherName(null), null, null, CreateMode.Persistent);
break;
}
catch (RingMasterException ex)
{
if (ex.ErrorCode == RingMasterException.Code.Connectionloss || ex.ErrorCode == RingMasterException.Code.Operationtimeout)
{
if (DateTime.UtcNow > maxTime)
{
return(false);
}
continue;
}
if (ex.ErrorCode == RingMasterException.Code.Nodeexists)
{
break;
}
onChange(ex.ErrorCode, null);
return(true);
}
}
while (true)
{
try
{
await ringMaster.Create(path, data, null, CreateMode.Ephemeral);
break;
}
catch (RingMasterException ex)
{
if (ex.ErrorCode == RingMasterException.Code.Connectionloss || ex.ErrorCode == RingMasterException.Code.Operationtimeout)
{
if (DateTime.UtcNow > maxTime)
{
return(false);
}
continue;
}
if (ex.ErrorCode == RingMasterException.Code.Nodeexists)
{
break;
}
onChange(ex.ErrorCode, null);
return(true);
}
}
while (true)
{
try
{
await ringMaster.Exists(path, watcher, false);
break;
}
catch (RingMasterException ex)
{
if (ex.ErrorCode == RingMasterException.Code.Connectionloss)
{
if (DateTime.UtcNow > maxTime)
{
return(false);
}
continue;
}
onChange(ex.ErrorCode, null);
return(true);
}
}
return(true);
}
/// <summary>
/// Queries the <see cref="Stat"/> of the node with the given path.
/// </summary>
/// <param name="ringMaster">Interface to ringmaster</param>
/// <param name="path">Node path</param>
/// <param name="watcher">Watcher interface that receives notifications for changes to this path or null</param>
/// <returns>Task that will resolve on success to the <see cref="Stat"/> associated with the node</returns>
public static Task <IStat> Exists(this IRingMasterRequestHandler ringMaster, string path, IWatcher watcher)
{
return(ringMaster.Exists(path, watcher, ignoreNonodeError: false));
}