public void TestExists()
{
using (var requestHandler = new TestRequestHandler())
using (var client = new RingMasterClient(requestHandler))
{
string path = this.GetRandomString();
var version = this.GetRandomInt();
var watcher = new TestWatcher(this.GetRandomInt());
var expectedStat = new Stat()
{
Version = this.GetRandomInt()
};
requestHandler.Implementation = request =>
{
Assert.IsTrue(request is RequestExists);
var existsRequest = (RequestExists)request;
Assert.AreEqual(path, existsRequest.Path);
Assert.AreEqual(watcher.Id, existsRequest.Watcher.Id);
return(new RequestResponse()
{
ResultCode = (int)RingMasterException.Code.Ok,
Content = expectedStat
});
};
client.SetWatcher(watcher);
IStat stat = client.Exists(path, watch: true).Result;
Assert.IsNotNull(stat);
Assert.AreEqual(expectedStat.Version, stat.Version);
}
}
/// <summary>
/// Thread to mock a NSM / LNM which is publishing VNET data
/// </summary>
/// <param name="cancel">Cancellation token</param>
/// <param name="threadId">Thread sequence number</param>
/// <param name="operationCount">Object to store operation statistics</param>
private void MockLnmThread(CancellationToken cancel, int threadId, OperationCount operationCount)
{
using (var client = new RingMasterClient(serverAddress, null, null, 10000))
{
var rnd = new Random();
while (!cancel.IsCancellationRequested)
{
Task.Run(async() =>
{
try
{
var vnet = $"/mud/vnets/{CreateSpanningVnetId(threadId)}";
var stat = await client.Exists(vnet, null, true);
var ops = new List <Op>();
if (stat == null)
{
ops.Add(Op.Create($"{vnet}/mappings/v4ca", null, null, CreateMode.PersistentAllowPathCreation));
ops.Add(Op.Create($"{vnet}/lnms/thread-{threadId}", null, null, CreateMode.PersistentAllowPathCreation));
await client.Multi(ops, true);
ops.Clear();
operationCount.AddCreate(2);
}
var mappingCount = rnd.Next(1, 1024 * 8);
for (int i = 0; i < mappingCount; i++)
{
ops.Add(Op.Create($"{vnet}/mappings/v4ca/{i}", null, null, CreateMode.PersistentAllowPathCreation));
operationCount.AddCreate(1);
}
await client.Multi(ops, true);
ops.Clear();
for (int i = 0; i < mappingCount; i++)
{
var data = new byte[rnd.Next(minPayloadSize, maxPayloadSize)];
ops.Add(Op.SetData($"{vnet}/mappings/v4ca/{i}", data, -1));
operationCount.AddSet(1);
operationCount.AddData(data.Length);
}
await client.Multi(ops, true);
ops.Clear();
}
catch (Exception ex)
{
operationCount.IncrementFailure();
// Ignore and keep going
log($"FAIL in {threadId}: {ex.Message}");
}
}).GetAwaiter().GetResult();
}
}
}
private void ConnectPerformanceTest(ConfigurationSection config, CancellationToken cancellationToken)
{
try
{
var instrumentation = new ConnectPerformanceInstrumentation(this.MetricsFactory);
var random = new Random();
string connectionString = config.GetStringValue("TargetConnectionString");
connectionString = Helpers.GetServerAddressIfNotProvided(connectionString);
IPEndPoint[] endpoints = SecureTransport.ParseConnectionString(connectionString);
string testPath = config.GetStringValue("TestPath");
int numConnections = config.GetIntValue("NumberOfConnections");
int maxConcurrentRequests = config.GetIntValue("MaxConcurrentRequests");
int minConnectionLifetimeSeconds = config.GetIntValue("MinConnectionLifetimeSeconds");
int maxConnectionLifetimeSeconds = config.GetIntValue("MaxConnectionLifetimeSeconds");
Func <IRingMasterRequestHandler> createConnection = () =>
{
var connectionConfiguration = new SecureTransport.Configuration
{
UseSecureConnection = false,
CommunicationProtocolVersion = RingMasterCommunicationProtocol.MaximumSupportedVersion,
MaxConnectionLifespan = TimeSpan.FromSeconds(random.Next(minConnectionLifetimeSeconds, maxConnectionLifetimeSeconds))
};
var protocol = new RingMasterCommunicationProtocol();
var transport = new SecureTransport(connectionConfiguration, instrumentation, cancellationToken);
var client = new RingMasterClient(protocol, transport);
transport.StartClient(endpoints);
ConnectionStressServiceEventSource.Log.CreateConnection(
connectionConfiguration.UseSecureConnection,
connectionConfiguration.CommunicationProtocolVersion,
(long)connectionConfiguration.MaxConnectionLifespan.TotalSeconds);
client.Exists("/", watcher: null).Wait();
return((IRingMasterRequestHandler)client);
};
using (var connectPerformanceTest = new ConnectPerformance(instrumentation, maxConcurrentRequests, cancellationToken))
{
ConnectionStressServiceEventSource.Log.ConnectPerformanceTestStarted(testPath, numConnections, minConnectionLifetimeSeconds, maxConnectionLifetimeSeconds);
connectPerformanceTest.EstablishConnections(createConnection, numConnections);
connectPerformanceTest.QueueRequests(testPath);
}
ConnectionStressServiceEventSource.Log.ConnectPerformanceTestCompleted();
}
catch (Exception ex)
{
ConnectionStressServiceEventSource.Log.ConnectPerformanceTestFailed(ex.ToString());
}
}
private void ConnectPerformanceTest(SecureTransport.Configuration configuration, IPEndPoint[] endpoints, int numConnections)
{
var instrumentation = new ConnectPerformanceInstrumentation();
var random = new Random();
int minConnectionLifetimeSeconds = int.Parse(appSettings["ConnectPerformance.MinConnectionLifetimeSeconds"]);
int maxConnectionLifetimeSeconds = int.Parse(appSettings["ConnectPerformance.MaxConnectionLifetimeSeconds"]);
Func <IRingMasterRequestHandler> createConnection = () =>
{
var connectionConfiguration = new SecureTransport.Configuration
{
UseSecureConnection = configuration.UseSecureConnection,
ClientCertificates = configuration.ClientCertificates,
ServerCertificates = configuration.ServerCertificates,
CommunicationProtocolVersion = configuration.CommunicationProtocolVersion,
MaxConnectionLifespan = TimeSpan.FromSeconds(random.Next(minConnectionLifetimeSeconds, maxConnectionLifetimeSeconds)),
};
var protocol = new RingMasterCommunicationProtocol();
var transport = new SecureTransport(connectionConfiguration, instrumentation, CancellationToken.None);
var client = new RingMasterClient(protocol, transport);
transport.StartClient(endpoints);
client.Exists("/", watcher: null).Wait();
return((IRingMasterRequestHandler)client);
};
using (var connectPerformanceTest = new ConnectPerformance(instrumentation, this.MaxConcurrency, CancellationToken.None))
{
Trace.TraceInformation($"Connect performance test numConnections={numConnections}, path={this.TestPath}, minConnectionLifetimeSeconds={minConnectionLifetimeSeconds}, maxConnectionLifetimeSeconds={maxConnectionLifetimeSeconds}");
connectPerformanceTest.EstablishConnections(createConnection, numConnections);
var task = Task.Run(() => connectPerformanceTest.QueueRequests(this.TestPath));
long lastSuccessCount = 0;
var timer = Stopwatch.StartNew();
while (!task.Wait(5000))
{
timer.Stop();
long rate = (long)((instrumentation.Success - lastSuccessCount) * 1000) / timer.ElapsedMilliseconds;
Trace.TraceInformation($"Connections created={instrumentation.ConnectionCreatedCount}, closed={instrumentation.ConnectionClosedCount}, requestSuccess={instrumentation.Success}, requestFailure={instrumentation.Failure}, rate={rate}");
timer.Restart();
lastSuccessCount = instrumentation.Success;
}
}
}
/// <summary>
/// Wait until a stable connection can be established with the given ringmaster.
/// </summary>
/// <param name="ringMaster">RingMaster client</param>
/// <param name="maxAttempts">Maximum number of attempts</param>
/// <returns>A Task that resolves to <c>true</c> if the connection was successful</returns>
private static async Task <bool> WaitForStableConnection(RingMasterClient ringMaster, int maxAttempts)
{
for (int retryCount = 0; retryCount < maxAttempts; retryCount++)
{
try
{
await ringMaster.Exists("/", watcher : null);
return(true);
}
catch (Exception ex)
{
Trace.TraceError("WaitForStableConnection. exception={0}, retryCount={1}, maxAttempts={2}", ex.Message, retryCount, maxAttempts);
}
await Task.Delay(1000);
}
return(false);
}
public void TestWrongChildrenCountAfterFailedMulti()
{
TestWrongNumChildrenInStastAsync().GetAwaiter().GetResult();
async Task TestWrongNumChildrenInStastAsync()
{
const string path = "/$rmbvt/test";
var stop = false;
// Create a parent node with 3 children. During the test, the number of children is not expected
// to change.
using (var rm = new RingMasterClient(serverAddress, null, null, 10000))
{
var ops = new List <Op>
{
Op.Create($"{path}/parent/child1", null, null, CreateMode.PersistentAllowPathCreation),
Op.Create($"{path}/parent/child2", null, null, CreateMode.PersistentAllowPathCreation),
Op.Create($"{path}/parent/child3", null, null, CreateMode.PersistentAllowPathCreation),
};
await rm.Multi(ops);
}
// Start multiple threads to stress the backend
var tasks = Enumerable.Range(0, 2).Select(_ => Task.Run(async() =>
{
using (var rm = new RingMasterClient(serverAddress, null, null, 10000))
{
var ops = new List <Op>();
while (!stop)
{
// Randomly add or delete children in Multi
ops.Clear();
ops.AddRange(
Enumerable.Range(1, 3).Select(
x => Op.Delete($"{path}/parent/child{x}", -1, false)));
// Add one more operation to fail the multi, so nothing get committed, in other words the
// locklist will always abort.
ops.Add(Op.GetData(
$"{path}/parent/nonexisting/node",
Azure.Networking.Infrastructure.RingMaster.Requests.RequestGetData.GetDataOptions.None,
null));
var result = (await rm.Multi(ops)).Last();
Assert.AreEqual(OpCode.Error, result.ResultType);
Assert.AreEqual(RingMasterException.Code.Nonode, result.ErrCode);
var children = await rm.GetChildren($"{path}/parent", null);
var stat = await rm.Exists($"{path}/parent", null);
Assert.AreEqual(
children.Count,
stat.NumChildren,
$"Children count {children.Count} should be consistent with Stat {stat.NumChildren}");
Assert.AreEqual(
3,
stat.NumChildren,
"Number of children returned by Exists should not change");
}
}
})).ToArray();
var clock = Stopwatch.StartNew();
while (clock.Elapsed.TotalMinutes < 60)
{
await Task.Delay(1000);
if (tasks.Any(t => t.IsCompleted))
{
break;
}
}
stop = true;
await Task.WhenAll(tasks);
}
}
public void TestConflictingCreateDeleteExists()
{
TestConflictingCreateDeleteExistsAsync().GetAwaiter().GetResult();
async Task TestConflictingCreateDeleteExistsAsync()
{
const string path = "/$rmbvt/test";
var stop = false;
var taskW = Task.Run(async() =>
{
while (!stop)
{
try
{
using (var rm = new RingMasterClient(serverAddress, null, null, 10000))
{
while (!stop)
{
await rm.Create(path, null, null, CreateMode.PersistentAllowPathCreation | CreateMode.SuccessEvenIfNodeExistsFlag);
await rm.Delete(path, -1, DeleteMode.SuccessEvenIfNodeDoesntExist);
}
}
}
catch (Exception ex)
{
var rmException = ex as RingMasterException;
if (rmException != null && rmException.ErrorCode == RingMasterException.Code.Operationtimeout)
{
// Do thing. Continue test.
}
else
{
stop = true;
throw;
}
}
}
});
var taskR = Task.Run(async() =>
{
while (!stop)
{
try
{
using (var rm = new RingMasterClient(serverAddress, null, null, 10000))
{
while (!stop)
{
await Task.WhenAll(
rm.Exists(path, null, true));
}
}
}
catch (Exception ex)
{
var rmException = ex as RingMasterException;
if (rmException != null && rmException.ErrorCode == RingMasterException.Code.Operationtimeout)
{
// Do thing. Continue test.
}
else
{
stop = true;
throw;
}
}
}
});
var clock = Stopwatch.StartNew();
SpinWait.SpinUntil(() => clock.Elapsed.TotalSeconds >= 3600 || stop);
stop = true;
await Task.WhenAll(taskW, taskR).ContinueWith(t => log(t.Exception?.ToString()));
}
}