public void TestSingleClientConnection()
{
var connectionEstablished = new ManualResetEventSlim();
var connectionLost = new ManualResetEventSlim();
SecureTransport serverTransport = null;
SecureTransport clientTransport = null;
try
{
serverTransport = CreateTransport();
serverTransport.StartServer(this.serverListenPort);
clientTransport = CreateTransport();
clientTransport.OnNewConnection = _ => connectionEstablished.Set();
clientTransport.OnConnectionLost = () => connectionLost.Set();
clientTransport.StartClient(this.clientEndPoints);
connectionEstablished.Wait();
}
finally
{
try
{
serverTransport.Dispose();
Assert.IsTrue(connectionLost.Wait(5000));
}
finally
{
clientTransport?.Dispose();
}
}
}
public void TestClientConnectionRetryDifferentServer()
{
var connectionEstablished = new ManualResetEventSlim();
var connectionLost = new ManualResetEventSlim();
int server1ListenPort = GetAvailablePort(9997);
int server2ListenPort = GetAvailablePort(server1ListenPort + 1);
IPEndPoint[] clientEndPoints = SecureTransport.ParseConnectionString(string.Format("127.0.0.1:{0},127.0.0.1:{1}", server1ListenPort, server2ListenPort));
SecureTransport serverTransport1 = null;
SecureTransport clientTransport = null;
try
{
serverTransport1 = CreateTransport();
serverTransport1.StartServer(server1ListenPort);
clientTransport = CreateTransport();
clientTransport.OnNewConnection = _ => connectionEstablished.Set();
clientTransport.OnConnectionLost = () => connectionLost.Set();
clientTransport.StartClient(clientEndPoints);
connectionEstablished.Wait();
// Now stop the server to break the established connection
serverTransport1.Stop();
connectionLost.Wait();
connectionEstablished.Reset();
connectionLost.Reset();
using (var serverTransport2 = CreateTransport())
{
// Start a new server at a different port and verify that the client automatically connects to it
serverTransport2.StartServer(server2ListenPort);
connectionEstablished.Wait();
}
}
finally
{
try
{
serverTransport1.Dispose();
Assert.IsTrue(connectionLost.Wait(5000));
}
finally
{
clientTransport?.Dispose();
}
}
}
public void TestMultipleServerConnections()
{
var connection1Established = new ManualResetEventSlim();
var connection1Lost = new ManualResetEventSlim();
var connection2Established = new ManualResetEventSlim();
var connection2Lost = new ManualResetEventSlim();
var port = GetAvailablePort(10000);
var clientEndpoint = SecureTransport.ParseConnectionString($"127.0.0.1:{port}");
SecureTransport serverTransport = null;
SecureTransport clientTransport1 = null;
SecureTransport clientTransport2 = null;
try
{
serverTransport = CreateTransport();
serverTransport.StartServer(port);
clientTransport1 = CreateTransport();
serverTransport.OnNewConnection = _ => connection1Established.Set();
clientTransport1.OnConnectionLost = () => connection1Lost.Set();
clientTransport1.StartClient(clientEndpoint);
connection1Established.Wait();
clientTransport2 = CreateTransport();
serverTransport.OnNewConnection = _ => connection2Established.Set();
clientTransport2.OnConnectionLost = () => connection2Lost.Set();
clientTransport2.StartClient(clientEndpoint);
connection2Established.Wait();
}
finally
{
try
{
serverTransport.Dispose();
Assert.IsTrue(connection1Lost.Wait(5000));
Assert.IsTrue(connection2Lost.Wait(5000));
}
finally
{
clientTransport1?.Dispose();
clientTransport2?.Dispose();
}
}
}
public void TestClientConnectionRetrySameServer()
{
var connectionEstablished = new ManualResetEventSlim();
var connectionLost = new ManualResetEventSlim();
SecureTransport serverTransport = null;
SecureTransport clientTransport = null;
try
{
serverTransport = CreateTransport();
serverTransport.StartServer(this.serverListenPort);
clientTransport = CreateTransport();
clientTransport.OnNewConnection = _ => connectionEstablished.Set();
clientTransport.OnConnectionLost = () => connectionLost.Set();
clientTransport.StartClient(this.clientEndPoints);
connectionEstablished.Wait();
// Now stop the server to break the established connection
serverTransport.Stop();
connectionLost.Wait();
connectionEstablished.Reset();
connectionLost.Reset();
// Restart the server and verify that the client automatically connects to it
serverTransport.StartServer(this.serverListenPort);
connectionEstablished.Wait();
}
finally
{
try
{
serverTransport.Dispose();
Assert.IsTrue(connectionLost.Wait(5000));
}
finally
{
clientTransport?.Dispose();
}
}
}
/// <inheritdoc />
public override void Send(IRingMasterBackendRequest req)
{
if (req == null)
{
throw new ArgumentNullException("req");
}
if (this.client == null)
{
SecureTransport transport = null;
try
{
var endpoints = SecureTransport.ParseConnectionString(this.ConnectString);
transport = new SecureTransport(this.transportConfiguration);
this.client = new RingMasterClient(this.protocol, transport);
this.secureTransport = transport;
// The lifetime of transport is now owned by RingMasterClient
transport = null;
this.secureTransport.StartClient(endpoints);
}
finally
{
transport?.Dispose();
}
}
this.client.Request(req.WrappedRequest).ContinueWith(responseTask =>
{
try
{
RequestResponse response = responseTask.Result;
req.NotifyComplete(response.ResultCode, response.Content, response.Stat, response.ResponsePath);
}
catch (System.Exception)
{
req.NotifyComplete((int)RingMasterException.Code.Systemerror, null, null, null);
}
});
}
private static void SecureTransportInternal(Action <string> log, string testCaseName, int numberOfClients)
{
var packetCount = 0;
log($"Starting {testCaseName} ...");
SecureTransport server = null;
SecureTransport[] clients = new SecureTransport[0];
try
{
// Start the service
server = new SecureTransport(new SecureTransport.Configuration
{
UseSecureConnection = false,
IsClientCertificateRequired = false,
CommunicationProtocolVersion = RingMasterCommunicationProtocol.MaximumSupportedVersion,
});
var serverSendTask = Task.CompletedTask;
server.OnNewConnection = connection =>
{
connection.OnPacketReceived = packet =>
{
Interlocked.Increment(ref packetCount);
serverSendTask.GetAwaiter().GetResult();
serverSendTask = connection.SendAsync(packet);
};
Trace.TraceInformation("Server accepted a new connection: {0}", connection.RemoteIdentity);
};
server.StartServer(Port);
// Start the client
clients = Enumerable.Range(0, numberOfClients).Select(
_ => new SecureTransport(new SecureTransport.Configuration
{
UseSecureConnection = false,
IsClientCertificateRequired = false,
CommunicationProtocolVersion = RingMasterCommunicationProtocol.MaximumSupportedVersion,
}))
.ToArray();
Parallel.ForEach(
clients,
client => client.OnNewConnection = connection =>
{
var clientSendTask = Task.CompletedTask;
connection.OnPacketReceived = packet =>
{
clientSendTask.GetAwaiter().GetResult();
clientSendTask = connection.SendAsync(packet);
};
clientSendTask = connection.SendAsync(new byte[PacketLength]);
});
Parallel.ForEach(
clients,
client => client.StartClient(new IPEndPoint(IPAddress.Loopback, Port)));
log($" Warming up for {WarmupMilliSeconds / 1000} seconds");
Thread.Sleep(WarmupMilliSeconds);
log($" Start measuring for {MeasureMilliSeconds} seconds");
var sw = new Stopwatch();
int gen0 = GC.CollectionCount(0), gen1 = GC.CollectionCount(1), gen2 = GC.CollectionCount(2);
sw.Start();
Interlocked.Exchange(ref packetCount, 0);
for (int i = 0; i < MeasureMilliSeconds / PrintStatusInterval; i++)
{
Thread.Sleep(PrintStatusInterval);
log($" {DateTime.Now.ToString()} count={packetCount}");
}
sw.Stop();
var totalCount = packetCount;
var rate = totalCount / sw.Elapsed.TotalSeconds;
Parallel.ForEach(clients, client => client.Close());
server.Close();
log($"{testCaseName}: {totalCount} in {sw.Elapsed} with {numberOfClients} clients. QPS={rate}");
log($" Gen0={GC.CollectionCount(0) - gen0} Gen1={GC.CollectionCount(1) - gen1} Gen2={GC.CollectionCount(2) - gen2}\n");
log(string.Empty);
}
finally
{
server?.Dispose();
Parallel.ForEach(clients, client => client.Dispose());
}
}
public RingMasterClient(
ServerSpec serverSpec,
Configuration configuration,
IRingMasterClientInstrumentation instrumentation,
IWatcher watcher,
CancellationToken cancellationToken)
{
if (serverSpec == null)
{
throw new ArgumentNullException(nameof(serverSpec));
}
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
IPEndPoint[] endpoints = serverSpec.Endpoints;
if ((endpoints == null) || (endpoints.Length == 0))
{
throw new ArgumentException("Endpoints were not specified");
}
if (watcher != null)
{
this.SetWatcher(watcher);
}
SecureTransport secureTransport = null;
try
{
var transportConfiguration = new SecureTransport.Configuration();
transportConfiguration.UseSecureConnection = serverSpec.UseSecureConnection;
transportConfiguration.ClientCertificates = new X509Certificate[] { serverSpec.ClientCertificate };
transportConfiguration.ServerCertificates = serverSpec.AcceptedServerCertificates;
transportConfiguration.RemoteCertificateValidationCallback = serverSpec.ServerCertificateValidationCallback;
transportConfiguration.LocalCertificateSelectionCallback = serverSpec.ClientCertificationSelectionCallback;
transportConfiguration.MustCheckCertificateRevocation = serverSpec.MustCheckCertificateRevocation;
transportConfiguration.MustCheckCertificateTrustChain = serverSpec.MustCheckCertificateTrustChain;
transportConfiguration.CommunicationProtocolVersion = serverSpec.CommunicationProtocolVersion;
transportConfiguration.SendBufferSize = configuration.BufferSize;
transportConfiguration.ReceiveBufferSize = configuration.BufferSize;
transportConfiguration.SendQueueLength = configuration.RequestQueueLength;
transportConfiguration.AuthAsClient = true;
List <SecureTransport.SubjectRuleValidation> subjectRules = new List <SecureTransport.SubjectRuleValidation>();
if (!string.IsNullOrWhiteSpace(serverSpec.CertValidationASubject) && serverSpec.CertValidationASigningThumbprints != null)
{
SecureTransport.SubjectValidation subjectA = new SecureTransport.SubjectValidation()
{
CertificateSubject = serverSpec.CertValidationASubject,
SigningCertThumbprints = serverSpec.CertValidationASigningThumbprints,
};
subjectRules.Add(new SecureTransport.SubjectRuleValidation(CertificateRules.AbstractCertificateRule.RoleToApply.ServerCert, subjectA));
}
if (!string.IsNullOrWhiteSpace(serverSpec.CertValidationBSubject) && serverSpec.CertValidationBSigningThumbprints != null)
{
SecureTransport.SubjectValidation subjectB = new SecureTransport.SubjectValidation()
{
CertificateSubject = serverSpec.CertValidationBSubject,
SigningCertThumbprints = serverSpec.CertValidationBSigningThumbprints,
};
subjectRules.Add(new SecureTransport.SubjectRuleValidation(CertificateRules.AbstractCertificateRule.RoleToApply.ServerCert, subjectB));
}
if (subjectRules != null && subjectRules.Count > 0)
{
transportConfiguration.SubjectValidations = subjectRules;
}
secureTransport = new SecureTransport(transportConfiguration, instrumentation: null, cancellationToken: cancellationToken);
ICommunicationProtocol protocol = new RingMasterCommunicationProtocol();
var handlerConfiguration = new RingMasterRequestHandler.Configuration();
handlerConfiguration.DefaultTimeout = configuration.DefaultTimeout;
handlerConfiguration.HeartBeatInterval = configuration.HeartBeatInterval;
handlerConfiguration.RequestQueueLength = configuration.RequestQueueLength;
handlerConfiguration.RequireLockForReadOnlyOperations = configuration.RequireLockForReadOnlyOperations;
handlerConfiguration.MustTransparentlyForwardRequests = configuration.MustTransparentlyForwardRequests;
var handlerInstrumentation = new RingMasterRequestHandlerInstrumentation(instrumentation);
this.requestHandler = new RingMasterRequestHandler(
handlerConfiguration,
handlerInstrumentation,
protocol,
secureTransport,
cancellationToken);
foreach (var endpoint in endpoints)
{
RingMasterClientEventSource.Log.Start(endpoint.ToString());
}
secureTransport.StartClient(endpoints);
secureTransport = null;
}
finally
{
if (secureTransport != null)
{
secureTransport.Dispose();
}
}
}
