public void TestSingleClientConnection()
{
var connectionEstablished = new ManualResetEventSlim();
var connectionLost = new ManualResetEventSlim();
SecureTransport serverTransport = null;
SecureTransport clientTransport = null;
try
{
serverTransport = CreateTransport();
serverTransport.StartServer(this.serverListenPort);
clientTransport = CreateTransport();
clientTransport.OnNewConnection = _ => connectionEstablished.Set();
clientTransport.OnConnectionLost = () => connectionLost.Set();
clientTransport.StartClient(this.clientEndPoints);
connectionEstablished.Wait();
}
finally
{
try
{
serverTransport.Dispose();
Assert.IsTrue(connectionLost.Wait(5000));
}
finally
{
clientTransport?.Dispose();
}
}
}
private void ConnectPerformanceTest(ConfigurationSection config, CancellationToken cancellationToken)
{
try
{
var instrumentation = new ConnectPerformanceInstrumentation(this.MetricsFactory);
var random = new Random();
string connectionString = config.GetStringValue("TargetConnectionString");
connectionString = Helpers.GetServerAddressIfNotProvided(connectionString);
IPEndPoint[] endpoints = SecureTransport.ParseConnectionString(connectionString);
string testPath = config.GetStringValue("TestPath");
int numConnections = config.GetIntValue("NumberOfConnections");
int maxConcurrentRequests = config.GetIntValue("MaxConcurrentRequests");
int minConnectionLifetimeSeconds = config.GetIntValue("MinConnectionLifetimeSeconds");
int maxConnectionLifetimeSeconds = config.GetIntValue("MaxConnectionLifetimeSeconds");
Func <IRingMasterRequestHandler> createConnection = () =>
{
var connectionConfiguration = new SecureTransport.Configuration
{
UseSecureConnection = false,
CommunicationProtocolVersion = RingMasterCommunicationProtocol.MaximumSupportedVersion,
MaxConnectionLifespan = TimeSpan.FromSeconds(random.Next(minConnectionLifetimeSeconds, maxConnectionLifetimeSeconds))
};
var protocol = new RingMasterCommunicationProtocol();
var transport = new SecureTransport(connectionConfiguration, instrumentation, cancellationToken);
var client = new RingMasterClient(protocol, transport);
transport.StartClient(endpoints);
ConnectionStressServiceEventSource.Log.CreateConnection(
connectionConfiguration.UseSecureConnection,
connectionConfiguration.CommunicationProtocolVersion,
(long)connectionConfiguration.MaxConnectionLifespan.TotalSeconds);
client.Exists("/", watcher: null).Wait();
return((IRingMasterRequestHandler)client);
};
using (var connectPerformanceTest = new ConnectPerformance(instrumentation, maxConcurrentRequests, cancellationToken))
{
ConnectionStressServiceEventSource.Log.ConnectPerformanceTestStarted(testPath, numConnections, minConnectionLifetimeSeconds, maxConnectionLifetimeSeconds);
connectPerformanceTest.EstablishConnections(createConnection, numConnections);
connectPerformanceTest.QueueRequests(testPath);
}
ConnectionStressServiceEventSource.Log.ConnectPerformanceTestCompleted();
}
catch (Exception ex)
{
ConnectionStressServiceEventSource.Log.ConnectPerformanceTestFailed(ex.ToString());
}
}
public void TestClientConnectionRetryDifferentServer()
{
var connectionEstablished = new ManualResetEventSlim();
var connectionLost = new ManualResetEventSlim();
int server1ListenPort = GetAvailablePort(9997);
int server2ListenPort = GetAvailablePort(server1ListenPort + 1);
IPEndPoint[] clientEndPoints = SecureTransport.ParseConnectionString(string.Format("127.0.0.1:{0},127.0.0.1:{1}", server1ListenPort, server2ListenPort));
SecureTransport serverTransport1 = null;
SecureTransport clientTransport = null;
try
{
serverTransport1 = CreateTransport();
serverTransport1.StartServer(server1ListenPort);
clientTransport = CreateTransport();
clientTransport.OnNewConnection = _ => connectionEstablished.Set();
clientTransport.OnConnectionLost = () => connectionLost.Set();
clientTransport.StartClient(clientEndPoints);
connectionEstablished.Wait();
// Now stop the server to break the established connection
serverTransport1.Stop();
connectionLost.Wait();
connectionEstablished.Reset();
connectionLost.Reset();
using (var serverTransport2 = CreateTransport())
{
// Start a new server at a different port and verify that the client automatically connects to it
serverTransport2.StartServer(server2ListenPort);
connectionEstablished.Wait();
}
}
finally
{
try
{
serverTransport1.Dispose();
Assert.IsTrue(connectionLost.Wait(5000));
}
finally
{
clientTransport?.Dispose();
}
}
}
public void TestMultipleServerConnections()
{
var connection1Established = new ManualResetEventSlim();
var connection1Lost = new ManualResetEventSlim();
var connection2Established = new ManualResetEventSlim();
var connection2Lost = new ManualResetEventSlim();
var port = GetAvailablePort(10000);
var clientEndpoint = SecureTransport.ParseConnectionString($"127.0.0.1:{port}");
SecureTransport serverTransport = null;
SecureTransport clientTransport1 = null;
SecureTransport clientTransport2 = null;
try
{
serverTransport = CreateTransport();
serverTransport.StartServer(port);
clientTransport1 = CreateTransport();
serverTransport.OnNewConnection = _ => connection1Established.Set();
clientTransport1.OnConnectionLost = () => connection1Lost.Set();
clientTransport1.StartClient(clientEndpoint);
connection1Established.Wait();
clientTransport2 = CreateTransport();
serverTransport.OnNewConnection = _ => connection2Established.Set();
clientTransport2.OnConnectionLost = () => connection2Lost.Set();
clientTransport2.StartClient(clientEndpoint);
connection2Established.Wait();
}
finally
{
try
{
serverTransport.Dispose();
Assert.IsTrue(connection1Lost.Wait(5000));
Assert.IsTrue(connection2Lost.Wait(5000));
}
finally
{
clientTransport1?.Dispose();
clientTransport2?.Dispose();
}
}
}
private void ConnectPerformanceTest(SecureTransport.Configuration configuration, IPEndPoint[] endpoints, int numConnections)
{
var instrumentation = new ConnectPerformanceInstrumentation();
var random = new Random();
int minConnectionLifetimeSeconds = int.Parse(appSettings["ConnectPerformance.MinConnectionLifetimeSeconds"]);
int maxConnectionLifetimeSeconds = int.Parse(appSettings["ConnectPerformance.MaxConnectionLifetimeSeconds"]);
Func <IRingMasterRequestHandler> createConnection = () =>
{
var connectionConfiguration = new SecureTransport.Configuration
{
UseSecureConnection = configuration.UseSecureConnection,
ClientCertificates = configuration.ClientCertificates,
ServerCertificates = configuration.ServerCertificates,
CommunicationProtocolVersion = configuration.CommunicationProtocolVersion,
MaxConnectionLifespan = TimeSpan.FromSeconds(random.Next(minConnectionLifetimeSeconds, maxConnectionLifetimeSeconds)),
};
var protocol = new RingMasterCommunicationProtocol();
var transport = new SecureTransport(connectionConfiguration, instrumentation, CancellationToken.None);
var client = new RingMasterClient(protocol, transport);
transport.StartClient(endpoints);
client.Exists("/", watcher: null).Wait();
return((IRingMasterRequestHandler)client);
};
using (var connectPerformanceTest = new ConnectPerformance(instrumentation, this.MaxConcurrency, CancellationToken.None))
{
Trace.TraceInformation($"Connect performance test numConnections={numConnections}, path={this.TestPath}, minConnectionLifetimeSeconds={minConnectionLifetimeSeconds}, maxConnectionLifetimeSeconds={maxConnectionLifetimeSeconds}");
connectPerformanceTest.EstablishConnections(createConnection, numConnections);
var task = Task.Run(() => connectPerformanceTest.QueueRequests(this.TestPath));
long lastSuccessCount = 0;
var timer = Stopwatch.StartNew();
while (!task.Wait(5000))
{
timer.Stop();
long rate = (long)((instrumentation.Success - lastSuccessCount) * 1000) / timer.ElapsedMilliseconds;
Trace.TraceInformation($"Connections created={instrumentation.ConnectionCreatedCount}, closed={instrumentation.ConnectionClosedCount}, requestSuccess={instrumentation.Success}, requestFailure={instrumentation.Failure}, rate={rate}");
timer.Restart();
lastSuccessCount = instrumentation.Success;
}
}
}
public void TestClientConnectionRetrySameServer()
{
var connectionEstablished = new ManualResetEventSlim();
var connectionLost = new ManualResetEventSlim();
SecureTransport serverTransport = null;
SecureTransport clientTransport = null;
try
{
serverTransport = CreateTransport();
serverTransport.StartServer(this.serverListenPort);
clientTransport = CreateTransport();
clientTransport.OnNewConnection = _ => connectionEstablished.Set();
clientTransport.OnConnectionLost = () => connectionLost.Set();
clientTransport.StartClient(this.clientEndPoints);
connectionEstablished.Wait();
// Now stop the server to break the established connection
serverTransport.Stop();
connectionLost.Wait();
connectionEstablished.Reset();
connectionLost.Reset();
// Restart the server and verify that the client automatically connects to it
serverTransport.StartServer(this.serverListenPort);
connectionEstablished.Wait();
}
finally
{
try
{
serverTransport.Dispose();
Assert.IsTrue(connectionLost.Wait(5000));
}
finally
{
clientTransport?.Dispose();
}
}
}
public void TestClientCertificateSelectionCallback()
{
var certificates = GetLocalCertificates(2);
X509Certificate[] clientCertificates = new X509Certificate[] { certificates[0] };
X509Certificate[] serverCertificates = new X509Certificate[] { certificates[1] };
int port = TestSecureTransport.GetAvailablePort(10000);
var serverAcceptedClient = new ManualResetEventSlim();
var clientConnected = new ManualResetEventSlim();
var clientCertificateSelectionCallbackCalled = new ManualResetEventSlim();
var configuration = new SecureTransport.Configuration()
{
UseSecureConnection = true,
ClientCertificates = certificates,
ServerCertificates = serverCertificates,
CommunicationProtocolVersion = 1,
MustCheckCertificateTrustChain = false,
LocalCertificateSelectionCallback = (sender, targetHost, localCertificates, remoteCertificate, acceptableIssuers) =>
{
clientCertificateSelectionCallbackCalled.Set();
return(certificates[0]);
}
};
using (var server = CreateTransport(clientCertificates, serverCertificates))
using (var client = new SecureTransport(configuration, null, CancellationToken.None))
{
server.OnNewConnection = _ => serverAcceptedClient.Set();
client.OnNewConnection = _ => clientConnected.Set();
server.StartServer(port);
client.StartClient(new IPEndPoint(IPAddress.Loopback, port));
serverAcceptedClient.Wait(30000).Should().BeTrue();
// Client certificate selection callback must be called before
// the client accepts the connection.
clientCertificateSelectionCallbackCalled.Wait(3000).Should().BeTrue();
clientConnected.Wait(30000).Should().BeTrue();
}
}
/// <summary>
/// Defines the entry point of the application.
/// </summary>
/// <param name="args">Command line arguments</param>
public static void Main(string[] args)
{
Trace.Listeners.Add(new TextWriterTraceListener(Console.Out));
if (args == null || args.Length < 1)
{
Trace.TraceError("USAGE: SecureTransportClientTool.exe <connection string> [<request length>]");
return;
}
string connectionString = args[0];
int requestLength = 128;
int waitTimeoutMs = 30000;
if (args.Length > 1)
{
requestLength = int.Parse(args[1]);
}
var path = System.Reflection.Assembly.GetExecutingAssembly().Location;
var builder = new ConfigurationBuilder().SetBasePath(Path.GetDirectoryName(path)).AddJsonFile("appSettings.json");
var appSettings = builder.Build();
var configuration = new SecureTransport.Configuration();
int maxConcurrentRequests = int.Parse(appSettings["MaxConcurrentRequests"]);
configuration.SendBufferSize = int.Parse(appSettings["SendBufferSize"]);
configuration.ReceiveBufferSize = int.Parse(appSettings["ReceiveBufferSize"]);
configuration.UseSecureConnection = bool.Parse(appSettings["SSL.UseSSL"]);
if (configuration.UseSecureConnection)
{
string[] clientThumbprints = appSettings["SSL.ClientCerts"].Split(new char[] { ';', ',' });
string[] serviceThumbprints = appSettings["SSL.ServerCerts"].Split(new char[] { ';', ',' });
configuration.ClientCertificates = SecureTransport.GetCertificatesFromThumbPrintOrFileName(clientThumbprints);
configuration.ServerCertificates = SecureTransport.GetCertificatesFromThumbPrintOrFileName(serviceThumbprints);
}
Trace.TraceInformation(
"Connecting to {0}. Using SSL={1} RequestLength={2} MaxConcurrentRequests={3}, SendBufferSize={4}, ReceiveBufferSize={5}",
connectionString,
configuration.UseSecureConnection,
requestLength,
maxConcurrentRequests,
configuration.SendBufferSize,
configuration.ReceiveBufferSize);
var connectionAvailable = new ManualResetEvent(false);
IPEndPoint[] endpoints = SecureTransport.ParseConnectionString(connectionString);
using (var transport = new SecureTransport(configuration))
{
transport.StartClient(endpoints);
Console.CancelKeyPress += (sender, eventArgs) =>
{
Trace.TraceInformation("Attempting to close client transport");
transport.Close();
};
IConnection currentConnection = null;
transport.OnNewConnection = connection =>
{
currentConnection = connection;
connectionAvailable.Set();
};
transport.OnConnectionLost = () =>
{
connectionAvailable.Reset();
};
var random = new Random();
var timer = new Stopwatch();
timer.Start();
long requestsSent = 0;
byte[] request = new byte[requestLength];
random.NextBytes(request);
var sendSemaphore = new SemaphoreSlim(maxConcurrentRequests, maxConcurrentRequests);
while (true)
{
if (!connectionAvailable.WaitOne(waitTimeoutMs))
{
Trace.TraceWarning("Connection is not available. retrying...");
continue;
}
currentConnection.OnPacketReceived = packet =>
{
};
try
{
if (!sendSemaphore.Wait(waitTimeoutMs))
{
Trace.TraceError("Timedout waiting for send semaphore...");
continue;
}
currentConnection.SendAsync(request).ContinueWith(_ => sendSemaphore.Release());
}
catch (IOException ex)
{
Trace.TraceError("IO Exception: {0}", ex.Message);
}
Interlocked.Increment(ref requestsSent);
if (timer.ElapsedMilliseconds > 5000)
{
long rate = (requestsSent * 1000) / timer.ElapsedMilliseconds;
int inflightCount = maxConcurrentRequests - sendSemaphore.CurrentCount;
Trace.TraceInformation($"Send rate={rate} InFlight={inflightCount}");
requestsSent = 0;
timer.Restart();
}
}
}
}
public RingMasterClient(
ServerSpec serverSpec,
Configuration configuration,
IRingMasterClientInstrumentation instrumentation,
IWatcher watcher,
CancellationToken cancellationToken)
{
if (serverSpec == null)
{
throw new ArgumentNullException(nameof(serverSpec));
}
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
IPEndPoint[] endpoints = serverSpec.Endpoints;
if ((endpoints == null) || (endpoints.Length == 0))
{
throw new ArgumentException("Endpoints were not specified");
}
if (watcher != null)
{
this.SetWatcher(watcher);
}
SecureTransport secureTransport = null;
try
{
var transportConfiguration = new SecureTransport.Configuration();
transportConfiguration.UseSecureConnection = serverSpec.UseSecureConnection;
transportConfiguration.ClientCertificates = new X509Certificate[] { serverSpec.ClientCertificate };
transportConfiguration.ServerCertificates = serverSpec.AcceptedServerCertificates;
transportConfiguration.RemoteCertificateValidationCallback = serverSpec.ServerCertificateValidationCallback;
transportConfiguration.LocalCertificateSelectionCallback = serverSpec.ClientCertificationSelectionCallback;
transportConfiguration.MustCheckCertificateRevocation = serverSpec.MustCheckCertificateRevocation;
transportConfiguration.MustCheckCertificateTrustChain = serverSpec.MustCheckCertificateTrustChain;
transportConfiguration.CommunicationProtocolVersion = serverSpec.CommunicationProtocolVersion;
transportConfiguration.SendBufferSize = configuration.BufferSize;
transportConfiguration.ReceiveBufferSize = configuration.BufferSize;
transportConfiguration.SendQueueLength = configuration.RequestQueueLength;
transportConfiguration.AuthAsClient = true;
List <SecureTransport.SubjectRuleValidation> subjectRules = new List <SecureTransport.SubjectRuleValidation>();
if (!string.IsNullOrWhiteSpace(serverSpec.CertValidationASubject) && serverSpec.CertValidationASigningThumbprints != null)
{
SecureTransport.SubjectValidation subjectA = new SecureTransport.SubjectValidation()
{
CertificateSubject = serverSpec.CertValidationASubject,
SigningCertThumbprints = serverSpec.CertValidationASigningThumbprints,
};
subjectRules.Add(new SecureTransport.SubjectRuleValidation(CertificateRules.AbstractCertificateRule.RoleToApply.ServerCert, subjectA));
}
if (!string.IsNullOrWhiteSpace(serverSpec.CertValidationBSubject) && serverSpec.CertValidationBSigningThumbprints != null)
{
SecureTransport.SubjectValidation subjectB = new SecureTransport.SubjectValidation()
{
CertificateSubject = serverSpec.CertValidationBSubject,
SigningCertThumbprints = serverSpec.CertValidationBSigningThumbprints,
};
subjectRules.Add(new SecureTransport.SubjectRuleValidation(CertificateRules.AbstractCertificateRule.RoleToApply.ServerCert, subjectB));
}
if (subjectRules != null && subjectRules.Count > 0)
{
transportConfiguration.SubjectValidations = subjectRules;
}
secureTransport = new SecureTransport(transportConfiguration, instrumentation: null, cancellationToken: cancellationToken);
ICommunicationProtocol protocol = new RingMasterCommunicationProtocol();
var handlerConfiguration = new RingMasterRequestHandler.Configuration();
handlerConfiguration.DefaultTimeout = configuration.DefaultTimeout;
handlerConfiguration.HeartBeatInterval = configuration.HeartBeatInterval;
handlerConfiguration.RequestQueueLength = configuration.RequestQueueLength;
handlerConfiguration.RequireLockForReadOnlyOperations = configuration.RequireLockForReadOnlyOperations;
handlerConfiguration.MustTransparentlyForwardRequests = configuration.MustTransparentlyForwardRequests;
var handlerInstrumentation = new RingMasterRequestHandlerInstrumentation(instrumentation);
this.requestHandler = new RingMasterRequestHandler(
handlerConfiguration,
handlerInstrumentation,
protocol,
secureTransport,
cancellationToken);
foreach (var endpoint in endpoints)
{
RingMasterClientEventSource.Log.Start(endpoint.ToString());
}
secureTransport.StartClient(endpoints);
secureTransport = null;
}
finally
{
if (secureTransport != null)
{
secureTransport.Dispose();
}
}
}
