public async Task ReconnectWhenHavingIntermittentConnectivityFailure()
{
var connectionFactory = new TestConnectionFactory(new IntermittentConnectivityFailure().ConnectCallback);
var proxy = new ServiceConnectionProxy(connectionFactory: connectionFactory);
var serverTask = proxy.WaitForServerConnectionAsync(1);
_ = proxy.StartAsync();
// fail 3 times, 1~2 + 2~3 + 4~5 = 7~10
await serverTask.OrTimeout(11 * 1000);
var connectionId = Guid.NewGuid().ToString("N");
var connectionTask = proxy.WaitForConnectionAsync(connectionId);
await proxy.WriteMessageAsync(new OpenConnectionMessage(connectionId, null));
await connectionTask.OrTimeout();
Assert.True(proxy.IsConnected);
var list = connectionFactory.Times;
Assert.True(TimeSpan.FromSeconds(0.9) < list[1] - list[0]);
Assert.True(TimeSpan.FromSeconds(2.1) > list[1] - list[0]);
Assert.True(TimeSpan.FromSeconds(1.9) < list[2] - list[1]);
Assert.True(TimeSpan.FromSeconds(3.1) > list[2] - list[1]);
Assert.True(TimeSpan.FromSeconds(3.9) < list[3] - list[2]);
Assert.True(TimeSpan.FromSeconds(5.1) > list[3] - list[2]);
}
public async Task ReconnectWhenHandshakeThrowException()
{
var connectionFactory = new TestConnectionFactory(new IntermittentInvalidHandshakeResponseMessage().ConnectCallback);
var proxy = new ServiceConnectionProxy(connectionFactory: connectionFactory);
// Throw exception for 3 times and will be success in the 4th retry
var serverTask = proxy.WaitForServerConnectionAsync(4);
_ = proxy.StartAsync();
// fail 3 times, 1~2 + 2~3 + 4~5 = 7~10
await serverTask.OrTimeout(11 * 1000);
var connectionId = Guid.NewGuid().ToString("N");
var connectionTask = proxy.WaitForConnectionAsync(connectionId);
await proxy.WriteMessageAsync(new OpenConnectionMessage(connectionId, null));
await connectionTask.OrTimeout();
Assert.True(proxy.IsConnected);
var list = connectionFactory.Times;
Assert.True(TimeSpan.FromSeconds(0.9) < list[1] - list[0]);
Assert.True(TimeSpan.FromSeconds(2.1) > list[1] - list[0]);
Assert.True(TimeSpan.FromSeconds(1.9) < list[2] - list[1]);
Assert.True(TimeSpan.FromSeconds(3.1) > list[2] - list[1]);
Assert.True(TimeSpan.FromSeconds(3.9) < list[3] - list[2]);
Assert.True(TimeSpan.FromSeconds(5.1) > list[3] - list[2]);
}
public async Task ClientConnectionOutgoingAbortCanEndLifeTime()
{
using (StartVerifiableLog(out var loggerFactory, LogLevel.Warning, expectedErrors: c => true,
logChecker: logs =>
{
Assert.Equal(2, logs.Count);
Assert.Equal("SendLoopStopped", logs[0].Write.EventId.Name);
Assert.Equal("ApplicationTaskFailed", logs[1].Write.EventId.Name);
return(true);
}))
{
var ccm = new TestClientConnectionManager();
var ccf = new ClientConnectionFactory();
var protocol = new ServiceProtocol();
TestConnection transportConnection = null;
var connectionFactory = new TestConnectionFactory(conn =>
{
transportConnection = conn;
return(Task.CompletedTask);
});
var services = new ServiceCollection();
var connectionHandler = new EndlessConnectionHandler();
services.AddSingleton(connectionHandler);
var builder = new ConnectionBuilder(services.BuildServiceProvider());
builder.UseConnectionHandler <EndlessConnectionHandler>();
ConnectionDelegate handler = builder.Build();
var connection = new ServiceConnection(protocol, ccm, connectionFactory, loggerFactory, handler, ccf,
"serverId", Guid.NewGuid().ToString("N"), null, null,
closeTimeOutMilliseconds: 500);
var connectionTask = connection.StartAsync();
// completed handshake
await connection.ConnectionInitializedTask.OrTimeout();
Assert.Equal(ServiceConnectionStatus.Connected, connection.Status);
var clientConnectionId = Guid.NewGuid().ToString();
await transportConnection.Application.Output.WriteAsync(
protocol.GetMessageBytes(new OpenConnectionMessage(clientConnectionId, new Claim[] { })));
var clientConnection = await ccm.WaitForClientConnectionAsync(clientConnectionId).OrTimeout();
clientConnection.CancelOutgoing();
await clientConnection.LifetimeTask.OrTimeout();
// complete reading to end the connection
transportConnection.Application.Output.Complete();
// 1s for application task to timeout
await connectionTask.OrTimeout(1000);
Assert.Equal(ServiceConnectionStatus.Disconnected, connection.Status);
Assert.Empty(ccm.ClientConnections);
connectionHandler.CancellationToken.Cancel();
}
}
public async Task TestServiceConnectionWithEndlessApplicationTaskNeverEnds()
{
var clientConnectionId = Guid.NewGuid().ToString();
using (StartVerifiableLog(out var loggerFactory, LogLevel.Warning, expectedErrors: c => true,
logChecker: logs =>
{
Assert.Single(logs);
Assert.Equal("DetectedLongRunningApplicationTask", logs[0].Write.EventId.Name);
Assert.Equal($"The connection {clientConnectionId} has a long running application logic that prevents the connection from complete.", logs[0].Write.Message);
return(true);
}))
{
var ccm = new TestClientConnectionManager();
var ccf = new ClientConnectionFactory();
var protocol = new ServiceProtocol();
TestConnection transportConnection = null;
var connectionFactory = new TestConnectionFactory(conn =>
{
transportConnection = conn;
return(Task.CompletedTask);
});
var services = new ServiceCollection();
var connectionHandler = new EndlessConnectionHandler();
services.AddSingleton(connectionHandler);
var builder = new ConnectionBuilder(services.BuildServiceProvider());
builder.UseConnectionHandler <EndlessConnectionHandler>();
ConnectionDelegate handler = builder.Build();
var connection = new ServiceConnection(protocol, ccm, connectionFactory, loggerFactory, handler, ccf,
"serverId", Guid.NewGuid().ToString("N"),
null, null, null, closeTimeOutMilliseconds: 1);
var connectionTask = connection.StartAsync();
// completed handshake
await connection.ConnectionInitializedTask.OrTimeout();
Assert.Equal(ServiceConnectionStatus.Connected, connection.Status);
await transportConnection.Application.Output.WriteAsync(
protocol.GetMessageBytes(new OpenConnectionMessage(clientConnectionId, new Claim[] { })));
var clientConnection = await ccm.WaitForClientConnectionAsync(clientConnectionId).OrTimeout();
// complete reading to end the connection
transportConnection.Application.Output.Complete();
// Assert timeout
var lifetime = clientConnection.LifetimeTask;
var task = await Task.WhenAny(lifetime, Task.Delay(1000));
Assert.NotEqual(lifetime, task);
Assert.Equal(ServiceConnectionStatus.Disconnected, connection.Status);
// since the service connection ends, the client connection is cleaned up from the collection...
Assert.Empty(ccm.ClientConnections);
}
}
public async Task ClientConnectionWithDiagnosticClientTagTest()
{
using (StartVerifiableLog(out var loggerFactory))
{
var ccm = new TestClientConnectionManager();
var ccf = new ClientConnectionFactory();
var protocol = new ServiceProtocol();
TestConnection transportConnection = null;
var connectionFactory = new TestConnectionFactory(conn =>
{
transportConnection = conn;
return(Task.CompletedTask);
});
var diagnosticClientConnectionId = "diagnosticClient";
var normalClientConnectionId = "normalClient";
var services = new ServiceCollection();
var connectionHandler = new DiagnosticClientConnectionHandler(diagnosticClientConnectionId);
services.AddSingleton(connectionHandler);
var builder = new ConnectionBuilder(services.BuildServiceProvider());
builder.UseConnectionHandler <DiagnosticClientConnectionHandler>();
ConnectionDelegate handler = builder.Build();
var connection = new ServiceConnection(protocol, ccm, connectionFactory, loggerFactory, handler, ccf,
"serverId", Guid.NewGuid().ToString("N"), null, null, closeTimeOutMilliseconds: 500);
var connectionTask = connection.StartAsync();
// completed handshake
await connection.ConnectionInitializedTask.OrTimeout();
Assert.Equal(ServiceConnectionStatus.Connected, connection.Status);
await transportConnection.Application.Output.WriteAsync(
protocol.GetMessageBytes(new OpenConnectionMessage(diagnosticClientConnectionId, null, new Dictionary <string, StringValues>
{
{ Constants.AsrsIsDiagnosticClient, "true" }
}, null)));
await transportConnection.Application.Output.WriteAsync(
protocol.GetMessageBytes(new OpenConnectionMessage(normalClientConnectionId, null)));
var connections = await Task.WhenAll(ccm.WaitForClientConnectionAsync(normalClientConnectionId).OrTimeout(),
ccm.WaitForClientConnectionAsync(diagnosticClientConnectionId).OrTimeout());
await Task.WhenAll(from c in connections select c.LifetimeTask.OrTimeout());
// complete reading to end the connection
transportConnection.Application.Output.Complete();
// 1s for application task to timeout
await connectionTask.OrTimeout(1000);
Assert.Equal(ServiceConnectionStatus.Disconnected, connection.Status);
Assert.Empty(ccm.ClientConnections);
}
}
public async Task TestServiceConnectionWithNormalApplicationTask()
{
using (StartVerifiableLog(out var loggerFactory, LogLevel.Debug))
{
var ccm = new TestClientConnectionManager();
var ccf = new ClientConnectionFactory();
var protocol = new ServiceProtocol();
TestConnection transportConnection = null;
var connectionFactory = new TestConnectionFactory(conn =>
{
transportConnection = conn;
return(Task.CompletedTask);
});
var services = new ServiceCollection();
var builder = new ConnectionBuilder(services.BuildServiceProvider());
builder.UseConnectionHandler <TestConnectionHandler>();
ConnectionDelegate handler = builder.Build();
var connection = new ServiceConnection(protocol, ccm, connectionFactory, loggerFactory, handler, ccf,
"serverId", Guid.NewGuid().ToString("N"), null, null);
var connectionTask = connection.StartAsync();
// completed handshake
await connection.ConnectionInitializedTask.OrTimeout();
Assert.Equal(ServiceConnectionStatus.Connected, connection.Status);
var clientConnectionId = Guid.NewGuid().ToString();
await transportConnection.Application.Output.WriteAsync(
protocol.GetMessageBytes(new OpenConnectionMessage(clientConnectionId, new Claim[] { })));
var clientConnection = await ccm.WaitForClientConnectionAsync(clientConnectionId).OrTimeout();
await transportConnection.Application.Output.WriteAsync(
protocol.GetMessageBytes(new CloseConnectionMessage(clientConnectionId)));
// Normal end with close message
await ccm.WaitForClientConnectionRemovalAsync(clientConnectionId).OrTimeout();
// another connection comes in
clientConnectionId = Guid.NewGuid().ToString();
await transportConnection.Application.Output.WriteAsync(
protocol.GetMessageBytes(new OpenConnectionMessage(clientConnectionId, new Claim[] { })));
clientConnection = await ccm.WaitForClientConnectionAsync(clientConnectionId).OrTimeout();
// complete reading to end the connection
transportConnection.Application.Output.Complete();
await connectionTask.OrTimeout();
Assert.Equal(ServiceConnectionStatus.Disconnected, connection.Status);
Assert.Empty(ccm.ClientConnections);
}
}
private static TestServiceConnection CreateServiceConnection(Action <ConnectionBuilder> use = null,
TestClientConnectionManager clientConnectionManager = null,
string serverId = null,
string connectionId = null,
GracefulShutdownMode?mode = null,
IServiceMessageHandler messageHandler = null,
IServiceEventHandler eventHandler = null,
IClientConnectionFactory clientConnectionFactory = null,
ILoggerFactory loggerFactory = null)
{
clientConnectionManager ??= new TestClientConnectionManager();
clientConnectionFactory ??= new ClientConnectionFactory();
var container = new TestConnectionContainer();
var connectionFactory = new TestConnectionFactory(conn =>
{
container.Instance = conn;
return(Task.CompletedTask);
});
var services = new ServiceCollection();
var builder = new ConnectionBuilder(services.BuildServiceProvider());
if (use == null)
{
use = (builder) => builder.UseConnectionHandler <TestConnectionHandler>();
}
use(builder);
builder.UseConnectionHandler <TestConnectionHandler>();
ConnectionDelegate handler = builder.Build();
return(new TestServiceConnection(
container,
new ServiceProtocol(),
clientConnectionManager,
connectionFactory,
loggerFactory ?? NullLoggerFactory.Instance,
handler,
clientConnectionFactory,
serverId ?? "serverId",
connectionId ?? Guid.NewGuid().ToString("N"),
null,
messageHandler ?? new TestServiceMessageHandler(),
eventHandler ?? new TestServiceEventHandler(),
mode: mode ?? GracefulShutdownMode.Off
));
}
public ServiceConnectionProxy(ConnectionDelegate callback = null, PipeOptions clientPipeOptions = null,
TestConnectionFactory connectionFactory = null)
{
ConnectionFactory = connectionFactory ?? new TestConnectionFactory();
ClientConnectionManager = new ClientConnectionManager();
_clientPipeOptions = clientPipeOptions;
ServiceConnection = new ServiceConnection(
ServiceProtocol,
this,
ConnectionFactory,
NullLoggerFactory.Instance,
callback ?? OnConnectionAsync,
this,
Guid.NewGuid().ToString("N"));
}
public async Task ReconnectWhenHavingIntermittentConnectivityFailure()
{
var connectionFactory = new TestConnectionFactory(new IntermittentConnectivityFailure().ConnectCallback);
var proxy = new ServiceConnectionProxy(connectionFactory: connectionFactory);
var serverTask = proxy.WaitForServerConnectionAsync(1);
_ = proxy.StartAsync();
await serverTask.OrTimeout();
var connectionId = Guid.NewGuid().ToString("N");
var connectionTask = proxy.WaitForConnectionAsync(connectionId);
await proxy.WriteMessageAsync(new OpenConnectionMessage(connectionId, null));
await connectionTask.OrTimeout();
}
private static TestServiceConnection CreateServiceConnection(ConnectionHandler handler = null,
TestClientConnectionManager clientConnectionManager = null,
string serverId = null,
string connectionId = null,
GracefulShutdownMode?mode = null,
IServiceMessageHandler messageHandler = null,
IServiceEventHandler eventHandler = null,
IClientConnectionFactory clientConnectionFactory = null,
HubServiceEndpoint hubServiceEndpoint = null,
ILoggerFactory loggerFactory = null)
{
clientConnectionManager ??= new TestClientConnectionManager();
clientConnectionFactory ??= new TestClientConnectionFactory();
var container = new TestConnectionContainer();
var connectionFactory = new TestConnectionFactory(conn =>
{
container.Instance = conn;
return(Task.CompletedTask);
});
var services = new ServiceCollection();
var builder = new ConnectionBuilder(services.BuildServiceProvider());
if (handler == null)
{
handler = new TestConnectionHandler();
}
return(new TestServiceConnection(
container,
new ServiceProtocol(),
clientConnectionManager,
connectionFactory,
loggerFactory ?? NullLoggerFactory.Instance,
handler.OnConnectedAsync,
clientConnectionFactory,
serverId ?? "serverId",
connectionId ?? Guid.NewGuid().ToString("N"),
hubServiceEndpoint ?? new TestHubServiceEndpoint(),
messageHandler ?? new TestServiceMessageHandler(),
eventHandler ?? new TestServiceEventHandler(),
mode: mode ?? GracefulShutdownMode.Off
));
}
public async Task ReconnectWhenHandshakeThrowException()
{
var connectionFactory = new TestConnectionFactory(new IntermittentInvalidHandshakeResponseMessage().ConnectCallback);
var proxy = new ServiceConnectionProxy(connectionFactory: connectionFactory);
// Throw exception for 3 times and will be success in the 4th retry
var serverTask = proxy.WaitForServerConnectionAsync(4);
_ = proxy.StartAsync();
await serverTask.OrTimeout();
var connectionId = Guid.NewGuid().ToString("N");
var connectionTask = proxy.WaitForConnectionAsync(connectionId);
await proxy.WriteMessageAsync(new OpenConnectionMessage(connectionId, null));
await connectionTask.OrTimeout();
}
public ServiceConnectionProxy(ConnectionDelegate callback = null, PipeOptions clientPipeOptions = null,
TestConnectionFactory connectionFactory = null, IServiceConnectionManager serviceConnectionManager = null)
{
ConnectionFactory = connectionFactory ?? new TestConnectionFactory();
ClientConnectionManager = new ClientConnectionManager();
_clientPipeOptions = clientPipeOptions;
ServiceConnection = new ServiceConnection(
SharedServiceProtocol,
this,
ConnectionFactory,
NullLoggerFactory.Instance,
callback ?? OnConnectionAsync,
this,
Guid.NewGuid().ToString("N"),
serviceConnectionManager);
ServiceConnectionContainer = new StrongServiceConnectionContainer(null, connectionFactory, new List <IServiceConnection>()
{
ServiceConnection
}, new ServiceEndpoint("", ""));
}
public async Task ClientConnectionContextAbortCanSendOutCloseMessage()
{
using (StartVerifiableLog(out var loggerFactory, LogLevel.Trace, expectedErrors: c => true,
logChecker: logs =>
{
return(true);
}))
{
var ccm = new TestClientConnectionManager();
var ccf = new ClientConnectionFactory();
var protocol = new ServiceProtocol();
TestConnection transportConnection = null;
var connectionFactory = new TestConnectionFactory(conn =>
{
transportConnection = conn;
return(Task.CompletedTask);
});
var services = new ServiceCollection();
var lastWill = "This is the last will";
var connectionHandler = new LastWillConnectionHandler(lastWill);
services.AddSingleton(connectionHandler);
var builder = new ConnectionBuilder(services.BuildServiceProvider());
builder.UseConnectionHandler <LastWillConnectionHandler>();
ConnectionDelegate handler = builder.Build();
var connection = new ServiceConnection(protocol, ccm, connectionFactory, loggerFactory, handler, ccf,
"serverId", Guid.NewGuid().ToString("N"), null, null, closeTimeOutMilliseconds: 500);
var connectionTask = connection.StartAsync();
// completed handshake
await connection.ConnectionInitializedTask.OrTimeout();
Assert.Equal(ServiceConnectionStatus.Connected, connection.Status);
var clientConnectionId = Guid.NewGuid().ToString();
await transportConnection.Application.Output.WriteAsync(
protocol.GetMessageBytes(new OpenConnectionMessage(clientConnectionId, new Claim[] { })));
var clientConnection = await ccm.WaitForClientConnectionAsync(clientConnectionId).OrTimeout();
await clientConnection.LifetimeTask.OrTimeout();
transportConnection.Transport.Output.Complete();
var input = await transportConnection.Application.Input.ReadAsync();
var buffer = input.Buffer;
var canParse = protocol.TryParseMessage(ref buffer, out var msg);
Assert.True(canParse);
var message = msg as ConnectionDataMessage;
Assert.NotNull(message);
Assert.Equal(clientConnectionId, message.ConnectionId);
Assert.Equal(lastWill, Encoding.UTF8.GetString(message.Payload.First.ToArray()));
// complete reading to end the connection
transportConnection.Application.Output.Complete();
// 1s for application task to timeout
await connectionTask.OrTimeout(1000);
Assert.Equal(ServiceConnectionStatus.Disconnected, connection.Status);
Assert.Empty(ccm.ClientConnections);
}
}
public async Task TestServiceConnectionWithErrorApplicationTask()
{
using (StartVerifiableLog(out var loggerFactory, LogLevel.Warning, expectedErrors: c => true,
logChecker: logs =>
{
Assert.Equal(2, logs.Count);
Assert.Equal("SendLoopStopped", logs[0].Write.EventId.Name);
Assert.Equal("ApplicationTaskFailed", logs[1].Write.EventId.Name);
return(true);
}))
{
var ccm = new TestClientConnectionManager();
var ccf = new ClientConnectionFactory();
var protocol = new ServiceProtocol();
TestConnection transportConnection = null;
var connectionFactory = new TestConnectionFactory(conn =>
{
transportConnection = conn;
return(Task.CompletedTask);
});
var services = new ServiceCollection();
var errorTcs = new TaskCompletionSource <Exception>();
var connectionHandler = new ErrorConnectionHandler(errorTcs);
services.AddSingleton(connectionHandler);
var builder = new ConnectionBuilder(services.BuildServiceProvider());
builder.UseConnectionHandler <ErrorConnectionHandler>();
ConnectionDelegate handler = builder.Build();
var connection = new ServiceConnection(protocol, ccm, connectionFactory, loggerFactory, handler, ccf,
"serverId", Guid.NewGuid().ToString("N"), null, null);
var connectionTask = connection.StartAsync();
// completed handshake
await connection.ConnectionInitializedTask.OrTimeout();
Assert.Equal(ServiceConnectionStatus.Connected, connection.Status);
var clientConnectionId = Guid.NewGuid().ToString();
await transportConnection.Application.Output.WriteAsync(
protocol.GetMessageBytes(new OpenConnectionMessage(clientConnectionId, new Claim[] { })));
var clientConnection = await ccm.WaitForClientConnectionAsync(clientConnectionId).OrTimeout();
errorTcs.SetException(new InvalidOperationException("error operation"));
await clientConnection.LifetimeTask.OrTimeout();
// Should complete the connection when application throws
await ccm.WaitForClientConnectionRemovalAsync(clientConnectionId).OrTimeout();
// Application task should not affect the underlying service connection
Assert.Equal(ServiceConnectionStatus.Connected, connection.Status);
// complete reading to end the connection
transportConnection.Application.Output.Complete();
await connectionTask.OrTimeout();
Assert.Equal(ServiceConnectionStatus.Disconnected, connection.Status);
Assert.Empty(ccm.ClientConnections);
}
}