public async Task TestContainerWithTwoEndpointWithAllConnectedSucceedsWithGoodRouter()
{
var sem = new TestServiceEndpointManager(
new ServiceEndpoint(ConnectionString1),
new ServiceEndpoint(ConnectionString2));
var router = new TestEndpointRouter();
var writeTcs = new TaskCompletionSource <object>();
var containers = new Dictionary <ServiceEndpoint, TestServiceConnectionContainer>();
var container = new MultiEndpointServiceConnectionContainer("hub",
e => containers[e] = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
}, e), sem, router, NullLoggerFactory.Instance);
// All the connections started
_ = container.StartAsync();
await container.ConnectionInitializedTask;
var task = container.WriteAckableMessageAsync(DefaultGroupMessage);
await writeTcs.Task.OrTimeout();
containers.First().Value.HandleAck(new AckMessage(1, (int)AckStatus.Ok));
await task.OrTimeout();
}
private async Task TestEndpointOfflineInner(IServiceEndpointManager manager, IEndpointRouter router, bool migratable)
{
var containers = new List <TestServiceConnectionContainer>();
var container = new TestMultiEndpointServiceConnectionContainer("hub", e =>
{
var c = new TestServiceConnectionContainer(new List <IServiceConnection>
{
new TestSimpleServiceConnection(),
new TestSimpleServiceConnection()
});
c.MockOffline = true;
containers.Add(c);
return(c);
}, manager, router, NullLoggerFactory.Instance);
foreach (var c in containers)
{
Assert.False(c.IsOffline);
}
var expected = container.OfflineAsync(migratable);
var actual = await Task.WhenAny(
expected,
Task.Delay(TimeSpan.FromSeconds(1))
);
Assert.Equal(expected, actual);
foreach (var c in containers)
{
Assert.True(c.IsOffline);
}
}
public async Task TestContainerWithOneEndpointWithAllConnectedSucceeeds()
{
var sem = new TestServiceEndpointManager(new ServiceEndpoint(ConnectionString1));
var throws = new ServiceConnectionNotActiveException();
var router = new TestEndpointRouter(throws);
var writeTcs = new TaskCompletionSource <object>();
TestServiceConnectionContainer innerContainer = null;
var container = new MultiEndpointServiceConnectionContainer("hub",
e => innerContainer = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
}, e), sem, router, NullLoggerFactory.Instance);
// All the connections started
_ = container.StartAsync();
await container.ConnectionInitializedTask;
var task = container.WriteAckableMessageAsync(DefaultGroupMessage);
await writeTcs.Task.OrTimeout();
innerContainer.HandleAck(new AckMessage(1, (int)AckStatus.Ok));
await task.OrTimeout();
}
public async Task TestServersPing(int startCount, int stopCount, int expectedWarn)
{
using (StartVerifiableLog(out var loggerFactory, LogLevel.Warning, logChecker: logs =>
{
var warns = logs.Where(s => s.Write.LogLevel == LogLevel.Warning).ToList();
Assert.Equal(expectedWarn, warns.Count);
if (expectedWarn > 0)
{
Assert.Contains(warns, s => s.Write.Message.Contains("Failed to stop Servers timer as it's not started"));
}
return(true);
}))
{
List <IServiceConnection> connections = new List <IServiceConnection>
{
new SimpleTestServiceConnection(),
new SimpleTestServiceConnection(),
new SimpleTestServiceConnection()
};
using TestServiceConnectionContainer container =
new TestServiceConnectionContainer(
connections,
factory: new SimpleTestServiceConnectionFactory(),
logger: loggerFactory.CreateLogger <TestServiceConnectionContainer>());
await container.StartAsync();
var tasks = new List <Task>();
while (startCount > 0)
{
tasks.Add(container.StartGetServersPing());
startCount--;
}
await Task.WhenAll(tasks);
// default interval is 5s, add 2s for delay, validate any one connection write servers ping.
if (tasks.Count > 0)
{
await Task.WhenAny(connections.Select(c =>
{
var connection = c as SimpleTestServiceConnection;
return(connection.ServersPingTask.OrTimeout(7000));
}));
}
tasks.Clear();
while (stopCount > 0)
{
tasks.Add(container.StopGetServersPing());
stopCount--;
}
await Task.WhenAll(tasks);
}
}
public async Task TestTwoEndpointsWithAllNotFoundAck()
{
var sem = new TestServiceEndpointManager(
new ServiceEndpoint(ConnectionString1),
new ServiceEndpoint(ConnectionString2, name: "online"));
var router = new TestEndpointRouter();
var writeTcs = new TaskCompletionSource <object>();
var containers = new Dictionary <ServiceEndpoint, TestServiceConnectionContainer>();
var container = new MultiEndpointServiceConnectionContainer("hub", e =>
{
if (string.IsNullOrEmpty(e.Name))
{
return(containers[e] = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
}, e));
}
return(containers[e] = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
}, e));
}, sem, router, NullLoggerFactory.Instance);
// All the connections started
_ = container.StartAsync();
await container.ConnectionInitializedTask;
var task = container.WriteAckableMessageAsync(DefaultGroupMessage);
await writeTcs.Task.OrTimeout();
foreach (var c in containers)
{
c.Value.HandleAck(new AckMessage(1, (int)AckStatus.NotFound));
}
var result = await task.OrTimeout();
Assert.False(result);
}
public async Task TestContainerWithTwoEndpointWithPrimaryOfflineAndConnectionStartedSucceeds()
{
var sem = new TestServiceEndpointManager(
new ServiceEndpoint(ConnectionString1),
new ServiceEndpoint(ConnectionString2, EndpointType.Secondary, "online"));
var router = new TestEndpointRouter();
var writeTcs = new TaskCompletionSource <object>();
var containers = new Dictionary <ServiceEndpoint, TestServiceConnectionContainer>();
var container = new MultiEndpointServiceConnectionContainer("hub", e =>
{
if (string.IsNullOrEmpty(e.Name))
{
return(containers[e] = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
}, e));
}
return(containers[e] = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
}, e));
}, sem, router, NullLoggerFactory.Instance);
_ = container.StartAsync();
var task = container.WriteAckableMessageAsync(DefaultGroupMessage);
await writeTcs.Task.OrTimeout();
containers.First(p => !string.IsNullOrEmpty(p.Key.Name)).Value.HandleAck(new AckMessage(1, (int)AckStatus.Ok));
await task.OrTimeout();
var endpoints = container.GetOnlineEndpoints().ToArray();
Assert.Single(endpoints);
Assert.Equal("online", endpoints.First().Name);
}
public async Task TestServiceConnectionContainerWithSomeThrows2WriteWithPartitionCanPass()
{
var container = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestServiceConnection(),
new TestServiceConnection(throws: true),
new TestServiceConnection(throws: true),
new TestServiceConnection(throws: true),
new TestServiceConnection(throws: true),
new TestServiceConnection(throws: true),
new TestServiceConnection(throws: true),
});
await container.WriteAsync(new HandshakeResponseMessage());
await container.WriteAsync("1", new HandshakeResponseMessage());
}
public async Task TestServiceConnectionContainerWithAllConnectedSucceeeds()
{
var container = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestServiceConnection(),
new TestServiceConnection(),
new TestServiceConnection(),
new TestServiceConnection(),
new TestServiceConnection(),
new TestServiceConnection(),
new TestServiceConnection(),
});
await container.WriteAsync(new HandshakeResponseMessage());
await container.WriteAsync("1", new HandshakeResponseMessage());
}
public async Task TestTwoEndpointsWithCancellationToken()
{
var sem = new TestServiceEndpointManager(
new ServiceEndpoint(ConnectionString1),
new ServiceEndpoint(ConnectionString2, name: "online"));
var router = new TestEndpointRouter();
var writeTcs = new TaskCompletionSource <object>();
var containers = new Dictionary <ServiceEndpoint, TestServiceConnectionContainer>();
var container = new MultiEndpointServiceConnectionContainer("hub", e =>
{
if (string.IsNullOrEmpty(e.Name))
{
return(containers[e] = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
}, e, new AckHandler(100, 200)));
}
return(containers[e] = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
}, e, new AckHandler(100, 200)));
}, sem, router, NullLoggerFactory.Instance);
// All the connections started
_ = container.StartAsync();
await container.ConnectionInitializedTask;
var task = container.WriteAckableMessageAsync(DefaultGroupMessage, new CancellationToken(true));
await writeTcs.Task.OrTimeout();
foreach (var c in containers)
{
c.Value.HandleAck(new AckMessage(1, (int)AckStatus.Timeout));
}
await Assert.ThrowsAnyAsync <OperationCanceledException>(async() => await task).OrTimeout();
}
public async Task TestContainerWithTwoEndpointWithOneOfflineSucceeds()
{
using (StartVerifiableLog(out var loggerFactory, LogLevel.Warning))
{
var sem = new TestServiceEndpointManager(
new ServiceEndpoint(ConnectionString1),
new ServiceEndpoint(ConnectionString2, name: "online"));
var router = new TestEndpointRouter();
var writeTcs = new TaskCompletionSource <object>();
var containers = new Dictionary <ServiceEndpoint, TestServiceConnectionContainer>();
var container = new MultiEndpointServiceConnectionContainer("hub", e =>
{
if (string.IsNullOrEmpty(e.Name))
{
return(containers[e] = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(ServiceConnectionStatus.Disconnected, writeAsyncTcs: writeTcs),
}, e));
}
return(containers[e] = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
new TestSimpleServiceConnection(writeAsyncTcs: writeTcs),
}, e));
}, sem, router, loggerFactory);
_ = container.StartAsync();
var task = container.WriteAckableMessageAsync(DefaultGroupMessage);
await writeTcs.Task.OrTimeout();
containers.First(p => !string.IsNullOrEmpty(p.Key.Name)).Value.HandleAck(new AckMessage(1, (int)AckStatus.Ok));
await task.OrTimeout();
}
}
public async Task TestIfConnectionWillRestartAfterShutdown()
{
List <IServiceConnection> connections = new List <IServiceConnection>
{
new SimpleTestServiceConnection(),
new SimpleTestServiceConnection() // A connection which is not in Connected status could be replaced.
};
IServiceConnection connection = connections[1];
using TestServiceConnectionContainer container = new TestServiceConnectionContainer(connections, factory: new SimpleTestServiceConnectionFactory());
container.ShutdownForTest();
await container.OnConnectionCompleteForTestShutdown(connection);
// connection should be replaced, but it's StartAsync method should not be called.
Assert.NotEqual(container.Connections[1], connection);
Assert.NotEqual(ServiceConnectionStatus.Connected, container.Connections[1].Status);
}
public async Task TestServiceConnectionContainerWithAllThrowsThrows()
{
var container = new TestServiceConnectionContainer(new List <IServiceConnection> {
new TestServiceConnection(throws: true),
new TestServiceConnection(throws: true),
new TestServiceConnection(throws: true),
new TestServiceConnection(throws: true),
new TestServiceConnection(throws: true),
new TestServiceConnection(throws: true),
new TestServiceConnection(throws: true),
});
await Assert.ThrowsAsync <ServiceConnectionNotActiveException>(
() => container.WriteAsync(new HandshakeResponseMessage())
);
await Assert.ThrowsAsync <ServiceConnectionNotActiveException>(
() => container.WriteAsync("1", new HandshakeResponseMessage())
);
}
public async Task TestOffline()
{
List <IServiceConnection> connections = new List <IServiceConnection>
{
new SimpleTestServiceConnection(),
new SimpleTestServiceConnection()
};
using TestServiceConnectionContainer container = new TestServiceConnectionContainer(connections, factory: new SimpleTestServiceConnectionFactory());
foreach (SimpleTestServiceConnection c in connections)
{
Assert.False(c.ConnectionOfflineTask.IsCompleted);
}
await container.OfflineAsync();
foreach (SimpleTestServiceConnection c in connections)
{
Assert.True(c.ConnectionOfflineTask.IsCompleted);
}
}
internal async Task TestIfConnectionWillNotRestartAfterShutdown(ServiceConnectionStatus status)
{
List <IServiceConnection> connections = new List <IServiceConnection>
{
new SimpleTestServiceConnection(),
new SimpleTestServiceConnection(status: status)
};
IServiceConnection connection = connections[1];
using TestServiceConnectionContainer container = new TestServiceConnectionContainer(connections, factory: new SimpleTestServiceConnectionFactory());
container.ShutdownForTest();
await container.OnConnectionCompleteForTestShutdown(connection);
// the connection should not be replaced when shutting down
Assert.Equal(container.Connections[1], connection);
// its status is not changed
Assert.Equal(status, container.Connections[1].Status);
// the container is not listening to the connection's status changes after shutdown
Assert.Equal(1, (connection as SimpleTestServiceConnection).ConnectionStatusChangedRemoveCount);
}