public async Task Active_UnaryCall_MultipleStreams_UnavailableAddress_FallbackToWorkingAddress() { // Ignore errors SetExpectedErrorsFilter(writeContext => { return(true); }); var tcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); string?host = null; async Task <HelloReply> UnaryMethod(HelloRequest request, ServerCallContext context) { var protocol = context.GetHttpContext().Request.Protocol; Logger.LogInformation("Received protocol: " + protocol); await tcs.Task; host = context.Host; return(new HelloReply { Message = request.Name }); } // Arrange using var endpoint1 = BalancerHelpers.CreateGrpcEndpoint <HelloRequest, HelloReply>(50051, UnaryMethod, nameof(UnaryMethod), HttpProtocols.Http1AndHttp2, isHttps: true); using var endpoint2 = BalancerHelpers.CreateGrpcEndpoint <HelloRequest, HelloReply>(50052, UnaryMethod, nameof(UnaryMethod), HttpProtocols.Http1AndHttp2, isHttps: true); var services = new ServiceCollection(); services.AddSingleton <ResolverFactory>(new StaticResolverFactory(_ => new[] { new BalancerAddress(endpoint1.Address.Host, endpoint1.Address.Port), new BalancerAddress(endpoint2.Address.Host, endpoint2.Address.Port) })); var socketsHttpHandler = new SocketsHttpHandler { EnableMultipleHttp2Connections = true, SslOptions = new System.Net.Security.SslClientAuthenticationOptions() { RemoteCertificateValidationCallback = (_, __, ___, ____) => true } }; var grpcWebHandler = new GrpcWebHandler(GrpcWebMode.GrpcWeb, new RequestVersionHandler(socketsHttpHandler)); var channel = GrpcChannel.ForAddress("static:///localhost", new GrpcChannelOptions { LoggerFactory = LoggerFactory, HttpHandler = grpcWebHandler, ServiceProvider = services.BuildServiceProvider(), Credentials = new SslCredentials() }); var client = TestClientFactory.Create(channel, endpoint1.Method); // Act grpcWebHandler.HttpVersion = new Version(1, 1); var http11CallTasks = new List <Task <HelloReply> >(); for (int i = 0; i < 10; i++) { Logger.LogInformation($"Sending gRPC call {i}"); http11CallTasks.Add(client.UnaryCall(new HelloRequest { Name = "Balancer" }).ResponseAsync); } Logger.LogInformation($"Done sending gRPC calls"); var balancer = BalancerHelpers.GetInnerLoadBalancer <PickFirstBalancer>(channel) !; var subchannel = balancer._subchannel !; var transport = (SocketConnectivitySubchannelTransport)subchannel.Transport; var activeStreams = transport.GetActiveStreams(); // Assert Assert.AreEqual(HttpHandlerType.SocketsHttpHandler, channel.HttpHandlerType); await TestHelpers.AssertIsTrueRetryAsync(() => { activeStreams = transport.GetActiveStreams(); return(activeStreams.Count == 10); }, "Wait for connections to start."); foreach (var t in activeStreams) { Assert.AreEqual(new DnsEndPoint("127.0.0.1", 50051), t.Address.EndPoint); } // Act grpcWebHandler.HttpVersion = new Version(2, 0); var http2CallTasks = new List <Task <HelloReply> >(); for (int i = 0; i < 10; i++) { Logger.LogInformation($"Sending gRPC call {i}"); http2CallTasks.Add(client.UnaryCall(new HelloRequest { Name = "Balancer" }).ResponseAsync); } Logger.LogInformation($"Done sending gRPC calls"); // Assert await TestHelpers.AssertIsTrueRetryAsync(() => { activeStreams = transport.GetActiveStreams(); return(activeStreams.Count == 11); }, "Wait for connections to start."); Assert.AreEqual(new DnsEndPoint("127.0.0.1", 50051), activeStreams.Last().Address.EndPoint); tcs.SetResult(null); await Task.WhenAll(http11CallTasks).DefaultTimeout(); await Task.WhenAll(http2CallTasks).DefaultTimeout(); Assert.AreEqual(ConnectivityState.Ready, channel.State); Logger.LogInformation($"Closing {endpoint1}"); endpoint1.Dispose(); // There are still be 10 HTTP/1.1 connections because they aren't immediately removed // when the server is shutdown and connectivity is lost. await TestHelpers.AssertIsTrueRetryAsync(() => { activeStreams = transport.GetActiveStreams(); return(activeStreams.Count == 10); }, "Wait for HTTP/2 connection to end."); grpcWebHandler.HttpVersion = new Version(1, 1); await Task.Delay(1000); Logger.LogInformation($"Starting failed call"); var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => client.UnaryCall(new HelloRequest { Name = "Balancer" }).ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); // Removed by failed call. activeStreams = transport.GetActiveStreams(); Assert.AreEqual(0, activeStreams.Count); Assert.AreEqual(ConnectivityState.Idle, channel.State); Logger.LogInformation($"Next call goes to fallback address."); var reply = await client.UnaryCall(new HelloRequest { Name = "Balancer" }).ResponseAsync.TimeoutAfter(TimeSpan.FromSeconds(20)); Assert.AreEqual("Balancer", reply.Message); Assert.AreEqual("127.0.0.1:50052", host); activeStreams = transport.GetActiveStreams(); Assert.AreEqual(1, activeStreams.Count); Assert.AreEqual(new DnsEndPoint("127.0.0.1", 50052), activeStreams[0].Address.EndPoint); }