public void AConnectionMustBeMadeEvenIfAllNodesAreDead() { using (var fake = new AutoFake(callsDoNothing: true)) { //make sure we retry one more time then we have nodes //original call + 4 retries == 5 fake.Provide <IConnectionConfigurationValues>( new ConnectionConfiguration(_connectionPool) .MaximumRetries(4) ); //set up our GET to / to return 4 503's followed by a 200 var getCall = A.CallTo(() => fake.Resolve <IConnection>().GetSync(A <Uri> ._, A <IRequestConfiguration> ._)); getCall.ReturnsNextFromSequence( _bad, _bad, _bad, _bad, _ok ); var transport = this.ProvideTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(_ok); //setup client var client = fake.Resolve <ElasticsearchClient>(); //Do not throw because by miracle the 4th retry manages to give back a 200 //even if all nodes have been marked dead. Assert.DoesNotThrow(() => client.Info()); //original call + 4 retries == 5 getCall.MustHaveHappened(Repeated.Exactly.Times(5)); //ping must have been send out 4 times to the 4 nodes being used for the first time pingCall.MustHaveHappened(Repeated.Exactly.Times(4)); } }
public void ShouldRetryOnPingConnectionException() { using (var fake = new AutoFake(callsDoNothing: true)) { var connectionPool = new StaticConnectionPool(_uris, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool); fake.Provide <IConnectionConfigurationValues>(config); FakeCalls.ProvideDefaultTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); var seenPorts = new List <int>(); pingCall.ReturnsLazily((Uri u, IRequestConfiguration c) => { seenPorts.Add(u.Port); throw new Exception("Something bad happened"); }); var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.Ok(config)); var client = fake.Resolve <ElasticsearchClient>(); var e = Assert.Throws <MaxRetryException>(() => client.Info()); pingCall.MustHaveHappened(Repeated.Exactly.Times(_retries + 1)); getCall.MustNotHaveHappened(); //make sure that if a ping throws an exception it wont //keep retrying to ping the same node but failover to the next seenPorts.ShouldAllBeEquivalentTo(_uris.Select(u => u.Port)); var aggregateException = e.InnerException as AggregateException; aggregateException.Should().NotBeNull(); aggregateException.InnerExceptions.Should().Contain(ex => ex.GetType().Name == "PingException"); } }
public void ShouldNotThrowAndNotRetry401() { using (var fake = new AutoFake(callsDoNothing: true)) { var uris = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201"), new Uri("http://localhost:9202") }; var connectionPool = new StaticConnectionPool(uris, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool); fake.Provide <IConnectionConfigurationValues>(config); FakeCalls.ProvideDefaultTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.Any(config, 401)); var client = fake.Resolve <ElasticsearchClient>(); Assert.DoesNotThrow(() => client.Info()); pingCall.MustHaveHappened(Repeated.Exactly.Once); getCall.MustHaveHappened(Repeated.Exactly.Once); } }
public void HardRetryLimitTakesPrecedenceOverNumberOfNodes() { using (var fake = new AutoFake(callsDoNothing: true)) { //setup config values with a hight retry count fake.Provide <IConnectionConfigurationValues>( new ConnectionConfiguration(_connectionPool) .MaximumRetries(7) ); var getCall = A.CallTo(() => fake.Resolve <IConnection>().GetSync(A <Uri> ._, A <IRequestConfiguration> ._)); getCall.Throws <Exception>(); var transport = this.ProvideTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(_ok); var client = fake.Resolve <ElasticsearchClient>(); Assert.Throws <MaxRetryException>(() => client.Info()); //We want to see 8 attempt to do a GET on / //(original call + retry of 7 == 8) getCall.MustHaveHappened(Repeated.Exactly.Times(8)); } }
public void Should_Not_Retry_On_IConnection_Exception() { using (var fake = new AutoFake(callsDoNothing: true)) { //set up connection configuration that holds a connection pool //with '_uris' (see the constructor) fake.Provide <IConnectionConfigurationValues>(_config); //prove a real HttpTransport with its unspecified dependencies //as fakes this.ProvideTransport(fake); //set up fake for a call on IConnection.GetSync so that it always throws //an exception var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.AnyWithException(_config, -1, innerException: new Exception("inner"))); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(_ok); //create a real ElasticsearchClient with it unspecified dependencies //as fakes var client = fake.Resolve <ElasticsearchClient>(); //our settings dictate retrying 2 times client.Settings.MaxRetries.Should().Be(2); //the single node connection pool always reports 0 for maxretries client.Settings.ConnectionPool.MaxRetries.Should().Be(0); // var exception = Assert.Throws <Exception>(() => client.Info()); exception.Message.Should().Be("inner"); //the GetSync method must in total have been called the number of nodes times. getCall.MustHaveHappened(Repeated.Exactly.Once); } }
public void ThrowsOutOfNodesException_AndRetriesTheSpecifiedTimes() { using (var fake = new AutoFake(callsDoNothing: true)) { //set up connection configuration that holds a connection pool //with '_uris' (see the constructor) fake.Provide <IConnectionConfigurationValues>(_config); //prove a real HttpTransport with its unspecified dependencies //as fakes this.ProvideTransport(fake); //set up fake for a call on IConnection.GetSync so that it always throws //an exception var getCall = FakeCalls.GetSyncCall(fake); getCall.Throws <Exception>(); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(_ok); //create a real ElasticsearchClient with it unspecified dependencies //as fakes var client = fake.Resolve <ElasticsearchClient>(); //we don't specify our own value so it should be up to the connection pool client.Settings.MaxRetries.Should().Be(null); //the default for the connection pool should be the number of nodes - 1; client.Settings.ConnectionPool.MaxRetries.Should().Be(_retries); //calling GET / should throw an OutOfNodesException because our fake //will always throw an exception Assert.Throws <MaxRetryException>(() => client.Info()); //the GetSync method must in total have been called the number of nodes times. getCall.MustHaveHappened(Repeated.Exactly.Times(_retries + 1)); } }
public void IfResponseIsKnowError_DoNotRetry_ThrowServerException(int status, string exceptionType, string exceptionMessage) { var response = CreateServerExceptionResponse(status, exceptionType, exceptionMessage); using (var fake = new AutoFake(callsDoNothing: true)) { var connectionPool = new StaticConnectionPool(new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201"), }); var connectionConfiguration = new ConnectionConfiguration(connectionPool) .ThrowOnElasticsearchServerExceptions() .ExposeRawResponse(false); fake.Provide <IConnectionConfigurationValues>(connectionConfiguration); FakeCalls.ProvideDefaultTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(connectionConfiguration)); var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.Any(connectionConfiguration, status, response: response)); var client = fake.Resolve <ElasticsearchClient>(); var e = Assert.Throws <ElasticsearchServerException>(() => client.Info()); AssertServerErrorsOnResponse(e, status, exceptionType, exceptionMessage); //make sure a know ElasticsearchServerException does not cause a retry //In this case we want to fail early getCall.MustHaveHappened(Repeated.Exactly.Once); } }
public void AllNodesWillBeMarkedDead() { using (var fake = new AutoFake(callsDoNothing: true)) { //set up a fake datetimeprovider var dateTimeProvider = fake.Resolve <IDateTimeProvider>(); fake.Provide(dateTimeProvider); //create a connectionpool that uses the fake datetimeprovider var connectionPool = new StaticConnectionPool( _uris, dateTimeProvider: dateTimeProvider ); var config = new ConnectionConfiguration(connectionPool); fake.Provide <IConnectionConfigurationValues>(config); //Now() on the fake still means Now() A.CallTo(() => dateTimeProvider.Now()).Returns(DateTime.UtcNow); //Set up individual mocks for each DeadTime(Uri uri,...) call //where uri matches one of the node ports var calls = _uris.Select(u => A.CallTo(() => dateTimeProvider.DeadTime( A <Uri> .That.Matches(uu => uu.Port == u.Port), A <int> ._, A <int?> ._, A <int?> ._ ))).ToList(); //all the fake mark dead calls return 60 seconds into the future foreach (var call in calls) { call.Returns(DateTime.UtcNow.AddSeconds(60)); } //When we do a GET on / we always recieve a 503 var getCall = A.CallTo(() => fake.Resolve <IConnection>().GetSync(A <Uri> ._, A <IRequestConfiguration> ._)); getCall.Returns(_bad); var transport = this.ProvideTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(_ok); var client = fake.Resolve <ElasticsearchClient>(); //Since we always get a 503 we should see an out of nodes exception Assert.Throws <MaxRetryException>(() => client.Info()); pingCall.MustHaveHappened(Repeated.Exactly.Times(4)); //The call should be tried on all the nodes getCall.MustHaveHappened(Repeated.Exactly.Times(4)); //We should see each individual node being marked as dead foreach (var call in calls) { call.MustHaveHappened(Repeated.Exactly.Once); } } }
public void IfAConnectionComesBackToLifeOnItsOwnItShouldBeMarked() { using (var fake = new AutoFake(callsDoNothing: true)) { //Setting up a datetime provider so that can track dead/alive marks var dateTimeProvider = fake.Resolve <IDateTimeProvider>(); A.CallTo(() => dateTimeProvider.Now()).Returns(DateTime.UtcNow); var markDeadCall = A.CallTo(() => dateTimeProvider.DeadTime(A <Uri> ._, A <int> ._, A <int?> ._, A <int?> ._)); var markAliveCall = A.CallTo(() => dateTimeProvider.AliveTime(A <Uri> ._, A <int> ._)); markDeadCall.Returns(DateTime.UtcNow.AddSeconds(60)); markAliveCall.Returns(new DateTime()); fake.Provide(dateTimeProvider); var connectionPool = new StaticConnectionPool( _uris, dateTimeProvider: dateTimeProvider); //set retries to 4 fake.Provide <IConnectionConfigurationValues>( new ConnectionConfiguration(connectionPool) .MaximumRetries(4) ); //fake getsync handler that return a 503 4 times and then a 200 //this will cause all 4 nodes to be marked dead on the first client call var getCall = FakeCalls.GetSyncCall(fake); getCall.ReturnsNextFromSequence( _bad, _bad, _bad, _bad, _ok ); //provide a transport with all the dependencies resolved var transport = this.ProvideTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(_ok); //instantiate connection with faked dependencies var client = fake.Resolve <ElasticsearchClient>(); //Do not throw because by miracle the 4th retry manages to give back a 200 //even if all nodes have been marked dead. Assert.DoesNotThrow(() => client.Info()); //original call + 4 retries is 5 getCall.MustHaveHappened(Repeated.Exactly.Times(5)); //4 nodes must be marked dead markDeadCall.MustHaveHappened(Repeated.Exactly.Times(4)); //atleast one of them sprung back to live so markAlive must be called once markAliveCall.MustHaveHappened(Repeated.Exactly.Times(1)); } }
public void SniffCalledOnceAndEachEnpointPingedOnce() { using (var fake = new AutoFake(callsDoNothing: true)) { //It's recommended to only have on instance of your connection pool //Be sure to register it as Singleton in your IOC var uris = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201") }; var connectionPool = new SniffingConnectionPool(uris); var config = new ConnectionConfiguration(connectionPool) .SniffOnStartup(); fake.Provide <IConnectionConfigurationValues>(config); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake, config, uris); var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.Ok(config)); var transport1 = FakeCalls.ProvideRealTranportInstance(fake); var transport2 = FakeCalls.ProvideRealTranportInstance(fake); var transport3 = FakeCalls.ProvideRealTranportInstance(fake); var transport4 = FakeCalls.ProvideRealTranportInstance(fake); transport1.Should().NotBe(transport2); var client1 = new ElasticsearchClient(config, transport: transport1); client1.Info(); client1.Info(); client1.Info(); client1.Info(); var client2 = new ElasticsearchClient(config, transport: transport2); client2.Info(); client2.Info(); client2.Info(); client2.Info(); var client3 = new ElasticsearchClient(config, transport: transport3); client3.Info(); client3.Info(); client3.Info(); client3.Info(); var client4 = new ElasticsearchClient(config, transport: transport4); client4.Info(); client4.Info(); client4.Info(); client4.Info(); sniffCall.MustHaveHappened(Repeated.Exactly.Once); //sniff validates first node, one new node should be pinged before usage. pingCall.MustHaveHappened(Repeated.Exactly.Once); } }
public void SniffIsCalledAfterItHasGoneOutOfDate_NotWhenItSeesA503() { using (var fake = new AutoFake()) { var dateTimeProvider = fake.Resolve <IDateTimeProvider>(); var nowCall = A.CallTo(() => dateTimeProvider.Now()); nowCall.ReturnsNextFromSequence( DateTime.UtcNow, //initial sniff time (set even if not sniff_on_startup DateTime.UtcNow, //info call 1 DateTime.UtcNow, //info call 2 DateTime.UtcNow.AddMinutes(10), //info call 3 DateTime.UtcNow.AddMinutes(10), //set now after sniff 3 DateTime.UtcNow.AddMinutes(10), //info call 4 DateTime.UtcNow.AddMinutes(12) //info call 5 ); var uris = new[] { new Uri("http://localhost:9200") }; var connectionPool = new SniffingConnectionPool(uris); var config = new ConnectionConfiguration(connectionPool) .SniffLifeSpan(TimeSpan.FromMinutes(4)) .ExposeRawResponse(); fake.Provide <IConnectionConfigurationValues>(config); var transport = FakeCalls.ProvideDefaultTransport(fake, dateTimeProvider); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake, config, uris); var getCall = FakeCalls.GetSyncCall(fake); getCall.ReturnsNextFromSequence( FakeResponse.Ok(config), //info 1 FakeResponse.Ok(config), //info 2 FakeResponse.Ok(config), //info 3 FakeResponse.Ok(config), //sniff FakeResponse.Ok(config), //info 4 FakeResponse.Bad(config) //info 5 ); var client1 = fake.Resolve <ElasticsearchClient>(); var result = client1.Info(); //info call 1 result = client1.Info(); //info call 2 result = client1.Info(); //info call 3 result = client1.Info(); //info call 4 result = client1.Info(); //info call 5 sniffCall.MustHaveHappened(Repeated.Exactly.Once); nowCall.MustHaveHappened(Repeated.Exactly.Times(7)); //var nowCall = A.CallTo(() => fake.Resolve<IDateTimeProvider>().Sniff(A<Uri>._, A<int>._)); } }
public void ShouldRetryOnSniffConnectionException_Async() { using (var fake = new AutoFake(callsDoNothing: true)) { var uris = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201"), new Uri("http://localhost:9202") }; var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool) .SniffOnConnectionFault(); fake.Provide <IConnectionConfigurationValues>(config); var pingAsyncCall = FakeCalls.PingAtConnectionLevelAsync(fake); pingAsyncCall.Returns(FakeResponse.OkAsync(config)); //sniffing is always synchronous and in turn will issue synchronous pings var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake); var seenPorts = new List <int>(); sniffCall.ReturnsLazily((Uri u, IRequestConfiguration c) => { seenPorts.Add(u.Port); throw new Exception("Something bad happened"); }); var getCall = FakeCalls.GetCall(fake); getCall.Returns(FakeResponse.BadAsync(config)); FakeCalls.ProvideDefaultTransport(fake); var client = fake.Resolve <ElasticsearchClient>(); var e = Assert.Throws <MaxRetryException>(async() => await client.NodesHotThreadsAsync("nodex")); //all nodes must be tried to sniff for more information sniffCall.MustHaveHappened(Repeated.Exactly.Times(uris.Count())); //make sure we only saw one call to hot threads (the one that failed initially) getCall.MustHaveHappened(Repeated.Exactly.Once); //make sure the sniffs actually happened on all the individual nodes seenPorts.ShouldAllBeEquivalentTo(uris.Select(u => u.Port)); e.InnerException.Message.Should().Contain("Sniffing known nodes"); } }
public void SniffOnConnectionFaultCausesSniffOn503() { using (var fake = new AutoFake()) { var dateTimeProvider = fake.Resolve <IDateTimeProvider>(); var nowCall = A.CallTo(() => dateTimeProvider.Now()); nowCall.Invokes(() => { }); nowCall.Returns(DateTime.UtcNow); var nodes = new[] { new Uri("http://localhost:9200") }; var connectionPool = new SniffingConnectionPool(nodes); var config = new ConnectionConfiguration(connectionPool) .SniffOnConnectionFault(); fake.Provide <IConnectionConfigurationValues>(config); var transport = FakeCalls.ProvideDefaultTransport(fake, dateTimeProvider); var connection = fake.Resolve <IConnection>(); var sniffNewNodes = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201") }; var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake, config, sniffNewNodes); var getCall = FakeCalls.GetSyncCall(fake); getCall.ReturnsNextFromSequence( FakeResponse.Ok(config), //info 1 FakeResponse.Ok(config), //info 2 FakeResponse.Ok(config), //info 3 FakeResponse.Ok(config), //info 4 FakeResponse.Bad(config) //info 5 ); var client1 = fake.Resolve <ElasticsearchClient>(); client1.Info(); //info call 1 client1.Info(); //info call 2 client1.Info(); //info call 3 client1.Info(); //info call 4 Assert.Throws <MaxRetryException>(() => client1.Info()); //info call 5 sniffCall.MustHaveHappened(Repeated.Exactly.Once); nowCall.MustHaveHappened(Repeated.Exactly.Times(8)); } }
public void DeadNodesAreNotVisited_AndPingedAppropiately() { using (var fake = new AutoFake()) { var dateTimeProvider = ProvideDateTimeProvider(fake); var config = ProvideConfiguration(dateTimeProvider); var connection = ProvideConnection(fake, config); var getCall = FakeCalls.GetSyncCall(fake); var ok = FakeResponse.Ok(config); var bad = FakeResponse.Bad(config); getCall.ReturnsNextFromSequence( ok, //info 1 - 9204 bad, //info 2 - 9203 DEAD ok, //info 2 retry - 9202 ok, //info 3 - 9201 ok, //info 4 - 9204 ok, //info 5 - 9202 ok, //info 6 - 9201 ok, //info 7 - 9204 ok, //info 8 - 9203 (Now > Timeout) ok //info 9 - 9202 ); var seenNodes = new List <Uri>(); getCall.Invokes((Uri u, IRequestConnectionConfiguration o) => seenNodes.Add(u)); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(ok); var client1 = fake.Resolve <ElasticsearchClient>(); client1.Info(); //info call 1 client1.Info(); //info call 2 client1.Info(); //info call 3 client1.Info(); //info call 4 client1.Info(); //info call 5 client1.Info(); //info call 6 client1.Info(); //info call 7 client1.Info(); //info call 8 client1.Info(); //info call 9 AssertSeenNodesAreInExpectedOrder(seenNodes); //4 nodes first time usage + 1 time after the first time 9203 came back to live pingCall.MustHaveHappened(Repeated.Exactly.Times(5)); } }
public void ShouldRetryOnSniff500() { using (var fake = new AutoFake(callsDoNothing: true)) { var uris = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201"), new Uri("http://localhost:9202") }; var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool) .SniffOnConnectionFault(); fake.Provide <IConnectionConfigurationValues>(config); FakeCalls.ProvideDefaultTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake); var seenPorts = new List <int>(); sniffCall.ReturnsLazily((Uri u, IRequestConfiguration c) => { seenPorts.Add(u.Port); return(FakeResponse.Bad(config)); }); var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.Bad(config)); var client = fake.Resolve <ElasticsearchClient>(); var e = Assert.Throws <MaxRetryException>(() => client.Info()); sniffCall.MustHaveHappened(Repeated.Exactly.Times(uris.Count())); getCall.MustHaveHappened(Repeated.Exactly.Once); //make sure that if a ping throws an exception it wont //keep retrying to ping the same node but failover to the next seenPorts.ShouldAllBeEquivalentTo(uris.Select(u => u.Port)); var sniffException = e.InnerException as SniffException; sniffException.Should().NotBeNull(); } }
public void ShouldThrowAndNotRetrySniffOnConnectionFault401_Async() { using (var fake = new AutoFake(callsDoNothing: true)) { var uris = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201"), new Uri("http://localhost:9202") }; var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool) .SniffOnConnectionFault() .ThrowOnElasticsearchServerExceptions(); fake.Provide <IConnectionConfigurationValues>(config); FakeCalls.ProvideDefaultTransport(fake); var pingAsyncCall = FakeCalls.PingAtConnectionLevelAsync(fake); pingAsyncCall.Returns(FakeResponse.OkAsync(config)); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake); var seenPorts = new List <int>(); sniffCall.ReturnsLazily((Uri u, IRequestConfiguration c) => { seenPorts.Add(u.Port); return(FakeResponse.Any(config, 401)); }); var getCall = FakeCalls.GetCall(fake); getCall.Returns(FakeResponse.BadAsync(config)); var client = fake.Resolve <ElasticsearchClient>(); var e = Assert.Throws <ElasticsearchServerException>(async() => await client.InfoAsync()); e.Status.Should().Be(401); sniffCall.MustHaveHappened(Repeated.Exactly.Once); getCall.MustHaveHappened(Repeated.Exactly.Once); } }
public void ShouldNotThrowAndNotRetrySniffInformationIsTooOld401() { using (var fake = new AutoFake(callsDoNothing: true)) { var uris = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201"), new Uri("http://localhost:9202") }; var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool) .SniffLifeSpan(new TimeSpan(1)); fake.Provide <IConnectionConfigurationValues>(config); FakeCalls.ProvideDefaultTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake); var seenPorts = new List <int>(); sniffCall.ReturnsLazily((Uri u, IRequestConfiguration c) => { seenPorts.Add(u.Port); return(FakeResponse.Any(config, 401)); }); var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.Bad(config)); var client = fake.Resolve <ElasticsearchClient>(); Assert.DoesNotThrow(() => client.Info()); sniffCall.MustHaveHappened(Repeated.Exactly.Once); getCall.MustNotHaveHappened(); } }
public void ShouldThrowAndNotRetrySniffOnStartup401() { using (var fake = new AutoFake(callsDoNothing: true)) { var uris = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201"), new Uri("http://localhost:9202") }; var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool) .SniffOnStartup(); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake); var seenPorts = new List <int>(); sniffCall.ReturnsLazily((Uri u, IRequestConfiguration c) => { seenPorts.Add(u.Port); return(FakeResponse.Any(config, 401)); }); var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.Bad(config)); fake.Provide <IConnectionConfigurationValues>(config); var e = Assert.Throws <ElasticsearchAuthenticationException>(() => FakeCalls.ProvideRealTranportInstance(fake)); sniffCall.MustHaveHappened(Repeated.Exactly.Once); getCall.MustNotHaveHappened(); } }
public void FailEarlyIfTimeoutIsExhausted() { using (var fake = new AutoFake()) { var dateTimeProvider = ProvideDateTimeProvider(fake); var config = ProvideConfiguration(dateTimeProvider); var connection = ProvideConnection(fake, config, dateTimeProvider); var getCall = FakeCalls.GetSyncCall(fake); var ok = FakeResponse.Ok(config); var bad = FakeResponse.Bad(config); getCall.ReturnsNextFromSequence( bad, //info 1 - 9204 bad, //info 2 - 9203 DEAD ok //info 2 retry - 9202 ); var seenNodes = new List <Uri>(); getCall.Invokes((Uri u, IRequestConfiguration o) => seenNodes.Add(u)); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(ok); var client1 = fake.Resolve <ElasticsearchClient>(); //event though the third node should have returned ok, the first 2 calls took a minute var e = Assert.Throws <MaxRetryException>(() => client1.Info()); e.Message.Should() .StartWith("Retry timeout 00:01:20 was hit after retrying 1 times:"); IElasticsearchResponse response = null; Assert.DoesNotThrow(() => response = client1.Info()); response.Should().NotBeNull(); response.Success.Should().BeTrue(); } }
public void HostsReturnedBySniffAreVisited() { using (var fake = new AutoFake()) { var dateTimeProvider = fake.Resolve <IDateTimeProvider>(); var nowCall = A.CallTo(() => dateTimeProvider.Now()); nowCall.Returns(DateTime.UtcNow); var connectionPool = new SniffingConnectionPool(new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201") }, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool) .SniffOnConnectionFault(); fake.Provide <IConnectionConfigurationValues>(config); FakeCalls.ProvideDefaultTransport(fake); FakeCalls.PingAtConnectionLevel(fake) .Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake, config, new List <Uri>() { new Uri("http://localhost:9204"), new Uri("http://localhost:9203"), new Uri("http://localhost:9202"), new Uri("http://localhost:9201") }); var connection = fake.Resolve <IConnection>(); var seenNodes = new List <Uri>(); //var getCall = FakeResponse.GetSyncCall(fake); var getCall = A.CallTo(() => connection.GetSync( A <Uri> .That.Not.Matches(u => u.AbsolutePath.StartsWith("/_nodes")), A <IRequestConfiguration> ._)); getCall.ReturnsNextFromSequence( FakeResponse.Ok(config), //info 1 FakeResponse.Bad(config), //info 2 FakeResponse.Ok(config), //info 2 retry FakeResponse.Ok(config), //info 3 FakeResponse.Ok(config), //info 4 FakeResponse.Ok(config), //info 5 FakeResponse.Ok(config), //info 6 FakeResponse.Ok(config), //info 7 FakeResponse.Ok(config), //info 8 FakeResponse.Ok(config) //info 9 ); getCall.Invokes((Uri u, IRequestConfiguration o) => seenNodes.Add(u)); var client1 = fake.Resolve <ElasticsearchClient>(); client1.Info(); //info call 1 client1.Info(); //info call 2 client1.Info(); //info call 3 client1.Info(); //info call 4 client1.Info(); //info call 5 client1.Info(); //info call 6 client1.Info(); //info call 7 client1.Info(); //info call 8 client1.Info(); //info call 9 sniffCall.MustHaveHappened(Repeated.Exactly.Once); seenNodes.Should().NotBeEmpty().And.HaveCount(10); seenNodes[0].Port.Should().Be(9200); seenNodes[1].Port.Should().Be(9201); //after sniff seenNodes[2].Port.Should().Be(9202, string.Join(",", seenNodes.Select(n => n.Port))); seenNodes[3].Port.Should().Be(9204); seenNodes[4].Port.Should().Be(9203); seenNodes[5].Port.Should().Be(9202); seenNodes[6].Port.Should().Be(9201); } }
public void IfAllButOneConnectionDiesSubsequentRequestsMustUseTheOneAliveConnection() { using (var fake = new AutoFake(callsDoNothing: true)) { //Setting up a datetime provider so that we can measure which //nodes have been marked alive and dead. //we provide a different fake for the method call with the last node //as argument. var dateTimeProvider = fake.Resolve <IDateTimeProvider>(); A.CallTo(() => dateTimeProvider.Now()).Returns(DateTime.UtcNow); var markOthersDeadCall = A.CallTo(() => dateTimeProvider .DeadTime(A <Uri> .That.Not.Matches(u => u.Port == 9203), A <int> ._, A <int?> ._, A <int?> ._)); var markLastDead = A.CallTo(() => dateTimeProvider .DeadTime(A <Uri> .That.Matches(u => u.Port == 9203), A <int> ._, A <int?> ._, A <int?> ._)); var markOthersAliveCall = A.CallTo(() => dateTimeProvider .AliveTime(A <Uri> .That.Not.Matches(u => u.Port == 9203), A <int> ._)); var markLastAlive = A.CallTo(() => dateTimeProvider .AliveTime(A <Uri> .That.Matches(u => u.Port == 9203), A <int> ._)); markOthersDeadCall.Returns(DateTime.UtcNow.AddSeconds(60)); markLastAlive.Returns(new DateTime()); fake.Provide(dateTimeProvider); //Creating the connection pool making sure nodes are not randomized //So we are sure 9203 is that last node in the pool var connectionPool = new StaticConnectionPool( _uris, randomizeOnStartup: false, dateTimeProvider: dateTimeProvider ); var config = new ConnectionConfiguration(connectionPool); fake.Provide <IConnectionConfigurationValues>(config); // provide a simple fake for synchronous get var getCall = FakeCalls.GetSyncCall(fake); //The first three tries get a 503 causing the first 3 nodes to be marked dead //all the subsequent requests should be handled by 9203 which gives a 200 4 times getCall.ReturnsNextFromSequence( FakeResponse.Bad(config), FakeResponse.Bad(config), FakeResponse.Bad(config), FakeResponse.Ok(config), FakeResponse.Ok(config), FakeResponse.Ok(config), FakeResponse.Ok(config) ); //provide a transport with all the dependencies resolved var transport = this.ProvideTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(_ok); //instantiate connection with faked dependencies var client = fake.Resolve <ElasticsearchClient>(); //We call the root for each node 4 times, eventhough the first 3 nodes //give back a timeout the default ammount of retries is 4 (each provided node) //and the last retry will hit our 9203 node. Assert.DoesNotThrow(() => client.Info()); //These calls should not hit the dead nodes and go straight to the active 9203 Assert.DoesNotThrow(() => client.Info()); Assert.DoesNotThrow(() => client.Info()); Assert.DoesNotThrow(() => client.Info()); //The last node should never be marked dead markLastDead.MustNotHaveHappened(); //the other nodes should never be marked alive markOthersAliveCall.MustNotHaveHappened(); //marking the other 3 nodes dead should only happen once for each markOthersDeadCall.MustHaveHappened(Repeated.Exactly.Times(3)); //every time a connection succeeds on a node it will be marked //alive therefor the last node should be marked alive 4 times markLastAlive.MustHaveHappened(Repeated.Exactly.Times(4)); } }
public void DeadNodeIsSkipped_AndRevivedAfterTimeout() { using (var fake = new AutoFake()) { var dateTimeProvider = ProvideDateTimeProvider(fake); var config = ProvideConfiguration(dateTimeProvider); var connection = ProvideConnection(fake, config); var getCall = FakeCalls.GetSyncCall(fake); var ok = FakeResponse.Ok(config); var bad = FakeResponse.Bad(config); getCall.ReturnsNextFromSequence( ok, //info 1 - 9204 bad, //info 2 - 9203 DEAD ok, //info 2 retry - 9202 ok, //info 3 - 9201 ok, //info 4 - 9204 ok, //info 5 - 9202 ok, //info 6 - 9201 ok, //info 7 - 9204 ok, //info 8 - 9203 (Now > Timeout) ok //info 9 - 9202 ); var seenNodes = new List <Uri>(); getCall.Invokes((Uri u, IRequestConfiguration o) => seenNodes.Add(u)); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(ok); var client1 = fake.Resolve <ElasticsearchClient>(); var result = client1.Info(); //info call 1//first time node is used so a ping is sent first result.Metrics.Requests.Count.Should().Be(2); result.Metrics.Requests.First().RequestType.Should().Be(RequestType.Ping); result.Metrics.Requests.First().Node.Port.Should().Be(9204); result.Metrics.Requests.Last().RequestType.Should().Be(RequestType.ElasticsearchCall); result.Metrics.Requests.Last().Node.Port.Should().Be(9204); result = client1.Info(); //info call 2 //using 9203 for the first time ping succeeds result.Metrics.Requests.First().RequestType.Should().Be(RequestType.Ping); result.Metrics.Requests.First().Node.Port.Should().Be(9203); result.Metrics.Requests.First().Success.Should().BeTrue(); //call on 9203 fails result.Metrics.Requests[1].RequestType.Should().Be(RequestType.ElasticsearchCall); result.Metrics.Requests[1].Node.Port.Should().Be(9203); result.Metrics.Requests[1].Success.Should().BeFalse(); result.Metrics.Requests[1].HttpStatusCode.Should().Be(503); //using 9202 for the first time ping succeeds result.Metrics.Requests[2].RequestType.Should().Be(RequestType.Ping); result.Metrics.Requests[2].Node.Port.Should().Be(9202); //call on 9203 fails result.Metrics.Requests[3].RequestType.Should().Be(RequestType.ElasticsearchCall); result.Metrics.Requests[3].Node.Port.Should().Be(9202); result.Metrics.Requests[3].Success.Should().BeTrue(); result.Metrics.Requests[3].HttpStatusCode.Should().Be(200); result = client1.Info(); //info call 3 result = client1.Info(); //info call 4 result = client1.Info(); //info call 5 result = client1.Info(); //info call 6 result = client1.Info(); //info call 7 result = client1.Info(); //info call 8 result = client1.Info(); //info call 9 AssertSeenNodesAreInExpectedOrder(seenNodes); //4 nodes first time usage + 1 time after the first time 9203 came back to live pingCall.MustHaveHappened(Repeated.Exactly.Times(5)); } }