public void WhenPostExceedsHttpLimit_DoNotRetry_UsingConnectionPooling() { var pool = new StaticConnectionPool(new [] { new Uri("http://localhost:9200"), new Uri("http://127.0.0.1:9200"), }); var settings = new ConnectionSettings(pool); var client = new ElasticClient(settings); var index = ElasticsearchConfiguration.NewUniqueIndexName(); var projects = NestTestData.Data; var people = NestTestData.People; var boolTerms = NestTestData.BoolTerms; var bulk = client.Bulk(b => b .FixedPath(index) .IndexMany(projects) .IndexMany(people) .IndexMany(boolTerms) ); bulk.IsValid.Should().BeFalse(); bulk.ConnectionStatus.NumberOfRetries.Should().Be(0); }
public void IfResponseIsKnowError_DoNotRetry_ThrowServerException_Async(int status, string exceptionType, string exceptionMessage) { var response = CreateServerExceptionResponse(status, exceptionType, exceptionMessage); using (var fake = new AutoFake(callsDoNothing: true)) { var connectionPool = new StaticConnectionPool(new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201"), }); var connectionConfiguration = new ConnectionConfiguration(connectionPool) .ThrowOnElasticsearchServerExceptions() .ExposeRawResponse(false); fake.Provide<IConnectionConfigurationValues>(connectionConfiguration); FakeCalls.ProvideDefaultTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevelAsync(fake); pingCall.Returns(FakeResponse.OkAsync(connectionConfiguration)); var getCall = FakeCalls.GetCall(fake); getCall.Returns(FakeResponse.AnyAsync(connectionConfiguration, status, response: response)); var client = fake.Resolve<ElasticsearchClient>(); var e = Assert.Throws<ElasticsearchServerException>(async ()=>await client.InfoAsync()); AssertServerErrorsOnResponse(e, status, exceptionType, exceptionMessage); //make sure a know ElasticsearchServerException does not cause a retry //In this case we want to fail early getCall.MustHaveHappened(Repeated.Exactly.Once); } }
public StaticConnectionPoolTests() { _connectionPool = new StaticConnectionPool(_uris); _config = new ConnectionConfiguration(_connectionPool); _ok = FakeResponse.Ok(_config); _bad = FakeResponse.Bad(_config); }
public ConcurrencyTests() { _connectionPool = new SniffingConnectionPool(_uris); _config = new ConnectionConfiguration(_connectionPool) .SniffOnConnectionFault() .SniffOnStartup() .MaximumRetries(5); }
public static ElasticClient Create(IEnumerable<string> addresses) { var uris = addresses.Select(address => new Uri(address)); var connectionPool = new StaticConnectionPool(uris); var settings = new ConnectionSettings(connectionPool); settings.MaximumRetries(3); settings.SetPingTimeout(500); return new ElasticClient(settings); }
/// <summary> /// Configures the elasticsearch sink /// </summary> /// <param name="nodes">The nodes to write to</param> public ElasticsearchSinkOptions(IEnumerable<Uri> nodes) : this() { nodes = nodes != null && nodes.Any(n => n != null) ? nodes.Where(n => n != null) : new[] { new Uri("http://localhost:9200") }; if (nodes.Count() == 1) ConnectionPool = new SingleNodeConnectionPool(nodes.First()); else ConnectionPool = new StaticConnectionPool(nodes); }
public IElasticClient GetClient(IEnumerable<Uri> serverUris) { var connectionPool = new StaticConnectionPool(serverUris); var indexes = GetIndexes(); var settings = new ConnectionSettings(connectionPool) .MapDefaultTypeIndices(t => t.AddRange(indexes.GetTypeIndices())) .MapDefaultTypeNames(t => t.AddRange(indexes.GetIndexTypeNames())) .SetDefaultTypeNameInferrer(p => p.Name.ToLowerUnderscoredWords()) .SetDefaultPropertyNameInferrer(p => p.ToLowerUnderscoredWords()) .MaximumRetries(5); settings.SetJsonSerializerSettingsModifier(s => { s.ContractResolver = new EmptyCollectionElasticContractResolver(settings); s.AddModelConverters(); }); var client = new ElasticClient(settings, new KeepAliveHttpConnection(settings)); ConfigureIndexes(client); return client; }
public void MaxRetryExceptionInnerExceptionIsNullAsync() { var nodes = new Uri[] { new Uri("http://invalid_host:9300"), new Uri("http://invalid_host:9400"), new Uri("http://invalid_host:9500") }; var connectionPool = new StaticConnectionPool(nodes); var settings = new ConnectionSettings(connectionPool) .DisablePing(); var client = new ElasticClient(settings); var maxRetryException = Assert.Throws<MaxRetryException>(async () => await client.GetIndexAsync(g => g.Index("foo"))); maxRetryException.InnerException.Should().NotBeNull(); var aggregateException = maxRetryException.InnerException as AggregateException; aggregateException.Should().NotBeNull(); aggregateException.InnerExceptions.Count.Should().Be(nodes.Count()); }
public void ShouldFailoverOnThriftConnections() { var uris = new [] { new Uri("http://INVALID_HOST"), new Uri("http://INVALID_HOST2"), new Uri("http://localhost:9500") }; var connectionPool = new StaticConnectionPool(uris, randomizeOnStartup: false); var settings = new ConnectionSettings(connectionPool, ElasticsearchConfiguration.DefaultIndex) .ExposeRawResponse() .SetTimeout(2000); var client = new ElasticClient(settings, new ThriftConnection(settings)); var results = client.Search<dynamic>(s => s.MatchAll()); results.IsValid.Should().BeTrue("{0}", results.ConnectionStatus.ToString()); results.ConnectionStatus.NumberOfRetries.Should().Be(2); results = client.Search<dynamic>(s => s.MatchAll()); results.IsValid.Should().BeTrue("{0}", results.ConnectionStatus.ToString()); results.ConnectionStatus.NumberOfRetries.Should().Be(0); }
public StaticConnectionPoolRetryTests() { _connectionPool = new StaticConnectionPool(_uris); _config = new ConnectionConfiguration(_connectionPool); }
public void IfAllButOneConnectionDiesSubsequentRequestsMustUseTheOneAliveConnection() { using (var fake = new AutoFake(callsDoNothing: true)) { //Setting up a datetime provider so that we can measure which //nodes have been marked alive and dead. //we provide a different fake for the method call with the last node //as argument. var dateTimeProvider = fake.Resolve<IDateTimeProvider>(); A.CallTo(() => dateTimeProvider.Now()).Returns(DateTime.UtcNow); var markOthersDeadCall = A.CallTo(() => dateTimeProvider .DeadTime(A<Uri>.That.Not.Matches(u=>u.Port == 9203), A<int>._, A<int?>._, A<int?>._)); var markLastDead = A.CallTo(() => dateTimeProvider .DeadTime(A<Uri>.That.Matches(u=>u.Port == 9203), A<int>._, A<int?>._, A<int?>._)); var markOthersAliveCall = A.CallTo(() => dateTimeProvider .AliveTime(A<Uri>.That.Not.Matches(u=>u.Port == 9203), A<int>._)); var markLastAlive = A.CallTo(() => dateTimeProvider .AliveTime(A<Uri>.That.Matches(u=>u.Port == 9203), A<int>._)); markOthersDeadCall.Returns(DateTime.UtcNow.AddSeconds(60)); markLastAlive.Returns(new DateTime()); fake.Provide(dateTimeProvider); //Creating the connection pool making sure nodes are not randomized //So we are sure 9203 is that last node in the pool var connectionPool = new StaticConnectionPool( _uris, randomizeOnStartup: false, dateTimeProvider: dateTimeProvider ); var config = new ConnectionConfiguration(connectionPool); fake.Provide<IConnectionConfigurationValues>(config); // provide a simple fake for synchronous get var getCall = FakeCalls.GetSyncCall(fake); //The first three tries get a 503 causing the first 3 nodes to be marked dead //all the subsequent requests should be handled by 9203 which gives a 200 4 times getCall.ReturnsNextFromSequence( FakeResponse.Bad(config), FakeResponse.Bad(config), FakeResponse.Bad(config), FakeResponse.Ok(config), FakeResponse.Ok(config), FakeResponse.Ok(config), FakeResponse.Ok(config) ); var pingCall = A.CallTo(() => fake.Resolve<IConnection>().Ping(A<Uri>._)); pingCall.Returns(true); //provide a transport with all the dependencies resolved this.ProvideTransport(fake); //instantiate connection with faked dependencies var client = fake.Resolve<ElasticsearchClient>(); //We call the root for each node 4 times, eventhough the first 3 nodes //give back a timeout the default ammount of retries is 4 (each provided node) //and the last retry will hit our 9203 node. Assert.DoesNotThrow(()=> client.Info()); //These calls should not hit the dead nodes and go straight to the active 9203 Assert.DoesNotThrow(()=> client.Info()); Assert.DoesNotThrow(()=> client.Info()); Assert.DoesNotThrow(()=> client.Info()); //The last node should never be marked dead markLastDead.MustNotHaveHappened(); //the other nodes should never be marked alive markOthersAliveCall.MustNotHaveHappened(); //marking the other 3 nodes dead should only happen once for each markOthersDeadCall.MustHaveHappened(Repeated.Exactly.Times(3)); //every time a connection succeeds on a node it will be marked //alive therefor the last node should be marked alive 4 times markLastAlive.MustHaveHappened(Repeated.Exactly.Times(4)); } }
public void IfAConnectionComesBackToLifeOnItsOwnItShouldBeMarked() { using (var fake = new AutoFake(callsDoNothing: true)) { //Setting up a datetime provider so that can track dead/alive marks var dateTimeProvider = fake.Resolve<IDateTimeProvider>(); A.CallTo(() => dateTimeProvider.Now()).Returns(DateTime.UtcNow); var markDeadCall = A.CallTo(() => dateTimeProvider.DeadTime(A<Uri>._, A<int>._, A<int?>._, A<int?>._)); var markAliveCall = A.CallTo(() => dateTimeProvider.AliveTime(A<Uri>._, A<int>._)); markDeadCall.Returns(DateTime.UtcNow.AddSeconds(60)); markAliveCall.Returns(new DateTime()); fake.Provide(dateTimeProvider); var connectionPool = new StaticConnectionPool( _uris, dateTimeProvider: dateTimeProvider); //set retries to 4 fake.Provide<IConnectionConfigurationValues>( new ConnectionConfiguration(connectionPool) .MaximumRetries(4) ); //fake getsync handler that return a 503 4 times and then a 200 //this will cause all 4 nodes to be marked dead on the first client call var getCall = A.CallTo(() => fake.Resolve<IConnection>().GetSync<Dictionary<string, object>>(A<Uri>._, A<object>._)); getCall.ReturnsNextFromSequence( ElasticsearchResponse<Dictionary<string, object>>.Create(_config, 503, "GET", "/", null), ElasticsearchResponse<Dictionary<string, object>>.Create(_config, 503, "GET", "/", null), ElasticsearchResponse<Dictionary<string, object>>.Create(_config, 503, "GET", "/", null), ElasticsearchResponse<Dictionary<string, object>>.Create(_config, 503, "GET", "/", null), ElasticsearchResponse<Dictionary<string, object>>.Create(_config, 200, "GET", "/", null) ); var pingCall = A.CallTo(() => fake.Resolve<IConnection>().Ping(A<Uri>._)); pingCall.Returns(true); //provide a transport with all the dependencies resolved this.ProvideTransport(fake); //instantiate connection with faked dependencies var client = fake.Resolve<ElasticsearchClient>(); //Do not throw because by miracle the 4th retry manages to give back a 200 //even if all nodes have been marked dead. Assert.DoesNotThrow(()=> client.Info()); //original call + 4 retries is 5 getCall.MustHaveHappened(Repeated.Exactly.Times(5)); //4 nodes must be marked dead markDeadCall.MustHaveHappened(Repeated.Exactly.Times(4)); //atleast one of them sprung back to live so markAlive must be called once markAliveCall.MustHaveHappened(Repeated.Exactly.Times(1)); } }
public void AllNodesWillBeMarkedDead() { using (var fake = new AutoFake(callsDoNothing: true)) { //set up a fake datetimeprovider var dateTimeProvider = fake.Resolve<IDateTimeProvider>(); fake.Provide(dateTimeProvider); //create a connectionpool that uses the fake datetimeprovider var connectionPool = new StaticConnectionPool( _uris, dateTimeProvider: dateTimeProvider ); var config = new ConnectionConfiguration(connectionPool); fake.Provide<IConnectionConfigurationValues>(config); //Now() on the fake still means Now() A.CallTo(()=>dateTimeProvider.Now()).Returns(DateTime.UtcNow); //Set up individual mocks for each DeadTime(Uri uri,...) call //where uri matches one of the node ports var calls = _uris.Select(u => A.CallTo(()=> dateTimeProvider.DeadTime( A<Uri>.That.Matches(uu=>uu.Port == u.Port), A<int>._, A<int?>._, A<int?>._ ))).ToList(); //all the fake mark dead calls return 60 seconds into the future foreach (var call in calls) call.Returns(DateTime.UtcNow.AddSeconds(60)); //When we do a GET on / we always recieve a 503 var getCall = A.CallTo(() => fake.Resolve<IConnection>().GetSync<Dictionary<string, object>>(A<Uri>._, A<object>._)); getCall.Returns( ElasticsearchResponse<Dictionary<string, object>>.Create(_config, 503, "GET", "/", null) ); var pingCall = A.CallTo(() => fake.Resolve<IConnection>().Ping(A<Uri>._)); pingCall.Returns(true); this.ProvideTransport(fake); var client = fake.Resolve<ElasticsearchClient>(); //Since we always get a 503 we should see an out of nodes exception Assert.Throws<OutOfNodesException>(()=> client.Info()); //The call should be tried on all the nodes getCall.MustHaveHappened(Repeated.Exactly.Times(4)); //We should see each individual node being marked as dead foreach (var call in calls) call.MustHaveHappened(Repeated.Exactly.Once); } }
public void DeadNodesAreNotVisited() { using (var fake = new AutoFake()) { var now = DateTime.UtcNow; var dateTimeProvider = fake.Resolve<IDateTimeProvider>(); var nowCall = A.CallTo(()=>dateTimeProvider.Now()); nowCall.ReturnsNextFromSequence( DateTime.UtcNow, //info 1 DateTime.UtcNow, //info 2 DateTime.UtcNow, //info 2 retry DateTime.UtcNow, //info 3 DateTime.UtcNow, //info 4 DateTime.UtcNow, //info 5 pass over node 3 DateTime.UtcNow, //info 5 DateTime.UtcNow, //info 6 DateTime.UtcNow.AddMinutes(2), //info 7 DateTime.UtcNow.AddMinutes(2), //info 8 DateTime.UtcNow.AddMinutes(2) //info 9 ); A.CallTo(()=>dateTimeProvider.AliveTime(A<Uri>._, A<int>._)) .Returns(new DateTime()); A.CallTo(() => dateTimeProvider.DeadTime(A<Uri>._, A<int>._, A<int?>._, A<int?>._)) .Returns(DateTime.UtcNow.AddMinutes(1)); //make sure the transport layer uses a different datetimeprovider fake.Provide<IDateTimeProvider>(new DateTimeProvider()); var connectionPool = new StaticConnectionPool(new[] { new Uri("http://localhost:9204"), new Uri("http://localhost:9203"), new Uri("http://localhost:9202"), new Uri("http://localhost:9201") }, randomizeOnStartup: false, dateTimeProvider: dateTimeProvider); var config = new ConnectionConfiguration(connectionPool); fake.Provide<IConnectionConfigurationValues>(config); fake.Provide<ITransport>(fake.Resolve<Transport>()); var connection = fake.Resolve<IConnection>(); var seenNodes = new List<Uri>(); var getCall = A.CallTo(() => connection.GetSync(A<Uri>._)); getCall.ReturnsNextFromSequence( ElasticsearchResponse.Create(config, 200, "GET", "/", null, null), //info 1 - 9204 ElasticsearchResponse.Create(config, 503, "GET", "/", null, null), //info 2 - 9203 DEAD ElasticsearchResponse.Create(config, 200, "GET", "/", null, null), //info 2 retry - 9202 ElasticsearchResponse.Create(config, 200, "GET", "/", null, null), //info 3 - 9201 ElasticsearchResponse.Create(config, 200, "GET", "/", null, null), //info 4 - 9204 ElasticsearchResponse.Create(config, 200, "GET", "/", null, null), //info 5 - 9202 ElasticsearchResponse.Create(config, 200, "GET", "/", null, null), //info 6 - 9201 ElasticsearchResponse.Create(config, 200, "GET", "/", null, null), //info 7 - 9204 ElasticsearchResponse.Create(config, 200, "GET", "/", null, null), //info 8 - 9203 (Now > Timeout) ElasticsearchResponse.Create(config, 200, "GET", "/", null, null) //info 9 - 9202 ); getCall.Invokes((Uri u) => seenNodes.Add(u)); var pingCall = A.CallTo(() => fake.Resolve<IConnection>().Ping(A<Uri>._)); pingCall.Returns(true); var client1 = fake.Resolve<ElasticsearchClient>(); client1.Info(); //info call 1 client1.Info(); //info call 2 client1.Info(); //info call 3 client1.Info(); //info call 4 client1.Info(); //info call 5 client1.Info(); //info call 6 client1.Info(); //info call 7 client1.Info(); //info call 8 client1.Info(); //info call 9 seenNodes.Should().NotBeEmpty().And.HaveCount(10); seenNodes[0].Port.Should().Be(9204); seenNodes[1].Port.Should().Be(9203); //after sniff seenNodes[2].Port.Should().Be(9202); seenNodes[3].Port.Should().Be(9201); seenNodes[4].Port.Should().Be(9204); seenNodes[5].Port.Should().Be(9202); seenNodes[6].Port.Should().Be(9201); seenNodes[7].Port.Should().Be(9204); seenNodes[8].Port.Should().Be(9203); //4 nodes first time usage + 1 time after the first time 9203 came back to live pingCall.MustHaveHappened(Repeated.Exactly.Times(5)); //var nowCall = A.CallTo(() => fake.Resolve<IDateTimeProvider>().Sniff(A<Uri>._, A<int>._)); } }
public async void DeadNodesAreNotVisited_Async() { using (var fake = new AutoFake()) { var now = DateTime.UtcNow; var dateTimeProvider = fake.Resolve<IDateTimeProvider>(); var nowCall = A.CallTo(()=>dateTimeProvider.Now()); nowCall.ReturnsNextFromSequence( DateTime.UtcNow, //info 1 DateTime.UtcNow, //info 2 DateTime.UtcNow, //info 2 retry DateTime.UtcNow, //info 3 DateTime.UtcNow, //info 4 DateTime.UtcNow, //info 5 pass over node 3 DateTime.UtcNow, //info 5 DateTime.UtcNow, //info 6 DateTime.UtcNow.AddMinutes(2), //info 7 DateTime.UtcNow.AddMinutes(2), //info 8 DateTime.UtcNow.AddMinutes(2) //info 9 ); A.CallTo(()=>dateTimeProvider.AliveTime(A<Uri>._, A<int>._)) .Returns(new DateTime()); A.CallTo(() => dateTimeProvider.DeadTime(A<Uri>._, A<int>._, A<int?>._, A<int?>._)) .Returns(DateTime.UtcNow.AddMinutes(1)); //make sure the transport layer uses a different datetimeprovider fake.Provide<IDateTimeProvider>(new DateTimeProvider()); var connectionPool = new StaticConnectionPool(new[] { new Uri("http://localhost:9204"), new Uri("http://localhost:9203"), new Uri("http://localhost:9202"), new Uri("http://localhost:9201") }, randomizeOnStartup: false, dateTimeProvider: dateTimeProvider); var config = new ConnectionConfiguration(connectionPool); fake.Provide<IConnectionConfigurationValues>(config); fake.Provide<ITransport>(fake.Resolve<Transport>()); var connection = fake.Resolve<IConnection>(); var ok = Task.FromResult(ElasticsearchResponse.Create(config, 200, "GET", "/", null, null)); var bad = Task.FromResult(ElasticsearchResponse.Create(config, 503, "GET", "/", null, null)); var seenNodes = new List<Uri>(); var getCall = A.CallTo(() => connection.Get(A<Uri>._)); getCall.ReturnsNextFromSequence( ok, //info 1 - 9204 bad, //info 2 - 9203 DEAD ok, //info 2 retry - 9202 ok, //info 3 - 9201 ok, //info 4 - 9204 ok, //info 5 - 9202 ok, //info 6 - 9201 ok, //info 7 - 9204 ok, //info 8 - 9203 (Now > Timeout) ok //info 9 - 9202 ); getCall.Invokes((Uri u) => seenNodes.Add(u)); var client1 = fake.Resolve<ElasticsearchClient>(); await client1.InfoAsync(); //info call 1 await client1.InfoAsync(); //info call 2 await client1.InfoAsync(); //info call 3 await client1.InfoAsync(); //info call 4 await client1.InfoAsync(); //info call 5 await client1.InfoAsync(); //info call 6 await client1.InfoAsync(); //info call 7 await client1.InfoAsync(); //info call 8 await client1.InfoAsync(); //info call 9 seenNodes.Should().NotBeEmpty().And.HaveCount(10); seenNodes[0].Port.Should().Be(9204); seenNodes[1].Port.Should().Be(9203); //after sniff seenNodes[2].Port.Should().Be(9202); seenNodes[3].Port.Should().Be(9201); seenNodes[4].Port.Should().Be(9204); seenNodes[5].Port.Should().Be(9202); seenNodes[6].Port.Should().Be(9201); seenNodes[7].Port.Should().Be(9204); seenNodes[8].Port.Should().Be(9203); //var nowCall = A.CallTo(() => fake.Resolve<IDateTimeProvider>().Sniff(A<Uri>._, A<int>._)); } }
private static ConnectionConfiguration ProvideConfiguration(IDateTimeProvider dateTimeProvider) { var connectionPool = new StaticConnectionPool(new[] { new Uri("http://localhost:9204"), new Uri("http://localhost:9203"), new Uri("http://localhost:9202"), new Uri("http://localhost:9201") }, randomizeOnStartup: false, dateTimeProvider: dateTimeProvider); var config = new ConnectionConfiguration(connectionPool).EnableMetrics(); return config; }
public void ShouldNotThrowAndNotRetry401_Async() { using (var fake = new AutoFake(callsDoNothing: true)) { var uris = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201"), new Uri("http://localhost:9202") }; var connectionPool = new StaticConnectionPool(uris, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool); fake.Provide<IConnectionConfigurationValues>(config); FakeCalls.ProvideDefaultTransport(fake); var pingAsyncCall = FakeCalls.PingAtConnectionLevelAsync(fake); pingAsyncCall.Returns(FakeResponse.OkAsync(config)); //sniffing is always synchronous and in turn will issue synchronous pings var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var getCall = FakeCalls.GetCall(fake); getCall.Returns(FakeResponse.Any(config, 401)); var client = fake.Resolve<ElasticsearchClient>(); Assert.DoesNotThrow(async () => await client.InfoAsync()); getCall.MustHaveHappened(Repeated.Exactly.Once); } }
public void ShouldRetryOnPingConnectionException() { using (var fake = new AutoFake(callsDoNothing: true)) { var connectionPool = new StaticConnectionPool(_uris, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool); fake.Provide<IConnectionConfigurationValues>(config); FakeCalls.ProvideDefaultTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); var seenPorts = new List<int>(); pingCall.ReturnsLazily((Uri u, IRequestConfiguration c) => { seenPorts.Add(u.Port); throw new Exception("Something bad happened"); }); var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.Ok(config)); var client = fake.Resolve<ElasticsearchClient>(); var e = Assert.Throws<MaxRetryException>(() => client.Info()); pingCall.MustHaveHappened(Repeated.Exactly.Times(_retries + 1)); getCall.MustNotHaveHappened(); //make sure that if a ping throws an exception it wont //keep retrying to ping the same node but failover to the next seenPorts.ShouldAllBeEquivalentTo(_uris.Select(u=>u.Port)); var aggregateException = e.InnerException as AggregateException; aggregateException.Should().NotBeNull(); aggregateException.InnerExceptions.Should().Contain(ex => ex.GetType().Name == "PingException"); } }