public void ShouldNotThrowAndNotRetry401_Async()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
var uris = new[]
{
new Uri("http://localhost:9200"),
new Uri("http://localhost:9201"),
new Uri("http://localhost:9202")
};
var connectionPool = new StaticConnectionPool(uris, randomizeOnStartup: false);
var config = new ConnectionConfiguration(connectionPool);
fake.Provide <IConnectionConfigurationValues>(config);
FakeCalls.ProvideDefaultTransport(fake);
var pingAsyncCall = FakeCalls.PingAtConnectionLevelAsync(fake);
pingAsyncCall.Returns(FakeResponse.OkAsync(config));
//sniffing is always synchronous and in turn will issue synchronous pings
var pingCall = FakeCalls.PingAtConnectionLevel(fake);
pingCall.Returns(FakeResponse.Ok(config));
var getCall = FakeCalls.GetCall(fake);
getCall.Returns(FakeResponse.Any(config, 401));
var client = fake.Resolve <ElasticsearchClient>();
Assert.DoesNotThrow(async() => await client.InfoAsync());
getCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public void IfResponseIsKnowError_DoNotRetry_ThrowServerException_Async(int status, string exceptionType, string exceptionMessage)
{
var response = CreateServerExceptionResponse(status, exceptionType, exceptionMessage);
using (var fake = new AutoFake(callsDoNothing: true))
{
var connectionPool = new StaticConnectionPool(new[]
{
new Uri("http://localhost:9200"),
new Uri("http://localhost:9201"),
});
var connectionConfiguration = new ConnectionConfiguration(connectionPool)
.ThrowOnElasticsearchServerExceptions()
.ExposeRawResponse(false);
fake.Provide <IConnectionConfigurationValues>(connectionConfiguration);
FakeCalls.ProvideDefaultTransport(fake);
var pingCall = FakeCalls.PingAtConnectionLevelAsync(fake);
pingCall.Returns(FakeResponse.OkAsync(connectionConfiguration));
var getCall = FakeCalls.GetCall(fake);
getCall.Returns(FakeResponse.AnyAsync(connectionConfiguration, status, response: response));
var client = fake.Resolve <ElasticsearchClient>();
var e = Assert.Throws <ElasticsearchServerException>(async() => await client.InfoAsync());
AssertServerErrorsOnResponse(e, status, exceptionType, exceptionMessage);
//make sure a know ElasticsearchServerException does not cause a retry
//In this case we want to fail early
getCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public async Task <AsyncMemorySetup <T> > Init()
{
this.Fake = new AutoFake(callsDoNothing: true);
var connectionConfiguration = _configSetup(new ConnectionConfiguration());
this.ResponseStream = CreateServerExceptionResponse(_responseValue);
var response = _responseSetup(connectionConfiguration, this.ResponseStream);
this.Fake.Provide <IConnectionConfigurationValues>(connectionConfiguration);
this.MemoryProvider = this.Fake.Resolve <IMemoryStreamProvider>();
A.CallTo(() => this.MemoryProvider.New()).ReturnsLazily((o) =>
{
var memoryStream = new TrackableMemoryStream();
this._createdMemoryStreams.Add(memoryStream);
return(memoryStream);
});
FakeCalls.ProvideDefaultTransport(this.Fake, memoryStreamProvider: this.MemoryProvider);
this.GetCall = FakeCalls.GetCall(this.Fake);
this.GetCall.Returns(response);
var client = this.Fake.Resolve <ElasticsearchClient>();
this.Result = await(_call != null ? _call(client) : client.InfoAsync <T>());
this.GetCall.MustHaveHappened(Repeated.Exactly.Once);
return(this);
}
public void ShouldRetryOnPingConnectionException_Async()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
var connectionPool = new StaticConnectionPool(_uris, randomizeOnStartup: false);
var config = new ConnectionConfiguration(connectionPool);
fake.Provide <IConnectionConfigurationValues>(config);
FakeCalls.ProvideDefaultTransport(fake);
var pingCall = FakeCalls.PingAtConnectionLevelAsync(fake);
var seenPorts = new List <int>();
pingCall.ReturnsLazily((Uri u, IRequestConfiguration c) =>
{
seenPorts.Add(u.Port);
throw new Exception("Something bad happened");
});
var getCall = FakeCalls.GetCall(fake);
getCall.Returns(FakeResponse.OkAsync(config));
var client = fake.Resolve <ElasticsearchClient>();
var e = Assert.Throws <MaxRetryException>(async() => await client.InfoAsync());
pingCall.MustHaveHappened(Repeated.Exactly.Times(_retries + 1));
getCall.MustNotHaveHappened();
//make sure that if a ping throws an exception it wont
//keep retrying to ping the same node but failover to the next
seenPorts.ShouldAllBeEquivalentTo(_uris.Select(u => u.Port));
var ae = e.InnerException as AggregateException;
ae = ae.Flatten();
ae.InnerException.Message.Should().Contain("Pinging");
}
}
public void FailEarlyIfTimeoutIsExhausted_Async()
{
using (var fake = new AutoFake())
{
var dateTimeProvider = ProvideDateTimeProvider(fake);
var config = ProvideConfiguration(dateTimeProvider);
var connection = ProvideConnection(fake, config, dateTimeProvider);
var getCall = FakeCalls.GetCall(fake);
var ok = Task.FromResult(FakeResponse.Ok(config));
var bad = Task.FromResult(FakeResponse.Bad(config));
getCall.ReturnsNextFromSequence(
bad,
bad,
ok
);
var seenNodes = new List <Uri>();
getCall.Invokes((Uri u, IRequestConfiguration o) => seenNodes.Add(u));
var pingCall = FakeCalls.PingAtConnectionLevelAsync(fake);
pingCall.Returns(ok);
var client1 = fake.Resolve <ElasticsearchClient>();
//event though the third node should have returned ok, the first 2 calls took a minute
var e = Assert.Throws <MaxRetryException>(async() => await client1.InfoAsync());
e.Message.Should()
.StartWith("Retry timeout 00:01:20 was hit after retrying 1 times:");
IElasticsearchResponse response = null;
Assert.DoesNotThrow(async() => response = await client1.InfoAsync());
response.Should().NotBeNull();
response.Success.Should().BeTrue();
}
}
public async void ThrowsOutOfNodesException_AndRetriesTheSpecifiedTimes_Async()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
fake.Provide <IConnectionConfigurationValues>(_connectionConfig);
FakeCalls.ProvideDefaultTransport(fake);
var getCall = FakeCalls.GetCall(fake);
//return a started task that throws
Func <ElasticsearchResponse <Stream> > badTask = () => { throw new Exception(); };
var t = new Task <ElasticsearchResponse <Stream> >(badTask);
t.Start();
getCall.Returns(t);
var client = fake.Resolve <ElasticsearchClient>();
client.Settings.MaxRetries.Should().Be(_retries);
try
{
var result = await client.InfoAsync();
}
catch (MaxRetryException e)
{
Assert.AreEqual(typeof(MaxRetryException), e.GetType());
}
getCall.MustHaveHappened(Repeated.Exactly.Times(_retries + 1));
}
}
public async void DeadNodesAreNotVisited_AndPingedAppropiately_Async()
{
using (var fake = new AutoFake())
{
var dateTimeProvider = ProvideDateTimeProvider(fake);
var config = ProvideConfiguration(dateTimeProvider);
var connection = ProvideConnection(fake, config);
var getCall = FakeCalls.GetCall(fake);
var ok = Task.FromResult(FakeResponse.Ok(config));
var bad = Task.FromResult(FakeResponse.Bad(config));
getCall.ReturnsNextFromSequence(
ok, //info 1 - 9204
bad, //info 2 - 9203 DEAD
ok, //info 2 retry - 9202
ok, //info 3 - 9201
ok, //info 4 - 9204
ok, //info 5 - 9202
ok, //info 6 - 9201
ok, //info 7 - 9204
ok, //info 8 - 9203 (Now > Timeout)
ok //info 9 - 9202
);
var seenNodes = new List <Uri>();
getCall.Invokes((Uri u, IRequestConnectionConfiguration o) => seenNodes.Add(u));
var pingCall = FakeCalls.PingAtConnectionLevelAsync(fake);
pingCall.Returns(ok);
var client1 = fake.Resolve <ElasticsearchClient>();
await client1.InfoAsync(); //info call 1
await client1.InfoAsync(); //info call 2
await client1.InfoAsync(); //info call 3
await client1.InfoAsync(); //info call 4
await client1.InfoAsync(); //info call 5
await client1.InfoAsync(); //info call 6
await client1.InfoAsync(); //info call 7
await client1.InfoAsync(); //info call 8
await client1.InfoAsync(); //info call 9
AssertSeenNodesAreInExpectedOrder(seenNodes);
//4 nodes first time usage + 1 time after the first time 9203 came back to live
pingCall.MustHaveHappened(Repeated.Exactly.Times(5));
}
}
public void ShouldRetryOnSniffConnectionException_Async()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
var uris = new[]
{
new Uri("http://localhost:9200"),
new Uri("http://localhost:9201"),
new Uri("http://localhost:9202")
};
var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false);
var config = new ConnectionConfiguration(connectionPool)
.SniffOnConnectionFault();
fake.Provide <IConnectionConfigurationValues>(config);
var pingAsyncCall = FakeCalls.PingAtConnectionLevelAsync(fake);
pingAsyncCall.Returns(FakeResponse.OkAsync(config));
//sniffing is always synchronous and in turn will issue synchronous pings
var pingCall = FakeCalls.PingAtConnectionLevel(fake);
pingCall.Returns(FakeResponse.Ok(config));
var sniffCall = FakeCalls.Sniff(fake);
var seenPorts = new List <int>();
sniffCall.ReturnsLazily((Uri u, IRequestConfiguration c) =>
{
seenPorts.Add(u.Port);
throw new Exception("Something bad happened");
});
var getCall = FakeCalls.GetCall(fake);
getCall.Returns(FakeResponse.BadAsync(config));
FakeCalls.ProvideDefaultTransport(fake);
var client = fake.Resolve <ElasticsearchClient>();
var e = Assert.Throws <MaxRetryException>(async() => await client.NodesHotThreadsAsync("nodex"));
//all nodes must be tried to sniff for more information
sniffCall.MustHaveHappened(Repeated.Exactly.Times(uris.Count()));
//make sure we only saw one call to hot threads (the one that failed initially)
getCall.MustHaveHappened(Repeated.Exactly.Once);
//make sure the sniffs actually happened on all the individual nodes
seenPorts.ShouldAllBeEquivalentTo(uris.Select(u => u.Port));
e.InnerException.Message.Should().Contain("Sniffing known nodes");
}
}
public void ShouldRetryOn503_Async()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
fake.Provide <IConnectionConfigurationValues>(_connectionConfig);
FakeCalls.ProvideDefaultTransport(fake);
var getCall = FakeCalls.GetCall(fake);
getCall.Returns(Task.FromResult(FakeResponse.Bad(_connectionConfig)));
var client = fake.Resolve <ElasticsearchClient>();
Assert.Throws <MaxRetryException>(async() => await client.InfoAsync());
getCall.MustHaveHappened(Repeated.Exactly.Times(_retries + 1));
}
}
public void ShouldNotRetryWhenMaxRetriesIs0_Async()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
var connectionConfiguration = new ConnectionConfiguration().MaximumRetries(0);
fake.Provide <IConnectionConfigurationValues>(connectionConfiguration);
FakeCalls.ProvideDefaultTransport(fake);
var getCall = FakeCalls.GetCall(fake);
getCall.Returns(FakeResponse.Bad(connectionConfiguration));
var client = fake.Resolve <ElasticsearchClient>();
Assert.DoesNotThrow(async() => await client.InfoAsync());
getCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public async void ShouldNotRetryOn400_Async()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
fake.Provide <IConnectionConfigurationValues>(_connectionConfig);
FakeCalls.ProvideDefaultTransport(fake);
var getCall = FakeCalls.GetCall(fake);
var task = Task.FromResult(FakeResponse.Any(_connectionConfig, 400));
getCall.Returns(task);
var client = fake.Resolve <ElasticsearchClient>();
var result = await client.InfoAsync();
getCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public void Async_CallThrowsHardException_ShouldBubbleToCallee()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
fake.Provide <IConnectionConfigurationValues>(_connectionConfig);
FakeCalls.ProvideDefaultTransport(fake);
var getCall = FakeCalls.GetCall(fake);
//return a started task that throws
getCall.Throws((c) => new Exception("hard exception!"));
var client = fake.Resolve <ElasticsearchClient>();
var e = Assert.Throws <Exception>(async() => await client.InfoAsync());
e.Message.Should().Be("hard exception!");
getCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public void ThrowsMaxRetryException_AndRetriesTheSpecifiedTimes_HardIConnectionException_Async()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
fake.Provide <IConnectionConfigurationValues>(_connectionConfig);
FakeCalls.ProvideDefaultTransport(fake);
var getCall = FakeCalls.GetCall(fake);
//return a started task that throws
getCall.Throws <Exception>();
var client = fake.Resolve <ElasticsearchClient>();
client.Settings.MaxRetries.Should().Be(_retries);
Assert.Throws <MaxRetryException>(async() => await client.InfoAsync());
getCall.MustHaveHappened(Repeated.Exactly.Times(_retries + 1));
}
}
public void ShouldThrowAndNotRetrySniffOnConnectionFault401_Async()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
var uris = new[]
{
new Uri("http://localhost:9200"),
new Uri("http://localhost:9201"),
new Uri("http://localhost:9202")
};
var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false);
var config = new ConnectionConfiguration(connectionPool)
.SniffOnConnectionFault()
.ThrowOnElasticsearchServerExceptions();
fake.Provide <IConnectionConfigurationValues>(config);
FakeCalls.ProvideDefaultTransport(fake);
var pingAsyncCall = FakeCalls.PingAtConnectionLevelAsync(fake);
pingAsyncCall.Returns(FakeResponse.OkAsync(config));
var pingCall = FakeCalls.PingAtConnectionLevel(fake);
pingCall.Returns(FakeResponse.Ok(config));
var sniffCall = FakeCalls.Sniff(fake);
var seenPorts = new List <int>();
sniffCall.ReturnsLazily((Uri u, IRequestConfiguration c) =>
{
seenPorts.Add(u.Port);
return(FakeResponse.Any(config, 401));
});
var getCall = FakeCalls.GetCall(fake);
getCall.Returns(FakeResponse.BadAsync(config));
var client = fake.Resolve <ElasticsearchClient>();
var e = Assert.Throws <ElasticsearchServerException>(async() => await client.InfoAsync());
e.Status.Should().Be(401);
sniffCall.MustHaveHappened(Repeated.Exactly.Once);
getCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public void ThrowsException_Async()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
fake.Provide <IConnectionConfigurationValues>(_connectionConfig);
FakeCalls.ProvideDefaultTransport(fake);
var getCall = FakeCalls.GetCall(fake);
//return a started task that throws
Func <ElasticsearchResponse <Stream> > badTask = () => { throw new Exception(); };
var t = new Task <ElasticsearchResponse <Stream> >(badTask);
t.Start();
getCall.Returns(t);
var client = fake.Resolve <ElasticsearchClient>();
Assert.Throws <Exception>(async() => await client.InfoAsync());
getCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public async Task Init()
{
var responseStream = CreateServerExceptionResponse(_responseValue);
this.Fake = new AutoFake(callsDoNothing: true);
var connectionConfiguration = _configSetup(new ConnectionConfiguration());
var response = _responseSetup(connectionConfiguration, responseStream);
this.Fake.Provide <IConnectionConfigurationValues>(connectionConfiguration);
FakeCalls.ProvideDefaultTransport(this.Fake);
this.GetCall = FakeCalls.GetCall(this.Fake);
this.GetCall.Returns(response);
var client = this.Fake.Resolve <ElasticsearchClient>();
this.Result = await(_call != null ? _call(client) : client.InfoAsync <T>());
this.GetCall.MustHaveHappened(Repeated.Exactly.Once);
}
public void Hard_IConnectionException_AsyncCall_WithoutPooling_DoesNot_Retry_AndRethrows()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
fake.Provide <IConnectionConfigurationValues>(_connectionConfig);
FakeCalls.ProvideDefaultTransport(fake);
var getCall = FakeCalls.GetCall(fake);
//return a started task that throws
getCall.Throws((o) => new Exception("inner"));
var client = fake.Resolve <ElasticsearchClient>();
client.Settings.MaxRetries.Should().Be(_retries);
var e = Assert.Throws <Exception>(async() => await client.InfoAsync());
e.Message.Should().Be("inner");
getCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
private void CallAsync(int status, string exceptionType, string exceptionMessage, AutoFake fake, MemoryStream response, bool exposeRawResponse = false)
{
var connectionConfiguration = new ConnectionConfiguration()
.ThrowOnElasticsearchServerExceptions()
.ExposeRawResponse(exposeRawResponse);
fake.Provide <IConnectionConfigurationValues>(connectionConfiguration);
FakeCalls.ProvideDefaultTransport(fake);
var getCall = FakeCalls.GetCall(fake);
getCall.Returns(FakeResponse.BadAsync(connectionConfiguration, response: response));
var client = fake.Resolve <ElasticsearchClient>();
var e = Assert.Throws <ElasticsearchServerException>(async() => await client.InfoAsync());
AssertServerErrorsException(e, status, exceptionType, exceptionMessage);
getCall.MustHaveHappened(Repeated.Exactly.Once);
}
private async Task <ElasticsearchResponse <DynamicDictionary> > CallAsync(int status, string exceptionType, string exceptionMessage, AutoFake fake, MemoryStream response, bool exposeRawResponse = false)
{
var connectionConfiguration = new ConnectionConfiguration()
.ExposeRawResponse(exposeRawResponse);
fake.Provide <IConnectionConfigurationValues>(connectionConfiguration);
FakeCalls.ProvideDefaultTransport(fake);
var getCall = FakeCalls.GetCall(fake);
getCall.Returns(FakeResponse.BadAsync(connectionConfiguration, response: response));
var client = fake.Resolve <ElasticsearchClient>();
var result = await client.InfoAsync();
result.Success.Should().BeFalse();
AssertServerErrorsOnResponse(result, status, exceptionType, exceptionMessage);
getCall.MustHaveHappened(Repeated.Exactly.Once);
return(result);
}
public void Soft_IConnectionException_AsyncCall_WithoutPooling_Retries_AndThrows()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
fake.Provide <IConnectionConfigurationValues>(_connectionConfig);
FakeCalls.ProvideDefaultTransport(fake);
var getCall = FakeCalls.GetCall(fake);
//return a started task that throws
Func <ElasticsearchResponse <Stream> > badTask = () => { throw new Exception("inner"); };
var t = new Task <ElasticsearchResponse <Stream> >(badTask);
t.Start();
getCall.Returns(t);
var client = fake.Resolve <ElasticsearchClient>();
client.Settings.MaxRetries.Should().NotHaveValue();
var e = Assert.Throws <Exception>(async() => await client.InfoAsync());
e.Message.Should().Be("inner");
getCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public async void DeadNodeIsSkipped_AndRevivedAfterTimeout_Async()
{
using (var fake = new AutoFake())
{
var dateTimeProvider = ProvideDateTimeProvider(fake);
var config = ProvideConfiguration(dateTimeProvider);
var connection = ProvideConnection(fake, config);
var getCall = FakeCalls.GetCall(fake);
var ok = Task.FromResult(FakeResponse.Ok(config));
var bad = Task.FromResult(FakeResponse.Bad(config));
getCall.ReturnsNextFromSequence(
ok, //info 1 - 9204
bad, //info 2 - 9203 DEAD
ok, //info 2 retry - 9202
ok, //info 3 - 9201
ok, //info 4 - 9204
ok, //info 5 - 9202
ok, //info 6 - 9201
ok, //info 7 - 9204
ok, //info 8 - 9203 (Now > Timeout)
ok //info 9 - 9202
);
var seenNodes = new List <Uri>();
getCall.Invokes((Uri u, IRequestConfiguration o) => seenNodes.Add(u));
var pingCall = FakeCalls.PingAtConnectionLevelAsync(fake);
pingCall.Returns(ok);
var client1 = fake.Resolve <ElasticsearchClient>();
var result = await client1.InfoAsync(); //info call 1
//first time node is used so a ping is sent first
result.Metrics.Requests.Count.Should().Be(2);
result.Metrics.Requests.First().RequestType.Should().Be(RequestType.Ping);
result.Metrics.Requests.First().Node.Port.Should().Be(9204);
result.Metrics.Requests.Last().RequestType.Should().Be(RequestType.ElasticsearchCall);
result.Metrics.Requests.Last().Node.Port.Should().Be(9204);
result = await client1.InfoAsync(); //info call 2
result.Metrics.Requests.Count.Should().Be(4);
//using 9203 for the first time ping succeeds
result.Metrics.Requests.First().RequestType.Should().Be(RequestType.Ping);
result.Metrics.Requests.First().Node.Port.Should().Be(9203);
result.Metrics.Requests.First().Success.Should().BeTrue();
//call on 9203 fails
result.Metrics.Requests[1].RequestType.Should().Be(RequestType.ElasticsearchCall);
result.Metrics.Requests[1].Node.Port.Should().Be(9203);
result.Metrics.Requests[1].Success.Should().BeFalse();
result.Metrics.Requests[1].HttpStatusCode.Should().Be(503);
//using 9202 for the first time ping succeeds
result.Metrics.Requests[2].RequestType.Should().Be(RequestType.Ping);
result.Metrics.Requests[2].Node.Port.Should().Be(9202);
//call on 9203 fails
result.Metrics.Requests[3].RequestType.Should().Be(RequestType.ElasticsearchCall);
result.Metrics.Requests[3].Node.Port.Should().Be(9202);
result.Metrics.Requests[3].Success.Should().BeTrue();
result.Metrics.Requests[3].HttpStatusCode.Should().Be(200);
result = await client1.InfoAsync(); //info call 3
result = await client1.InfoAsync(); //info call 4
result = await client1.InfoAsync(); //info call 5
result = await client1.InfoAsync(); //info call 6
result = await client1.InfoAsync(); //info call 7
result = await client1.InfoAsync(); //info call 8
result = await client1.InfoAsync(); //info call 9
AssertSeenNodesAreInExpectedOrder(seenNodes);
//4 nodes first time usage + 1 time after the first time 9203 came back to live
pingCall.MustHaveHappened(Repeated.Exactly.Times(5));
}
}
