public static IElasticLowLevelClient CreateClient(Func <VirtualCluster, VirtualCluster> setup)
{
var cluster = VirtualClusterWith.Nodes(1).Ping(c => c.SucceedAlways());
var virtualSettings = setup(cluster)
.StaticConnectionPool()
.AllDefaults();
var settings = new ConnectionConfiguration(virtualSettings.ConnectionPool, virtualSettings.Connection, Serializer)
.DisablePing()
.EnableDebugMode();
return(new ElasticLowLevelClient(settings));
}
[U] public async Task ThrowsExceptionWithNoRules()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(1)
.StaticConnectionPool()
.Settings(s => s.DisablePing().EnableDebugMode())
);
var e = await Assert.ThrowsAsync <UnexpectedElasticsearchClientException>(
async() => await audit.TraceCalls(new ClientCall {
}));
e.Message.Should().Contain("No ClientCalls defined for the current VirtualCluster, so we do not know how to respond");
}
public async Task PicksADifferentNodeEachTimeAnodeIsDown()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(4)
.ClientCalls(p => p.Fails(Always))
.StickySniffingConnectionPool()
.Settings(p => p.DisablePing().SniffOnStartup(false).SniffOnConnectionFault(false))
);
await audit.TraceCalls(
/** All the calls fail */
new ClientCall {
{ BadResponse, 9200 },
{ BadResponse, 9201 },
{ BadResponse, 9202 },
{ BadResponse, 9203 },
{ MaxRetriesReached },
{ FailedOverAllNodes },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(4) }
},
/** After all our registered nodes are marked dead we want to sample a single dead node
* each time to quickly see if the cluster is back up. We do not want to retry all 4
* nodes
*/
new ClientCall {
{ AllNodesDead },
{ Resurrection, 9200 },
{ BadResponse, 9200 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(4) }
},
new ClientCall {
{ AllNodesDead },
{ Resurrection, 9201 },
{ BadResponse, 9201 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(4) }
},
new ClientCall {
{ AllNodesDead },
{ Resurrection, 9202 },
{ BadResponse, 9202 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(4) }
},
new ClientCall {
{ AllNodesDead },
{ Resurrection, 9203 },
{ BadResponse, 9203 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(4) }
}
);
}
public async Task FailOverResultsInSpans()
{
var payloadSender = new MockPayloadSender();
var cluster = VirtualClusterWith.Nodes(10)
.ClientCalls(c => c.FailAlways())
.ClientCalls(r => r.OnPort(9209).SucceedAlways())
.StaticConnectionPool()
.AllDefaults();
var client = cluster.Client;
using (var agent = new ApmAgent(new TestAgentComponents(payloadSender: payloadSender)))
using (agent.Subscribe(new ElasticsearchDiagnosticsSubscriber()))
{
var searchResponse = await agent.Tracer.CaptureTransaction("Call Client", ApiConstants.ActionExec,
async() => await client.SearchAsync <StringResponse>(PostData.Empty)
);
searchResponse.Should().NotBeNull();
searchResponse.Success.Should().BeTrue();
searchResponse.AuditTrail.Should().NotBeEmpty();
var failed = searchResponse.AuditTrail.Where(a => a.Event == AuditEvent.BadResponse);
failed.Should().HaveCount(9);
var spans = payloadSender.SpansOnFirstTransaction;
spans.Should().NotBeEmpty();
var pings = spans.Where(s => s.Action == "Ping");
pings.Should().HaveCount(10);
spans.Where(s => s.Action == "CallElasticsearch").Should().HaveCount(10);
spans.Should().OnlyContain(s => s.Context != null);
spans.Should().OnlyContain(s => s.Context.Db != null);
spans.Should().OnlyContain(s => s.Context.Db.Statement != null);
spans.First(n => n.Subtype == ApiConstants.SubtypeElasticsearch).Context.Destination.Should().NotBeNull();
spans.First(n => n.Subtype == ApiConstants.SubtypeElasticsearch).Context.Destination.Address.Should().Be("localhost");
spans.First(n => n.Subtype == ApiConstants.SubtypeElasticsearch).Context.Destination.Port.Should().Be(9200);
spans.First(n => n.Subtype == ApiConstants.SubtypeElasticsearch).Context.Destination.Service.Should().NotBeNull();
spans.First(n => n.Subtype == ApiConstants.SubtypeElasticsearch).Context.Destination.Service.Type.Should().Be(ApiConstants.TypeDb);
spans.First(n => n.Subtype == ApiConstants.SubtypeElasticsearch)
.Context.Destination.Service.Name.Should()
.Be(ApiConstants.SubtypeElasticsearch);
spans.First(n => n.Subtype == ApiConstants.SubtypeElasticsearch)
.Context.Destination.Service.Resource.Should()
.Be(ApiConstants.SubtypeElasticsearch);
spans.First(n => n.Subtype == ApiConstants.SubtypeElasticsearch).Outcome.Should().Be(Outcome.Success);
}
}
public async Task SniffOnStartupReseedsPutsMostWeightedNodeToFront()
{
IEnumerable <Node> Nodes(int start) => Enumerable.Range(start, 4)
.Select(i => new Uri($"http://localhost:{9200 + i}"))
.Select((u, i) => new Node(u)
{
Settings = new Dictionary <string, object> {
{ "rack", $"rack_{u.Port - 9200}" }
}
});
/** We seed a cluster with an array of 4 Uri's starting at port 9200.
* Our sniffing sorted connection pool is set up to favor nodes in rack_2
*/
var audit = new Auditor(() => VirtualClusterWith
.Nodes(4)
.ClientCalls(p => p.SucceedAlways())
.Sniff(s => s.SucceedAlways(VirtualClusterWith
.Nodes(Nodes(0))
.ClientCalls(p => p.SucceedAlways()))
)
.StickySniffingConnectionPool(n =>
(n.Settings.TryGetValue("rack", out var v) && v.ToString() == "rack_2" ? 10 : 0)
)
.Settings(p => p.DisablePing())
);
/** Sniff happens on 9200 because our seed has no knowledge of rack ids
* However when we reseed the nodes from the sniff response we sort 9202 to to top
* because it lives in rack_2
*/
await audit.TraceCalls(
new ClientCall
{
{ SniffOnStartup },
{ SniffSuccess, 9200 },
{ HealthyResponse, 9202 },
},
/** We are sticky on 9202 for as long as it keeps returning valid responses */
new ClientCall { { HealthyResponse, 9202 } },
new ClientCall { { HealthyResponse, 9202 } },
new ClientCall { { HealthyResponse, 9202 } },
new ClientCall { { HealthyResponse, 9202 } },
new ClientCall { { HealthyResponse, 9202 } },
new ClientCall { { HealthyResponse, 9202 } },
new ClientCall { { HealthyResponse, 9202 } },
new ClientCall { { HealthyResponse, 9202 } }
);
}
/**==== Client uses publish address
*
*/
[U] public async Task UsesPublishAddress()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(2)
.MasterEligible(9200)
.ClientCalls(c => c.SucceedAlways())
.Ping(r => r.OnPort(9200).Fails(Once))
.Ping(c => c.SucceedAlways())
.Sniff(p => p.SucceedAlways(VirtualClusterWith
.Nodes(10)
.MasterEligible(9200, 9202, 9201)
.PublishAddress("10.0.12.1")
.ClientCalls(c => c.SucceedAlways())
.Ping(c => c.SucceedAlways())
))
.SniffingConnectionPool()
.Settings(s => s.SniffOnStartup(false))
);
void HostAssert(Audit a, string host, int expectedPort)
{
a.Node.Uri.Host.Should().Be(host);
a.Node.Uri.Port.Should().Be(expectedPort);
}
void SniffUrlAssert(Audit a, string host, int expectedPort)
{
HostAssert(a, host, expectedPort);
var sniffUri = new UriBuilder(a.Node.Uri)
{
Path = RequestPipeline.SniffPath,
Query = "flat_settings=true&timeout=2s"
}.Uri;
sniffUri.PathEquals(a.Path, nameof(SniffUrlAssert));
}
audit = await audit.TraceCalls(
new ClientCall {
{ PingFailure, a => HostAssert(a, "localhost", 9200) },
{ SniffOnFail },
{ SniffSuccess, a => SniffUrlAssert(a, "localhost", 9200) },
{ PingSuccess, a => HostAssert(a, "10.0.12.1", 9200) },
{ HealthyResponse, a => HostAssert(a, "10.0.12.1", 9200) },
{ pool => pool.Nodes.Count.Should().Be(10) } // <1> Our pool should now have 10 nodes
}
);
}
public async Task HttpTeapotDoesNotFallOver()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.FailAlways(418))
.ClientCalls(r => r.OnPort(9201).SucceedAlways())
.StaticConnectionPool()
.Settings(s => s.DisablePing())
);
audit = await audit.TraceCall(
new ClientCall {
{ AuditEvent.BadResponse, 9200 },
}
);
}
public async Task DoesNotRetryOnSingleNodeConnectionPool()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.FailAlways().Takes(TimeSpan.FromSeconds(3)))
.ClientCalls(r => r.OnPort(9209).SucceedAlways())
.SingleNodeConnection()
.Settings(s => s.DisablePing().MaximumRetries(10))
);
audit = await audit.TraceCall(
new ClientCall(r => r.MaxRetries(10)) {
{ BadResponse, 9200 }
}
);
}
public async Task OnlyCallsForcedNode()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.SucceedAlways())
.ClientCalls(r => r.OnPort(9208).FailAlways())
.StaticConnectionPool()
.Settings(s => s.DisablePing())
);
audit = await audit.TraceCall(
new ClientCall(r => r.ForceNode(new Uri("http://localhost:9208"))) {
{ BadResponse, 9208 }
}
);
}
/** Finally, let's demonstrate disabling both sniff and ping on the request */
[U] public async Task DisableSniffAndPing()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.SucceedAlways())
.SniffingConnectionPool()
.Settings(s => s.SniffOnStartup())
);
audit = await audit.TraceCall(
new ClientCall(r => r.DisableSniffing().DisablePing()) // <1> disable ping and sniff
{
{ HealthyResponse, 9200 } // <2> no ping or sniff before the call
}
);
}
[U] public async Task AGlobalRuleStaysValidForever()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(1)
.ClientCalls(c => c.SucceedAlways())
.StaticConnectionPool()
.Settings(s => s.DisablePing())
);
audit = await audit.TraceCalls(
Enumerable.Range(0, 1000)
.Select(i => new ClientCall {
{ AuditEvent.HealthyResponse, 9200 },
})
.ToArray()
);
}
public async Task FallsOverDeadNodes()
{
/** A cluster with 2 nodes where the second node fails on ping */
var audit = new Auditor(() => VirtualClusterWith
.Nodes(4)
.ClientCalls(p => p.Succeeds(Always))
.ClientCalls(p => p.OnPort(9201).FailAlways())
.ClientCalls(p => p.OnPort(9203).FailAlways())
.StaticConnectionPool()
.Settings(p => p.DisablePing())
);
await audit.TraceCalls(
new ClientCall {
{ HealthyResponse, 9200 }, // <1> The first call goes to 9200 which succeeds
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(0) }
},
new ClientCall {
{ BadResponse, 9201 }, // <2> The 2nd call does a ping on 9201 because its used for the first time. It fails so we wrap over to node 9202
{ HealthyResponse, 9202 },
/** Finally we assert that the connectionpool has one node that is marked as dead */
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(1) }
},
new ClientCall {
{ BadResponse, 9203 }, // <3> The next call goes to 9203 which fails so we should wrap over
{ HealthyResponse, 9200 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(2) }
},
new ClientCall {
{ HealthyResponse, 9202 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(2) }
},
new ClientCall {
{ HealthyResponse, 9200 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(2) }
},
new ClientCall {
{ HealthyResponse, 9202 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(2) }
},
new ClientCall {
{ HealthyResponse, 9200 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(2) }
}
);
}
public async Task PicksADifferentNodeEachTimeAnodeIsDown()
{
/** A cluster with 2 nodes where the second node fails on ping */
var audit = new Auditor(() => VirtualClusterWith
.Nodes(4)
.ClientCalls(p => p.Fails(Always))
.StaticConnectionPool()
.Settings(p => p.DisablePing())
);
await audit.TraceCalls(
new ClientCall {
{ BadResponse, 9200 }, // <1> All the calls fail
{ BadResponse, 9201 },
{ BadResponse, 9202 },
{ BadResponse, 9203 },
{ MaxRetriesReached },
{ FailedOverAllNodes },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(4) }
},
new ClientCall {
{ AllNodesDead }, // <2> After all our registered nodes are marked dead we want to sample a single dead node each time to quickly see if the cluster is back up. We do not want to retry all 4 nodes
{ Resurrection, 9201 },
{ BadResponse, 9201 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(4) }
},
new ClientCall {
{ AllNodesDead },
{ Resurrection, 9202 },
{ BadResponse, 9202 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(4) }
},
new ClientCall {
{ AllNodesDead },
{ Resurrection, 9203 },
{ BadResponse, 9203 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(4) }
},
new ClientCall {
{ AllNodesDead },
{ Resurrection, 9200 },
{ BadResponse, 9200 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(4) }
}
);
}
public async Task PingAfterRevival()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(3)
.ClientCalls(r => r.SucceedAlways())
.ClientCalls(r => r.OnPort(9202).Fails(Once))
.Ping(p => p.SucceedAlways())
.StaticConnectionPool()
.AllDefaults()
);
audit = await audit.TraceCalls(
new ClientCall { { PingSuccess, 9200 }, { HealthyResponse, 9200 } },
new ClientCall { { PingSuccess, 9201 }, { HealthyResponse, 9201 } },
new ClientCall {
{ PingSuccess, 9202 },
{ BadResponse, 9202 },
{ HealthyResponse, 9200 },
{ pool => pool.Nodes.Where(n => !n.IsAlive).Should().HaveCount(1) }
},
new ClientCall { { HealthyResponse, 9201 } },
new ClientCall { { HealthyResponse, 9200 } },
new ClientCall { { HealthyResponse, 9201 } },
new ClientCall {
{ HealthyResponse, 9200 },
{ pool => pool.Nodes.First(n => !n.IsAlive).DeadUntil.Should().BeAfter(DateTime.UtcNow) }
}
);
audit = await audit.TraceCalls(
new ClientCall { { HealthyResponse, 9201 } },
new ClientCall { { HealthyResponse, 9200 } },
new ClientCall { { HealthyResponse, 9201 } }
);
audit.ChangeTime(d => d.AddMinutes(20));
audit = await audit.TraceCalls(
new ClientCall { { HealthyResponse, 9201 } },
new ClientCall {
{ Resurrection, 9202 },
{ PingSuccess, 9202 },
{ HealthyResponse, 9202 }
}
);
}
public async Task ExceptionFallsOverToNextNode()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.FailAlways())
.ClientCalls(r => r.OnPort(9201).SucceedAlways())
.StaticConnectionPool()
.Settings(s => s.DisablePing())
);
audit = await audit.TraceCall(
new ClientCall {
{ AuditEvent.BadResponse, 9200 },
{ AuditEvent.HealthyResponse, 9201 },
}
);
}
public async Task CustomPredicateYieldingNothingThrows()
{
var totalNodesInTheCluster = 20;
var audit = new Auditor(() => VirtualClusterWith
.Nodes(totalNodesInTheCluster)
.Sniff(s => s.SucceedAlways()
.Succeeds(Always, VirtualClusterWith.Nodes(totalNodesInTheCluster))
)
.SniffingConnectionPool()
.Settings(s => s
.DisablePing()
.NodePredicate(node => false) // <1> A _bad_ predicate that declines *all* nodes
)
);
/**
* Now when making the client calls, the audit trail indicates that a sniff on startup succeeds, but the subsequent
* API call fails because the node predicate filters out all nodes as targets on which to execute API calls
*/
await audit.TraceUnexpectedElasticsearchException(new ClientCall
{
{ SniffOnStartup }, // <1> The audit trail indicates a sniff for the very first time on startup
{ SniffSuccess }, // <2> The sniff succeeds because the node predicate is ignored when sniffing
{ NoNodesAttempted } // <3> when trying to do an actual API call however, the predicate prevents any nodes from being attempted
}, e =>
{
e.FailureReason.Should().Be(PipelineFailure.Unexpected);
Func <string> debug = () => e.DebugInformation;
debug.Invoking(s => s.Invoke()).Should().NotThrow();
});
/**
* An example of the debug information that the client response returns for the previous exception
*
* ....
* # FailureReason: Unrecoverable/Unexpected NoNodesAttempted while attempting POST on default-index/project/_search on an empty node, likely a node predicate on ConnectionSettings not matching ANY nodes
* - [1] SniffOnStartup: Took: 00:00:00
* - [2] SniffSuccess: Node: http://localhost:9200/ Took: 00:00:00
* - [3] NoNodesAttempted: Took: 00:00:00
* # Inner Exception: No nodes were attempted, this can happen when a node predicate does not match any nodes
* ....
*/
}
public async Task CanOverrideBadResponse()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.FailAlways(400))
.StaticConnectionPool()
.Settings(s => s.DisablePing().MaximumRetries(0))
);
audit = await audit.TraceCalls(
new ClientCall {
{ BadResponse, 9200 }
},
new ClientCall(r => r.AllowedStatusCodes(400)) {
{ HealthyResponse, 9201 }
}
);
}
/** Now, let's disable pinging on the request */
[U] public async Task DisablePing()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.SucceedAlways())
.SniffingConnectionPool()
.Settings(s => s.SniffOnStartup())
);
audit = await audit.TraceCall(
new ClientCall(r => r.DisablePing()) // <1> disable ping
{
{ SniffOnStartup },
{ SniffSuccess, 9200 }, // <2> No ping after sniffing
{ HealthyResponse, 9200 }
}
);
}
public async Task RespectsOveralRequestTimeout()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.FailAlways().Takes(TimeSpan.FromSeconds(10)))
.ClientCalls(r => r.OnPort(9209).SucceedAlways())
.StaticConnectionPool()
.Settings(s => s.DisablePing().RequestTimeout(TimeSpan.FromSeconds(20)))
);
audit = await audit.TraceCall(
new ClientCall {
{ BadResponse, 9200 },
{ BadResponse, 9201 },
{ MaxTimeoutReached }
}
);
}
public async Task RetriesAreLimitedByNodesInPool()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(2)
.ClientCalls(r => r.FailAlways().Takes(TimeSpan.FromSeconds(3)))
.ClientCalls(r => r.OnPort(9209).SucceedAlways())
.StaticConnectionPool()
.Settings(s => s.DisablePing().RequestTimeout(TimeSpan.FromSeconds(2)).MaxRetryTimeout(TimeSpan.FromSeconds(10)))
);
audit = await audit.TraceCall(
new ClientCall {
{ BadResponse, 9200 },
{ BadResponse, 9201 },
{ MaxRetriesReached },
{ FailedOverAllNodes }
}
);
}
public async Task SniffOnStartUpTakesNewClusterState()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.Sniff(s => s.Fails(Always))
.Sniff(s => s.OnPort(9202).Succeeds(Always, VirtualClusterWith.Nodes(8, startFrom: 9204))) // <1> Sniffing returns 8 nodes, starting from 9204
.SniffingConnectionPool()
.AllDefaults()
);
await audit.TraceCall(new ClientCall {
{ SniffOnStartup },
{ SniffFailure, 9200 },
{ SniffFailure, 9201 },
{ SniffSuccess, 9202 },
{ PingSuccess, 9204 }, // <2> After successfully sniffing, the ping now happens on 9204
{ HealthyResponse, 9204 }
});
}
public async Task FixedMaximumNumberOfRetries()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.FailAlways())
.ClientCalls(r => r.OnPort(9209).SucceedAlways())
.StaticConnectionPool()
.Settings(s => s.DisablePing().MaximumRetries(5))
);
audit = await audit.TraceCall(
new ClientCall(r => r.MaxRetries(2)) {
{ BadResponse, 9200 },
{ BadResponse, 9201 },
{ BadResponse, 9202 },
{ MaxRetriesReached }
}
);
}
public async Task FixedMaximumNumberOfRetries()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.FailAlways())
.ClientCalls(r => r.OnPort(9209).SucceedAlways())
.StaticConnectionPool()
.Settings(s => s.DisablePing().MaximumRetries(3)) // <1> Set the maximum number of retries to 3
);
audit = await audit.TraceCall(
new ClientCall {
{ BadResponse, 9200 },
{ BadResponse, 9201 },
{ BadResponse, 9202 },
{ BadResponse, 9203 },
{ MaxRetriesReached } // <2> The client call trace returns an `MaxRetriesReached` audit after the initial attempt and the number of retries allowed
}
);
}
public async Task AllNodesArePingedOnlyOnFirstUseProvidedTheyAreHealthy()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(4)
.Ping(p => p.SucceedAlways()) // <1> Pings on nodes always succeed
.StaticConnectionPool()
.AllDefaults()
);
await audit.TraceCalls(
new ClientCall { { PingSuccess, 9200 }, { HealthyResponse, 9200 } }, // <2> A successful ping on each node
new ClientCall { { PingSuccess, 9201 }, { HealthyResponse, 9201 } },
new ClientCall { { PingSuccess, 9202 }, { HealthyResponse, 9202 } },
new ClientCall { { PingSuccess, 9203 }, { HealthyResponse, 9203 } },
new ClientCall { { HealthyResponse, 9200 } },
new ClientCall { { HealthyResponse, 9201 } },
new ClientCall { { HealthyResponse, 9202 } },
new ClientCall { { HealthyResponse, 9203 } },
new ClientCall { { HealthyResponse, 9200 } }
);
}
public async Task RespectsConnectTimeoutOverride()
{
/** We set up a 10 node cluster with a global time out of 20 seconds.
* Each call on a node takes 10 seconds. So we can only try this call on 2 nodes
* before the max request time out kills the client call.
*/
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.Ping(p => p.SucceedAlways().Takes(TimeSpan.FromSeconds(20)))
.ClientCalls(r => r.SucceedAlways())
.StaticConnectionPool()
.Settings(s => s.RequestTimeout(TimeSpan.FromSeconds(10)).PingTimeout(TimeSpan.FromSeconds(10)))
);
audit = await audit.TraceCalls(
/**
* The first call uses the configured global settings, request times out after 10 seconds and ping
* calls always take 20, so we should see a single ping failure
*/
new ClientCall {
{ PingFailure, 9200 },
{ MaxTimeoutReached }
},
/**
* On the second request we set a request ping timeout override of 2 seconds
* We should now see more nodes being tried before the request timeout is hit.
*/
new ClientCall(r => r.PingTimeout(TimeSpan.FromSeconds(2)))
{
{ PingFailure, 9202 },
{ PingFailure, 9203 },
{ PingFailure, 9204 },
{ PingFailure, 9205 },
{ PingFailure, 9206 },
{ MaxTimeoutReached }
}
);
}
/**[[disable-sniff-ping-per-request]]
* === Disable sniffing and pinging per request
*
* Even if you are using a sniffing connection pool thats set up to sniff on start/failure
* and pinging enabled, you can opt out of this behaviour on a _per request_ basis.
*
* In our first test we set up a cluster that pings and sniffs on startup
* but we disable the sniffing on our first request so we only see the ping and the response
*/
[U] public async Task DisableSniff()
{
/** Let's set up the cluster and configure clients to **always** sniff on startup */
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.SucceedAlways())
.Sniff(c => c.SucceedAlways())
.Ping(c => c.SucceedAlways())
.SniffingConnectionPool()
.Settings(s => s.SniffOnStartup()) // <1> sniff on startup
);
/** Now We disable sniffing on the request so even though it's our first call,
* we do not want to sniff on startup.
*
* Instead, the sniff on startup is deferred to the second call into the cluster that
* does not disable sniffing on a per request basis.
*
* And after that no sniff on startup will happen again
*/
audit = await audit.TraceCalls(
new ClientCall(r => r.DisableSniffing()) // <1> disable sniffing
{
{ PingSuccess, 9200 }, // <2> first call is a successful ping
{ HealthyResponse, 9200 }
},
new ClientCall()
{
{ SniffOnStartup }, // <3> sniff on startup call happens here, on the second call
{ SniffSuccess, 9200 },
{ PingSuccess, 9200 },
{ HealthyResponse, 9200 }
},
new ClientCall()
{
{ PingSuccess, 9201 }, // <4> No sniff on startup again
{ HealthyResponse, 9201 }
}
);
}
public async Task ExceptionDoesNotCauseLoseOfSpan()
{
var payloadSender = new MockPayloadSender();
var cluster = VirtualClusterWith.Nodes(2)
.ClientCalls(r => r.OnPort(9200).FailAlways())
.ClientCalls(c => c.OnPort(9201).FailAlways(new Exception("boom!")))
.StaticConnectionPool()
.AllDefaults();
var client = cluster.Client;
using (var agent = new ApmAgent(new TestAgentComponents(payloadSender: payloadSender)))
using (agent.Subscribe(new ElasticsearchDiagnosticsSubscriber()))
{
try
{
var searchResponse = await agent.Tracer.CaptureTransaction("Call Client", ApiConstants.ActionExec,
async() => await client.SearchAsync <StringResponse>(PostData.Empty)
);
searchResponse.Should().NotBeNull();
}
catch (Exception)
{
// ignored
}
var spans = payloadSender.SpansOnFirstTransaction;
spans.Should().NotBeEmpty();
spans.Should().Contain(s => s.Context.Db.Statement != null);
//ensure we the last span is closed even if the listener does not receive a response
spans.Where(s => s.Action == "Ping").Should().HaveCount(2);
spans.Where(s => s.Action == "Ping").All(n => n.Outcome == Outcome.Success).Should().BeTrue();
spans.Where(s => s.Action == "CallElasticsearch").Should().HaveCount(2);
spans.Where(s => s.Action == "CallElasticsearch").All(n => n.Outcome == Outcome.Failure).Should().BeTrue();
payloadSender.Errors.Should().Contain(e => ((Error)e).Exception.Message == "boom!");
}
}
/**
* When a bad authentication response occurs, the client attempts to deserialize the response body returned;
*
* In some setups you might be running behind a proxy and you might need to prevent the client from trying to deserialize
* bad json. In the following example an HTML response is return but with an application/json content type. If the proxy is not
* under your control you would need to be able to fix this in the client. Here we make the client aware that 401 responses
* should never be deserialized by calling `SkipDeserializationForStatusCodes()` on `ConnectionSettings`.
*/
[U] public async Task BadAuthenticationHtmlResponseIsIgnored()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.Ping(r => r.SucceedAlways())
.ClientCalls(r => r.FailAlways(401).ReturnByteResponse(HtmlNginx401Response, "application/json")) // <1> Always return a 401 bad response with a HTML response on client calls
.StaticConnectionPool()
.Settings(s => s.SkipDeserializationForStatusCodes(401))
);
audit = await audit.TraceElasticsearchException(
new ClientCall {
{ AuditEvent.PingSuccess, 9200 },
{ AuditEvent.BadResponse, 9201 },
},
(e) =>
{
e.FailureReason.Should().Be(PipelineFailure.BadAuthentication);
e.Response.HttpStatusCode.Should().Be(401);
e.Response.ResponseBodyInBytes.Should().BeNull(); // <2> Assert that the response body bytes are null
}
);
}
/**
* Sometimes, an unexpected exception happens further down in the pipeline. In this scenario, we
* wrap them inside an `UnexpectedElasticsearchClientException` so that information about where
* in the pipeline the exception happened is not lost.
*
* In this next example, a call to 9200 fails with a `WebException`.
* The call then rolls over to retry on 9201, which throws an hard exception from within `IConnection`.
* Finally, we assert that we can still see the audit trail for the whole coordinated request.
*/
[U] public async Task WillFailOverKnowConnectionExceptionButNotUnexpected()
{
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.OnPort(9200).FailAlways(new System.Net.Http.HttpRequestException("recover"))) // <1> calls on 9200 set up to throw a `HttpRequestException`
.ClientCalls(r => r.OnPort(9201).FailAlways(new Exception("boom!"))) // <2> calls on 9201 set up to throw an `Exception`
.StaticConnectionPool()
.Settings(s => s.DisablePing())
);
audit = await audit.TraceUnexpectedException(
new ClientCall {
{ AuditEvent.BadResponse, 9200 },
{ AuditEvent.BadResponse, 9201 }, // <3> Assert that the audit trail for the client call includes the bad response from 9200 and 9201
},
(e) =>
{
e.FailureReason.Should().Be(PipelineFailure.Unexpected);
e.InnerException.Should().NotBeNull();
e.InnerException.Message.Should().Be("boom!");
}
);
}
public async Task RespectsRequestTimeoutOverride()
{
/** we set up a 10 node cluster with a global time out of 20 seconds.
* Each call on a node takes 10 seconds. So we can only try this call on 2 nodes
* before the max request time out kills the client call.
*/
var audit = new Auditor(() => VirtualClusterWith
.Nodes(10)
.ClientCalls(r => r.FailAlways().Takes(TimeSpan.FromSeconds(10)))
.ClientCalls(r => r.OnPort(9209).SucceedAlways())
.StaticConnectionPool()
.Settings(s => s.DisablePing().RequestTimeout(TimeSpan.FromSeconds(20)))
);
audit = await audit.TraceCalls(
new ClientCall {
{ BadResponse, 9200 },
{ BadResponse, 9201 },
{ MaxTimeoutReached }
},
/**
* On the second request we specify a request timeout override to 80 seconds
* We should now see more nodes being tried.
*/
new ClientCall(r => r.RequestTimeout(TimeSpan.FromSeconds(80)))
{
{ BadResponse, 9203 },
{ BadResponse, 9204 },
{ BadResponse, 9205 },
{ BadResponse, 9206 },
{ BadResponse, 9207 },
{ BadResponse, 9208 },
{ HealthyResponse, 9209 },
}
);
}
