/**== Custom Connection Implementations
*
* The client abstracts sending the request and creating a response behind `IConnection`
*
* By default the client will use a WebRequest based version on the desktop CLR (.NET 4.5 and up targets)
* and a HttpClient based HttpConnection specifically build for the Core CLR (netstandard 1.6).
*
* The reason for the split is because WebRequest and ServicePoint are not directly available on netstandard 1.6
*
* However the implementation written against WebRequest is the most matured implementation that we weren't ready to it give up.
* There are also a couple of important toggles that are easy to set against a `ServicePoint` that we'd have to give up
* had we jumped on the `HttpClient` completely.
*
* Another limitation is that `HttpClient` has no synchronous code paths and supporting that means doing hacky async patches which definitely
* need time to bake.
*
* So why would you ever want to pass your own `IConnection`? Let's look at a couple of examples
*
*/
public void OverrideHow()
{
var connection = new InMemoryConnection();
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(connectionPool, connection);
var client = new ElasticClient(settings);
}
/** == Transports
*
* The ITransport interface can be seen as the motor block of the client. It's interface is deceitfully simple.
* Its ultimately responsible from translating a client call to a response. If for some reasons you do not agree with the way we wrote
* the internals of the client by implementing a custom ITransport you can circumvent all of it and introduce your own.
*/
public async Task InterfaceExplained()
{
/**
* Transport is generically typed to a type that implements IConnectionConfigurationValues
* This is the minimum ITransport needs to report back for the client to function.
*
* e.g in the low level client transport is instantiated like this:
*/
var lowLevelTransport = new Transport<ConnectionConfiguration>(new ConnectionConfiguration());
/** In the high level client like this. */
var highlevelTransport = new Transport<ConnectionSettings>(new ConnectionSettings());
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var inMemoryTransport = new Transport<ConnectionSettings>(new ConnectionSettings(connectionPool, new InMemoryConnection()));
/**
* The only two methods on ITransport are Request and DoRequestAsync, the default ITransport implementation is responsible for introducing
* many of the building blocks in the client, if these do not work for you can swap them out for your own custom ITransport implementation.
* If you feel this need report back to us we would love to learn why you'd go down this route!
*/
var response = inMemoryTransport.Request<SearchResponse<Project>>(HttpMethod.GET, "/_search", new { query = new { match_all = new { } } });
response = await inMemoryTransport.RequestAsync<SearchResponse<Project>>(HttpMethod.GET, "/_search", new { query = new { match_all = new { } } });
}
/**== Transports
*
* The `ITransport` interface can be seen as the motor block of the client. It's interface is deceitfully simple and
* it's ultimately responsible from translating a client call to a response.
*
* If for some reason you do not agree with the way we wrote the internals of the client,
* by implementing a custom `ITransport`, you can circumvent all of it and introduce your own.
*/
public async Task InterfaceExplained()
{
/**
* Transport is generically typed to a type that implements `IConnectionConfigurationValues`
* This is the minimum `ITransport` needs to report back for the client to function.
*
* In the low level client, `ElasticLowLevelClient`, a `Transport` is instantiated like this:
*/
var lowLevelTransport = new Transport<ConnectionConfiguration>(new ConnectionConfiguration());
/** and in the high level client, `ElasticClient`, like this: */
var highlevelTransport = new Transport<ConnectionSettings>(new ConnectionSettings());
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var inMemoryTransport = new Transport<ConnectionSettings>(new ConnectionSettings(connectionPool, new InMemoryConnection()));
/**
* The only two methods on `ITransport` are `Request()` and `RequestAsync()`; the default `ITransport` implementation is responsible for introducing
* many of the building blocks in the client. If you feel that the defaults do not work for you then you can swap them out for your own
* custom `ITransport` implementation and if you do, {github}/issues[please let us know] as we'd love to learn why you've go down this route!
*/
var response = inMemoryTransport.Request<SearchResponse<Project>>(
HttpMethod.GET,
"/_search",
new { query = new { match_all = new { } } });
response = await inMemoryTransport.RequestAsync<SearchResponse<Project>>(
HttpMethod.GET,
"/_search",
default(CancellationToken),
new { query = new { match_all = new { } } });
}
public void Ping_should_work()
{
var httpConnection = new AwsHttpConnection(TestConfig.AwsSettings);
var pool = new SingleNodeConnectionPool(new Uri(TestConfig.Endpoint));
var config = new ConnectionConfiguration(pool, httpConnection);
var client = new ElasticLowLevelClient(config);
var response = client.Ping<object>();
Assert.AreEqual(200, response.HttpStatusCode.GetValueOrDefault(-1));
}
public void NestPing_should_work()
{
var httpConnection = new AwsHttpConnection(TestConfig.AwsSettings);
var pool = new SingleNodeConnectionPool(new Uri(TestConfig.Endpoint));
var config = new Nest.ConnectionSettings(pool, httpConnection);
var client = new Nest.ElasticClient(config);
var response = client.Ping();
Assert.AreEqual(true, response.IsValid);
}
public void Asterisk_encoded_url_should_work()
{
var httpConnection = new AwsHttpConnection(TestConfig.AwsSettings);
var pool = new SingleNodeConnectionPool(new Uri(TestConfig.Endpoint));
var config = new ConnectionConfiguration(pool, httpConnection);
var client = new ElasticLowLevelClient(config);
var response = client.Get<Stream>("index*", "type", "id");
Assert.AreEqual(404, response.HttpStatusCode.GetValueOrDefault(-1));
}
public IElasticClient CreateClient(string jsonResponse, Action<JsonSerializerSettings, IConnectionSettingsValues> settingsOverride)
{
var connection = new InMemoryConnection(Encoding.UTF8.GetBytes(jsonResponse));
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var connectionSettings = new ConnectionSettings(connectionPool, connection, new MyCustomJsonFactory(settingsOverride))
.DefaultIndex("default-index");
var client = new ElasticClient(connectionSettings);
return client;
}
public void Random_encoded_url_should_work()
{
var randomString = Guid.NewGuid().ToString("N");
var httpConnection = new AwsHttpConnection(TestConfig.AwsSettings);
var pool = new SingleNodeConnectionPool(new Uri(TestConfig.Endpoint));
var config = new ConnectionConfiguration(pool, httpConnection);
var client = new ElasticLowLevelClient(config);
var response = client.Get<Stream>(randomString, string.Join(",", Enumerable.Repeat(randomString, 2)), randomString);
Assert.AreEqual(404, response.HttpStatusCode.GetValueOrDefault(-1));
}
public IElasticClient CreateClient(string jsonResponse, Action<JsonSerializerSettings, IConnectionSettingsValues> settingsOverride)
{
var connection = new InMemoryConnection(Encoding.UTF8.GetBytes(jsonResponse));
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
#pragma warning disable CS0618 // Type or member is obsolete
var connectionSettings = new ConnectionSettings(connectionPool, connection, settings =>
new LocalJsonNetSerializer(settings.DefaultIndex("default-index"), settingsOverride));
#pragma warning restore CS0618 // Type or member is obsolete
var client = new ElasticClient(connectionSettings);
return client;
}
/// <summary>
/// Configures the elasticsearch sink
/// </summary>
/// <param name="nodes">The nodes to write to</param>
public ElasticsearchSinkOptions(IEnumerable<Uri> nodes)
: this()
{
nodes = nodes != null && nodes.Any(n => n != null)
? nodes.Where(n => n != null)
: new[] { new Uri("http://localhost:9200") };
if (nodes.Count() == 1)
ConnectionPool = new SingleNodeConnectionPool(nodes.First());
else
ConnectionPool = new StaticConnectionPool(nodes);
}
public static IElasticClient GetFixedReturnClient(object responseJson)
{
var serializer = new NestSerializer(new ConnectionSettings());
byte[] fixedResult;
using (var ms = new MemoryStream())
{
serializer.Serialize(responseJson, ms);
fixedResult = ms.ToArray();
}
var connection = new InMemoryConnection(fixedResult);
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(connectionPool, connection);
return new ElasticClient(settings);
}
public void SeesInterfaceProperties()
{
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(pool, new InMemoryConnection());
var c = new ElasticClient(settings);
var serialized = c.Serializer.SerializeToString(new Nest.Analysis { CharFilters = new CharFilters() });
serialized.Should().NotContain("char_filters").And.NotContain("charFilters");
serialized.Should().Contain("char_filter");
serialized = c.Serializer.SerializeToString(new AnalysisDescriptor().CharFilters(cf=>cf));
serialized.Should().NotContain("char_filters").And.NotContain("charFilters");
serialized.Should().Contain("char_filter");
}
public void CanDeserializeCopyTo()
{
var json = "{\"test-events-v1-201412\":{\"mappings\":{\"events\":{\"dynamic\":\"false\",\"_size\":{\"enabled\":true},\"properties\":{\"created_utc\":{\"type\":\"date\"},\"data\":{\"properties\":{\"@environment\":{\"properties\":{\"o_s_name\":{\"type\":\"text\",\"index\":false,\"copy_to\":[\"os\"]}}}}},\"id\":{\"type\":\"keyword\"},\"os\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}}},\"test-events-v1-201502\":{\"mappings\":{\"events\":{\"dynamic\":\"false\",\"_size\":{\"enabled\":true},\"properties\":{\"created_utc\":{\"type\":\"date\"},\"data\":{\"properties\":{\"@environment\":{\"properties\":{\"o_s_name\":{\"type\":\"text\",\"index\":false,\"copy_to\":[\"os\"]}}}}},\"id\":{\"type\":\"keyword\"},\"os\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}}}}";
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var connection = new InMemoryConnection(Encoding.UTF8.GetBytes(json));
var settings = new ConnectionSettings(pool, connection).DefaultIndex("test-events-v1-201412");
var client = new ElasticClient(settings);
var mappingResponse = client.GetMapping<Events>();
mappingResponse.IsValid.Should().BeTrue();
var mappingWalker = new MappingWalker(new CopyToVisitor());
mappingWalker.Accept(mappingResponse);
}
/**=== Overriding Json.NET settings
*
* Overriding the default Json.NET behaviour in NEST is an expert behavior but if you need to get to the nitty gritty, this can be really useful.
*/
/**
* The easiest way is to create an instance of `SerializerFactory` that allows you to register a modification callback
* in the constructor
*/
public void EasyWay()
{
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var connectionSettings = new ConnectionSettings(
pool,
new HttpConnection(),
new SerializerFactory((jsonSettings, nestSettings) => jsonSettings.PreserveReferencesHandling = PreserveReferencesHandling.All));
var client = new ElasticClient(connectionSettings);
}
public static IElasticClient GetFixedReturnClient(
object response,
int statusCode = 200,
Func<ConnectionSettings, ConnectionSettings> modifySettings = null,
string contentType = "application/json",
Exception exception = null)
{
var serializer = new JsonNetSerializer(new ConnectionSettings());
byte[] fixedResult;
if (contentType == "application/json")
{
using (var ms = new MemoryStream())
{
serializer.Serialize(response, ms);
fixedResult = ms.ToArray();
}
}
else
{
fixedResult = Encoding.UTF8.GetBytes(response.ToString());
}
var connection = new InMemoryConnection(fixedResult, statusCode, exception);
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var defaultSettings = new ConnectionSettings(connectionPool, connection)
.DefaultIndex("default-index");
var settings = (modifySettings != null) ? modifySettings(defaultSettings) : defaultSettings;
return new ElasticClient(settings);
}
public void ModifyJsonSerializerSettingsIsCalled()
{
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(connectionPool, new InMemoryConnection(), s => new MyJsonNetSerializer(s));
var client = new ElasticClient(settings);
client.RootNodeInfo();
client.RootNodeInfo();
var serializer = ((IConnectionSettingsValues)settings).Serializer as MyJsonNetSerializer;
serializer.X.Should().BeGreaterThan(0);
}
/**
* An example of using `OnRequestCompleted()` for complex logging. Remember, if you would also like
* to capture the request and/or response bytes, you also need to set `.DisableDirectStreaming()`
* to `true`
*/
[U]public async Task UsingOnRequestCompletedForLogging()
{
var list = new List<string>();
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(connectionPool, new InMemoryConnection())
.DisableDirectStreaming()
.OnRequestCompleted(response =>
{
// log out the request
if (response.RequestBodyInBytes != null)
{
list.Add(
$"{response.HttpMethod} {response.Uri} \n" +
$"{Encoding.UTF8.GetString(response.RequestBodyInBytes)}");
}
else
{
list.Add($"{response.HttpMethod} {response.Uri}");
}
// log out the response
if (response.ResponseBodyInBytes != null)
{
list.Add($"Status: {response.HttpStatusCode}\n" +
$"{Encoding.UTF8.GetString(response.ResponseBodyInBytes)}\n" +
$"{new string('-', 30)}\n");
}
else
{
list.Add($"Status: {response.HttpStatusCode}\n" +
$"{new string('-', 30)}\n");
}
});
var client = new ElasticClient(settings);
var syncResponse = client.Search<object>(s => s
.Scroll("2m")
.Sort(ss => ss
.Ascending(SortSpecialField.DocumentIndexOrder)
)
);
list.Count.Should().Be(2);
var asyncResponse = await client.SearchAsync<object>(s => s
.Scroll("2m")
.Sort(ss => ss
.Ascending(SortSpecialField.DocumentIndexOrder)
)
);
list.Count.Should().Be(4);
list.ShouldAllBeEquivalentTo(new []
{
"POST http://localhost:9200/_search?scroll=2m \n{\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}",
"Status: 200\n------------------------------\n",
"POST http://localhost:9200/_search?scroll=2m \n{\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}",
"Status: 200\n------------------------------\n"
});
}
public void OnRequestCompletedIsCalled()
{
var counter = 0;
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(connectionPool, new InMemoryConnection())
.OnRequestCompleted(r => counter++);
var client = new ElasticClient(settings);
client.RootNodeInfo();
counter.Should().Be(1);
client.RootNodeInfoAsync();
counter.Should().Be(2);
}
public static IElasticClient GetFixedReturnClient(
object response,
int statusCode = 200,
Func<ConnectionSettings, ConnectionSettings> modifySettings = null,
string contentType = "application/json",
Exception exception = null)
{
var serializer = Default.Serializer;
var fixedResult = contentType == "application/json"
? serializer.SerializeToBytes(response)
: Encoding.UTF8.GetBytes(response.ToString());
var connection = new InMemoryConnection(fixedResult, statusCode, exception);
var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var defaultSettings = new ConnectionSettings(connectionPool, connection)
.DefaultIndex("default-index");
var settings = (modifySettings != null) ? modifySettings(defaultSettings) : defaultSettings;
return new ElasticClient(settings);
}
[U] public void SearchDoesNotTakeDefaultIndexIntoAccount()
{
var node = new Uri("http://localhost:9200");
var connectionPool = new SingleNodeConnectionPool(node);
var connectionSettings = new ConnectionSettings(connectionPool, connection: new InMemoryConnection())
.DefaultIndex("logstash-*")
.DefaultFieldNameInferrer(p => p)
.OnRequestCompleted(info =>
{
// info.Uri is /_search/ without the default index
// my ES instance throws an error on the .kibana index (@timestamp field not mapped because I sort on @timestamp)
});
var client = new ElasticClient(connectionSettings);
var response = client.Search<ESLogEvent>(s=>s);
response.ApiCall.Uri.AbsolutePath.Should().Be("/logstash-%2A/eslogevent/_search");
response = client.Search<ESLogEvent>(new SearchRequest<ESLogEvent>{ });
response.ApiCall.Uri.AbsolutePath.Should().Be("/logstash-%2A/eslogevent/_search");
response = client.Search<ESLogEvent>(new SearchRequest { });
response.ApiCall.Uri.AbsolutePath.Should().Be("/_search");
}
/** = Connection Pooling
* Connection pooling is the internal mechanism that takes care of registering what nodes there are in the cluster and which
* we can use to issue client calls on.
*/
/** == SingleNodeConnectionPool
* The simplest of all connection pools, this takes a single `Uri` and uses that to connect to elasticsearch for all the calls
* It doesn't opt in to sniffing and pinging behavior, and will never mark nodes dead or alive. The one `Uri` it holds is always
* ready to go.
*/
[U] public void SingleNode()
{
var uri = new Uri("http://localhost:9201");
var pool = new SingleNodeConnectionPool(uri);
pool.Nodes.Should().HaveCount(1);
var node = pool.Nodes.First();
node.Uri.Port.Should().Be(9201);
/** This type of pool is hardwired to optout out sniffing*/
pool.SupportsReseeding.Should().BeFalse();
/** and pinging */
pool.SupportsPinging.Should().BeFalse();
/** When you use the low ceremony ElasticClient constructor that takes a single Uri
* We default to this SingleNodeConnectionPool */
var client = new ElasticClient(uri);
client.ConnectionSettings.ConnectionPool.Should().BeOfType<SingleNodeConnectionPool>();
/** However we urge that you always pass your connection settings explicitly */
client = new ElasticClient(new ConnectionSettings(uri));
client.ConnectionSettings.ConnectionPool.Should().BeOfType<SingleNodeConnectionPool>();
/** or even better pass the connection pool explicitly */
client = new ElasticClient(new ConnectionSettings(pool));
client.ConnectionSettings.ConnectionPool.Should().BeOfType<SingleNodeConnectionPool>();
}
/// <summary>
/// Connects to server.
/// </summary>
protected override void ConnectToServer()
{
if ( GetAttributeValue( "NodeUrl" ).IsNotNullOrWhitespace() )
{
try {
var node = new Uri( GetAttributeValue( "NodeUrl" ) );
var region = GetAttributeValue( "Region" );
var accessKey = GetAttributeValue( "AccessKey" );
var secretKey = GetAttributeValue( "SecretKey" );
var httpConnection = new AwsHttpConnection( new AwsSettings
{
AccessKey = accessKey,
SecretKey = secretKey,
Region = region,
} );
var pool = new SingleNodeConnectionPool( node );
var config = new ConnectionSettings( pool, httpConnection );
_client = new ElasticClient( config );
}
catch { }
}
}