private void ApplySettings(dynamic config)
{
_clients.Clear();
_config = (ESConfig)(config?.elasticsearch?.ToObject <ESConfig>()) ?? new ESConfig();
_connectionPools = _config.ConnectionPools?.ToDictionary(kvp => kvp.Key, kvp =>
{
var c = kvp.Value;
IConnectionPool pool;
var endpoints = c.Endpoints.DefaultIfEmpty("http://localhost:9200").Select(endpoint => new Uri(endpoint));
if (c.Sniffing)
{
// var connectionEndpoints = ((JArray)config.esEndpoints).ToObject<string[]>();
pool = new Elasticsearch.Net.SniffingConnectionPool(endpoints);
}
else
{
pool = new Elasticsearch.Net.StaticConnectionPool(endpoints);
}
return(new ConnectionPool(pool, c.Credentials));
}) ?? new Dictionary <string, ConnectionPool>();
if (!_connectionPools.ContainsKey("default"))
{
_connectionPools.Add("default", new ConnectionPool(new Elasticsearch.Net.StaticConnectionPool(new[] { new Uri("http://localhost:9200") }), null));
}
}
[U] public void StaticPoolWithstandsConcurrentReadAndWrites()
{
var uris = Enumerable.Range(9200, NumberOfNodes).Select(p => new Uri("http://localhost:" + p));
var staticPool = new StaticConnectionPool(uris, randomize: false);
Action callStatic = () => this.AssertCreateView(staticPool);
callStatic.ShouldNotThrow();
}
[U] public void EachViewStartsAtNexPositionAndWrapsOver()
{
var uris = Enumerable.Range(9200, NumberOfNodes).Select(p => new Uri("http://localhost:" + p));
var staticPool = new StaticConnectionPool(uris, randomize: false);
var sniffingPool = new SniffingConnectionPool(uris, randomize: false);
this.AssertCreateView(staticPool);
this.AssertCreateView(sniffingPool);
}
/// <summary>
/// Configures the elasticsearch sink
/// </summary>
/// <param name="nodes">The nodes to write to</param>
public ElasticsearchSinkOptions(IEnumerable<Uri> nodes)
: this()
{
nodes = nodes != null && nodes.Any(n => n != null)
? nodes.Where(n => n != null)
: new[] { new Uri("http://localhost:9200") };
if (nodes.Count() == 1)
ConnectionPool = new SingleNodeConnectionPool(nodes.First());
else
ConnectionPool = new StaticConnectionPool(nodes);
}
[U] public void EachViewSkipsAheadWithOne()
{
var seeds = Enumerable.Range(9200, NumberOfNodes).Select(p => new Node(new Uri("http://localhost:" + p))).ToList();
var pool = new StaticConnectionPool(seeds, randomize: false);
for (var i = 0; i < 20; i++)
{
var node = pool.CreateView().First();
node.Uri.Port.Should().Be(9200);
node = pool.CreateView().First();
node.Uri.Port.Should().Be(9201);
node = pool.CreateView().First();
node.Uri.Port.Should().Be(9202);
}
}
[U] public void ViewSeesResurrectedNodes()
{
var dateTimeProvider = new TestableDateTimeProvider();
var seeds = Enumerable.Range(9200, NumberOfNodes).Select(p => new Node(new Uri("http://localhost:" + p))).ToList();
seeds.First().MarkDead(dateTimeProvider.Now().AddDays(1));
var pool = new StaticConnectionPool(seeds, randomize: false, dateTimeProvider: dateTimeProvider);
for (var i = 0; i < 20; i++)
{
var node = pool.CreateView().First();
node.Uri.Port.Should().Be(9201);
node = pool.CreateView().First();
node.Uri.Port.Should().Be(9202);
}
/** If we roll the clock forward two days, the node that was marked dead until tomorrow (or yesterday!) should be resurrected */
dateTimeProvider.ChangeTime(d => d.AddDays(2));
var n = pool.CreateView().First();
n.Uri.Port.Should().Be(9201);
n = pool.CreateView().First();
n.Uri.Port.Should().Be(9202);
n = pool.CreateView().First();
n.Uri.Port.Should().Be(9200);
n.IsResurrected.Should().BeTrue();
}
/** == StaticConnectionPool
* The static connection pool is great if you have a known small sized cluster and do no want to enable
* sniffing to find out the cluster topology.
*/
[U] public void Static()
{
var uris = Enumerable.Range(9200, 5).Select(p => new Uri("http://localhost:" + p));
/** a connection pool can be seeded using an enumerable of `Uri` */
var pool = new StaticConnectionPool(uris);
/** Or using an enumerable of `Node` */
var nodes = uris.Select(u=>new Node(u));
pool = new StaticConnectionPool(nodes);
/** This type of pool is hardwirted to optout out sniffing*/
pool.SupportsReseeding.Should().BeFalse();
/** but supports pinging when enabled */
pool.SupportsPinging.Should().BeTrue();
/** To create a client using this static connection pool pass
* the connection pool to the connectionsettings you pass to ElasticClient
*/
var client = new ElasticClient(new ConnectionSettings(pool));
client.ConnectionSettings.ConnectionPool.Should().BeOfType<StaticConnectionPool>();
}
[U] public void EachViewSeesNextButSkipsTheDeadNode()
{
var seeds = Enumerable.Range(9200, NumberOfNodes).Select(p => new Node(new Uri("http://localhost:" + p))).ToList();
seeds.First().MarkDead(DateTime.Now.AddDays(1));
var pool = new StaticConnectionPool(seeds, randomize: false);
for (var i = 0; i < 20; i++)
{
var node = pool.CreateView().First();
node.Uri.Port.Should().Be(9201);
node = pool.CreateView().First();
node.Uri.Port.Should().Be(9202);
}
/** After we marked the first node alive again, we expect it to be hit again*/
seeds.First().MarkAlive();
for (var i = 0; i < 20; i++)
{
var node = pool.CreateView().First();
node.Uri.Port.Should().Be(9201);
node = pool.CreateView().First();
node.Uri.Port.Should().Be(9202);
node = pool.CreateView().First();
node.Uri.Port.Should().Be(9200);
}
}
public void Init()
{
ESTraceLogger.Debug($"ElasticCacheProvider Init: Starting");
if (client == null)
{
var pool = new Elasticsearch.Net.StaticConnectionPool(
this.ElasticServers
.Where(m => !string.IsNullOrWhiteSpace(m))
.Select(u => new Uri(u))
);
var settings = new ConnectionSettings(pool)
.DefaultIndex(this.DBName)
.DisableAutomaticProxyDetection(false)
.RequestTimeout(TimeSpan.FromMilliseconds(60000))
.SniffLifeSpan(null)
.OnRequestCompleted(call =>
{
// log out the request and the request body, if one exists for the type of request
if (call.RequestBodyInBytes != null)
{
ESTraceLogger.Debug($"{call.HttpMethod}\t{call.Uri}\t" +
$"{Encoding.UTF8.GetString(call.RequestBodyInBytes)}");
}
else
{
ESTraceLogger.Debug($"{call.HttpMethod}\t{call.Uri}\t");
}
})
;
if (System.Diagnostics.Debugger.IsAttached || Devmasters.Config.GetWebConfigValue("ESDebugDataEnabled") == "true")
{
settings = settings.DisableDirectStreaming();
}
client = new ElasticClient(settings);
IndexSettings set = new IndexSettings();
set.NumberOfReplicas = this.NumberOfReplicas;
set.NumberOfShards = this.NumberOfShards;
IndexState idxSt = new IndexState();
idxSt.Settings = set;
ESTraceLogger.Debug($"ElasticCacheProvider Init: Check Elastic DB {client.ConnectionSettings.DefaultIndex}");
var resExists = client.Indices.Exists(client.ConnectionSettings.DefaultIndex);
//if (resExists.IsValid == false)
//{
// ESTraceLogger.Fatal($"ElasticCacheProvider: Cannot check db existance {resExists.ServerError?.ToString()}", resExists.OriginalException);
// throw new ApplicationException("Cannot check Elastic DB");
//}
if (resExists.Exists == false)
{
ESTraceLogger.Debug($"ElasticCacheProvider Init: DB {client.ConnectionSettings.DefaultIndex} doesn't exist. Creating new one");
var res = client.Indices.Create(client.ConnectionSettings.DefaultIndex,
i => i.InitializeUsing(idxSt)
.Map <Bag <T> >(map => map.AutoMap().DateDetection(false))
);
if (res.IsValid == false)
{
ESTraceLogger.Fatal($"ElasticCacheProvider: Created db error {res.ServerError?.ToString()}", res.OriginalException);
throw new ApplicationException("Cannot created Elastic DB");
}
}
ESTraceLogger.Debug($"ElasticCacheProvider Init: DB {client.ConnectionSettings.DefaultIndex} is live.");
cleaning.Elapsed += Cleaning_Elapsed;
cleaning.Start();
}
}
