public async void SniffOnStartShouldOnlyHit9200_WithoutPing_Async()
{
var seeds = new[]
{
ElasticsearchConfiguration.CreateBaseUri(9202),
ElasticsearchConfiguration.CreateBaseUri(9201),
ElasticsearchConfiguration.CreateBaseUri(9200)
};
var sniffingConnectionPool = new SniffingConnectionPool(seeds, randomizeOnStartup: false);
var connectionSettings = new ConnectionSettings(sniffingConnectionPool)
.SniffOnStartup();
var client = new ElasticClient(connectionSettings);
var rootNode = await client.RootNodeInfoAsync();
var metrics = rootNode.ConnectionStatus.Metrics;
//When the connectionpool is used for the first time the sniff call should already
//know only 9200 is on and live, no need to ping
metrics.Requests.Count.Should().Be(1);
metrics.Requests[0].Node.Port.Should().Be(9200);
metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall);
for (var i = 0; i < 3; i++)
{
rootNode = await client.RootNodeInfoAsync();
metrics = rootNode.ConnectionStatus.Metrics;
metrics.Requests.Count.Should().Be(1);
metrics.Requests[0].Node.Port.Should().Be(9200);
metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall);
}
}
public void FailoverShouldOnlyPingDeadNodes()
{
var seeds = new[]
{
ElasticsearchConfiguration.CreateBaseUri(9202),
ElasticsearchConfiguration.CreateBaseUri(9201),
ElasticsearchConfiguration.CreateBaseUri(9200)
};
var sniffingConnectionPool = new SniffingConnectionPool(seeds, randomizeOnStartup: false);
var connectionSettings = new ConnectionSettings(sniffingConnectionPool);
var client = new ElasticClient(connectionSettings);
var rootNode = client.RootNodeInfo();
var metrics = rootNode.ConnectionStatus.Metrics;
//ping 9202 + 9201 + 9200 and call 9200
metrics.Requests.Count.Should().Be(4);
metrics.Requests[0].Node.Port.Should().Be(9202);
metrics.Requests[0].RequestType.Should().Be(RequestType.Ping);
metrics.Requests[0].EllapsedMilliseconds.Should().BeLessOrEqualTo(1100);
metrics.Requests[1].Node.Port.Should().Be(9201);
metrics.Requests[1].RequestType.Should().Be(RequestType.Ping);
metrics.Requests[1].EllapsedMilliseconds.Should().BeLessOrEqualTo(1100);
metrics.Requests[2].Node.Port.Should().Be(9200);
metrics.Requests[2].RequestType.Should().Be(RequestType.Ping);
metrics.Requests[2].EllapsedMilliseconds.Should().BeLessOrEqualTo(1100);
metrics.Requests[3].Node.Port.Should().Be(9200);
metrics.Requests[3].RequestType.Should().Be(RequestType.ElasticsearchCall);
metrics.Requests[3].EllapsedMilliseconds.Should().BeLessOrEqualTo(1100);
rootNode = client.RootNodeInfo();
metrics = rootNode.ConnectionStatus.Metrics;
metrics.Requests.Count.Should().Be(1);
metrics.Requests[0].Node.Port.Should().Be(9200);
metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall);
rootNode = client.RootNodeInfo();
metrics = rootNode.ConnectionStatus.Metrics;
metrics.Requests.Count.Should().Be(1);
metrics.Requests[0].Node.Port.Should().Be(9200);
metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall);
rootNode = client.RootNodeInfo();
metrics = rootNode.ConnectionStatus.Metrics;
metrics.Requests.Count.Should().Be(1);
metrics.Requests[0].Node.Port.Should().Be(9200);
metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall);
rootNode = client.RootNodeInfo();
metrics = rootNode.ConnectionStatus.Metrics;
metrics.Requests.Count.Should().Be(1);
metrics.Requests[0].Node.Port.Should().Be(9200);
metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall);
}
public void IndexExistShouldNotThrowOn404()
{
var host = Test.Default.Host;
if (Process.GetProcessesByName("fiddler").Any())
host = "ipv4.fiddler";
var connectionPool = new SniffingConnectionPool(new[] { new Uri("http://{0}:9200".F(host)) });
var settings = new ConnectionSettings(connectionPool, ElasticsearchConfiguration.DefaultIndex)
.SniffOnStartup();
var client = new ElasticClient(settings);
}
public async void SniffOnFaultShouldGetCleanClusterState_Async()
{
var seeds = new[]
{
ElasticsearchConfiguration.CreateBaseUri(9202),
ElasticsearchConfiguration.CreateBaseUri(9201),
ElasticsearchConfiguration.CreateBaseUri(9200),
};
var sniffingConnectionPool = new SniffingConnectionPool(seeds, randomizeOnStartup: false);
var connectionSettings = new ConnectionSettings(sniffingConnectionPool)
.SniffOnConnectionFault();
var client = new ElasticClient(connectionSettings);
var rootNode = await client.RootNodeInfoAsync();
var metrics = rootNode.ConnectionStatus.Metrics;
metrics.Requests.Count.Should().Be(5);
metrics.Requests[0].Node.Port.Should().Be(9202);
metrics.Requests[0].RequestType.Should().Be(RequestType.Ping);
metrics.Requests[1].Node.Port.Should().Be(9201);
metrics.Requests[1].RequestType.Should().Be(RequestType.Ping);
metrics.Requests[2].Node.Port.Should().Be(9200);
metrics.Requests[2].RequestType.Should().Be(RequestType.Ping);
metrics.Requests[3].Node.Port.Should().Be(9200);
metrics.Requests[3].RequestType.Should().Be(RequestType.Sniff);
metrics.Requests[4].Node.Port.Should().Be(9200);
metrics.Requests[4].RequestType.Should().Be(RequestType.ElasticsearchCall);
for (var i = 0; i < 3; i++)
{
rootNode = await client.RootNodeInfoAsync();
metrics = rootNode.ConnectionStatus.Metrics;
metrics.Requests.Count.Should().Be(1);
metrics.Requests[0].Node.Port.Should().Be(9200);
metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall);
}
}
public void ShouldThrowAndNotRetrySniffOnConnectionFault403()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
var uris = new[]
{
new Uri("http://localhost:9200"),
new Uri("http://localhost:9201"),
new Uri("http://localhost:9202")
};
var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false);
var config = new ConnectionConfiguration(connectionPool)
.SniffOnConnectionFault()
.ThrowOnElasticsearchServerExceptions();
fake.Provide <IConnectionConfigurationValues>(config);
FakeCalls.ProvideDefaultTransport(fake);
var pingCall = FakeCalls.PingAtConnectionLevel(fake);
pingCall.Returns(FakeResponse.Ok(config));
var sniffCall = FakeCalls.Sniff(fake);
var seenPorts = new List <int>();
sniffCall.ReturnsLazily((Uri u, IRequestConfiguration c) =>
{
seenPorts.Add(u.Port);
return(FakeResponse.Any(config, 403));
});
var getCall = FakeCalls.GetSyncCall(fake);
getCall.Returns(FakeResponse.Bad(config));
var client = fake.Resolve <ElasticsearchClient>();
var e = Assert.Throws <ElasticsearchServerException>(() => client.Info());
e.Status.Should().Be(403);
sniffCall.MustHaveHappened(Repeated.Exactly.Once);
getCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public void SniffOnStartupCallsSniffOnlyOnce()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
//It's recommended to only have on instance of your connection pool
//Be sure to register it as Singleton in your IOC
var uris = new[] { new Uri("http://localhost:9200") };
var connectionPool = new SniffingConnectionPool(uris);
var config = new ConnectionConfiguration(connectionPool)
.DisablePing()
.SniffOnStartup();
fake.Provide <IConnectionConfigurationValues>(config);
var sniffCall = FakeCalls.Sniff(fake, config, uris);
var transport = FakeCalls.ProvideDefaultTransport(fake);
var client1 = new ElasticsearchClient(config, transport: transport);
var client2 = new ElasticsearchClient(config, transport: transport);
var client3 = new ElasticsearchClient(config, transport: transport);
var client4 = new ElasticsearchClient(config, transport: transport);
sniffCall.MustHaveHappened(Repeated.Exactly.Once);
}
}
public void SniffOnFaultShouldGetCleanClusterState()
{
var seeds = new[]
{
ElasticsearchConfiguration.CreateBaseUri(9202),
ElasticsearchConfiguration.CreateBaseUri(9201),
ElasticsearchConfiguration.CreateBaseUri(9200),
};
var sniffingConnectionPool = new SniffingConnectionPool(seeds, randomizeOnStartup: false);
var connectionSettings = new ConnectionSettings(sniffingConnectionPool)
.SniffOnConnectionFault();
var client = new ElasticClient(connectionSettings);
var rootNode = client.RootNodeInfo();
var metrics = rootNode.ConnectionStatus.Metrics;
metrics.Requests.Count.Should().Be(5);
metrics.Requests[0].Node.Port.Should().Be(9202);
metrics.Requests[0].RequestType.Should().Be(RequestType.Ping);
metrics.Requests[1].Node.Port.Should().Be(9201);
metrics.Requests[1].RequestType.Should().Be(RequestType.Ping);
metrics.Requests[2].Node.Port.Should().Be(9200);
metrics.Requests[2].RequestType.Should().Be(RequestType.Ping);
metrics.Requests[3].Node.Port.Should().Be(9200);
metrics.Requests[3].RequestType.Should().Be(RequestType.Sniff);
metrics.Requests[4].Node.Port.Should().Be(9200);
metrics.Requests[4].RequestType.Should().Be(RequestType.ElasticsearchCall);
for (var i = 0; i < 3; i++)
{
rootNode = client.RootNodeInfo();
metrics = rootNode.ConnectionStatus.Metrics;
metrics.Requests[0].Node.Port.Should().Be(9200);
metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall);
metrics.Requests.Count.Should().Be(1);
}
}
public void SniffIsCalledAfterItHasGoneOutOfDate()
{
using (var fake = new AutoFake())
{
var dateTimeProvider = fake.Resolve <IDateTimeProvider>();
var nowCall = A.CallTo(() => dateTimeProvider.Now());
nowCall.ReturnsNextFromSequence(
DateTime.UtcNow, //initial sniff time (set even if not sniff_on_startup
DateTime.UtcNow, //info call 1
DateTime.UtcNow, //info call 2
DateTime.UtcNow.AddMinutes(10), //info call 3
DateTime.UtcNow.AddMinutes(10), //set now after sniff 3
DateTime.UtcNow.AddMinutes(20), //info call 4
DateTime.UtcNow.AddMinutes(20), //set now after sniff 4
DateTime.UtcNow.AddMinutes(22) //info call 5
);
var uris = new[] { new Uri("http://localhost:9200") };
var connectionPool = new SniffingConnectionPool(uris);
var config = new ConnectionConfiguration(connectionPool)
.SniffLifeSpan(TimeSpan.FromMinutes(4));
fake.Provide <IConnectionConfigurationValues>(config);
var transport = FakeCalls.ProvideDefaultTransport(fake, dateTimeProvider);
var connection = fake.Resolve <IConnection>();
var sniffCall = FakeCalls.Sniff(fake, config, uris);
var getCall = FakeCalls.GetSyncCall(fake);
getCall.Returns(FakeResponse.Ok(config));
var client1 = fake.Resolve <ElasticsearchClient>();
client1.Info(); //info call 1
client1.Info(); //info call 2
client1.Info(); //info call 3
client1.Info(); //info call 4
client1.Info(); //info call 5
sniffCall.MustHaveHappened(Repeated.Exactly.Twice);
nowCall.MustHaveHappened(Repeated.Exactly.Times(8));
}
}
/// <summary>
/// Sets up the <see cref="ConnectionSettings"/> using a collection of <see cref="Uri"/> to create a <see cref="SniffingConnectionPool"/>
/// </summary>
/// <param name="uris"></param>
/// <param name="configureSettings"></param>
/// <returns>ElasticSearchMessageQueueOptions</returns>
public ElasticSearchMessageQueueOptions <TMessage> UseConnectionPool(IEnumerable <Uri> uris, Action <ConnectionSettings> configureSettings)
{
if (uris is null)
{
throw new ArgumentNullException(nameof(uris));
}
if (!uris.Any())
{
throw new ArgumentException($"{nameof(uris)} cannot be empty");
}
if (configureSettings is null)
{
throw new ArgumentNullException(nameof(configureSettings));
}
var connPool = new SniffingConnectionPool(uris);
ConnectionSettings = new ConnectionSettings(connPool);
configureSettings(ConnectionSettings);
return(this);
}
private static ElasticLowLevelClient Initialize()
{
var uris = new[]
{
new Uri("http://localhost:9200"),
//new Uri("http://localhost:9201"),
//new Uri("http://localhost:9202"),
};
var connectionPool = new SniffingConnectionPool(uris);
var settings = new ConnectionConfiguration(connectionPool)
.RequestTimeout(TimeSpan.FromMinutes(2))
.ThrowExceptions()
.OnRequestCompleted(apiCallDetails =>
{
if (apiCallDetails.HttpStatusCode == 400)
{
throw new ElasticsearchClientException("you probably provided a malformed request");
}
});
return(new ElasticLowLevelClient(settings));
}
public static void Main(string[] args)
{
var hosts = new[] {
new Uri("https://iadd1-s10538-0.es.objectrocket.com:20538"),
new Uri("https://iadd1-s10538-1.es.objectrocket.com:20538"),
new Uri("https://iadd1-s10538-2.es.objectrocket.com:20538"),
new Uri("https://iadd1-s10538-3.es.objectrocket.com:20538")
};
try
{
var connectionPool = new SniffingConnectionPool(hosts);
var config = new ConnectionConfiguration(connectionPool)
.BasicAuthentication("sooz", "xxxxxxxx");
_client = new ElasticLowLevelClient(config);
// TODO: Figure out format of a client-level action to trigger actual connection.
_client.Ping((PingRequestParameters x, PingRequestParameters y) => x);
}
catch (Exception e) {
Console.WriteLine("Error: " + e);
}
Console.WriteLine("Done!");
}
public void SniffOnConnectionFaultCausesSniffOn503()
{
using (var fake = new AutoFake())
{
var dateTimeProvider = fake.Resolve <IDateTimeProvider>();
var nowCall = A.CallTo(() => dateTimeProvider.Now());
nowCall.Returns(DateTime.UtcNow);
var nodes = new[] { new Uri("http://localhost:9200") };
var connectionPool = new SniffingConnectionPool(nodes);
var config = new ConnectionConfiguration(connectionPool)
.SniffOnConnectionFault();
fake.Provide <IConnectionConfigurationValues>(config);
var transport = FakeCalls.ProvideDefaultTransport(fake, dateTimeProvider);
var connection = fake.Resolve <IConnection>();
var sniffCall = FakeCalls.Sniff(fake, config, nodes);
var getCall = FakeCalls.GetSyncCall(fake);
getCall.ReturnsNextFromSequence(
FakeResponse.Ok(config), //info 1
FakeResponse.Ok(config), //info 2
FakeResponse.Ok(config), //info 3
FakeResponse.Ok(config), //info 4
FakeResponse.Bad(config) //info 5
);
var client1 = fake.Resolve <ElasticsearchClient>();
client1.Info(); //info call 1
client1.Info(); //info call 2
client1.Info(); //info call 3
client1.Info(); //info call 4
Assert.Throws <MaxRetryException>(() => client1.Info()); //info call 5
sniffCall.MustHaveHappened(Repeated.Exactly.Once);
nowCall.MustHaveHappened(Repeated.Exactly.Times(7));
}
}
public EsClientProvider(IOptions <EsConfig> esConfig,
ILogger <EsClientProvider> logger)
{
try
{
var uris = esConfig.Value.Uris;
if (uris == null || uris.Count < 1)
{
throw new Exception("urls can not be null");
}
ConnectionSettings connectionSetting;
if (uris.Count == 1)
{
var uri = uris.First();
connectionSetting = new ConnectionSettings(uri);
}
else
{
var connectionPool = new SniffingConnectionPool(uris);
connectionSetting = new ConnectionSettings(connectionPool).DefaultIndex("");
}
if (!string.IsNullOrWhiteSpace(esConfig.Value.UserName) && !string.IsNullOrWhiteSpace(esConfig.Value.Password))
{
connectionSetting.BasicAuthentication(esConfig.Value.UserName, esConfig.Value.Password);
}
Client = new ElasticClient(connectionSetting);
}
catch (Exception ex)
{
logger.LogError(ex, "ElasticSearch Initialized failed.");
throw;
}
}
public void ShouldThrowAndNotRetrySniffOnStartup401()
{
using (var fake = new AutoFake(callsDoNothing: true))
{
var uris = new[]
{
new Uri("http://localhost:9200"),
new Uri("http://localhost:9201"),
new Uri("http://localhost:9202")
};
var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false);
var config = new ConnectionConfiguration(connectionPool)
.SniffOnStartup();
var pingCall = FakeCalls.PingAtConnectionLevel(fake);
pingCall.Returns(FakeResponse.Ok(config));
var sniffCall = FakeCalls.Sniff(fake);
var seenPorts = new List <int>();
sniffCall.ReturnsLazily((Uri u, IRequestConfiguration c) =>
{
seenPorts.Add(u.Port);
return(FakeResponse.Any(config, 401));
});
var getCall = FakeCalls.GetSyncCall(fake);
getCall.Returns(FakeResponse.Bad(config));
fake.Provide <IConnectionConfigurationValues>(config);
var e = Assert.Throws <ElasticsearchAuthenticationException>(() => FakeCalls.ProvideRealTranportInstance(fake));
sniffCall.MustHaveHappened(Repeated.Exactly.Once);
getCall.MustNotHaveHappened();
}
}
private void Initialize(ElasticSearchOutputConfiguration esOutputConfiguration)
{
Debug.Assert(esOutputConfiguration != null);
Debug.Assert(this.healthReporter != null);
this.connectionData = new ElasticSearchConnectionData
{
Configuration = esOutputConfiguration
};
var esServiceUri = esOutputConfiguration.ServiceUri
.Split(';')
.Where(x => Uri.IsWellFormedUriString(x, UriKind.Absolute))
.Select(x => new Uri(x))
.ToList();
string errorMessage;
if (!esServiceUri.Any())
{
errorMessage =
$"{nameof(ElasticSearchOutput)}: required 'serviceUri' configuration parameter is invalid";
this.healthReporter.ReportProblem(errorMessage, EventFlowContextIdentifiers.Configuration);
throw new Exception(errorMessage);
}
string userName = esOutputConfiguration.BasicAuthenticationUserName;
string password = esOutputConfiguration.BasicAuthenticationUserPassword;
bool credentialsIncomplete = string.IsNullOrWhiteSpace(userName) ^ string.IsNullOrWhiteSpace(password);
if (credentialsIncomplete)
{
errorMessage = $"{nameof(ElasticSearchOutput)}: for basic authentication to work both user name and password must be specified";
healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Configuration);
userName = password = null;
}
var pool = new SniffingConnectionPool(esServiceUri);
ConnectionSettings connectionSettings = new ConnectionSettings(pool);
if (!string.IsNullOrWhiteSpace(userName) && !string.IsNullOrWhiteSpace(password))
{
connectionSettings = connectionSettings.BasicAuthentication(userName, password);
}
this.connectionData.Client = new ElasticClient(connectionSettings);
this.connectionData.LastIndexName = null;
if (string.IsNullOrWhiteSpace(esOutputConfiguration.IndexNamePrefix))
{
esOutputConfiguration.IndexNamePrefix = string.Empty;
}
else
{
string lowerCaseIndexNamePrefix = esOutputConfiguration.IndexNamePrefix.ToLowerInvariant();
if (lowerCaseIndexNamePrefix != esOutputConfiguration.IndexNamePrefix)
{
healthReporter.ReportWarning($"{nameof(ElasticSearchOutput)}: The chosen index name prefix '{esOutputConfiguration.IndexNamePrefix}' "
+ "contains uppercase characters, which are not allowed by Elasticsearch. The prefix will be converted to lowercase.",
EventFlowContextIdentifiers.Configuration);
}
esOutputConfiguration.IndexNamePrefix = lowerCaseIndexNamePrefix + Dash;
}
if (string.IsNullOrWhiteSpace(esOutputConfiguration.EventDocumentTypeName))
{
string warning = $"{nameof(ElasticSearchOutput)}: '{nameof(ElasticSearchOutputConfiguration.EventDocumentTypeName)}' configuration parameter "
+ "should not be empty";
healthReporter.ReportWarning(warning, EventFlowContextIdentifiers.Configuration);
esOutputConfiguration.EventDocumentTypeName = ElasticSearchOutputConfiguration.DefaultEventDocumentTypeName;
}
}
public async void SniffingConnectionPool_DoesNotThrowOrRetry_On_401_Async()
{
var nodes = new Uri[] {
new Uri("http://*****:*****@localhost:9200"),
new Uri("http://*****:*****@localhost:9201")
};
var pool = new SniffingConnectionPool(nodes);
var settings = new ConnectionSettings(pool)
.MaximumRetries(3);
var client = new ElasticClient(settings);
var r = await client.RootNodeInfoAsync();
r.IsValid.Should().BeFalse();
r.ConnectionStatus.HttpStatusCode.Should().Be(401);
r.ConnectionStatus.Metrics.Requests.Count.Should().Be(1);
r.ConnectionStatus.NumberOfRetries.Should().Be(0);
}
public void SniffingConnectionPool_ThrowsAndDoesNotRetry_On_401_Async()
{
var nodes = new Uri[] {
new Uri("http://*****:*****@localhost:9200"),
new Uri("http://*****:*****@localhost:9201")
};
var pool = new SniffingConnectionPool(nodes);
var settings = new ConnectionSettings(pool)
.ThrowOnElasticsearchServerExceptions()
.MaximumRetries(3);
var client = new ElasticClient(settings);
var e = Assert.Throws<ElasticsearchServerException>(async () => await client.RootNodeInfoAsync());
e.Status.Should().Be(401);
}
/// <summary>
/// Configures the services.
/// </summary>
/// <returns>The services.</returns>
/// <param name="services">Services.</param>
public void ConfigureServices(IServiceCollection services)
{
/*
* Do we still need this? Or is this .Net 1.1 code?
*/
services.AddLogging();
//Turn on the OptionsManager that supports IOptions
services.AddOptions();
//Add Configuration mappings.
services.Configure <ElasticsearchOptions>(Configuration.GetSection("Elasticsearch"));
AddAdditionalConfigurationMappings(services);
/*
* Do we still need this? Or is this Best Bets code which never should
* have been in the base class to begin with?
*/
//Add HttpClient singleton, which is used by the display service.
services.AddSingleton <HttpClient, HttpClient>();
// This will inject an IElasticClient using our configuration into any
// controllers that take an IElasticClient parameter into its constructor.
//
// AddTransient means that it will instantiate a new instance of our client
// for each instance of the controller. So the function below will be called
// on each request.
services.AddTransient <IElasticClient>(p =>
{
// Get the ElasticSearch credentials.
string username = Configuration["Elasticsearch:Userid"];
string password = Configuration["Elasticsearch:Password"];
//Get the ElasticSearch servers that we will be connecting to.
List <Uri> uris = GetElasticsearchServerUriList();
// Create the connection pool, the SniffingConnectionPool will
// keep tabs on the health of the servers in the cluster and
// probe them to ensure they are healthy. This is how we handle
// redundancy and load balancing.
var connectionPool = new SniffingConnectionPool(uris);
//Return a new instance of an ElasticClient with our settings
ConnectionSettings settings = new ConnectionSettings(connectionPool, sourceSerializer: JsonNetSerializer.Default);
//Let's only try and use credentials if the username is set.
if (!string.IsNullOrWhiteSpace(username))
{
settings.BasicAuthentication(username, password);
}
return(new ElasticClient(settings));
});
//Add in Application specific services
AddAppServices(services);
// Create CORS policies.
services.AddCors();
// Make the application's routes available.
services.AddControllers()
.AddNewtonsoftJson();
// Enable Swagger
// This creates the Swagger Json
services.AddOpenApiDocument(config =>
{
if (!string.IsNullOrEmpty(Configuration["NSwag:Title"]))
{
config.Title = Configuration["NSwag:Title"];
}
if (!string.IsNullOrEmpty(Configuration["NSwag:Description"]))
{
config.Description = Configuration["NSwag:Description"];
}
});
}
public void HostsReturnedBySniffAreVisited()
{
using (var fake = new AutoFake())
{
var dateTimeProvider = fake.Resolve <IDateTimeProvider>();
var nowCall = A.CallTo(() => dateTimeProvider.Now());
nowCall.Returns(DateTime.UtcNow);
var connectionPool = new SniffingConnectionPool(new[]
{
new Uri("http://localhost:9200"),
new Uri("http://localhost:9201")
}, randomizeOnStartup: false);
var config = new ConnectionConfiguration(connectionPool)
.SniffOnConnectionFault();
fake.Provide <IConnectionConfigurationValues>(config);
FakeCalls.ProvideDefaultTransport(fake);
FakeCalls.PingAtConnectionLevel(fake)
.Returns(FakeResponse.Ok(config));
var sniffCall = FakeCalls.Sniff(fake, config, new List <Uri>()
{
new Uri("http://localhost:9204"),
new Uri("http://localhost:9203"),
new Uri("http://localhost:9202"),
new Uri("http://localhost:9201")
});
var connection = fake.Resolve <IConnection>();
var seenNodes = new List <Uri>();
//var getCall = FakeResponse.GetSyncCall(fake);
var getCall = A.CallTo(() => connection.GetSync(
A <Uri> .That.Not.Matches(u => u.AbsolutePath.StartsWith("/_nodes")),
A <IRequestConfiguration> ._));
getCall.ReturnsNextFromSequence(
FakeResponse.Ok(config), //info 1
FakeResponse.Bad(config), //info 2
FakeResponse.Ok(config), //info 2 retry
FakeResponse.Ok(config), //info 3
FakeResponse.Ok(config), //info 4
FakeResponse.Ok(config), //info 5
FakeResponse.Ok(config), //info 6
FakeResponse.Ok(config), //info 7
FakeResponse.Ok(config), //info 8
FakeResponse.Ok(config) //info 9
);
getCall.Invokes((Uri u, IRequestConfiguration o) => seenNodes.Add(u));
var client1 = fake.Resolve <ElasticsearchClient>();
client1.Info(); //info call 1
client1.Info(); //info call 2
client1.Info(); //info call 3
client1.Info(); //info call 4
client1.Info(); //info call 5
client1.Info(); //info call 6
client1.Info(); //info call 7
client1.Info(); //info call 8
client1.Info(); //info call 9
sniffCall.MustHaveHappened(Repeated.Exactly.Once);
seenNodes.Should().NotBeEmpty().And.HaveCount(10);
seenNodes[0].Port.Should().Be(9200);
seenNodes[1].Port.Should().Be(9201);
//after sniff
seenNodes[2].Port.Should().Be(9202, string.Join(",", seenNodes.Select(n => n.Port)));
seenNodes[3].Port.Should().Be(9204);
seenNodes[4].Port.Should().Be(9203);
seenNodes[5].Port.Should().Be(9202);
seenNodes[6].Port.Should().Be(9201);
}
}
private static ElasticClient GetClient(IList <string> endpoints,
string defaultIndex,
ConnectionConfig config,
ConnectionPool connectionPool)
{
if (!endpoints.HasItems())
{
throw new ArgumentException(string.Format(Utils.ARGUMENT_EMPTY_LIST_MESSAGE, nameof(endpoints)),
nameof(endpoints));
}
var uris = endpoints.Select(x => new Uri(x));
IConnectionPool pool;
switch (connectionPool)
{
case ConnectionPool.SingleNode:
pool = new SingleNodeConnectionPool(new Uri(endpoints.First()));
break;
case ConnectionPool.Static:
pool = new StaticConnectionPool(uris);
break;
case ConnectionPool.Sniffing:
pool = new SniffingConnectionPool(uris);
break;
case ConnectionPool.Sticky:
pool = new StickyConnectionPool(uris);
break;
default:
throw new ArgumentOutOfRangeException(nameof(connectionPool), connectionPool, null);
}
var connectionSettings = new ConnectionSettings(pool)
.ConnectionLimit(config?.ConnectionLimit > 0 ? config.ConnectionLimit : 80)
.DisableAutomaticProxyDetection(config?.DisableAutomaticProxyDetection ?? false)
.DisableDirectStreaming(config?.EnableDebugMode ?? false)
.DisablePing(config?.DisablePings ?? false)
.DefaultIndex(defaultIndex)
.EnableHttpCompression(config?.EnableHttpCompression ?? true)
.EnableHttpPipelining(config?.EnableHttpPipelining ?? true)
.EnableTcpKeepAlive(config?.KeepAliveTime.Ticks > 0
? config.KeepAliveTime
: TimeSpan.FromMilliseconds(2000),
config?.KeepAliveInterval.Ticks > 0
? config.KeepAliveInterval
: TimeSpan.FromMilliseconds(2000))
.MaximumRetries(config?.MaxRetries > 0 ? config.MaxRetries : 10)
.MaxRetryTimeout(
config?.MaxRetryTimeout.Ticks > 0 ? config.MaxRetryTimeout : TimeSpan.FromSeconds(60))
.PrettyJson(config?.EnableDebugMode ?? false)
.RequestTimeout(config?.RequestTimeout.Ticks > 0
? config.RequestTimeout
: TimeSpan.FromSeconds(60))
.SniffLifeSpan(config?.SniffLifeSpan.Ticks > 0
? config.SniffLifeSpan
: TimeSpan.FromHours(1))
.SniffOnConnectionFault(config?.SniffsOnConnectionFault ?? true)
.SniffOnStartup(config?.SniffsOnStartup ?? true)
.ThrowExceptions(config?.ThrowExceptions ?? false);
if (!string.IsNullOrWhiteSpace(config?.BasicAuthenticationUsername) &&
!string.IsNullOrWhiteSpace(config.BasicAuthenticationPassword))
{
connectionSettings.BasicAuthentication(config.BasicAuthenticationUsername, config.BasicAuthenticationPassword);
}
if (config?.ClientCerfificates != null && config.ClientCerfificates.HasItems())
{
connectionSettings.ClientCertificates(config.ClientCerfificates);
}
else if (config?.ClientCerfificate != null)
{
connectionSettings.ClientCertificate(config.ClientCerfificate);
}
if (config?.DeadTimeout.Ticks > 0)
{
connectionSettings.DeadTimeout(config.DeadTimeout);
}
if (config?.Headers != null && config.Headers.HasItems())
{
connectionSettings.GlobalHeaders(config.Headers);
}
if (config?.QueryStringParameters != null && config.QueryStringParameters.HasItems())
{
connectionSettings.GlobalQueryStringParameters(config.QueryStringParameters);
}
if (config?.MaxDeadTimeout.Ticks > 0)
{
connectionSettings.MaxDeadTimeout(config.DeadTimeout);
}
if (config?.NodePredicate != null)
{
connectionSettings.NodePredicate(config.NodePredicate);
}
if (config?.RequestCompletedHandler != null)
{
connectionSettings.OnRequestCompleted(config.RequestCompletedHandler);
}
if (config?.RequestDataCreatedHandler != null)
{
connectionSettings.OnRequestDataCreated(config.RequestDataCreatedHandler);
}
if (config?.PingTimeout.Ticks > 0)
{
connectionSettings.PingTimeout(config.PingTimeout);
}
if (!string.IsNullOrWhiteSpace(config?.ProxyAddress))
{
connectionSettings.Proxy(new Uri(config.ProxyAddress), config?.ProxyUsername, config?.ProxyPassword);
}
if (config?.ServerCertificateValidationCallback != null)
{
connectionSettings.ServerCertificateValidationCallback(config.ServerCertificateValidationCallback);
}
return(new ElasticClient(connectionSettings));
}
static void Main(string[] args)
{
var setting = new ConnectionConfiguration(new Uri("http://120.27.213.67:9200/")).RequestTimeout(TimeSpan.FromMinutes(2));
//setting.BasicAuthentication("", "");
var lowlevelClient = new ElasticLowLevelClient(setting);
var uris = new List <Uri>
{
new Uri("http://localhost:9200"),
new Uri("http://120.27.213.67:9200/"),
};
var connectPool = new SniffingConnectionPool(uris);
var settings1 = new ConnectionConfiguration(connectPool);
var lowlevelClient1 = new ElasticLowLevelClient(settings1);
//var person = new Person
//{
// FirstName = "Martijn",
// LastName = "Laarman"
//};
//var indexRespose = lowlevelClient.Index<BytesResponse>("people", "1", PostData.Serializable(person));
//byte[] responseBytes = indexRespose.Body;
//var asyncIndexResponse = await lowlevelClient.IndexAsync<StringResponse>("people", "1", PostData.Serializable(person));
//string responseString = asyncIndexResponse.Body;
//var peoples = new object[]
//{
// new { index = new { _index = "people", _type = "_doc", _id = "1" }},
// new { FirstName = "Martijn", LastName = "Laarman" },
// new { index = new { _index = "people", _type = "_doc", _id = "2" }},
// new { FirstName = "Greg", LastName = "Marzouka" },
// new { index = new { _index = "people", _type = "_doc", _id = "3" }},
// new { FirstName = "Russ", LastName = "Cam" },
//};
//var ndexResponse = lowlevelClient.Bulk<StringResponse>(PostData.MultiJson(peoples));
//string responseStream = ndexResponse.Body;
//var searchResponse = lowlevelClient.Search<StringResponse>("people", PostData.Serializable(new
//{
// from = 0,
// size = 10,
// query = new
// {
// match = new
// {
// firstName = new
// {
// query = "Martijn"
// }
// }
// }
//}));
//var successful = searchResponse.Success;
//var responseJson = searchResponse.Body;
//var searchResponse1 = lowlevelClient.Search<BytesResponse>("people", PostData.Serializable(new { match_all = new { } }));
//var success = searchResponse1.Success;
//var successOrKnownError = searchResponse1.SuccessOrKnownError;
//var exception = searchResponse1.OriginalException;
}
static ElasticConnector()
{
_sniffingConnectionPool = new SniffingConnectionPool(GetUris());
_connectionSettings = new ConnectionSettings(_sniffingConnectionPool).DisableDirectStreaming();
}
/// <summary>
/// Configures the services.
/// </summary>
/// <returns>The services.</returns>
/// <param name="services">Services.</param>
public IServiceProvider ConfigureServices(IServiceCollection services)
{
services.AddLogging();
//Turn on the OptionsManager that supports IOptions
services.AddOptions();
services.AddCors();
//This allows us to easily generate URLs to routes
services.AddSingleton <IActionContextAccessor, ActionContextAccessor>();
services.AddScoped <IUrlHelper>(factory =>
{
var actionContext = factory.GetService <IActionContextAccessor>()
.ActionContext;
return(new UrlHelper(actionContext));
});
//Add Configuration mappings.
// services.Configure<NSwagOptions>(Configuration.GetSection("NSwag"));
services.Configure <ElasticsearchOptions>(Configuration.GetSection("Elasticsearch"));
AddAdditionalConfigurationMappings(services);
//Add HttpClient singleton, which is used by the display service.
services.AddSingleton <HttpClient, HttpClient>();
// This will inject an IElasticClient using our configuration into any
// controllers that take an IElasticClient parameter into its constructor.
//
// AddTransient means that it will instantiate a new instance of our client
// for each instance of the controller. So the function below will be called
// on each request.
services.AddTransient <IElasticClient>(p => {
// Get the ElasticSearch credentials.
string username = Configuration["Elasticsearch:Userid"];
string password = Configuration["Elasticsearch:Password"];
//Get the ElasticSearch servers that we will be connecting to.
List <Uri> uris = GetServerUriList();
// Create the connection pool, the SniffingConnectionPool will
// keep tabs on the health of the servers in the cluster and
// probe them to ensure they are healthy. This is how we handle
// redundancy and load balancing.
var connectionPool = new SniffingConnectionPool(uris);
//Return a new instance of an ElasticClient with our settings
ConnectionSettings settings = new ConnectionSettings(connectionPool);
//Let's only try and use credentials if the username is set.
if (!string.IsNullOrWhiteSpace(username))
{
settings.BasicAuthentication(username, password);
}
return(new ElasticClient(settings));
});
//Add in Application specific services
AddAppServices(services);
// Create CORS policies.
services.AddCors();
// Add framework services.
services.AddMvc();
// Enable Swagger
// This creates the Swagger Json
services.AddOpenApiDocument(config => {
if (!string.IsNullOrEmpty(Configuration["NSwag:Title"]))
{
config.Title = Configuration["NSwag:Title"];
}
if (!string.IsNullOrEmpty(Configuration["NSwag:Description"]))
{
config.Description = Configuration["NSwag:Description"];
}
});
// Add framework services
return(services.BuildServiceProvider());
}
/// <summary>
/// Gets the search results from this SiteWideSearch provider.
/// </summary>
/// <param name="searchCollection">The search collection.</param>
/// <param name="term">The term.</param>
/// <param name="pageSize">Size of the page.</param>
/// <param name="offset">The offset.</param>
/// <returns></returns>
public override ISiteWideSearchResultCollection GetSearchResults(string searchCollection, string term, int pageSize, int offset)
{
ESSiteWideSearchCollectionElement searchCollConfig = ElasticSearchConfig.GetSearchCollectionConfig(searchCollection);
ClusterElement clusterConfig = searchCollConfig.ClusterElementDetails;
var connectionPool = new SniffingConnectionPool(searchCollConfig.ClusterNodes);
var config = new ConnectionConfiguration(connectionPool)
.UsePrettyResponses()
.MaximumRetries(clusterConfig.MaximumRetries)//try 5 times if the server is down
.ExposeRawResponse()
.ThrowOnElasticsearchServerExceptions();
var esClient = new ElasticsearchClient(config);
string[] fieldList = ElasticSearchConfig.GetFields(searchCollection);
//do search_template
var _body = new
{
template = new
{
id = searchCollConfig.Template
},
@params = new
{
my_value = term,
my_size = pageSize.ToString(),
my_from = offset.ToString(),
my_site = searchCollConfig.Site,
my_fields = fieldList
}
};
ElasticsearchResponse <DynamicDictionary> results = null;
try
{
results = esClient.SearchTemplate(searchCollConfig.Index, _body);
}
catch (MaxRetryException ex)
{
try
{
//log the maximum retry excpetion
log.Error("Error using the ESClient Search Template method. Maximum retry exception: ", ex);
//sleep for 5 seconds
Thread.Sleep(clusterConfig.ConnectionTimeoutDelay);
//try to fetch results again
results = esClient.SearchTemplate(searchCollConfig.Index, _body);
}
catch (Exception e)
{
log.Error("Error using the ESClient Search Template method.", e);
throw e;
}
}//retry catch log error sleep retry
List <ESSiteWideSearchResult> foundTerms = new List <ESSiteWideSearchResult>(pageSize);
if (results.Success)
{
var stringResponse = results.Response;
try
{
foreach (var hit in results.Response["hits"].hits)
{
//title is a special case and when it is empty the value needs to be Untitled
string title = SetFieldValue(hit, "title");
if (string.IsNullOrEmpty(title))
{
title = "Untitled";
}
string url = SetFieldValue(hit, "url");
string description = SetFieldValue(hit, "metatag.description");
string type = SetFieldValue(hit, "metatag.dcterms.type");
foundTerms.Add(new ESSiteWideSearchResult(title, url, description, type));
}
}
catch (Exception ex)
{
log.Error("Error retrieving search results.", ex);
throw ex;
}
}
else
{
string prettyResponse = "";
//log the raw response message
//if there is not response them log with message - "No message in response"
if (!string.IsNullOrEmpty(System.Text.Encoding.Default.GetString(results.ResponseRaw)))
{
prettyResponse = System.Text.Encoding.Default.GetString(results.ResponseRaw);
}
else
{
prettyResponse = "No message in response";
}
log.ErrorFormat("Search failed. Http Status Code: {0}. Message: {1}", results.HttpStatusCode.Value, prettyResponse);
throw (new Exception("Search failed. Http Status Code:" + results.HttpStatusCode.Value.ToString() + ". Message: " + prettyResponse));
}
//Return the results
return(new ESSiteWideSearchResultCollection(foundTerms.AsEnumerable())
{
ResultCount = results.Response["hits"].total
});
}
public void NodesDiscoveredDuringSniffShouldBeHttps()
{
ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) =>
{
if (errors == SslPolicyErrors.None)
return true;
string certificateHash = certificate.GetCertHashString();
bool knownThumbprintIsValid = false;
if (_knownPrints.TryGetValue(certificateHash, out knownThumbprintIsValid))
return knownThumbprintIsValid;
var isValid = IsValidCertificate(certificate, chain);
_knownPrints.AddOrUpdate(certificateHash, isValid, (s, b) => isValid);
return isValid;
};
var uris = new[]
{
new Uri("https://localhost:9200")
};
var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false);
var settings = new ConnectionSettings(connectionPool, ElasticsearchConfiguration.DefaultIndex)
.SniffOnStartup()
.SetBasicAuthentication("mpdreamz", "blahblah")
.ExposeRawResponse()
//.SetPingTimeout(1000)
.SetTimeout(2000);
var client = new ElasticClient(settings);
var results = client.NodesInfo();
results.IsValid.Should().BeTrue("{0}", results.ConnectionStatus.ToString());
results.ConnectionStatus.NumberOfRetries.Should().Be(0);
var uri = new Uri(results.ConnectionStatus.RequestUrl);
uri.Port.Should().Be(9200);
uri.Scheme.Should().Be("https");
results = client.NodesInfo();
results.IsValid.Should().BeTrue("{0}", results.ConnectionStatus.ToString());
results.ConnectionStatus.NumberOfRetries.Should().Be(0);
uri = new Uri(results.ConnectionStatus.RequestUrl);
uri.Port.Should().Be(9201);
uri.Scheme.Should().Be("https");
}
static void Main(string[] args)
{
Program p=new Program();
const string elUri = "http://*****:*****@localhost:9200/";
Node = new Uri(elUri);
var connectionPool = new SniffingConnectionPool(new[] { Node });
Settings = new ConnectionSettings(Node);
var config = new ConnectionConfiguration(connectionPool)
.EnableTrace()
.ExposeRawResponse()
.SetBasicAuthentication("aotuo", "123456")
.SetTimeout(4000);
Client = new ElasticClient(Settings);
/*var newPost = new activitylog
{
IpAddress = "21",
Time = DateTime.Now.ToString(CultureInfo.CurrentCulture),
Desciption = "azure"
};
Client.Index(newPost, i => i
.Index("azure")
.Type("activitylog")
);
var indexsettings = new IndexSettings();
try
{
var res = Client.Count(c=>c.AllIndices().Type("logs"));
var res2 =Client.Search<object>(s => s.AllIndices()
.From(0)
.Size(50)
.Query(q =>q.Term("ident","BingLi"))
);
Console.WriteLine(res.Count);
Escount = (int)res.Count;
}
catch (Exception)
{
Console.WriteLine("error");
}
Termquery();*/
Console.ReadLine();
}
private static void Main(string[] args)
{
using (SqlConnection conn = new SqlConnection("Data Source=192.168.30.128;Initial Catalog=SmartBusExtension;User Id=sa;Password=1qaz2wsx123_@;Connect Timeout=120"))
{
SqlCommand cmd = new SqlCommand(@"
SELECT * FROM MQUser"
, conn);
List <SyncArrayConfiguration> arrayConfig = new List <SyncArrayConfiguration>()
{
new SyncArrayConfiguration
{
SqlCommand = new SqlCommand(@"
SELECT id, userid, login,username,avatar
FROM MQUser
where UserName='******'"
, conn),
AttributeName = "tags"
},
/**
* new SyncArrayConfiguration
* {
* SqlCommand = new SqlCommand(@"
* SELECT id_object AS '_id', id, description, xmlData, languageId
* FROM dbo.Categories
* WHERE languageId = 'es'"
* , conn),
* AttributeName = "categories",
* XmlFields = new string[] { "xmlData" }
* }
**/
};
SyncDeleteConfiguration deleteCmd = new SyncDeleteConfiguration
{
SqlCommand = new SqlCommand(@"
SELECT id_object AS '_id', createdOn
FROM dbo.DeleteLog
WHERE id_language = 'es'"
, conn),
ColumnsToCompareWithLastSyncDate = new string[] { "[createdOn]" },
};
var nodes =
new Uri[] {
new Uri("http://192.168.30.135:9200"),
//new Uri("http://localhost:9201")
};
var connectionPool = new SniffingConnectionPool(nodes);
var esConfig = new ConnectionConfiguration(connectionPool).UsePrettyResponses();
var syncConfig = new SyncConfiguration()
{
SqlConnection = conn,
SqlCommand = cmd,
ArraysConfiguration = arrayConfig,
ColumnsToCompareWithLastSyncDate = new string[] { "[id]" },
//DeleteConfiguration = deleteCmd,
ElasticSearchConfiguration = esConfig,
BulkSize = 500,
_Index = new Index("mssql_30.128"),
_Type = "notes"
};
var sync = new Sync(syncConfig);
try
{
var response = sync.Exec();
if (response.BulkResponses.Any())
{
foreach (var bulkResponse in response.BulkResponses)
{
Console.WriteLine("success: " + bulkResponse.Success);
Console.WriteLine("http status code: " + bulkResponse.HttpStatusCode);
Console.WriteLine("affected documents: " + bulkResponse.AffectedDocuments);
Console.WriteLine("started on: " + bulkResponse.StartedOn);
Console.WriteLine("bulk duration: " + bulkResponse.Duration + "ms");
if (!response.Success)
{
Console.WriteLine("es original exception: " + bulkResponse.ESexception);
}
Console.WriteLine("\n");
}
Console.WriteLine("\nbulk avg duration: " + response.BulkResponses.Average(x => x.Duration) + "ms");
Console.WriteLine("\n");
}
}
catch (Exception ex)
{
Console.WriteLine("an error has occurred: " + ex.Message);
}
finally
{
Console.WriteLine("Execution has completed. Press any key to continue...");
Console.ReadKey();
}
}
}
public void CreateIndex(string name, string elUri)
{
Node = new Uri(elUri);
var connectionPool = new SniffingConnectionPool(new[] { Node });
Settings = new ConnectionSettings(Node);
Client = new ElasticClient(Settings);
var indexsettings = new IndexSettings();
indexsettings.NumberOfReplicas = 1;
indexsettings.NumberOfShards = 5;
Client.CreateIndex(c => c
.Index(name)
.InitializeUsing(indexsettings)
.AddMapping<activitylog>(m => m.MapFromAttributes()));
}
