public async void SniffOnStartShouldOnlyHit9200_WithoutPing_Async() { var seeds = new[] { ElasticsearchConfiguration.CreateBaseUri(9202), ElasticsearchConfiguration.CreateBaseUri(9201), ElasticsearchConfiguration.CreateBaseUri(9200) }; var sniffingConnectionPool = new SniffingConnectionPool(seeds, randomizeOnStartup: false); var connectionSettings = new ConnectionSettings(sniffingConnectionPool) .SniffOnStartup(); var client = new ElasticClient(connectionSettings); var rootNode = await client.RootNodeInfoAsync(); var metrics = rootNode.ConnectionStatus.Metrics; //When the connectionpool is used for the first time the sniff call should already //know only 9200 is on and live, no need to ping metrics.Requests.Count.Should().Be(1); metrics.Requests[0].Node.Port.Should().Be(9200); metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall); for (var i = 0; i < 3; i++) { rootNode = await client.RootNodeInfoAsync(); metrics = rootNode.ConnectionStatus.Metrics; metrics.Requests.Count.Should().Be(1); metrics.Requests[0].Node.Port.Should().Be(9200); metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall); } }
public void FailoverShouldOnlyPingDeadNodes() { var seeds = new[] { ElasticsearchConfiguration.CreateBaseUri(9202), ElasticsearchConfiguration.CreateBaseUri(9201), ElasticsearchConfiguration.CreateBaseUri(9200) }; var sniffingConnectionPool = new SniffingConnectionPool(seeds, randomizeOnStartup: false); var connectionSettings = new ConnectionSettings(sniffingConnectionPool); var client = new ElasticClient(connectionSettings); var rootNode = client.RootNodeInfo(); var metrics = rootNode.ConnectionStatus.Metrics; //ping 9202 + 9201 + 9200 and call 9200 metrics.Requests.Count.Should().Be(4); metrics.Requests[0].Node.Port.Should().Be(9202); metrics.Requests[0].RequestType.Should().Be(RequestType.Ping); metrics.Requests[0].EllapsedMilliseconds.Should().BeLessOrEqualTo(1100); metrics.Requests[1].Node.Port.Should().Be(9201); metrics.Requests[1].RequestType.Should().Be(RequestType.Ping); metrics.Requests[1].EllapsedMilliseconds.Should().BeLessOrEqualTo(1100); metrics.Requests[2].Node.Port.Should().Be(9200); metrics.Requests[2].RequestType.Should().Be(RequestType.Ping); metrics.Requests[2].EllapsedMilliseconds.Should().BeLessOrEqualTo(1100); metrics.Requests[3].Node.Port.Should().Be(9200); metrics.Requests[3].RequestType.Should().Be(RequestType.ElasticsearchCall); metrics.Requests[3].EllapsedMilliseconds.Should().BeLessOrEqualTo(1100); rootNode = client.RootNodeInfo(); metrics = rootNode.ConnectionStatus.Metrics; metrics.Requests.Count.Should().Be(1); metrics.Requests[0].Node.Port.Should().Be(9200); metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall); rootNode = client.RootNodeInfo(); metrics = rootNode.ConnectionStatus.Metrics; metrics.Requests.Count.Should().Be(1); metrics.Requests[0].Node.Port.Should().Be(9200); metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall); rootNode = client.RootNodeInfo(); metrics = rootNode.ConnectionStatus.Metrics; metrics.Requests.Count.Should().Be(1); metrics.Requests[0].Node.Port.Should().Be(9200); metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall); rootNode = client.RootNodeInfo(); metrics = rootNode.ConnectionStatus.Metrics; metrics.Requests.Count.Should().Be(1); metrics.Requests[0].Node.Port.Should().Be(9200); metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall); }
public void IndexExistShouldNotThrowOn404() { var host = Test.Default.Host; if (Process.GetProcessesByName("fiddler").Any()) host = "ipv4.fiddler"; var connectionPool = new SniffingConnectionPool(new[] { new Uri("http://{0}:9200".F(host)) }); var settings = new ConnectionSettings(connectionPool, ElasticsearchConfiguration.DefaultIndex) .SniffOnStartup(); var client = new ElasticClient(settings); }
public async void SniffOnFaultShouldGetCleanClusterState_Async() { var seeds = new[] { ElasticsearchConfiguration.CreateBaseUri(9202), ElasticsearchConfiguration.CreateBaseUri(9201), ElasticsearchConfiguration.CreateBaseUri(9200), }; var sniffingConnectionPool = new SniffingConnectionPool(seeds, randomizeOnStartup: false); var connectionSettings = new ConnectionSettings(sniffingConnectionPool) .SniffOnConnectionFault(); var client = new ElasticClient(connectionSettings); var rootNode = await client.RootNodeInfoAsync(); var metrics = rootNode.ConnectionStatus.Metrics; metrics.Requests.Count.Should().Be(5); metrics.Requests[0].Node.Port.Should().Be(9202); metrics.Requests[0].RequestType.Should().Be(RequestType.Ping); metrics.Requests[1].Node.Port.Should().Be(9201); metrics.Requests[1].RequestType.Should().Be(RequestType.Ping); metrics.Requests[2].Node.Port.Should().Be(9200); metrics.Requests[2].RequestType.Should().Be(RequestType.Ping); metrics.Requests[3].Node.Port.Should().Be(9200); metrics.Requests[3].RequestType.Should().Be(RequestType.Sniff); metrics.Requests[4].Node.Port.Should().Be(9200); metrics.Requests[4].RequestType.Should().Be(RequestType.ElasticsearchCall); for (var i = 0; i < 3; i++) { rootNode = await client.RootNodeInfoAsync(); metrics = rootNode.ConnectionStatus.Metrics; metrics.Requests.Count.Should().Be(1); metrics.Requests[0].Node.Port.Should().Be(9200); metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall); } }
public void ShouldThrowAndNotRetrySniffOnConnectionFault403() { using (var fake = new AutoFake(callsDoNothing: true)) { var uris = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201"), new Uri("http://localhost:9202") }; var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool) .SniffOnConnectionFault() .ThrowOnElasticsearchServerExceptions(); fake.Provide <IConnectionConfigurationValues>(config); FakeCalls.ProvideDefaultTransport(fake); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake); var seenPorts = new List <int>(); sniffCall.ReturnsLazily((Uri u, IRequestConfiguration c) => { seenPorts.Add(u.Port); return(FakeResponse.Any(config, 403)); }); var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.Bad(config)); var client = fake.Resolve <ElasticsearchClient>(); var e = Assert.Throws <ElasticsearchServerException>(() => client.Info()); e.Status.Should().Be(403); sniffCall.MustHaveHappened(Repeated.Exactly.Once); getCall.MustHaveHappened(Repeated.Exactly.Once); } }
public void SniffOnStartupCallsSniffOnlyOnce() { using (var fake = new AutoFake(callsDoNothing: true)) { //It's recommended to only have on instance of your connection pool //Be sure to register it as Singleton in your IOC var uris = new[] { new Uri("http://localhost:9200") }; var connectionPool = new SniffingConnectionPool(uris); var config = new ConnectionConfiguration(connectionPool) .DisablePing() .SniffOnStartup(); fake.Provide <IConnectionConfigurationValues>(config); var sniffCall = FakeCalls.Sniff(fake, config, uris); var transport = FakeCalls.ProvideDefaultTransport(fake); var client1 = new ElasticsearchClient(config, transport: transport); var client2 = new ElasticsearchClient(config, transport: transport); var client3 = new ElasticsearchClient(config, transport: transport); var client4 = new ElasticsearchClient(config, transport: transport); sniffCall.MustHaveHappened(Repeated.Exactly.Once); } }
public void SniffOnFaultShouldGetCleanClusterState() { var seeds = new[] { ElasticsearchConfiguration.CreateBaseUri(9202), ElasticsearchConfiguration.CreateBaseUri(9201), ElasticsearchConfiguration.CreateBaseUri(9200), }; var sniffingConnectionPool = new SniffingConnectionPool(seeds, randomizeOnStartup: false); var connectionSettings = new ConnectionSettings(sniffingConnectionPool) .SniffOnConnectionFault(); var client = new ElasticClient(connectionSettings); var rootNode = client.RootNodeInfo(); var metrics = rootNode.ConnectionStatus.Metrics; metrics.Requests.Count.Should().Be(5); metrics.Requests[0].Node.Port.Should().Be(9202); metrics.Requests[0].RequestType.Should().Be(RequestType.Ping); metrics.Requests[1].Node.Port.Should().Be(9201); metrics.Requests[1].RequestType.Should().Be(RequestType.Ping); metrics.Requests[2].Node.Port.Should().Be(9200); metrics.Requests[2].RequestType.Should().Be(RequestType.Ping); metrics.Requests[3].Node.Port.Should().Be(9200); metrics.Requests[3].RequestType.Should().Be(RequestType.Sniff); metrics.Requests[4].Node.Port.Should().Be(9200); metrics.Requests[4].RequestType.Should().Be(RequestType.ElasticsearchCall); for (var i = 0; i < 3; i++) { rootNode = client.RootNodeInfo(); metrics = rootNode.ConnectionStatus.Metrics; metrics.Requests[0].Node.Port.Should().Be(9200); metrics.Requests[0].RequestType.Should().Be(RequestType.ElasticsearchCall); metrics.Requests.Count.Should().Be(1); } }
public void SniffIsCalledAfterItHasGoneOutOfDate() { using (var fake = new AutoFake()) { var dateTimeProvider = fake.Resolve <IDateTimeProvider>(); var nowCall = A.CallTo(() => dateTimeProvider.Now()); nowCall.ReturnsNextFromSequence( DateTime.UtcNow, //initial sniff time (set even if not sniff_on_startup DateTime.UtcNow, //info call 1 DateTime.UtcNow, //info call 2 DateTime.UtcNow.AddMinutes(10), //info call 3 DateTime.UtcNow.AddMinutes(10), //set now after sniff 3 DateTime.UtcNow.AddMinutes(20), //info call 4 DateTime.UtcNow.AddMinutes(20), //set now after sniff 4 DateTime.UtcNow.AddMinutes(22) //info call 5 ); var uris = new[] { new Uri("http://localhost:9200") }; var connectionPool = new SniffingConnectionPool(uris); var config = new ConnectionConfiguration(connectionPool) .SniffLifeSpan(TimeSpan.FromMinutes(4)); fake.Provide <IConnectionConfigurationValues>(config); var transport = FakeCalls.ProvideDefaultTransport(fake, dateTimeProvider); var connection = fake.Resolve <IConnection>(); var sniffCall = FakeCalls.Sniff(fake, config, uris); var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.Ok(config)); var client1 = fake.Resolve <ElasticsearchClient>(); client1.Info(); //info call 1 client1.Info(); //info call 2 client1.Info(); //info call 3 client1.Info(); //info call 4 client1.Info(); //info call 5 sniffCall.MustHaveHappened(Repeated.Exactly.Twice); nowCall.MustHaveHappened(Repeated.Exactly.Times(8)); } }
/// <summary> /// Sets up the <see cref="ConnectionSettings"/> using a collection of <see cref="Uri"/> to create a <see cref="SniffingConnectionPool"/> /// </summary> /// <param name="uris"></param> /// <param name="configureSettings"></param> /// <returns>ElasticSearchMessageQueueOptions</returns> public ElasticSearchMessageQueueOptions <TMessage> UseConnectionPool(IEnumerable <Uri> uris, Action <ConnectionSettings> configureSettings) { if (uris is null) { throw new ArgumentNullException(nameof(uris)); } if (!uris.Any()) { throw new ArgumentException($"{nameof(uris)} cannot be empty"); } if (configureSettings is null) { throw new ArgumentNullException(nameof(configureSettings)); } var connPool = new SniffingConnectionPool(uris); ConnectionSettings = new ConnectionSettings(connPool); configureSettings(ConnectionSettings); return(this); }
private static ElasticLowLevelClient Initialize() { var uris = new[] { new Uri("http://localhost:9200"), //new Uri("http://localhost:9201"), //new Uri("http://localhost:9202"), }; var connectionPool = new SniffingConnectionPool(uris); var settings = new ConnectionConfiguration(connectionPool) .RequestTimeout(TimeSpan.FromMinutes(2)) .ThrowExceptions() .OnRequestCompleted(apiCallDetails => { if (apiCallDetails.HttpStatusCode == 400) { throw new ElasticsearchClientException("you probably provided a malformed request"); } }); return(new ElasticLowLevelClient(settings)); }
public static void Main(string[] args) { var hosts = new[] { new Uri("https://iadd1-s10538-0.es.objectrocket.com:20538"), new Uri("https://iadd1-s10538-1.es.objectrocket.com:20538"), new Uri("https://iadd1-s10538-2.es.objectrocket.com:20538"), new Uri("https://iadd1-s10538-3.es.objectrocket.com:20538") }; try { var connectionPool = new SniffingConnectionPool(hosts); var config = new ConnectionConfiguration(connectionPool) .BasicAuthentication("sooz", "xxxxxxxx"); _client = new ElasticLowLevelClient(config); // TODO: Figure out format of a client-level action to trigger actual connection. _client.Ping((PingRequestParameters x, PingRequestParameters y) => x); } catch (Exception e) { Console.WriteLine("Error: " + e); } Console.WriteLine("Done!"); }
public void SniffOnConnectionFaultCausesSniffOn503() { using (var fake = new AutoFake()) { var dateTimeProvider = fake.Resolve <IDateTimeProvider>(); var nowCall = A.CallTo(() => dateTimeProvider.Now()); nowCall.Returns(DateTime.UtcNow); var nodes = new[] { new Uri("http://localhost:9200") }; var connectionPool = new SniffingConnectionPool(nodes); var config = new ConnectionConfiguration(connectionPool) .SniffOnConnectionFault(); fake.Provide <IConnectionConfigurationValues>(config); var transport = FakeCalls.ProvideDefaultTransport(fake, dateTimeProvider); var connection = fake.Resolve <IConnection>(); var sniffCall = FakeCalls.Sniff(fake, config, nodes); var getCall = FakeCalls.GetSyncCall(fake); getCall.ReturnsNextFromSequence( FakeResponse.Ok(config), //info 1 FakeResponse.Ok(config), //info 2 FakeResponse.Ok(config), //info 3 FakeResponse.Ok(config), //info 4 FakeResponse.Bad(config) //info 5 ); var client1 = fake.Resolve <ElasticsearchClient>(); client1.Info(); //info call 1 client1.Info(); //info call 2 client1.Info(); //info call 3 client1.Info(); //info call 4 Assert.Throws <MaxRetryException>(() => client1.Info()); //info call 5 sniffCall.MustHaveHappened(Repeated.Exactly.Once); nowCall.MustHaveHappened(Repeated.Exactly.Times(7)); } }
public EsClientProvider(IOptions <EsConfig> esConfig, ILogger <EsClientProvider> logger) { try { var uris = esConfig.Value.Uris; if (uris == null || uris.Count < 1) { throw new Exception("urls can not be null"); } ConnectionSettings connectionSetting; if (uris.Count == 1) { var uri = uris.First(); connectionSetting = new ConnectionSettings(uri); } else { var connectionPool = new SniffingConnectionPool(uris); connectionSetting = new ConnectionSettings(connectionPool).DefaultIndex(""); } if (!string.IsNullOrWhiteSpace(esConfig.Value.UserName) && !string.IsNullOrWhiteSpace(esConfig.Value.Password)) { connectionSetting.BasicAuthentication(esConfig.Value.UserName, esConfig.Value.Password); } Client = new ElasticClient(connectionSetting); } catch (Exception ex) { logger.LogError(ex, "ElasticSearch Initialized failed."); throw; } }
public void ShouldThrowAndNotRetrySniffOnStartup401() { using (var fake = new AutoFake(callsDoNothing: true)) { var uris = new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201"), new Uri("http://localhost:9202") }; var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool) .SniffOnStartup(); var pingCall = FakeCalls.PingAtConnectionLevel(fake); pingCall.Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake); var seenPorts = new List <int>(); sniffCall.ReturnsLazily((Uri u, IRequestConfiguration c) => { seenPorts.Add(u.Port); return(FakeResponse.Any(config, 401)); }); var getCall = FakeCalls.GetSyncCall(fake); getCall.Returns(FakeResponse.Bad(config)); fake.Provide <IConnectionConfigurationValues>(config); var e = Assert.Throws <ElasticsearchAuthenticationException>(() => FakeCalls.ProvideRealTranportInstance(fake)); sniffCall.MustHaveHappened(Repeated.Exactly.Once); getCall.MustNotHaveHappened(); } }
private void Initialize(ElasticSearchOutputConfiguration esOutputConfiguration) { Debug.Assert(esOutputConfiguration != null); Debug.Assert(this.healthReporter != null); this.connectionData = new ElasticSearchConnectionData { Configuration = esOutputConfiguration }; var esServiceUri = esOutputConfiguration.ServiceUri .Split(';') .Where(x => Uri.IsWellFormedUriString(x, UriKind.Absolute)) .Select(x => new Uri(x)) .ToList(); string errorMessage; if (!esServiceUri.Any()) { errorMessage = $"{nameof(ElasticSearchOutput)}: required 'serviceUri' configuration parameter is invalid"; this.healthReporter.ReportProblem(errorMessage, EventFlowContextIdentifiers.Configuration); throw new Exception(errorMessage); } string userName = esOutputConfiguration.BasicAuthenticationUserName; string password = esOutputConfiguration.BasicAuthenticationUserPassword; bool credentialsIncomplete = string.IsNullOrWhiteSpace(userName) ^ string.IsNullOrWhiteSpace(password); if (credentialsIncomplete) { errorMessage = $"{nameof(ElasticSearchOutput)}: for basic authentication to work both user name and password must be specified"; healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Configuration); userName = password = null; } var pool = new SniffingConnectionPool(esServiceUri); ConnectionSettings connectionSettings = new ConnectionSettings(pool); if (!string.IsNullOrWhiteSpace(userName) && !string.IsNullOrWhiteSpace(password)) { connectionSettings = connectionSettings.BasicAuthentication(userName, password); } this.connectionData.Client = new ElasticClient(connectionSettings); this.connectionData.LastIndexName = null; if (string.IsNullOrWhiteSpace(esOutputConfiguration.IndexNamePrefix)) { esOutputConfiguration.IndexNamePrefix = string.Empty; } else { string lowerCaseIndexNamePrefix = esOutputConfiguration.IndexNamePrefix.ToLowerInvariant(); if (lowerCaseIndexNamePrefix != esOutputConfiguration.IndexNamePrefix) { healthReporter.ReportWarning($"{nameof(ElasticSearchOutput)}: The chosen index name prefix '{esOutputConfiguration.IndexNamePrefix}' " + "contains uppercase characters, which are not allowed by Elasticsearch. The prefix will be converted to lowercase.", EventFlowContextIdentifiers.Configuration); } esOutputConfiguration.IndexNamePrefix = lowerCaseIndexNamePrefix + Dash; } if (string.IsNullOrWhiteSpace(esOutputConfiguration.EventDocumentTypeName)) { string warning = $"{nameof(ElasticSearchOutput)}: '{nameof(ElasticSearchOutputConfiguration.EventDocumentTypeName)}' configuration parameter " + "should not be empty"; healthReporter.ReportWarning(warning, EventFlowContextIdentifiers.Configuration); esOutputConfiguration.EventDocumentTypeName = ElasticSearchOutputConfiguration.DefaultEventDocumentTypeName; } }
public async void SniffingConnectionPool_DoesNotThrowOrRetry_On_401_Async() { var nodes = new Uri[] { new Uri("http://*****:*****@localhost:9200"), new Uri("http://*****:*****@localhost:9201") }; var pool = new SniffingConnectionPool(nodes); var settings = new ConnectionSettings(pool) .MaximumRetries(3); var client = new ElasticClient(settings); var r = await client.RootNodeInfoAsync(); r.IsValid.Should().BeFalse(); r.ConnectionStatus.HttpStatusCode.Should().Be(401); r.ConnectionStatus.Metrics.Requests.Count.Should().Be(1); r.ConnectionStatus.NumberOfRetries.Should().Be(0); }
public void SniffingConnectionPool_ThrowsAndDoesNotRetry_On_401_Async() { var nodes = new Uri[] { new Uri("http://*****:*****@localhost:9200"), new Uri("http://*****:*****@localhost:9201") }; var pool = new SniffingConnectionPool(nodes); var settings = new ConnectionSettings(pool) .ThrowOnElasticsearchServerExceptions() .MaximumRetries(3); var client = new ElasticClient(settings); var e = Assert.Throws<ElasticsearchServerException>(async () => await client.RootNodeInfoAsync()); e.Status.Should().Be(401); }
/// <summary> /// Configures the services. /// </summary> /// <returns>The services.</returns> /// <param name="services">Services.</param> public void ConfigureServices(IServiceCollection services) { /* * Do we still need this? Or is this .Net 1.1 code? */ services.AddLogging(); //Turn on the OptionsManager that supports IOptions services.AddOptions(); //Add Configuration mappings. services.Configure <ElasticsearchOptions>(Configuration.GetSection("Elasticsearch")); AddAdditionalConfigurationMappings(services); /* * Do we still need this? Or is this Best Bets code which never should * have been in the base class to begin with? */ //Add HttpClient singleton, which is used by the display service. services.AddSingleton <HttpClient, HttpClient>(); // This will inject an IElasticClient using our configuration into any // controllers that take an IElasticClient parameter into its constructor. // // AddTransient means that it will instantiate a new instance of our client // for each instance of the controller. So the function below will be called // on each request. services.AddTransient <IElasticClient>(p => { // Get the ElasticSearch credentials. string username = Configuration["Elasticsearch:Userid"]; string password = Configuration["Elasticsearch:Password"]; //Get the ElasticSearch servers that we will be connecting to. List <Uri> uris = GetElasticsearchServerUriList(); // Create the connection pool, the SniffingConnectionPool will // keep tabs on the health of the servers in the cluster and // probe them to ensure they are healthy. This is how we handle // redundancy and load balancing. var connectionPool = new SniffingConnectionPool(uris); //Return a new instance of an ElasticClient with our settings ConnectionSettings settings = new ConnectionSettings(connectionPool, sourceSerializer: JsonNetSerializer.Default); //Let's only try and use credentials if the username is set. if (!string.IsNullOrWhiteSpace(username)) { settings.BasicAuthentication(username, password); } return(new ElasticClient(settings)); }); //Add in Application specific services AddAppServices(services); // Create CORS policies. services.AddCors(); // Make the application's routes available. services.AddControllers() .AddNewtonsoftJson(); // Enable Swagger // This creates the Swagger Json services.AddOpenApiDocument(config => { if (!string.IsNullOrEmpty(Configuration["NSwag:Title"])) { config.Title = Configuration["NSwag:Title"]; } if (!string.IsNullOrEmpty(Configuration["NSwag:Description"])) { config.Description = Configuration["NSwag:Description"]; } }); }
public void HostsReturnedBySniffAreVisited() { using (var fake = new AutoFake()) { var dateTimeProvider = fake.Resolve <IDateTimeProvider>(); var nowCall = A.CallTo(() => dateTimeProvider.Now()); nowCall.Returns(DateTime.UtcNow); var connectionPool = new SniffingConnectionPool(new[] { new Uri("http://localhost:9200"), new Uri("http://localhost:9201") }, randomizeOnStartup: false); var config = new ConnectionConfiguration(connectionPool) .SniffOnConnectionFault(); fake.Provide <IConnectionConfigurationValues>(config); FakeCalls.ProvideDefaultTransport(fake); FakeCalls.PingAtConnectionLevel(fake) .Returns(FakeResponse.Ok(config)); var sniffCall = FakeCalls.Sniff(fake, config, new List <Uri>() { new Uri("http://localhost:9204"), new Uri("http://localhost:9203"), new Uri("http://localhost:9202"), new Uri("http://localhost:9201") }); var connection = fake.Resolve <IConnection>(); var seenNodes = new List <Uri>(); //var getCall = FakeResponse.GetSyncCall(fake); var getCall = A.CallTo(() => connection.GetSync( A <Uri> .That.Not.Matches(u => u.AbsolutePath.StartsWith("/_nodes")), A <IRequestConfiguration> ._)); getCall.ReturnsNextFromSequence( FakeResponse.Ok(config), //info 1 FakeResponse.Bad(config), //info 2 FakeResponse.Ok(config), //info 2 retry FakeResponse.Ok(config), //info 3 FakeResponse.Ok(config), //info 4 FakeResponse.Ok(config), //info 5 FakeResponse.Ok(config), //info 6 FakeResponse.Ok(config), //info 7 FakeResponse.Ok(config), //info 8 FakeResponse.Ok(config) //info 9 ); getCall.Invokes((Uri u, IRequestConfiguration o) => seenNodes.Add(u)); var client1 = fake.Resolve <ElasticsearchClient>(); client1.Info(); //info call 1 client1.Info(); //info call 2 client1.Info(); //info call 3 client1.Info(); //info call 4 client1.Info(); //info call 5 client1.Info(); //info call 6 client1.Info(); //info call 7 client1.Info(); //info call 8 client1.Info(); //info call 9 sniffCall.MustHaveHappened(Repeated.Exactly.Once); seenNodes.Should().NotBeEmpty().And.HaveCount(10); seenNodes[0].Port.Should().Be(9200); seenNodes[1].Port.Should().Be(9201); //after sniff seenNodes[2].Port.Should().Be(9202, string.Join(",", seenNodes.Select(n => n.Port))); seenNodes[3].Port.Should().Be(9204); seenNodes[4].Port.Should().Be(9203); seenNodes[5].Port.Should().Be(9202); seenNodes[6].Port.Should().Be(9201); } }
private static ElasticClient GetClient(IList <string> endpoints, string defaultIndex, ConnectionConfig config, ConnectionPool connectionPool) { if (!endpoints.HasItems()) { throw new ArgumentException(string.Format(Utils.ARGUMENT_EMPTY_LIST_MESSAGE, nameof(endpoints)), nameof(endpoints)); } var uris = endpoints.Select(x => new Uri(x)); IConnectionPool pool; switch (connectionPool) { case ConnectionPool.SingleNode: pool = new SingleNodeConnectionPool(new Uri(endpoints.First())); break; case ConnectionPool.Static: pool = new StaticConnectionPool(uris); break; case ConnectionPool.Sniffing: pool = new SniffingConnectionPool(uris); break; case ConnectionPool.Sticky: pool = new StickyConnectionPool(uris); break; default: throw new ArgumentOutOfRangeException(nameof(connectionPool), connectionPool, null); } var connectionSettings = new ConnectionSettings(pool) .ConnectionLimit(config?.ConnectionLimit > 0 ? config.ConnectionLimit : 80) .DisableAutomaticProxyDetection(config?.DisableAutomaticProxyDetection ?? false) .DisableDirectStreaming(config?.EnableDebugMode ?? false) .DisablePing(config?.DisablePings ?? false) .DefaultIndex(defaultIndex) .EnableHttpCompression(config?.EnableHttpCompression ?? true) .EnableHttpPipelining(config?.EnableHttpPipelining ?? true) .EnableTcpKeepAlive(config?.KeepAliveTime.Ticks > 0 ? config.KeepAliveTime : TimeSpan.FromMilliseconds(2000), config?.KeepAliveInterval.Ticks > 0 ? config.KeepAliveInterval : TimeSpan.FromMilliseconds(2000)) .MaximumRetries(config?.MaxRetries > 0 ? config.MaxRetries : 10) .MaxRetryTimeout( config?.MaxRetryTimeout.Ticks > 0 ? config.MaxRetryTimeout : TimeSpan.FromSeconds(60)) .PrettyJson(config?.EnableDebugMode ?? false) .RequestTimeout(config?.RequestTimeout.Ticks > 0 ? config.RequestTimeout : TimeSpan.FromSeconds(60)) .SniffLifeSpan(config?.SniffLifeSpan.Ticks > 0 ? config.SniffLifeSpan : TimeSpan.FromHours(1)) .SniffOnConnectionFault(config?.SniffsOnConnectionFault ?? true) .SniffOnStartup(config?.SniffsOnStartup ?? true) .ThrowExceptions(config?.ThrowExceptions ?? false); if (!string.IsNullOrWhiteSpace(config?.BasicAuthenticationUsername) && !string.IsNullOrWhiteSpace(config.BasicAuthenticationPassword)) { connectionSettings.BasicAuthentication(config.BasicAuthenticationUsername, config.BasicAuthenticationPassword); } if (config?.ClientCerfificates != null && config.ClientCerfificates.HasItems()) { connectionSettings.ClientCertificates(config.ClientCerfificates); } else if (config?.ClientCerfificate != null) { connectionSettings.ClientCertificate(config.ClientCerfificate); } if (config?.DeadTimeout.Ticks > 0) { connectionSettings.DeadTimeout(config.DeadTimeout); } if (config?.Headers != null && config.Headers.HasItems()) { connectionSettings.GlobalHeaders(config.Headers); } if (config?.QueryStringParameters != null && config.QueryStringParameters.HasItems()) { connectionSettings.GlobalQueryStringParameters(config.QueryStringParameters); } if (config?.MaxDeadTimeout.Ticks > 0) { connectionSettings.MaxDeadTimeout(config.DeadTimeout); } if (config?.NodePredicate != null) { connectionSettings.NodePredicate(config.NodePredicate); } if (config?.RequestCompletedHandler != null) { connectionSettings.OnRequestCompleted(config.RequestCompletedHandler); } if (config?.RequestDataCreatedHandler != null) { connectionSettings.OnRequestDataCreated(config.RequestDataCreatedHandler); } if (config?.PingTimeout.Ticks > 0) { connectionSettings.PingTimeout(config.PingTimeout); } if (!string.IsNullOrWhiteSpace(config?.ProxyAddress)) { connectionSettings.Proxy(new Uri(config.ProxyAddress), config?.ProxyUsername, config?.ProxyPassword); } if (config?.ServerCertificateValidationCallback != null) { connectionSettings.ServerCertificateValidationCallback(config.ServerCertificateValidationCallback); } return(new ElasticClient(connectionSettings)); }
static void Main(string[] args) { var setting = new ConnectionConfiguration(new Uri("http://120.27.213.67:9200/")).RequestTimeout(TimeSpan.FromMinutes(2)); //setting.BasicAuthentication("", ""); var lowlevelClient = new ElasticLowLevelClient(setting); var uris = new List <Uri> { new Uri("http://localhost:9200"), new Uri("http://120.27.213.67:9200/"), }; var connectPool = new SniffingConnectionPool(uris); var settings1 = new ConnectionConfiguration(connectPool); var lowlevelClient1 = new ElasticLowLevelClient(settings1); //var person = new Person //{ // FirstName = "Martijn", // LastName = "Laarman" //}; //var indexRespose = lowlevelClient.Index<BytesResponse>("people", "1", PostData.Serializable(person)); //byte[] responseBytes = indexRespose.Body; //var asyncIndexResponse = await lowlevelClient.IndexAsync<StringResponse>("people", "1", PostData.Serializable(person)); //string responseString = asyncIndexResponse.Body; //var peoples = new object[] //{ // new { index = new { _index = "people", _type = "_doc", _id = "1" }}, // new { FirstName = "Martijn", LastName = "Laarman" }, // new { index = new { _index = "people", _type = "_doc", _id = "2" }}, // new { FirstName = "Greg", LastName = "Marzouka" }, // new { index = new { _index = "people", _type = "_doc", _id = "3" }}, // new { FirstName = "Russ", LastName = "Cam" }, //}; //var ndexResponse = lowlevelClient.Bulk<StringResponse>(PostData.MultiJson(peoples)); //string responseStream = ndexResponse.Body; //var searchResponse = lowlevelClient.Search<StringResponse>("people", PostData.Serializable(new //{ // from = 0, // size = 10, // query = new // { // match = new // { // firstName = new // { // query = "Martijn" // } // } // } //})); //var successful = searchResponse.Success; //var responseJson = searchResponse.Body; //var searchResponse1 = lowlevelClient.Search<BytesResponse>("people", PostData.Serializable(new { match_all = new { } })); //var success = searchResponse1.Success; //var successOrKnownError = searchResponse1.SuccessOrKnownError; //var exception = searchResponse1.OriginalException; }
static ElasticConnector() { _sniffingConnectionPool = new SniffingConnectionPool(GetUris()); _connectionSettings = new ConnectionSettings(_sniffingConnectionPool).DisableDirectStreaming(); }
/// <summary> /// Configures the services. /// </summary> /// <returns>The services.</returns> /// <param name="services">Services.</param> public IServiceProvider ConfigureServices(IServiceCollection services) { services.AddLogging(); //Turn on the OptionsManager that supports IOptions services.AddOptions(); services.AddCors(); //This allows us to easily generate URLs to routes services.AddSingleton <IActionContextAccessor, ActionContextAccessor>(); services.AddScoped <IUrlHelper>(factory => { var actionContext = factory.GetService <IActionContextAccessor>() .ActionContext; return(new UrlHelper(actionContext)); }); //Add Configuration mappings. // services.Configure<NSwagOptions>(Configuration.GetSection("NSwag")); services.Configure <ElasticsearchOptions>(Configuration.GetSection("Elasticsearch")); AddAdditionalConfigurationMappings(services); //Add HttpClient singleton, which is used by the display service. services.AddSingleton <HttpClient, HttpClient>(); // This will inject an IElasticClient using our configuration into any // controllers that take an IElasticClient parameter into its constructor. // // AddTransient means that it will instantiate a new instance of our client // for each instance of the controller. So the function below will be called // on each request. services.AddTransient <IElasticClient>(p => { // Get the ElasticSearch credentials. string username = Configuration["Elasticsearch:Userid"]; string password = Configuration["Elasticsearch:Password"]; //Get the ElasticSearch servers that we will be connecting to. List <Uri> uris = GetServerUriList(); // Create the connection pool, the SniffingConnectionPool will // keep tabs on the health of the servers in the cluster and // probe them to ensure they are healthy. This is how we handle // redundancy and load balancing. var connectionPool = new SniffingConnectionPool(uris); //Return a new instance of an ElasticClient with our settings ConnectionSettings settings = new ConnectionSettings(connectionPool); //Let's only try and use credentials if the username is set. if (!string.IsNullOrWhiteSpace(username)) { settings.BasicAuthentication(username, password); } return(new ElasticClient(settings)); }); //Add in Application specific services AddAppServices(services); // Create CORS policies. services.AddCors(); // Add framework services. services.AddMvc(); // Enable Swagger // This creates the Swagger Json services.AddOpenApiDocument(config => { if (!string.IsNullOrEmpty(Configuration["NSwag:Title"])) { config.Title = Configuration["NSwag:Title"]; } if (!string.IsNullOrEmpty(Configuration["NSwag:Description"])) { config.Description = Configuration["NSwag:Description"]; } }); // Add framework services return(services.BuildServiceProvider()); }
/// <summary> /// Gets the search results from this SiteWideSearch provider. /// </summary> /// <param name="searchCollection">The search collection.</param> /// <param name="term">The term.</param> /// <param name="pageSize">Size of the page.</param> /// <param name="offset">The offset.</param> /// <returns></returns> public override ISiteWideSearchResultCollection GetSearchResults(string searchCollection, string term, int pageSize, int offset) { ESSiteWideSearchCollectionElement searchCollConfig = ElasticSearchConfig.GetSearchCollectionConfig(searchCollection); ClusterElement clusterConfig = searchCollConfig.ClusterElementDetails; var connectionPool = new SniffingConnectionPool(searchCollConfig.ClusterNodes); var config = new ConnectionConfiguration(connectionPool) .UsePrettyResponses() .MaximumRetries(clusterConfig.MaximumRetries)//try 5 times if the server is down .ExposeRawResponse() .ThrowOnElasticsearchServerExceptions(); var esClient = new ElasticsearchClient(config); string[] fieldList = ElasticSearchConfig.GetFields(searchCollection); //do search_template var _body = new { template = new { id = searchCollConfig.Template }, @params = new { my_value = term, my_size = pageSize.ToString(), my_from = offset.ToString(), my_site = searchCollConfig.Site, my_fields = fieldList } }; ElasticsearchResponse <DynamicDictionary> results = null; try { results = esClient.SearchTemplate(searchCollConfig.Index, _body); } catch (MaxRetryException ex) { try { //log the maximum retry excpetion log.Error("Error using the ESClient Search Template method. Maximum retry exception: ", ex); //sleep for 5 seconds Thread.Sleep(clusterConfig.ConnectionTimeoutDelay); //try to fetch results again results = esClient.SearchTemplate(searchCollConfig.Index, _body); } catch (Exception e) { log.Error("Error using the ESClient Search Template method.", e); throw e; } }//retry catch log error sleep retry List <ESSiteWideSearchResult> foundTerms = new List <ESSiteWideSearchResult>(pageSize); if (results.Success) { var stringResponse = results.Response; try { foreach (var hit in results.Response["hits"].hits) { //title is a special case and when it is empty the value needs to be Untitled string title = SetFieldValue(hit, "title"); if (string.IsNullOrEmpty(title)) { title = "Untitled"; } string url = SetFieldValue(hit, "url"); string description = SetFieldValue(hit, "metatag.description"); string type = SetFieldValue(hit, "metatag.dcterms.type"); foundTerms.Add(new ESSiteWideSearchResult(title, url, description, type)); } } catch (Exception ex) { log.Error("Error retrieving search results.", ex); throw ex; } } else { string prettyResponse = ""; //log the raw response message //if there is not response them log with message - "No message in response" if (!string.IsNullOrEmpty(System.Text.Encoding.Default.GetString(results.ResponseRaw))) { prettyResponse = System.Text.Encoding.Default.GetString(results.ResponseRaw); } else { prettyResponse = "No message in response"; } log.ErrorFormat("Search failed. Http Status Code: {0}. Message: {1}", results.HttpStatusCode.Value, prettyResponse); throw (new Exception("Search failed. Http Status Code:" + results.HttpStatusCode.Value.ToString() + ". Message: " + prettyResponse)); } //Return the results return(new ESSiteWideSearchResultCollection(foundTerms.AsEnumerable()) { ResultCount = results.Response["hits"].total }); }
public void NodesDiscoveredDuringSniffShouldBeHttps() { ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => { if (errors == SslPolicyErrors.None) return true; string certificateHash = certificate.GetCertHashString(); bool knownThumbprintIsValid = false; if (_knownPrints.TryGetValue(certificateHash, out knownThumbprintIsValid)) return knownThumbprintIsValid; var isValid = IsValidCertificate(certificate, chain); _knownPrints.AddOrUpdate(certificateHash, isValid, (s, b) => isValid); return isValid; }; var uris = new[] { new Uri("https://localhost:9200") }; var connectionPool = new SniffingConnectionPool(uris, randomizeOnStartup: false); var settings = new ConnectionSettings(connectionPool, ElasticsearchConfiguration.DefaultIndex) .SniffOnStartup() .SetBasicAuthentication("mpdreamz", "blahblah") .ExposeRawResponse() //.SetPingTimeout(1000) .SetTimeout(2000); var client = new ElasticClient(settings); var results = client.NodesInfo(); results.IsValid.Should().BeTrue("{0}", results.ConnectionStatus.ToString()); results.ConnectionStatus.NumberOfRetries.Should().Be(0); var uri = new Uri(results.ConnectionStatus.RequestUrl); uri.Port.Should().Be(9200); uri.Scheme.Should().Be("https"); results = client.NodesInfo(); results.IsValid.Should().BeTrue("{0}", results.ConnectionStatus.ToString()); results.ConnectionStatus.NumberOfRetries.Should().Be(0); uri = new Uri(results.ConnectionStatus.RequestUrl); uri.Port.Should().Be(9201); uri.Scheme.Should().Be("https"); }
static void Main(string[] args) { Program p=new Program(); const string elUri = "http://*****:*****@localhost:9200/"; Node = new Uri(elUri); var connectionPool = new SniffingConnectionPool(new[] { Node }); Settings = new ConnectionSettings(Node); var config = new ConnectionConfiguration(connectionPool) .EnableTrace() .ExposeRawResponse() .SetBasicAuthentication("aotuo", "123456") .SetTimeout(4000); Client = new ElasticClient(Settings); /*var newPost = new activitylog { IpAddress = "21", Time = DateTime.Now.ToString(CultureInfo.CurrentCulture), Desciption = "azure" }; Client.Index(newPost, i => i .Index("azure") .Type("activitylog") ); var indexsettings = new IndexSettings(); try { var res = Client.Count(c=>c.AllIndices().Type("logs")); var res2 =Client.Search<object>(s => s.AllIndices() .From(0) .Size(50) .Query(q =>q.Term("ident","BingLi")) ); Console.WriteLine(res.Count); Escount = (int)res.Count; } catch (Exception) { Console.WriteLine("error"); } Termquery();*/ Console.ReadLine(); }
private static void Main(string[] args) { using (SqlConnection conn = new SqlConnection("Data Source=192.168.30.128;Initial Catalog=SmartBusExtension;User Id=sa;Password=1qaz2wsx123_@;Connect Timeout=120")) { SqlCommand cmd = new SqlCommand(@" SELECT * FROM MQUser" , conn); List <SyncArrayConfiguration> arrayConfig = new List <SyncArrayConfiguration>() { new SyncArrayConfiguration { SqlCommand = new SqlCommand(@" SELECT id, userid, login,username,avatar FROM MQUser where UserName='******'" , conn), AttributeName = "tags" }, /** * new SyncArrayConfiguration * { * SqlCommand = new SqlCommand(@" * SELECT id_object AS '_id', id, description, xmlData, languageId * FROM dbo.Categories * WHERE languageId = 'es'" * , conn), * AttributeName = "categories", * XmlFields = new string[] { "xmlData" } * } **/ }; SyncDeleteConfiguration deleteCmd = new SyncDeleteConfiguration { SqlCommand = new SqlCommand(@" SELECT id_object AS '_id', createdOn FROM dbo.DeleteLog WHERE id_language = 'es'" , conn), ColumnsToCompareWithLastSyncDate = new string[] { "[createdOn]" }, }; var nodes = new Uri[] { new Uri("http://192.168.30.135:9200"), //new Uri("http://localhost:9201") }; var connectionPool = new SniffingConnectionPool(nodes); var esConfig = new ConnectionConfiguration(connectionPool).UsePrettyResponses(); var syncConfig = new SyncConfiguration() { SqlConnection = conn, SqlCommand = cmd, ArraysConfiguration = arrayConfig, ColumnsToCompareWithLastSyncDate = new string[] { "[id]" }, //DeleteConfiguration = deleteCmd, ElasticSearchConfiguration = esConfig, BulkSize = 500, _Index = new Index("mssql_30.128"), _Type = "notes" }; var sync = new Sync(syncConfig); try { var response = sync.Exec(); if (response.BulkResponses.Any()) { foreach (var bulkResponse in response.BulkResponses) { Console.WriteLine("success: " + bulkResponse.Success); Console.WriteLine("http status code: " + bulkResponse.HttpStatusCode); Console.WriteLine("affected documents: " + bulkResponse.AffectedDocuments); Console.WriteLine("started on: " + bulkResponse.StartedOn); Console.WriteLine("bulk duration: " + bulkResponse.Duration + "ms"); if (!response.Success) { Console.WriteLine("es original exception: " + bulkResponse.ESexception); } Console.WriteLine("\n"); } Console.WriteLine("\nbulk avg duration: " + response.BulkResponses.Average(x => x.Duration) + "ms"); Console.WriteLine("\n"); } } catch (Exception ex) { Console.WriteLine("an error has occurred: " + ex.Message); } finally { Console.WriteLine("Execution has completed. Press any key to continue..."); Console.ReadKey(); } } }
public void CreateIndex(string name, string elUri) { Node = new Uri(elUri); var connectionPool = new SniffingConnectionPool(new[] { Node }); Settings = new ConnectionSettings(Node); Client = new ElasticClient(Settings); var indexsettings = new IndexSettings(); indexsettings.NumberOfReplicas = 1; indexsettings.NumberOfShards = 5; Client.CreateIndex(c => c .Index(name) .InitializeUsing(indexsettings) .AddMapping<activitylog>(m => m.MapFromAttributes())); }