public NestBatchPusher(string esUrl, IIndexNamer indexNamer, IConfigurationValueProvider configurationValueProvider) { _indexNamer = indexNamer; if (!int.TryParse(configurationValueProvider.GetValue(ConfigurationKeys.BulkBatchSize), out _batchSize)) { _batchSize = 500; } var endodedCreds = configurationValueProvider.GetValue(ConfigurationKeys.TabSeparatedCustomEsHttpHeaders) .Replace("Authorization: Basic", "").Trim(); var credentials = Encoding.UTF8.GetString(Convert.FromBase64String(endodedCreds)).Split(':'); if (!bool.TryParse(configurationValueProvider.GetValue(ConfigurationKeys.EsPipelineEnabled), out _setPipeline)) { _setPipeline = false; } var connectionConfiguration = new ConnectionSettings( new SingleNodeConnectionPool(new Uri(esUrl))) .ServerCertificateValidationCallback(delegate { return(true); }) .BasicAuthentication(credentials[0], credentials[1]) .RequestTimeout(TimeSpan.FromMinutes(2)) //.EnableDebugMode(_ => { if(!_.Success) Console.WriteLine(); }) .DefaultIndex("DafaultIndex"); _client = new ElasticClient(connectionConfiguration); }
public ElasticsearchBatchPusher(IHttpClient httpClient, IConfigurationValueProvider configurationValueProvider, string esUrl, IIndexNamer indexNamer, int batchSize = 100) { _indexNamer = indexNamer; _httpClient = httpClient; _batchSize = batchSize; _esUrl = esUrl; var esBackOffMinSecondsString = configurationValueProvider.GetValue(ConfigurationKeys.EsBackOffMinSeconds); var esBackOffMaxSecondsString = configurationValueProvider.GetValue(ConfigurationKeys.EsBackOffMaxSeconds); var esBackOffMinSeconds = string.IsNullOrWhiteSpace(esBackOffMinSecondsString) ? 5 : int.Parse(esBackOffMinSecondsString); var esBackOffMaxSeconds = string.IsNullOrWhiteSpace(esBackOffMaxSecondsString) ? 100 : int.Parse(esBackOffMaxSecondsString); _interval = new DoublyIncreasingInterval(TimeSpan.FromSeconds(esBackOffMinSeconds), TimeSpan.FromSeconds(esBackOffMaxSeconds), 5); }
public MasterScheduler(IEventQueueOperator eventQueueOperator, IConfigurationValueProvider configurationValueProvider, ISourceConfiguration sourceConfiguration, IElasticsearchClient elasticsearchClient, IServiceLocator locator, ILockStore lockStore, ITelemetryProvider telemetryProvider, IIndexNamer indexNamer, IKeyValueStore keyValueStore) { _keyValueStore = keyValueStore; _indexNamer = indexNamer; _lockStore = lockStore; _telemetryProvider = telemetryProvider; _sourceConfiguration = sourceConfiguration; _locator = locator; _elasticsearchClient = elasticsearchClient; _configurationValueProvider = configurationValueProvider; _eventQueueOperator = eventQueueOperator; _scheduleDurationInstrumentor = telemetryProvider.GetInstrumentor <MasterScheduler>(); }