public ReservationIndexRepository(IElasticLowLevelClient client, ReservationsApiEnvironment environment, IElasticSearchQueries elasticQueries, ILogger <ReservationIndexRepository> logger) { _client = client; _environment = environment; _elasticQueries = elasticQueries; _logger = logger; }
protected ServerErrorTestsBase() { var settings = TestClient.GetFixedReturnSettings(ResponseJson, 500); this.LowLevelClient = new ElasticLowLevelClient(settings); this.HighLevelClient = new ElasticClient(settings); }
public static void AddElasticsearch(this IServiceCollection services, IConfiguration configuration) { var url = configuration["elasticsearch:url"]; var defaultIndex = configuration["elasticsearch:index"]; _updateSearchService = Boolean.Parse(configuration["elasticsearch:updateIndex"]); _settings = new ConnectionSettings(new Uri(url)) .DefaultIndex(defaultIndex) .DefaultFieldNameInferrer(s => s) .DefaultMappingFor <ComplexStringIndex>(m => m .IndexName("complexstring") .TypeName("complexstring") ) .DefaultMappingFor <ProjectIndex>(m => m .IndexName("project") .TypeName("project") ); _elasticClient = new ElasticClient(_settings); services.AddSingleton <IElasticClient>(_elasticClient); var settingslow = new ConnectionConfiguration(new Uri(url)) .RequestTimeout(TimeSpan.FromMinutes(2)); _lowlevelClient = new ElasticLowLevelClient(settingslow); }
public ElasticReader( IConnectionContext context, Field[] fields, IElasticLowLevelClient client, IRowFactory rowFactory, ReadFrom readFrom ) { _context = context; _fields = fields; _fieldNames = fields.Select(f => _readFrom == ReadFrom.Input ? f.Name : f.Alias.ToLower()).ToArray(); _client = client; _rowFactory = rowFactory; _readFrom = readFrom; _typeName = readFrom == ReadFrom.Input ? context.Entity.Name : context.Entity.Alias.ToLower(); _context.Entity.ReadSize = _context.Entity.ReadSize == 0 ? DefaultSize : _context.Entity.ReadSize; if (_context.Entity.ReadSize > ElasticsearchDefaultSizeLimit) { _context.Warn("Elasticsearch's default size limit is 10000. {0} may be too high.", _context.Entity.ReadSize); } _version = ElasticVersionParser.ParseVersion(_context); }
public LogAutoscalingEvent(IElasticLowLevelClient es, IAmazonAutoScaling asg, LogToElasticsearch logToEs) : base(request => (from asgEvent in Observable.Return(request) let asgName = asgEvent.SelectToken("detail.AutoScalingGroupName")?.Value <string>() where !string.IsNullOrEmpty(asgName) let describeRequest = new DescribeAutoScalingGroupsRequest { AutoScalingGroupNames = new List <string> { asgName } } from describeResponse in asg.DescribeAutoScalingGroupsAsync(describeRequest) from description in describeResponse.AutoScalingGroups let tags = description.Tags let app = tags.GetValue("repo") ?? tags.GetValue("Application") ?? tags.GetValue("app") let env = tags.GetValue("env") ?? tags.GetValue("Environment") let sha = tags.GetValue("sha") let log = new { Name = asgEvent["detail-type"]?.Value <string>(), RequestId = asgName, Message = asgEvent } let time = asgEvent["time"].Value <DateTime>() let alteredLog = new AlteredLog { Time = time, Repo = app, Env = env, Sha = sha, Log = JObject.FromObject(log, AlteredJson.DefaultJsonSerializer) } from response in logToEs.Execute(alteredLog) select response).ToTask()) { }
protected override void InitializeTarget() { base.InitializeTarget(); var uri = ConnectionStringName.GetConnectionString() ?? Uri; var nodes = uri.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(url => new Uri(url)); var connectionPool = new StaticConnectionPool(nodes); var config = new ConnectionConfiguration(connectionPool); if (RequireAuth) { config.BasicAuthentication(Username, Password); } if (ElasticsearchSerializer != null) { config = new ConnectionConfiguration(connectionPool, _ => ElasticsearchSerializer); } _client = new ElasticLowLevelClient(config); if (!string.IsNullOrEmpty(ExcludedProperties)) { _excludedProperties = ExcludedProperties.Split(new [] { ',' }, StringSplitOptions.RemoveEmptyEntries).ToList(); } }
/// <summary> /// This will rebuild an entire index alias behind the scenes with no downtime. It creates a new index /// and populates it then uses Elasticsearch's hot-swapping technique to bring it online. /// </summary> public async static Task RebuildIndexWithHotSwapAsync( this IElasticLowLevelClient client, string alias, JObject indexMapping, Func <Task <IEnumerable <BulkIndexingDoc> > > reader, CancellationToken ctx = default(CancellationToken)) { var deletableIndices = JArray.Parse((await client.CatAliasesAsync <StringResponse>(alias, new CatAliasesRequestParameters { Format = "json" }, ctx)).Body) .Select(x => new { alias = x.Value <string>("alias"), index = x.Value <string>("index") }) .ToList(); var index = GenerateUniqueIndexNameForAlias(alias); await client.IndicesCreateAsync <StringResponse>(index, PostData.String(indexMapping?.ToString()), ctx : ctx); while (!ctx.IsCancellationRequested) { // TODO: If an exception is thrown, delete the half-created index var docs = await reader(); if (ctx.IsCancellationRequested || !docs.Any()) { break; } var body = docs.SelectMany(doc => doc.ToLines().Select(x => x.ToString(Formatting.None))); var bulkResponse = await client.BulkAsync <StringResponse>(index, PostData.MultiJson(body)); ThrowOnPartialBulkSuccess(bulkResponse); } if (ctx.IsCancellationRequested) { return; } var actions = deletableIndices.Select(idx => (object)new { remove = new { idx.index, idx.alias } }).ToList(); actions.Add(new { add = new { index, alias } }); // This is the hot-swap. The actions in the list are performed atomically await client.IndicesUpdateAliasesForAllAsync <StringResponse>(PostData.String(JObject.FromObject(new { actions }).ToString()), ctx : ctx); if (deletableIndices.Any()) { await client.IndicesDeleteAsync <StringResponse>(string.Join(",", deletableIndices.Select(x => x.index)), ctx : ctx); } }
public ElClient(string elasticURL) { var uri = new Uri(elasticURL); var settings = new ConnectionConfiguration(uri); Client = new ElasticLowLevelClient(settings); }
protected ServerErrorTestsBase() { var settings = FixedResponseClient.CreateConnectionSettings(ResponseJson, 500); this.LowLevelClient = new ElasticLowLevelClient(settings); this.HighLevelClient = new ElasticClient(settings); }
private async Task CreateIndexIfNeededAsync(IElasticLowLevelClient client) { if (!_seederConfiguration.CreateIndexIfNotExists) { return; } StringResponse result = await client.Indices.ExistsAsync <StringResponse>(_elasticsearchConfiguration.IndexName); if (result.HttpStatusCode == (int)HttpStatusCode.NotFound) { string json = await File.ReadAllTextAsync(_seederConfiguration.CreateIndexRequestBodyContentFilePath); StringResponse response = await client.Indices.CreateAsync <StringResponse>( _elasticsearchConfiguration.IndexName, PostData.String(json)); if (!response.Success) { const string messageConst = "Unable to create index. Response from server: "; ServerError error; var message = response.TryGetServerError(out error) ? $"{messageConst}{error.Error.Reason}" : $"{messageConst}{response.Body}"; throw new SystemException(message); } Log.Logger.Information($"Index '{_elasticsearchConfiguration.IndexName}' created"); } else if (!result.Success) { Log.Logger.Warning($"Unable to check if index exists. Server response: [{result.HttpStatusCode}] {result.Body}"); } }
/// <summary> /// Constructor with ElasticsearchProcessorSettings. /// </summary> /// <param name="settings">Settings.</param> public ElasticsearchProcessor(ElasticsearchProcessorSettings settings) { ConnectionConfiguration esSettings = new ConnectionConfiguration(settings.Address) .RequestTimeout(TimeSpan.FromSeconds(settings.TimeOutInSeconds)); Client = new ElasticLowLevelClient(esSettings); }
private void EnsureConnectionOpen() { if (_client == null) { var uri = ConnectionStringName.GetConnectionString() ?? Uri; var nodes = uri.Render(new LogEventInfo()).Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(url => new Uri(url)); var connectionPool = new StaticConnectionPool(nodes); var config = new ConnectionSettings(connectionPool, sourceSerializer: (builtin, settings) => new JsonNetSerializer( builtin, settings, () => new JsonSerializerSettings { NullValueHandling = NullValueHandling.Include, ReferenceLoopHandling = ReferenceLoopHandling.Ignore }, resolver => resolver.NamingStrategy = new SnakeCaseNamingStrategy() )); if (RequireAuth) { config.BasicAuthentication(Username, Password); } if (DisableAutomaticProxyDetection) { config.DisableAutomaticProxyDetection(); } _client = new ElasticLowLevelClient(config); } }
public static void AddElasticSearch(this IServiceCollection services, IConfiguration config) { var url = config["elasticsearch:url"]; var defaultIndex = config["elasticsearch:index"]; IsElasticUsed = bool.Parse(config["elasticsearch:updateIndex"]); _settings = new ConnectionSettings(new Uri(url)) .DefaultFieldNameInferrer(s => s) .DefaultMappingFor <UserIndex>(m => m .IndexName("user") .TypeName("user") ); _settings.EnableDebugMode(); if (!string.IsNullOrWhiteSpace(defaultIndex)) { _settings.DefaultIndex(defaultIndex); } Client = new ElasticClient(_settings); services.AddSingleton <IElasticClient>(Client); services.AddTransient <ISearchService>(a => new SearchService(Client)); var settingslow = new ConnectionConfiguration(new Uri(url)) .RequestTimeout(TimeSpan.FromMinutes(2)); _lowlevelClient = new ElasticLowLevelClient(settingslow); }
public ElasticPartialUpdater(OutputContext context, Configuration.Field[] fields, IElasticLowLevelClient client) { _context = context; _fields = fields; _client = client; _index = context.Connection.Index; _type = context.Entity.Alias.ToLower(); }
/// <summary> /// /// </summary> /// <param name="elasticLowLevelClient"></param> /// <param name="cleanPayload"></param> /// <param name="elasticOpType"></param> public ElasticsearchLogClient(IElasticLowLevelClient elasticLowLevelClient, Func <string, long?, string, string> cleanPayload, ElasticOpType elasticOpType) { _elasticLowLevelClient = elasticLowLevelClient; _cleanPayload = cleanPayload; _elasticOpType = elasticOpType; }
public ElasticAnalyticsService(IElasticLowLevelClient elasticClient, IDateTimeProvider dateTimeProvider, IWebHostEnvironment environment, ILogger <ElasticAnalyticsService> logger) { _elasticClient = elasticClient; _dateTimeProvider = dateTimeProvider; _environment = environment; _logger = logger; }
public ElasticsearchSession(IElasticLowLevelClient elasticClient, IUniqueNameResolver uniqueNameResolver, IIdGenerator documentsIdGenerator, IEsfStateInputValidator validator) { _elasticClient = elasticClient; _indexName = uniqueNameResolver.GetUniqueName(); _typeName = uniqueNameResolver.GetUniqueName(); _documentsIdGenerator = documentsIdGenerator; _validator = validator; }
/// <summary> /// Initializes a new instance of the <see cref="SearchService{T}"/> class. /// </summary> /// <param name="logger">The log handler for the controller.</param> /// <param name="serviceConfig">The service config settings <see cref="SearchServiceConfig"/></param> /// <param name="client">Interface to high level ElasticSearch client.</param> /// <param name="lowLevelClient">Interface to low level ElasticSearch client.</param> public SearchService( ILogger logger, SearchServiceConfig serviceConfig, IElasticClient client, IElasticLowLevelClient lowLevelClient) { _logger = logger; _serviceConfig = serviceConfig; _client = client; _lowLevelClient = lowLevelClient; }
public ElasticQueryReader( InputContext context, IElasticLowLevelClient client, IRowFactory rowFactory ) { _context = context; _client = client; _rowFactory = rowFactory; _fields = context.InputFields.ToDictionary(k => k.Name, v => v); }
public ElasticsearchSessionFactory( IElasticLowLevelClient elasticClient, IUniqueNameResolver uniqueNameResolver, IIdGenerator idGenerator, IEsfStateInputValidator validator) { _elasticClient = elasticClient; _uniqueNameResolver = uniqueNameResolver; _idGenerator = idGenerator; _validator = validator; }
public ElasticClient(ITransport <IConnectionSettingsValues> transport) { transport.ThrowIfNull(nameof(transport)); transport.Settings.ThrowIfNull(nameof(transport.Settings)); transport.Settings.Serializer.ThrowIfNull(nameof(transport.Settings.Serializer)); transport.Settings.Inferrer.ThrowIfNull(nameof(transport.Settings.Inferrer)); this.Transport = transport; this.LowLevel = new ElasticLowLevelClient(this.Transport); this.LowLevelDispatch = new LowLevelDispatch(this.LowLevel); }
/// <summary> /// This performs a bulk command against the index pointed at by an alias. This is to avoid the potential for /// a hotswap to change the underlying index from under us while we're reindexing. The index mapping is only /// used if the aliased index does not yet exist. /// </summary> public async static Task BulkTargetingAliasAsync( this IElasticLowLevelClient client, string alias, JObject indexMappingIfNotExists, Func <Task <IEnumerable <BulkIndexingDoc> > > reader, CancellationToken ctx = default(CancellationToken)) { var targetIndices = JArray.Parse((await client.CatAliasesAsync <StringResponse>(alias, new CatAliasesRequestParameters { Format = "json" }, ctx)).Body) .Select(x => x.Value <string>("index")) .ToList(); string index; if (targetIndices.Count == 0) { index = GenerateUniqueIndexNameForAlias(alias); await client.IndicesCreateAsync <StringResponse>(index, PostData.String(indexMappingIfNotExists?.ToString()), ctx : ctx); await client.IndicesUpdateAliasesForAllAsync <StringResponse>(PostData.String(JObject.FromObject(new { actions = new[] { new { add = new { index, alias } } } }).ToString()), ctx : ctx); } else if (targetIndices.Count > 1) { throw new ArgumentException( $"{nameof(BulkTargetingAliasAsync)} can only be used against an alias targeting a single index. The `{alias}` alias covers {targetIndices.Count} indices", nameof(alias)); } else { index = targetIndices.First(); } while (!ctx.IsCancellationRequested) { var docs = await reader(); if (ctx.IsCancellationRequested || !docs.Any()) { break; } var body = docs.SelectMany(doc => doc.ToLines().Select(x => x.ToString(Formatting.None))); var bulkResponse = await client.BulkAsync <StringResponse>(index, PostData.MultiJson(body)); ThrowOnPartialBulkSuccess(bulkResponse); } }
public ElasticWriter(OutputContext context, IElasticLowLevelClient client) { _context = context; _client = client; _prefix = "{\"index\": {\"_index\": \"" + context.Connection.Index + "\", \"_type\": \"" + context.Entity.Alias.ToLower() + "\", \"_id\": \""; _fields = context.OutputFields.Select(f => new AliasField { Alias = f.Alias.ToLower(), Field = f }).ToArray(); _settings = new JsonSerializerSettings { Formatting = Formatting.None, ContractResolver = new DefaultContractResolver() }; }
public LogToElasticsearch(IElasticLowLevelClient es) : base(async(logBatch) => { var multiJson = logBatch.ToEsMultiJson(); //let _ = Write(multiJson) var bulkRequest = PostData.String(multiJson); var bulkRequestParameters = new BulkRequestParameters { // todo }; var bulkResponse = await es.BulkAsync <StringResponse>(bulkRequest, bulkRequestParameters); return(bulkResponse); }) { }
/// <summary> /// Initializes the current instance. /// </summary> protected override void InitializeTarget() { base.InitializeTarget(); var uri = Uri; var nodes = uri.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(url => new Uri(url)); var connectionPool = new StaticConnectionPool(nodes); var config = new ConnectionConfiguration(connectionPool); config.DisableAutomaticProxyDetection(); _client = new ElasticLowLevelClient(config); }
public StoreToElasticsearch(IElasticLowLevelClient es, string indexName = "terraform") : base(async(request) => { var logBatch = JObject.FromObject(request.Data, AlteredJson.DefaultJsonSerializer); var multiJson = logBatch.ToEsMultiJson(indexName); //let _ = Write(multiJson) var bulkRequest = PostData.String(multiJson); var bulkRequestParameters = new BulkRequestParameters { // todo }; var bulkResponse = await es.BulkAsync <StringResponse>(bulkRequest, bulkRequestParameters); return(bulkResponse); }) { }
public ElasticOutputController( OutputContext context, IAction initializer, IVersionDetector inputVersionDetector, IVersionDetector outputVersionDetector, IElasticLowLevelClient client ) : base( context, initializer, inputVersionDetector, outputVersionDetector ) { _client = client; _stopWatch = new Stopwatch(); }
public ElasticReader( IConnectionContext context, Field[] fields, IElasticLowLevelClient client, IRowFactory rowFactory, ReadFrom readFrom ) { _context = context; _fields = fields; _fieldNames = fields.Select(f => _readFrom == ReadFrom.Input ? f.Name : f.Alias.ToLower()).ToArray(); _client = client; _rowFactory = rowFactory; _readFrom = readFrom; _typeName = readFrom == ReadFrom.Input ? context.Entity.Name : context.Entity.Alias.ToLower(); }
public ElasticOutputController( OutputContext context, IAction initializer, IInputProvider inputProvider, IOutputProvider outputProvider, IElasticLowLevelClient client ) : base( context, initializer, inputProvider, outputProvider ) { _client = client; _stopWatch = new Stopwatch(); }
private async Task IndexProductAsync(ProductForIndex productForIndex, IElasticLowLevelClient client) { string insertData = JsonConvert.SerializeObject(productForIndex); Log.Logger.Information($"Inserting product: [{productForIndex.SearchResultData.Id}]' {productForIndex.SearchResultData.Name}'"); StringResponse response = await client.IndexAsync <StringResponse>( _elasticsearchConfiguration.IndexName, productForIndex.SearchResultData.Id.ToString(), PostData.String(insertData)); if (response.Success) { Log.Logger.Information($"Product [{productForIndex.SearchResultData.Id}]' {productForIndex.SearchResultData.Name}' successfuly indexed"); } else { Log.Logger.Error($"Unable to index product (ID: {productForIndex.SearchResultData.Id}): {response.Body}"); } }
public LowLevelDispatch(IElasticLowLevelClient rawElasticClient) { this._lowLevel = rawElasticClient; }