コード例 #1
0
        /// <summary>
        /// This will rebuild an entire index alias behind the scenes with no downtime. It creates a new index
        /// and populates it then uses Elasticsearch's hot-swapping technique to bring it online.
        /// </summary>
        public async static Task RebuildIndexWithHotSwapAsync(
            this IElasticLowLevelClient client,
            string alias,
            JObject indexMapping,
            Func <Task <IEnumerable <BulkIndexingDoc> > > reader,
            CancellationToken ctx = default(CancellationToken))
        {
            var deletableIndices = JArray.Parse((await client.CatAliasesAsync <StringResponse>(alias, new CatAliasesRequestParameters {
                Format = "json"
            }, ctx)).Body)
                                   .Select(x => new
            {
                alias = x.Value <string>("alias"),
                index = x.Value <string>("index")
            })
                                   .ToList();

            var index = GenerateUniqueIndexNameForAlias(alias);

            await client.IndicesCreateAsync <StringResponse>(index, PostData.String(indexMapping?.ToString()), ctx : ctx);

            while (!ctx.IsCancellationRequested)
            {
                // TODO: If an exception is thrown, delete the half-created index
                var docs = await reader();

                if (ctx.IsCancellationRequested || !docs.Any())
                {
                    break;
                }

                var body         = docs.SelectMany(doc => doc.ToLines().Select(x => x.ToString(Formatting.None)));
                var bulkResponse = await client.BulkAsync <StringResponse>(index, PostData.MultiJson(body));

                ThrowOnPartialBulkSuccess(bulkResponse);
            }

            if (ctx.IsCancellationRequested)
            {
                return;
            }

            var actions = deletableIndices.Select(idx => (object)new
            {
                remove = new { idx.index, idx.alias }
            }).ToList();

            actions.Add(new { add = new { index, alias } });

            // This is the hot-swap. The actions in the list are performed atomically
            await client.IndicesUpdateAliasesForAllAsync <StringResponse>(PostData.String(JObject.FromObject(new
            {
                actions
            }).ToString()), ctx : ctx);

            if (deletableIndices.Any())
            {
                await client.IndicesDeleteAsync <StringResponse>(string.Join(",", deletableIndices.Select(x => x.index)), ctx : ctx);
            }
        }
コード例 #2
0
        /// <summary>
        /// This performs a bulk command against the index pointed at by an alias. This is to avoid the potential for
        /// a hotswap to change the underlying index from under us while we're reindexing. The index mapping is only
        /// used if the aliased index does not yet exist.
        /// </summary>
        public async static Task BulkTargetingAliasAsync(
            this IElasticLowLevelClient client,
            string alias,
            JObject indexMappingIfNotExists,
            Func <Task <IEnumerable <BulkIndexingDoc> > > reader,
            CancellationToken ctx = default(CancellationToken))
        {
            var targetIndices = JArray.Parse((await client.CatAliasesAsync <StringResponse>(alias, new CatAliasesRequestParameters {
                Format = "json"
            }, ctx)).Body)
                                .Select(x => x.Value <string>("index"))
                                .ToList();

            string index;

            if (targetIndices.Count == 0)
            {
                index = GenerateUniqueIndexNameForAlias(alias);

                await client.IndicesCreateAsync <StringResponse>(index, PostData.String(indexMappingIfNotExists?.ToString()), ctx : ctx);

                await client.IndicesUpdateAliasesForAllAsync <StringResponse>(PostData.String(JObject.FromObject(new
                {
                    actions = new[] { new { add = new { index, alias } } }
                }).ToString()), ctx : ctx);
            }
            else if (targetIndices.Count > 1)
            {
                throw new ArgumentException(
                          $"{nameof(BulkTargetingAliasAsync)} can only be used against an alias targeting a single index. The `{alias}` alias covers {targetIndices.Count} indices",
                          nameof(alias));
            }
            else
            {
                index = targetIndices.First();
            }

            while (!ctx.IsCancellationRequested)
            {
                var docs = await reader();

                if (ctx.IsCancellationRequested || !docs.Any())
                {
                    break;
                }

                var body         = docs.SelectMany(doc => doc.ToLines().Select(x => x.ToString(Formatting.None)));
                var bulkResponse = await client.BulkAsync <StringResponse>(index, PostData.MultiJson(body));

                ThrowOnPartialBulkSuccess(bulkResponse);
            }
        }
コード例 #3
0
 public LogToElasticsearch(IElasticLowLevelClient es) : base(async(logBatch) =>
 {
     var multiJson = logBatch.ToEsMultiJson();
     //let _ = Write(multiJson)
     var bulkRequest           = PostData.String(multiJson);
     var bulkRequestParameters = new BulkRequestParameters
     {
         // todo
     };
     var bulkResponse = await es.BulkAsync <StringResponse>(bulkRequest, bulkRequestParameters);
     return(bulkResponse);
 })
 {
 }
コード例 #4
0
 public StoreToElasticsearch(IElasticLowLevelClient es, string indexName = "terraform") : base(async(request) =>
 {
     var logBatch  = JObject.FromObject(request.Data, AlteredJson.DefaultJsonSerializer);
     var multiJson = logBatch.ToEsMultiJson(indexName);
     //let _ = Write(multiJson)
     var bulkRequest           = PostData.String(multiJson);
     var bulkRequestParameters = new BulkRequestParameters
     {
         // todo
     };
     var bulkResponse = await es.BulkAsync <StringResponse>(bulkRequest, bulkRequestParameters);
     return(bulkResponse);
 })
 {
 }
コード例 #5
0
        public async Task <SentPayloadResult> SendPayloadAsync(List <string> payload, bool first)
        {
            try
            {
                if (payload == null || !payload.Any())
                {
                    return(new SentPayloadResult(null, true));
                }
                var response = await _elasticLowLevelClient.BulkAsync <DynamicResponse>(PostData.MultiJson(payload));

                if (response.Success)
                {
                    var cleanPayload   = new List <string>();
                    var invalidPayload = GetInvalidPayloadAsync(response, payload, out cleanPayload);
                    if ((cleanPayload?.Any() == false) && first)
                    {
                        await SendPayloadAsync(cleanPayload, false);
                    }

                    return(new SentPayloadResult(response, true, invalidPayload));
                }
                else
                {
                    SelfLog.WriteLine("Received failed ElasticSearch shipping result {0}: {1}", response.HttpStatusCode,
                                      response.OriginalException);
                    return(new SentPayloadResult(response, false,
                                                 new InvalidResult()
                    {
                        StatusCode = response.HttpStatusCode ?? 500,
                        Content = response.OriginalException.ToString()
                    }));
                }
            }
            catch (Exception ex)
            {
                SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex);
                return(new SentPayloadResult(null, false, null, ex));
            }
        }
コード例 #6
0
        public async Task CreateMany(string name, IEnumerable <string> items)
        {
            var data = PostData.MultiJson(items);

            await _client.BulkAsync <ElasticSearchResponse>(name, data);
        }