/// <summary>
/// This will rebuild an entire index alias behind the scenes with no downtime. It creates a new index
/// and populates it then uses Elasticsearch's hot-swapping technique to bring it online.
/// </summary>
public async static Task RebuildIndexWithHotSwapAsync(
this IElasticLowLevelClient client,
string alias,
JObject indexMapping,
Func <Task <IEnumerable <BulkIndexingDoc> > > reader,
CancellationToken ctx = default(CancellationToken))
{
var deletableIndices = JArray.Parse((await client.CatAliasesAsync <StringResponse>(alias, new CatAliasesRequestParameters {
Format = "json"
}, ctx)).Body)
.Select(x => new
{
alias = x.Value <string>("alias"),
index = x.Value <string>("index")
})
.ToList();
var index = GenerateUniqueIndexNameForAlias(alias);
await client.IndicesCreateAsync <StringResponse>(index, PostData.String(indexMapping?.ToString()), ctx : ctx);
while (!ctx.IsCancellationRequested)
{
// TODO: If an exception is thrown, delete the half-created index
var docs = await reader();
if (ctx.IsCancellationRequested || !docs.Any())
{
break;
}
var body = docs.SelectMany(doc => doc.ToLines().Select(x => x.ToString(Formatting.None)));
var bulkResponse = await client.BulkAsync <StringResponse>(index, PostData.MultiJson(body));
ThrowOnPartialBulkSuccess(bulkResponse);
}
if (ctx.IsCancellationRequested)
{
return;
}
var actions = deletableIndices.Select(idx => (object)new
{
remove = new { idx.index, idx.alias }
}).ToList();
actions.Add(new { add = new { index, alias } });
// This is the hot-swap. The actions in the list are performed atomically
await client.IndicesUpdateAliasesForAllAsync <StringResponse>(PostData.String(JObject.FromObject(new
{
actions
}).ToString()), ctx : ctx);
if (deletableIndices.Any())
{
await client.IndicesDeleteAsync <StringResponse>(string.Join(",", deletableIndices.Select(x => x.index)), ctx : ctx);
}
}
/// <summary>
/// This performs a bulk command against the index pointed at by an alias. This is to avoid the potential for
/// a hotswap to change the underlying index from under us while we're reindexing. The index mapping is only
/// used if the aliased index does not yet exist.
/// </summary>
public async static Task BulkTargetingAliasAsync(
this IElasticLowLevelClient client,
string alias,
JObject indexMappingIfNotExists,
Func <Task <IEnumerable <BulkIndexingDoc> > > reader,
CancellationToken ctx = default(CancellationToken))
{
var targetIndices = JArray.Parse((await client.CatAliasesAsync <StringResponse>(alias, new CatAliasesRequestParameters {
Format = "json"
}, ctx)).Body)
.Select(x => x.Value <string>("index"))
.ToList();
string index;
if (targetIndices.Count == 0)
{
index = GenerateUniqueIndexNameForAlias(alias);
await client.IndicesCreateAsync <StringResponse>(index, PostData.String(indexMappingIfNotExists?.ToString()), ctx : ctx);
await client.IndicesUpdateAliasesForAllAsync <StringResponse>(PostData.String(JObject.FromObject(new
{
actions = new[] { new { add = new { index, alias } } }
}).ToString()), ctx : ctx);
}
else if (targetIndices.Count > 1)
{
throw new ArgumentException(
$"{nameof(BulkTargetingAliasAsync)} can only be used against an alias targeting a single index. The `{alias}` alias covers {targetIndices.Count} indices",
nameof(alias));
}
else
{
index = targetIndices.First();
}
while (!ctx.IsCancellationRequested)
{
var docs = await reader();
if (ctx.IsCancellationRequested || !docs.Any())
{
break;
}
var body = docs.SelectMany(doc => doc.ToLines().Select(x => x.ToString(Formatting.None)));
var bulkResponse = await client.BulkAsync <StringResponse>(index, PostData.MultiJson(body));
ThrowOnPartialBulkSuccess(bulkResponse);
}
}
public LogToElasticsearch(IElasticLowLevelClient es) : base(async(logBatch) =>
{
var multiJson = logBatch.ToEsMultiJson();
//let _ = Write(multiJson)
var bulkRequest = PostData.String(multiJson);
var bulkRequestParameters = new BulkRequestParameters
{
// todo
};
var bulkResponse = await es.BulkAsync <StringResponse>(bulkRequest, bulkRequestParameters);
return(bulkResponse);
})
{
}
public StoreToElasticsearch(IElasticLowLevelClient es, string indexName = "terraform") : base(async(request) =>
{
var logBatch = JObject.FromObject(request.Data, AlteredJson.DefaultJsonSerializer);
var multiJson = logBatch.ToEsMultiJson(indexName);
//let _ = Write(multiJson)
var bulkRequest = PostData.String(multiJson);
var bulkRequestParameters = new BulkRequestParameters
{
// todo
};
var bulkResponse = await es.BulkAsync <StringResponse>(bulkRequest, bulkRequestParameters);
return(bulkResponse);
})
{
}
public async Task <SentPayloadResult> SendPayloadAsync(List <string> payload, bool first)
{
try
{
if (payload == null || !payload.Any())
{
return(new SentPayloadResult(null, true));
}
var response = await _elasticLowLevelClient.BulkAsync <DynamicResponse>(PostData.MultiJson(payload));
if (response.Success)
{
var cleanPayload = new List <string>();
var invalidPayload = GetInvalidPayloadAsync(response, payload, out cleanPayload);
if ((cleanPayload?.Any() == false) && first)
{
await SendPayloadAsync(cleanPayload, false);
}
return(new SentPayloadResult(response, true, invalidPayload));
}
else
{
SelfLog.WriteLine("Received failed ElasticSearch shipping result {0}: {1}", response.HttpStatusCode,
response.OriginalException);
return(new SentPayloadResult(response, false,
new InvalidResult()
{
StatusCode = response.HttpStatusCode ?? 500,
Content = response.OriginalException.ToString()
}));
}
}
catch (Exception ex)
{
SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex);
return(new SentPayloadResult(null, false, null, ex));
}
}
public async Task CreateMany(string name, IEnumerable <string> items)
{
var data = PostData.MultiJson(items);
await _client.BulkAsync <ElasticSearchResponse>(name, data);
}
