public bool BulkIndexDesc <T>(IEnumerable <T> inputObject, bool isAsync = false) where T : class
{
try
{
var descriptor = new BulkDescriptor();
foreach (T pageview in inputObject)
{
T item = pageview;
descriptor.Index <T>(op => op.Document(item));
}
if (isAsync)
{
_client.BulkAsync(descriptor);
}
else
{
_client.Bulk(descriptor);
}
return(true);
}
catch (Exception ex)
{
throw;
}
}
public async Task <Response> AddEntitiesAsync <TEntity>(IList <TEntity> entities)
where TEntity : class
{
var response = new Response();
if (entities == null)
{
return(response);
}
if (!entities.Any())
{
return(response);
}
var indexName = ElasticSearchExtensions.GetIndexNameFrom <TEntity>();
var indexExistsResponse = await _elasticClient.Indices.ExistsAsync(indexName)
.ConfigureAwait(false);
if (!indexExistsResponse.Exists)
{
var indexCreateResponse = await CreateIndexAsync <TEntity>(indexName)
.ConfigureAwait(false);
if (!indexCreateResponse.IsValid)
{
response.IsOk = false;
response.ErrorMessage = indexCreateResponse.OriginalException.Message;
response.Exception = indexCreateResponse.OriginalException;
return(response);
}
}
var bulkInsert = await _elasticClient.BulkAsync(b => b
.IndexMany(entities, (op,
item) => op.Index(indexName)
.Id(GetEntityId(item)))
)
.ConfigureAwait(false);
await _elasticClient.Indices.RefreshAsync(indexName)
.ConfigureAwait(false);
if (!bulkInsert.Errors)
{
return(response);
}
response.IsOk = false;
response.ErrorMessage = bulkInsert.OriginalException.Message;
response.Exception = bulkInsert.OriginalException;
return(response);
}
public async Task BulkIndex(IEnumerable <T> documents,
Func <TypeMappingDescriptor <T>, ITypeMapping> mapper = null, bool switchIndex = true)
{
_indexRotator.InitAlias <T>(_aliasName, m => mapper != null ? mapper(m) : m.AutoMap());
var newIndexName = _indexRotator.GetSecondaryIndex(_aliasName, mapper);
var bulk = documents.Aggregate(new BulkDescriptor(newIndexName),
(acc, x) => acc.Index <T>(p => p.Index(newIndexName).Document(x)));
await _esClient.BulkAsync(bulk);
if (switchIndex)
{
await _indexRotator.SwitchIndex(_aliasName);
}
}
protected override async Task ProduceMessages(
StreamName stream,
IEnumerable <ProducedMessage> messages,
ElasticProduceOptions?options,
CancellationToken cancellationToken = default
)
{
var documents = messages.Select(x => x.Message);
var mode = options?.ProduceMode ?? ProduceMode.Create;
var bulk = GetOp(new BulkDescriptor(stream.ToString()));
var result = await _elasticClient.BulkAsync(bulk, cancellationToken);
if (!result.IsValid)
{
if (result.DebugInformation.Contains("version conflict"))
{
SubscriptionsEventSource.Log.Warn("ElasticProducer: version conflict");
}
else
{
throw result.OriginalException ?? throw new InvalidOperationException(result.DebugInformation);
}
}
BulkDescriptor GetOp(BulkDescriptor descriptor)
=> mode switch
{
ProduceMode.Create => descriptor.CreateMany(documents),
ProduceMode.Index => descriptor.IndexMany(documents),
_ => throw new ArgumentOutOfRangeException()
};
}
}
public async Task ReIndexBulkAsync()
{
await elasticClient.DeleteByQueryAsync <ProductViewModel>(q => q.Index("products").MatchAll());
var products = productRepository.GetAll().ToArray();
await elasticClient.BulkAsync(b => b.Index("products").IndexMany(products));
}
private Task <BulkResponse> BulkData(IBulkRequest request)
{
var response = _client.BulkAsync(request);
if (!response.Result.IsValid)
{
if ((response.Result.OriginalException != null && response.Result.OriginalException.Message.Contains("The underlying connection was closed: A connection that was expected to be kept alive was closed by the server")) ||
(response.Result.ItemsWithErrors != null && response.Result.ItemsWithErrors.First().Status == 503))
{
_logger.Warn("Elasticsearch overload, retrying");
throw new TooManyRequestsException();
}
if (response.Exception != null)
{
foreach (var ex in response.Exception.InnerExceptions)
{
_logger.Error(ex, ex.Message);
}
}
throw new HttpException(response.Result.ItemsWithErrors.First().Status, "Something failed trying to insert data into the bulk service", response.Exception);
}
return(response);
}
private static IEnumerable <Func <object> > DocumentCommandsAsync(IElasticClient elastic)
{
return(new List <Func <object> >
{
() => elastic.BulkAsync(
new BulkRequest("test_index")
{
Operations = new List <IBulkOperation>
{
new BulkCreateOperation <Post>(new Post {
Id = 1, Title = "BulkCreateOperation"
})
}
}),
() => elastic.CreateAsync(new CreateRequest <Post>(new Post {
Id = 2, Title = "CreateRequest"
}, "test_index")),
() => elastic.CreateDocumentAsync(new Post {
Id = 3, Title = "CreateDocument"
}),
() => elastic.CountAsync <Post>(),
() => elastic.SearchAsync <Post>(s => s.MatchAll()),
() => elastic.DeleteByQueryAsync(new DeleteByQueryRequest("test_index")
{
Size = 0
})
});
}
/// <summary>
/// Shortcut into the Bulk call that deletes the specified objects
/// <para> </para>http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html
/// </summary>
/// <param name="client"></param>
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
/// <param name="objects">List of objects to delete</param>
/// <param name="index">Override the inferred indexname for T</param>
/// <param name="type">Override the inferred typename for T</param>
public static Task <IBulkResponse> DeleteManyAsync <T>(this IElasticClient client, IEnumerable <T> objects, string index = null, string type = null, CancellationToken cancellationToken = default(CancellationToken))
where T : class
{
var bulkRequest = CreateDeleteBulkRequest(objects, index, type);
return(client.BulkAsync(bulkRequest, cancellationToken));
}
public static async Task <IBulkResponse> BulkAsync <T>(
this IElasticClient client,
IEnumerable <BulkItem <T> > items,
string routing = null,
string indexName = null)
where T : class
{
indexName = indexName ?? client.ConnectionSettings.DefaultIndex;
var response = await client.BulkAsync(bulkDescriptor =>
{
bulkDescriptor = bulkDescriptor.Index(indexName);
if (routing != null)
{
bulkDescriptor = bulkDescriptor.Routing(routing);
}
foreach (var item in items)
{
switch (item.Operation)
{
case BulkOperation.Index:
bulkDescriptor = bulkDescriptor.Index <T>(
indexDescriptor => indexDescriptor
.Routing(item.Routing)
.Document(item.Document)
.Id(item.Id));
break;
case BulkOperation.Create:
bulkDescriptor = bulkDescriptor.Create <T>(
createDescriptor => createDescriptor
.Routing(item.Routing)
.Document(item.Document)
.Id(item.Id));
break;
case BulkOperation.Delete:
bulkDescriptor = bulkDescriptor.Delete <T>(
deleteDescriptor => deleteDescriptor
.Routing(item.Routing)
.Id(item.Id));
break;
case BulkOperation.Upsert:
bulkDescriptor = bulkDescriptor.Update <T>(
updateDescriptor => updateDescriptor
.Routing(item.Routing)
.Doc(item.Document)
.DocAsUpsert(true)
.Id(item.Id));
break;
}
}
return(bulkDescriptor);
});
return(response);
}
/// <summary>
/// Shortcut into the Bulk call that indexes the specified objects
/// <para> </para>http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html
/// </summary>
/// <param name="client"></param>
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
/// <param name="objects">List of objects to index, Id will be inferred (Id property or IdProperty attribute on type)</param>
/// <param name="index">Override the inferred indexname for T</param>
/// <param name="type">Override the inferred typename for T</param>
public static Task <IBulkResponse> IndexManyAsync <T>(this IElasticClient client, IEnumerable <T> objects, string index = null, string type = null)
where T : class
{
var bulkRequest = CreateIndexBulkRequest(objects, index, type);
return(client.BulkAsync(bulkRequest));
}
public async Task InsertRangeAsync <T>(IEnumerable <T> entity, string index) where T : class
{
var indexName = index.GetIndex <T>();
var exists = await IndexExistsAsync(indexName);
if (!exists)
{
await((ElasticClient)_elasticClient).CreateIndexAsync <T>(indexName);
await AddAliasAsync(indexName, typeof(T).Name);
}
var bulkRequest = new BulkRequest(indexName)
{
Operations = new List <IBulkOperation>()
};
var operations = entity.Select(o => new BulkIndexOperation <T>(o)).Cast <IBulkOperation>().ToList();
bulkRequest.Operations = operations;
var response = await _elasticClient.BulkAsync(bulkRequest);
if (!response.IsValid)
{
throw new Exception("批量新增数据失败:" + response.OriginalException.Message);
}
}
public async Task <AppendEventsResult> AppendEvents(
StreamName stream,
ExpectedStreamVersion expectedVersion,
IReadOnlyCollection <StreamEvent> events,
CancellationToken cancellationToken
)
{
var streamName = stream.ToString();
var documents = events.Select(AsDocument).ToArray();
var bulk = new BulkDescriptor(_options.IndexName).CreateMany(documents).Refresh(Refresh.WaitFor);
var result = await _client.BulkAsync(bulk, cancellationToken);
return(result.IsValid
? new AppendEventsResult(0, documents.Last().StreamPosition + 1)
: throw new ApplicationException($"Unable to add events: {result.DebugInformation}"));
PersistedEvent AsDocument(StreamEvent evt)
=> new(
evt.Id.ToString(),
TypeMap.Instance.GetTypeName(evt.Payload !),
evt.Position + 1,
evt.ContentType,
streamName,
(ulong)evt.Position + 1,
evt.Payload,
evt.Metadata.ToHeaders(),
DateTime.Now
);
}
public virtual Task <IBulkResponse> BulkAsync(IBulkRequest request, string callerName = "")
{
var timer = Stopwatch.StartNew();
var result = _client.BulkAsync(request);
SendLog(null, null, timer.ElapsedMilliseconds, $"Bulk Async {callerName}");
return(result);
}
/// <summary>
/// Shortcut into the Bulk call that indexes the specified objects
/// <para> </para>
/// https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
/// </summary>
/// <param name="client"></param>
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
/// <param name="objects">List of objects to index, Id will be inferred (Id property or IdProperty attribute on type)</param>
/// <param name="index">Override the inferred indexname for T</param>
/// <param name="type">Override the inferred typename for T</param>
public static Task <BulkResponse> IndexManyAsync <T>(this IElasticClient client, IEnumerable <T> objects, IndexName index = null,
CancellationToken cancellationToken = default
)
where T : class
{
var bulkRequest = CreateIndexBulkRequest(objects, index);
return(client.BulkAsync(bulkRequest, cancellationToken));
}
public async Task <IActionResult> BulkInsert([FromBody] List <Company> companies)
{
try
{
var queryResponse = await _elasticClient.BulkAsync(x => x
.CreateMany(companies)
.Index(IndexName));
if (queryResponse.IsValid)
{
return(Ok(queryResponse.Items.Count));
}
return(BadRequest(queryResponse.ServerError.Error));
}
catch (System.Exception ex)
{
return(BadRequest(ex.Message));
}
}
/// <summary>
/// 添加到索引
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="client"></param>
/// <param name="indexName"></param>
/// <param name="data"></param>
/// <returns></returns>
public static async Task AddToIndexAsync <T>(this IElasticClient client, string indexName, params T[] data) where T : class, IElasticSearchIndex
{
var bulk = new BulkRequest(indexName)
{
Operations = ConvertHelper.NotNullList(data).Select(x => new BulkIndexOperation <T>(x)).ToArray()
};
var response = await client.BulkAsync(bulk);
response.ThrowIfException();
}
/// <summary>
/// 批量保存文档
/// </summary>
/// <typeparam name="TDocument">文档类型</typeparam>
/// <param name="documents">文档对象列表</param>
/// <param name="index">索引名称。注意：必须小写</param>
/// <param name="timeout">超时时间间隔。单位：毫秒，默认值：300000，即5分钟</param>
/// <param name="cancellationToken">取消令牌</param>
public async Task <BulkResponse> BulkSaveAsync <TDocument>(IEnumerable <TDocument> documents, string index = null, double timeout = 300000, CancellationToken cancellationToken = default)
where TDocument : class
{
index = GetIndexName(Helper.SafeIndexName <TDocument>(index));
return(await _client.BulkAsync(x => x
.Index(index)
.IndexMany(documents)
.Timeout(timeout),
cancellationToken));
}
private async Task BulkAddOrUpdate <T, TKey>(string indexName, List <T> list) where T : class
{
var bulk = new BulkRequest(indexName)
{
Operations = new List <IBulkOperation>()
};
foreach (var item in list)
{
bulk.Operations.Add(new BulkIndexOperation <T>(item));
}
var response = await _esClient.BulkAsync(bulk);
if (response.Errors)
{
throw new ElasticSearchException(
$"Bulk InsertOrUpdate Document failed at index {indexName} :{response.ServerError.Error.Reason}");
}
}
public static Task <IBulkResponse> IndexManyAsync <T>(this IElasticClient client, IEnumerable <T> objects, Func <T, string> getParent, Func <T, string> getIndex = null, string type = null) where T : class
{
if (getParent == null && getIndex == null)
{
return(client.IndexManyAsync(objects, null, type));
}
var indexBulkRequest = CreateIndexBulkRequest(objects, getIndex, type, getParent);
return(client.BulkAsync(indexBulkRequest));
}
public async Task PublishToElastic(IClass createdClass)
{
List <IBulkOperation> bulkOps = new List <IBulkOperation>();
var createReq = new BulkIndexOperation <Class>(createdClass as Class);
createReq.Index = ElasticConfig.Indices.Main.Name();
bulkOps.Add(createReq);
var createReqLog = new BulkIndexOperation <Activity <string> >(new Activity <string>()
{
Id = Guid.NewGuid().ToString(),
Timestamp = DateTime.UtcNow,
Payload = new ActivityPayload <string>()
{
Data = string.Format($"SearchProvider: Create(), Created Class - {JsonConvert.SerializeObject(createdClass)}")
},
ActivityType = ActivityType.ClassCreate
});
createReqLog.Index = ElasticConfig.Indices.Logs.Name();
bulkOps.Add(createReqLog);
await _elasticProvider.BulkAsync(new BulkRequest()
{
Operations = bulkOps
});
}
public async Task SaveBulkAsync(T[] entities)
{
var result = await _elasticClient.BulkAsync(b => b.Index("service_management").IndexMany(entities));
if (result.Errors)
{
// the response can be inspected for errors
foreach (var item in result.ItemsWithErrors)
{
Console.WriteLine("Failed to index document {0} : {1}", item.Id, item.Error);
}
}
}
/// <summary>
/// Inserts/ Index documents
/// </summary>
/// <param name="entities"></param>
/// <returns></returns>
public async Task BulkInsertAsync(IEnumerable <QuotesModel> entities)
{
var request = new BulkDescriptor();
foreach (var entity in entities)
{
request
.Index <QuotesModel>(op => op
.Id(Guid.NewGuid().ToString())
.Document(entity));
}
await Client.BulkAsync(request);
}
public async Task SaveBulkAsync(IEnumerable <AuditEntity> audits)
{
var result = await _elasticClient.BulkAsync(b => b.Index("audits").IndexMany(audits));
if (result.Errors)
{
// the response can be inspected for errors
foreach (var itemWithError in result.ItemsWithErrors)
{
_logger.LogError("Failed to index document {0}: {1}",
itemWithError.Id, itemWithError.Error);
}
}
}
public async Task SaveBulkAsync(Product[] products)
{
_cache.AddRange(products);
var result = await _elasticClient.BulkAsync(b => b.Index("products").IndexMany(products));
if (result.Errors)
{
// the response can be inspected for errors
foreach (var itemWithError in result.ItemsWithErrors)
{
_logger.LogError("Failed to index document {0}: {1}",
itemWithError.Id, itemWithError.Error);
}
}
}
public async Task SaveBulkAsync(Address[] addresses)
{
_cache.AddRange(addresses); // TODO : WARNING HARDCODED INDEX
var result = await _elasticClient.BulkAsync(b => b.Index("addresses").IndexMany(addresses));
if (result.Errors)
{
// the response can be inspected for errors
foreach (var itemWithError in result.ItemsWithErrors)
{
_logger.LogError("Failed to index document {0}: {1}",
itemWithError.Id, itemWithError.Error);
}
}
}
private async Task UpdateData(List <Feature> features, string alias)
{
var result = await _elasticClient.BulkAsync(bulk =>
{
foreach (var feature in features)
{
bulk.Index <Feature>(i => i.Index(alias).Document(feature).Id(GetId(feature)));
}
return(bulk);
});
if (result.IsValid == false)
{
result.ItemsWithErrors.ToList().ForEach(i => _logger.LogError($"Inserting {i.Id} failed with error: {i.Error?.Reason ?? string.Empty} caused by: {i.Error?.CausedBy?.Reason ?? string.Empty}"));
}
}
/**
* ==== Multiple documents with bulk
*
* If you require finer grained control over indexing many documents you can use the `Bulk` and `BulkAsync` methods and use the descriptors to
* customise the bulk calls.
*
* As with the `IndexMany` methods above, documents are sent to the `_bulk` endpoint in a single HTTP request.
* This does mean that consideration will need to be given to the overall size of the HTTP request. For indexing large numbers
* of documents it may be sensible to perform multiple separate `Bulk` calls.
*/
public async Task BulkIndexDocuments()
{
//hide
var people = new [] { new Person {
Id = 1, FirstName = "Martijn", LastName = "Laarman"
} };
var bulkIndexResponse = client.Bulk(b => b
.Index("people")
.IndexMany(people)); //<1> synchronous method that returns an IBulkResponse, the same as IndexMany and can be inspected in the same way for errors
// Alternatively, documents can be indexed asynchronously similar to IndexManyAsync
var asyncBulkIndexResponse = await client.BulkAsync(b => b
.Index("people")
.IndexMany(people)); //<2> asynchronous method that returns a Task<IBulkResponse> that can be awaited
}
/// <summary>
/// Inserts documents in bulk into Elastic Search.
/// </summary>
/// <param name="docs">The documents to insert.</param>
public virtual async Task <IBulkResponse> Index(IEnumerable <T> docs)
{
var request = new BulkDescriptor();
foreach (var req in docs)
{
var reqES = PrepareDoc(req);
request.Index <T>(op => op
.Id(reqES.Id)
.Index(_serviceConfig.IndexName)
.Document(reqES)
);
}
return(await _client.BulkAsync(request));
}
public override async Task ProfileAsync(IElasticClient client, ColoredConsoleWriter output)
{
for (var i = 0; i < _iterations; i++)
{
var bulkResponse = await client.BulkAsync(b => b
.IndexMany(Developer.Generator.Generate(_itemsPerIteration), (bd, d) => bd
.Index(Infer.Index<Developer>())
.Document(d)
)).ConfigureAwait(false);
if (!bulkResponse.IsValid)
if (bulkResponse.Errors)
foreach (var error in bulkResponse.ItemsWithErrors)
output.WriteOrange($"error with id {error.Id}. message: {error.Error.Reason}");
}
}
public async Task<IBulkResponse> Handle(SaveOrUpdateMovieTrailerCommand request, CancellationToken cancellationToken)
{
if (!request.MovieTrailers.Any())
{
return null;
}
var descriptor = new BulkDescriptor();
foreach (var movieTrailer in request.MovieTrailers)
{
descriptor.Index<MovieTrailer>(op => op.Document(movieTrailer));
}
var response = await elasticClient.BulkAsync(descriptor);
return response;
}
public async Task End(Exception ex = null)
{
if (ex != null)
{
return;
}
if (_pendingSize > 0)
{
// Index the pending docs
var response = await _client.BulkAsync(_pendingDocs.Consistency(Elasticsearch.Net.Consistency.Quorum));
if (response.Errors)
{
throw new StorageException(response);
}
}
}
