public async Task <int> CreateAsync(IList <T> items, Refresh refresh = Refresh.WaitFor)
{
if (!items.HasItems())
{
throw new ArgumentException(string.Format(Utils.ARGUMENT_EMPTY_LIST_MESSAGE, nameof(items)), nameof(items));
}
var descriptor = new BulkDescriptor();
foreach (var item in items)
{
if (string.IsNullOrWhiteSpace(item.Id))
{
descriptor.Index <T>(x => x
.Document(item));
}
else
{
descriptor.Create <T>(x => x
.Document(item));
}
}
descriptor.Refresh(refresh);
var response = await _client.BulkAsync(descriptor);
return(!response.IsValid ? 0 : response.Items.Count);
}
public async Task Publish(List <T> records, CancellationToken stopToken)
{
foreach (var batch in records.Batch(_publishBatchSize))
{
await Instance.BulkAsync(b => b.IndexMany(batch), stopToken);
}
}
public async Task AddTestDataInLoop()
{
log.LogInformation("Inside AddTest data");
try
{
await Task.Delay(5000);
for (int i = 0; i < 4; i++)
{
IPingResponse x = await client.PingAsync();
if (x.ApiCall.Success)
{
log.LogInformation(DateTime.Now.ToLocalTime() + " Connection successful to - " + x.ApiCall.Uri);
break;
}
else
{
log.LogWarning(DateTime.Now.ToLocalTime() + " Unable to connect to - " + x.ApiCall.Uri);
await Task.Delay(i * 1000);
}
}
var allProducts = new List <Product>();
var descriptor = new BulkDescriptor();
for (int i = 0; i < itemCount; i++)
{
Product p = new Product()
{
Id = i, Title = "test " + i, Price = i, CategoryId = 1
};
descriptor.Index <Product>(op => op.Document(p));
allProducts.Add(p);
}
for (int j = 0; j < 10; j++)
{
Category c = new Category()
{
Id = j,
Name = "Category " + j,
Products = allProducts.Where(p => p.CategoryId == j).ToList <Product>()
};
descriptor.Index <Category>(op => op.Document(c));
}
log.LogWarning("before bulk async");
var result = await client.BulkAsync(descriptor);
log.LogWarning("after bulk async");
}
catch (Exception ex)
{
log.LogError(DateTime.Now.ToLocalTime() + " - Ex Caught:" + ex.Message);
}
}
public async Task <bool> TryAddAsync(TKey key, ISet <T> value, bool overwrite = false, CancellationToken cancellationToken = default(CancellationToken))
{
var items = await value.AsEnumerableAsync(cancellationToken).ConfigureAwait(false);
var documents = new Dictionary <string, T>();
await items.ForEachAsync(
async item =>
{
documents.Add($"{key}:{GetSubKeyValue(item)}", item);
},
cancellationToken);
var descriptor = new BulkDescriptor();
foreach (var document in documents)
{
descriptor.Index <T>(op => op
.Id(document.Key)
.Document(document.Value));
}
var result = await ElasticClient.BulkAsync(descriptor, cancellationToken);
return(result.IsValid);
}
private static List <Func <object> > DocumentCommandsAsync(ElasticClient elastic)
{
return(new List <Func <object> >
{
() => elastic.BulkAsync(new BulkRequest("test_index")
{
Operations = new List <IBulkOperation>
{
new BulkCreateOperation <Post>(new Post
{
Id = 1,
Title = "BulkCreateOperation",
})
}
}),
() => elastic.CreateAsync <Post>(new Post
{
Id = 2,
Title = "Create",
}),
() => elastic.CountAsync <Post>(),
() => elastic.SearchAsync <Post>(s => s.MatchAll()),
() => elastic.DeleteByQueryAsync(new DeleteByQueryRequest("test_index")
{
Size = 0,
}),
});
}
private static Task <BulkResponse> IndexBatch(ElasticClient client, string indexName, IEnumerable <dynamic> documents)
{
return(client.BulkAsync(b => b
.Index(indexName)
.IndexMany(documents)
));
}
public async Task SyncData()
{
//1290213 = 13x100000
for (int i = 1; i < 14; i++)
{
var recepies = await _dbContext
.Recipes.Where(r => r.Id <= 100000 *i && r.Id > 100000 *(i - 1))
.Include(r => r.Ingredients)
.Include(r => r.RecipeTag)
.ThenInclude(rt => rt.Tag)
.ToListAsync();
var recepiesEsModel = _mapper.Map <List <RecipeSearchModel> >(recepies);
var request = new BulkRequest(indexName)
{
Operations = new List <IBulkOperation>()
};
var bulkOperations = recepiesEsModel.Select(r => new BulkIndexOperation <RecipeSearchModel>(r));
request.Operations.AddRange(bulkOperations);
await _elasticClient.BulkAsync(request);
}
Console.WriteLine("Sync completed");
}
private async Task <bool> PushBulk(IEnumerable <ElasticSDResult> results)
{
if (elasticClient == null)
{
return(false);
}
var descriptor = new BulkDescriptor();
descriptor.CreateMany <ElasticSDResult>(results);
Console.WriteLine($"Inserting {results.Count()} superdump results into ES...");
var sw = new Stopwatch();
sw.Start();
var result = await elasticClient.BulkAsync(descriptor);
sw.Stop();
if (result.IsValid)
{
Console.WriteLine($"Finished inserting in {sw.Elapsed})");
return(true);
}
// something failed
Console.WriteLine($"PushBulk failed for {result.ItemsWithErrors?.Count()} items. servererror: {result.ServerError?.ToString()}, error on first item: {result.ItemsWithErrors?.FirstOrDefault()?.Error}");
return(false);
}
private static async Task CreateIndexAndTestData(string indexName, int count)
{
var counter = 1;
var testUsers = new Faker <User>()
.RuleFor(x => x.Email, f => f.Person.Email)
.RuleFor(x => x.FirstName, f => f.Person.FirstName)
.RuleFor(x => x.LastName, f => f.Person.LastName)
.RuleFor(x => x.State, f => f.Person.Address.State)
.RuleFor(x => x.IsActive, f => f.Random.Bool(.8f))
.RuleFor(x => x.Number, f => counter++)
.RuleFor(x => x.RegistrationDate, f => f.Date.Recent(100))
.RuleFor(x => x.EmptyValue, f =>
{
if (counter % 2 == 0)
{
return(f.Random.AlphaNumeric(5));
}
if (counter % 3 == 0)
{
return(null);
}
return("");
})
.RuleFor(x => x.LastOrderDate, f => f.Date.Recent(365).OrNull(f));
var userBatch = testUsers.Generate(count);
var bulkResponse = await client.BulkAsync(x => x
.Index(indexName)
.IndexMany(userBatch));
Console.WriteLine(bulkResponse.ToString());
}
protected async override Task OnConsume(string example, TaskParameters taskParameters, ClientActiveTasks cleanup)
{
registerTask(cleanup, "BuildElasticIndexTask"); // may throw AlreadyInProgress exception
GetLogger().LogInformation("BuildElasticIndexTask Starting");
using (var _context = CTDbContext.CreateDbContext())
{
CaptionQueries captionQueries = new CaptionQueries(_context);
var all_transcriptions = await _context.Transcriptions.Where(t => t.Language == Languages.ENGLISH_AMERICAN).ToListAsync();
foreach (var transcription in all_transcriptions)
{
var all_captions = transcription.Captions;
// each index has the unique name "index_string_unique", the current in use one has the alias "index_string_alias"
var index_string_base = transcription.Id + "_" + Languages.ENGLISH_AMERICAN.ToLower(System.Globalization.CultureInfo.CurrentCulture);
var index_string_unique = index_string_base + "_" + $"{DateTime.Now:yyyyMMddHHmmss}";
var index_string_alias = index_string_base + "_" + "primary";
var asyncBulkIndexResponse = await _client.BulkAsync(b => b
.Index(index_string_unique)
.IndexMany(all_captions)
);
var alias_exist = await _client.Indices.ExistsAsync(index_string_alias);
if (alias_exist.Exists)
{
var oldIndices = await _client.GetIndicesPointingToAliasAsync(index_string_alias);
var oldIndexName = oldIndices.First().ToString();
var indexResponse = await _client.Indices.BulkAliasAsync(new BulkAliasRequest
{
Actions = new List <IAliasAction>
{
new AliasRemoveAction {
Remove = new AliasRemoveOperation {
Index = oldIndexName, Alias = index_string_alias
}
},
new AliasAddAction {
Add = new AliasAddOperation {
Index = index_string_unique, Alias = index_string_alias
}
}
}
});
}
else
{
var putAliasResponse = await _client.Indices.PutAliasAsync(new PutAliasRequest(index_string_unique, index_string_alias));
}
}
}
GetLogger().LogInformation("BuildElasticIndexTask Done");
}
public async Task <IActionResult> IndexBulkBad()
{
var bulkIndexResponse = await _client.BulkAsync(b => b
.Index("authorbooks")
.IndexMany(Data.AuthorBooks));
return(Ok(bulkIndexResponse.DebugInformation));
}
public async Task AddToIndex(List <TEntity> records, string indexName)
{
var bullkResult =
await Client
.BulkAsync(b => b
.Index(indexName)
.CreateMany(records)
);
}
public async Task <bool> Set(IEnumerable <ScoreCard> entities)
{
var result = await _elasticClient.BulkAsync(b => b
.Index(_defaultIndex)
.IndexMany(entities)
);
return(result.IsValid);
}
public async Task BulkInsertAsync <T>(IEnumerable <T> input) where T : class, new()
{
await _client.BulkAsync(new BulkRequest()
{
Operations = input.Select(o =>
(new BulkIndexOperation <T>(o)) as IBulkOperation
).ToList()
});
}
private async Task<BulkResponse> IndexMany(IEnumerable<T> models)
{
var descriptor = new BulkDescriptor();
descriptor.IndexMany(models, (bd, q) => bd
.Index(_indexName)
.Id(q.Id.ToString())
);
return await _client.BulkAsync(descriptor);
}
public void InsertFiles()
{
var bulkIndexResponse = client.BulkAsync(b => b
.Index("texts")
.IndexMany(files)
.Refresh(Elasticsearch.Net.Refresh.True)
);
bulkIndexResponse.Wait();
files.Clear();
}
public async Task <bool> indexManyUsers(List <User> newUsers)
{
var indexName = "users";
var indexResponse = await esClient.BulkAsync(f => f.Index(indexName).IndexMany(newUsers, (descriptor, s) => descriptor.Index(indexName)));
if (indexResponse.IsValid)
{
return(true);
}
return(false);
}
public async void IndexCustomers()
{
await _client.Indices.DeleteAsync("customers");
using (SqlConnection conn = new SqlConnection(_dbOptions.Value.ConnectionString))
{
try
{
conn.Open();
var customers = await conn.QueryAsync <Customers>("SELECT * FROM Customers WHERE IsActive = 1;");
await _client.BulkAsync(request => request
.Index("customers")
.IndexMany <Customers>(customers));
}
catch (SqlException exc)
{
Console.WriteLine(exc.Message);
}
}
}
public static Task <IBulkResponse> BulkInsertAsync <T>(IEnumerable <T> elasticIndexes, string parentId = null) where T : class, new()
{
var descriptor = new BulkDescriptor();
foreach (var i in elasticIndexes)
{
descriptor.Index <T>(op => op.Document(i).Parent(parentId));
}
var bulkresult = Client.BulkAsync(descriptor);
return(bulkresult);
}
public async Task IndexManyAsync(IEnumerable <TDoc> documents, Func <BulkIndexDescriptor <TDoc>, TDoc, IBulkIndexOperation <TDoc> > bulkIndexSelector, CancellationToken cancellationToken)
{
if (documents == null)
{
throw new ArgumentNullException(nameof(documents));
}
var indexResponse = await _cl.BulkAsync(bd => bd.IndexMany(documents, bulkIndexSelector), cancellationToken);
if (!indexResponse.IsValid)
{
throw new EsIndexManyException(indexResponse);
}
}
async Task IServcieOperationStorage.ServcieOperationStorage(IEnumerable <SpanServiceOperation> spanServiceOperation, CancellationToken cancellationToken)
{
if (spanServiceOperation == null)
{
return;
}
List <ServiceOperationModel> serviceOperationModels = new List <ServiceOperationModel>();
foreach (var serviceOperation in spanServiceOperation.Where(w => !string.IsNullOrEmpty(w.Operation) &&
w.Process != null &&
!string.IsNullOrEmpty(w.Process.ServiceName)))
{
var cacheKey = CreateCacheKey(serviceOperation.Process.ServiceName, serviceOperation.Operation);
if (MemoryCache.TryGetValue(cacheKey, out var _))
{
return;
}
MemoryCache.Set(cacheKey, true);
serviceOperationModels.Add(new ServiceOperationModel()
{
Operation = serviceOperation.Operation,
Service = serviceOperation.Process.ServiceName,
});
}
if (serviceOperationModels.Count > 0)
{
var bulkRequest = new BulkRequest {
Operations = new List <IBulkOperation>()
};
foreach (var serviceOperationModel in serviceOperationModels)
{
var operation = new BulkIndexOperation <ServiceOperationModel>(serviceOperationModel)
{
Index = _IndexName
};
bulkRequest.Operations.Add(operation);
}
var result = await ElasticClient.BulkAsync(bulkRequest, cancellationToken);
foreach (var item in result.ItemsWithErrors)
{
ServiceOperationModel source;
if ((source = item.GetResponse <ServiceOperationModel>()?.Source) != null)
{
var cacheKey = CreateCacheKey(source.Service, source.Operation);
MemoryCache.Remove(cacheKey);
}
}
}
}
public async Task RunAsync()
{
// if the index exists, let's delete it
// you probably don't want to do this kind of
// index management in a production environment
var index = await client.Indices.ExistsAsync(IndexName);
if (index.Exists)
{
await client.Indices.DeleteAsync(IndexName);
}
// let's create the index
var createResult =
await client.Indices.CreateAsync(IndexName, c => c
.Settings(s => s
.Analysis(a => a
// our custom search analyzer
.AddSearchAnalyzer()
)
)
.Map <CapitalSearchDocument>(m => m.AutoMap())
);
// let's load the data
var file = File.Open("capital_cities.csv", FileMode.Open);
using (var csv = new CsvReader(new StreamReader(file)))
{
csv.Configuration.Delimiter = ",";
// describe's the csv file
csv.Configuration.RegisterClassMap <CapitalCitiesMapping>();
var records = csv
.GetRecords <CapitalCityRecord>()
.Select(record => new CapitalSearchDocument(record))
.ToList();
// we are pushing all the data in at once
var bullkResult =
await client
.BulkAsync(b => b
.Index(IndexName)
.CreateMany(records)
);
}
}
public async Task <BulkResponse> Handle(CollectionCommand request, CancellationToken cancellationToken)
{
var response = request.AlbumCollection
.Select(x => new BulkIndexOperation <Album>(x))
.Cast <IBulkOperation>().ToList();
var bulkRequest = new BulkRequest()
{
Refresh = new Refresh(),
Operations = response
};
_bulkResponse = await _client.BulkAsync(bulkRequest, cancellationToken);
return(_bulkResponse);
}
private Task BulkStore(IEnumerable <Span> spans, CancellationToken cancellationToken)
{
var bulkRequest = new BulkRequest {
Operations = new List <IBulkOperation>()
};
foreach (var span in spans)
{
var operation = new BulkIndexOperation <Span>(span)
{
Index = _indexManager.CreateTracingIndex(DateTimeOffset.Now)
};
bulkRequest.Operations.Add(operation);
}
return(_elasticClient.BulkAsync(bulkRequest, cancellationToken));
}
/// <summary>
/// Upload docs into ES
/// </summary>
/// <returns></returns>
public async Task <bool> UploadDocs <T>(string indName, IEnumerable <T> items) where T : class
{
BulkResponse resp = null;
try
{
resp = await _client.BulkAsync(b => b.Index(indName).IndexMany(items));
_logger.LogInformation($"Documents have been uploaded into index {indName}");
}
catch (Exception)
{
_logger.LogInformation($"An error occurred while uploading documents");
return(false);
}
return(resp.IsValid);
}
public static async Task Add(ElasticClient client, IEnumerable <FileElementDTO> books)
{
var overflow = (books.Count() % 10000 == 0) ? 0 : 1;
var numRuns = books.Count() / 10000 + overflow;
for (int n = 0; n < numRuns; n++)
{
var page = books.Skip(n * 10000).Take(10000);
var bulkResponse = await client.BulkAsync(b => b.IndexMany(page));
if (!bulkResponse.IsValid)
{
throw new InvalidElasticSearchResponseException("AddOrUpdateMany-call to ElasticSearch did not return a valid response",
bulkResponse.OriginalException);
}
}
}
public async Task FlushAsync(CancellationToken cancellationToken = default(CancellationToken))
{
var bulk = new BulkDescriptor();
foreach (var document in _documents)
{
document.AddToBulkDescriptor(bulk, _attribute.Index, _attribute.IndexType);
}
var response = await _client.BulkAsync(bulk);
// TODO - handle logging on failure, throw exception, ...?
if (response.Errors)
{
foreach (var item in response.ItemsWithErrors)
{
Console.WriteLine(item.Error.ToString());
}
}
}
public virtual async Task UpsertAsync(IEnumerable <T> source, CancellationToken cancellationToken)
{
// Validate parameters.
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
// The request.
IBulkRequest request = new BulkDescriptor().
Index(Index.Name).
Type <T>().
IndexMany(source, (d, t) => d.Document(t));
// Send the request.
IBulkResponse response = await ElasticClient.BulkAsync(request, cancellationToken).ConfigureAwait(false);
// Throw if there is an error.
response.ThrowIfError();
}
private Task BulkStore(IEnumerable <Span> spans, CancellationToken cancellationToken)
{
var bulkRequest = new BulkRequest {
Operations = new List <IBulkOperation>()
};
foreach (var span in spans)
{
var operation = new BulkIndexOperation <Span>(span)
{
Index = _indexManager.CreateTracingIndex(DateTimeOffset.UtcNow)
};
bulkRequest.Operations.Add(operation);
}
//GeoDistanceQuery query = new GeoDistanceQuery
//{
// Distance = new Distance(100, DistanceUnit.Meters),
// Location = new GeoLocation(30, 120),
// DistanceType = GeoDistanceType.SloppyArc,
// Field = new Field("")
//};
//GeoDistanceSort sort = new GeoDistanceSort()
//{
// DistanceType = GeoDistanceType.SloppyArc,
// Field = new Field(""),
// GeoUnit = DistanceUnit.Meters,
// Order = SortOrder.Ascending,
// Points = new List<GeoLocation> { new GeoLocation(30, 120) }
//};
//SearchRequest search = new SearchRequest()
//{
// Query = new QueryContainer(query),
// Sort = new List<ISort> { sort },
// Size = 200
//};
//_elasticClient.Search<object>(search).Documents.ToList();
return(_elasticClient.BulkAsync(bulkRequest, cancellationToken));
}
public async Task RunAsync()
{
var index = await esClient.Indices.ExistsAsync(IndexName);
if (index.Exists)
{
await esClient.Indices.DeleteAsync(IndexName);
}
var createResult = await esClient.Indices.CreateAsync(IndexName, d => d
//d.Settings(s=> s.Analysis(ad=> ad.CharFilters(c=> c.
//.Settings(s => s.Analysis(ad => ad.Analyzers(a => a.Custom(IndexAnalyzerName, c => c.Filters("lowercase")))))
.Map <BookSearchDocument>(m => m.AutoMap()));
var books = await db.Books.ToListAsync();
if (books.Any())
{
var bulkResult = await esClient.BulkAsync(b => b.Index(IndexName).CreateMany(books));
}
}
