public void BulkIndexTest()
{
var objs = new[]
{
new Variable {
PackageId = "pack", Name = "name1"
},
new Variable {
PackageId = "pack", Name = "name2"
},
new Variable {
PackageId = "pack", Name = "name3"
},
new Variable {
PackageId = "pack", Name = "name4"
},
new Variable {
PackageId = "pack2", Name = "black"
}
};
var bulk = new BulkDescriptor();
bulk.IndexMany(objs,
(d, v) => d
.Index("test")
//.Type("variable")
.Document(v));
var objs2 = new[]
{
new Variable {
PackageId = "pack4", Name = "name1"
},
new Variable {
PackageId = "pack4", Name = "name2"
},
};
bulk.IndexMany(objs2,
(d, v) => d
.Index("test")
//.Type("variable")
.Document(v));
bulk.DeleteMany(new Variable[0], (d, v) => d.Index("test").Id("1"));
client.Bulk(bulk);
}
/// <summary>
/// Synchronize mediaRef picons
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <IBulkResponse> SynkPiconsAsync(CancellationToken cancellationToken)
{
var response = await _elasticConnectionClient.Client.Value.SearchAsync <MediaRef>(s => s
.Index(_elasticConnectionClient.ElasticConfig.MediaRefIndex)
.Size(_elasticConnectionClient.ElasticConfig.MaxResultWindow)
.From(0)
.Query(q => q.MatchAll())
, cancellationToken);
var tasks = response.Documents.Select(async m =>
{
//Match all displaynames with boost
var findLogoResponse = await _elasticConnectionClient.Client.Value.SearchAsync <Picon>(x =>
x.Query(q => q.Match(fz =>
fz.Field(f => f.Name)
.Query(m.DisplayNames.FirstOrDefault().RemoveDiacritics().RemoveChars(new char[] { ' ' })))).Size(1), cancellationToken);
if (findLogoResponse.Documents.Any())
{
m.Tvg.Logo = findLogoResponse.Documents.First().RawUrl;
}
});
await Task.WhenAll(tasks);
var descriptor = new BulkDescriptor();
descriptor.IndexMany(response.Documents);
return(await _elasticConnectionClient.Client.Value.BulkAsync(descriptor, cancellationToken));
}
private BulkDescriptor CreateBulk(IEnumerable <G> entities)
{
var bulkDescriptor = new BulkDescriptor();
bulkDescriptor.IndexMany <G>(entities, (descriptor, s) => descriptor.Index(indexName).Id(s.Id.ToString()));
return(bulkDescriptor);
}
public void IndexJsonObject(IEnumerable <Dictionary <string, object> > obj1, string indexName, string type)
{
var bdesc = new BulkDescriptor();
bdesc.IndexMany(obj1, (f, g) => f.Index(indexName).Type(type));
var x = _elasticClient.Bulk(bdesc);
}
public async Task <IBulkResponse> SaveAsync(List <MediaRef> mediasRef, CancellationToken cancellationToken)
{
var descriptor = new BulkDescriptor();
descriptor.IndexMany(mediasRef);
descriptor.Refresh(Elasticsearch.Net.Refresh.True);
return(await _elasticConnectionClient.Client.Value.BulkAsync(descriptor, cancellationToken));
}
private BulkDescriptor CreateBulk(IEnumerable <T> documents, string index, List <string> ids)
{
var bulkDescriptor = new BulkDescriptor();
int idx = 0;
bulkDescriptor.IndexMany(documents, (descriptor, s) => descriptor.Index(index).Id(ids[idx++]));
return(bulkDescriptor);
}
public async Task <IBulkResponse> SaveAsync(List <SitePackChannel> sitepacks, CancellationToken cancellationToken)
{
var descriptor = new BulkDescriptor();
descriptor.Pipeline(ElasticConnectionClient.SITE_PACK_PIPELINE);
descriptor.IndexMany(sitepacks);
descriptor.Refresh(Elasticsearch.Net.Refresh.True);
return(await _elasticConnectionClient.Client.Value.BulkAsync(descriptor, cancellationToken));
}
private async Task<BulkResponse> IndexMany(IEnumerable<T> models)
{
var descriptor = new BulkDescriptor();
descriptor.IndexMany(models, (bd, q) => bd
.Index(_indexName)
.Id(q.Id.ToString())
);
return await _client.BulkAsync(descriptor);
}
protected virtual BulkDescriptor BuildQueryCore(BulkDescriptor descriptor,
string index, bool refreshOnSave)
{
descriptor = descriptor
.IndexMany(_documents, (d, i) => d
.Type(i.GetType())
.Index(index)
)
.Refresh(refreshOnSave);
return(BuildQuery(descriptor));
}
public void BulkIndex <T>(IEnumerable <T> data, string indexName) where T : class
{
var descriptor = new BulkDescriptor();
descriptor.IndexMany(data, (desc, content) => desc.Type(content.GetType()).Index(indexName));
var response = _client.Bulk(descriptor);
if (response.Errors)
{
throw new Exception("Failed while indexing items.", response.OriginalException);
}
}
public void RecordChanges(INestedAspect compareTo, BulkDescriptor bulk)
{
var correlation = byLocalName.CorrelateWith((compareTo as VariableSet)?.byLocalName);
bulk.IndexMany(
correlation.SourceItems.Where(t => t.Item.IsChanged(t.PairedWith)).Select(t => t.Item),
(d, v) => d.Index(Constants.ControlIndex).Document(v));
bulk.DeleteMany(
correlation.NonSourceItems.Select(v => v.Item),
(d, v) => d.Index(Constants.ControlIndex).Id(v.Id));
}
public async Task <IBulkResponse> SynkAsync(CancellationToken cancellationToken)
{
var response = await _elasticConnectionClient.Client.Value.SearchAsync <SitePackChannel>(s => s
.Index(_elasticConnectionClient.ElasticConfig.SitePackIndex)
.Size(_elasticConnectionClient.ElasticConfig.MaxResultWindow)
.From(0)
.Aggregations(a => a
.Terms("unique", te => te
.Field(f => f.Channel_name.Suffix(ElasticConfig.ELK_KEYWORD_SUFFIX))
.Order(o => o
.KeyAscending()
.CountDescending())
)
)
, cancellationToken);
var mediasRef = response.Documents.Select(x => new MediaRef(x.Channel_name, x.Site, x.Country, x.Xmltv_id, x.Id)).ToList();
//var qc = new QueryContainerDescriptor<Picon>();
//var query = mediasRef.SelectMany(x => x.DisplayNames.Take(2))
// .Select(val => qc.Fuzzy(fz => fz.Field(f => f.Name).Value(val.Replace("+", "plus")))).Aggregate((a, b) => a || b);
//var query = mediasRef.Select(x => x.DisplayNames.FirstOrDefault())
// .Select(val => qc.Fuzzy(fz => fz.Field(f => f.Name).Value(val.Replace("+", "plus")))).FirstOrDefault();
//var searchPiconDisc = new SearchDescriptor<Picon>()
// .Size(1)
// .Query(x => query);
var tasks = mediasRef.Select(async m =>
{
var findLogoResponse = await _elasticConnectionClient.Client.Value.SearchAsync <Picon>(x =>
x.Query(q => q.Match(fz =>
fz.Field(f => f.Name)
.Query(m.DisplayNames.FirstOrDefault()))).Size(1), cancellationToken);
if (findLogoResponse.Documents.Any())
{
m.Tvg.Logo = findLogoResponse.Documents.First().RawUrl;
}
});
await Task.WhenAll(tasks);
var descriptor = new BulkDescriptor();
descriptor.IndexMany(mediasRef);
return(await _elasticConnectionClient.Client.Value.BulkAsync(descriptor, cancellationToken));
}
/// <summary>
/// Force update country field
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <IBulkResponse> SyncCountrySitePackAsync(CancellationToken cancellationToken = default)
{
_logger.LogInformation($"Lister les SitePackChannels");
var tvgSites = await _elasticConnectionClient.Client.Value.SearchAsync <SitePackChannel>(s => s
.Index(_elasticConnectionClient.ElasticConfig.SitePackIndex)
.Size(_elasticConnectionClient.ElasticConfig.MaxResultWindow)
.From(0)
, cancellationToken);
var descriptor = new BulkDescriptor();
descriptor.Pipeline(ElasticConnectionClient.SITE_PACK_PIPELINE);
descriptor.IndexMany(tvgSites.Documents);
descriptor.Refresh(Elasticsearch.Net.Refresh.True);
return(await _elasticConnectionClient.Client.Value.BulkAsync(descriptor, cancellationToken));
}
public virtual async Task <IndexingResult> IndexAsync(string documentType, IList <IndexDocument> documents)
{
var indexName = GetIndexName(documentType);
var providerFields = await GetMappingAsync(indexName, documentType);
var oldFieldsCount = providerFields.Count();
var providerDocuments = documents.Select(document => ConvertToProviderDocument(document, providerFields, documentType)).ToList();
var updateMapping = providerFields.Count() != oldFieldsCount;
var indexExits = await IndexExistsAsync(indexName);
if (!indexExits)
{
await CreateIndexAsync(indexName, documentType);
}
if (!indexExits || updateMapping)
{
await UpdateMappingAsync(indexName, documentType, providerFields);
}
var bulkDefinition = new BulkDescriptor();
bulkDefinition.IndexMany(providerDocuments).Index(indexName).Type(documentType);
var bulkResponse = await Client.BulkAsync(bulkDefinition);
await Client.RefreshAsync(indexName);
var result = new IndexingResult
{
Items = bulkResponse.Items.Select(i => new IndexingResultItem
{
Id = i.Id,
Succeeded = i.IsValid,
ErrorMessage = i.Error?.Reason,
}).ToArray(),
};
return(result);
}
/// <summary>
/// 添加一个文档集合到索引中(此方法查询数据会有延时)
/// </summary>
/// <typeparam name="TEntity"></typeparam>
/// <param name="entities"></param>
/// <param name="indexName"></param>
/// <returns></returns>
public virtual async Task <bool> BulkAsync <TEntity>(IEnumerable <TEntity> entities, string indexName = null) where TEntity : class
{
BulkDescriptor bulkDescriptor = new BulkDescriptor();
if (!string.IsNullOrWhiteSpace(indexName))
{
bulkDescriptor.Index(indexName);
}
bulkDescriptor.Refresh(Elasticsearch.Net.Refresh.True);
var response = await Context.BulkAsync(bulkDescriptor.IndexMany(entities));
if (!response.IsValid)
{
Logger.LogError($"[Success:{response.ApiCall.Success}]\t{response.ApiCall.Uri}");
Logger.LogError(response.ApiCall.DebugInformation);
Logger.LogError(response.ApiCall.OriginalException?.Message);
}
return(response.ApiCall.Success);
}
/// <summary>
/// 批量索引
/// </summary>
/// <param name="entities">实体集合</param>
/// <param name="routing">路由键</param>
/// <returns></returns>
public async Task <HttpResponseResultModel <bool> > IndexBatchAsync <T>(InsertUpdateModel <IList <T> > insertUpdateModel) where T : class, new()
{
HttpResponseResultModel <bool> httpResponseResultModel = new HttpResponseResultModel <bool>();
string indexName = insertUpdateModel.IndexName?.ToLower();
if (string.IsNullOrEmpty(indexName))
{
indexName = PocoIndexName;
}
BulkDescriptor bulkDescriptor = new BulkDescriptor();
bulkDescriptor.IndexMany(insertUpdateModel.Entity);
bulkDescriptor.Index(indexName);
if (!string.IsNullOrEmpty(insertUpdateModel.Routing))
{
bulkDescriptor.Routing(new Routing(insertUpdateModel.Routing));
}
var result = await client.BulkAsync(bulkDescriptor).ConfigureAwait(false);
GetDebugInfo(result);
//if (result.Errors || !result.ItemsWithErrors.IsNullOrEmpty())
//{
// Task.Run(() =>
// {
// var json = JsonHelper.SerializeObject(insertUpdateModel.Entity);
// WriteLog.WriteLogsAsync(json, "batchError");
// });
//}
var isSuccess = result.ItemsWithErrors.IsListNullOrEmpty();
string errorMessage = "";
if (!isSuccess)
{
var errorIdList = result.ItemsWithErrors.Select(t => t.Id);
errorMessage = string.Join(",", errorIdList);
}
httpResponseResultModel.IsSuccess = isSuccess;
httpResponseResultModel.ErrorMessage = errorMessage;
return(httpResponseResultModel);
}
/// <summary>
/// Synk picons
/// </summary>
/// <param name="picons"></param>
/// <param name="reset">Delete the index and recreate it</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <IBulkResponse> SynkAsync(IEnumerable <Picon> picons, bool reset = false, CancellationToken cancellationToken = default)
{
if (reset)
{
var res = await _elasticConnectionClient.Client.Value.DeleteByQueryAsync <Picon>(r => r.Query(rq => rq.Wildcard(f => f.Name, "*")), cancellationToken);
if (!res.IsValid)
{
if (res.ServerError.Status != 404)
{
throw new ApplicationException(res.ServerError.Error.Reason, res.OriginalException);
}
}
}
var descriptor = new BulkDescriptor();
descriptor.Pipeline(ElasticConnectionClient.PICONS_RETREIVE_CHANNEL_NUMBER_PIPELINE);
descriptor.IndexMany(picons);
descriptor.Refresh(Elasticsearch.Net.Refresh.True);
_logger.LogInformation($"Sync picons count : {picons.Count()}");
return(await _elasticConnectionClient.Client.Value.BulkAsync(descriptor, cancellationToken));
}
