public NestBulkResponse ParallelBulkIndex <T>(IEnumerable <T> elasticObjects, int parallelLimit, decimal objectsSizeInBytes) where T : class, IModel, new()
{
// it can be -1 (for Parallel.ForEach means not set), for calculation we need an exact number greater than 0
if (parallelLimit < 1)
{
parallelLimit = Environment.ProcessorCount;
}
var allCount = elasticObjects.Count();
if (allCount == 0)
{
return(new NestBulkResponse());
}
var batchSize = 0;
var divider = 0;
if (objectsSizeInBytes != 0)
{
divider = objectsSizeInBytes < SiteConfig.Resources.MaxIndexBulkSize ? 1 : Math.Max(Convert.ToInt32(objectsSizeInBytes / (SiteConfig.Resources.MaxIndexBulkSize)), 1);
}
else
{
divider = allCount < SiteConfig.Resources.MaxIndexBulkCount ? 1 : Math.Max(Convert.ToInt32((double)allCount / (SiteConfig.Resources.MaxIndexBulkCount)), 1);
}
batchSize = (int)Math.Ceiling((double)allCount / divider);
parallelLimit = Math.Min(divider, parallelLimit);
var parallelBatchSize = Math.Max(1, (int)Math.Ceiling(allCount / (double)parallelLimit));
var parallelBatchs = elasticObjects.Batch(parallelBatchSize);
var bulkResponseStructs = new ConcurrentBag <NestBulkResponse>();
try
{
Parallel.ForEach(parallelBatchs, documents =>
{
var bulkResponse = Index(documents, false, batchSize);
bulkResponseStructs.Add(bulkResponse);
});
}
finally
{
Client.Flush(IndexName);
}
var response = new NestBulkResponse
{
Items = new List <BulkResponseItemBase>(),
ItemsWithErrors = new List <BulkResponseItemBase>()
};
foreach (var bResp in bulkResponseStructs)
{
response.Items.AddRange(bResp.Items);
response.ItemsWithErrors.AddRange(bResp.ItemsWithErrors);
}
return(response);
}
public static List <BulkResult> ToBulkResult(this NestBulkResponse response)
{
var result = new List <BulkResult>();
if (response.Items != null)
{
result.AddRange(response.Items.Where(i => i.IsValid)
.Select(item => BulkResult.Create(item.Id, StatusCodes.Status200OK, string.Empty)));
}
if (response.ItemsWithErrors != null)
{
result.AddRange(response.ItemsWithErrors
.Select(item => BulkResult.Create(item.Id, StatusCodes.Status406NotAcceptable, item.Error.Reason)));
}
return(result);
}
protected NestBulkResponse Index <T>(IEnumerable <T> elasticObjects, bool doFlush, int batchSize = 0) where T : class, IModel, new()
{
var allErroredItems = new List <BulkResponseItemBase>();
var allSucceedItems = new List <BulkResponseItemBase>();
var elasticObjectList = elasticObjects.ToList();
if (batchSize == 0)
{
batchSize = SiteConfig.Resources.MaxIndexBulkCount;
}
do
{
var actualBatch = elasticObjectList.Count > batchSize?elasticObjectList.Take(batchSize) : elasticObjectList;
if (!actualBatch.Any())
{
break;
}
var bulkResponse = Client.IndexMany(actualBatch);
ResponseValidator(bulkResponse);
// a sikereseket kikapjuk az indexelendők listájából
var succeedIdsDic = bulkResponse.Items.Select(i => i.Id).Distinct().ToDictionary(i => i, i => i);
allSucceedItems.AddRange(bulkResponse.Items);
elasticObjectList.RemoveAll(d => succeedIdsDic.ContainsKey(d.Id));
// ha voltak nem indexálható elemek
if (bulkResponse.Errors)
{
// ezek a státusz kódok utalnak szerver túlterhelésre
var tryAgainItems = bulkResponse.ItemsWithErrors.Where(i => i.Status == 503 || i.Status == 429);
// ezek az elemek nem fognak tudni bemenni, nem a túlterhelés a gond, elrakjuk őket, és kiszedjük a listából
var erroredItems = bulkResponse.ItemsWithErrors.Except(tryAgainItems);
allErroredItems.AddRange(erroredItems);
var erroredIdsDic = erroredItems.Select(i => i.Id).Distinct().ToDictionary(id => id, id => id);
elasticObjectList.RemoveAll(d => erroredIdsDic.ContainsKey(d.Id));
// a batchsize még csökkenthető
if (tryAgainItems.Count() > 0 && batchSize > 1)
{
batchSize = Math.Max(batchSize / 2, 1);
Thread.Sleep(5000);
}
/*
* // már nincs remény, ezek biza nem akarnak bemenni, elrakjuk őket, és kiszedjük a listából
* else
* {
* allErroredItems.AddRange(tryAgainItems);
* var tryAgainIdsDic = tryAgainItems.Select(i => i.Id).ToDictionary(id => id, id => id);
* elasticObjectList.RemoveAll(d => tryAgainIdsDic.ContainsKey(d.Id));
* }
*/
}
} while (elasticObjectList.Any());
if (doFlush)
{
ResponseValidator(Client.Flush(IndexName));
}
var response = new NestBulkResponse
{
Items = allSucceedItems,
ItemsWithErrors = allErroredItems
};
return(response);
}
public static List <string> ToList(this NestBulkResponse response)
{
return(response.ItemsWithErrors.Select(i => i.Error.Reason).ToList());
}
public static ErrorsModel ToErrorsModel(this NestBulkResponse response)
{
return(ErrorsModel.Create(response.ToList()));
}
