public NestBulkResponse ParallelBulkIndex <T>(IEnumerable <T> elasticObjects, int parallelLimit, decimal objectsSizeInBytes) where T : class, IModel, new() { // it can be -1 (for Parallel.ForEach means not set), for calculation we need an exact number greater than 0 if (parallelLimit < 1) { parallelLimit = Environment.ProcessorCount; } var allCount = elasticObjects.Count(); if (allCount == 0) { return(new NestBulkResponse()); } var batchSize = 0; var divider = 0; if (objectsSizeInBytes != 0) { divider = objectsSizeInBytes < SiteConfig.Resources.MaxIndexBulkSize ? 1 : Math.Max(Convert.ToInt32(objectsSizeInBytes / (SiteConfig.Resources.MaxIndexBulkSize)), 1); } else { divider = allCount < SiteConfig.Resources.MaxIndexBulkCount ? 1 : Math.Max(Convert.ToInt32((double)allCount / (SiteConfig.Resources.MaxIndexBulkCount)), 1); } batchSize = (int)Math.Ceiling((double)allCount / divider); parallelLimit = Math.Min(divider, parallelLimit); var parallelBatchSize = Math.Max(1, (int)Math.Ceiling(allCount / (double)parallelLimit)); var parallelBatchs = elasticObjects.Batch(parallelBatchSize); var bulkResponseStructs = new ConcurrentBag <NestBulkResponse>(); try { Parallel.ForEach(parallelBatchs, documents => { var bulkResponse = Index(documents, false, batchSize); bulkResponseStructs.Add(bulkResponse); }); } finally { Client.Flush(IndexName); } var response = new NestBulkResponse { Items = new List <BulkResponseItemBase>(), ItemsWithErrors = new List <BulkResponseItemBase>() }; foreach (var bResp in bulkResponseStructs) { response.Items.AddRange(bResp.Items); response.ItemsWithErrors.AddRange(bResp.ItemsWithErrors); } return(response); }
public static List <BulkResult> ToBulkResult(this NestBulkResponse response) { var result = new List <BulkResult>(); if (response.Items != null) { result.AddRange(response.Items.Where(i => i.IsValid) .Select(item => BulkResult.Create(item.Id, StatusCodes.Status200OK, string.Empty))); } if (response.ItemsWithErrors != null) { result.AddRange(response.ItemsWithErrors .Select(item => BulkResult.Create(item.Id, StatusCodes.Status406NotAcceptable, item.Error.Reason))); } return(result); }
protected NestBulkResponse Index <T>(IEnumerable <T> elasticObjects, bool doFlush, int batchSize = 0) where T : class, IModel, new() { var allErroredItems = new List <BulkResponseItemBase>(); var allSucceedItems = new List <BulkResponseItemBase>(); var elasticObjectList = elasticObjects.ToList(); if (batchSize == 0) { batchSize = SiteConfig.Resources.MaxIndexBulkCount; } do { var actualBatch = elasticObjectList.Count > batchSize?elasticObjectList.Take(batchSize) : elasticObjectList; if (!actualBatch.Any()) { break; } var bulkResponse = Client.IndexMany(actualBatch); ResponseValidator(bulkResponse); // a sikereseket kikapjuk az indexelendők listájából var succeedIdsDic = bulkResponse.Items.Select(i => i.Id).Distinct().ToDictionary(i => i, i => i); allSucceedItems.AddRange(bulkResponse.Items); elasticObjectList.RemoveAll(d => succeedIdsDic.ContainsKey(d.Id)); // ha voltak nem indexálható elemek if (bulkResponse.Errors) { // ezek a státusz kódok utalnak szerver túlterhelésre var tryAgainItems = bulkResponse.ItemsWithErrors.Where(i => i.Status == 503 || i.Status == 429); // ezek az elemek nem fognak tudni bemenni, nem a túlterhelés a gond, elrakjuk őket, és kiszedjük a listából var erroredItems = bulkResponse.ItemsWithErrors.Except(tryAgainItems); allErroredItems.AddRange(erroredItems); var erroredIdsDic = erroredItems.Select(i => i.Id).Distinct().ToDictionary(id => id, id => id); elasticObjectList.RemoveAll(d => erroredIdsDic.ContainsKey(d.Id)); // a batchsize még csökkenthető if (tryAgainItems.Count() > 0 && batchSize > 1) { batchSize = Math.Max(batchSize / 2, 1); Thread.Sleep(5000); } /* * // már nincs remény, ezek biza nem akarnak bemenni, elrakjuk őket, és kiszedjük a listából * else * { * allErroredItems.AddRange(tryAgainItems); * var tryAgainIdsDic = tryAgainItems.Select(i => i.Id).ToDictionary(id => id, id => id); * elasticObjectList.RemoveAll(d => tryAgainIdsDic.ContainsKey(d.Id)); * } */ } } while (elasticObjectList.Any()); if (doFlush) { ResponseValidator(Client.Flush(IndexName)); } var response = new NestBulkResponse { Items = allSucceedItems, ItemsWithErrors = allErroredItems }; return(response); }
public static List <string> ToList(this NestBulkResponse response) { return(response.ItemsWithErrors.Select(i => i.Error.Reason).ToList()); }
public static ErrorsModel ToErrorsModel(this NestBulkResponse response) { return(ErrorsModel.Create(response.ToList())); }