private static BulkLoadResponse LoadIndex(string fileName) { var bulkResponse = new BulkLoadResponse(); int recordsPerLoad = 5000; var orders = GetAllOrders(fileName); var search = new FlnSearch.AwsSearch(); var remainingCount = orders.Count(); var startIndex = 0; while (remainingCount > 0) { var count = remainingCount > recordsPerLoad ? recordsPerLoad : remainingCount; var batch = orders.GetRange(startIndex, count); Console.WriteLine(string.Format("\tprocessing records :{0} thru {1}...", startIndex, startIndex + count)); var response = search.BulkLoad(batch); bulkResponse.RecordsInBatch += response.RecordsInBatch; Console.WriteLine(string.Format("\t\tProcess took: {0}", response.Took)); if (response.Errors) { bulkResponse.Errors = true; bulkResponse.FailedItems.AddRange(response.FailedItems); Console.WriteLine(string.Format("\t{0} errors found", response.FailedItems.Count)); } remainingCount -= count; startIndex += count; } return(bulkResponse); }
public BulkLoadResponse LoadDataFromSource(DateTime startDate, DateTime enddate) { logger.Debug("+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+"); logger.Debug(string.Format("LoadDataFromSource:: StartDate:{0} EndDate:{1}", startDate, enddate)); var orders = GetDataFromSql(startDate, enddate); var bulkResponse = new BulkLoadResponse(); int recordsPerLoad = 2500; var remainingCount = orders.Count(); var startIndex = 0; while (remainingCount > 0) { var count = remainingCount > recordsPerLoad ? recordsPerLoad : remainingCount; var batch = orders.GetRange(startIndex, count); logger.Debug("+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+"); logger.Debug(string.Format("processing records<start> :{0} thru {1}...", startIndex, startIndex + count)); var response = BulkLoad(batch); bulkResponse.RecordsInBatch += response.RecordsInBatch; logger.Debug(string.Format("Process took: {0}", response.Took)); if (response.Errors) { bulkResponse.Errors = true; bulkResponse.FailedItems.AddRange(response.FailedItems); logger.Debug(string.Format("{0} errors found", response.FailedItems.Count)); } logger.Debug(string.Format("processing records<end> :{0} thru {1}", startIndex, startIndex + count)); remainingCount -= count; startIndex += count; } return(bulkResponse); }
public BulkLoadResponse BulkLoad(List <OrderRecord> records) { var response = new BulkLoadResponse(); var builder = new StringBuilder(); foreach (var record in records) { builder.AppendFormat("{{\"index\":{{\"_index\":\"{0}\",\"_type\":\"order\",\"_id\":\"{1}\"}}}}", _index, record.OrderNumber); builder.Append(Environment.NewLine); builder.Append("{"); builder.AppendFormat("\"OrderNumber\":{0}", record.OrderNumber); builder.AppendFormat(",\"OrderDate\":\"{0}\"", record.OrderDate.ToString("yyyy-MM-ddTHH:mm:ssZ")); builder.AppendFormat(",\"OrderDateTicks\":{0}", record.OrderDate.Ticks); builder.AppendFormat(",\"ServiceType\":{0}", record.ServiceType); if (!string.IsNullOrEmpty(record.OrderStatusCode)) { builder.AppendFormat(",\"OrderStatusCode\":\"{0}\"", record.OrderStatusCode.Trim().Replace(@"\", @"\\").Replace("\"", "\\\"")); builder.AppendFormat(",\"OrderStatus\":\"{0}\"", record.OrderStatus.Trim().Replace(@"\", @"\\").Replace("\"", "\\\"")); } builder.AppendFormat(",\"CustomerNumber\":{0}", record.CustomerNumber); if (!string.IsNullOrEmpty(record.BolNumber)) { builder.AppendFormat(",\"BolNumber\":\"{0}\"", record.BolNumber.Trim().Replace(@"\", @"\\").Replace("\"", "\\\"")); } if (!string.IsNullOrEmpty(record.RecipientCompany)) { builder.AppendFormat(",\"RecipientCompany\":\"{0}\"", record.RecipientCompany.Trim().Replace(@"\", @"\\").Replace("\"", "\\\"")); } if (!string.IsNullOrEmpty(record.RecipientName)) { builder.AppendFormat(",\"RecipientName\":\"{0}\"", record.RecipientName.Trim().Replace(@"\", @"\\").Replace("\"", "\\\"")); } if (!string.IsNullOrEmpty(record.ClientMatter)) { builder.AppendFormat(",\"ClientMatter\":\"{0}\"", record.ClientMatter.Trim().Replace(@"\", @"\\").Replace("\"", "\\\"")); } builder.AppendFormat(",\"LastUpdateDate\":\"{0}\"", record.LastUpdateDate.ToString("yyyy-MM-ddTHH:mm:ssZ")); builder.AppendFormat(",\"LastUpdateDateTicks\":{0}", record.LastUpdateDate.Ticks); builder.Append("}"); builder.Append(Environment.NewLine); } JToken root = null; var content = builder.ToString(); try { var result = PostAsync(string.Format("{0}/_bulk", _baseUrl), content).Result; root = JToken.Parse(result); } catch (Exception ex) { logger.Error(string.Format("{0} Failed.\r\n:{1}", "BulkLoad", ex.Message)); response.Errors = true; response.ResponseMessage = string.Format("{0} failed.\r\n{1}", "BulkLoad", ex.Message); return(response); } response.Took = (int)root["took"]; response.Errors = (bool)root["errors"]; List <JToken> itemResults = root["items"].Children().ToList(); response.RecordsInBatch = itemResults.Count; //only get the error items. Getting all items has a big impact on performance if (response.Errors) { foreach (var token in itemResults) { var tokenError = token["index"]["error"]; if (tokenError != null) { LoadItemResult item = token["index"].ToObject <LoadItemResult>(); var error = tokenError.ToObject <LoadError>(); error.SubType = tokenError["caused_by"]["type"].ToString(); error.Message = tokenError["caused_by"]["reason"].ToString(); item.Error = error; response.FailedItems.Add(item); logger.Debug(string.Format("Error:{0}::id:{1}\r\n\t{2}\r\n\t{3}\r\n------------------------------------", item.Error, item.Id, item.Error.Reason, item.Error.Message)); } } } response.ResponseMessage = string.Format("BulkLoad::{0} Total items; {1} items failed", response.RecordsInBatch, response.FailedItems.Count); return(response); }