private static BulkLoadResponse LoadIndex(string fileName)
{
var bulkResponse = new BulkLoadResponse();
int recordsPerLoad = 5000;
var orders = GetAllOrders(fileName);
var search = new FlnSearch.AwsSearch();
var remainingCount = orders.Count();
var startIndex = 0;
while (remainingCount > 0)
{
var count = remainingCount > recordsPerLoad ? recordsPerLoad : remainingCount;
var batch = orders.GetRange(startIndex, count);
Console.WriteLine(string.Format("\tprocessing records :{0} thru {1}...", startIndex, startIndex + count));
var response = search.BulkLoad(batch);
bulkResponse.RecordsInBatch += response.RecordsInBatch;
Console.WriteLine(string.Format("\t\tProcess took: {0}", response.Took));
if (response.Errors)
{
bulkResponse.Errors = true;
bulkResponse.FailedItems.AddRange(response.FailedItems);
Console.WriteLine(string.Format("\t{0} errors found", response.FailedItems.Count));
}
remainingCount -= count;
startIndex += count;
}
return(bulkResponse);
}
public BulkLoadResponse LoadDataFromSource(DateTime startDate, DateTime enddate)
{
logger.Debug("+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+");
logger.Debug(string.Format("LoadDataFromSource:: StartDate:{0} EndDate:{1}", startDate, enddate));
var orders = GetDataFromSql(startDate, enddate);
var bulkResponse = new BulkLoadResponse();
int recordsPerLoad = 2500;
var remainingCount = orders.Count();
var startIndex = 0;
while (remainingCount > 0)
{
var count = remainingCount > recordsPerLoad ? recordsPerLoad : remainingCount;
var batch = orders.GetRange(startIndex, count);
logger.Debug("+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+");
logger.Debug(string.Format("processing records<start> :{0} thru {1}...", startIndex, startIndex + count));
var response = BulkLoad(batch);
bulkResponse.RecordsInBatch += response.RecordsInBatch;
logger.Debug(string.Format("Process took: {0}", response.Took));
if (response.Errors)
{
bulkResponse.Errors = true;
bulkResponse.FailedItems.AddRange(response.FailedItems);
logger.Debug(string.Format("{0} errors found", response.FailedItems.Count));
}
logger.Debug(string.Format("processing records<end> :{0} thru {1}", startIndex, startIndex + count));
remainingCount -= count;
startIndex += count;
}
return(bulkResponse);
}
public BulkLoadResponse BulkLoad(List <OrderRecord> records)
{
var response = new BulkLoadResponse();
var builder = new StringBuilder();
foreach (var record in records)
{
builder.AppendFormat("{{\"index\":{{\"_index\":\"{0}\",\"_type\":\"order\",\"_id\":\"{1}\"}}}}", _index, record.OrderNumber);
builder.Append(Environment.NewLine);
builder.Append("{");
builder.AppendFormat("\"OrderNumber\":{0}", record.OrderNumber);
builder.AppendFormat(",\"OrderDate\":\"{0}\"", record.OrderDate.ToString("yyyy-MM-ddTHH:mm:ssZ"));
builder.AppendFormat(",\"OrderDateTicks\":{0}", record.OrderDate.Ticks);
builder.AppendFormat(",\"ServiceType\":{0}", record.ServiceType);
if (!string.IsNullOrEmpty(record.OrderStatusCode))
{
builder.AppendFormat(",\"OrderStatusCode\":\"{0}\"", record.OrderStatusCode.Trim().Replace(@"\", @"\\").Replace("\"", "\\\""));
builder.AppendFormat(",\"OrderStatus\":\"{0}\"", record.OrderStatus.Trim().Replace(@"\", @"\\").Replace("\"", "\\\""));
}
builder.AppendFormat(",\"CustomerNumber\":{0}", record.CustomerNumber);
if (!string.IsNullOrEmpty(record.BolNumber))
{
builder.AppendFormat(",\"BolNumber\":\"{0}\"", record.BolNumber.Trim().Replace(@"\", @"\\").Replace("\"", "\\\""));
}
if (!string.IsNullOrEmpty(record.RecipientCompany))
{
builder.AppendFormat(",\"RecipientCompany\":\"{0}\"", record.RecipientCompany.Trim().Replace(@"\", @"\\").Replace("\"", "\\\""));
}
if (!string.IsNullOrEmpty(record.RecipientName))
{
builder.AppendFormat(",\"RecipientName\":\"{0}\"", record.RecipientName.Trim().Replace(@"\", @"\\").Replace("\"", "\\\""));
}
if (!string.IsNullOrEmpty(record.ClientMatter))
{
builder.AppendFormat(",\"ClientMatter\":\"{0}\"", record.ClientMatter.Trim().Replace(@"\", @"\\").Replace("\"", "\\\""));
}
builder.AppendFormat(",\"LastUpdateDate\":\"{0}\"", record.LastUpdateDate.ToString("yyyy-MM-ddTHH:mm:ssZ"));
builder.AppendFormat(",\"LastUpdateDateTicks\":{0}", record.LastUpdateDate.Ticks);
builder.Append("}");
builder.Append(Environment.NewLine);
}
JToken root = null;
var content = builder.ToString();
try
{
var result = PostAsync(string.Format("{0}/_bulk", _baseUrl), content).Result;
root = JToken.Parse(result);
}
catch (Exception ex)
{
logger.Error(string.Format("{0} Failed.\r\n:{1}", "BulkLoad", ex.Message));
response.Errors = true;
response.ResponseMessage = string.Format("{0} failed.\r\n{1}", "BulkLoad", ex.Message);
return(response);
}
response.Took = (int)root["took"];
response.Errors = (bool)root["errors"];
List <JToken> itemResults = root["items"].Children().ToList();
response.RecordsInBatch = itemResults.Count;
//only get the error items. Getting all items has a big impact on performance
if (response.Errors)
{
foreach (var token in itemResults)
{
var tokenError = token["index"]["error"];
if (tokenError != null)
{
LoadItemResult item = token["index"].ToObject <LoadItemResult>();
var error = tokenError.ToObject <LoadError>();
error.SubType = tokenError["caused_by"]["type"].ToString();
error.Message = tokenError["caused_by"]["reason"].ToString();
item.Error = error;
response.FailedItems.Add(item);
logger.Debug(string.Format("Error:{0}::id:{1}\r\n\t{2}\r\n\t{3}\r\n------------------------------------", item.Error, item.Id, item.Error.Reason, item.Error.Message));
}
}
}
response.ResponseMessage = string.Format("BulkLoad::{0} Total items; {1} items failed", response.RecordsInBatch, response.FailedItems.Count);
return(response);
}
