/// <summary>
/// Processing message for one product
/// </summary>
/// <param name="checkingProductId"></param>
/// <param name="xmlMessages"></param>
/// <param name="productSettings"></param>
/// <param name="messagesCount"></param>
/// <param name="enableInsertingArrMessages"></param>
public void ProcessArrivalMessagesForProduct(int checkingProductId, List <string> xmlMessages, IList <ValidationSetting> productSettings, int messagesCount, bool enableInsertingArrMessages)
{
var startedAt = DateTime.UtcNow;
_log.Info("Count of messages: " + xmlMessages.Count);
try
{
var allArrivalMessages = new ConcurrentBag <ArrivalMessage>();
var allEvents = new ConcurrentBag <Event>();
var processTasks = new List <Task>();
var counter = 10;
var step = messagesCount / counter;
// First: we parse all xml-messages and build sometimes xml-tree(as update)
// also it is parallel process by each N messages(10 tasks for all messages)
for (var i = 0; i < counter; i++)
{
var index = i * 1;
var processingMessages = xmlMessages.Skip(index * step).Take(step).ToList();
var task = Task.Factory.StartNew(() =>
{
foreach (var xmlMessage in processingMessages)
{
var message = _arrivalMessageService.CreateArrivalMessage(checkingProductId, "", "", "");
// Parsing and checking: sometimes xml contains only keep-alive messages
if (_downloadService.ProcessEventMessageAndParse(message, xmlMessage, enableInsertingArrMessages, ref allEvents))
{
message.IsProcessed = true;
allArrivalMessages.Add(message);
} // pass
}
});
processTasks.Add(task);
}
Task.WaitAll(processTasks.ToArray());
// Now we get all events
var events = allEvents.ToList();
var eventsCount = events.Count;
CreateSuccessSchedulerHistory(startedAt, checkingProductId, SchedulerTypes.ParseXmlFile, "Count: " + eventsCount + " events");
_log.Info("Processing: " + eventsCount + " events; unique: " + events.DistinctBy(x => x.EventId).Count());
if (eventsCount > 0)
{
var startedOn = DateTime.UtcNow;
// Then we also parallel all processing(inserting arrival messages and etc)
var waitingTasks = new List <Task>();
//1 task, if it is push-product, but not product #5
if (enableInsertingArrMessages)
{
// buld insert the packet of messages
var taskUpdateArrivalMessages = Task.Factory.StartNew(() =>
{
_arrivalMessageService.BulkInsert(allArrivalMessages.ToList(), "");
});
waitingTasks.Add(taskUpdateArrivalMessages);
}
//2 task
var taskMergeEvents = Task.Factory.StartNew(() =>
{
var startMergeEvents = DateTime.UtcNow;
// Ae get all old events
var oldEvents = _merger.GetEventsByEventIds(events.Select(x => x.EventId).ToList(), checkingProductId);
// And we add them all as repeated, because there are a lot of not-unique events. It is such optimization.
sportsEventsStores[checkingProductId.ToString()].AddRepeated(oldEvents, events);
CreateSuccessSchedulerHistory(startMergeEvents, checkingProductId, SchedulerTypes.Merger, "Merged: " + oldEvents.Count + events.Count);
});
// if it is't product #5 we create common arrival message
if (!enableInsertingArrMessages)
{
var taskInseringCommonArrMessage = Task.Factory.StartNew(() =>
{
var commonArrivalMessage = new ArrivalMessage
{
ProductId = checkingProductId,
IsProcessed = true,
Url = "Count of processed arrival messages: " + messagesCount,
};
var bets = allArrivalMessages.SelectMany(x => x.Bets ?? new List <string>()).ToList();
var messagesEvents = allArrivalMessages.SelectMany(x => x.Events ?? new List <string>()).ToList();
var leagues = allArrivalMessages.SelectMany(x => x.Leagues ?? new List <string>()).ToList();
var markets = allArrivalMessages.SelectMany(x => x.Markets ?? new List <string>()).ToList();
var sports = allArrivalMessages.SelectMany(x => x.Sports ?? new List <string>()).ToList();
var locations =
allArrivalMessages.SelectMany(x => x.Locations ?? new List <string>()).ToList();
var providers =
allArrivalMessages.SelectMany(x => x.Providers ?? new List <string>()).ToList();
var statuses = allArrivalMessages.SelectMany(x => x.Statuses ?? new List <string>()).ToList();
commonArrivalMessage.SetFields(true, messagesEvents, sports, leagues, locations, statuses, markets,
providers, bets);
_arrivalMessageRepository.Insert(commonArrivalMessage);
});
waitingTasks.Add(taskInseringCommonArrMessage);
}
_log.Info(String.Format("merged"));
// We can't validate messages, while not merge all of them(for NumberOfChanges and DistinctNumberOfChanges)
taskMergeEvents.Wait();
// Then we get all events for processing
var eventsForProcessing = sportsEventsStores[checkingProductId.ToString()].GetEvents();
// And...cal saving dropdowns, validation, saving events!
StartProcessingTasks(startedOn, eventsForProcessing, checkingProductId, productSettings);
// Clearing from memory.
sportsEventsStores[checkingProductId.ToString()].Clear();
Task.WaitAll(waitingTasks.ToArray());
_log.Info(String.Format("end processing-creating-merging-validating"));
}
}
catch (Exception e)
{
var errorMessage = GetExceptionInfo(e);
_log.Error(errorMessage);
CreateFailedSchedulerHistory(startedAt, checkingProductId, SchedulerTypes.ProcessingAllArrivalMessages,
errorMessage);
}
}
public override void Run()
{
// datetime variable for logging
var startedOn = DateTime.UtcNow;
int currentProductId = 1;
int?currentArrivalMessageId = null;
try
{
// false - pull-products, if true - only push-products
var products = _productRepository.GetListByType(false);
// for each product in pull-products(now only one product)
foreach (var product in products)
{
var messages = new List <ArrivalMessage>();
var notProccessedMessages = GetLastArrivalMessagesNotProcessed(product.Id);
// if we have a lot of not processed messages we stop downloading.
// because downloading always works faster than message processing.
if (notProccessedMessages.Count > Sports.sports.Count / 2)
{
continue;
}
currentProductId = product.Id;
// one file - on task
var tasks = new List <Task>();
// variable for logging common file size
var commonSize = 0L;
// timestamp variable
var downloadFrom = notProccessedMessages.FirstOrDefault()?.CreatedOn ?? DateTime.UtcNow.AddDays(-1);
foreach (var sport in Sports.sports)
{
var downloadTask = Task.Factory.StartNew(() =>
{
try
{
// building gateway url with sportid and timestamp
var gateway = product.GatewayAPI + "×tamp=" + ConvertToUnixTimestamp(downloadFrom) +
"&sports=" + sport.Id;
var filePath = downloadService.DownloadFile(product, gateway);
var isContainsEvents = downloadService.IsContainsEvents(product.Id, filePath);
// if downloaded file contains events...
if (isContainsEvents)
{
commonSize += (new System.IO.FileInfo(filePath)).Length;
// new not processed arrival message
var message = CreateArrivalMessage(product.Id, filePath, gateway,
sport.Id.ToString());
_arrivalMessageRepository.Insert(message);
messages.Add(message);
}
}
// else logging error
catch (WebException webError)
{
var httpResponse = (HttpWebResponse)webError.Response;
var message = "";
if (httpResponse != null)
{
message =
$"{webError.Message}; {httpResponse.StatusDescription}; {GetExceptionInfo(webError)}";
}
else
{
message = $"{webError.Message}";
}
_log.Error(message);
CreateFailedSchedulerHistory(startedOn, currentProductId, SchedulerTypes.Downloader, message);
}
catch (Exception e)
{
var message = GetExceptionInfo(e);
_log.Error(message);
CreateFailedSchedulerHistory(startedOn, currentProductId, SchedulerTypes.Downloader, message);
}
});
tasks.Add(downloadTask);
}
// waiting of all downloaders
Task.WaitAll(tasks.ToArray());
// logging common result
_log.Info("Saved: " + messages.Count + " messages");
var fileSizeString = $"Size: {Math.Round((double)commonSize / 1024/1024, 2)} MB";
CreateSuccessSchedulerHistory(startedOn, currentProductId, SchedulerTypes.Downloader, fileSizeString);
}
}
catch (WebException webError)
{
var httpResponse = (HttpWebResponse)webError.Response;
var message = "";
if (httpResponse != null)
{
message = $"{webError.Message}; {httpResponse.StatusDescription}; {GetExceptionInfo(webError)}";
}
else
{
message = $"{webError.Message}";
}
_log.Error(message);
CreateFailedSchedulerHistory(startedOn, currentProductId, SchedulerTypes.Downloader, message);
}
catch (Exception e)
{
var message = GetExceptionInfo(e);
_log.Error(message);
CreateFailedSchedulerHistory(startedOn, currentProductId, SchedulerTypes.Downloader, message);
}
finally
{
// _log.Info(String.Format("End downloading"));
}
}