/// <summary> /// /// </summary> /// <returns></returns> public TaskResult Do(TransactionParameter transactionParameter) { var isBackOfficeException = false; var sleepInterval = 5; _importProcessor.PurgeTrackers(transactionParameter); while (true) { if (isBackOfficeException) { Logger.Info($"Performing sleep for {sleepInterval} minutes"); Thread.Sleep(TimeSpan.FromMinutes(sleepInterval)); sleepInterval *= 2; Logger.Info("awoken from sleep"); } _batchProcessor.SkipCounter = 0; foreach (var taskActor in _taskActors) { var batch = _batchProcessor .Process(ImportMode.Batch) .GetData <List <Import> >(); if (batch.Count == 0) { continue; } Thread.Sleep(TimeSpan.FromSeconds(30)); taskActor.Do(batch, transactionParameter); } isBackOfficeException = _taskProcessor .Process() .IsSuccessful .Value; _taskProcessor.Dispose(); var isCompleted = _importProcessor.IsCompleted(); if (isCompleted) { break; } //send signal to recovery actor } return(new TaskResult { IsSuccessful = true }); }
/// <summary> /// /// </summary> /// <returns></returns> public TaskResult Do(TransactionParameter transactionParameter) { var isBackOfficeException = false; var sleepInterval = 5; while (true) { if (isBackOfficeException) { Logger.Info($"Performing sleep for {sleepInterval} minutes"); Thread.Sleep(TimeSpan.FromMinutes(sleepInterval)); sleepInterval *= 2; Logger.Info("awoken from sleep"); } var batch = new List <Import> { new Import { Symbol = transactionParameter.Symbol } }; var taskResult = _taskActor.Do(batch, transactionParameter); Logger.Info($"Thread id: {_taskActor.TaskInstance.Id} will processing the following symbols: { transactionParameter.Symbol }"); _taskProcessor.AddTask(_taskActor); isBackOfficeException = _taskProcessor .Process() .IsSuccessful.Value; if (taskResult.IsSuccessful.Value) { break; } } _taskActor.Dispose(); var importTracker = _importTrackerRepository .GetAll() .GetData <List <ImportTracker> >() .ToList() .First(x => x.Equals(new ImportTracker(transactionParameter.Symbol))); importTracker.Status = "Success"; _importTrackerRepository.PutAsync(importTracker); return(new TaskResult { IsSuccessful = true }); }
/// <summary> /// /// </summary> /// <returns></returns> public TaskResult Do(TransactionParameter transactionParameter) { var isBackOfficeException = false; var sleepInterval = 5; _importProcessor.PurgeTrackers(transactionParameter); while (true) { if (isBackOfficeException) { Logger.Info($"Performing sleep for {sleepInterval} minutes"); Thread.Sleep(TimeSpan.FromMinutes(sleepInterval)); sleepInterval *= 2; Logger.Info("awoken from sleep"); } var workerCount = (!string.IsNullOrEmpty(transactionParameter.Symbol)) ? 1 : Convert.ToInt32(_configuration["WorkerCount"]); Logger.Info($"setting worker count {workerCount}"); var webDrivers = new List <IWebDriver>(); var tasks = new List <Task>(); var cancellationTokenSource = new CancellationTokenSource(); var cancellationToken = cancellationTokenSource.Token; for (var index = 0; index <= workerCount; index++) { var batch = (!string.IsNullOrEmpty(transactionParameter.Symbol)) ? new List <Import> { new Import { Symbol = transactionParameter.Symbol } } : _batchProcessor .Process() .GetData <List <Import> >(); if (batch.Count == 0) { continue; } if (index > 0) { Logger.Info($"sleeping for 30 seconds to allow breathing for previously logged sessions"); Thread.Sleep(TimeSpan.FromSeconds(30)); } var webDriver = WebDriverFactory.Create(_configuration); var task = Task .Factory .StartNew(() => _brokerTransactionScraper.Scrape(transactionParameter, batch, webDriver), cancellationToken); var symbols = string.Join(",", batch.Select(b => b.Symbol)); webDrivers.Add(webDriver); Logger.Info($"Thread id: {task.Id} will processing the following symbols: {symbols}"); LogicalThreadContext.Properties["thread-id"] = task.Id; tasks.Add(task); } Logger.Info($"waiting for all complete, thread count: {tasks.Count}"); isBackOfficeException = _taskProcessor .Process(tasks, cancellationTokenSource) .IsSuccessful.Value; Logger.Info("performing webdriver cleanup"); foreach (var webDriver in webDrivers) { webDriver.Quit(); webDriver.Dispose(); } Logger.Info("performing task cleanup"); foreach (var task in tasks) { task.Dispose(); } if (transactionParameter.Frequency.Equals("daily", StringComparison.CurrentCultureIgnoreCase)) { break; } var isCompleted = _importProcessor.IsCompleted(); if (isCompleted) { break; } } return(new TaskResult { IsSuccessful = true }); }