private async Task ProcessAllWaitingWebsites() { var allTasks = new List <Task>(); var websites = _waitingWebsites.ToList(); foreach (var waitingWebsite in websites) { _emptyRunsCounter = 0; await SemaphoreDownload.WaitAsync(); waitingWebsite.Crawled = true; waitingWebsite.CrawledTimestamp = DateTime.Now; await _context.SaveChangesAsync(); allTasks.Add( Task.Run(async() => { try { _logger.LogInformation($"Processing webpage: {waitingWebsite.Url}"); _downloadStrategy.Download(waitingWebsite); await _entityManager.Persist(waitingWebsite); await _entityManager.Flush(); } catch (Exception e) { _logger.LogError(e.Message); } finally { SemaphoreDownload.Release(); } }) ); } try { await Task.WhenAll(allTasks.ToArray()); } catch (Exception e) { _logger.LogError(e.Message); } }
public void Process(string sources) { //TODO : Move this logic to separate windows service and make the service listen on Message Queue like Rabbitmq logger.AddInformationLog($"sources value: {sources}"); //Data validations if (string.IsNullOrWhiteSpace(sources)) { throw new ArgumentException($"The sources Is null or empty string."); } string delimiter = ConfigurationManager.AppSettings["Delimiter"]; logger.AddInformationLog($"Delimiter config value: {delimiter}"); string sourcesParsingDelimiter = string.IsNullOrWhiteSpace(delimiter) ? "," : delimiter; logger.AddInformationLog($"Sources parsing delimiter: {sourcesParsingDelimiter}"); string localPath = ConfigurationManager.AppSettings["LocalDirectory"]; logger.AddInformationLog($"LocalDirectory config value: {localPath}"); List <string> soursesUrls = parser.Parse(sources, sourcesParsingDelimiter).ToList(); logger.AddInformationLog($"sourses Urls counts after parsing: {soursesUrls?.Count}"); ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>(); using (repoDownloadedFile) { Parallel.ForEach(soursesUrls, url => { try { IDownloadStrategy downloadStrategy = downloadStrategyFactory.Build(url); string physicalPath = downloadStrategy.Download(url, localPath); string virtualPath = $"{localPath.Replace('\\', '/')}/{Path.GetFileName(physicalPath)}"; logger.AddInformationLog($"Local path is: {localPath} for URL: {url} "); repoDownloadedFile.Add(new DomainModels.DownloadedFile() { FileRemotePath = url, LocalPath = virtualPath, ProcessingStatus = (int)DownloadStatutes.ReadyForProcessing, }); repoDownloadedFile.SaveChanges(); logger.AddInformationLog($"Source :{url} saved to the database with status ready for processing"); } catch (Exception ex) { exceptions.Enqueue(ex); logger.AddErrorLog(ex); } }); if (exceptions.Any()) { throw new AggregateException(exceptions); } } }
public (IStatusCode, string) Download(IDownloadStrategy strategy, IUrl toDownload) { (IStatusCode, string)result = strategy.Download(toDownload); return(result.Item1, result.Item2); }