public void Start() { var log = LogManager.GetCurrentClassLogger(); VerifyConfiguration(log); taskTokenSource = new CancellationTokenSource(); var taskToken = taskTokenSource.Token; var downloader = new DownloaderService(LogManager.GetLogger("Downloader"), new HmaCliService(new FreeGeoIpService(log), Settings.ExcludedIps, Settings.HMAInstallFullPath, log)); var parser = new ParsingService(LogManager.GetLogger("Parser")); downlaodTask = new Task(() => { if (Program.TestAccount == null) { while (!taskToken.IsCancellationRequested) { try { downloader.Download(taskToken); } catch (Exception ex) { log.Error("Uncaught exception in Downloader", ex); } if (!taskToken.IsCancellationRequested) Thread.Sleep(Settings.DownloaderBatchWaitTime); } } else { downloader.Download(taskToken, Program.TestAccount); log.Info("Single account download complete"); } }, taskTokenSource.Token); if (Program.RunDownloader) { downlaodTask.Start(); } parseTask = new Task(() => { while (!taskToken.IsCancellationRequested) { try { parser.ParseEmlFiles(taskToken); } catch (Exception ex) { log.Error("Uncaught exception in Parser", ex); } if (!taskToken.IsCancellationRequested) Thread.Sleep(Settings.ParserBatchWaitTime); } }, taskTokenSource.Token); parseTask.Start(); }
public async Task HttpsDontStart() { var downloadRequest = new DownloadRequest() { Threads = 2, Links = new System.Collections.Generic.List <LinkSave> { new LinkSave { Filename = "file1", Link = "https://someValidLink" }, new LinkSave { Filename = "file2", Link = "http://someValidLink" } } }; var fileDownloadStatus = new FileDonwladStatus(); var downloadService = new DownloaderService(fileDownloadStatus , NSubstitute.Substitute.For <Microsoft.Extensions.Configuration.IConfiguration>() , NSubstitute.Substitute.For <Microsoft.Extensions.Logging.ILogger <DownloaderService> >()); Assert.False(downloadService.ValidateHttpLinks(downloadRequest)); await downloadService.Download(downloadRequest); Assert.Empty(fileDownloadStatus.Definitions); }
public async Task EmptyLinksDontStartDownload() { var downloadRequest = new DownloadRequest() { Threads = 2, Links = null }; var fileDownloadStatus = new FileDonwladStatus(); var downloadService = new DownloaderService(fileDownloadStatus , NSubstitute.Substitute.For <Microsoft.Extensions.Configuration.IConfiguration>() , NSubstitute.Substitute.For <Microsoft.Extensions.Logging.ILogger <DownloaderService> >()); await downloadService.Download(downloadRequest); Assert.Empty(fileDownloadStatus.Definitions); }
public IActionResult Download(DownloadRequest request) { if (request.Links == null || request.Links.Count == 0) { return(new BadRequestObjectResult("no links provided")); } if (!_downloaderService.ValidateHttpLinks(request)) { return(new BadRequestObjectResult("http links only")); } var duplicated = _downloaderService.GetDuplicatedFiles(request); if (duplicated.Count != 0) { return(new BadRequestObjectResult($"duplicated filenames to save: {duplicated.Aggregate((i, j) => i + ", " + j)}")); } _downloaderService.Download(request); return(Ok()); }