public void Start()
{
var log = LogManager.GetCurrentClassLogger();
VerifyConfiguration(log);
taskTokenSource = new CancellationTokenSource();
var taskToken = taskTokenSource.Token;
var downloader = new DownloaderService(LogManager.GetLogger("Downloader"), new HmaCliService(new FreeGeoIpService(log), Settings.ExcludedIps, Settings.HMAInstallFullPath, log));
var parser = new ParsingService(LogManager.GetLogger("Parser"));
downlaodTask = new Task(() =>
{
if (Program.TestAccount == null)
{
while (!taskToken.IsCancellationRequested)
{
try
{
downloader.Download(taskToken);
}
catch (Exception ex)
{
log.Error("Uncaught exception in Downloader", ex);
}
if (!taskToken.IsCancellationRequested) Thread.Sleep(Settings.DownloaderBatchWaitTime);
}
}
else
{
downloader.Download(taskToken, Program.TestAccount);
log.Info("Single account download complete");
}
}, taskTokenSource.Token);
if (Program.RunDownloader)
{
downlaodTask.Start();
}
parseTask = new Task(() =>
{
while (!taskToken.IsCancellationRequested)
{
try
{
parser.ParseEmlFiles(taskToken);
}
catch (Exception ex)
{
log.Error("Uncaught exception in Parser", ex);
}
if (!taskToken.IsCancellationRequested) Thread.Sleep(Settings.ParserBatchWaitTime);
}
}, taskTokenSource.Token);
parseTask.Start();
}
public async Task HttpsDontStart()
{
var downloadRequest = new DownloadRequest()
{
Threads = 2,
Links = new System.Collections.Generic.List <LinkSave>
{
new LinkSave {
Filename = "file1", Link = "https://someValidLink"
},
new LinkSave {
Filename = "file2", Link = "http://someValidLink"
}
}
};
var fileDownloadStatus = new FileDonwladStatus();
var downloadService = new DownloaderService(fileDownloadStatus
, NSubstitute.Substitute.For <Microsoft.Extensions.Configuration.IConfiguration>()
, NSubstitute.Substitute.For <Microsoft.Extensions.Logging.ILogger <DownloaderService> >());
Assert.False(downloadService.ValidateHttpLinks(downloadRequest));
await downloadService.Download(downloadRequest);
Assert.Empty(fileDownloadStatus.Definitions);
}
public async Task EmptyLinksDontStartDownload()
{
var downloadRequest = new DownloadRequest()
{
Threads = 2,
Links = null
};
var fileDownloadStatus = new FileDonwladStatus();
var downloadService = new DownloaderService(fileDownloadStatus
, NSubstitute.Substitute.For <Microsoft.Extensions.Configuration.IConfiguration>()
, NSubstitute.Substitute.For <Microsoft.Extensions.Logging.ILogger <DownloaderService> >());
await downloadService.Download(downloadRequest);
Assert.Empty(fileDownloadStatus.Definitions);
}
public IActionResult Download(DownloadRequest request)
{
if (request.Links == null || request.Links.Count == 0)
{
return(new BadRequestObjectResult("no links provided"));
}
if (!_downloaderService.ValidateHttpLinks(request))
{
return(new BadRequestObjectResult("http links only"));
}
var duplicated = _downloaderService.GetDuplicatedFiles(request);
if (duplicated.Count != 0)
{
return(new BadRequestObjectResult($"duplicated filenames to save: {duplicated.Aggregate((i, j) => i + ", " + j)}"));
}
_downloaderService.Download(request);
return(Ok());
}