public DiaryScraperNew(ILogger <DiaryScraperNew> logger, ScrapeContext context, DiaryScraperOptions options)
{
_logger = logger;
_cookieContainer = new CookieContainer();
_webClient = new CF_WebClient(_cookieContainer);
_context = context;
_options = options;
_downloadExistingChecker = new DownloadExistingChecker(Path.Combine(_options.WorkingDir, _options.DiaryName), context, _logger);
_downloader = new DataDownloader($"http://{_options.DiaryName}.diary.ru",
Path.Combine(_options.WorkingDir, _options.DiaryName),
_cookieContainer,
_logger);
_downloader.BeforeDownload += (s, e) =>
{
if (!(e.Resource is DiaryImage))
{
Progress.Values[ScrapeProgressNames.CurrentUrl] = e.Resource.Url.ToLower();
}
};
_downloader.AfterDownload += OnResourceDownloaded;
var config = new Configuration().WithCss();
_parser = new HtmlParser(config);
_moreFixer = new DiaryMoreLinksFixer(_downloader, _options.WorkingDir, _options.DiaryName);
}
public DiaryArchiver(ILogger <DiaryArchiver> logger, ArchiveTaskDescriptor descriptor, ScrapeContext context)
{
_logger = logger;
_descriptor = descriptor;
var config = new Configuration().WithCss();
_parser = new HtmlParser(config);
_context = context;
}
protected ScrapeContext GetContext(string dbPath)
{
var optionsBuilder = new DbContextOptionsBuilder();
optionsBuilder.UseSqlite($@"Data Source={dbPath}");
optionsBuilder.EnableSensitiveDataLogging();
var context = new ScrapeContext(optionsBuilder.Options);
context.Database.Migrate();
return(context);
}
public DownloadExistingChecker(string diaryDir, ScrapeContext context, ILogger logger)
{
_context = context;
_logger = logger;
_diaryDir = diaryDir;
}