private void CopyImages(IHtmlDocument document, string url)
{
var imgSources = FilterForFileSource(document.Images.Select(i => i.Source));
foreach (var imgSource in imgSources)
{
SafeRunner.Run(() => _fileWrapper.CreateFile(imgSource, url));
}
}
public void Dispose()
{
this.transaction.Do(tran => tran.Dispose());
this.connection.Do(delegate(SQLiteConnection conn)
{
SafeRunner.Run(conn.Close);
SafeRunner.Run(conn.Dispose);
});
}
private void CopyScripts(IHtmlDocument document, string url)
{
var scriptsSource = FilterForFileSource(document.Scripts.Select(s => s.Source));
foreach (var scriptSource in scriptsSource)
{
SafeRunner.Run(() => _fileWrapper.CreateFile(scriptSource, url));
}
}
private void CopyCss(IHtmlDocument document, string url)
{
var stylesHref = FilterForFileSource(document.QuerySelectorAll("link[href]")
.OfType <IHtmlLinkElement>().Select(s => s.Href));
foreach (var styleSource in stylesHref)
{
SafeRunner.Run(() => _fileWrapper.CreateFile(styleSource, url));
}
}
private void WaitRunningTasks()
{
foreach (
var task in
this.runningTasks.Keys.Where(
task =>
task.Status == TaskStatus.Running || task.Status == TaskStatus.WaitingForChildrenToComplete ||
task.Status == TaskStatus.WaitingForActivation))
{
SafeRunner.Run(task.Wait);
}
}
private void DisposeRunningTasks()
{
foreach (
var task in
this.runningTasks.Keys.Where(
task =>
task.Status == TaskStatus.RanToCompletion || task.Status == TaskStatus.Faulted ||
task.Status == TaskStatus.Canceled))
{
SafeRunner.Run(task.Dispose);
}
}
private void CancelPreviousTask()
{
if (!this.cancellation.IsCancellationRequested)
{
this.view.SetLogProgressCustomText(Resources.CancelPrevious);
SafeRunner.Run(this.cancellation.Cancel);
}
this.queue.CleanupPendingTasks();
this.WaitRunningTasks();
SafeRunner.Run(this.cancellation.Dispose);
this.DisposeRunningTasks();
this.runningTasks.Clear();
}
public void Dispose()
{
this.queue.Shutdown(true);
try
{
SafeRunner.Run(this.CancelPreviousTask);
}
finally
{
this.store.Do(logStore => SafeRunner.Run(logStore.Dispose));
this.kernel.Dispose();
}
}
private void CreatePage(string url, uint maxStep, List <string> analyzedUrls, uint stepCount = 0)
{
if (stepCount > maxStep)
{
return;
}
if (!DomainValidator.IsValid(_domainLimit, url, _rootUrl))
{
return;
}
if (_isVerbose)
{
Console.WriteLine($"Analyzing url: {url}");
}
SafeRunner.Run(() => _directoryWrapper.Create(url));
var html = SafeRunner.Run(() => _reader.Read(url));
SafeRunner.Run(() => _fileWrapper.CreateHtmlFile(url, html));
var doc = _parser.Parse(html);
CopyFiles(doc, url);
analyzedUrls.Add(url);
var links = GetLinks(doc, url);
foreach (var link in links)
{
var linkUrl = PathConverter.ToUrl(url, link.Href);
if (analyzedUrls.Contains(linkUrl))
{
continue;
}
CreatePage(linkUrl, maxStep, analyzedUrls, stepCount + 1);
}
}
