private void CopyImages(IHtmlDocument document, string url)
        {
            var imgSources = FilterForFileSource(document.Images.Select(i => i.Source));

            foreach (var imgSource in imgSources)
            {
                SafeRunner.Run(() => _fileWrapper.CreateFile(imgSource, url));
            }
        }
Beispiel #2
0
 public void Dispose()
 {
     this.transaction.Do(tran => tran.Dispose());
     this.connection.Do(delegate(SQLiteConnection conn)
     {
         SafeRunner.Run(conn.Close);
         SafeRunner.Run(conn.Dispose);
     });
 }
        private void CopyScripts(IHtmlDocument document, string url)
        {
            var scriptsSource = FilterForFileSource(document.Scripts.Select(s => s.Source));

            foreach (var scriptSource in scriptsSource)
            {
                SafeRunner.Run(() => _fileWrapper.CreateFile(scriptSource, url));
            }
        }
        private void CopyCss(IHtmlDocument document, string url)
        {
            var stylesHref = FilterForFileSource(document.QuerySelectorAll("link[href]")
                                                 .OfType <IHtmlLinkElement>().Select(s => s.Href));

            foreach (var styleSource in stylesHref)
            {
                SafeRunner.Run(() => _fileWrapper.CreateFile(styleSource, url));
            }
        }
Beispiel #5
0
 private void WaitRunningTasks()
 {
     foreach (
         var task in
         this.runningTasks.Keys.Where(
             task =>
             task.Status == TaskStatus.Running || task.Status == TaskStatus.WaitingForChildrenToComplete ||
             task.Status == TaskStatus.WaitingForActivation))
     {
         SafeRunner.Run(task.Wait);
     }
 }
Beispiel #6
0
 private void DisposeRunningTasks()
 {
     foreach (
         var task in
         this.runningTasks.Keys.Where(
             task =>
             task.Status == TaskStatus.RanToCompletion || task.Status == TaskStatus.Faulted ||
             task.Status == TaskStatus.Canceled))
     {
         SafeRunner.Run(task.Dispose);
     }
 }
Beispiel #7
0
 private void CancelPreviousTask()
 {
     if (!this.cancellation.IsCancellationRequested)
     {
         this.view.SetLogProgressCustomText(Resources.CancelPrevious);
         SafeRunner.Run(this.cancellation.Cancel);
     }
     this.queue.CleanupPendingTasks();
     this.WaitRunningTasks();
     SafeRunner.Run(this.cancellation.Dispose);
     this.DisposeRunningTasks();
     this.runningTasks.Clear();
 }
Beispiel #8
0
 public void Dispose()
 {
     this.queue.Shutdown(true);
     try
     {
         SafeRunner.Run(this.CancelPreviousTask);
     }
     finally
     {
         this.store.Do(logStore => SafeRunner.Run(logStore.Dispose));
         this.kernel.Dispose();
     }
 }
        private void CreatePage(string url, uint maxStep, List <string> analyzedUrls, uint stepCount = 0)
        {
            if (stepCount > maxStep)
            {
                return;
            }

            if (!DomainValidator.IsValid(_domainLimit, url, _rootUrl))
            {
                return;
            }

            if (_isVerbose)
            {
                Console.WriteLine($"Analyzing url: {url}");
            }

            SafeRunner.Run(() => _directoryWrapper.Create(url));
            var html = SafeRunner.Run(() => _reader.Read(url));

            SafeRunner.Run(() => _fileWrapper.CreateHtmlFile(url, html));

            var doc = _parser.Parse(html);

            CopyFiles(doc, url);
            analyzedUrls.Add(url);
            var links = GetLinks(doc, url);

            foreach (var link in links)
            {
                var linkUrl = PathConverter.ToUrl(url, link.Href);

                if (analyzedUrls.Contains(linkUrl))
                {
                    continue;
                }

                CreatePage(linkUrl, maxStep, analyzedUrls, stepCount + 1);
            }
        }