public DownloadedResource(RemoteResource remoteResource, string mediaType, string charSet, byte[] content) { _remoteResource = remoteResource; _charSet = charSet; _resourceType = mediaType.Equals(TEXT_HTML) ? ResourceType.Content : ResourceType.Resource; _content = content; _textContent = string.Empty; }
public void Start() { RemoteResource root = new RemoteResource(_options, _options.BaseUrlString); _downloadableResources.Add(root); Download(); while (IsDownloadInProgress()) { } _progressBar.Close(); }
private void ExtractLinks(IDownloadedResource resource) { foreach (string url in resource.ExtractLinks()) { RemoteResource remoteResource = new RemoteResource(_options, url); if (!IsDownloadableResource(remoteResource)) { continue; } _uniqueResources.Add(remoteResource); _downloadableResources.Add(remoteResource); } }
private void ProcessLink(RemoteResource link) { IDownloadedResource resource = _downloader.Download(link); resource.Save(); _downloadedResources.Add(link); ExtractLinks(resource); UpdateProgressBar(); _semaphore.Release(); if (IsDownloadInProgress()) { return; } _downloadableResources.CompleteAdding(); }
public NotFoundResource(RemoteResource remoteResource) { _remoteResource = remoteResource; }
public MovedResource(RemoteResource remoteResource, string newUri) { _remoteResource = remoteResource; _newUri = newUri; }
private bool IsRequestAndResponseUrlSame(RemoteResource remoteResource, HttpResponseMessage response) { return(remoteResource.Url.Equals(response.Headers.Location)); }
private bool Is404(RemoteResource remoteResource, HttpResponseMessage response) { return(!IsRequestAndResponseUrlSame(remoteResource, response) && response.StatusCode == HttpStatusCode.Found); }
private bool IsDownloadableResource(RemoteResource remoteResource) { return(remoteResource.IsHttpResource && remoteResource.IsAtBaseSite && !_uniqueResources.Contains(remoteResource)); }