public DownloadedResource(RemoteResource remoteResource, string mediaType, string charSet, byte[] content)
{
_remoteResource = remoteResource;
_charSet = charSet;
_resourceType = mediaType.Equals(TEXT_HTML) ? ResourceType.Content : ResourceType.Resource;
_content = content;
_textContent = string.Empty;
}
public void Start()
{
RemoteResource root = new RemoteResource(_options, _options.BaseUrlString);
_downloadableResources.Add(root);
Download();
while (IsDownloadInProgress())
{
}
_progressBar.Close();
}
private void ExtractLinks(IDownloadedResource resource)
{
foreach (string url in resource.ExtractLinks())
{
RemoteResource remoteResource = new RemoteResource(_options, url);
if (!IsDownloadableResource(remoteResource))
{
continue;
}
_uniqueResources.Add(remoteResource);
_downloadableResources.Add(remoteResource);
}
}
private void ProcessLink(RemoteResource link)
{
IDownloadedResource resource = _downloader.Download(link);
resource.Save();
_downloadedResources.Add(link);
ExtractLinks(resource);
UpdateProgressBar();
_semaphore.Release();
if (IsDownloadInProgress())
{
return;
}
_downloadableResources.CompleteAdding();
}
public NotFoundResource(RemoteResource remoteResource)
{
_remoteResource = remoteResource;
}
public MovedResource(RemoteResource remoteResource, string newUri)
{
_remoteResource = remoteResource;
_newUri = newUri;
}
private bool IsRequestAndResponseUrlSame(RemoteResource remoteResource, HttpResponseMessage response)
{
return(remoteResource.Url.Equals(response.Headers.Location));
}
private bool Is404(RemoteResource remoteResource, HttpResponseMessage response)
{
return(!IsRequestAndResponseUrlSame(remoteResource, response) && response.StatusCode == HttpStatusCode.Found);
}
private bool IsDownloadableResource(RemoteResource remoteResource)
{
return(remoteResource.IsHttpResource && remoteResource.IsAtBaseSite && !_uniqueResources.Contains(remoteResource));
}
