public bool ShouldDownloadThisUrl(string url)
{
// If the url is not downloadable
if (!HTMLDocumentDownloader.IsDownloadableUrl(url))
{
return(false);
}
// If the url isn't in the restricted domain
if (RestrictToDomain && RootDomain != null && UrlHelper.GetDomain(url) != UrlHelper.GetDomain(RootDomain))
{
return(false);
}
// If we should skip timed out hosts, and this host has timed out
if (IsTimedOutUrl(url))
{
return(false);
}
return(true);
}
private PageToDownload DownloadUrl(string url, PageToDownload parent, IProgressHost progress)
{
PageToDownload thisPageToDownload = null;
// Download the current page
LightWeightHTMLDocument lightWeightDoc = null;
using (HTMLDocumentDownloader downloader = new HTMLDocumentDownloader(_parentControl, url, null, _context.CookieString, _context.TimeoutMS, true))
{
downloader.DownloadHTMLDocument(progress);
lightWeightDoc = LightWeightHTMLDocument.FromIHTMLDocument2(downloader.HtmlDocument, downloader.Url);
thisPageToDownload = new PageToDownload(lightWeightDoc, url, null, parent);
// Reset the url in the event that a redirect occurred
thisPageToDownload.AbsoluteUrl = downloader.Url;
}
foreach (HTMLDocumentHelper.ResourceUrlInfo styleUrl in lightWeightDoc.StyleResourcesUrls)
{
thisPageToDownload.AddReference(new ReferenceToDownload(styleUrl.ResourceUrl, thisPageToDownload, styleUrl.ResourceAbsoluteUrl));
}
return(thisPageToDownload);
}
/// <summary>
/// Actually downloads the pages
/// </summary>
private PageToDownload[] DownloadPages(IProgressHost progress, string url, LightWeightHTMLDocument lightWeightDocument, PageToDownload parentPageToDownload)
{
// Check for cancel
if (progress.CancelRequested)
{
throw new OperationCancelledException();
}
_currentDepth++;
ArrayList downloadedPages = new ArrayList();
// Set up our progress
int thisPageTicks = FIRSTPAGETICKS;
if (_context.Depth == _currentDepth)
{
thisPageTicks = TOTALTICKS;
}
ProgressTick firstPagedownloadProgress = new ProgressTick(progress, thisPageTicks, TOTALTICKS);
string safeUrl = UrlHelper.GetUrlWithoutAnchorIdentifier(url);
// Look up the content type of this pageToDownload
UrlContentTypeInfo headerInfo = null;
if (_headerInfo.ContainsKey(safeUrl))
{
headerInfo = (UrlContentTypeInfo)_headerInfo[safeUrl];
}
else
{
if (lightWeightDocument != null)
{
headerInfo = new UrlContentTypeInfo("text/html", url);
}
else if (headerInfo == null && !_context.IsTimedOutUrl(url) && _context.ShouldDownloadThisUrl(url))
{
progress.UpdateProgress(string.Format(CultureInfo.CurrentCulture, Res.Get(StringId.ProgressDeterminingType), url));
if (lightWeightDocument == null)
{
headerInfo = ContentTypeHelper.ExpensivelyGetUrlContentType(url, _context.TimeoutMS);
}
else
{
headerInfo = ContentTypeHelper.InexpensivelyGetUrlContentType(url);
}
}
_headerInfo.Add(safeUrl, headerInfo);
}
// If this is a web page and we should download it, do it!
if ((lightWeightDocument != null && IsDownloadablePageResource(headerInfo)) ||
(lightWeightDocument == null && IsDownloadablePageResource(headerInfo) && _context.ShouldDownloadThisUrl(headerInfo))
)
{
bool downloadWorked = false;
int downloadAttempts = -1;
bool timedOut = true;
// Max sure we are retrying the correct number of times
ProgressTick pageDownloadProgress = new ProgressTick(firstPagedownloadProgress, 80, 100);
while (!downloadWorked && downloadAttempts++ < _context.RetryCount && timedOut)
{
timedOut = false;
pageDownloadProgress.UpdateProgress(0, 1);
try
{
// If we haven't downloaded this page yet download it
PageToDownload thisPageToDownload = null;
if (!_context.UrlAlreadyDownloaded(safeUrl))
{
if (lightWeightDocument == null)
{
thisPageToDownload = DownloadUrl(url, parentPageToDownload, pageDownloadProgress);
}
else
{
LightWeightHTMLDocument htmlDoc = lightWeightDocument;
// Only redownload if we absolutely need to
if (htmlDoc.HasFramesOrStyles && (htmlDoc.Frames == null || htmlDoc.StyleResourcesUrls == null))
{
string html = htmlDoc.GenerateHtml();
string tempFile = TempFileManager.Instance.CreateTempFile("temp.htm");
using (StreamWriter writer = new StreamWriter(tempFile, false, Encoding.UTF8))
writer.Write(html);
using (HTMLDocumentDownloader downloader = new HTMLDocumentDownloader(_parentControl, UrlHelper.GetLocalFileUrl(tempFile), htmlDoc.Title, _context.CookieString, _context.TimeoutMS, false))
{
downloader.DownloadHTMLDocument(pageDownloadProgress);
htmlDoc.UpdateBasedUponHTMLDocumentData(downloader.HtmlDocument, url);
}
}
thisPageToDownload = new PageToDownload(htmlDoc, url, null, parentPageToDownload);
if (htmlDoc.StyleResourcesUrls != null)
{
foreach (HTMLDocumentHelper.ResourceUrlInfo styleUrl in htmlDoc.StyleResourcesUrls)
{
thisPageToDownload.AddReference(new ReferenceToDownload(styleUrl.ResourceUrl, thisPageToDownload, styleUrl.ResourceAbsoluteUrl));
}
}
}
// Add this page to our lists
_context.AddPageToDownload(safeUrl, thisPageToDownload, true);
downloadedPages.Add(thisPageToDownload);
}
else
{
thisPageToDownload = (PageToDownload)_context.CreatedPageToDownloadTable[safeUrl];
}
// If we're downloading a site, add a second copy of the root page in the references subdir
// This was, if the root page gets renamed, links back to it will still work correctly
// This is a bit of a hack, but otherwise, we'll need to escape urls whenever we output
// the site and change the root file name
if (thisPageToDownload.IsRootPage && _context.Depth > 0)
{
PageToDownload copyOfThisPageToDownload = new PageToDownload(thisPageToDownload.LightWeightHTMLDocument.Clone(), thisPageToDownload.UrlToReplace, thisPageToDownload.FileName, thisPageToDownload);
downloadedPages.Add(copyOfThisPageToDownload);
}
// enumerate the frames of this page and add them to the list of pages
PageToDownload[] subFramesToDownload = GetFramePagesToDownload(thisPageToDownload);
downloadedPages.AddRange(subFramesToDownload);
foreach (PageToDownload pageToDownload in subFramesToDownload)
{
_context.AddPageToDownload(pageToDownload.AbsoluteUrl, pageToDownload, false);
}
// Now drill down based upon the depth configuration
if (_context.ShouldContinue(_currentDepth))
{
ProgressTick otherPagesdownloadProgress = new ProgressTick(progress, TOTALTICKS - thisPageTicks, TOTALTICKS);
downloadedPages.AddRange(GetSubPagesToDownload(otherPagesdownloadProgress, downloadedPages, thisPageToDownload));
}
downloadWorked = true;
firstPagedownloadProgress.UpdateProgress(1, 1);
}
catch (OperationTimedOutException)
{
timedOut = true;
}
catch (WebPageDownloaderException htex)
{
HandleException(new Exception(htex.Message, htex));
}
catch (Exception ex)
{
HandleException(new Exception(String.Format(CultureInfo.CurrentCulture, "{0} could not be downloaded", _url), ex));
}
}
// If we never got the download to succeed, add it to the list of timed out Urls
if (!downloadWorked && timedOut)
{
_context.AddTimedOutUrl(_url);
firstPagedownloadProgress.UpdateProgress(1, 1);
}
}
// If it isn't a page we'll just add the file to the reference list for the parent page
// There is not an else, because we could be looking at a reference, but a reference that
// should not be downloaded (in which case we just ignore it)
else if (headerInfo != null && _context.ShouldDownloadThisUrl(headerInfo))
{
parentPageToDownload.AddReference(new ReferenceToDownload(url, parentPageToDownload));
progress.UpdateProgress(1, 1);
}
progress.UpdateProgress(1, 1);
_currentDepth--;
return((PageToDownload[])downloadedPages.ToArray(typeof(PageToDownload)));
}
private PageToDownload DownloadUrl(string url, PageToDownload parent, IProgressHost progress)
{
PageToDownload thisPageToDownload = null;
// Download the current page
LightWeightHTMLDocument lightWeightDoc = null;
using (HTMLDocumentDownloader downloader = new HTMLDocumentDownloader(_parentControl, url, null, _context.CookieString, _context.TimeoutMS, true))
{
downloader.DownloadHTMLDocument(progress);
lightWeightDoc = LightWeightHTMLDocument.FromIHTMLDocument2(downloader.HtmlDocument, downloader.Url);
thisPageToDownload = new PageToDownload(lightWeightDoc, url, null, parent);
// Reset the url in the event that a redirect occurred
thisPageToDownload.AbsoluteUrl = downloader.Url;
}
foreach (HTMLDocumentHelper.ResourceUrlInfo styleUrl in lightWeightDoc.StyleResourcesUrls)
thisPageToDownload.AddReference(new ReferenceToDownload(styleUrl.ResourceUrl, thisPageToDownload, styleUrl.ResourceAbsoluteUrl));
return thisPageToDownload;
}
/// <summary>
/// Actually downloads the pages
/// </summary>
private PageToDownload[] DownloadPages(IProgressHost progress, string url, LightWeightHTMLDocument lightWeightDocument, PageToDownload parentPageToDownload)
{
// Check for cancel
if (progress.CancelRequested)
throw new OperationCancelledException();
_currentDepth++;
ArrayList downloadedPages = new ArrayList();
// Set up our progress
int thisPageTicks = FIRSTPAGETICKS;
if (_context.Depth == _currentDepth)
thisPageTicks = TOTALTICKS;
ProgressTick firstPagedownloadProgress = new ProgressTick(progress, thisPageTicks, TOTALTICKS);
string safeUrl = UrlHelper.GetUrlWithoutAnchorIdentifier(url);
// Look up the content type of this pageToDownload
UrlContentTypeInfo headerInfo = null;
if (_headerInfo.ContainsKey(safeUrl))
{
headerInfo = (UrlContentTypeInfo)_headerInfo[safeUrl];
}
else
{
if (lightWeightDocument != null)
headerInfo = new UrlContentTypeInfo("text/html", url);
else if (headerInfo == null && !_context.IsTimedOutUrl(url) && _context.ShouldDownloadThisUrl(url))
{
progress.UpdateProgress(string.Format(CultureInfo.CurrentCulture, Res.Get(StringId.ProgressDeterminingType), url));
if (lightWeightDocument == null)
headerInfo = ContentTypeHelper.ExpensivelyGetUrlContentType(url, _context.TimeoutMS);
else
headerInfo = ContentTypeHelper.InexpensivelyGetUrlContentType(url);
}
_headerInfo.Add(safeUrl, headerInfo);
}
// If this is a web page and we should download it, do it!
if ((lightWeightDocument != null && IsDownloadablePageResource(headerInfo)) ||
(lightWeightDocument == null && IsDownloadablePageResource(headerInfo) && _context.ShouldDownloadThisUrl(headerInfo))
)
{
bool downloadWorked = false;
int downloadAttempts = -1;
bool timedOut = true;
// Max sure we are retrying the correct number of times
ProgressTick pageDownloadProgress = new ProgressTick(firstPagedownloadProgress, 80, 100);
while (!downloadWorked && downloadAttempts++ < _context.RetryCount && timedOut)
{
timedOut = false;
pageDownloadProgress.UpdateProgress(0, 1);
try
{
// If we haven't downloaded this page yet download it
PageToDownload thisPageToDownload = null;
if (!_context.UrlAlreadyDownloaded(safeUrl))
{
if (lightWeightDocument == null)
thisPageToDownload = DownloadUrl(url, parentPageToDownload, pageDownloadProgress);
else
{
LightWeightHTMLDocument htmlDoc = lightWeightDocument;
// Only redownload if we absolutely need to
if (htmlDoc.HasFramesOrStyles && (htmlDoc.Frames == null || htmlDoc.StyleResourcesUrls == null))
{
string html = htmlDoc.GenerateHtml();
string tempFile = TempFileManager.Instance.CreateTempFile("temp.htm");
using (StreamWriter writer = new StreamWriter(tempFile, false, Encoding.UTF8))
writer.Write(html);
using (HTMLDocumentDownloader downloader = new HTMLDocumentDownloader(_parentControl, UrlHelper.GetLocalFileUrl(tempFile), htmlDoc.Title, _context.CookieString, _context.TimeoutMS, false))
{
downloader.DownloadHTMLDocument(pageDownloadProgress);
htmlDoc.UpdateBasedUponHTMLDocumentData(downloader.HtmlDocument, url);
}
}
thisPageToDownload = new PageToDownload(htmlDoc, url, null, parentPageToDownload);
if (htmlDoc.StyleResourcesUrls != null)
foreach (HTMLDocumentHelper.ResourceUrlInfo styleUrl in htmlDoc.StyleResourcesUrls)
thisPageToDownload.AddReference(new ReferenceToDownload(styleUrl.ResourceUrl, thisPageToDownload, styleUrl.ResourceAbsoluteUrl));
}
// Add this page to our lists
_context.AddPageToDownload(safeUrl, thisPageToDownload, true);
downloadedPages.Add(thisPageToDownload);
}
else
thisPageToDownload = (PageToDownload)_context.CreatedPageToDownloadTable[safeUrl];
// If we're downloading a site, add a second copy of the root page in the references subdir
// This was, if the root page gets renamed, links back to it will still work correctly
// This is a bit of a hack, but otherwise, we'll need to escape urls whenever we output
// the site and change the root file name
if (thisPageToDownload.IsRootPage && _context.Depth > 0)
{
PageToDownload copyOfThisPageToDownload = new PageToDownload(thisPageToDownload.LightWeightHTMLDocument.Clone(), thisPageToDownload.UrlToReplace, thisPageToDownload.FileName, thisPageToDownload);
downloadedPages.Add(copyOfThisPageToDownload);
}
// enumerate the frames of this page and add them to the list of pages
PageToDownload[] subFramesToDownload = GetFramePagesToDownload(thisPageToDownload);
downloadedPages.AddRange(subFramesToDownload);
foreach (PageToDownload pageToDownload in subFramesToDownload)
_context.AddPageToDownload(pageToDownload.AbsoluteUrl, pageToDownload, false);
// Now drill down based upon the depth configuration
if (_context.ShouldContinue(_currentDepth))
{
ProgressTick otherPagesdownloadProgress = new ProgressTick(progress, TOTALTICKS - thisPageTicks, TOTALTICKS);
downloadedPages.AddRange(GetSubPagesToDownload(otherPagesdownloadProgress, downloadedPages, thisPageToDownload));
}
downloadWorked = true;
firstPagedownloadProgress.UpdateProgress(1, 1);
}
catch (OperationTimedOutException)
{
timedOut = true;
}
catch (WebPageDownloaderException htex)
{
HandleException(new Exception(htex.Message, htex));
}
catch (Exception ex)
{
HandleException(new Exception(String.Format(CultureInfo.CurrentCulture, "{0} could not be downloaded", _url), ex));
}
}
// If we never got the download to succeed, add it to the list of timed out Urls
if (!downloadWorked && timedOut)
{
_context.AddTimedOutUrl(_url);
firstPagedownloadProgress.UpdateProgress(1, 1);
}
}
// If it isn't a page we'll just add the file to the reference list for the parent page
// There is not an else, because we could be looking at a reference, but a reference that
// should not be downloaded (in which case we just ignore it)
else if (headerInfo != null && _context.ShouldDownloadThisUrl(headerInfo))
{
parentPageToDownload.AddReference(new ReferenceToDownload(url, parentPageToDownload));
progress.UpdateProgress(1, 1);
}
progress.UpdateProgress(1, 1);
_currentDepth--;
return (PageToDownload[])downloadedPages.ToArray(typeof(PageToDownload));
}