/// <summary>
/// Used to actually commit the HTML to disk
/// </summary>
/// <param name="pageInfo">The PageToDownload to write</param>
/// <param name="downloadedReferences">A hashtable of download references</param>
/// <param name="storage">The storage to write the file into</param>
private void WriteHtmlToDisk(PageToDownload pageInfo, FileBasedSiteStorage storage)
{
// Set the character set for this document
pageInfo.LightWeightHTMLDocument.MetaData.Charset = Encoding.UTF8.WebName;
string html = string.Empty;
// Replace references to any URL that we downloaded!
foreach (PageToDownload pageToDownload in _pagesToDownload)
{
if (!pageToDownload.IsRootPage)
{
pageInfo.LightWeightHTMLDocument.AddUrlToEscape(new UrlToReplace(pageToDownload.UrlToReplace, pageInfo.GetRelativeUrlForReferencedPage(pageToDownload)));
}
}
foreach (ReferenceToDownload referenceToDownload in _referencesToDownload.Values)
{
ReferenceToDownload downloadedReference = ((ReferenceToDownload)_referencesToDownload[referenceToDownload.AbsoluteUrl]);
// Since we consolidated references, replace the UrToReplace from the original reference
// with the relativePath to the reference that actually got downloaded
string path = downloadedReference.GetRelativeUrlForReference(pageInfo);
pageInfo.LightWeightHTMLDocument.AddUrlToEscape(new UrlToReplace(referenceToDownload.AbsoluteUrl, path));
}
html = pageInfo.LightWeightHTMLDocument.GenerateHtml();
// finally, write the html out to disk
string destination = Path.Combine(_siteStorage.BasePath, pageInfo.RelativePath);
Stream htmlStream = _siteStorage.Open(destination, AccessMode.Write);
using (StreamWriter writer = new StreamWriter(htmlStream, Encoding.UTF8))
{
writer.Write(html);
}
// if this is the entry page, write the path token and root file name
if (pageInfo.IsRootPage)
{
this._pathToken = pageInfo.ReferencedFileRelativePath;
_siteStorage.RootFile = pageInfo.FileName;
}
}
/// <summary>
///
/// </summary>
/// <param name="reference"></param>
public void AddReference(ReferenceToDownload reference)
{
// if we haven't already generated the list of references, we can safely add this
if (_referencesToDownload == null)
{
if (!_explicitReferences.Contains(reference))
{
_explicitReferences.Add(reference);
}
}
else
{
// Since we've already generated the new reference list, we should go ahead
// and add this to the end
ReferenceToDownload[] newReferenceList = new ReferenceToDownload[_referencesToDownload.Length + 1];
_referencesToDownload.CopyTo(newReferenceList, 0);
newReferenceList[_referencesToDownload.Length + 1] = reference;
_referencesToDownload = newReferenceList;
}
}
/// <summary>
///
/// </summary>
/// <param name="reference"></param>
public void AddReference(ReferenceToDownload reference)
{
// if we haven't already generated the list of references, we can safely add this
if (_referencesToDownload == null)
{
if (!_explicitReferences.Contains(reference))
_explicitReferences.Add(reference);
}
else
{
// Since we've already generated the new reference list, we should go ahead
// and add this to the end
ReferenceToDownload[] newReferenceList = new ReferenceToDownload[_referencesToDownload.Length + 1];
_referencesToDownload.CopyTo(newReferenceList, 0);
newReferenceList[_referencesToDownload.Length + 1] = reference;
_referencesToDownload = newReferenceList;
}
}
public DownloadWorkItem(ReferenceToDownload reference, FileBasedSiteStorage siteStorage, IProgressHost progressHost)
{
this.reference = reference;
this.siteStorage = siteStorage;
this.progressHost = progressHost;
}
/// <summary>
/// Download a reference, providing progress
/// </summary>
/// <param name="reference">The reference to download</param>
/// <param name="fileStorage">The storage to download the refernce into</param>
/// <param name="progressHost">The progressHost to provide feedback to</param>
private void DownloadReference(ReferenceToDownload reference, FileBasedSiteStorage fileStorage, IProgressHost progressHost)
{
UrlDownloadToFile downloader;
string fullPath;
downloader = new UrlDownloadToFile();
downloader.TimeoutMs = 30000;
if (progressHost.CancelRequested)
throw new OperationCancelledException();
// make sure that the directory exists
fullPath = Path.Combine(fileStorage.BasePath, reference.RelativePath);
string directory = Path.GetDirectoryName(fullPath);
if (!Directory.Exists(directory))
Directory.CreateDirectory(directory);
// Make sure there aren't any conflicts
lock (this)
{
string newFileName = Path.GetFileName(fullPath);
do
{
fullPath = PathHelper.GetNonConflictingPath(fullPath);
newFileName = Path.GetFileName(fullPath);
reference.SetFileName(ref newFileName);
} while (newFileName != Path.GetFileName(fullPath));
using (File.Open(fullPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
{
}
}
string downloadUrl = reference.AbsoluteUrl;
if (UrlHelper.IsFileUrl(downloadUrl))
downloadUrl = HttpUtility.UrlDecode(downloadUrl);
downloader.Url = downloadUrl.Trim();
downloader.FilePath = fullPath;
downloader.ShowSecurityUI = true;
if (CredentialsContext != null)
downloader.CredentialsContext = CredentialsContext;
try
{
downloader.Download(progressHost);
}
catch (COMException e)
{
// If the file couldn't be downloaded, this doesn't matter. But log it
Trace.WriteLine("Didn't download file: " + downloader.Url + " " + e.ToString());
}
// Fix the filename of the downloaded entity to have the correct extension
string contentType = downloader.ContentType;
if (contentType != null && File.Exists(fullPath))
{
string suggestedExtension = MimeHelper.GetExtensionFromContentType(contentType);
if (suggestedExtension == null)
suggestedExtension = UrlHelper.GetExtensionForUrl(downloader.FinalUrl);
if (Path.GetExtension(fullPath) != suggestedExtension)
{
string newFilePath = Path.ChangeExtension(fullPath, suggestedExtension);
string newFileName = Path.GetFileName(newFilePath);
// Try to reset the name until we can both agree
while (true)
{
newFilePath = PathHelper.GetNonConflictingPath(newFilePath);
newFileName = Path.GetFileName(newFilePath);
FileHelper.Rename(fullPath, newFilePath);
reference.SetFileName(ref newFileName);
if (newFileName != Path.GetFileName(newFilePath))
{
try
{
File.Delete(newFilePath);
}
catch (Exception e) { Debug.Fail("Unable to delete failed temp file: " + e.ToString()); }
}
else
{
break;
}
}
fullPath = newFilePath;
}
}
// Handle anything special we need to do for stylesheet and js file references
if (Path.GetExtension(fullPath) == ".css" && File.Exists(fullPath))
{
string fileContents = string.Empty;
using (StreamReader reader = new StreamReader(fullPath))
fileContents = reader.ReadToEnd();
if (fileContents != string.Empty)
{
LightWeightCSSReplacer cssReplacer = new LightWeightCSSReplacer(fileContents);
// fix up references
foreach (ReferenceToDownload referenceInfo in _referencesToDownload.Values)
cssReplacer.AddUrlToReplace(new UrlToReplace(referenceInfo.UrlToReplace, referenceInfo.FileName));
string newCss = cssReplacer.DoReplace();
using (StreamWriter writer = new StreamWriter(fullPath, false))
writer.Write(newCss);
}
}
}
/// <summary>
/// Download a reference, providing progress
/// </summary>
/// <param name="reference">The reference to download</param>
/// <param name="fileStorage">The storage to download the reference into</param>
/// <param name="progressHost">The progressHost to provide feedback to</param>
private void DownloadReference(ReferenceToDownload reference, FileBasedSiteStorage fileStorage, IProgressHost progressHost)
{
if (IsBase64EmbededImage(reference.AbsoluteUrl))
{
return;
}
UrlDownloadToFile downloader;
string fullPath;
downloader = new UrlDownloadToFile();
downloader.TimeoutMs = 30000;
if (progressHost.CancelRequested)
{
throw new OperationCancelledException();
}
// make sure that the directory exists
fullPath = Path.Combine(fileStorage.BasePath, reference.RelativePath);
string directory = Path.GetDirectoryName(fullPath);
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
// Make sure there aren't any conflicts
lock (this)
{
string newFileName = Path.GetFileName(fullPath);
do
{
fullPath = PathHelper.GetNonConflictingPath(fullPath);
newFileName = Path.GetFileName(fullPath);
reference.SetFileName(ref newFileName);
} while (newFileName != Path.GetFileName(fullPath));
using (File.Open(fullPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
{
}
}
string downloadUrl = reference.AbsoluteUrl;
if (UrlHelper.IsFileUrl(downloadUrl))
{
downloadUrl = HttpUtility.UrlDecode(downloadUrl);
}
downloader.Url = downloadUrl.Trim();
downloader.FilePath = fullPath;
downloader.ShowSecurityUI = true;
if (CredentialsContext != null)
{
downloader.CredentialsContext = CredentialsContext;
}
try
{
downloader.Download(progressHost);
}
catch (COMException e)
{
// If the file couldn't be downloaded, this doesn't matter. But log it
Trace.WriteLine("Didn't download file: " + downloader.Url + " " + e.ToString());
}
// Fix the filename of the downloaded entity to have the correct extension
string contentType = downloader.ContentType;
if (contentType != null && File.Exists(fullPath))
{
string suggestedExtension = MimeHelper.GetExtensionFromContentType(contentType);
if (suggestedExtension == null)
{
suggestedExtension = UrlHelper.GetExtensionForUrl(downloader.FinalUrl);
}
if (Path.GetExtension(fullPath) != suggestedExtension)
{
string newFilePath = Path.ChangeExtension(fullPath, suggestedExtension);
string newFileName = Path.GetFileName(newFilePath);
// Try to reset the name until we can both agree
while (true)
{
newFilePath = PathHelper.GetNonConflictingPath(newFilePath);
newFileName = Path.GetFileName(newFilePath);
FileHelper.Rename(fullPath, newFilePath);
reference.SetFileName(ref newFileName);
if (newFileName != Path.GetFileName(newFilePath))
{
try
{
File.Delete(newFilePath);
}
catch (Exception e) { Debug.Fail("Unable to delete failed temp file: " + e.ToString()); }
}
else
{
break;
}
}
fullPath = newFilePath;
}
}
// Handle anything special we need to do for stylesheet and js file references
if (Path.GetExtension(fullPath) == ".css" && File.Exists(fullPath))
{
string fileContents = string.Empty;
using (StreamReader reader = new StreamReader(fullPath))
fileContents = reader.ReadToEnd();
if (fileContents != string.Empty)
{
LightWeightCSSReplacer cssReplacer = new LightWeightCSSReplacer(fileContents);
// fix up references
foreach (ReferenceToDownload referenceInfo in _referencesToDownload.Values)
{
cssReplacer.AddUrlToReplace(new UrlToReplace(referenceInfo.UrlToReplace, referenceInfo.FileName));
}
string newCss = cssReplacer.DoReplace();
using (StreamWriter writer = new StreamWriter(fullPath, false))
writer.Write(newCss);
}
}
}
public DownloadWorkItem(ReferenceToDownload reference, FileBasedSiteStorage siteStorage, IProgressHost progressHost)
{
this.reference = reference;
this.siteStorage = siteStorage;
this.progressHost = progressHost;
}
