public void GeneratePageInAllLanguages(IContentRepository contentRepository, SiteConfigurationElement configuration, PageData page, bool ignoreHtmlDependencies)
{
// This page type should be ignored
if (page is IStaticWebIgnoreGenerate)
{
return;
}
var languages = page.ExistingLanguages;
foreach (var lang in languages)
{
var langPage = contentRepository.Get <PageData>(page.ContentLink.ToReferenceWithoutVersion(), lang);
var langContentLink = langPage.ContentLink.ToReferenceWithoutVersion();
// This page type has a conditional for when we should generate it
if (langPage is IStaticWebIgnoreGenerateDynamically generateDynamically)
{
if (!generateDynamically.ShouldGenerate())
{
if (generateDynamically.ShouldDeleteGenerated())
{
RemoveGeneratedPage(configuration, langContentLink, lang);
}
// This page should not be generated at this time, ignore it.
continue;
}
}
bool?useTemporaryAttribute = configuration.UseTemporaryAttribute.HasValue ? false : configuration.UseTemporaryAttribute;
GeneratePage(configuration, langPage, lang, useTemporaryAttribute, ignoreHtmlDependencies);
}
}
public void GeneratePagesDependingOnContent(SiteConfigurationElement configuration, ContentReference contentReference, bool?useTemporaryAttribute, bool ignoreHtmlDependencies)
{
var contentRepository = ServiceLocator.Current.GetInstance <IContentRepository>();
var references = contentRepository.GetReferencesToContent(contentReference, false).GroupBy(x => x.OwnerID + "-" + x.OwnerLanguage);
foreach (var group in references)
{
var item = group.FirstOrDefault();
if (item == null)
{
continue;
}
if (item.ReferenceType != (int)EPiServer.DataAbstraction.ReferenceType.PageLinkReference)
{
continue;
}
if (item.OwnerID == null)
{
continue;
}
var referencedContent = contentRepository.Get <IContent>(item.OwnerID.ToReferenceWithoutVersion(), item.OwnerLanguage);
if (referencedContent is PageData)
{
GeneratePage(item.OwnerID, referencedContent, useTemporaryAttribute, ignoreHtmlDependencies);
}
else if (referencedContent is BlockData)
{
GeneratePagesDependingOnBlock(configuration, item.OwnerID, useTemporaryAttribute, ignoreHtmlDependencies);
}
}
}
protected void EnsureSourceTagSupport(IStaticWebService staticWebService, SiteConfigurationElement configuration, ref string html, ref Dictionary <string, string> currentPageResourcePairs, ref ConcurrentDictionary <string, string> replaceResourcePairs, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, int callDepth = 0)
{
if (configuration == null || !configuration.Enabled)
{
return;
}
var sourceSetMatches = REGEX_FIND_SOURCE_REFERENCE.Matches(html);
foreach (Match sourceSetMatch in sourceSetMatches)
{
var imageCandidatesGroup = sourceSetMatch.Groups["imageCandidates"];
if (imageCandidatesGroup.Success)
{
var imageCandidates = imageCandidatesGroup.Value;
// Take into account that we can have many image candidates, for example: logo-768.png 768w, logo-768-1.5x.png 1.5x
var resourceMatches = REGEX_FIND_SOURCE_RESOUCE_REFERENCE.Matches(imageCandidates);
foreach (Match match in resourceMatches)
{
var group = match.Groups["resource"];
if (group.Success)
{
var resourceUrl = group.Value;
if (currentPageResourcePairs.ContainsValue(resourceUrl))
{
/**
* Website is probably using a 404 page that is not returning HTTP StatusCode 404, ignore this...
**/
continue;
}
if (replaceResourcePairs.ContainsKey(resourceUrl))
{
/**
* If we have already downloaded resource, we don't need to download it again.
* Not only usefull for pages repeating same resource but also in our Scheduled job where we try to generate all pages.
**/
if (!currentPageResourcePairs.ContainsKey(resourceUrl))
{
// current page has no info regarding this resource, add it
currentPageResourcePairs.Add(resourceUrl, replaceResourcePairs[resourceUrl]);
}
continue;
}
var newResourceUrl = staticWebService.EnsureResource(configuration, resourceUrl, currentPageResourcePairs, replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);
if (!replaceResourcePairs.ContainsKey(resourceUrl))
{
replaceResourcePairs.TryAdd(resourceUrl, newResourceUrl);
}
if (!currentPageResourcePairs.ContainsKey(resourceUrl))
{
currentPageResourcePairs.Add(resourceUrl, newResourceUrl);
}
}
}
}
}
}
protected void RemoveObsoletePages(SiteConfigurationElement configuration)
{
if (configuration == null || !configuration.Enabled)
{
return;
}
if (!Directory.Exists(configuration.OutputPath))
{
// Directory doesn't exist, nothing to remove :)
return;
}
try
{
DirectoryInfo directoryInfo = new DirectoryInfo(configuration.OutputPath);
var fileInfos = directoryInfo.GetFiles("index.html", SearchOption.AllDirectories);
var nOfCandidates = fileInfos.Length;
var candidateIndex = 1;
var siteUrl = configuration.Url.TrimEnd('/');
foreach (FileInfo info in fileInfos)
{
OnStatusChanged($"Looking for obsolete pages, candidate {candidateIndex} of {nOfCandidates}");
var url = info.FullName;
// c:\websites\A\
url = "/" + url.Replace(configuration.OutputPath, "").Replace("\\", "/");
// index.html
url = url.Replace("index.html", "");
IContent contentData = UrlResolver.Current.Route(new UrlBuilder(siteUrl + url));
// Does page exists?
if (!(contentData is PageData page) || !page.CheckPublishedStatus(PagePublishedStatus.Published))
{
// remove index.html file as it doesn't exist in EpiServer
info.Delete();
var dir = info.Directory;
// remove folder if no more files in it
if (dir.GetFiles().Length == 0 && dir.GetDirectories().Length == 0)
{
dir.Delete();
}
_numberOfObsoletePages++;
}
candidateIndex++;
}
}
catch (Exception)
{
// something was wrong, but this is just extra cleaning so ignore it
return;
}
}
public void RemoveGeneratedPage(SiteConfigurationElement configuration, ContentReference contentLink, CultureInfo language, bool removeSubFolders)
{
string orginalUrl = GetPageUrl(contentLink, language);
if (orginalUrl == null)
{
return;
}
RemoveGeneratedPage(configuration, orginalUrl, removeSubFolders);
}
public void CreateRedirectPages(SiteConfigurationElement configuration, string oldUrl, string newUrl)
{
if (configuration == null || !configuration.Enabled)
{
return;
}
if (!Directory.Exists(configuration.OutputPath + oldUrl))
{
// Directory doesn't exist, nothing to remove :)
return;
}
if (!File.Exists(configuration.OutputPath + oldUrl + "index.html"))
{
// File doesn't exist, nothing to remove :)
return;
}
try
{
DirectoryInfo directoryInfo = new DirectoryInfo(configuration.OutputPath + oldUrl);
var fileInfos = directoryInfo.GetFiles("index.html", SearchOption.AllDirectories);
foreach (FileInfo info in fileInfos)
{
var tempPath = info.FullName;
// c:\websites\A\
tempPath = "/" + tempPath.Replace(configuration.OutputPath, "").Replace("\\", "/");
// /old/
tempPath = tempPath.Replace(oldUrl, "");
// index.html
tempPath = tempPath.Replace("index.html", "");
// /new/
tempPath = newUrl + tempPath;
// Create redirect html file
var redirectHtml = $"<!doctype html><html><head><meta charset=\"utf-8\"><meta http-equiv=\"refresh\" content=\"0; URL='{tempPath}'\" /></head><body><a href=\"{tempPath}\">{tempPath}</a></body></html>";
File.WriteAllText(info.FullName, redirectHtml);
AfterIOWrite?.Invoke(this, new StaticWebIOEvent {
FilePath = info.FullName, Data = Encoding.UTF8.GetBytes(redirectHtml)
});
}
}
catch (Exception)
{
// something was wrong, but this is just extra cleaning so ignore it
return;
}
}
private IEnumerable <KeyValuePair <string, string> > GetPages(SiteConfigurationElement configuration, ParallelOptions parallelOptions, ContentReference startPage)
{
var page = _contentRepository.Get <PageData>(startPage);
AddAllPagesInAllLanguagesForConfiguration(configuration, page);
// Add pages url(s) into generated resouces from begining to prohibit it for downloading pages as resources
_generatedResources = new ConcurrentDictionary <string, string>(_sitePages[configuration.Name]);
// We now probably have most common
IEnumerable <KeyValuePair <string, string> > pages = SortPages(configuration, _sitePages[configuration.Name].Skip(1));
return(pages);
}
private void RemoveObsoleteResources(SiteConfigurationElement configuration)
{
if (configuration == null || !configuration.Enabled)
{
return;
}
if (!Directory.Exists(configuration.OutputPath + configuration.ResourceFolder))
{
// Directory doesn't exist, nothing to remove :)
return;
}
var resources = _generatedResources.Values.ToHashSet <string>();
try
{
DirectoryInfo directoryInfo = new DirectoryInfo(configuration.OutputPath + configuration.ResourceFolder);
var fileInfos = directoryInfo.GetFiles("*", SearchOption.AllDirectories);
var nOfCandidates = fileInfos.Length;
var candidateIndex = 1;
foreach (FileInfo info in fileInfos)
{
OnStatusChanged($"Looking for obsolete resources, candidate {candidateIndex} of {nOfCandidates}");
var resourcePath = info.FullName;
if (resourcePath.EndsWith("index.html"))
{
// Ignore index.html files (for example when having a empty resource folder name)
continue;
}
resourcePath = "/" + resourcePath.Replace(configuration.OutputPath, "").Replace("\\", "/");
if (!resources.Contains(resourcePath))
{
info.Delete();
_numberOfObsoleteResources++;
}
candidateIndex++;
}
}
catch (Exception)
{
// something was wrong, but this is just extra cleaning so ignore it
return;
}
}
public void RemovePageInAllLanguages(IContentRepository contentRepository, SiteConfigurationElement configuration, ContentReference contentReference)
{
var page = contentRepository.Get <PageData>(contentReference);
var languages = page.ExistingLanguages;
foreach (var lang in languages)
{
var langPage = contentRepository.Get <PageData>(page.ContentLink, lang);
var langContentLink = langPage.ContentLink;
var removeSubFolders = true;
RemoveGeneratedPage(configuration, langContentLink, lang, removeSubFolders);
}
}
protected void UpdateScheduledJobStatus(SiteConfigurationElement configuration, string pageUrl)
{
if (pageUrl.StartsWith("//"))
{
return;
}
// NOTE: If publishing event comes from scheduled publishing (orginalUrl includes protocol, domain and port number)
if (!pageUrl.StartsWith("/"))
{
pageUrl = new Uri(pageUrl).AbsolutePath;
}
OnStatusChanged($"{configuration.Name} - {_numberOfPages} pages found. currently on: {pageUrl}");
}
public void RemoveGeneratedPage(SiteConfigurationElement configuration, string orginalUrl, bool removeSubFolders = false)
{
if (configuration == null || !configuration.Enabled)
{
return;
}
string relativePath = GetPageRelativePath(orginalUrl);
if (!Directory.Exists(configuration.OutputPath + relativePath))
{
// Directory doesn't exist, nothing to remove :)
return;
}
if (!File.Exists(configuration.OutputPath + relativePath + "index.html"))
{
// File doesn't exist, nothing to remove :)
return;
}
File.Delete(configuration.OutputPath + relativePath + "index.html");
AfterIODelete?.Invoke(this, new StaticWebIOEvent {
FilePath = relativePath + "index.html"
});
bool hasFiles;
bool hasDirectories;
try
{
DirectoryInfo directoryInfo = new DirectoryInfo(configuration.OutputPath + relativePath);
hasFiles = directoryInfo.GetFiles().Length > 0;
hasDirectories = directoryInfo.GetDirectories().Length > 0;
}
catch (Exception)
{
// something was wrong, but this is just extra cleaning so ignore it
return;
}
// Directory is empty, remove empty directory
if (removeSubFolders || (!hasFiles && !hasDirectories))
{
Directory.Delete(configuration.OutputPath + relativePath, removeSubFolders);
}
}
private void GeneratingPages(SiteConfigurationElement configuration, ParallelOptions parallelOptions, IEnumerable <KeyValuePair <string, string> > pages)
{
OnStatusChanged($"{configuration.Name} - Generating Pages");
bool?useTemporaryAttribute = configuration.UseTemporaryAttribute.HasValue ? false : configuration.UseTemporaryAttribute;
// Runt first page first to get most of the common resources
var firstPageUrl = _sitePages[configuration.Name].FirstOrDefault();
_staticWebService.GeneratePage(configuration, firstPageUrl.Key, useTemporaryAttribute, IGNORE_HTML_DEPENDENCIES, null, _generatedResources);
Parallel.ForEach(pages, parallelOptions, (pageInfo, _) =>
{
// TODO: Change this to handle SimpleAddress...
_staticWebService.GeneratePage(configuration, pageInfo.Key, useTemporaryAttribute, IGNORE_HTML_DEPENDENCIES, null, _generatedResources);
});
OnStatusChanged($"{configuration.Name} - Generated Pages");
}
public void GeneratePage(SiteConfigurationElement configuration, PageData page, CultureInfo lang, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, ConcurrentDictionary <string, string> generatedResources = null)
{
var urls = GetUrlsForPage(configuration, page, lang, out string simpleAddress);
var firstUrl = urls.FirstOrDefault();
if (!string.IsNullOrEmpty(firstUrl))
{
GeneratePage(configuration, firstUrl, useTemporaryAttribute, ignoreHtmlDependencies, simpleAddress, generatedResources);
}
urls = urls.Skip(1).ToList();
foreach (var url in urls)
{
GeneratePage(configuration, url, useTemporaryAttribute, ignoreHtmlDependencies, null, generatedResources);
}
}
public List <string> GetUrlsForPage(SiteConfigurationElement configuration, PageData page, CultureInfo lang, out string simpleAddress)
{
var urls = new List <string>();
simpleAddress = string.IsNullOrEmpty(page.ExternalURL) ? null : "/" + page.ExternalURL;
var pageUrl = GetPageUrl(page.ContentLink.ToReferenceWithoutVersion(), lang);
if (!string.IsNullOrEmpty(pageUrl))
{
urls.Add(pageUrl);
// TODO: Add configuration.UseFallbackLanguage?
if (true)
{
urls.AddRange(GetFallbackUrlsForPageUrl(page.Language.TwoLetterISOLanguageName, pageUrl));
}
}
return(urls);
}
public void GeneratePagesDependingOnBlock(SiteConfigurationElement configuration, ContentReference contentLink, bool?useTemporaryAttribute, bool ignoreHtmlDependencies)
{
if (configuration == null || !configuration.Enabled)
{
return;
}
var repository = ServiceLocator.Current.GetInstance <IContentRepository>();
var pages = GetPageReferencesToContent(repository, contentLink);
foreach (var page in pages)
{
// This page or block type should be ignored
if (page is IStaticWebIgnoreGenerate)
{
continue;
}
// This page type has a conditional for when we should generate it
if (page is IStaticWebIgnoreGenerateDynamically generateDynamically)
{
if (!generateDynamically.ShouldGenerate())
{
if (generateDynamically.ShouldDeleteGenerated())
{
RemoveGeneratedPage(configuration, contentLink, page.Language);
}
// This page should not be generated at this time, ignore it.
continue;
}
}
var languages = page.ExistingLanguages;
foreach (var lang in languages)
{
GeneratePage(configuration, page, lang, useTemporaryAttribute, ignoreHtmlDependencies);
}
}
}
private IEnumerable <KeyValuePair <string, string> > SortPages(SiteConfigurationElement configuration, IEnumerable <KeyValuePair <string, string> > pages)
{
OnStatusChanged($"{configuration.Name} - Sorting Pages {configuration.GenerateOrderForScheduledJob}");
switch (configuration.GenerateOrderForScheduledJob)
{
default:
case Models.GenerateOrderForScheduledJob.Default:
pages = pages.ToList();
break;
case Models.GenerateOrderForScheduledJob.UrlDepthFirst:
pages = pages.OrderBy(pair => pair.Key.TrimEnd('/')).ToList();
break;
case Models.GenerateOrderForScheduledJob.UrlBreadthFirst:
// Smallest folder structure ordered alphabetic
pages = pages.OrderBy(pair => pair.Key.TrimEnd('/').Count(character => character == '/').ToString().PadLeft(5, '0') + '|' + pair.Key).ToList();
break;
}
OnStatusChanged($"{configuration.Name} - Sorted Pages {configuration.GenerateOrderForScheduledJob}");
return(pages);
}
protected string EnsureDependencies(string referencingUrl, string content, SiteConfigurationElement siteConfiguration, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, AllowedResourceTypeConfigurationElement typeConfiguration, Dictionary <string, string> currentPageResourcePairs = null, ConcurrentDictionary <string, string> replaceResourcePairs = null, int callDepth = 0)
{
string workingContent = content;
if (typeConfiguration.DenendencyLookup.HasFlag(ResourceDependencyLookup.Html))
{
if (!ignoreHtmlDependencies || callDepth == 0)
{
workingContent = _htmlDependencyService.EnsureDependencies(referencingUrl, workingContent, this, siteConfiguration, useTemporaryAttribute, ignoreHtmlDependencies, currentPageResourcePairs, replaceResourcePairs, ++callDepth);
}
}
if (typeConfiguration.DenendencyLookup.HasFlag(ResourceDependencyLookup.Svg))
{
workingContent = _svgDependencyService.EnsureDependencies(referencingUrl, workingContent, this, siteConfiguration, useTemporaryAttribute, ignoreHtmlDependencies, currentPageResourcePairs, replaceResourcePairs, ++callDepth);
}
if (typeConfiguration.DenendencyLookup.HasFlag(ResourceDependencyLookup.Css))
{
workingContent = _cssDependencyService.EnsureDependencies(referencingUrl, workingContent, this, siteConfiguration, useTemporaryAttribute, ignoreHtmlDependencies, currentPageResourcePairs, replaceResourcePairs, ++callDepth);
}
return(workingContent);
}
public string EnsureResource(SiteConfigurationElement siteConfiguration, string resourceUrl, Dictionary <string, string> currentPageResourcePairs, ConcurrentDictionary <string, string> replaceResourcePairs, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, int callDepth = 0)
{
var extension = GetExtension(resourceUrl);
bool preventDownload = IsDownloadPrevented(resourceUrl + extension);
if (preventDownload)
{
// don't download resource as it is of a known type we don't want to download
return(null);
}
var fileExtensionConfiguration = StaticWebConfiguration.AllowedResourceTypes.FirstOrDefault(r => r.FileExtension == extension);
if (fileExtensionConfiguration == null)
{
// don't download resource as it is of a known type we don't want to download
return(null);
}
if (replaceResourcePairs.TryGetValue(resourceUrl, out string oldResultingResourceUrl))
{
return(oldResultingResourceUrl);
}
var resourceInfo = DownloadResource(siteConfiguration.Url, resourceUrl, fileExtensionConfiguration);
if (resourceInfo == null)
{
// error occured OR resource type not allowed, ignore
return(null);
}
if (ignoreHtmlDependencies && callDepth != 0 && resourceInfo.TypeConfiguration.DenendencyLookup.HasFlag(ResourceDependencyLookup.Html))
{
return(null);
}
// For approved file extensions that we don't need to do any changes on
string newResourceUrl = GetNewResourceUrl(siteConfiguration.ResourceFolder, resourceUrl, resourceInfo.Data, resourceInfo.TypeConfiguration);
if (!replaceResourcePairs.ContainsKey(resourceUrl))
{
replaceResourcePairs.TryAdd(resourceUrl, newResourceUrl);
}
if (!currentPageResourcePairs.ContainsKey(resourceUrl))
{
currentPageResourcePairs.Add(resourceUrl, newResourceUrl);
}
if (!resourceInfo.TypeConfiguration.DenendencyLookup.HasFlag(ResourceDependencyLookup.None))
{
var content = Encoding.UTF8.GetString(resourceInfo.Data);
content = EnsureDependencies(resourceUrl, content, siteConfiguration, useTemporaryAttribute, ignoreHtmlDependencies, resourceInfo.TypeConfiguration, currentPageResourcePairs, replaceResourcePairs, callDepth);
resourceInfo.Data = Encoding.UTF8.GetBytes(content);
}
var hasContent = resourceInfo.Data != null && resourceInfo.Data.LongLength > 0;
if (newResourceUrl != null && hasContent)
{
var filepath = siteConfiguration.OutputPath + newResourceUrl.Replace("/", "\\");
if (resourceInfo.TypeConfiguration.UseHash)
{
// We are using hash ONLY as file name so no need to replace file that already exists
if (File.Exists(filepath))
{
return(newResourceUrl);
}
}
var shouldMaintainUrl = !resourceInfo.TypeConfiguration.UseHash && !resourceInfo.TypeConfiguration.UseResourceFolder && resourceInfo.TypeConfiguration.UseResourceUrl;
if (filepath.EndsWith("\\") && shouldMaintainUrl)
{
filepath = filepath + resourceInfo.TypeConfiguration.DefaultName + resourceInfo.TypeConfiguration.FileExtension;
}
WriteFile(filepath, resourceInfo.Data, useTemporaryAttribute);
return(newResourceUrl);
}
else
{
// Resource is not valid, return null
return(null);
}
}
protected void AddAllPagesInAllLanguagesForConfiguration(SiteConfigurationElement configuration, PageData page)
{
//For long running jobs periodically check if stop is signaled and if so stop execution
if (_stopSignaled)
{
OnStatusChanged("Stop of job was called");
return;
}
// Only add pages once (have this because of how websites can be setup to have a circle reference
if (page.ContentLink == null || _generatedPages.ContainsKey(page.ContentLink.ID))
{
return;
}
_generatedPages.Add(page.ContentLink.ID, null);
// This page type should be ignored
var ignorePageType = page is IStaticWebIgnoreGenerate;
var languages = page.ExistingLanguages;
foreach (var lang in languages)
{
var ignorePage = ignorePageType;
var langPage = _contentRepository.Get <PageData>(page.ContentLink.ToReferenceWithoutVersion(), lang);
var langContentLink = langPage.ContentLink.ToReferenceWithoutVersion();
if (!langPage.CheckPublishedStatus(PagePublishedStatus.Published))
{
ignorePage = true;
}
// This page type has a conditional for when we should generate it
if (langPage is IStaticWebIgnoreGenerateDynamically generateDynamically)
{
if (!generateDynamically.ShouldGenerate())
{
if (generateDynamically.ShouldDeleteGenerated())
{
_staticWebService.RemoveGeneratedPage(configuration, langContentLink, lang);
}
// This page should not be generated at this time, ignore it.
ignorePage = true;
}
}
if (!ignorePage)
{
var urls = _staticWebService.GetUrlsForPage(configuration, page, lang, out string simpleAddress);
foreach (var url in urls)
{
UpdateScheduledJobStatus(configuration, url);
_sitePages[configuration.Name].TryAdd(url, null);
}
if (!string.IsNullOrEmpty(simpleAddress))
{
_sitePages[configuration.Name].TryAdd(simpleAddress, null);
}
//_staticWebService.GeneratePage(configuration, langPage, lang, _generatedResources);
_numberOfPages++;
}
var children = _contentRepository.GetChildren <PageData>(langContentLink, lang);
foreach (PageData child in children)
{
AddAllPagesInAllLanguagesForConfiguration(configuration, child);
//For long running jobs periodically check if stop is signaled and if so stop execution
if (_stopSignaled)
{
OnStatusChanged("Stop of job was called");
return;
}
}
//For long running jobs periodically check if stop is signaled and if so stop execution
if (_stopSignaled)
{
OnStatusChanged("Stop of job was called");
return;
}
}
}
public void GeneratePage(SiteConfigurationElement configuration, string pageUrl, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, string simpleAddress = null, ConcurrentDictionary <string, string> generatedResources = null)
{
if (configuration == null || !configuration.Enabled)
{
return;
}
//string orginalUrl = GetPageUrl(contentLink, language);
if (pageUrl == null)
{
return;
}
string relativePath = GetPageRelativePath(pageUrl);
string relativeSimplePath = GetPageRelativePath(simpleAddress);
var hasSimpleAddress = !string.IsNullOrEmpty(simpleAddress);
var fullPageUrl = configuration.Url + pageUrl;
var fullSimpeAddress = configuration.Url + simpleAddress;
var generatePageEvent = new StaticWebGeneratePageEventArgs(fullPageUrl, simpleAddress);
if (generatedResources != null)
{
generatePageEvent.Resources = generatedResources;
}
else
{
generatePageEvent.Resources = new ConcurrentDictionary <string, string>();
}
string html = null;
var pageTypeConfiguration = StaticWebConfiguration.AllowedResourceTypes.FirstOrDefault(r => r.FileExtension == "");
if (pageTypeConfiguration == null)
{
// don't download resource as it is of a known type we don't want to download
generatePageEvent.CancelAction = true;
generatePageEvent.CancelReason = "AllowedResourceTypes configuration for file extension '' is missing (read: used by page).";
}
else
{
generatePageEvent.TypeConfiguration = pageTypeConfiguration;
}
BeforeGeneratePage?.Invoke(this, generatePageEvent);
// someone wants to cancel this generation of this event.
if (generatePageEvent.CancelAction)
{
return;
}
BeforeGetPageContent?.Invoke(this, generatePageEvent);
if (configuration.UseRouting)
{
StaticWebRouting.Remove(relativePath);
if (hasSimpleAddress)
{
StaticWebRouting.Remove(relativeSimplePath);
}
}
// someone wants to cancel this generation of this event.
if (!generatePageEvent.CancelAction)
{
var resourceInfo = DownloadResource(configuration.Url, pageUrl, generatePageEvent.TypeConfiguration);
if (resourceInfo != null)
{
if (generatePageEvent.TypeConfiguration.MimeType.StartsWith("*"))
{
generatePageEvent.TypeConfiguration = resourceInfo.TypeConfiguration;
}
if (resourceInfo.Data != null)
{
html = Encoding.UTF8.GetString(resourceInfo.Data);
}
}
}
generatePageEvent.Content = html;
// We don't care about a cancel action here
AfterGetPageContent?.Invoke(this, generatePageEvent);
if (generatePageEvent.Content == null)
{
return;
}
// reset cancel action and reason
generatePageEvent.CancelAction = false;
generatePageEvent.CancelReason = null;
BeforeTryToFixLinkUrls?.Invoke(this, generatePageEvent);
// someone wants to cancel/ignore ensuring resources.
if (!generatePageEvent.CancelAction)
{
generatePageEvent.Content = TryToFixLinkUrls(generatePageEvent.Content);
}
// reset cancel action and reason
generatePageEvent.CancelAction = false;
generatePageEvent.CancelReason = null;
BeforeEnsurePageResources?.Invoke(this, generatePageEvent);
// someone wants to cancel/ignore ensuring resources.
if (!generatePageEvent.CancelAction)
{
generatePageEvent.Content = EnsureDependencies(generatePageEvent.PageUrl, generatePageEvent.Content, configuration, useTemporaryAttribute, ignoreHtmlDependencies, generatePageEvent.TypeConfiguration, generatePageEvent.CurrentResources, generatePageEvent.Resources, 0);
}
// reset cancel action and reason
generatePageEvent.CancelAction = false;
generatePageEvent.CancelReason = null;
// We don't care about a cancel action here
AfterEnsurePageResources?.Invoke(this, generatePageEvent);
string filePath = configuration.OutputPath + relativePath + generatePageEvent.TypeConfiguration.DefaultName + generatePageEvent.TypeConfiguration.FileExtension;
var filePaths = new List <string>
{
filePath
};
if (hasSimpleAddress)
{
string filePath2 = configuration.OutputPath + relativeSimplePath + generatePageEvent.TypeConfiguration.DefaultName + generatePageEvent.TypeConfiguration.FileExtension;
filePaths.Add(filePath2);
}
generatePageEvent.FilePaths = filePaths;
// reset cancel action and reason
generatePageEvent.CancelAction = false;
generatePageEvent.CancelReason = null;
BeforeGeneratePageWrite?.Invoke(this, generatePageEvent);
// someone wants to cancel this page write.
if (!generatePageEvent.CancelAction)
{
// only write and route content if it is not empty
if (!string.IsNullOrEmpty(generatePageEvent.Content))
{
foreach (string outputFilePath in generatePageEvent.FilePaths)
{
var directory = Path.GetDirectoryName(outputFilePath);
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
WriteFile(outputFilePath, Encoding.UTF8.GetBytes(generatePageEvent.Content), useTemporaryAttribute);
if (configuration.UseRouting)
{
StaticWebRouting.Add(relativePath);
if (hasSimpleAddress)
{
StaticWebRouting.Add(relativeSimplePath);
}
}
}
}
}
AfterGeneratePageWrite?.Invoke(this, generatePageEvent);
// someone wants to cancel AFTER page write.
if (generatePageEvent.CancelAction)
{
return;
}
AfterGeneratePage?.Invoke(this, generatePageEvent);
}
public void RemoveGeneratedPage(SiteConfigurationElement configuration, ContentReference contentLink, CultureInfo language)
{
bool removeSubFolders = false;
RemoveGeneratedPage(configuration, contentLink, language, removeSubFolders);
}
public string EnsureDependencies(string referencingUrl, string content, IStaticWebService staticWebService, SiteConfigurationElement configuration, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, Dictionary <string, string> currentPageResourcePairs = null, ConcurrentDictionary <string, string> replaceResourcePairs = null, int callDepth = 0)
{
if (configuration == null || !configuration.Enabled)
{
return(content);
}
if (currentPageResourcePairs == null)
{
currentPageResourcePairs = new Dictionary <string, string>();
}
if (replaceResourcePairs == null)
{
replaceResourcePairs = new ConcurrentDictionary <string, string>();
}
// make sure we have all resources from script, link and img tags for current page
// <(script|link|img).*(href|src)="(?<resource>[^"]+)
EnsureScriptAndLinkAndImgAndATagSupport(staticWebService, configuration, ref content, ref currentPageResourcePairs, ref replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);
// make sure we have all source resources for current page
// <(source).*(srcset)="(?<resource>[^"]+)"
EnsureSourceTagSupport(staticWebService, configuration, ref content, ref currentPageResourcePairs, ref replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);
// TODO: make sure we have all meta resources for current page
// Below matches ALL meta content that is a URL
// <(meta).*(content)="(?<resource>(http:\/\/|https:\/\/|\/)[^"]+)"
// Below matches ONLY known properties
// <(meta).*(property|name)="(twitter:image|og:image)".*(content)="(?<resource>[http:\/\/|https:\/\/|\/][^"]+)"
var sbHtml = new StringBuilder(content);
foreach (KeyValuePair <string, string> pair in replaceResourcePairs)
{
// We have a value if we want to replace orginal url with a new one
if (pair.Value != null)
{
sbHtml = sbHtml.Replace(pair.Key, pair.Value);
}
}
return(sbHtml.ToString());
}
protected void EnsureScriptAndLinkAndImgAndATagSupport(IStaticWebService staticWebService, SiteConfigurationElement configuration, ref string html, ref Dictionary <string, string> currentPageResourcePairs, ref ConcurrentDictionary <string, string> replaceResourcePairs, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, int callDepth = 0)
{
if (configuration == null || !configuration.Enabled)
{
return;
}
var matches = REGEX_FIND_SCRIPT_OR_LINK_OR_IMG_OR_A_URL_REFERENCE.Matches(html);
foreach (Match match in matches)
{
var group = match.Groups["resource"];
if (group.Success)
{
var resourceUrl = group.Value;
if (currentPageResourcePairs.ContainsValue(resourceUrl))
{
/**
* Website is probably using a 404 page that is not returning HTTP StatusCode 404, ignore this...
**/
continue;
}
if (replaceResourcePairs.ContainsKey(resourceUrl))
{
/**
* If we have already downloaded resource, we don't need to download it again.
* Not only usefull for pages repeating same resource but also in our Scheduled job where we try to generate all pages.
**/
if (!currentPageResourcePairs.ContainsKey(resourceUrl))
{
// current page has no info regarding this resource, add it
currentPageResourcePairs.Add(resourceUrl, replaceResourcePairs[resourceUrl]);
}
continue;
}
var newResourceUrl = staticWebService.EnsureResource(configuration, resourceUrl, currentPageResourcePairs, replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);
if (!replaceResourcePairs.ContainsKey(resourceUrl))
{
replaceResourcePairs.TryAdd(resourceUrl, newResourceUrl);
}
if (!currentPageResourcePairs.ContainsKey(resourceUrl))
{
currentPageResourcePairs.Add(resourceUrl, newResourceUrl);
}
}
}
}
public string EnsureDependencies(string referencingUrl, string content, IStaticWebService staticWebService, SiteConfigurationElement configuration, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, Dictionary <string, string> currentPageResourcePairs = null, ConcurrentDictionary <string, string> replaceResourcePairs = null, int callDepth = 0)
{
if (configuration == null || !configuration.Enabled)
{
return(content);
}
if (currentPageResourcePairs == null)
{
currentPageResourcePairs = new Dictionary <string, string>();
}
if (replaceResourcePairs == null)
{
replaceResourcePairs = new ConcurrentDictionary <string, string>();
}
content = EnsureUrlReferenceSupport(referencingUrl, content, staticWebService, configuration, useTemporaryAttribute, ignoreHtmlDependencies, currentPageResourcePairs, replaceResourcePairs);
return(content);
}
private static string EnsureUrlReferenceSupport(string referencingUrl, string content, IStaticWebService staticWebService, SiteConfigurationElement configuration, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, Dictionary <string, string> currentPageResourcePairs, ConcurrentDictionary <string, string> replaceResourcePairs, int callDepth = 0)
{
// Download and ensure files referenced are downloaded also
var matches = REGEX_FIND_URL_REFERENCE.Matches(content);
foreach (Match match in matches)
{
var group = match.Groups["resource"];
if (group.Success)
{
var orginalUrl = group.Value;
var resourceUrl = orginalUrl;
var changedDir = false;
var directory = referencingUrl.Substring(0, referencingUrl.LastIndexOf('/'));
while (resourceUrl.StartsWith("../"))
{
changedDir = true;
resourceUrl = resourceUrl.Remove(0, 3);
directory = directory.Substring(0, directory.LastIndexOf('/'));
}
if (changedDir)
{
resourceUrl = directory.Replace(@"\", "/") + "/" + resourceUrl;
}
if (currentPageResourcePairs.ContainsValue(resourceUrl))
{
/**
* Website is probably using a 404 page that is not returning HTTP StatusCode 404, ignore this...
**/
continue;
}
if (replaceResourcePairs.ContainsKey(resourceUrl))
{
/**
* If we have already downloaded resource, we don't need to download it again.
* Not only usefull for pages repeating same resource but also in our Scheduled job where we try to generate all pages.
**/
if (!currentPageResourcePairs.ContainsKey(resourceUrl))
{
// current page has no info regarding this resource, add it
currentPageResourcePairs.Add(resourceUrl, replaceResourcePairs[resourceUrl]);
}
continue;
}
string newResourceUrl = staticWebService.EnsureResource(configuration, resourceUrl, currentPageResourcePairs, replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);
if (!string.IsNullOrEmpty(newResourceUrl))
{
content = content.Replace(orginalUrl, newResourceUrl);
if (!replaceResourcePairs.ContainsKey(resourceUrl))
{
replaceResourcePairs.TryAdd(resourceUrl, newResourceUrl);
}
if (!currentPageResourcePairs.ContainsKey(resourceUrl))
{
currentPageResourcePairs.Add(resourceUrl, newResourceUrl);
}
}
else
{
content = content.Replace(orginalUrl, "/" + configuration.ResourceFolder.Replace(@"\", "/") + resourceUrl);
if (!replaceResourcePairs.ContainsKey(resourceUrl))
{
replaceResourcePairs.TryAdd(resourceUrl, null);
}
if (!currentPageResourcePairs.ContainsKey(resourceUrl))
{
currentPageResourcePairs.Add(resourceUrl, null);
}
}
}
}
return(content);
}
public string EnsureDependencies(string referencingUrl, string content, IStaticWebService staticWebService, SiteConfigurationElement configuration, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, Dictionary <string, string> currentPageResourcePairs = null, ConcurrentDictionary <string, string> replaceResourcePairs = null, int callDepth = 0)
{
if (configuration == null || !configuration.Enabled)
{
return(content);
}
if (currentPageResourcePairs == null)
{
currentPageResourcePairs = new Dictionary <string, string>();
}
if (replaceResourcePairs == null)
{
replaceResourcePairs = new ConcurrentDictionary <string, string>();
}
// make sure we have all resources from script, link and img tags for current page
// <(use).*(xlink:href)="(?<resource>[^"]+)
EnsureUseTagSupport(staticWebService, configuration, ref content, ref currentPageResourcePairs, ref replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);
var sbHtml = new StringBuilder(content);
foreach (KeyValuePair <string, string> pair in replaceResourcePairs)
{
// We have a value if we want to replace orginal url with a new one
if (pair.Value != null)
{
sbHtml = sbHtml.Replace(pair.Key, pair.Value);
}
}
return(sbHtml.ToString());
}
