public void GeneratePageInAllLanguages(IContentRepository contentRepository, SiteConfigurationElement configuration, PageData page, bool ignoreHtmlDependencies)
        {
            // This page type should be ignored
            if (page is IStaticWebIgnoreGenerate)
            {
                return;
            }

            var languages = page.ExistingLanguages;

            foreach (var lang in languages)
            {
                var langPage = contentRepository.Get <PageData>(page.ContentLink.ToReferenceWithoutVersion(), lang);

                var langContentLink = langPage.ContentLink.ToReferenceWithoutVersion();

                // This page type has a conditional for when we should generate it
                if (langPage is IStaticWebIgnoreGenerateDynamically generateDynamically)
                {
                    if (!generateDynamically.ShouldGenerate())
                    {
                        if (generateDynamically.ShouldDeleteGenerated())
                        {
                            RemoveGeneratedPage(configuration, langContentLink, lang);
                        }

                        // This page should not be generated at this time, ignore it.
                        continue;
                    }
                }

                bool?useTemporaryAttribute = configuration.UseTemporaryAttribute.HasValue ? false : configuration.UseTemporaryAttribute;
                GeneratePage(configuration, langPage, lang, useTemporaryAttribute, ignoreHtmlDependencies);
            }
        }
        public void GeneratePagesDependingOnContent(SiteConfigurationElement configuration, ContentReference contentReference, bool?useTemporaryAttribute, bool ignoreHtmlDependencies)
        {
            var contentRepository = ServiceLocator.Current.GetInstance <IContentRepository>();
            var references        = contentRepository.GetReferencesToContent(contentReference, false).GroupBy(x => x.OwnerID + "-" + x.OwnerLanguage);

            foreach (var group in references)
            {
                var item = group.FirstOrDefault();
                if (item == null)
                {
                    continue;
                }

                if (item.ReferenceType != (int)EPiServer.DataAbstraction.ReferenceType.PageLinkReference)
                {
                    continue;
                }

                if (item.OwnerID == null)
                {
                    continue;
                }

                var referencedContent = contentRepository.Get <IContent>(item.OwnerID.ToReferenceWithoutVersion(), item.OwnerLanguage);
                if (referencedContent is PageData)
                {
                    GeneratePage(item.OwnerID, referencedContent, useTemporaryAttribute, ignoreHtmlDependencies);
                }
                else if (referencedContent is BlockData)
                {
                    GeneratePagesDependingOnBlock(configuration, item.OwnerID, useTemporaryAttribute, ignoreHtmlDependencies);
                }
            }
        }
예제 #3
0
        protected void EnsureSourceTagSupport(IStaticWebService staticWebService, SiteConfigurationElement configuration, ref string html, ref Dictionary <string, string> currentPageResourcePairs, ref ConcurrentDictionary <string, string> replaceResourcePairs, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, int callDepth = 0)
        {
            if (configuration == null || !configuration.Enabled)
            {
                return;
            }

            var sourceSetMatches = REGEX_FIND_SOURCE_REFERENCE.Matches(html);

            foreach (Match sourceSetMatch in sourceSetMatches)
            {
                var imageCandidatesGroup = sourceSetMatch.Groups["imageCandidates"];
                if (imageCandidatesGroup.Success)
                {
                    var imageCandidates = imageCandidatesGroup.Value;
                    // Take into account that we can have many image candidates, for example: logo-768.png 768w, logo-768-1.5x.png 1.5x
                    var resourceMatches = REGEX_FIND_SOURCE_RESOUCE_REFERENCE.Matches(imageCandidates);
                    foreach (Match match in resourceMatches)
                    {
                        var group = match.Groups["resource"];
                        if (group.Success)
                        {
                            var resourceUrl = group.Value;

                            if (currentPageResourcePairs.ContainsValue(resourceUrl))
                            {
                                /**
                                 * Website is probably using a 404 page that is not returning HTTP StatusCode 404, ignore this...
                                 **/
                                continue;
                            }

                            if (replaceResourcePairs.ContainsKey(resourceUrl))
                            {
                                /**
                                 * If we have already downloaded resource, we don't need to download it again.
                                 * Not only usefull for pages repeating same resource but also in our Scheduled job where we try to generate all pages.
                                 **/
                                if (!currentPageResourcePairs.ContainsKey(resourceUrl))
                                {
                                    // current page has no info regarding this resource, add it
                                    currentPageResourcePairs.Add(resourceUrl, replaceResourcePairs[resourceUrl]);
                                }
                                continue;
                            }
                            var newResourceUrl = staticWebService.EnsureResource(configuration, resourceUrl, currentPageResourcePairs, replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);
                            if (!replaceResourcePairs.ContainsKey(resourceUrl))
                            {
                                replaceResourcePairs.TryAdd(resourceUrl, newResourceUrl);
                            }
                            if (!currentPageResourcePairs.ContainsKey(resourceUrl))
                            {
                                currentPageResourcePairs.Add(resourceUrl, newResourceUrl);
                            }
                        }
                    }
                }
            }
        }
        protected void RemoveObsoletePages(SiteConfigurationElement configuration)
        {
            if (configuration == null || !configuration.Enabled)
            {
                return;
            }

            if (!Directory.Exists(configuration.OutputPath))
            {
                // Directory doesn't exist, nothing to remove :)
                return;
            }

            try
            {
                DirectoryInfo directoryInfo = new DirectoryInfo(configuration.OutputPath);
                var           fileInfos     = directoryInfo.GetFiles("index.html", SearchOption.AllDirectories);

                var nOfCandidates  = fileInfos.Length;
                var candidateIndex = 1;

                var siteUrl = configuration.Url.TrimEnd('/');
                foreach (FileInfo info in fileInfos)
                {
                    OnStatusChanged($"Looking for obsolete pages, candidate {candidateIndex} of {nOfCandidates}");

                    var url = info.FullName;
                    // c:\websites\A\
                    url = "/" + url.Replace(configuration.OutputPath, "").Replace("\\", "/");

                    // index.html
                    url = url.Replace("index.html", "");

                    IContent contentData = UrlResolver.Current.Route(new UrlBuilder(siteUrl + url));

                    // Does page exists?
                    if (!(contentData is PageData page) || !page.CheckPublishedStatus(PagePublishedStatus.Published))
                    {
                        // remove index.html file as it doesn't exist in EpiServer
                        info.Delete();
                        var dir = info.Directory;

                        // remove folder if no more files in it
                        if (dir.GetFiles().Length == 0 && dir.GetDirectories().Length == 0)
                        {
                            dir.Delete();
                        }

                        _numberOfObsoletePages++;
                    }
                    candidateIndex++;
                }
            }
            catch (Exception)
            {
                // something was wrong, but this is just extra cleaning so ignore it
                return;
            }
        }
        public void RemoveGeneratedPage(SiteConfigurationElement configuration, ContentReference contentLink, CultureInfo language, bool removeSubFolders)
        {
            string orginalUrl = GetPageUrl(contentLink, language);

            if (orginalUrl == null)
            {
                return;
            }

            RemoveGeneratedPage(configuration, orginalUrl, removeSubFolders);
        }
        public void CreateRedirectPages(SiteConfigurationElement configuration, string oldUrl, string newUrl)
        {
            if (configuration == null || !configuration.Enabled)
            {
                return;
            }

            if (!Directory.Exists(configuration.OutputPath + oldUrl))
            {
                // Directory doesn't exist, nothing to remove :)
                return;
            }

            if (!File.Exists(configuration.OutputPath + oldUrl + "index.html"))
            {
                // File doesn't exist, nothing to remove :)
                return;
            }

            try
            {
                DirectoryInfo directoryInfo = new DirectoryInfo(configuration.OutputPath + oldUrl);
                var           fileInfos     = directoryInfo.GetFiles("index.html", SearchOption.AllDirectories);

                foreach (FileInfo info in fileInfos)
                {
                    var tempPath = info.FullName;
                    // c:\websites\A\
                    tempPath = "/" + tempPath.Replace(configuration.OutputPath, "").Replace("\\", "/");
                    // /old/
                    tempPath = tempPath.Replace(oldUrl, "");

                    // index.html
                    tempPath = tempPath.Replace("index.html", "");

                    // /new/
                    tempPath = newUrl + tempPath;

                    // Create redirect html file
                    var redirectHtml = $"<!doctype html><html><head><meta charset=\"utf-8\"><meta http-equiv=\"refresh\" content=\"0; URL='{tempPath}'\" /></head><body><a href=\"{tempPath}\">{tempPath}</a></body></html>";
                    File.WriteAllText(info.FullName, redirectHtml);

                    AfterIOWrite?.Invoke(this, new StaticWebIOEvent {
                        FilePath = info.FullName, Data = Encoding.UTF8.GetBytes(redirectHtml)
                    });
                }
            }
            catch (Exception)
            {
                // something was wrong, but this is just extra cleaning so ignore it
                return;
            }
        }
        private IEnumerable <KeyValuePair <string, string> > GetPages(SiteConfigurationElement configuration, ParallelOptions parallelOptions, ContentReference startPage)
        {
            var page = _contentRepository.Get <PageData>(startPage);

            AddAllPagesInAllLanguagesForConfiguration(configuration, page);

            // Add pages url(s) into generated resouces from begining to prohibit it for downloading pages as resources
            _generatedResources = new ConcurrentDictionary <string, string>(_sitePages[configuration.Name]);

            // We now probably have most common
            IEnumerable <KeyValuePair <string, string> > pages = SortPages(configuration, _sitePages[configuration.Name].Skip(1));

            return(pages);
        }
        private void RemoveObsoleteResources(SiteConfigurationElement configuration)
        {
            if (configuration == null || !configuration.Enabled)
            {
                return;
            }

            if (!Directory.Exists(configuration.OutputPath + configuration.ResourceFolder))
            {
                // Directory doesn't exist, nothing to remove :)
                return;
            }

            var resources = _generatedResources.Values.ToHashSet <string>();

            try
            {
                DirectoryInfo directoryInfo = new DirectoryInfo(configuration.OutputPath + configuration.ResourceFolder);
                var           fileInfos     = directoryInfo.GetFiles("*", SearchOption.AllDirectories);

                var nOfCandidates  = fileInfos.Length;
                var candidateIndex = 1;
                foreach (FileInfo info in fileInfos)
                {
                    OnStatusChanged($"Looking for obsolete resources, candidate {candidateIndex} of {nOfCandidates}");

                    var resourcePath = info.FullName;

                    if (resourcePath.EndsWith("index.html"))
                    {
                        // Ignore index.html files (for example when having a empty resource folder name)
                        continue;
                    }

                    resourcePath = "/" + resourcePath.Replace(configuration.OutputPath, "").Replace("\\", "/");

                    if (!resources.Contains(resourcePath))
                    {
                        info.Delete();
                        _numberOfObsoleteResources++;
                    }
                    candidateIndex++;
                }
            }
            catch (Exception)
            {
                // something was wrong, but this is just extra cleaning so ignore it
                return;
            }
        }
        public void RemovePageInAllLanguages(IContentRepository contentRepository, SiteConfigurationElement configuration, ContentReference contentReference)
        {
            var page      = contentRepository.Get <PageData>(contentReference);
            var languages = page.ExistingLanguages;

            foreach (var lang in languages)
            {
                var langPage = contentRepository.Get <PageData>(page.ContentLink, lang);

                var langContentLink = langPage.ContentLink;

                var removeSubFolders = true;
                RemoveGeneratedPage(configuration, langContentLink, lang, removeSubFolders);
            }
        }
        protected void UpdateScheduledJobStatus(SiteConfigurationElement configuration, string pageUrl)
        {
            if (pageUrl.StartsWith("//"))
            {
                return;
            }

            // NOTE: If publishing event comes from scheduled publishing (orginalUrl includes protocol, domain and port number)
            if (!pageUrl.StartsWith("/"))
            {
                pageUrl = new Uri(pageUrl).AbsolutePath;
            }

            OnStatusChanged($"{configuration.Name} - {_numberOfPages} pages found. currently on: {pageUrl}");
        }
        public void RemoveGeneratedPage(SiteConfigurationElement configuration, string orginalUrl, bool removeSubFolders = false)
        {
            if (configuration == null || !configuration.Enabled)
            {
                return;
            }

            string relativePath = GetPageRelativePath(orginalUrl);

            if (!Directory.Exists(configuration.OutputPath + relativePath))
            {
                // Directory doesn't exist, nothing to remove :)
                return;
            }

            if (!File.Exists(configuration.OutputPath + relativePath + "index.html"))
            {
                // File doesn't exist, nothing to remove :)
                return;
            }

            File.Delete(configuration.OutputPath + relativePath + "index.html");

            AfterIODelete?.Invoke(this, new StaticWebIOEvent {
                FilePath = relativePath + "index.html"
            });
            bool hasFiles;
            bool hasDirectories;

            try
            {
                DirectoryInfo directoryInfo = new DirectoryInfo(configuration.OutputPath + relativePath);
                hasFiles       = directoryInfo.GetFiles().Length > 0;
                hasDirectories = directoryInfo.GetDirectories().Length > 0;
            }
            catch (Exception)
            {
                // something was wrong, but this is just extra cleaning so ignore it
                return;
            }

            // Directory is empty, remove empty directory
            if (removeSubFolders || (!hasFiles && !hasDirectories))
            {
                Directory.Delete(configuration.OutputPath + relativePath, removeSubFolders);
            }
        }
        private void GeneratingPages(SiteConfigurationElement configuration, ParallelOptions parallelOptions, IEnumerable <KeyValuePair <string, string> > pages)
        {
            OnStatusChanged($"{configuration.Name} - Generating Pages");
            bool?useTemporaryAttribute = configuration.UseTemporaryAttribute.HasValue ? false : configuration.UseTemporaryAttribute;

            // Runt first page first to get most of the common resources
            var firstPageUrl = _sitePages[configuration.Name].FirstOrDefault();

            _staticWebService.GeneratePage(configuration, firstPageUrl.Key, useTemporaryAttribute, IGNORE_HTML_DEPENDENCIES, null, _generatedResources);

            Parallel.ForEach(pages, parallelOptions, (pageInfo, _) =>
            {
                // TODO: Change this to handle SimpleAddress...
                _staticWebService.GeneratePage(configuration, pageInfo.Key, useTemporaryAttribute, IGNORE_HTML_DEPENDENCIES, null, _generatedResources);
            });
            OnStatusChanged($"{configuration.Name} - Generated Pages");
        }
        public void GeneratePage(SiteConfigurationElement configuration, PageData page, CultureInfo lang, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, ConcurrentDictionary <string, string> generatedResources = null)
        {
            var urls = GetUrlsForPage(configuration, page, lang, out string simpleAddress);

            var firstUrl = urls.FirstOrDefault();

            if (!string.IsNullOrEmpty(firstUrl))
            {
                GeneratePage(configuration, firstUrl, useTemporaryAttribute, ignoreHtmlDependencies, simpleAddress, generatedResources);
            }

            urls = urls.Skip(1).ToList();
            foreach (var url in urls)
            {
                GeneratePage(configuration, url, useTemporaryAttribute, ignoreHtmlDependencies, null, generatedResources);
            }
        }
        public List <string> GetUrlsForPage(SiteConfigurationElement configuration, PageData page, CultureInfo lang, out string simpleAddress)
        {
            var urls = new List <string>();

            simpleAddress = string.IsNullOrEmpty(page.ExternalURL) ? null : "/" + page.ExternalURL;
            var pageUrl = GetPageUrl(page.ContentLink.ToReferenceWithoutVersion(), lang);

            if (!string.IsNullOrEmpty(pageUrl))
            {
                urls.Add(pageUrl);

                // TODO: Add configuration.UseFallbackLanguage?
                if (true)
                {
                    urls.AddRange(GetFallbackUrlsForPageUrl(page.Language.TwoLetterISOLanguageName, pageUrl));
                }
            }

            return(urls);
        }
        public void GeneratePagesDependingOnBlock(SiteConfigurationElement configuration, ContentReference contentLink, bool?useTemporaryAttribute, bool ignoreHtmlDependencies)
        {
            if (configuration == null || !configuration.Enabled)
            {
                return;
            }

            var repository = ServiceLocator.Current.GetInstance <IContentRepository>();
            var pages      = GetPageReferencesToContent(repository, contentLink);

            foreach (var page in pages)
            {
                // This page or block type should be ignored
                if (page is IStaticWebIgnoreGenerate)
                {
                    continue;
                }

                // This page type has a conditional for when we should generate it
                if (page is IStaticWebIgnoreGenerateDynamically generateDynamically)
                {
                    if (!generateDynamically.ShouldGenerate())
                    {
                        if (generateDynamically.ShouldDeleteGenerated())
                        {
                            RemoveGeneratedPage(configuration, contentLink, page.Language);
                        }

                        // This page should not be generated at this time, ignore it.
                        continue;
                    }
                }

                var languages = page.ExistingLanguages;
                foreach (var lang in languages)
                {
                    GeneratePage(configuration, page, lang, useTemporaryAttribute, ignoreHtmlDependencies);
                }
            }
        }
        private IEnumerable <KeyValuePair <string, string> > SortPages(SiteConfigurationElement configuration, IEnumerable <KeyValuePair <string, string> > pages)
        {
            OnStatusChanged($"{configuration.Name} - Sorting Pages {configuration.GenerateOrderForScheduledJob}");
            switch (configuration.GenerateOrderForScheduledJob)
            {
            default:
            case Models.GenerateOrderForScheduledJob.Default:
                pages = pages.ToList();
                break;

            case Models.GenerateOrderForScheduledJob.UrlDepthFirst:
                pages = pages.OrderBy(pair => pair.Key.TrimEnd('/')).ToList();
                break;

            case Models.GenerateOrderForScheduledJob.UrlBreadthFirst:
                // Smallest folder structure ordered alphabetic
                pages = pages.OrderBy(pair => pair.Key.TrimEnd('/').Count(character => character == '/').ToString().PadLeft(5, '0') + '|' + pair.Key).ToList();
                break;
            }
            OnStatusChanged($"{configuration.Name} - Sorted Pages {configuration.GenerateOrderForScheduledJob}");
            return(pages);
        }
        protected string EnsureDependencies(string referencingUrl, string content, SiteConfigurationElement siteConfiguration, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, AllowedResourceTypeConfigurationElement typeConfiguration, Dictionary <string, string> currentPageResourcePairs = null, ConcurrentDictionary <string, string> replaceResourcePairs = null, int callDepth = 0)
        {
            string workingContent = content;

            if (typeConfiguration.DenendencyLookup.HasFlag(ResourceDependencyLookup.Html))
            {
                if (!ignoreHtmlDependencies || callDepth == 0)
                {
                    workingContent = _htmlDependencyService.EnsureDependencies(referencingUrl, workingContent, this, siteConfiguration, useTemporaryAttribute, ignoreHtmlDependencies, currentPageResourcePairs, replaceResourcePairs, ++callDepth);
                }
            }
            if (typeConfiguration.DenendencyLookup.HasFlag(ResourceDependencyLookup.Svg))
            {
                workingContent = _svgDependencyService.EnsureDependencies(referencingUrl, workingContent, this, siteConfiguration, useTemporaryAttribute, ignoreHtmlDependencies, currentPageResourcePairs, replaceResourcePairs, ++callDepth);
            }
            if (typeConfiguration.DenendencyLookup.HasFlag(ResourceDependencyLookup.Css))
            {
                workingContent = _cssDependencyService.EnsureDependencies(referencingUrl, workingContent, this, siteConfiguration, useTemporaryAttribute, ignoreHtmlDependencies, currentPageResourcePairs, replaceResourcePairs, ++callDepth);
            }

            return(workingContent);
        }
        public string EnsureResource(SiteConfigurationElement siteConfiguration, string resourceUrl, Dictionary <string, string> currentPageResourcePairs, ConcurrentDictionary <string, string> replaceResourcePairs, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, int callDepth = 0)
        {
            var  extension       = GetExtension(resourceUrl);
            bool preventDownload = IsDownloadPrevented(resourceUrl + extension);

            if (preventDownload)
            {
                // don't download resource as it is of a known type we don't want to download
                return(null);
            }

            var fileExtensionConfiguration = StaticWebConfiguration.AllowedResourceTypes.FirstOrDefault(r => r.FileExtension == extension);

            if (fileExtensionConfiguration == null)
            {
                // don't download resource as it is of a known type we don't want to download
                return(null);
            }

            if (replaceResourcePairs.TryGetValue(resourceUrl, out string oldResultingResourceUrl))
            {
                return(oldResultingResourceUrl);
            }

            var resourceInfo = DownloadResource(siteConfiguration.Url, resourceUrl, fileExtensionConfiguration);

            if (resourceInfo == null)
            {
                // error occured OR resource type not allowed, ignore
                return(null);
            }

            if (ignoreHtmlDependencies && callDepth != 0 && resourceInfo.TypeConfiguration.DenendencyLookup.HasFlag(ResourceDependencyLookup.Html))
            {
                return(null);
            }

            // For approved file extensions that we don't need to do any changes on
            string newResourceUrl = GetNewResourceUrl(siteConfiguration.ResourceFolder, resourceUrl, resourceInfo.Data, resourceInfo.TypeConfiguration);

            if (!replaceResourcePairs.ContainsKey(resourceUrl))
            {
                replaceResourcePairs.TryAdd(resourceUrl, newResourceUrl);
            }
            if (!currentPageResourcePairs.ContainsKey(resourceUrl))
            {
                currentPageResourcePairs.Add(resourceUrl, newResourceUrl);
            }

            if (!resourceInfo.TypeConfiguration.DenendencyLookup.HasFlag(ResourceDependencyLookup.None))
            {
                var content = Encoding.UTF8.GetString(resourceInfo.Data);
                content           = EnsureDependencies(resourceUrl, content, siteConfiguration, useTemporaryAttribute, ignoreHtmlDependencies, resourceInfo.TypeConfiguration, currentPageResourcePairs, replaceResourcePairs, callDepth);
                resourceInfo.Data = Encoding.UTF8.GetBytes(content);
            }

            var hasContent = resourceInfo.Data != null && resourceInfo.Data.LongLength > 0;

            if (newResourceUrl != null && hasContent)
            {
                var filepath = siteConfiguration.OutputPath + newResourceUrl.Replace("/", "\\");
                if (resourceInfo.TypeConfiguration.UseHash)
                {
                    // We are using hash ONLY as file name so no need to replace file that already exists
                    if (File.Exists(filepath))
                    {
                        return(newResourceUrl);
                    }
                }

                var shouldMaintainUrl = !resourceInfo.TypeConfiguration.UseHash && !resourceInfo.TypeConfiguration.UseResourceFolder && resourceInfo.TypeConfiguration.UseResourceUrl;
                if (filepath.EndsWith("\\") && shouldMaintainUrl)
                {
                    filepath = filepath + resourceInfo.TypeConfiguration.DefaultName + resourceInfo.TypeConfiguration.FileExtension;
                }

                WriteFile(filepath, resourceInfo.Data, useTemporaryAttribute);
                return(newResourceUrl);
            }
            else
            {
                // Resource is not valid, return null
                return(null);
            }
        }
        protected void AddAllPagesInAllLanguagesForConfiguration(SiteConfigurationElement configuration, PageData page)
        {
            //For long running jobs periodically check if stop is signaled and if so stop execution
            if (_stopSignaled)
            {
                OnStatusChanged("Stop of job was called");
                return;
            }

            // Only add pages once (have this because of how websites can be setup to have a circle reference
            if (page.ContentLink == null || _generatedPages.ContainsKey(page.ContentLink.ID))
            {
                return;
            }
            _generatedPages.Add(page.ContentLink.ID, null);

            // This page type should be ignored
            var ignorePageType = page is IStaticWebIgnoreGenerate;

            var languages = page.ExistingLanguages;

            foreach (var lang in languages)
            {
                var ignorePage = ignorePageType;
                var langPage   = _contentRepository.Get <PageData>(page.ContentLink.ToReferenceWithoutVersion(), lang);

                var langContentLink = langPage.ContentLink.ToReferenceWithoutVersion();

                if (!langPage.CheckPublishedStatus(PagePublishedStatus.Published))
                {
                    ignorePage = true;
                }

                // This page type has a conditional for when we should generate it
                if (langPage is IStaticWebIgnoreGenerateDynamically generateDynamically)
                {
                    if (!generateDynamically.ShouldGenerate())
                    {
                        if (generateDynamically.ShouldDeleteGenerated())
                        {
                            _staticWebService.RemoveGeneratedPage(configuration, langContentLink, lang);
                        }

                        // This page should not be generated at this time, ignore it.
                        ignorePage = true;
                    }
                }

                if (!ignorePage)
                {
                    var urls = _staticWebService.GetUrlsForPage(configuration, page, lang, out string simpleAddress);

                    foreach (var url in urls)
                    {
                        UpdateScheduledJobStatus(configuration, url);
                        _sitePages[configuration.Name].TryAdd(url, null);
                    }
                    if (!string.IsNullOrEmpty(simpleAddress))
                    {
                        _sitePages[configuration.Name].TryAdd(simpleAddress, null);
                    }

                    //_staticWebService.GeneratePage(configuration, langPage, lang, _generatedResources);
                    _numberOfPages++;
                }

                var children = _contentRepository.GetChildren <PageData>(langContentLink, lang);
                foreach (PageData child in children)
                {
                    AddAllPagesInAllLanguagesForConfiguration(configuration, child);

                    //For long running jobs periodically check if stop is signaled and if so stop execution
                    if (_stopSignaled)
                    {
                        OnStatusChanged("Stop of job was called");
                        return;
                    }
                }

                //For long running jobs periodically check if stop is signaled and if so stop execution
                if (_stopSignaled)
                {
                    OnStatusChanged("Stop of job was called");
                    return;
                }
            }
        }
        public void GeneratePage(SiteConfigurationElement configuration, string pageUrl, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, string simpleAddress = null, ConcurrentDictionary <string, string> generatedResources = null)
        {
            if (configuration == null || !configuration.Enabled)
            {
                return;
            }

            //string orginalUrl = GetPageUrl(contentLink, language);
            if (pageUrl == null)
            {
                return;
            }

            string relativePath       = GetPageRelativePath(pageUrl);
            string relativeSimplePath = GetPageRelativePath(simpleAddress);
            var    hasSimpleAddress   = !string.IsNullOrEmpty(simpleAddress);
            var    fullPageUrl        = configuration.Url + pageUrl;
            var    fullSimpeAddress   = configuration.Url + simpleAddress;

            var generatePageEvent = new StaticWebGeneratePageEventArgs(fullPageUrl, simpleAddress);

            if (generatedResources != null)
            {
                generatePageEvent.Resources = generatedResources;
            }
            else
            {
                generatePageEvent.Resources = new ConcurrentDictionary <string, string>();
            }


            string html = null;

            var pageTypeConfiguration = StaticWebConfiguration.AllowedResourceTypes.FirstOrDefault(r => r.FileExtension == "");

            if (pageTypeConfiguration == null)
            {
                // don't download resource as it is of a known type we don't want to download
                generatePageEvent.CancelAction = true;
                generatePageEvent.CancelReason = "AllowedResourceTypes configuration for file extension '' is missing (read: used by page).";
            }
            else
            {
                generatePageEvent.TypeConfiguration = pageTypeConfiguration;
            }


            BeforeGeneratePage?.Invoke(this, generatePageEvent);
            // someone wants to cancel this generation of this event.
            if (generatePageEvent.CancelAction)
            {
                return;
            }

            BeforeGetPageContent?.Invoke(this, generatePageEvent);

            if (configuration.UseRouting)
            {
                StaticWebRouting.Remove(relativePath);
                if (hasSimpleAddress)
                {
                    StaticWebRouting.Remove(relativeSimplePath);
                }
            }

            // someone wants to cancel this generation of this event.
            if (!generatePageEvent.CancelAction)
            {
                var resourceInfo = DownloadResource(configuration.Url, pageUrl, generatePageEvent.TypeConfiguration);
                if (resourceInfo != null)
                {
                    if (generatePageEvent.TypeConfiguration.MimeType.StartsWith("*"))
                    {
                        generatePageEvent.TypeConfiguration = resourceInfo.TypeConfiguration;
                    }

                    if (resourceInfo.Data != null)
                    {
                        html = Encoding.UTF8.GetString(resourceInfo.Data);
                    }
                }
            }
            generatePageEvent.Content = html;

            // We don't care about a cancel action here
            AfterGetPageContent?.Invoke(this, generatePageEvent);

            if (generatePageEvent.Content == null)
            {
                return;
            }

            // reset cancel action and reason
            generatePageEvent.CancelAction = false;
            generatePageEvent.CancelReason = null;

            BeforeTryToFixLinkUrls?.Invoke(this, generatePageEvent);
            // someone wants to cancel/ignore ensuring resources.
            if (!generatePageEvent.CancelAction)
            {
                generatePageEvent.Content = TryToFixLinkUrls(generatePageEvent.Content);
            }

            // reset cancel action and reason
            generatePageEvent.CancelAction = false;
            generatePageEvent.CancelReason = null;

            BeforeEnsurePageResources?.Invoke(this, generatePageEvent);
            // someone wants to cancel/ignore ensuring resources.
            if (!generatePageEvent.CancelAction)
            {
                generatePageEvent.Content = EnsureDependencies(generatePageEvent.PageUrl, generatePageEvent.Content, configuration, useTemporaryAttribute, ignoreHtmlDependencies, generatePageEvent.TypeConfiguration, generatePageEvent.CurrentResources, generatePageEvent.Resources, 0);
            }

            // reset cancel action and reason
            generatePageEvent.CancelAction = false;
            generatePageEvent.CancelReason = null;

            // We don't care about a cancel action here
            AfterEnsurePageResources?.Invoke(this, generatePageEvent);

            string filePath  = configuration.OutputPath + relativePath + generatePageEvent.TypeConfiguration.DefaultName + generatePageEvent.TypeConfiguration.FileExtension;
            var    filePaths = new List <string>
            {
                filePath
            };

            if (hasSimpleAddress)
            {
                string filePath2 = configuration.OutputPath + relativeSimplePath + generatePageEvent.TypeConfiguration.DefaultName + generatePageEvent.TypeConfiguration.FileExtension;
                filePaths.Add(filePath2);
            }
            generatePageEvent.FilePaths = filePaths;

            // reset cancel action and reason
            generatePageEvent.CancelAction = false;
            generatePageEvent.CancelReason = null;

            BeforeGeneratePageWrite?.Invoke(this, generatePageEvent);
            // someone wants to cancel this page write.
            if (!generatePageEvent.CancelAction)
            {
                // only write and route content if it is not empty
                if (!string.IsNullOrEmpty(generatePageEvent.Content))
                {
                    foreach (string outputFilePath in generatePageEvent.FilePaths)
                    {
                        var directory = Path.GetDirectoryName(outputFilePath);
                        if (!Directory.Exists(directory))
                        {
                            Directory.CreateDirectory(directory);
                        }

                        WriteFile(outputFilePath, Encoding.UTF8.GetBytes(generatePageEvent.Content), useTemporaryAttribute);

                        if (configuration.UseRouting)
                        {
                            StaticWebRouting.Add(relativePath);
                            if (hasSimpleAddress)
                            {
                                StaticWebRouting.Add(relativeSimplePath);
                            }
                        }
                    }
                }
            }

            AfterGeneratePageWrite?.Invoke(this, generatePageEvent);
            // someone wants to cancel AFTER page write.
            if (generatePageEvent.CancelAction)
            {
                return;
            }

            AfterGeneratePage?.Invoke(this, generatePageEvent);
        }
        public void RemoveGeneratedPage(SiteConfigurationElement configuration, ContentReference contentLink, CultureInfo language)
        {
            bool removeSubFolders = false;

            RemoveGeneratedPage(configuration, contentLink, language, removeSubFolders);
        }
예제 #22
0
        public string EnsureDependencies(string referencingUrl, string content, IStaticWebService staticWebService, SiteConfigurationElement configuration, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, Dictionary <string, string> currentPageResourcePairs = null, ConcurrentDictionary <string, string> replaceResourcePairs = null, int callDepth = 0)
        {
            if (configuration == null || !configuration.Enabled)
            {
                return(content);
            }

            if (currentPageResourcePairs == null)
            {
                currentPageResourcePairs = new Dictionary <string, string>();
            }

            if (replaceResourcePairs == null)
            {
                replaceResourcePairs = new ConcurrentDictionary <string, string>();
            }

            // make sure we have all resources from script, link and img tags for current page
            // <(script|link|img).*(href|src)="(?<resource>[^"]+)
            EnsureScriptAndLinkAndImgAndATagSupport(staticWebService, configuration, ref content, ref currentPageResourcePairs, ref replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);

            // make sure we have all source resources for current page
            // <(source).*(srcset)="(?<resource>[^"]+)"
            EnsureSourceTagSupport(staticWebService, configuration, ref content, ref currentPageResourcePairs, ref replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);

            // TODO: make sure we have all meta resources for current page
            // Below matches ALL meta content that is a URL
            // <(meta).*(content)="(?<resource>(http:\/\/|https:\/\/|\/)[^"]+)"
            // Below matches ONLY known properties
            // <(meta).*(property|name)="(twitter:image|og:image)".*(content)="(?<resource>[http:\/\/|https:\/\/|\/][^"]+)"

            var sbHtml = new StringBuilder(content);

            foreach (KeyValuePair <string, string> pair in replaceResourcePairs)
            {
                // We have a value if we want to replace orginal url with a new one
                if (pair.Value != null)
                {
                    sbHtml = sbHtml.Replace(pair.Key, pair.Value);
                }
            }

            return(sbHtml.ToString());
        }
예제 #23
0
        protected void EnsureScriptAndLinkAndImgAndATagSupport(IStaticWebService staticWebService, SiteConfigurationElement configuration, ref string html, ref Dictionary <string, string> currentPageResourcePairs, ref ConcurrentDictionary <string, string> replaceResourcePairs, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, int callDepth = 0)
        {
            if (configuration == null || !configuration.Enabled)
            {
                return;
            }

            var matches = REGEX_FIND_SCRIPT_OR_LINK_OR_IMG_OR_A_URL_REFERENCE.Matches(html);

            foreach (Match match in matches)
            {
                var group = match.Groups["resource"];
                if (group.Success)
                {
                    var resourceUrl = group.Value;
                    if (currentPageResourcePairs.ContainsValue(resourceUrl))
                    {
                        /**
                         * Website is probably using a 404 page that is not returning HTTP StatusCode 404, ignore this...
                         **/
                        continue;
                    }

                    if (replaceResourcePairs.ContainsKey(resourceUrl))
                    {
                        /**
                         * If we have already downloaded resource, we don't need to download it again.
                         * Not only usefull for pages repeating same resource but also in our Scheduled job where we try to generate all pages.
                         **/

                        if (!currentPageResourcePairs.ContainsKey(resourceUrl))
                        {
                            // current page has no info regarding this resource, add it
                            currentPageResourcePairs.Add(resourceUrl, replaceResourcePairs[resourceUrl]);
                        }
                        continue;
                    }

                    var newResourceUrl = staticWebService.EnsureResource(configuration, resourceUrl, currentPageResourcePairs, replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);
                    if (!replaceResourcePairs.ContainsKey(resourceUrl))
                    {
                        replaceResourcePairs.TryAdd(resourceUrl, newResourceUrl);
                    }
                    if (!currentPageResourcePairs.ContainsKey(resourceUrl))
                    {
                        currentPageResourcePairs.Add(resourceUrl, newResourceUrl);
                    }
                }
            }
        }
        public string EnsureDependencies(string referencingUrl, string content, IStaticWebService staticWebService, SiteConfigurationElement configuration, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, Dictionary <string, string> currentPageResourcePairs = null, ConcurrentDictionary <string, string> replaceResourcePairs = null, int callDepth = 0)
        {
            if (configuration == null || !configuration.Enabled)
            {
                return(content);
            }

            if (currentPageResourcePairs == null)
            {
                currentPageResourcePairs = new Dictionary <string, string>();
            }

            if (replaceResourcePairs == null)
            {
                replaceResourcePairs = new ConcurrentDictionary <string, string>();
            }

            content = EnsureUrlReferenceSupport(referencingUrl, content, staticWebService, configuration, useTemporaryAttribute, ignoreHtmlDependencies, currentPageResourcePairs, replaceResourcePairs);
            return(content);
        }
        private static string EnsureUrlReferenceSupport(string referencingUrl, string content, IStaticWebService staticWebService, SiteConfigurationElement configuration, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, Dictionary <string, string> currentPageResourcePairs, ConcurrentDictionary <string, string> replaceResourcePairs, int callDepth = 0)
        {
            // Download and ensure files referenced are downloaded also
            var matches = REGEX_FIND_URL_REFERENCE.Matches(content);

            foreach (Match match in matches)
            {
                var group = match.Groups["resource"];
                if (group.Success)
                {
                    var orginalUrl  = group.Value;
                    var resourceUrl = orginalUrl;
                    var changedDir  = false;
                    var directory   = referencingUrl.Substring(0, referencingUrl.LastIndexOf('/'));
                    while (resourceUrl.StartsWith("../"))
                    {
                        changedDir  = true;
                        resourceUrl = resourceUrl.Remove(0, 3);
                        directory   = directory.Substring(0, directory.LastIndexOf('/'));
                    }

                    if (changedDir)
                    {
                        resourceUrl = directory.Replace(@"\", "/") + "/" + resourceUrl;
                    }

                    if (currentPageResourcePairs.ContainsValue(resourceUrl))
                    {
                        /**
                         * Website is probably using a 404 page that is not returning HTTP StatusCode 404, ignore this...
                         **/
                        continue;
                    }

                    if (replaceResourcePairs.ContainsKey(resourceUrl))
                    {
                        /**
                         * If we have already downloaded resource, we don't need to download it again.
                         * Not only usefull for pages repeating same resource but also in our Scheduled job where we try to generate all pages.
                         **/

                        if (!currentPageResourcePairs.ContainsKey(resourceUrl))
                        {
                            // current page has no info regarding this resource, add it
                            currentPageResourcePairs.Add(resourceUrl, replaceResourcePairs[resourceUrl]);
                        }
                        continue;
                    }

                    string newResourceUrl = staticWebService.EnsureResource(configuration, resourceUrl, currentPageResourcePairs, replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);
                    if (!string.IsNullOrEmpty(newResourceUrl))
                    {
                        content = content.Replace(orginalUrl, newResourceUrl);
                        if (!replaceResourcePairs.ContainsKey(resourceUrl))
                        {
                            replaceResourcePairs.TryAdd(resourceUrl, newResourceUrl);
                        }
                        if (!currentPageResourcePairs.ContainsKey(resourceUrl))
                        {
                            currentPageResourcePairs.Add(resourceUrl, newResourceUrl);
                        }
                    }
                    else
                    {
                        content = content.Replace(orginalUrl, "/" + configuration.ResourceFolder.Replace(@"\", "/") + resourceUrl);
                        if (!replaceResourcePairs.ContainsKey(resourceUrl))
                        {
                            replaceResourcePairs.TryAdd(resourceUrl, null);
                        }
                        if (!currentPageResourcePairs.ContainsKey(resourceUrl))
                        {
                            currentPageResourcePairs.Add(resourceUrl, null);
                        }
                    }
                }
            }

            return(content);
        }
        public string EnsureDependencies(string referencingUrl, string content, IStaticWebService staticWebService, SiteConfigurationElement configuration, bool?useTemporaryAttribute, bool ignoreHtmlDependencies, Dictionary <string, string> currentPageResourcePairs = null, ConcurrentDictionary <string, string> replaceResourcePairs = null, int callDepth = 0)
        {
            if (configuration == null || !configuration.Enabled)
            {
                return(content);
            }

            if (currentPageResourcePairs == null)
            {
                currentPageResourcePairs = new Dictionary <string, string>();
            }

            if (replaceResourcePairs == null)
            {
                replaceResourcePairs = new ConcurrentDictionary <string, string>();
            }

            // make sure we have all resources from script, link and img tags for current page
            // <(use).*(xlink:href)="(?<resource>[^"]+)
            EnsureUseTagSupport(staticWebService, configuration, ref content, ref currentPageResourcePairs, ref replaceResourcePairs, useTemporaryAttribute, ignoreHtmlDependencies, callDepth);

            var sbHtml = new StringBuilder(content);

            foreach (KeyValuePair <string, string> pair in replaceResourcePairs)
            {
                // We have a value if we want to replace orginal url with a new one
                if (pair.Value != null)
                {
                    sbHtml = sbHtml.Replace(pair.Key, pair.Value);
                }
            }

            return(sbHtml.ToString());
        }