public void CheckEmptyTag(SiteObject siteobject, Kooboo.Dom.Document dom) { if (siteobject == null || dom == null) { return; } string name = Hardcoded.GetValue("Empty tag", session.context); HTMLCollection col = new HTMLCollection(); getEmptyTag(dom.body, col); foreach (var item in col.item) { string message = HttpUtility.HtmlEncode(item.OuterHtml); message += DiagnosisHelper.DisplayUsedBy(session.context, siteobject); session.AddMessage(name, message, MessageType.Warning); } }
private static void DumpDependencies(SiteObject site, string type, PageCollection pages) { foreach (var page in pages) { if (page.Discard) { continue; } Console.WriteLine($"Dependency {type} [{page.Path.FullName ?? page.Url}]"); foreach (var dep in page.Dependencies) { foreach (var file in dep.GetFiles()) { Console.WriteLine($" -> {file}"); } } } }
public static void FixOnSave(SiteDb SiteDb, SiteObject SiteObject, string Language = null) { if (SiteDb == null || SiteObject == null) { return; } foreach (var item in GetCheckers(SiteObject.GetType())) { if (item.FixOnSave) { List <object> paras = new List <object>(); paras.Add(SiteDb); paras.Add(SiteObject); paras.Add(Language); item.Fix.Invoke(item.ClassInstance, paras.ToArray()); } } }
public void CheckWrongNestedTag(SiteObject siteobject, Kooboo.Dom.Document dom) { if (siteobject == null || dom == null) { return; } string name = Hardcoded.GetValue("Wrong nested tag", session.context); HTMLCollection col = new HTMLCollection(); getWrongNestedTags(dom.body, col); foreach (var item in col.item) { string message = GetTagInfo(item); message += DiagnosisHelper.DisplayUsedBy(session.context, siteobject); session.AddMessage(name, message, MessageType.Warning); } }
static int Main(string[] args) { var site = new SiteObject() .Register <BundlePlugin>() .Register <MenuPlugin>() .Register <ExtendsPlugin>() .Register <LayoutPlugin>() .Register <ResourcePlugin>() .Register <DatasPlugin>() .Register <WatcherPlugin>() .Register <ServerPlugin>() .Register <MarkdownPlugin>() .Register <MinifierPlugin>() .Register <ScssPlugin>() .Register <TaxonomyPlugin>() .Register <YamlPlugin>(); return(site.Run(args)); }
public void CheckDepth(SiteObject siteobject, Kooboo.Dom.Document dom) { if (siteobject == null || dom == null) { return; } string name = Hardcoded.GetValue("Element too deep", session.context); HTMLCollection col = new HTMLCollection(); getTooDepthElements(dom.documentElement, col); foreach (var item in col.item) { string message = GetTagInfo(item); message += DiagnosisHelper.DisplayUsedBy(session.context, siteobject); session.AddMessage(name, message, MessageType.Warning); } }
public static string ParsePaging(this string input, int numOfPages, int selectedPage) { string result = string.Empty; if (numOfPages > 1) { StringBuilder builder = new StringBuilder(); SiteObject obj = DataObject.SiteObject; for (int i = 0; i < numOfPages; i++) { string filename = i == 0 ? "index.html" : string.Format("page-{0}.html", i + 1); builder.Append(DataObject.PagingItem .Replace("[page:url]", string.Format("{0}/{1}", obj.RootUrl, filename)) .Replace("[page:number]", (i + 1).ToString()) .Replace("[page:selected]", (selectedPage == i ? "selected" : ""))); } result = DataObject.PagingFrame.Replace("[content]", builder.ToString()); } return(input.Replace("[paging]", result)); }
public static void FixConstraint(SiteDb SiteDb, SiteObject SiteObject, string ConstrainName, string Language = null) { if (string.IsNullOrEmpty(ConstrainName) || SiteDb == null || SiteObject == null) { return; } foreach (var item in GetCheckers(SiteObject.GetType())) { if (item.HasFix && item.Name.ToLower() == ConstrainName.ToLower()) { List <object> paras = new List <object>(); paras.Add(SiteDb); paras.Add(SiteObject); paras.Add(Language); item.Fix.Invoke(item.ClassInstance, paras.ToArray()); } } }
public ServerPlugin(SiteObject site, Lazy <WatcherPlugin> watcher) : base(site) { _watcher = watcher; AppBuilders = new OrderedList <Action <IApplicationBuilder> >(); sockets = new HashSet <WebSocket>(); // Setup by default livereload Site.SetLiveReload(true); // Adds the server command ServerCommand = Site.CommandLine.Command("serve", newApp => { newApp.Description = "Builds the website, runs a web server and watches for changes"; newApp.HelpOption("-h|--help"); newApp.Invoke = Server; }, false); noWatchOption = ServerCommand.Option("-n|--no-watch", "Disables watching files and triggering of a new run", CommandOptionType.NoValue); }
public static void Convert(SiteDb SiteDb, SiteObject DownloadObject) { if (SiteDb.WebSite.ContinueConvert) { if (DownloadObject is Page) { var page = DownloadObject as Page; var continueconverters = SiteDb.ContinueConverter.All(); var converters = ConvertManager.GetConverters(); foreach (var item in continueconverters) { var converter = converters.Find(o => o.Type == item.ConvertType); if (converter != null && converter is IContinueConverter) { IContinueConverter continueconverter = converter as IContinueConverter; continueconverter.ContinueConvert(SiteDb, item.OriginalPageId, item.ConvertedTag, item.ObjectNameOrId, item.KoobooId, page, item.ElementPaths); } } } } }
public ActionResult Search(string typeName, int id, int?pageIndex) { if (!pageIndex.HasValue) { return(RedirectToAction(() => Search(typeName, id, 1))); } var type = SiteObject.GetType(typeName); if (type == null) { return(null); } var news = SiteObjectService.GetByRelationObject <News>(type, id) .IsActive().OrderByDescending(n => n.PublishDate) .ToPagedList(pageIndex.GetValueOrDefault(1) - 1, 5); return(View( new RelationNewsVM { News = news, SiteObject = SiteObjectService.GetBy( LinqToSqlUtils.GetTableName(type), id) })); }
private void ProceedBakeSite() { Log.LogInformation("Executing pre-build tasks"); { var prebuildCmds = BuildConfig.BeforeBuild; if (prebuildCmds.Count > 0) { if (!SkipPreBuild) { Log.LogInformation("Executing commands scheduled to run before build"); foreach (var cmd in prebuildCmds) { Log.LogInformation(cmd); var exitCode = ShellHelper.Run(cmd); if (exitCode != 0) { Environment.Exit(exitCode); } } } else { Log.LogWarning("The pre-build tasks have been requested to be skipped."); } } SiteObject.PreBuild(); } Log.LogInformation("Building site content"); SiteObject.Build(); Log.LogInformation("Executing post-build tasks"); SiteObject.PostBuild(); }
public async Task Execute() { if (this.SiteDb == null || this.TransferTask == null) { return; } List <TransferPage> pagelist = SiteDb.TransferPages.TableScan.Where(o => o.taskid == TransferTask.Id && o.done == false).SelectAll(); DownloadManager manager = new DownloadManager() { SiteDb = SiteDb, UserId = this.TransferTask.UserId }; List <TransferPage> transpages = new List <TransferPage>(); string baseurl = null; bool defaultstart = true; foreach (var item in pagelist) { if (baseurl == null) { baseurl = item.absoluteUrl; } var down = await DownloadHelper.DownloadUrlAsync(item.absoluteUrl, manager.CookieContainer, "GET", null, null); if (down == null || string.IsNullOrEmpty(down.GetString())) { item.done = true; SiteDb.TransferPages.AddOrUpdate(item); continue; } Page page = null; string downloadbody = down.GetString(); Guid sourcehash = Lib.Security.Hash.ComputeHashGuid(downloadbody); item.HtmlSourceHash = sourcehash; if (!string.IsNullOrEmpty(downloadbody)) { var result = SiteDb.TransferPages.Query.Where(o => o.HtmlSourceHash == sourcehash).SelectAll(); if (result != null && result.Count > 0) { var transferpage = result[0]; TransferHelper.AddPageRoute(SiteDb, transferpage.PageId, item.absoluteUrl, baseurl); item.done = true; item.PageId = transferpage.PageId; SiteDb.TransferPages.AddOrUpdate(item); continue; } } transpages.Add(item); SiteObject downloadobject = TransferHelper.AddDownload(manager, down, item.absoluteUrl, defaultstart, true, baseurl); if (downloadobject != null && downloadobject is Page) { page = downloadobject as Page; } if (page != null) { item.PageId = page.Id; } UpdateTransferPage(transpages, manager); if (defaultstart) { defaultstart = false; } // DownloadOnePage(manager, item); } while (!manager.IsComplete) { System.Threading.Thread.Sleep(20); } this.SiteDb.TransferTasks.SetDone(this.TransferTask.Id); }
public IEnumerable <ExtendDescription> FindAll(SiteObject site) { if (loaded) { return(cacheList); } if (RegistryUrl == null) { site.Error($"The registry Url for {nameof(DefaultExtendProvider)} cannot be null"); return(cacheList); } if (site.CanTrace()) { site.Trace($"Checking remote registry [{RegistryUrl}] for available extensions/themes"); } if (site == null) { throw new ArgumentNullException(nameof(site)); } string themeRegistryStr; try { using (HttpClient client = new HttpClient()) { themeRegistryStr = client.GetStringAsync(RegistryUrl).Result; } } catch (Exception ex) { site.Error($"Unable to load theme registry from Url [{RegistryUrl}]. Reason:{ex.GetReason()}"); return(cacheList); } var registryObject = new DynamicObject(); object result; if (site.Scripts.TryImportScript(themeRegistryStr, Name, registryObject, ScriptFlags.None, out result)) { var themeList = registryObject["extensions"] as ScriptArray; if (themeList != null) { foreach (var theme in themeList) { var themeObject = theme as ScriptObject; if (themeObject != null) { var themeName = (string)themeObject["name"]; var themeDescription = (string)themeObject["description"]; var themeUrl = (string)themeObject["url"]; var themeDirectory = (string)themeObject["directory"]; cacheList.Add(new ExtendDescription(themeName, themeDescription, themeUrl, themeDirectory)); } } } } loaded = true; return(cacheList); }
public static async Task <SiteObject> continueDownload(SiteDb siteDb, string RelativeUrl) { if (!siteDb.WebSite.ContinueDownload) { return(null); } string orgimport = null; var history = siteDb.TransferTasks.History().ToList(); if (history.Count() == 0) { return(null); } else { orgimport = history.First(); } /// track failed history... Guid downloadid = RelativeUrl.ToHashGuid(); DownloadFailTrack failtrack = siteDb.DownloadFailedLog.Get(downloadid); if (failtrack != null) { if (failtrack.HistoryTime.Where(o => o > DateTime.Now.AddMinutes(-30)).Any()) { return(null); } if (failtrack.HistoryTime.Count() > 3) { return(null); } } else { failtrack = new DownloadFailTrack(); failtrack.Id = downloadid; } var oktoDownload = await siteDb.TransferTasks.CanStartDownload(RelativeUrl); if (!oktoDownload) { return(null); } string fullurl = string.Empty; DownloadContent download = null; if (RelativeUrl.EndsWith("favicon.ico")) { return(null); } string hostname = TransferHelper.GetPossibleHostName(RelativeUrl); if (!string.IsNullOrEmpty(hostname)) { var findurl = history.Find(o => o.ToLower().EndsWith(hostname.ToLower())); if (!string.IsNullOrEmpty(findurl)) { string newrelative = RelativeUrl.Replace(hostname + "/", ""); fullurl = UrlHelper.Combine(findurl, newrelative); var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl); download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer); } else { string newrelative = RelativeUrl.Replace(hostname + "/", ""); // check whether it is https or not. // fullurl = UrlHelper.Combine(hostname, newrelative); var protocol = OrgProtocol(orgimport); fullurl = protocol + hostname + newrelative; var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl); download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer); } } if (download == null) { foreach (var item in history) { fullurl = UrlHelper.Combine(item, RelativeUrl); var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl); download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer); if (download != null) { break; } } } ///// 301, 302, will be converted to 200 and return back as well. So it is safe to == 200. if (download != null && download.StatusCode == 200) { DownloadManager downloadManager = new DownloadManager() { SiteDb = siteDb }; SiteObject downloadobject = TransferHelper.AddDownload(downloadManager, download, fullurl, false, true, orgimport); if (downloadobject is Page || downloadobject is View) { siteDb.TransferPages.AddOrUpdate(new TransferPage() { absoluteUrl = fullurl, PageId = downloadobject.Id }); } siteDb.TransferTasks.ReleaseDownload(RelativeUrl); ///for continue download content... Continue.ContinueTask.Convert(siteDb, downloadobject); return(downloadobject); } else { siteDb.TransferTasks.ReleaseDownload(RelativeUrl); } //download failed. failtrack.HistoryTime.Add(DateTime.Now); siteDb.DownloadFailedLog.AddOrUpdate(failtrack); return(null); }
private async Task Downloads(SiteDb siteDb, TransferProgress progress, DownloadManager manager) { List <TransferPage> transferingPages = new List <TransferPage>(); List <TransferPage> lowerPriorityPages = new List <TransferPage>(); var query = siteDb.TransferPages.Query.Where(o => o.taskid == progress.TaskId && o.done == false); while (true) { List <TransferPage> pagelist = query.SelectAll(); pagelist.RemoveAll(o => DoneUrlHash.Contains(o.Id)); if (pagelist == null || pagelist.Count == 0) { if (progress.counter < progress.TotalPages && lowerPriorityPages.Count() > 0) { var needed = progress.TotalPages - progress.counter; var neededpages = lowerPriorityPages.Take(needed); foreach (var item in neededpages) { progress.counter += 1; siteDb.TransferPages.AddOrUpdate(item); lowerPriorityPages.Remove(item); } continue; } else { break; } } foreach (var item in pagelist) { DoneUrlHash.Add(item.Id); var down = await DownloadHelper.DownloadUrlAsync(item.absoluteUrl, manager.CookieContainer); siteDb.TransferTasks.UpdateCookie(progress.TaskId, manager.CookieContainer); if (down == null || string.IsNullOrEmpty(down.GetString())) { item.done = true; siteDb.TransferPages.AddOrUpdate(item); continue; } Page page = null; string downloadbody = down.GetString(); Guid sourcehash = Lib.Security.Hash.ComputeHashGuid(downloadbody); item.HtmlSourceHash = sourcehash; if (!string.IsNullOrEmpty(downloadbody)) { var result = SiteDb.TransferPages.Query.Where(o => o.HtmlSourceHash == sourcehash).SelectAll(); if (result != null && result.Count > 0) { var transferpage = result[0]; TransferHelper.AddPageRoute(SiteDb, transferpage.PageId, item.absoluteUrl, progress.BaseUrl); item.done = true; item.PageId = transferpage.PageId; SiteDb.TransferPages.AddOrUpdate(item); continue; } } transferingPages.Add(item); SiteObject downloadobject = TransferHelper.AddDownload(manager, down, item.absoluteUrl, item.DefaultStartPage, true, progress.BaseUrl); if (downloadobject != null && downloadobject is Page) { page = downloadobject as Page; } if (page != null) { item.PageId = page.Id; } if (page == null || page.Dom == null) { item.done = true; manager.SiteDb.TransferPages.AddOrUpdate(item); continue; } if (progress.counter < progress.TotalPages && item.depth < progress.Levels) { page.Dom.URL = item.absoluteUrl; var links = TransferHelper.GetAbsoluteLinks(page.Dom, page.Dom.getBaseUrl()); foreach (var linkitem in links) { if (progress.counter >= progress.TotalPages) { break; } if (!UrlHelper.isSameHost(linkitem, progress.BaseUrl)) { continue; } if (!TransferHelper.IsPageUrl(linkitem)) { continue; } TransferPage newpage = new TransferPage(); newpage.absoluteUrl = linkitem; newpage.depth = item.depth + 1; newpage.taskid = progress.TaskId; if (!IsDuplicate(siteDb, newpage)) { if (TransferHelper.IsLowerPrioUrl(linkitem)) { if (lowerPriorityPages.Find(o => o.Id == newpage.Id) == null) { lowerPriorityPages.Add(newpage); } } else { progress.counter += 1; siteDb.TransferPages.AddOrUpdate(newpage); } } } } UpdateTransferPage(transferingPages, manager); } } while (transferingPages.Count() > 0) { System.Threading.Thread.Sleep(1000); UpdateTransferPage(transferingPages, manager); } }
public MenuPlugin(SiteObject site) : base(site) { Processor = new MenuProcessor(this); Site.SetValue("menu", this, true); Site.Content.BeforeLoadingProcessors.Insert(0, Processor); }
public LayoutContentObject(SiteObject site, FileEntry sourceFileInfo, ScriptInstance scriptInstance) : base(site, ContentObjectType.File, sourceFileInfo, scriptInstance) { ScriptObjectLocal = new ScriptObject(); }
public SiteChangeToken(SiteObject site, string filter) { _site = site; _filter = filter; }
public LayoutPlugin(SiteObject site) : base(site) { Processor = new LayoutProcessor(this); site.Content.ContentProcessors.Insert(0, Processor); }
public static bool GetLiveReload(this SiteObject site) { return(site.GetSafeValue <bool>(LiveReload)); }
public TemplateLoaderFromIncludes(SiteObject site) { this.site = site; IncludeFiles = new HashSet <FileEntry>(); }
public SiteFileProvider(SiteObject site) { _site = site; }
public static void SetLiveReload(this SiteObject site, bool value) { site[LiveReload] = value; }
public MarkdownPlugin(SiteObject site) : base(site) { site.Content.BeforeLoadingProcessors.Add(new MarkdownProcessor(this)); }
public TemplateLoaderUnauthorized(SiteObject site) { this.site = site; }
public MarkdownPlugin(SiteObject site) : base(site) { var processor = new MarkdownProcessor(this); site.Content.ContentProcessors.Add(processor); }
public ScssPlugin(SiteObject site) : base(site) { site.SetValue("scss", new ScssObject(this), true); site.Content.ContentProcessors.Add(new ScssProcessor(this)); }
public static async Task <SiteObject> continueDownload(SiteDb siteDb, string RelativeUrl) { if (!siteDb.WebSite.ContinueDownload) { return(null); } var oktoDownload = await siteDb.TransferTasks.CanStartDownload(RelativeUrl); if (!oktoDownload) { return(null); } var history = siteDb.TransferTasks.History().ToList(); if (history.Count() == 0) { return(null); } /// track failed history... Guid downloadid = RelativeUrl.ToHashGuid(); DownloadFailTrack failtrack = siteDb.DownloadFailedLog.Get(downloadid); if (failtrack != null) { if (failtrack.HistoryTime.Where(o => o > DateTime.Now.AddMinutes(-30)).Any()) { return(null); } } else { failtrack = new DownloadFailTrack(); failtrack.Id = downloadid; } string fullurl = string.Empty; DownloadContent download = null; string hostname = TransferHelper.GetPossibleHostName(RelativeUrl); if (!string.IsNullOrEmpty(hostname)) { var findurl = history.Find(o => o.ToLower().EndsWith(hostname.ToLower())); if (!string.IsNullOrEmpty(findurl)) { string newrelative = RelativeUrl.Replace(hostname + "/", ""); fullurl = UrlHelper.Combine(findurl, newrelative); var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl); download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer); } } if (download == null) { foreach (var item in history) { fullurl = UrlHelper.Combine(item, RelativeUrl); var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl); download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer); if (download != null) { break; } } } ///// 301, 302, will be converted to 200 and return back as well. So it is safe to == 200. if (download != null && download.StatusCode == 200) { DownloadManager downloadManager = new DownloadManager() { SiteDb = siteDb }; SiteObject downloadobject = TransferHelper.AddDownload(downloadManager, download, fullurl, false, true, fullurl); if (downloadobject is Page || downloadobject is View) { siteDb.TransferPages.AddOrUpdate(new TransferPage() { absoluteUrl = fullurl, PageId = downloadobject.Id }); } /// for continue download content... Continue.ContinueTask.Convert(siteDb, downloadobject); siteDb.TransferTasks.ReleaseDownload(RelativeUrl); return(downloadobject); } else { siteDb.TransferTasks.ReleaseDownload(RelativeUrl); } //download failed. failtrack.HistoryTime.Add(DateTime.Now); siteDb.DownloadFailedLog.AddOrUpdate(failtrack); if (failtrack.HistoryTime.Count() > 5) { var filetype = Kooboo.Lib.Helper.UrlHelper.GetFileType(RelativeUrl); byte consttype; switch (filetype) { case UrlHelper.UrlFileType.Image: consttype = ConstObjectType.Image; break; case UrlHelper.UrlFileType.JavaScript: consttype = ConstObjectType.Script; break; case UrlHelper.UrlFileType.Style: consttype = ConstObjectType.Style; break; case UrlHelper.UrlFileType.File: consttype = ConstObjectType.CmsFile; break; case UrlHelper.UrlFileType.PageOrView: consttype = ConstObjectType.Page; break; default: consttype = 0; break; } // siteDb.Routes.EnsureExists(RelativeUrl, consttype, default(Guid)); } return(null); }