public void CheckEmptyTag(SiteObject siteobject, Kooboo.Dom.Document dom)
{
if (siteobject == null || dom == null)
{
return;
}
string name = Hardcoded.GetValue("Empty tag", session.context);
HTMLCollection col = new HTMLCollection();
getEmptyTag(dom.body, col);
foreach (var item in col.item)
{
string message = HttpUtility.HtmlEncode(item.OuterHtml);
message += DiagnosisHelper.DisplayUsedBy(session.context, siteobject);
session.AddMessage(name, message, MessageType.Warning);
}
}
private static void DumpDependencies(SiteObject site, string type, PageCollection pages)
{
foreach (var page in pages)
{
if (page.Discard)
{
continue;
}
Console.WriteLine($"Dependency {type} [{page.Path.FullName ?? page.Url}]");
foreach (var dep in page.Dependencies)
{
foreach (var file in dep.GetFiles())
{
Console.WriteLine($" -> {file}");
}
}
}
}
public static void FixOnSave(SiteDb SiteDb, SiteObject SiteObject, string Language = null)
{
if (SiteDb == null || SiteObject == null)
{
return;
}
foreach (var item in GetCheckers(SiteObject.GetType()))
{
if (item.FixOnSave)
{
List <object> paras = new List <object>();
paras.Add(SiteDb);
paras.Add(SiteObject);
paras.Add(Language);
item.Fix.Invoke(item.ClassInstance, paras.ToArray());
}
}
}
public void CheckWrongNestedTag(SiteObject siteobject, Kooboo.Dom.Document dom)
{
if (siteobject == null || dom == null)
{
return;
}
string name = Hardcoded.GetValue("Wrong nested tag", session.context);
HTMLCollection col = new HTMLCollection();
getWrongNestedTags(dom.body, col);
foreach (var item in col.item)
{
string message = GetTagInfo(item);
message += DiagnosisHelper.DisplayUsedBy(session.context, siteobject);
session.AddMessage(name, message, MessageType.Warning);
}
}
static int Main(string[] args)
{
var site = new SiteObject()
.Register <BundlePlugin>()
.Register <MenuPlugin>()
.Register <ExtendsPlugin>()
.Register <LayoutPlugin>()
.Register <ResourcePlugin>()
.Register <DatasPlugin>()
.Register <WatcherPlugin>()
.Register <ServerPlugin>()
.Register <MarkdownPlugin>()
.Register <MinifierPlugin>()
.Register <ScssPlugin>()
.Register <TaxonomyPlugin>()
.Register <YamlPlugin>();
return(site.Run(args));
}
public void CheckDepth(SiteObject siteobject, Kooboo.Dom.Document dom)
{
if (siteobject == null || dom == null)
{
return;
}
string name = Hardcoded.GetValue("Element too deep", session.context);
HTMLCollection col = new HTMLCollection();
getTooDepthElements(dom.documentElement, col);
foreach (var item in col.item)
{
string message = GetTagInfo(item);
message += DiagnosisHelper.DisplayUsedBy(session.context, siteobject);
session.AddMessage(name, message, MessageType.Warning);
}
}
public static string ParsePaging(this string input, int numOfPages, int selectedPage)
{
string result = string.Empty;
if (numOfPages > 1)
{
StringBuilder builder = new StringBuilder();
SiteObject obj = DataObject.SiteObject;
for (int i = 0; i < numOfPages; i++)
{
string filename = i == 0 ? "index.html" : string.Format("page-{0}.html", i + 1);
builder.Append(DataObject.PagingItem
.Replace("[page:url]", string.Format("{0}/{1}", obj.RootUrl, filename))
.Replace("[page:number]", (i + 1).ToString())
.Replace("[page:selected]", (selectedPage == i ? "selected" : "")));
}
result = DataObject.PagingFrame.Replace("[content]", builder.ToString());
}
return(input.Replace("[paging]", result));
}
public static void FixConstraint(SiteDb SiteDb, SiteObject SiteObject, string ConstrainName, string Language = null)
{
if (string.IsNullOrEmpty(ConstrainName) || SiteDb == null || SiteObject == null)
{
return;
}
foreach (var item in GetCheckers(SiteObject.GetType()))
{
if (item.HasFix && item.Name.ToLower() == ConstrainName.ToLower())
{
List <object> paras = new List <object>();
paras.Add(SiteDb);
paras.Add(SiteObject);
paras.Add(Language);
item.Fix.Invoke(item.ClassInstance, paras.ToArray());
}
}
}
public ServerPlugin(SiteObject site, Lazy <WatcherPlugin> watcher) : base(site)
{
_watcher = watcher;
AppBuilders = new OrderedList <Action <IApplicationBuilder> >();
sockets = new HashSet <WebSocket>();
// Setup by default livereload
Site.SetLiveReload(true);
// Adds the server command
ServerCommand = Site.CommandLine.Command("serve", newApp =>
{
newApp.Description = "Builds the website, runs a web server and watches for changes";
newApp.HelpOption("-h|--help");
newApp.Invoke = Server;
}, false);
noWatchOption = ServerCommand.Option("-n|--no-watch", "Disables watching files and triggering of a new run", CommandOptionType.NoValue);
}
public static void Convert(SiteDb SiteDb, SiteObject DownloadObject)
{
if (SiteDb.WebSite.ContinueConvert)
{
if (DownloadObject is Page)
{
var page = DownloadObject as Page;
var continueconverters = SiteDb.ContinueConverter.All();
var converters = ConvertManager.GetConverters();
foreach (var item in continueconverters)
{
var converter = converters.Find(o => o.Type == item.ConvertType);
if (converter != null && converter is IContinueConverter)
{
IContinueConverter continueconverter = converter as IContinueConverter;
continueconverter.ContinueConvert(SiteDb, item.OriginalPageId, item.ConvertedTag, item.ObjectNameOrId, item.KoobooId, page, item.ElementPaths);
}
}
}
}
}
public ActionResult Search(string typeName, int id, int?pageIndex)
{
if (!pageIndex.HasValue)
{
return(RedirectToAction(() => Search(typeName, id, 1)));
}
var type = SiteObject.GetType(typeName);
if (type == null)
{
return(null);
}
var news = SiteObjectService.GetByRelationObject <News>(type, id)
.IsActive().OrderByDescending(n => n.PublishDate)
.ToPagedList(pageIndex.GetValueOrDefault(1) - 1, 5);
return(View(
new RelationNewsVM {
News = news,
SiteObject = SiteObjectService.GetBy(
LinqToSqlUtils.GetTableName(type), id)
}));
}
private void ProceedBakeSite()
{
Log.LogInformation("Executing pre-build tasks");
{
var prebuildCmds = BuildConfig.BeforeBuild;
if (prebuildCmds.Count > 0)
{
if (!SkipPreBuild)
{
Log.LogInformation("Executing commands scheduled to run before build");
foreach (var cmd in prebuildCmds)
{
Log.LogInformation(cmd);
var exitCode = ShellHelper.Run(cmd);
if (exitCode != 0)
{
Environment.Exit(exitCode);
}
}
}
else
{
Log.LogWarning("The pre-build tasks have been requested to be skipped.");
}
}
SiteObject.PreBuild();
}
Log.LogInformation("Building site content");
SiteObject.Build();
Log.LogInformation("Executing post-build tasks");
SiteObject.PostBuild();
}
public async Task Execute()
{
if (this.SiteDb == null || this.TransferTask == null)
{
return;
}
List <TransferPage> pagelist = SiteDb.TransferPages.TableScan.Where(o => o.taskid == TransferTask.Id && o.done == false).SelectAll();
DownloadManager manager = new DownloadManager()
{
SiteDb = SiteDb, UserId = this.TransferTask.UserId
};
List <TransferPage> transpages = new List <TransferPage>();
string baseurl = null;
bool defaultstart = true;
foreach (var item in pagelist)
{
if (baseurl == null)
{
baseurl = item.absoluteUrl;
}
var down = await DownloadHelper.DownloadUrlAsync(item.absoluteUrl, manager.CookieContainer, "GET", null, null);
if (down == null || string.IsNullOrEmpty(down.GetString()))
{
item.done = true;
SiteDb.TransferPages.AddOrUpdate(item);
continue;
}
Page page = null;
string downloadbody = down.GetString();
Guid sourcehash = Lib.Security.Hash.ComputeHashGuid(downloadbody);
item.HtmlSourceHash = sourcehash;
if (!string.IsNullOrEmpty(downloadbody))
{
var result = SiteDb.TransferPages.Query.Where(o => o.HtmlSourceHash == sourcehash).SelectAll();
if (result != null && result.Count > 0)
{
var transferpage = result[0];
TransferHelper.AddPageRoute(SiteDb, transferpage.PageId, item.absoluteUrl, baseurl);
item.done = true;
item.PageId = transferpage.PageId;
SiteDb.TransferPages.AddOrUpdate(item);
continue;
}
}
transpages.Add(item);
SiteObject downloadobject = TransferHelper.AddDownload(manager, down, item.absoluteUrl, defaultstart, true, baseurl);
if (downloadobject != null && downloadobject is Page)
{
page = downloadobject as Page;
}
if (page != null)
{
item.PageId = page.Id;
}
UpdateTransferPage(transpages, manager);
if (defaultstart)
{
defaultstart = false;
}
// DownloadOnePage(manager, item);
}
while (!manager.IsComplete)
{
System.Threading.Thread.Sleep(20);
}
this.SiteDb.TransferTasks.SetDone(this.TransferTask.Id);
}
public IEnumerable <ExtendDescription> FindAll(SiteObject site)
{
if (loaded)
{
return(cacheList);
}
if (RegistryUrl == null)
{
site.Error($"The registry Url for {nameof(DefaultExtendProvider)} cannot be null");
return(cacheList);
}
if (site.CanTrace())
{
site.Trace($"Checking remote registry [{RegistryUrl}] for available extensions/themes");
}
if (site == null)
{
throw new ArgumentNullException(nameof(site));
}
string themeRegistryStr;
try
{
using (HttpClient client = new HttpClient())
{
themeRegistryStr = client.GetStringAsync(RegistryUrl).Result;
}
}
catch (Exception ex)
{
site.Error($"Unable to load theme registry from Url [{RegistryUrl}]. Reason:{ex.GetReason()}");
return(cacheList);
}
var registryObject = new DynamicObject();
object result;
if (site.Scripts.TryImportScript(themeRegistryStr, Name, registryObject, ScriptFlags.None, out result))
{
var themeList = registryObject["extensions"] as ScriptArray;
if (themeList != null)
{
foreach (var theme in themeList)
{
var themeObject = theme as ScriptObject;
if (themeObject != null)
{
var themeName = (string)themeObject["name"];
var themeDescription = (string)themeObject["description"];
var themeUrl = (string)themeObject["url"];
var themeDirectory = (string)themeObject["directory"];
cacheList.Add(new ExtendDescription(themeName, themeDescription, themeUrl, themeDirectory));
}
}
}
}
loaded = true;
return(cacheList);
}
public static async Task <SiteObject> continueDownload(SiteDb siteDb, string RelativeUrl)
{
if (!siteDb.WebSite.ContinueDownload)
{
return(null);
}
string orgimport = null;
var history = siteDb.TransferTasks.History().ToList();
if (history.Count() == 0)
{
return(null);
}
else
{
orgimport = history.First();
}
/// track failed history...
Guid downloadid = RelativeUrl.ToHashGuid();
DownloadFailTrack failtrack = siteDb.DownloadFailedLog.Get(downloadid);
if (failtrack != null)
{
if (failtrack.HistoryTime.Where(o => o > DateTime.Now.AddMinutes(-30)).Any())
{
return(null);
}
if (failtrack.HistoryTime.Count() > 3)
{
return(null);
}
}
else
{
failtrack = new DownloadFailTrack();
failtrack.Id = downloadid;
}
var oktoDownload = await siteDb.TransferTasks.CanStartDownload(RelativeUrl);
if (!oktoDownload)
{
return(null);
}
string fullurl = string.Empty;
DownloadContent download = null;
if (RelativeUrl.EndsWith("favicon.ico"))
{
return(null);
}
string hostname = TransferHelper.GetPossibleHostName(RelativeUrl);
if (!string.IsNullOrEmpty(hostname))
{
var findurl = history.Find(o => o.ToLower().EndsWith(hostname.ToLower()));
if (!string.IsNullOrEmpty(findurl))
{
string newrelative = RelativeUrl.Replace(hostname + "/", "");
fullurl = UrlHelper.Combine(findurl, newrelative);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
}
else
{
string newrelative = RelativeUrl.Replace(hostname + "/", "");
// check whether it is https or not.
// fullurl = UrlHelper.Combine(hostname, newrelative);
var protocol = OrgProtocol(orgimport);
fullurl = protocol + hostname + newrelative;
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
}
}
if (download == null)
{
foreach (var item in history)
{
fullurl = UrlHelper.Combine(item, RelativeUrl);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
if (download != null)
{
break;
}
}
}
///// 301, 302, will be converted to 200 and return back as well. So it is safe to == 200.
if (download != null && download.StatusCode == 200)
{
DownloadManager downloadManager = new DownloadManager()
{
SiteDb = siteDb
};
SiteObject downloadobject = TransferHelper.AddDownload(downloadManager, download, fullurl, false, true, orgimport);
if (downloadobject is Page || downloadobject is View)
{
siteDb.TransferPages.AddOrUpdate(new TransferPage()
{
absoluteUrl = fullurl, PageId = downloadobject.Id
});
}
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
///for continue download content...
Continue.ContinueTask.Convert(siteDb, downloadobject);
return(downloadobject);
}
else
{
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
}
//download failed.
failtrack.HistoryTime.Add(DateTime.Now);
siteDb.DownloadFailedLog.AddOrUpdate(failtrack);
return(null);
}
private async Task Downloads(SiteDb siteDb, TransferProgress progress, DownloadManager manager)
{
List <TransferPage> transferingPages = new List <TransferPage>();
List <TransferPage> lowerPriorityPages = new List <TransferPage>();
var query = siteDb.TransferPages.Query.Where(o => o.taskid == progress.TaskId && o.done == false);
while (true)
{
List <TransferPage> pagelist = query.SelectAll();
pagelist.RemoveAll(o => DoneUrlHash.Contains(o.Id));
if (pagelist == null || pagelist.Count == 0)
{
if (progress.counter < progress.TotalPages && lowerPriorityPages.Count() > 0)
{
var needed = progress.TotalPages - progress.counter;
var neededpages = lowerPriorityPages.Take(needed);
foreach (var item in neededpages)
{
progress.counter += 1;
siteDb.TransferPages.AddOrUpdate(item);
lowerPriorityPages.Remove(item);
}
continue;
}
else
{
break;
}
}
foreach (var item in pagelist)
{
DoneUrlHash.Add(item.Id);
var down = await DownloadHelper.DownloadUrlAsync(item.absoluteUrl, manager.CookieContainer);
siteDb.TransferTasks.UpdateCookie(progress.TaskId, manager.CookieContainer);
if (down == null || string.IsNullOrEmpty(down.GetString()))
{
item.done = true;
siteDb.TransferPages.AddOrUpdate(item);
continue;
}
Page page = null;
string downloadbody = down.GetString();
Guid sourcehash = Lib.Security.Hash.ComputeHashGuid(downloadbody);
item.HtmlSourceHash = sourcehash;
if (!string.IsNullOrEmpty(downloadbody))
{
var result = SiteDb.TransferPages.Query.Where(o => o.HtmlSourceHash == sourcehash).SelectAll();
if (result != null && result.Count > 0)
{
var transferpage = result[0];
TransferHelper.AddPageRoute(SiteDb, transferpage.PageId, item.absoluteUrl, progress.BaseUrl);
item.done = true;
item.PageId = transferpage.PageId;
SiteDb.TransferPages.AddOrUpdate(item);
continue;
}
}
transferingPages.Add(item);
SiteObject downloadobject = TransferHelper.AddDownload(manager, down, item.absoluteUrl, item.DefaultStartPage, true, progress.BaseUrl);
if (downloadobject != null && downloadobject is Page)
{
page = downloadobject as Page;
}
if (page != null)
{
item.PageId = page.Id;
}
if (page == null || page.Dom == null)
{
item.done = true;
manager.SiteDb.TransferPages.AddOrUpdate(item);
continue;
}
if (progress.counter < progress.TotalPages && item.depth < progress.Levels)
{
page.Dom.URL = item.absoluteUrl;
var links = TransferHelper.GetAbsoluteLinks(page.Dom, page.Dom.getBaseUrl());
foreach (var linkitem in links)
{
if (progress.counter >= progress.TotalPages)
{
break;
}
if (!UrlHelper.isSameHost(linkitem, progress.BaseUrl))
{
continue;
}
if (!TransferHelper.IsPageUrl(linkitem))
{
continue;
}
TransferPage newpage = new TransferPage();
newpage.absoluteUrl = linkitem;
newpage.depth = item.depth + 1;
newpage.taskid = progress.TaskId;
if (!IsDuplicate(siteDb, newpage))
{
if (TransferHelper.IsLowerPrioUrl(linkitem))
{
if (lowerPriorityPages.Find(o => o.Id == newpage.Id) == null)
{
lowerPriorityPages.Add(newpage);
}
}
else
{
progress.counter += 1;
siteDb.TransferPages.AddOrUpdate(newpage);
}
}
}
}
UpdateTransferPage(transferingPages, manager);
}
}
while (transferingPages.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
UpdateTransferPage(transferingPages, manager);
}
}
public MenuPlugin(SiteObject site) : base(site)
{
Processor = new MenuProcessor(this);
Site.SetValue("menu", this, true);
Site.Content.BeforeLoadingProcessors.Insert(0, Processor);
}
public LayoutContentObject(SiteObject site, FileEntry sourceFileInfo, ScriptInstance scriptInstance) : base(site, ContentObjectType.File, sourceFileInfo, scriptInstance)
{
ScriptObjectLocal = new ScriptObject();
}
public SiteChangeToken(SiteObject site, string filter)
{
_site = site;
_filter = filter;
}
public LayoutPlugin(SiteObject site) : base(site)
{
Processor = new LayoutProcessor(this);
site.Content.ContentProcessors.Insert(0, Processor);
}
public static bool GetLiveReload(this SiteObject site)
{
return(site.GetSafeValue <bool>(LiveReload));
}
public TemplateLoaderFromIncludes(SiteObject site)
{
this.site = site;
IncludeFiles = new HashSet <FileEntry>();
}
public SiteFileProvider(SiteObject site)
{
_site = site;
}
public static void SetLiveReload(this SiteObject site, bool value)
{
site[LiveReload] = value;
}
public MarkdownPlugin(SiteObject site) : base(site)
{
site.Content.BeforeLoadingProcessors.Add(new MarkdownProcessor(this));
}
public TemplateLoaderUnauthorized(SiteObject site)
{
this.site = site;
}
public MarkdownPlugin(SiteObject site) : base(site)
{
var processor = new MarkdownProcessor(this);
site.Content.ContentProcessors.Add(processor);
}
public ScssPlugin(SiteObject site) : base(site)
{
site.SetValue("scss", new ScssObject(this), true);
site.Content.ContentProcessors.Add(new ScssProcessor(this));
}
public static async Task <SiteObject> continueDownload(SiteDb siteDb, string RelativeUrl)
{
if (!siteDb.WebSite.ContinueDownload)
{
return(null);
}
var oktoDownload = await siteDb.TransferTasks.CanStartDownload(RelativeUrl);
if (!oktoDownload)
{
return(null);
}
var history = siteDb.TransferTasks.History().ToList();
if (history.Count() == 0)
{
return(null);
}
/// track failed history...
Guid downloadid = RelativeUrl.ToHashGuid();
DownloadFailTrack failtrack = siteDb.DownloadFailedLog.Get(downloadid);
if (failtrack != null)
{
if (failtrack.HistoryTime.Where(o => o > DateTime.Now.AddMinutes(-30)).Any())
{
return(null);
}
}
else
{
failtrack = new DownloadFailTrack();
failtrack.Id = downloadid;
}
string fullurl = string.Empty;
DownloadContent download = null;
string hostname = TransferHelper.GetPossibleHostName(RelativeUrl);
if (!string.IsNullOrEmpty(hostname))
{
var findurl = history.Find(o => o.ToLower().EndsWith(hostname.ToLower()));
if (!string.IsNullOrEmpty(findurl))
{
string newrelative = RelativeUrl.Replace(hostname + "/", "");
fullurl = UrlHelper.Combine(findurl, newrelative);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
}
}
if (download == null)
{
foreach (var item in history)
{
fullurl = UrlHelper.Combine(item, RelativeUrl);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
if (download != null)
{
break;
}
}
}
///// 301, 302, will be converted to 200 and return back as well. So it is safe to == 200.
if (download != null && download.StatusCode == 200)
{
DownloadManager downloadManager = new DownloadManager()
{
SiteDb = siteDb
};
SiteObject downloadobject = TransferHelper.AddDownload(downloadManager, download, fullurl, false, true, fullurl);
if (downloadobject is Page || downloadobject is View)
{
siteDb.TransferPages.AddOrUpdate(new TransferPage()
{
absoluteUrl = fullurl, PageId = downloadobject.Id
});
}
/// for continue download content...
Continue.ContinueTask.Convert(siteDb, downloadobject);
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
return(downloadobject);
}
else
{
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
}
//download failed.
failtrack.HistoryTime.Add(DateTime.Now);
siteDb.DownloadFailedLog.AddOrUpdate(failtrack);
if (failtrack.HistoryTime.Count() > 5)
{
var filetype = Kooboo.Lib.Helper.UrlHelper.GetFileType(RelativeUrl);
byte consttype;
switch (filetype)
{
case UrlHelper.UrlFileType.Image:
consttype = ConstObjectType.Image;
break;
case UrlHelper.UrlFileType.JavaScript:
consttype = ConstObjectType.Script;
break;
case UrlHelper.UrlFileType.Style:
consttype = ConstObjectType.Style;
break;
case UrlHelper.UrlFileType.File:
consttype = ConstObjectType.CmsFile;
break;
case UrlHelper.UrlFileType.PageOrView:
consttype = ConstObjectType.Page;
break;
default:
consttype = 0;
break;
}
// siteDb.Routes.EnsureExists(RelativeUrl, consttype, default(Guid));
}
return(null);
}
