public static string SyncToDisk(SiteDb SiteDb, ISiteObject Value, ChangeType ChangeType, string StoreName)
{
var manager = GetSyncManager(SiteDb.WebSite.Id);
return(manager.SyncToDisk(SiteDb, Value, ChangeType, StoreName));
}
public static void ComputeUrlRelation(SiteDb sitedb, Guid objectId, byte constType, List <string> urllist, byte DestConstType)
{
List <Guid> internalRoutes = new List <Guid>();
List <Guid> ExternalResource = new List <Guid>();
var FinalDestConstType = DestConstType;
var oldRouteRelations = sitedb.Relations.GetRelationViaRoutes(objectId, DestConstType);
var oldExternalResourceRelations = sitedb.Relations.GetExternalRelations(objectId, DestConstType);
foreach (var item in urllist.Distinct())
{
if (DestConstType == ConstObjectType.Link)
{
FinalDestConstType = Kooboo.Sites.Service.ConstTypeService.GetConstTypeByUrl(item);
}
if (Service.DomUrlService.IsExternalLink(item))
{
Guid externalid = Kooboo.Data.IDGenerator.Generate(item, ConstObjectType.ExternalResource);
ExternalResource.Add(externalid);
if (oldExternalResourceRelations.Find(o => o.objectYId == externalid) == null)
{
sitedb.ExternalResource.AddOrUpdate(item, FinalDestConstType);
}
}
else
{
var routeids = GetRouteIds(sitedb, item);
internalRoutes.AddRange(routeids);
if (routeids.Count == 1 && oldRouteRelations.Find(o => o.objectYId == routeids[0]) == null)
{
sitedb.Routes.EnsureExists(item, FinalDestConstType);
}
}
}
foreach (var item in oldRouteRelations)
{
if (!internalRoutes.Contains(item.objectYId))
{
sitedb.Relations.Delete(item.Id);
}
}
foreach (var item in internalRoutes)
{
if (oldRouteRelations.Find(o => o.objectYId == item) == null)
{
sitedb.Relations.AddOrUpdate(objectId, item, constType, ConstObjectType.Route, DestConstType);
}
}
foreach (var item in oldExternalResourceRelations)
{
if (!ExternalResource.Contains(item.objectYId))
{
sitedb.Relations.Delete(item.Id);
}
}
foreach (var item in ExternalResource)
{
if (oldExternalResourceRelations.Find(o => o.objectYId == item) == null)
{
sitedb.Relations.AddOrUpdate(objectId, item, constType, ConstObjectType.ExternalResource, DestConstType);
}
}
}
public SiteClusterManager(SiteDb sitedb)
{
this.SiteDb = sitedb;
}
/// <summary>
/// Compute the embedded image that use base64 encoded image.
/// </summary>
/// <param name="page"></param>
/// <param name="sitedb"></param>
public static void ComputeEmbededImage(Document doc, SiteDb sitedb, Guid objectId, byte constType)
{
return;
}
public static async Task <SiteObject> continueDownload(SiteDb siteDb, string RelativeUrl, RenderContext Context)
{
if (!siteDb.WebSite.ContinueDownload)
{
return(null);
}
string orgimport = null;
var history = siteDb.TransferTasks.History().ToList();
if (history.Count() == 0)
{
return(null);
}
else
{
orgimport = history.First();
}
/// track failed history...
Guid downloadid = RelativeUrl.ToHashGuid();
DownloadFailTrack failtrack = siteDb.DownloadFailedLog.Get(downloadid);
if (failtrack != null)
{
if (failtrack.HistoryTime.Where(o => o > DateTime.Now.AddMinutes(-30)).Any())
{
return(null);
}
if (failtrack.HistoryTime.Count() > 3)
{
return(null);
}
}
else
{
failtrack = new DownloadFailTrack();
failtrack.Id = downloadid;
}
var oktoDownload = await siteDb.TransferTasks.CanStartDownload(RelativeUrl);
if (!oktoDownload)
{
return(null);
}
string fullurl = string.Empty;
DownloadContent download = null;
if (RelativeUrl.EndsWith("favicon.ico"))
{
return(null);
}
string hostname = TransferHelper.GetPossibleHostName(RelativeUrl);
if (!string.IsNullOrEmpty(hostname))
{
var findurl = history.Find(o => o.ToLower().EndsWith(hostname.ToLower()));
if (!string.IsNullOrEmpty(findurl))
{
string newrelative = RelativeUrl.Replace(hostname + "/", "");
fullurl = UrlHelper.Combine(findurl, newrelative);
}
else
{
string newrelative = RelativeUrl.Replace(hostname + "/", "");
var protocol = OrgProtocol(orgimport);
fullurl = protocol + hostname + newrelative;
}
download = await DownloadUrl(siteDb, fullurl, Context);
}
if (download == null)
{
foreach (var item in history)
{
fullurl = UrlHelper.Combine(item, RelativeUrl);
download = await DownloadUrl(siteDb, fullurl, Context);
if (download != null)
{
break;
}
}
}
///// 301, 302, will be converted to 200 and return back as well. So it is safe to == 200.
if (download != null && download.StatusCode == 200)
{
DownloadManager downloadManager = new DownloadManager()
{
SiteDb = siteDb
};
SiteObject downloadobject = TransferHelper.AddDownload(downloadManager, download, fullurl, false, true, orgimport);
if (downloadobject is Page || downloadobject is View)
{
siteDb.TransferPages.AddOrUpdate(new TransferPage()
{
absoluteUrl = fullurl, PageId = downloadobject.Id
});
}
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
///for continue download content...
Continue.ContinueTask.Convert(siteDb, downloadobject);
return(downloadobject);
}
else
{
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
}
//download failed.
failtrack.HistoryTime.Add(DateTime.Now);
siteDb.DownloadFailedLog.AddOrUpdate(failtrack);
return(null);
}
private async Task Downloads(SiteDb siteDb, TransferProgress progress, DownloadManager manager)
{
List <TransferPage> transferingPages = new List <TransferPage>();
List <TransferPage> lowerPriorityPages = new List <TransferPage>();
var query = siteDb.TransferPages.Query.Where(o => o.taskid == progress.TaskId && o.done == false);
while (true)
{
List <TransferPage> pagelist = query.SelectAll();
pagelist.RemoveAll(o => DoneUrlHash.Contains(o.Id));
if (pagelist == null || pagelist.Count == 0)
{
if (progress.counter < progress.TotalPages && lowerPriorityPages.Count() > 0)
{
var needed = progress.TotalPages - progress.counter;
var neededpages = lowerPriorityPages.Take(needed);
foreach (var item in neededpages)
{
progress.counter += 1;
siteDb.TransferPages.AddOrUpdate(item);
lowerPriorityPages.Remove(item);
}
continue;
}
else
{
break;
}
}
foreach (var item in pagelist)
{
DoneUrlHash.Add(item.Id);
var down = await DownloadHelper.DownloadUrlAsync(item.absoluteUrl, manager.CookieContainer);
siteDb.TransferTasks.UpdateCookie(progress.TaskId, manager.CookieContainer);
if (down == null || string.IsNullOrEmpty(down.GetString()))
{
item.done = true;
siteDb.TransferPages.AddOrUpdate(item);
continue;
}
Page page = null;
string downloadbody = down.GetString();
Guid sourcehash = Lib.Security.Hash.ComputeHashGuid(downloadbody);
item.HtmlSourceHash = sourcehash;
if (!string.IsNullOrEmpty(downloadbody))
{
var result = SiteDb.TransferPages.Query.Where(o => o.HtmlSourceHash == sourcehash).SelectAll();
if (result != null && result.Count > 0)
{
var transferpage = result[0];
TransferHelper.AddPageRoute(SiteDb, transferpage.PageId, item.absoluteUrl, progress.BaseUrl);
item.done = true;
item.PageId = transferpage.PageId;
SiteDb.TransferPages.AddOrUpdate(item);
continue;
}
}
transferingPages.Add(item);
SiteObject downloadobject = TransferHelper.AddDownload(manager, down, item.absoluteUrl, item.DefaultStartPage, true, progress.BaseUrl);
if (downloadobject != null && downloadobject is Page)
{
page = downloadobject as Page;
}
if (page != null)
{
item.PageId = page.Id;
}
if (page == null || page.Dom == null)
{
item.done = true;
manager.SiteDb.TransferPages.AddOrUpdate(item);
continue;
}
if (progress.counter < progress.TotalPages && item.depth < progress.Levels)
{
page.Dom.URL = item.absoluteUrl;
var links = TransferHelper.GetAbsoluteLinks(page.Dom, page.Dom.getBaseUrl());
foreach (var linkitem in links)
{
if (progress.counter >= progress.TotalPages)
{
break;
}
if (!UrlHelper.isSameHost(linkitem, progress.BaseUrl))
{
continue;
}
if (!TransferHelper.IsPageUrl(linkitem))
{
continue;
}
TransferPage newpage = new TransferPage();
newpage.absoluteUrl = linkitem;
newpage.depth = item.depth + 1;
newpage.taskid = progress.TaskId;
if (!IsDuplicate(siteDb, newpage))
{
if (TransferHelper.IsLowerPrioUrl(linkitem))
{
if (lowerPriorityPages.Find(o => o.Id == newpage.Id) == null)
{
lowerPriorityPages.Add(newpage);
}
}
else
{
progress.counter += 1;
siteDb.TransferPages.AddOrUpdate(newpage);
}
}
}
}
UpdateTransferPage(transferingPages, manager);
}
}
while (transferingPages.Count() > 0)
{
System.Threading.Thread.Sleep(300);
UpdateTransferPage(transferingPages, manager);
}
}
public static ObjectInfo GetObjectInfo(SiteDb SiteDb, ISiteObject siteobject)
{
if (siteobject == null)
{
return(null);
}
ObjectInfo info = new ObjectInfo();
info.ObjectId = siteobject.Id;
info.ConstType = siteobject.ConstType;
info.ModelType = ConstTypeService.GetModelType(siteobject.ConstType);
if (siteobject is IBinaryFile)
{
info.Size = ((IBinaryFile)siteobject).Size;
}
else if (siteobject is ITextObject)
{
info.Size = (((ITextObject)siteobject).Body ?? "").Length;
}
if (Kooboo.Lib.Reflection.TypeHelper.HasInterface(info.ModelType, typeof(IEmbeddable)))
{
var embeddable = siteobject as IEmbeddable;
if (embeddable.IsEmbedded)
{
return(GetObjectInfo(SiteDb, embeddable.OwnerObjectId, embeddable.OwnerConstType));
}
}
if (Attributes.AttributeHelper.IsRoutable(siteobject))
{
info.Url = GetObjectRelativeUrl(SiteDb, siteobject as SiteObject);
info.DisplayName = info.Url;
info.Name = siteobject.Name;
return(info);
}
else
{
if (info.ModelType == typeof(CmsCssRule))
{
var rule = siteobject as CmsCssRule;
if (rule == null)
{
return(null);
}
if (rule.IsInline)
{
return(GetObjectInfo(SiteDb, rule.OwnerObjectId, rule.OwnerObjectConstType));
}
else
{
return(GetObjectInfo(SiteDb, rule.ParentStyleId, ConstObjectType.Style));
}
}
info.Url = "/__kb/" + info.ModelType.Name + "/" + info.ObjectId.ToString();
info.DisplayName = siteobject.Name;
info.Name = siteobject.Name;
if (info.ModelType == typeof(TextContent))
{
info.Name = Kooboo.Sites.Helper.ContentHelper.GetSummary(siteobject as TextContent, SiteDb.WebSite.DefaultCulture);
info.DisplayName = info.Name;
}
if (info.ModelType == typeof(DataMethodSetting))
{
var datamethod = siteobject as DataMethodSetting;
if (datamethod != null)
{
info.Name = datamethod.OriginalMethodName;
info.DisplayName = datamethod.OriginalMethodName;
}
}
return(info);
}
}
public static string GetObjectRelativeUrl(SiteDb SiteDb, Guid ObjectId, byte ConstType)
{
var siteobject = GetSiteObject(SiteDb, ObjectId, ConstType);
return(GetObjectRelativeUrl(SiteDb, siteobject as SiteObject));
}