private void ProcessExternal(AnalyzerContext Context)
{
HTMLCollection styletags = Context.Dom.getElementsByTagName("link");
foreach (var item in styletags.item)
{
if (item.hasAttribute("rel") && item.getAttribute("rel").ToLower().Contains("stylesheet"))
{
string itemurl = Service.DomUrlService.GetLinkOrSrc(item);
if (!string.IsNullOrEmpty(itemurl))
{
itemurl = TransferHelper.TrimQuestionMark(itemurl);
string absoluteUrl = UrlHelper.Combine(Context.AbsoluteUrl, itemurl);
bool issamehost = UrlHelper.isSameHost(Context.OriginalImportUrl, absoluteUrl);
string relativeurl = UrlHelper.RelativePath(absoluteUrl, issamehost);
if (itemurl != relativeurl)
{
string oldstring = Kooboo.Sites.Service.DomService.GetOpenTag(item);
string newstring = oldstring.Replace(itemurl, relativeurl);
Context.Changes.Add(new AnalyzerUpdate()
{
StartIndex = item.location.openTokenStartIndex,
EndIndex = item.location.openTokenEndIndex,
NewValue = newstring
});
}
Context.DownloadManager.AddTask(new Download.DownloadTask()
{
AbsoluteUrl = absoluteUrl,
RelativeUrl = relativeurl,
ConstType = ConstObjectType.Style,
OwnerObjectId = Context.ObjectId
});
}
}
}
}
public void Execute(AnalyzerContext Context)
{
int embeddedItemIndex = 0;
HTMLCollection scripts = Context.Dom.getElementsByTagName("script");
foreach (var item in scripts.item)
{
if (item.hasAttribute("src"))
{
string srcurl = Service.DomUrlService.GetLinkOrSrc(item);
if (string.IsNullOrEmpty(srcurl))
{
/// script tag with a src source. does not consider as a script.
continue;
}
string fullurl = UrlHelper.Combine(Context.AbsoluteUrl, srcurl);
bool issamehost = Kooboo.Lib.Helper.UrlHelper.isSameHost(Context.OriginalImportUrl, fullurl);
if (issamehost)
{
string relativeurl = UrlHelper.RelativePath(fullurl, issamehost);
relativeurl = TransferHelper.TrimQuestionMark(relativeurl);
if (srcurl != relativeurl)
{
string oldstring = Kooboo.Sites.Service.DomService.GetOpenTag(item);
string newstring = oldstring.Replace(srcurl, relativeurl);
Context.Changes.Add(new AnalyzerUpdate()
{
StartIndex = item.location.openTokenStartIndex,
EndIndex = item.location.openTokenEndIndex,
NewValue = newstring
});
}
Context.DownloadManager.AddTask(new Download.DownloadTask()
{
AbsoluteUrl = fullurl,
RelativeUrl = relativeurl,
ConstType = ConstObjectType.Script,
OwnerObjectId = Context.ObjectId
});
}
}
else
{
//string text = item.InnerHtml;
//if (!string.IsNullOrEmpty(text))
//{
// // this is an embedded script.
// var script = new Script
// {
// IsEmbedded = true,
// Body = text,
// OwnerObjectId = Context.ObjectId,
// OwnerConstType = Context.ObjectType,
// ItemIndex = embeddedItemIndex,
// Name = UrlHelper.FileName(Context.AbsoluteUrl)
// };
// embeddedItemIndex += 1;
// Context.SiteDb.Scripts.AddOrUpdate(script);
//}
}
}
}
private List <JsTreeDataItem> GetPageLinks(Dictionary <string, string> urlContentDictionary, string rootUrl, List <JsTreeDataItem> list, int level, int maxLevel, int maxPages, Action <List <JsTreeDataItem> > callback)
{
level++;
var content = DownloadHelper.DownloadUrl(rootUrl);
if (content != null)
{
var html = content.ContentString;
if (urlContentDictionary.ContainsKey(rootUrl))
{
return(list);
}
urlContentDictionary[rootUrl] = html;
var rootItem = new JsTreeDataItem
{
Id = rootUrl.ToHashGuid().ToString(),
Parent = level == 1 ? "#" : rootUrl.ToHashGuid().ToString(),
Text = rootUrl
};
if (level == 1)
{
rootItem.TreeState = new JsTreeState
{
Selected = true,
Disabled = true
};
}
list.Add(rootItem);
callback(list);
var links = TransferHelper.GetLinkElements(html);
var rootUri = new Uri(rootUrl);
var rootHost = rootUri.Host;
IEnumerable <string> urls = links.Where(link => link.attributes != null &&
link.attributes.Any(attr => attr.name.Equals("href", StringComparison.OrdinalIgnoreCase) &&
!attr.value.StartsWith("javascript:", StringComparison.OrdinalIgnoreCase) &&
!attr.value.StartsWith("#", StringComparison.OrdinalIgnoreCase)
)).Select(link =>
{
var firstOrDefault = link.attributes.FirstOrDefault(it => it.name.Equals("href", StringComparison.OrdinalIgnoreCase));
if (firstOrDefault == null)
{
return(null);
}
var currentUri = new Uri(firstOrDefault.value, UriKind.RelativeOrAbsolute);
if (currentUri.IsAbsoluteUri)
{
if (!currentUri.Host.Equals(rootHost, StringComparison.OrdinalIgnoreCase))
{
return(null);
}
}
var absoluteUri = new Uri(new Uri(rootUrl), firstOrDefault.value);
return(absoluteUri.AbsoluteUri);
}).Where(url => !String.IsNullOrEmpty(url))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
foreach (var url in urls)
{
if (CancelToken.IsCancellationRequested)
{
break;
}
if (!urlContentDictionary.ContainsKey(url))
{
var subContent = DownloadHelper.DownloadUrl(rootUrl);
if (subContent != null)
{
urlContentDictionary[url] = subContent.ContentString;
list.Add(new JsTreeDataItem
{
Id = url.ToHashGuid().ToString(),
Parent = rootUrl.ToHashGuid().ToString(),
Text = url
});
callback(list);
if (list.Count >= maxPages)
{
break;
}
if (list.Count < maxPages && level < maxLevel)
{
GetPageLinks(urlContentDictionary, url, list, level, maxLevel, maxPages, callback);
}
}
else
{
urlContentDictionary[url] = String.Empty;
}
}
}
}
return(list);
}
public static async Task <SiteObject> continueDownload(SiteDb siteDb, string RelativeUrl)
{
if (!siteDb.WebSite.ContinueDownload)
{
return(null);
}
var oktoDownload = await siteDb.TransferTasks.CanStartDownload(RelativeUrl);
if (!oktoDownload)
{
return(null);
}
var history = siteDb.TransferTasks.History().ToList();
if (history.Count() == 0)
{
return(null);
}
/// track failed history...
Guid downloadid = RelativeUrl.ToHashGuid();
DownloadFailTrack failtrack = siteDb.DownloadFailedLog.Get(downloadid);
if (failtrack != null)
{
if (failtrack.HistoryTime.Where(o => o > DateTime.Now.AddMinutes(-30)).Any())
{
return(null);
}
}
else
{
failtrack = new DownloadFailTrack();
failtrack.Id = downloadid;
}
string fullurl = string.Empty;
DownloadContent download = null;
string hostname = TransferHelper.GetPossibleHostName(RelativeUrl);
if (!string.IsNullOrEmpty(hostname))
{
var findurl = history.Find(o => o.ToLower().EndsWith(hostname.ToLower()));
if (!string.IsNullOrEmpty(findurl))
{
string newrelative = RelativeUrl.Replace(hostname + "/", "");
fullurl = UrlHelper.Combine(findurl, newrelative);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
}
}
if (download == null)
{
foreach (var item in history)
{
fullurl = UrlHelper.Combine(item, RelativeUrl);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
if (download != null)
{
break;
}
}
}
///// 301, 302, will be converted to 200 and return back as well. So it is safe to == 200.
if (download != null && download.StatusCode == 200)
{
DownloadManager downloadManager = new DownloadManager()
{
SiteDb = siteDb
};
SiteObject downloadobject = TransferHelper.AddDownload(downloadManager, download, fullurl, false, true, fullurl);
if (downloadobject is Page || downloadobject is View)
{
siteDb.TransferPages.AddOrUpdate(new TransferPage()
{
absoluteUrl = fullurl, PageId = downloadobject.Id
});
}
/// for continue download content...
Continue.ContinueTask.Convert(siteDb, downloadobject);
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
return(downloadobject);
}
else
{
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
}
//download failed.
failtrack.HistoryTime.Add(DateTime.Now);
siteDb.DownloadFailedLog.AddOrUpdate(failtrack);
if (failtrack.HistoryTime.Count() > 5)
{
var filetype = Kooboo.Lib.Helper.UrlHelper.GetFileType(RelativeUrl);
byte consttype;
switch (filetype)
{
case UrlHelper.UrlFileType.Image:
consttype = ConstObjectType.Image;
break;
case UrlHelper.UrlFileType.JavaScript:
consttype = ConstObjectType.Script;
break;
case UrlHelper.UrlFileType.Style:
consttype = ConstObjectType.Style;
break;
case UrlHelper.UrlFileType.File:
consttype = ConstObjectType.CmsFile;
break;
case UrlHelper.UrlFileType.PageOrView:
consttype = ConstObjectType.Page;
break;
default:
consttype = 0;
break;
}
// siteDb.Routes.EnsureExists(RelativeUrl, consttype, default(Guid));
}
return(null);
}
public void Execute(AnalyzerContext Context)
{
HTMLCollection scripts = Context.Dom.getElementsByTagName("script");
foreach (var item in scripts.item)
{
if (item.hasAttribute("src"))
{
string srcurl = Service.DomUrlService.GetLinkOrSrc(item);
if (string.IsNullOrEmpty(srcurl))
{
/// script tag with a src source. does not consider as a script.
continue;
}
string fullurl = UrlHelper.Combine(Context.AbsoluteUrl, srcurl);
bool issamehost = Kooboo.Lib.Helper.UrlHelper.isSameHost(Context.OriginalImportUrl, fullurl);
if (issamehost)
{
string relativeurl = UrlHelper.RelativePath(fullurl, issamehost);
relativeurl = TransferHelper.TrimQuestionMark(relativeurl);
if (srcurl != relativeurl)
{
string oldstring = Kooboo.Sites.Service.DomService.GetOpenTag(item);
string newstring = oldstring.Replace(srcurl, relativeurl);
Context.Changes.Add(new AnalyzerUpdate()
{
StartIndex = item.location.openTokenStartIndex,
EndIndex = item.location.openTokenEndIndex,
NewValue = newstring
});
}
Context.DownloadManager.AddTask(new Download.DownloadTask()
{
AbsoluteUrl = fullurl,
RelativeUrl = relativeurl,
ConstType = ConstObjectType.Script,
OwnerObjectId = Context.ObjectId
});
}
}
else
{
///<script>if (document.location.protocol != "https:") {document.location = document.URL.replace(/^http:/i, "https:");}</script>
string text = item.InnerHtml;
if (!string.IsNullOrWhiteSpace(text) && text.Length < 200)
{
var lower = text.ToLower();
var hasProtocolOperator = lower.Contains("document.location.protocol") && lower.Contains("document.url.replace") && lower.Contains("https");
var hasLocationOperator = Regex.IsMatch(lower, "window.top.location\\s*=");
if (hasProtocolOperator || hasLocationOperator)
{
Context.Changes.Add(new AnalyzerUpdate()
{
StartIndex = item.location.openTokenStartIndex,
EndIndex = item.location.endTokenEndIndex,
NewValue = "<script>/* https redirect removed */</script>"
});
}
}
//string text = item.InnerHtml;
//if (!string.IsNullOrEmpty(text))
//{
// // this is an embedded script.
// var script = new Script
// {
// IsEmbedded = true,
// Body = text,
// OwnerObjectId = Context.ObjectId,
// OwnerConstType = Context.ObjectType,
// ItemIndex = embeddedItemIndex,
// Name = UrlHelper.FileName(Context.AbsoluteUrl)
// };
// embeddedItemIndex += 1;
// Context.SiteDb.Scripts.AddOrUpdate(script);
//}
}
}
}
public static async Task <SiteObject> continueDownload(SiteDb siteDb, string RelativeUrl)
{
if (!siteDb.WebSite.ContinueDownload)
{
return(null);
}
string orgimport = null;
var history = siteDb.TransferTasks.History().ToList();
if (history.Count() == 0)
{
return(null);
}
else
{
orgimport = history.First();
}
/// track failed history...
Guid downloadid = RelativeUrl.ToHashGuid();
DownloadFailTrack failtrack = siteDb.DownloadFailedLog.Get(downloadid);
if (failtrack != null)
{
if (failtrack.HistoryTime.Where(o => o > DateTime.Now.AddMinutes(-30)).Any())
{
return(null);
}
if (failtrack.HistoryTime.Count() > 3)
{
return(null);
}
}
else
{
failtrack = new DownloadFailTrack();
failtrack.Id = downloadid;
}
var oktoDownload = await siteDb.TransferTasks.CanStartDownload(RelativeUrl);
if (!oktoDownload)
{
return(null);
}
string fullurl = string.Empty;
DownloadContent download = null;
if (RelativeUrl.EndsWith("favicon.ico"))
{
return(null);
}
string hostname = TransferHelper.GetPossibleHostName(RelativeUrl);
if (!string.IsNullOrEmpty(hostname))
{
var findurl = history.Find(o => o.ToLower().EndsWith(hostname.ToLower()));
if (!string.IsNullOrEmpty(findurl))
{
string newrelative = RelativeUrl.Replace(hostname + "/", "");
fullurl = UrlHelper.Combine(findurl, newrelative);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
}
else
{
string newrelative = RelativeUrl.Replace(hostname + "/", "");
// check whether it is https or not.
// fullurl = UrlHelper.Combine(hostname, newrelative);
var protocol = OrgProtocol(orgimport);
fullurl = protocol + hostname + newrelative;
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
}
}
if (download == null)
{
foreach (var item in history)
{
fullurl = UrlHelper.Combine(item, RelativeUrl);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
if (download != null)
{
break;
}
}
}
///// 301, 302, will be converted to 200 and return back as well. So it is safe to == 200.
if (download != null && download.StatusCode == 200)
{
DownloadManager downloadManager = new DownloadManager()
{
SiteDb = siteDb
};
SiteObject downloadobject = TransferHelper.AddDownload(downloadManager, download, fullurl, false, true, orgimport);
if (downloadobject is Page || downloadobject is View)
{
siteDb.TransferPages.AddOrUpdate(new TransferPage()
{
absoluteUrl = fullurl, PageId = downloadobject.Id
});
}
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
///for continue download content...
Continue.ContinueTask.Convert(siteDb, downloadobject);
return(downloadobject);
}
else
{
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
}
//download failed.
failtrack.HistoryTime.Add(DateTime.Now);
siteDb.DownloadFailedLog.AddOrUpdate(failtrack);
return(null);
}