public static async Task <SiteObject> continueDownload(SiteDb siteDb, string RelativeUrl)
{
if (!siteDb.WebSite.ContinueDownload)
{
return(null);
}
var oktoDownload = await siteDb.TransferTasks.CanStartDownload(RelativeUrl);
if (!oktoDownload)
{
return(null);
}
var history = siteDb.TransferTasks.History().ToList();
if (history.Count() == 0)
{
return(null);
}
/// track failed history...
Guid downloadid = RelativeUrl.ToHashGuid();
DownloadFailTrack failtrack = siteDb.DownloadFailedLog.Get(downloadid);
if (failtrack != null)
{
if (failtrack.HistoryTime.Where(o => o > DateTime.Now.AddMinutes(-30)).Any())
{
return(null);
}
}
else
{
failtrack = new DownloadFailTrack();
failtrack.Id = downloadid;
}
string fullurl = string.Empty;
DownloadContent download = null;
string hostname = TransferHelper.GetPossibleHostName(RelativeUrl);
if (!string.IsNullOrEmpty(hostname))
{
var findurl = history.Find(o => o.ToLower().EndsWith(hostname.ToLower()));
if (!string.IsNullOrEmpty(findurl))
{
string newrelative = RelativeUrl.Replace(hostname + "/", "");
fullurl = UrlHelper.Combine(findurl, newrelative);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
}
}
if (download == null)
{
foreach (var item in history)
{
fullurl = UrlHelper.Combine(item, RelativeUrl);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
if (download != null)
{
break;
}
}
}
///// 301, 302, will be converted to 200 and return back as well. So it is safe to == 200.
if (download != null && download.StatusCode == 200)
{
DownloadManager downloadManager = new DownloadManager()
{
SiteDb = siteDb
};
SiteObject downloadobject = TransferHelper.AddDownload(downloadManager, download, fullurl, false, true, fullurl);
if (downloadobject is Page || downloadobject is View)
{
siteDb.TransferPages.AddOrUpdate(new TransferPage()
{
absoluteUrl = fullurl, PageId = downloadobject.Id
});
}
/// for continue download content...
Continue.ContinueTask.Convert(siteDb, downloadobject);
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
return(downloadobject);
}
else
{
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
}
//download failed.
failtrack.HistoryTime.Add(DateTime.Now);
siteDb.DownloadFailedLog.AddOrUpdate(failtrack);
if (failtrack.HistoryTime.Count() > 5)
{
var filetype = Kooboo.Lib.Helper.UrlHelper.GetFileType(RelativeUrl);
byte consttype;
switch (filetype)
{
case UrlHelper.UrlFileType.Image:
consttype = ConstObjectType.Image;
break;
case UrlHelper.UrlFileType.JavaScript:
consttype = ConstObjectType.Script;
break;
case UrlHelper.UrlFileType.Style:
consttype = ConstObjectType.Style;
break;
case UrlHelper.UrlFileType.File:
consttype = ConstObjectType.CmsFile;
break;
case UrlHelper.UrlFileType.PageOrView:
consttype = ConstObjectType.Page;
break;
default:
consttype = 0;
break;
}
// siteDb.Routes.EnsureExists(RelativeUrl, consttype, default(Guid));
}
return(null);
}
public static async Task <SiteObject> continueDownload(SiteDb siteDb, string RelativeUrl)
{
if (!siteDb.WebSite.ContinueDownload)
{
return(null);
}
string orgimport = null;
var history = siteDb.TransferTasks.History().ToList();
if (history.Count() == 0)
{
return(null);
}
else
{
orgimport = history.First();
}
/// track failed history...
Guid downloadid = RelativeUrl.ToHashGuid();
DownloadFailTrack failtrack = siteDb.DownloadFailedLog.Get(downloadid);
if (failtrack != null)
{
if (failtrack.HistoryTime.Where(o => o > DateTime.Now.AddMinutes(-30)).Any())
{
return(null);
}
if (failtrack.HistoryTime.Count() > 3)
{
return(null);
}
}
else
{
failtrack = new DownloadFailTrack();
failtrack.Id = downloadid;
}
var oktoDownload = await siteDb.TransferTasks.CanStartDownload(RelativeUrl);
if (!oktoDownload)
{
return(null);
}
string fullurl = string.Empty;
DownloadContent download = null;
if (RelativeUrl.EndsWith("favicon.ico"))
{
return(null);
}
string hostname = TransferHelper.GetPossibleHostName(RelativeUrl);
if (!string.IsNullOrEmpty(hostname))
{
var findurl = history.Find(o => o.ToLower().EndsWith(hostname.ToLower()));
if (!string.IsNullOrEmpty(findurl))
{
string newrelative = RelativeUrl.Replace(hostname + "/", "");
fullurl = UrlHelper.Combine(findurl, newrelative);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
}
else
{
string newrelative = RelativeUrl.Replace(hostname + "/", "");
// check whether it is https or not.
// fullurl = UrlHelper.Combine(hostname, newrelative);
var protocol = OrgProtocol(orgimport);
fullurl = protocol + hostname + newrelative;
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
}
}
if (download == null)
{
foreach (var item in history)
{
fullurl = UrlHelper.Combine(item, RelativeUrl);
var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl);
download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer);
if (download != null)
{
break;
}
}
}
///// 301, 302, will be converted to 200 and return back as well. So it is safe to == 200.
if (download != null && download.StatusCode == 200)
{
DownloadManager downloadManager = new DownloadManager()
{
SiteDb = siteDb
};
SiteObject downloadobject = TransferHelper.AddDownload(downloadManager, download, fullurl, false, true, orgimport);
if (downloadobject is Page || downloadobject is View)
{
siteDb.TransferPages.AddOrUpdate(new TransferPage()
{
absoluteUrl = fullurl, PageId = downloadobject.Id
});
}
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
///for continue download content...
Continue.ContinueTask.Convert(siteDb, downloadobject);
return(downloadobject);
}
else
{
siteDb.TransferTasks.ReleaseDownload(RelativeUrl);
}
//download failed.
failtrack.HistoryTime.Add(DateTime.Now);
siteDb.DownloadFailedLog.AddOrUpdate(failtrack);
return(null);
}
