public static async Task <SiteObject> continueDownload(SiteDb siteDb, string RelativeUrl) { if (!siteDb.WebSite.ContinueDownload) { return(null); } var oktoDownload = await siteDb.TransferTasks.CanStartDownload(RelativeUrl); if (!oktoDownload) { return(null); } var history = siteDb.TransferTasks.History().ToList(); if (history.Count() == 0) { return(null); } /// track failed history... Guid downloadid = RelativeUrl.ToHashGuid(); DownloadFailTrack failtrack = siteDb.DownloadFailedLog.Get(downloadid); if (failtrack != null) { if (failtrack.HistoryTime.Where(o => o > DateTime.Now.AddMinutes(-30)).Any()) { return(null); } } else { failtrack = new DownloadFailTrack(); failtrack.Id = downloadid; } string fullurl = string.Empty; DownloadContent download = null; string hostname = TransferHelper.GetPossibleHostName(RelativeUrl); if (!string.IsNullOrEmpty(hostname)) { var findurl = history.Find(o => o.ToLower().EndsWith(hostname.ToLower())); if (!string.IsNullOrEmpty(findurl)) { string newrelative = RelativeUrl.Replace(hostname + "/", ""); fullurl = UrlHelper.Combine(findurl, newrelative); var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl); download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer); } } if (download == null) { foreach (var item in history) { fullurl = UrlHelper.Combine(item, RelativeUrl); var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl); download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer); if (download != null) { break; } } } ///// 301, 302, will be converted to 200 and return back as well. So it is safe to == 200. if (download != null && download.StatusCode == 200) { DownloadManager downloadManager = new DownloadManager() { SiteDb = siteDb }; SiteObject downloadobject = TransferHelper.AddDownload(downloadManager, download, fullurl, false, true, fullurl); if (downloadobject is Page || downloadobject is View) { siteDb.TransferPages.AddOrUpdate(new TransferPage() { absoluteUrl = fullurl, PageId = downloadobject.Id }); } /// for continue download content... Continue.ContinueTask.Convert(siteDb, downloadobject); siteDb.TransferTasks.ReleaseDownload(RelativeUrl); return(downloadobject); } else { siteDb.TransferTasks.ReleaseDownload(RelativeUrl); } //download failed. failtrack.HistoryTime.Add(DateTime.Now); siteDb.DownloadFailedLog.AddOrUpdate(failtrack); if (failtrack.HistoryTime.Count() > 5) { var filetype = Kooboo.Lib.Helper.UrlHelper.GetFileType(RelativeUrl); byte consttype; switch (filetype) { case UrlHelper.UrlFileType.Image: consttype = ConstObjectType.Image; break; case UrlHelper.UrlFileType.JavaScript: consttype = ConstObjectType.Script; break; case UrlHelper.UrlFileType.Style: consttype = ConstObjectType.Style; break; case UrlHelper.UrlFileType.File: consttype = ConstObjectType.CmsFile; break; case UrlHelper.UrlFileType.PageOrView: consttype = ConstObjectType.Page; break; default: consttype = 0; break; } // siteDb.Routes.EnsureExists(RelativeUrl, consttype, default(Guid)); } return(null); }
public static async Task <SiteObject> continueDownload(SiteDb siteDb, string RelativeUrl) { if (!siteDb.WebSite.ContinueDownload) { return(null); } string orgimport = null; var history = siteDb.TransferTasks.History().ToList(); if (history.Count() == 0) { return(null); } else { orgimport = history.First(); } /// track failed history... Guid downloadid = RelativeUrl.ToHashGuid(); DownloadFailTrack failtrack = siteDb.DownloadFailedLog.Get(downloadid); if (failtrack != null) { if (failtrack.HistoryTime.Where(o => o > DateTime.Now.AddMinutes(-30)).Any()) { return(null); } if (failtrack.HistoryTime.Count() > 3) { return(null); } } else { failtrack = new DownloadFailTrack(); failtrack.Id = downloadid; } var oktoDownload = await siteDb.TransferTasks.CanStartDownload(RelativeUrl); if (!oktoDownload) { return(null); } string fullurl = string.Empty; DownloadContent download = null; if (RelativeUrl.EndsWith("favicon.ico")) { return(null); } string hostname = TransferHelper.GetPossibleHostName(RelativeUrl); if (!string.IsNullOrEmpty(hostname)) { var findurl = history.Find(o => o.ToLower().EndsWith(hostname.ToLower())); if (!string.IsNullOrEmpty(findurl)) { string newrelative = RelativeUrl.Replace(hostname + "/", ""); fullurl = UrlHelper.Combine(findurl, newrelative); var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl); download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer); } else { string newrelative = RelativeUrl.Replace(hostname + "/", ""); // check whether it is https or not. // fullurl = UrlHelper.Combine(hostname, newrelative); var protocol = OrgProtocol(orgimport); fullurl = protocol + hostname + newrelative; var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl); download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer); } } if (download == null) { foreach (var item in history) { fullurl = UrlHelper.Combine(item, RelativeUrl); var cookiecontianer = siteDb.TransferTasks.GetCookieContainerByFullUrl(fullurl); download = await DownloadHelper.DownloadUrlAsync(fullurl, cookiecontianer); if (download != null) { break; } } } ///// 301, 302, will be converted to 200 and return back as well. So it is safe to == 200. if (download != null && download.StatusCode == 200) { DownloadManager downloadManager = new DownloadManager() { SiteDb = siteDb }; SiteObject downloadobject = TransferHelper.AddDownload(downloadManager, download, fullurl, false, true, orgimport); if (downloadobject is Page || downloadobject is View) { siteDb.TransferPages.AddOrUpdate(new TransferPage() { absoluteUrl = fullurl, PageId = downloadobject.Id }); } siteDb.TransferTasks.ReleaseDownload(RelativeUrl); ///for continue download content... Continue.ContinueTask.Convert(siteDb, downloadobject); return(downloadobject); } else { siteDb.TransferTasks.ReleaseDownload(RelativeUrl); } //download failed. failtrack.HistoryTime.Add(DateTime.Now); siteDb.DownloadFailedLog.AddOrUpdate(failtrack); return(null); }