public TransferViewModel(TransferPage view)
{
_view = view;
ApplicationContext.UpdateSourceElementViewModelsSelected = new List <UpdateSourceElementViewModel>();
ViewList = new List <UpdateSource>();
TabSelectedCommand = new RelayCommand <Button>(OnMenuSelected, CanMenuAction);
BuidPage();
}
/// <summary>
/// Check to make sure that this is not a duplicate page.
/// </summary>
/// <param name="newpage"></param>
/// <returns></returns>
private bool IsDuplicate(SiteDb sitedb, TransferPage newpage)
{
var oldpage = sitedb.TransferPages.Get(newpage.Id);
if (oldpage != null)
{
return(true);
}
return(false);
}
public async Task Execute()
{
if (TransferTask == null || string.IsNullOrEmpty(TransferTask.FullStartUrl) || SiteDb == null)
{
return;
}
string relativeurl = UrlHelper.RelativePath(TransferTask.FullStartUrl, true);
TransferProgress progress = new TransferProgress();
progress.TaskId = TransferTask.Id;
progress.counter = 0;
progress.SetDomain(TransferTask.FullStartUrl);
progress.FullStartUrl = TransferTask.FullStartUrl;
progress.TotalPages = TransferTask.Totalpages;
progress.Levels = TransferTask.Levels;
TransferPage page = new TransferPage();
page.absoluteUrl = TransferTask.FullStartUrl;
page.depth = 0;
page.taskid = TransferTask.Id;
page.DefaultStartPage = true;
page.done = false;
var oldpage = SiteDb.TransferPages.Get(page.Id);
if (oldpage == null)
{
SiteDb.TransferPages.AddOrUpdate(page);
progress.counter += 1;
}
DownloadManager manager = new DownloadManager()
{
SiteDb = SiteDb, OriginalImportUrl = TransferTask.FullStartUrl, UserId = this.TransferTask.UserId
};
await Downloads(SiteDb, progress, manager);
while (!manager.IsComplete)
{
System.Threading.Thread.Sleep(500);
}
SiteDb.TransferTasks.SetDone(TransferTask.Id);
}
public async Task Execute()
{
if (this.SiteDb == null || this.TransferTask == null || string.IsNullOrEmpty(TransferTask.FullStartUrl))
{
return;
}
var cookieContainer = new System.Net.CookieContainer();
TransferPage transpage = new TransferPage();
transpage.absoluteUrl = TransferTask.FullStartUrl;
transpage.taskid = TransferTask.Id;
var oldpage = SiteDb.TransferPages.Get(transpage.Id);
if (oldpage == null || oldpage.PageId == default(Guid))
{
var down = await DownloadHelper.DownloadUrlAsync(transpage.absoluteUrl, cookieContainer);
if (down == null)
{
return;
}
string htmlsource = down.GetString();
var savepage = new Page();
string name = TransferTask.RelativeName;
if (!string.IsNullOrWhiteSpace(name))
{
if (name.EndsWith("\\"))
{
name = name.TrimEnd('\\');
}
if (name.EndsWith("/"))
{
name = name.TrimEnd('/');
}
name = System.IO.Path.GetFileNameWithoutExtension(name);
}
if (!string.IsNullOrWhiteSpace(name))
{
savepage.Name = name;
}
htmlsource = UrlHelper.ReplaceMetaCharSet(htmlsource);
string FirstImportUrl = SiteDb.TransferTasks.FirstImportHost();
if (string.IsNullOrEmpty(FirstImportUrl))
{
FirstImportUrl = TransferTask.FullStartUrl;
}
DownloadManager manager = new DownloadManager()
{
SiteDb = SiteDb
};
manager.OriginalImportUrl = FirstImportUrl;
var context = AnalyzerManager.Execute(htmlsource, transpage.absoluteUrl, savepage.Id, savepage.ConstType, manager, FirstImportUrl);
htmlsource = context.HtmlSource;
savepage.Body = htmlsource;
string PageRelativeName = TransferTask.RelativeName;
if (string.IsNullOrWhiteSpace(PageRelativeName))
{
bool issamehost = Kooboo.Lib.Helper.UrlHelper.isSameHost(TransferTask.FullStartUrl, FirstImportUrl);
PageRelativeName = UrlHelper.RelativePath(TransferTask.FullStartUrl, issamehost);
}
if (!PageRelativeName.StartsWith("/"))
{
PageRelativeName = "/" + PageRelativeName;
}
SiteDb.Routes.AddOrUpdate(PageRelativeName, savepage.ConstType, savepage.Id, this.TransferTask.UserId);
SiteDb.Pages.AddOrUpdate(savepage, this.TransferTask.UserId);
while (!manager.IsComplete)
{
System.Threading.Thread.Sleep(15);
}
transpage.done = true;
transpage.PageId = savepage.Id;
SiteDb.TransferPages.AddOrUpdate(transpage);
}
this.SiteDb.TransferTasks.SetDone(this.TransferTask.Id);
}
private async Task Downloads(SiteDb siteDb, TransferProgress progress, DownloadManager manager)
{
List <TransferPage> transferingPages = new List <TransferPage>();
List <TransferPage> lowerPriorityPages = new List <TransferPage>();
var query = siteDb.TransferPages.Query.Where(o => o.taskid == progress.TaskId && o.done == false);
while (true)
{
List <TransferPage> pagelist = query.SelectAll();
pagelist.RemoveAll(o => DoneUrlHash.Contains(o.Id));
if (pagelist == null || pagelist.Count == 0)
{
if (progress.counter < progress.TotalPages && lowerPriorityPages.Count() > 0)
{
var needed = progress.TotalPages - progress.counter;
var neededpages = lowerPriorityPages.Take(needed);
foreach (var item in neededpages)
{
progress.counter += 1;
siteDb.TransferPages.AddOrUpdate(item);
lowerPriorityPages.Remove(item);
}
continue;
}
else
{
break;
}
}
foreach (var item in pagelist)
{
DoneUrlHash.Add(item.Id);
var down = await DownloadHelper.DownloadUrlAsync(item.absoluteUrl, manager.CookieContainer);
siteDb.TransferTasks.UpdateCookie(progress.TaskId, manager.CookieContainer);
if (down == null || string.IsNullOrEmpty(down.GetString()))
{
item.done = true;
siteDb.TransferPages.AddOrUpdate(item);
continue;
}
Page page = null;
string downloadbody = down.GetString();
Guid sourcehash = Lib.Security.Hash.ComputeHashGuid(downloadbody);
item.HtmlSourceHash = sourcehash;
if (!string.IsNullOrEmpty(downloadbody))
{
var result = SiteDb.TransferPages.Query.Where(o => o.HtmlSourceHash == sourcehash).SelectAll();
if (result != null && result.Count > 0)
{
var transferpage = result[0];
TransferHelper.AddPageRoute(SiteDb, transferpage.PageId, item.absoluteUrl, progress.BaseUrl);
item.done = true;
item.PageId = transferpage.PageId;
SiteDb.TransferPages.AddOrUpdate(item);
continue;
}
}
transferingPages.Add(item);
SiteObject downloadobject = TransferHelper.AddDownload(manager, down, item.absoluteUrl, item.DefaultStartPage, true, progress.BaseUrl);
if (downloadobject != null && downloadobject is Page)
{
page = downloadobject as Page;
}
if (page != null)
{
item.PageId = page.Id;
}
if (page == null || page.Dom == null)
{
item.done = true;
manager.SiteDb.TransferPages.AddOrUpdate(item);
continue;
}
if (progress.counter < progress.TotalPages && item.depth < progress.Levels)
{
page.Dom.URL = item.absoluteUrl;
var links = TransferHelper.GetAbsoluteLinks(page.Dom, page.Dom.getBaseUrl());
foreach (var linkitem in links)
{
if (progress.counter >= progress.TotalPages)
{
break;
}
if (!UrlHelper.isSameHost(linkitem, progress.BaseUrl))
{
continue;
}
if (!TransferHelper.IsPageUrl(linkitem))
{
continue;
}
TransferPage newpage = new TransferPage();
newpage.absoluteUrl = linkitem;
newpage.depth = item.depth + 1;
newpage.taskid = progress.TaskId;
if (!IsDuplicate(siteDb, newpage))
{
if (TransferHelper.IsLowerPrioUrl(linkitem))
{
if (lowerPriorityPages.Find(o => o.Id == newpage.Id) == null)
{
lowerPriorityPages.Add(newpage);
}
}
else
{
progress.counter += 1;
siteDb.TransferPages.AddOrUpdate(newpage);
}
}
}
}
UpdateTransferPage(transferingPages, manager);
}
}
while (transferingPages.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
UpdateTransferPage(transferingPages, manager);
}
}
