public void CheckLabelAndAddPage(List <Page> pages, string url, string label, string dashboardID = null)
{
if (!DBHelpers.HasLabel(url, label))
{
List <string> docNumber = GetDocNumberByUrl(url);
if (docNumber != null && docNumber.Count > 0)
{
int pageNumber = 1;
while (true)
{
string filePath = GetFilePath(Code, dashboardID, docNumber[0], pageNumber + 1);
if (!File.Exists(filePath) || !DBHelpers.HasPage(url))
{
break;
}
else
{
pageNumber++;
}
}
pages.Add(new Page()
{
DashboardURL = url,
DashboardID = dashboardID,
URL = GetUrlByDocNumber(docNumber[0], pageNumber, dashboardID),
DocNumber = docNumber[0],
PageNumber = pageNumber
});
DBHelpers.SaveLabel(this.Code, url, label);
}
}
}
public void Run(object data)
{
while (true)
{
//this.LastErrorMessage = null;
List <Page> pages = new List <Page>();
try
{
//dashboards
List <Page> dashboards = GetDashboards();
//WriteLog("Download dashboards", dashboards.Count.ToString());
foreach (Page page in dashboards)
{
this.CurrentPage = page;
this.Progress = "Dashboard: " + (dashboards.IndexOf(page) + 1).ToString() + " of " + dashboards.Count;
try
{
this.DownloadWebPage(page);
}
catch (Exception ex)
{
this.LastErrorMessage = ex.Message;
Thread.Sleep(this.PageDelay);
continue;
}
this.TotalSize += page.HtmlContent.Length;
this.AmountPages++;
pages.AddRange(this.OnDashboardLoaded(page));
Thread.Sleep(this.PageDelay);
}
//WriteLog("Process pages", pages.Count.ToString());
}
catch (Exception ex)
{
if (this.CurrentPage != null)
{
//clear labels
for (int i = 0; i < pages.Count; i++)
{
//lock (Labels)
//{
// Labels[pages[i].DashboardURL] = string.Empty;
//}
DBHelpers.SaveLabel(this.Code, pages[i].DashboardURL, string.Empty);
}
this.LastErrorMessage = string.Format("Failed page: {0};{1}",
this.GetUrlByDocNumber(this.CurrentPage.DocNumber,
this.CurrentPage.PageNumber,
this.CurrentPage.DashboardID),
ex.Message);
//MessageBox.Show(ex.Message + this.CurrentPage.ToString() + ex.StackTrace);
}
else
{
this.LastErrorMessage = ex.Message;
//MessageBox.Show(ex.Message + ex.StackTrace);
}
this.WaitNextUpdate(true);
continue;
}
try
{
//pages
for (int i = 0; i < pages.Count; i++)
{
Page page = pages[i];
this.CurrentPage = page;
this.Progress = "Article: " + (i + 1).ToString() + " of " + pages.Count;
this.ProcessPage(pages[0], page);
this.TotalSize += page.HtmlContent.Length;
this.AmountPages++;
while (page.NeedLoadNextPage)
{
page.PageNumber++;
if (page.PageNumber > 5000)
{
break;
}
page.URL = GetUrlByDocNumber(page.DocNumber, page.PageNumber, page.DashboardID);
this.CurrentPage = page;
this.ProcessPage(pages[0], page);
if (page.URL != page.RedirectURL) //avoid cycles
{
break;
}
this.TotalSize += page.HtmlContent.Length;
this.AmountPages++;
Thread.Sleep(this.PageDelay);
}
Thread.Sleep(this.PageDelay);
}
//WriteLog("Site " + Code + " sleep to ", DateTime.Now.Add(this.SiteDelay).ToString());
//this.SaveLabels();
this.WaitNextUpdate(false);
}
catch (Exception ex)
{
//WriteLog("Exception in site " + Code, ex.Message + ex.StackTrace);
//WriteLog("Site " + Code + " sleep to ", DateTime.Now.Add(this.SiteDelay).ToString());
if (this.CurrentPage != null)
{
//clear labels
for (int i = pages.IndexOf(this.CurrentPage); i < pages.Count; i++)
{
//lock (Labels)
//{
// Labels[pages[i].DashboardURL] = string.Empty;
//}
DBHelpers.SaveLabel(this.Code, pages[i].DashboardURL, string.Empty);
}
this.LastErrorMessage = string.Format("Failed page: {0};{1}",
this.GetUrlByDocNumber(this.CurrentPage.DocNumber,
this.CurrentPage.PageNumber,
this.CurrentPage.DashboardID),
ex.Message);
//MessageBox.Show(ex.Message + this.CurrentPage.ToString() + ex.StackTrace);
}
else
{
this.LastErrorMessage = ex.Message;
//MessageBox.Show(ex.Message + ex.StackTrace);
}
this.WaitNextUpdate(true);
}
}
}