/**************************************************************************/
public async void Execute()
{
int MaxFetches = MacroscopePreferencesManager.GetMaxFetchesPerWorker();
while (MaxFetches > 0)
{
if (this.JobMaster.GetThreadsStop())
{
this.DebugMsg(string.Format("JobMaster.GetThreadsStop: {0}", this.JobMaster.GetThreadsStop()));
break;
}
else
{
MacroscopeJobItem JobItem = this.JobMaster.GetUrlQueueItem();
string Url = null;
string RedirectedFromUrl = null;
if (JobItem != null)
{
Url = JobItem.GetItemUrl();
RedirectedFromUrl = JobItem.GetItemRedirectedFromUrl();
}
if (!string.IsNullOrEmpty(Url))
{
if (!this.CheckIncludeExcludeUrl(Url))
{
Url = null;
}
}
if (!string.IsNullOrEmpty(Url))
{
if (
!MacroscopePreferencesManager.GetCrawlParentDirectories() &&
!MacroscopePreferencesManager.GetCrawlChildDirectories() &&
Url != this.JobMaster.GetStartUrl())
{
Url = null;
}
else if (
!MacroscopePreferencesManager.GetCrawlParentDirectories() ||
!MacroscopePreferencesManager.GetCrawlChildDirectories())
{
this.DebugMsg(string.Format("Running Parent/Child Check: {0}", Url));
if (
MacroscopePreferencesManager.GetCrawlParentDirectories() &&
(!string.IsNullOrEmpty(Url)))
{
if (!MacroscopeHttpUrlUtils.IsWithinParentDirectory(StartUrl: this.JobMaster.GetParentStartingDirectory(), Url: Url))
{
Url = null;
}
}
if (
MacroscopePreferencesManager.GetCrawlChildDirectories() &&
(!string.IsNullOrEmpty(Url)))
{
if (!MacroscopeHttpUrlUtils.IsWithinChildDirectory(StartUrl: this.JobMaster.GetChildStartingDirectory(), Url: Url))
{
Url = null;
}
}
}
else
{
this.DebugMsg(string.Format("Skipping Parent/Child Check: {0}", Url));
}
}
if (!string.IsNullOrEmpty(Url))
{
if (MacroscopePreferencesManager.GetDepth() >= 0)
{
if (MacroscopeHttpUrlUtils.FindUrlDepth(Url: Url) > MacroscopePreferencesManager.GetDepth())
{
this.DebugMsg(string.Format("URL Too Deep: {0}", Url));
Url = null;
}
}
}
if (!string.IsNullOrEmpty(Url))
{
this.DebugMsg(string.Format("Execute: {0}", Url));
int Tries = MacroscopePreferencesManager.GetMaxRetries();
JobHistory.AddHistoryItem(Url: Url);
do
{
this.DebugMsg(string.Format("Trying Fetch: {0} :: {1}", Tries, Url));
MacroscopeConstants.FetchStatus FetchStatus = MacroscopeConstants.FetchStatus.VOID;
try
{
if (!string.IsNullOrEmpty(RedirectedFromUrl))
{
FetchStatus = await this.Fetch(Url, RedirectedFromUrl);
}
else
{
FetchStatus = await this.Fetch(Url);
}
}
catch (Exception ex)
{
this.DebugMsg(string.Format("FetchStatus: {0}", ex.Message));
this.DebugMsg(string.Format("Url: {0}", Url));
this.DebugMsg(string.Format("FetchStatus: {0}", FetchStatus));
}
switch (FetchStatus)
{
case MacroscopeConstants.FetchStatus.ERROR:
this.DebugMsg(string.Format("Fetch Failed: {0} :: {1}", Tries, Url));
Thread.Sleep(25);
break;
case MacroscopeConstants.FetchStatus.NETWORK_ERROR:
this.DebugMsg(string.Format("Fetch Failed: {0} :: {1}", Tries, Url));
Thread.Sleep(25);
break;
default:
this.JobMaster.NotifyWorkersFetched(Url: Url);
Tries = 0;
break;
}
Tries--;
} while(Tries > 0);
if (this.CrawlDelay > 0)
{
this.DebugMsg(string.Format("CRAWL DELAY: Sleeping for {0} seconds...", this.CrawlDelay));
Thread.Sleep(CrawlDelay * 1000);
}
}
}
MaxFetches--;
//Thread.Yield();
}
this.JobMaster.NotifyWorkersDone();
}
/**************************************************************************/
public void Execute()
{
int MaxFetches = MacroscopePreferencesManager.GetMaxFetchesPerWorker();
while (MaxFetches > 0)
{
if (this.JobMaster.GetThreadsStop())
{
DebugMsg(string.Format("JobMaster.GetThreadsStop: {0}", this.JobMaster.GetThreadsStop()));
break;
}
else
{
MacroscopeJobItem JobItem = this.JobMaster.GetUrlQueueItem();
string Url = null;
if (JobItem != null)
{
Url = JobItem.GetItemUrl();
}
if (!string.IsNullOrEmpty(Url))
{
if (!this.CheckIncludeExcludeUrl(Url))
{
Url = null;
}
}
if (!string.IsNullOrEmpty(Url))
{
if (
!MacroscopePreferencesManager.GetCrawlParentDirectories() &&
!MacroscopePreferencesManager.GetCrawlChildDirectories() &&
Url != this.JobMaster.GetStartUrl())
{
Url = null;
}
else if (
!MacroscopePreferencesManager.GetCrawlParentDirectories() ||
!MacroscopePreferencesManager.GetCrawlChildDirectories())
{
DebugMsg(string.Format("Running Parent/Child Check: {0}", Url));
if (
MacroscopePreferencesManager.GetCrawlParentDirectories() &&
(!string.IsNullOrEmpty(Url)))
{
if (!this.JobMaster.IsWithinParentDirectory(Url))
{
Url = null;
}
}
if (
MacroscopePreferencesManager.GetCrawlChildDirectories() &&
(!string.IsNullOrEmpty(Url)))
{
if (!this.JobMaster.IsWithinChildDirectory(Url))
{
Url = null;
}
}
}
else
{
DebugMsg(string.Format("Skipping Parent/Child Check: {0}", Url));
}
}
if (!string.IsNullOrEmpty(Url))
{
DebugMsg(string.Format("Execute: {0}", Url));
int Tries = MacroscopePreferencesManager.GetMaxRetries();
do
{
DebugMsg(string.Format("Trying Fetch: {0} :: {1}", Tries, Url));
if (
(this.Fetch(Url) == MacroscopeConstants.FetchStatus.ERROR) ||
(this.Fetch(Url) == MacroscopeConstants.FetchStatus.ERROR))
{
DebugMsg(string.Format("Fetch Failed: {0} :: {1}", Tries, Url));
Thread.Sleep(1000);
}
else
{
this.JobMaster.NotifyWorkersFetched(Url);
break;
}
Tries--;
} while(Tries > 0);
if (this.CrawlDelay > 0)
{
DebugMsg(string.Format("CRAWL DELAY: Sleeping for {0} seconds...", this.CrawlDelay));
Thread.Sleep(CrawlDelay * 1000);
}
}
}
MaxFetches--;
Thread.Yield();
}
this.JobMaster.NotifyWorkersDone();
}
