public void TestGenerateKey()
{
MacroscopeCredentialsHttp Credentials = new MacroscopeCredentialsHttp();
const string Domain = "www.companyname.com";
const string Realm = "Realm of Chaos";
string Digest = Credentials.TestGenerateKey(Domain, Realm);
DebugMsg(string.Format("Digest: {0}", Digest));
Assert.IsNotEmpty(Digest, string.Format("FAIL: {0} :: {1}", Domain, Realm));
}
/**************************************************************************/
public MacroscopeCredential(
MacroscopeCredentialsHttp CredentialsHttp,
string Domain,
string Realm,
string Username,
string Password
)
{
this.SuppressDebugMsg = true;
this.CredentialsHttp = CredentialsHttp;
this.Domain = Domain;
this.Realm = Realm;
this.Username = Username;
this.Password = Password;
}
public MacroscopeCredentialsHttp IGetCredentialsHttp()
{
MacroscopeCredentialsHttp CredentialsHttp = new MacroscopeCredentialsHttp();
return(CredentialsHttp);
}
/**************************************************************************/
private async Task <MacroscopeConstants.FetchStatus> Fetch(string Url, string RedirectedFromUrl = null)
{
MacroscopeDocument msDoc = null;
MacroscopeConstants.FetchStatus FetchStatus = MacroscopeConstants.FetchStatus.VOID;
bool BlockedByRobotsRule;
if (MacroscopePreferencesManager.GetPageLimit() > -1)
{
int PagesFound = this.JobMaster.GetPagesFound();
int PageLimit = MacroscopePreferencesManager.GetPageLimit();
if (PagesFound >= PageLimit)
{
this.DebugMsg(string.Format("PAGE LIMIT REACHED: {0} :: {1}", PageLimit, PagesFound));
return(FetchStatus);
}
}
if (this.DocCollection.ContainsDocument(Url: Url))
{
msDoc = this.DocCollection.GetDocumentByUrl(Url: Url);
if (msDoc.GetAuthenticationRealm() != null)
{
if (msDoc.GetAuthenticationType() == MacroscopeConstants.AuthenticationType.BASIC)
{
MacroscopeCredential Credential;
Credential = this.JobMaster.GetCredentialsHttp().GetCredential(
msDoc.GetHostAndPort(),
msDoc.GetAuthenticationRealm()
);
if (Credential != null)
{
msDoc = this.DocCollection.CreateDocument(
Credential: Credential,
Url: Url
);
}
}
}
}
else
{
msDoc = this.DocCollection.CreateDocument(Url: Url);
}
if (!string.IsNullOrEmpty(RedirectedFromUrl))
{
msDoc.SetUrlRedirectFrom(Url: RedirectedFromUrl);
}
msDoc.SetFetchStatus(MacroscopeConstants.FetchStatus.OK);
if (!MacroscopeDnsTools.CheckValidHostname(Url: Url))
{
this.DebugMsg(string.Format("Fetch :: CheckValidHostname: {0}", "NOT OK"));
msDoc.SetStatusCode(HttpStatusCode.BadGateway);
FetchStatus = MacroscopeConstants.FetchStatus.NETWORK_ERROR;
msDoc.SetFetchStatus(FetchStatus);
}
if (await this.JobMaster.GetRobots().CheckRobotRule(Url: Url))
{
msDoc.SetAllowedByRobots(true);
}
else
{
msDoc.SetAllowedByRobots(false);
}
BlockedByRobotsRule = await this.JobMaster.GetRobots().ApplyRobotRule(Url: Url);
if (!BlockedByRobotsRule)
{
this.DebugMsg(string.Format("Disallowed by robots.txt: {0}", Url));
this.JobMaster.AddToBlockedByRobots(Url);
FetchStatus = MacroscopeConstants.FetchStatus.ROBOTS_DISALLOWED;
msDoc.SetFetchStatus(FetchStatus);
JobHistory.VisitedHistoryItem(Url: msDoc.GetUrl());
}
else
{
this.JobMaster.RemoveFromBlockedByRobots(Url);
}
if (this.AllowedHosts.IsExternalUrl(Url: Url))
{
this.DebugMsg(string.Format("IsExternalUrl: {0}", Url));
msDoc.SetIsExternal(State: true);
}
if (this.DocCollection.ContainsDocument(Url: Url))
{
if (!this.DocCollection.GetDocumentByUrl(Url: Url).GetIsDirty())
{
FetchStatus = MacroscopeConstants.FetchStatus.ALREADY_SEEN;
return(FetchStatus);
}
}
if (MacroscopePreferencesManager.GetDepth() >= 0)
{
int Depth = MacroscopeHttpUrlUtils.FindUrlDepth(Url: Url);
if (Depth > MacroscopePreferencesManager.GetDepth())
{
this.DebugMsg(string.Format("URL Too Deep: {0}", Depth));
FetchStatus = MacroscopeConstants.FetchStatus.SKIPPED;
return(FetchStatus);
}
}
/** ------------------------------------------------------------------ **/
if (!await msDoc.Execute())
{
this.DebugMsg(string.Format("EXECUTE FAILED: {0}", Url));
FetchStatus = MacroscopeConstants.FetchStatus.ERROR;
}
/** ------------------------------------------------------------------ **/
/** ------------------------------------------------------------------ **/
{
if (msDoc.GetStatusCode() == HttpStatusCode.Unauthorized)
{
if (msDoc.GetAuthenticationType() == MacroscopeConstants.AuthenticationType.BASIC)
{
MacroscopeCredentialsHttp CredentialsHttp = this.JobMaster.GetCredentialsHttp();
CredentialsHttp.EnqueueCredentialRequest(
Domain: msDoc.GetHostAndPort(),
Realm: msDoc.GetAuthenticationRealm(),
Url: msDoc.GetUrl()
);
this.JobMaster.AddUrlQueueItem(Url: msDoc.GetUrl());
}
}
if (msDoc.GetIsRedirect())
{
this.DebugMsg(string.Format("REDIRECTION DETECTED GetUrl: {0}", msDoc.GetUrl()));
this.DebugMsg(string.Format("REDIRECTION DETECTED From: {0}", msDoc.GetUrlRedirectFrom()));
if (MacroscopePreferencesManager.GetCheckRedirects())
{
string Hostname = msDoc.GetHostAndPort();
string HostnameFrom = MacroscopeAllowedHosts.ParseHostnameFromUrl(msDoc.GetUrlRedirectFrom());
string UrlRedirectTo = msDoc.GetUrlRedirectTo();
string HostnameTo = MacroscopeAllowedHosts.ParseHostnameFromUrl(UrlRedirectTo);
this.DebugMsg(string.Format("REDIRECTION DETECTED UrlRedirectTo: {0}", UrlRedirectTo));
this.DebugMsg(string.Format("REDIRECTION DETECTED HostnameTo: {0}", HostnameTo));
if (MacroscopePreferencesManager.GetFollowRedirects())
{
if (MacroscopePreferencesManager.GetCheckExternalLinks())
{
this.AllowedHosts.AddFromUrl(Url: UrlRedirectTo);
}
else
{
if (this.AllowedHosts.IsInternalUrl(Url: UrlRedirectTo))
{
this.AllowedHosts.AddFromUrl(Url: UrlRedirectTo);
}
}
}
}
this.JobMaster.AddUrlQueueItem(Url: msDoc.GetUrlRedirectTo());
}
else
{
this.ProcessHrefLangLanguages(msDoc); // Process Languages from HrefLang
this.JobMaster.ProcessOutlinks(msDoc: msDoc); // Process Outlinks from document
}
FetchStatus = MacroscopeConstants.FetchStatus.SUCCESS;
}
/** ------------------------------------------------------------------ **/
if (DocCollection.ContainsDocument(msDoc: msDoc))
{
JobHistory.VisitedHistoryItem(Url: Url);
}
else
{
this.DebugMsg(string.Format("OOPS: {0}", Url));
}
/** ------------------------------------------------------------------ **/
return(FetchStatus);
}
/**************************************************************************/
private void InitializeJobMaster(MacroscopeConstants.RunTimeMode JobRunTimeMode)
{
GC.Collect();
/*
* {
* this.JobMasterLog = new EventLog ();
* this.JobMasterLog.Source = MacroscopeConstants.MainEventLogSourceName;
* this.JobGuid = Guid.NewGuid();
* this.LogEntry( string.Format( "Starting Job" ) );
* }
*/
this.RunTimeMode = JobRunTimeMode;
if (this.TaskController != null)
{
this.CredentialsHttp = this.TaskController.IGetCredentialsHttp();
}
this.DocCollection = new MacroscopeDocumentCollection(JobMaster: this);
this.AllowedHosts = new MacroscopeAllowedHosts();
/** BEGIN: Named Queues *************************************************/
this.NamedQueueJobItems = new MacroscopeNamedQueue <MacroscopeJobItem> ();
this.NamedQueueJobItems.CreateNamedQueue(
Name: MacroscopeConstants.NamedQueueUrlList,
QueueMode: MacroscopeNamedQueue <MacroscopeJobItem> .MODE.USE_HISTORY
);
this.NamedQueue = new MacroscopeNamedQueue <string> ();
{
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayQueue);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayStructure);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayStructureLinkCounts);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayHierarchy);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayCanonicalAnalysis);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayHrefLang);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayErrors);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayHostnames);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayRedirectsAudit);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayLinks);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayHyperlinks);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayUriAnalysis);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayPageTitles);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayPageDescriptions);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayPageKeywords);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayPageHeadings);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayPageText);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayStylesheets);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayImages);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayJavascripts);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayAudios);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayVideos);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplaySitemaps);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayEmailAddresses);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayTelephoneNumbers);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayCustomFilters);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayDataExtractorsCssSelectors);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayDataExtractorsRegexes);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayDataExtractorsXpaths);
this.NamedQueue.CreateNamedQueue(Name: MacroscopeConstants.NamedQueueDisplayRemarks);
}
/** END: Named Queues ***************************************************/
this.CrawlDelay = 0;
this.AdjustThreadsMax();
this.ThreadsRunning = 0;
this.ThreadsStop = false;
this.ThreadsDict = new Dictionary <int, Boolean> ();
this.SemaphoreWorkers = new Semaphore(0, this.ThreadsMax);
this.SemaphoreWorkers.Release(this.ThreadsMax);
this.Depth = MacroscopePreferencesManager.GetDepth();
this.PageLimit = MacroscopePreferencesManager.GetPageLimit();
this.PageLimitCount = 0;
this.PagesFound = 0;
{
this.ParentStartingDirectory = "";
this.ChildStartingDirectory = "";
}
this.JobHistory = new MacroscopeJobHistory();
this.InitProgress();
this.Locales = new Dictionary <string, string> (32);
this.Robots = new MacroscopeRobots();
this.BlockedByRobots = new Dictionary <string, Boolean> ();
}
/**************************************************************************/
private MacroscopeConstants.FetchStatus Fetch(string Url)
{
MacroscopeDocument msDoc = this.DocCollection.GetDocument(Url);
MacroscopeConstants.FetchStatus FetchStatus = MacroscopeConstants.FetchStatus.VOID;
if (msDoc != null)
{
if (msDoc.GetAuthenticationRealm() != null)
{
if (msDoc.GetAuthenticationType() == MacroscopeConstants.AuthenticationType.BASIC)
{
MacroscopeCredential Credential;
Credential = this.JobMaster.GetCredentialsHttp().GetCredential(
msDoc.GetHostAndPort(),
msDoc.GetAuthenticationRealm()
);
if (Credential != null)
{
msDoc = this.DocCollection.CreateDocument(
Credential: Credential,
Url: Url
);
}
}
}
}
else
{
msDoc = this.DocCollection.CreateDocument(Url);
}
msDoc.SetFetchStatus(MacroscopeConstants.FetchStatus.OK);
if (!MacroscopeDnsTools.CheckValidHostname(Url: Url))
{
DebugMsg(string.Format("Fetch :: CheckValidHostname: {0}", "NOT OK"));
msDoc.SetStatusCode(HttpStatusCode.BadGateway);
FetchStatus = MacroscopeConstants.FetchStatus.NETWORK_ERROR;
msDoc.SetFetchStatus(MacroscopeConstants.FetchStatus.NETWORK_ERROR);
}
if (!this.JobMaster.GetRobots().ApplyRobotRule(Url))
{
DebugMsg(string.Format("Disallowed by robots.txt: {0}", Url));
this.JobMaster.AddToBlockedByRobots(Url);
FetchStatus = MacroscopeConstants.FetchStatus.ROBOTS_DISALLOWED;
msDoc.SetFetchStatus(MacroscopeConstants.FetchStatus.ROBOTS_DISALLOWED);
this.JobMaster.GetJobHistory().VisitedHistoryItem(msDoc.GetUrl());
}
else
{
this.JobMaster.RemoveFromBlockedByRobots(Url);
}
this.JobMaster.GetJobHistory().AddHistoryItem(Url);
if (this.AllowedHosts.IsExternalUrl(Url: Url))
{
DebugMsg(string.Format("IsExternalUrl: {0}", Url));
msDoc.SetIsExternal(State: true);
}
if (this.DocCollection.ContainsDocument(Url))
{
if (!this.DocCollection.GetDocument(Url).GetIsDirty())
{
FetchStatus = MacroscopeConstants.FetchStatus.ALREADY_SEEN;
return(FetchStatus);
}
}
if (this.JobMaster.GetDepth() > 0)
{
int Depth = MacroscopeUrlUtils.FindUrlDepth(Url);
if (Depth > this.JobMaster.GetDepth())
{
DebugMsg(string.Format("TOO DEEP: {0}", Depth));
FetchStatus = MacroscopeConstants.FetchStatus.SKIPPED;
return(FetchStatus);
}
}
if (msDoc.Execute())
{
this.DocCollection.AddDocument(Url, msDoc);
if (msDoc.GetStatusCode() == HttpStatusCode.Unauthorized)
{
if (msDoc.GetAuthenticationType() == MacroscopeConstants.AuthenticationType.BASIC)
{
MacroscopeCredentialsHttp CredentialsHttp = this.JobMaster.GetCredentialsHttp();
CredentialsHttp.EnqueueCredentialRequest(
Domain: msDoc.GetHostAndPort(),
Realm: msDoc.GetAuthenticationRealm(),
Url: msDoc.GetUrl()
);
this.JobMaster.AddUrlQueueItem(Url: msDoc.GetUrl());
}
}
this.JobMaster.GetJobHistory().VisitedHistoryItem(msDoc.GetUrl());
this.JobMaster.IncPageLimitCount();
if (msDoc.GetIsRedirect())
{
DebugMsg(string.Format("REDIRECTION DETECTED GetUrl: {0}", msDoc.GetUrl()));
DebugMsg(string.Format("REDIRECTION DETECTED From: {0}", msDoc.GetUrlRedirectFrom()));
if (MacroscopePreferencesManager.GetFollowRedirects())
{
string Hostname = msDoc.GetHostAndPort();
string HostnameFrom = MacroscopeAllowedHosts.ParseHostnameFromUrl(msDoc.GetUrlRedirectFrom());
string UrlRedirectTo = msDoc.GetUrlRedirectTo();
string HostnameTo = MacroscopeAllowedHosts.ParseHostnameFromUrl(UrlRedirectTo);
DebugMsg(string.Format("REDIRECTION DETECTED UrlRedirectTo: {0}", UrlRedirectTo));
DebugMsg(string.Format("REDIRECTION DETECTED HostnameTo: {0}", HostnameTo));
}
this.JobMaster.AddUrlQueueItem(Url: msDoc.GetUrlRedirectTo());
}
else
{
this.ProcessHrefLangLanguages(msDoc); // Process Languages from HrefLang
this.ProcessOutlinks(msDoc); // Process Outlinks from document
}
FetchStatus = MacroscopeConstants.FetchStatus.SUCCESS;
}
else
{
DebugMsg(string.Format("EXECUTE FAILED: {0}", Url));
FetchStatus = MacroscopeConstants.FetchStatus.ERROR;
}
return(FetchStatus);
}