public void TestDetectLanguage()
{
List <string> UrlList = new List <string> ();
UrlList.Add("https://nazuke.github.io/SEOMacroscope/");
MacroscopePreferencesManager.SetDetectLanguage(Enabled: true);
MacroscopePreferencesManager.SetRequestTimeout(Seconds: 10);
for (int i = 0; i < 10; i++)
{
foreach (string Url in UrlList)
{
MacroscopeDocument msDoc = new MacroscopeDocument(Url: Url);
Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));
Assert.IsTrue(msDoc.Execute(), string.Format("FAIL: {0}", "Execute()"));
Assert.IsTrue(msDoc.GetIsHtml(), string.Format("FAIL: {0}", Url));
Assert.IsNotNullOrEmpty(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));
string LanguageTitle = msDoc.GetTitleLanguage();
string LanguageDescription = msDoc.GetDescriptionLanguage();
string LanguageBodyText = msDoc.GetDocumentTextLanguage();
Assert.AreEqual("en", LanguageTitle, string.Format("FAIL: {0} :: {1}", "LanguageTitle", LanguageTitle));
Assert.AreEqual("en", LanguageDescription, string.Format("FAIL: {0} :: {1}", "LanguageDescription", LanguageDescription));
Assert.AreEqual("en", LanguageBodyText, string.Format("FAIL: {0} :: {1}", "LanguageBodyText", LanguageBodyText));
}
}
}
public async Task TestHtmlDocument()
{
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
List <string> UrlList = new List <string>();
UrlList.Add("https://nazuke.github.io/");
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
foreach (string Url in UrlList)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));
bool ExecuteResult = await msDoc.Execute();
Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
Assert.AreEqual(Url, msDoc.GetUrl(), string.Format("FAIL: {0}", Url));
Assert.IsTrue(msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML), string.Format("FAIL: {0}", Url));
}
}
public async Task TestNRequests()
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE, TaskController: this);
MacroscopeDocumentCollection DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
Assert.AreEqual(0, DocCollection.CountDocuments());
foreach (string Url in this.Urls)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
await msDoc.Execute();
}
Assert.AreEqual(this.MaxUrls, DocCollection.CountDocuments());
}
public async Task TestDocumentCookiesAreSet()
{
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
List <string> UrlList = new List <string>(2);
UrlList.Add("https://httpbin.org/cookies/set?first=bongo&second=bongobongo");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
{ // Set Cookies
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: UrlList[0]);
bool ExecuteResult = await msDoc.Execute();
//Assert.IsTrue( ExecuteResult, string.Format( "FAIL: {0}", "Execute()" ) );
}
// Get Cookies
for (int i = 1; i < UrlList.Count; i++)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: UrlList[i]);
bool ExecuteResult = await msDoc.Execute();
Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
}
}
public async Task TestDetectLanguage()
{
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
List <string> UrlList = new List <string>();
UrlList.Add("https://nazuke.github.io/SEOMacroscope/");
MacroscopePreferencesManager.SetDefaultValues();
MacroscopePreferencesManager.SetDetectLanguage(Enabled: true);
MacroscopePreferencesManager.SetRequestTimeout(Seconds: 10);
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
for (int i = 0; i < 10; i++)
{
foreach (string Url in UrlList)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));
bool ExecuteResult = await msDoc.Execute();
Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
Assert.IsTrue(msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML), string.Format("FAIL: {0}", Url));
Assert.IsNotNull(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));
Assert.IsNotEmpty(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));
string LanguageTitle = msDoc.GetTitleLanguage();
string LanguageDescription = msDoc.GetDescriptionLanguage();
string LanguageBodyText = msDoc.GetDocumentTextLanguage();
Assert.AreEqual("en", LanguageTitle, string.Format("FAIL: {0} :: {1}", "LanguageTitle", LanguageTitle));
Assert.AreEqual("en", LanguageDescription, string.Format("FAIL: {0} :: {1}", "LanguageDescription", LanguageDescription));
Assert.AreEqual("en", LanguageBodyText, string.Format("FAIL: {0} :: {1}", "LanguageBodyText", LanguageBodyText));
}
}
}
public void TestDuplicate()
{
const string StartUrl = "https://nazuke.github.io/SEOMacroscope/";
const string DupeUrl = "https://nazuke.github.io/SEOMacroscope/index.html";
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
MacroscopeDocumentCollection DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
Dictionary <string, Boolean> CrossCheckList = MacroscopeLevenshteinAnalysis.GetCrossCheckList(Capacity: DocCollection.CountDocuments());
MacroscopeDocument msDoc = DocCollection.CreateDocument(StartUrl);
MacroscopeDocument msDocDifferent = DocCollection.CreateDocument(DupeUrl);
msDoc.Execute();
msDocDifferent.Execute();
DocCollection.AddDocument(msDoc);
DocCollection.AddDocument(msDocDifferent);
DebugMsg(string.Format("msDoc: {0}", msDoc.GetStatusCode()));
DebugMsg(string.Format("msDocDifferent: {0}", msDocDifferent.GetStatusCode()));
for (int i = 1; i <= 100; i++)
{
MacroscopeLevenshteinAnalysis LevenshteinAnalysis = new MacroscopeLevenshteinAnalysis(
msDoc: msDoc,
SizeDifference: 64,
Threshold: 16,
CrossCheckList: CrossCheckList
);
Dictionary <MacroscopeDocument, int> DocList = LevenshteinAnalysis.AnalyzeDocCollection(DocCollection: DocCollection);
DebugMsg(string.Format("DocList: {0}", DocList.Count));
foreach (MacroscopeDocument msDocAnalyzed in DocList.Keys)
{
DebugMsg(string.Format("msDocAnalyzed: {0} => {1}", DocList[msDocAnalyzed], msDocAnalyzed.GetUrl()));
Assert.AreEqual(DocList[msDocAnalyzed], 0, string.Format("FAIL: {0} => {1}", DocList[msDocAnalyzed], msDocAnalyzed.GetUrl()));
}
}
}
public void TestHtmlDocument()
{
List <string> UrlList = new List <string> ();
UrlList.Add("https://nazuke.github.io/SEOMacroscope/");
foreach (string Url in UrlList)
{
MacroscopeDocument msDoc = new MacroscopeDocument(Url: Url);
Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));
Boolean ExecuteResult = msDoc.Execute();
Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
Assert.AreEqual(Url, msDoc.GetUrl(), string.Format("FAIL: {0}", Url));
Assert.IsTrue(msDoc.GetIsHtml(), string.Format("FAIL: {0}", Url));
}
}
public async Task TestTextDocument()
{
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
List <string> UrlList = new List <string>();
UrlList.Add("https://nazuke.github.io/robots.txt");
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
foreach (string Url in UrlList)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));
bool ExecuteResult = await msDoc.Execute();
Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
Assert.AreEqual(Url, msDoc.GetUrl(), string.Format("FAIL: {0}", Url));
Assert.IsTrue(msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.TEXT), string.Format("FAIL: {0}", Url));
/** Content Property Assertions ------------------------------------ **/
Assert.AreEqual("text/plain", msDoc.GetMimeType());
Assert.Greater(msDoc.GetContentLength(), 0);
}
}
/**************************************************************************/
private async Task <MacroscopeConstants.FetchStatus> Fetch(string Url, string RedirectedFromUrl = null)
{
MacroscopeDocument msDoc = null;
MacroscopeConstants.FetchStatus FetchStatus = MacroscopeConstants.FetchStatus.VOID;
bool BlockedByRobotsRule;
if (MacroscopePreferencesManager.GetPageLimit() > -1)
{
int PagesFound = this.JobMaster.GetPagesFound();
int PageLimit = MacroscopePreferencesManager.GetPageLimit();
if (PagesFound >= PageLimit)
{
this.DebugMsg(string.Format("PAGE LIMIT REACHED: {0} :: {1}", PageLimit, PagesFound));
return(FetchStatus);
}
}
if (this.DocCollection.ContainsDocument(Url: Url))
{
msDoc = this.DocCollection.GetDocumentByUrl(Url: Url);
if (msDoc.GetAuthenticationRealm() != null)
{
if (msDoc.GetAuthenticationType() == MacroscopeConstants.AuthenticationType.BASIC)
{
MacroscopeCredential Credential;
Credential = this.JobMaster.GetCredentialsHttp().GetCredential(
msDoc.GetHostAndPort(),
msDoc.GetAuthenticationRealm()
);
if (Credential != null)
{
msDoc = this.DocCollection.CreateDocument(
Credential: Credential,
Url: Url
);
}
}
}
}
else
{
msDoc = this.DocCollection.CreateDocument(Url: Url);
}
if (!string.IsNullOrEmpty(RedirectedFromUrl))
{
msDoc.SetUrlRedirectFrom(Url: RedirectedFromUrl);
}
msDoc.SetFetchStatus(MacroscopeConstants.FetchStatus.OK);
if (!MacroscopeDnsTools.CheckValidHostname(Url: Url))
{
this.DebugMsg(string.Format("Fetch :: CheckValidHostname: {0}", "NOT OK"));
msDoc.SetStatusCode(HttpStatusCode.BadGateway);
FetchStatus = MacroscopeConstants.FetchStatus.NETWORK_ERROR;
msDoc.SetFetchStatus(FetchStatus);
}
if (await this.JobMaster.GetRobots().CheckRobotRule(Url: Url))
{
msDoc.SetAllowedByRobots(true);
}
else
{
msDoc.SetAllowedByRobots(false);
}
BlockedByRobotsRule = await this.JobMaster.GetRobots().ApplyRobotRule(Url: Url);
if (!BlockedByRobotsRule)
{
this.DebugMsg(string.Format("Disallowed by robots.txt: {0}", Url));
this.JobMaster.AddToBlockedByRobots(Url);
FetchStatus = MacroscopeConstants.FetchStatus.ROBOTS_DISALLOWED;
msDoc.SetFetchStatus(FetchStatus);
JobHistory.VisitedHistoryItem(Url: msDoc.GetUrl());
}
else
{
this.JobMaster.RemoveFromBlockedByRobots(Url);
}
if (this.AllowedHosts.IsExternalUrl(Url: Url))
{
this.DebugMsg(string.Format("IsExternalUrl: {0}", Url));
msDoc.SetIsExternal(State: true);
}
if (this.DocCollection.ContainsDocument(Url: Url))
{
if (!this.DocCollection.GetDocumentByUrl(Url: Url).GetIsDirty())
{
FetchStatus = MacroscopeConstants.FetchStatus.ALREADY_SEEN;
return(FetchStatus);
}
}
if (MacroscopePreferencesManager.GetDepth() >= 0)
{
int Depth = MacroscopeHttpUrlUtils.FindUrlDepth(Url: Url);
if (Depth > MacroscopePreferencesManager.GetDepth())
{
this.DebugMsg(string.Format("URL Too Deep: {0}", Depth));
FetchStatus = MacroscopeConstants.FetchStatus.SKIPPED;
return(FetchStatus);
}
}
/** ------------------------------------------------------------------ **/
if (!await msDoc.Execute())
{
this.DebugMsg(string.Format("EXECUTE FAILED: {0}", Url));
FetchStatus = MacroscopeConstants.FetchStatus.ERROR;
}
/** ------------------------------------------------------------------ **/
/** ------------------------------------------------------------------ **/
{
if (msDoc.GetStatusCode() == HttpStatusCode.Unauthorized)
{
if (msDoc.GetAuthenticationType() == MacroscopeConstants.AuthenticationType.BASIC)
{
MacroscopeCredentialsHttp CredentialsHttp = this.JobMaster.GetCredentialsHttp();
CredentialsHttp.EnqueueCredentialRequest(
Domain: msDoc.GetHostAndPort(),
Realm: msDoc.GetAuthenticationRealm(),
Url: msDoc.GetUrl()
);
this.JobMaster.AddUrlQueueItem(Url: msDoc.GetUrl());
}
}
if (msDoc.GetIsRedirect())
{
this.DebugMsg(string.Format("REDIRECTION DETECTED GetUrl: {0}", msDoc.GetUrl()));
this.DebugMsg(string.Format("REDIRECTION DETECTED From: {0}", msDoc.GetUrlRedirectFrom()));
if (MacroscopePreferencesManager.GetCheckRedirects())
{
string Hostname = msDoc.GetHostAndPort();
string HostnameFrom = MacroscopeAllowedHosts.ParseHostnameFromUrl(msDoc.GetUrlRedirectFrom());
string UrlRedirectTo = msDoc.GetUrlRedirectTo();
string HostnameTo = MacroscopeAllowedHosts.ParseHostnameFromUrl(UrlRedirectTo);
this.DebugMsg(string.Format("REDIRECTION DETECTED UrlRedirectTo: {0}", UrlRedirectTo));
this.DebugMsg(string.Format("REDIRECTION DETECTED HostnameTo: {0}", HostnameTo));
if (MacroscopePreferencesManager.GetFollowRedirects())
{
if (MacroscopePreferencesManager.GetCheckExternalLinks())
{
this.AllowedHosts.AddFromUrl(Url: UrlRedirectTo);
}
else
{
if (this.AllowedHosts.IsInternalUrl(Url: UrlRedirectTo))
{
this.AllowedHosts.AddFromUrl(Url: UrlRedirectTo);
}
}
}
}
this.JobMaster.AddUrlQueueItem(Url: msDoc.GetUrlRedirectTo());
}
else
{
this.ProcessHrefLangLanguages(msDoc); // Process Languages from HrefLang
this.JobMaster.ProcessOutlinks(msDoc: msDoc); // Process Outlinks from document
}
FetchStatus = MacroscopeConstants.FetchStatus.SUCCESS;
}
/** ------------------------------------------------------------------ **/
if (DocCollection.ContainsDocument(msDoc: msDoc))
{
JobHistory.VisitedHistoryItem(Url: Url);
}
else
{
this.DebugMsg(string.Format("OOPS: {0}", Url));
}
/** ------------------------------------------------------------------ **/
return(FetchStatus);
}
public async Task TestDifferent()
{
const string StartUrl = "https://nazuke.github.io/SEOMacroscope/";
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
Dictionary <string, bool> CrossCheckList;
MacroscopeDocument msDoc;
MacroscopeLevenshteinAnalysis LevenshteinAnalysis;
List <string> TargetUrls;
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
CrossCheckList = MacroscopeLevenshteinAnalysis.GetCrossCheckList(Capacity: DocCollection.CountDocuments());
msDoc = DocCollection.CreateDocument(StartUrl);
await msDoc.Execute();
DebugMsg(string.Format("msDoc: {0}", msDoc.GetStatusCode()));
LevenshteinAnalysis = new MacroscopeLevenshteinAnalysis(
msDoc: msDoc,
SizeDifference: 64,
Threshold: 16,
CrossCheckList: CrossCheckList
);
TargetUrls = new List <string>();
TargetUrls.Add("https://nazuke.github.io/SEOMacroscope/blog/");
TargetUrls.Add("https://nazuke.github.io/SEOMacroscope/downloads/");
TargetUrls.Add("https://nazuke.github.io/SEOMacroscope/manual/");
foreach (string TargetUrl in TargetUrls)
{
MacroscopeDocument msDocTarget = DocCollection.CreateDocument(TargetUrl);
await msDocTarget.Execute();
DebugMsg(string.Format("msDocTarget: {0}", msDocTarget.GetStatusCode()));
}
for (int i = 1; i <= 10; i++)
{
Dictionary <MacroscopeDocument, int> DocList;
DocList = LevenshteinAnalysis.AnalyzeDocCollection(
DocCollection: DocCollection
);
DebugMsg(string.Format("DocList: {0}", DocList.Count));
foreach (MacroscopeDocument msDocAnalyzed in DocList.Keys)
{
DebugMsg(string.Format("msDocAnalyzed: {0} => {1}", DocList[msDocAnalyzed], msDocAnalyzed.GetUrl()));
Assert.AreNotEqual(
DocList[msDocAnalyzed],
0,
string.Format(
"FAIL: {0} => {1}",
DocList[msDocAnalyzed],
msDocAnalyzed.GetUrl()
)
);
}
}
}
/**************************************************************************/
private MacroscopeConstants.FetchStatus Fetch(string Url)
{
MacroscopeDocument msDoc = this.DocCollection.GetDocument(Url);
MacroscopeConstants.FetchStatus FetchStatus = MacroscopeConstants.FetchStatus.VOID;
if (msDoc != null)
{
if (msDoc.GetAuthenticationRealm() != null)
{
if (msDoc.GetAuthenticationType() == MacroscopeConstants.AuthenticationType.BASIC)
{
MacroscopeCredential Credential;
Credential = this.JobMaster.GetCredentialsHttp().GetCredential(
msDoc.GetHostAndPort(),
msDoc.GetAuthenticationRealm()
);
if (Credential != null)
{
msDoc = this.DocCollection.CreateDocument(
Credential: Credential,
Url: Url
);
}
}
}
}
else
{
msDoc = this.DocCollection.CreateDocument(Url);
}
msDoc.SetFetchStatus(MacroscopeConstants.FetchStatus.OK);
if (!MacroscopeDnsTools.CheckValidHostname(Url: Url))
{
DebugMsg(string.Format("Fetch :: CheckValidHostname: {0}", "NOT OK"));
msDoc.SetStatusCode(HttpStatusCode.BadGateway);
FetchStatus = MacroscopeConstants.FetchStatus.NETWORK_ERROR;
msDoc.SetFetchStatus(MacroscopeConstants.FetchStatus.NETWORK_ERROR);
}
if (!this.JobMaster.GetRobots().ApplyRobotRule(Url))
{
DebugMsg(string.Format("Disallowed by robots.txt: {0}", Url));
this.JobMaster.AddToBlockedByRobots(Url);
FetchStatus = MacroscopeConstants.FetchStatus.ROBOTS_DISALLOWED;
msDoc.SetFetchStatus(MacroscopeConstants.FetchStatus.ROBOTS_DISALLOWED);
this.JobMaster.GetJobHistory().VisitedHistoryItem(msDoc.GetUrl());
}
else
{
this.JobMaster.RemoveFromBlockedByRobots(Url);
}
this.JobMaster.GetJobHistory().AddHistoryItem(Url);
if (this.AllowedHosts.IsExternalUrl(Url: Url))
{
DebugMsg(string.Format("IsExternalUrl: {0}", Url));
msDoc.SetIsExternal(State: true);
}
if (this.DocCollection.ContainsDocument(Url))
{
if (!this.DocCollection.GetDocument(Url).GetIsDirty())
{
FetchStatus = MacroscopeConstants.FetchStatus.ALREADY_SEEN;
return(FetchStatus);
}
}
if (this.JobMaster.GetDepth() > 0)
{
int Depth = MacroscopeUrlUtils.FindUrlDepth(Url);
if (Depth > this.JobMaster.GetDepth())
{
DebugMsg(string.Format("TOO DEEP: {0}", Depth));
FetchStatus = MacroscopeConstants.FetchStatus.SKIPPED;
return(FetchStatus);
}
}
if (msDoc.Execute())
{
this.DocCollection.AddDocument(Url, msDoc);
if (msDoc.GetStatusCode() == HttpStatusCode.Unauthorized)
{
if (msDoc.GetAuthenticationType() == MacroscopeConstants.AuthenticationType.BASIC)
{
MacroscopeCredentialsHttp CredentialsHttp = this.JobMaster.GetCredentialsHttp();
CredentialsHttp.EnqueueCredentialRequest(
Domain: msDoc.GetHostAndPort(),
Realm: msDoc.GetAuthenticationRealm(),
Url: msDoc.GetUrl()
);
this.JobMaster.AddUrlQueueItem(Url: msDoc.GetUrl());
}
}
this.JobMaster.GetJobHistory().VisitedHistoryItem(msDoc.GetUrl());
this.JobMaster.IncPageLimitCount();
if (msDoc.GetIsRedirect())
{
DebugMsg(string.Format("REDIRECTION DETECTED GetUrl: {0}", msDoc.GetUrl()));
DebugMsg(string.Format("REDIRECTION DETECTED From: {0}", msDoc.GetUrlRedirectFrom()));
if (MacroscopePreferencesManager.GetFollowRedirects())
{
string Hostname = msDoc.GetHostAndPort();
string HostnameFrom = MacroscopeAllowedHosts.ParseHostnameFromUrl(msDoc.GetUrlRedirectFrom());
string UrlRedirectTo = msDoc.GetUrlRedirectTo();
string HostnameTo = MacroscopeAllowedHosts.ParseHostnameFromUrl(UrlRedirectTo);
DebugMsg(string.Format("REDIRECTION DETECTED UrlRedirectTo: {0}", UrlRedirectTo));
DebugMsg(string.Format("REDIRECTION DETECTED HostnameTo: {0}", HostnameTo));
}
this.JobMaster.AddUrlQueueItem(Url: msDoc.GetUrlRedirectTo());
}
else
{
this.ProcessHrefLangLanguages(msDoc); // Process Languages from HrefLang
this.ProcessOutlinks(msDoc); // Process Outlinks from document
}
FetchStatus = MacroscopeConstants.FetchStatus.SUCCESS;
}
else
{
DebugMsg(string.Format("EXECUTE FAILED: {0}", Url));
FetchStatus = MacroscopeConstants.FetchStatus.ERROR;
}
return(FetchStatus);
}