public async Task TestDuplicate()
{
const string StartUrl = "https://nazuke.github.io/SEOMacroscope/";
const string DupeUrl = "https://nazuke.github.io/SEOMacroscope/index.html";
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
Dictionary <string, bool> CrossCheckList;
MacroscopeDocument msDoc;
MacroscopeDocument msDocDifferent;
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
CrossCheckList = MacroscopeLevenshteinAnalysis.GetCrossCheckList(Capacity: DocCollection.CountDocuments());
msDoc = DocCollection.CreateDocument(StartUrl);
msDocDifferent = DocCollection.CreateDocument(DupeUrl);
await msDoc.Execute();
await msDocDifferent.Execute();
DebugMsg(string.Format("msDoc: {0}", msDoc.GetStatusCode()));
DebugMsg(string.Format("msDocDifferent: {0}", msDocDifferent.GetStatusCode()));
for (int i = 1; i <= 100; i++)
{
MacroscopeLevenshteinAnalysis LevenshteinAnalysis;
Dictionary <MacroscopeDocument, int> DocList;
LevenshteinAnalysis = new MacroscopeLevenshteinAnalysis(
msDoc: msDoc,
SizeDifference: 64,
Threshold: 16,
CrossCheckList: CrossCheckList
);
DocList = LevenshteinAnalysis.AnalyzeDocCollection(DocCollection: DocCollection);
DebugMsg(string.Format("DocList: {0}", DocList.Count));
foreach (MacroscopeDocument msDocAnalyzed in DocList.Keys)
{
DebugMsg(string.Format("msDocAnalyzed: {0} => {1}", DocList[msDocAnalyzed], msDocAnalyzed.GetUrl()));
Assert.AreEqual(
DocList[msDocAnalyzed],
0,
string.Format("FAIL: {0} => {1}", DocList[msDocAnalyzed], msDocAnalyzed.GetUrl())
);
}
}
}
public async Task TestDocumentCookiesAreSet()
{
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
List <string> UrlList = new List <string>(2);
UrlList.Add("https://httpbin.org/cookies/set?first=bongo&second=bongobongo");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
{ // Set Cookies
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: UrlList[0]);
bool ExecuteResult = await msDoc.Execute();
//Assert.IsTrue( ExecuteResult, string.Format( "FAIL: {0}", "Execute()" ) );
}
// Get Cookies
for (int i = 1; i < UrlList.Count; i++)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: UrlList[i]);
bool ExecuteResult = await msDoc.Execute();
Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
}
}
public void TestLinksInTextDocs()
{
string Url = @"https://nazuke.github.io/dummy.txt";
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));
msDoc.ProcessPureTextOutlinks(TextDoc: this.TextDoc, LinkType: MacroscopeConstants.InOutLinkType.PURETEXT);
foreach (MacroscopeLink Outlink in msDoc.IterateOutlinks())
{
Assert.Contains(Outlink.GetTargetUrl(), this.TextLinks);
}
Assert.AreEqual(5, msDoc.CountOutlinks());
}
public async Task TestHtmlDocument()
{
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
List <string> UrlList = new List <string>();
UrlList.Add("https://nazuke.github.io/");
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
foreach (string Url in UrlList)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));
bool ExecuteResult = await msDoc.Execute();
Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
Assert.AreEqual(Url, msDoc.GetUrl(), string.Format("FAIL: {0}", Url));
Assert.IsTrue(msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML), string.Format("FAIL: {0}", Url));
}
}
public async Task TestNHeadRequests()
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE, TaskController: this);
MacroscopeDocumentCollection DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
Assert.AreEqual(0, DocCollection.CountDocuments());
foreach (string Url in this.Urls)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
await msDoc.ExecuteHeadRequest();
}
Assert.AreEqual(this.MaxUrls, DocCollection.CountDocuments());
}
public async Task TestDetectLanguage()
{
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
List <string> UrlList = new List <string>();
UrlList.Add("https://nazuke.github.io/SEOMacroscope/");
MacroscopePreferencesManager.SetDefaultValues();
MacroscopePreferencesManager.SetDetectLanguage(Enabled: true);
MacroscopePreferencesManager.SetRequestTimeout(Seconds: 10);
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
for (int i = 0; i < 10; i++)
{
foreach (string Url in UrlList)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));
bool ExecuteResult = await msDoc.Execute();
Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
Assert.IsTrue(msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML), string.Format("FAIL: {0}", Url));
Assert.IsNotNull(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));
Assert.IsNotEmpty(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));
string LanguageTitle = msDoc.GetTitleLanguage();
string LanguageDescription = msDoc.GetDescriptionLanguage();
string LanguageBodyText = msDoc.GetDocumentTextLanguage();
Assert.AreEqual("en", LanguageTitle, string.Format("FAIL: {0} :: {1}", "LanguageTitle", LanguageTitle));
Assert.AreEqual("en", LanguageDescription, string.Format("FAIL: {0} :: {1}", "LanguageDescription", LanguageDescription));
Assert.AreEqual("en", LanguageBodyText, string.Format("FAIL: {0} :: {1}", "LanguageBodyText", LanguageBodyText));
}
}
}
public async Task TestTextDocument()
{
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
List <string> UrlList = new List <string>();
UrlList.Add("https://nazuke.github.io/robots.txt");
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
foreach (string Url in UrlList)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));
bool ExecuteResult = await msDoc.Execute();
Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
Assert.AreEqual(Url, msDoc.GetUrl(), string.Format("FAIL: {0}", Url));
Assert.IsTrue(msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.TEXT), string.Format("FAIL: {0}", Url));
/** Content Property Assertions ------------------------------------ **/
Assert.AreEqual("text/plain", msDoc.GetMimeType());
Assert.Greater(msDoc.GetContentLength(), 0);
}
}
public void TestDifferent()
{
const string StartUrl = "https://nazuke.github.io/SEOMacroscope/";
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
MacroscopeDocumentCollection DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
Dictionary <string, Boolean> CrossCheckList = MacroscopeLevenshteinAnalysis.GetCrossCheckList(Capacity: DocCollection.CountDocuments());
MacroscopeDocument msDoc = DocCollection.CreateDocument(StartUrl);
msDoc.Execute();
DocCollection.AddDocument(msDoc);
DebugMsg(string.Format("msDoc: {0}", msDoc.GetStatusCode()));
MacroscopeLevenshteinAnalysis LevenshteinAnalysis = new MacroscopeLevenshteinAnalysis(
msDoc: msDoc,
SizeDifference: 64,
Threshold: 16,
CrossCheckList: CrossCheckList
);
List <string> TargetUrls = new List <string> ()
{
{
"https://nazuke.github.io/SEOMacroscope/blog/"
},
{
"https://nazuke.github.io/SEOMacroscope/downloads/"
},
{
"https://nazuke.github.io/SEOMacroscope/manual/"
}
};
foreach (string TargetUrl in TargetUrls)
{
MacroscopeDocument msDocTarget = DocCollection.CreateDocument(TargetUrl);
msDocTarget.Execute();
DocCollection.AddDocument(msDocTarget);
DebugMsg(string.Format("msDocTarget: {0}", msDocTarget.GetStatusCode()));
}
for (int i = 1; i <= 10; i++)
{
Dictionary <MacroscopeDocument, int> DocList;
DocList = LevenshteinAnalysis.AnalyzeDocCollection(
DocCollection: DocCollection
);
DebugMsg(string.Format("DocList: {0}", DocList.Count));
foreach (MacroscopeDocument msDocAnalyzed in DocList.Keys)
{
DebugMsg(string.Format("msDocAnalyzed: {0} => {1}", DocList[msDocAnalyzed], msDocAnalyzed.GetUrl()));
Assert.AreNotEqual(
DocList[msDocAnalyzed],
0,
string.Format(
"FAIL: {0} => {1}",
DocList[msDocAnalyzed],
msDocAnalyzed.GetUrl()
)
);
}
}
}