public void TestDetectLanguage() { List <string> UrlList = new List <string> (); UrlList.Add("https://nazuke.github.io/SEOMacroscope/"); MacroscopePreferencesManager.SetDetectLanguage(Enabled: true); MacroscopePreferencesManager.SetRequestTimeout(Seconds: 10); for (int i = 0; i < 10; i++) { foreach (string Url in UrlList) { MacroscopeDocument msDoc = new MacroscopeDocument(Url: Url); Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url)); Assert.IsTrue(msDoc.Execute(), string.Format("FAIL: {0}", "Execute()")); Assert.IsTrue(msDoc.GetIsHtml(), string.Format("FAIL: {0}", Url)); Assert.IsNotNullOrEmpty(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle())); string LanguageTitle = msDoc.GetTitleLanguage(); string LanguageDescription = msDoc.GetDescriptionLanguage(); string LanguageBodyText = msDoc.GetDocumentTextLanguage(); Assert.AreEqual("en", LanguageTitle, string.Format("FAIL: {0} :: {1}", "LanguageTitle", LanguageTitle)); Assert.AreEqual("en", LanguageDescription, string.Format("FAIL: {0} :: {1}", "LanguageDescription", LanguageDescription)); Assert.AreEqual("en", LanguageBodyText, string.Format("FAIL: {0} :: {1}", "LanguageBodyText", LanguageBodyText)); } } }
public async Task TestDetectLanguage() { MacroscopeJobMaster JobMaster; MacroscopeDocumentCollection DocCollection; List <string> UrlList = new List <string>(); UrlList.Add("https://nazuke.github.io/SEOMacroscope/"); MacroscopePreferencesManager.SetDefaultValues(); MacroscopePreferencesManager.SetDetectLanguage(Enabled: true); MacroscopePreferencesManager.SetRequestTimeout(Seconds: 10); JobMaster = new MacroscopeJobMaster( JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE, TaskController: this ); DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster); for (int i = 0; i < 10; i++) { foreach (string Url in UrlList) { MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url); Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url)); bool ExecuteResult = await msDoc.Execute(); Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()")); Assert.IsTrue(msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML), string.Format("FAIL: {0}", Url)); Assert.IsNotNull(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle())); Assert.IsNotEmpty(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle())); string LanguageTitle = msDoc.GetTitleLanguage(); string LanguageDescription = msDoc.GetDescriptionLanguage(); string LanguageBodyText = msDoc.GetDocumentTextLanguage(); Assert.AreEqual("en", LanguageTitle, string.Format("FAIL: {0} :: {1}", "LanguageTitle", LanguageTitle)); Assert.AreEqual("en", LanguageDescription, string.Format("FAIL: {0} :: {1}", "LanguageDescription", LanguageDescription)); Assert.AreEqual("en", LanguageBodyText, string.Format("FAIL: {0} :: {1}", "LanguageBodyText", LanguageBodyText)); } } }
/** Render One ************************************************************/ protected override void RenderListView( List <ListViewItem> ListViewItems, MacroscopeDocumentCollection DocCollection, MacroscopeDocument msDoc, string Url ) { lock (this.DisplayListViewLock) { Dictionary <string, string> StructureItems = new Dictionary <string, string>(); ListViewItem lvItem = null; string TitleLanguage = msDoc.GetTitleLanguage(); string DescriptionLanguage = msDoc.GetDescriptionLanguage(); string BodyTextLanguage = msDoc.GetDocumentTextLanguage(); int StatusCode = (int)msDoc.GetStatusCode(); string PairKey = UrlToDigest(Url).ToString(); if (string.IsNullOrEmpty(TitleLanguage)) { TitleLanguage = ""; } if (string.IsNullOrEmpty(DescriptionLanguage)) { DescriptionLanguage = ""; } if (string.IsNullOrEmpty(BodyTextLanguage)) { BodyTextLanguage = ""; } // BEGIN: Columns ----------------------------------------------------// StructureItems.Add(MacroscopeConstants.Url, msDoc.GetUrl()); StructureItems.Add(MacroscopeConstants.StatusCode, StatusCode.ToString()); StructureItems.Add(MacroscopeConstants.Status, msDoc.GetStatusCode().ToString()); StructureItems.Add(MacroscopeConstants.IsRedirect, msDoc.GetIsRedirect().ToString()); StructureItems.Add(MacroscopeConstants.RobotsRule, msDoc.GetAllowedByRobotsAsString()); StructureItems.Add(MacroscopeConstants.Duration, msDoc.GetDurationInSecondsFormatted()); StructureItems.Add(MacroscopeConstants.ContentType, msDoc.GetMimeType()); { string Charset = msDoc.GetCharacterSet(); if (string.IsNullOrEmpty(Charset)) { Charset = ""; } StructureItems.Add(MacroscopeConstants.Charset, Charset); } { string LocaleCode = msDoc.GetLocale(); if (string.IsNullOrEmpty(LocaleCode)) { LocaleCode = ""; } StructureItems.Add(MacroscopeConstants.Locale, LocaleCode); } { string LanguageCode = msDoc.GetIsoLanguageCode(); if (string.IsNullOrEmpty(LanguageCode)) { LanguageCode = ""; } StructureItems.Add(MacroscopeConstants.Language, LanguageCode); } StructureItems.Add(MacroscopeConstants.DateCrawled, msDoc.GetCrawledDate()); StructureItems.Add(MacroscopeConstants.DateServer, msDoc.GetDateServer()); StructureItems.Add(MacroscopeConstants.DateModified, msDoc.GetDateModified()); StructureItems.Add(MacroscopeConstants.DateExpires, msDoc.GetDateExpires()); StructureItems.Add(MacroscopeConstants.Canonical, msDoc.GetCanonical()); StructureItems.Add(MacroscopeConstants.PageDepth, msDoc.GetDepth().ToString()); StructureItems.Add(MacroscopeConstants.Inlinks, msDoc.CountInlinks().ToString()); StructureItems.Add(MacroscopeConstants.Outlinks, msDoc.CountOutlinks().ToString()); StructureItems.Add(MacroscopeConstants.HyperlinksIn, msDoc.CountHyperlinksIn().ToString()); StructureItems.Add(MacroscopeConstants.HyperlinksOut, msDoc.CountHyperlinksOut().ToString()); // TODO: This is too slow: /* * { * List<decimal> HyperlinkRatio = DocCollection.GetDocumentHyperlinksRatio( Url: Url ); * StructureItems.Add( MacroscopeConstants.HyperlinksInRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 0 ] ) ); * StructureItems.Add( MacroscopeConstants.HyperlinksOutRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 1 ] ) ); * } */ StructureItems.Add(MacroscopeConstants.Author, msDoc.GetAuthor()); StructureItems.Add(MacroscopeConstants.Title, msDoc.GetTitle()); StructureItems.Add(MacroscopeConstants.TitleLen, msDoc.GetTitleLength().ToString()); StructureItems.Add(MacroscopeConstants.TitleLang, TitleLanguage); StructureItems.Add(MacroscopeConstants.Description, msDoc.GetDescription()); StructureItems.Add(MacroscopeConstants.DescriptionLen, msDoc.GetDescriptionLength().ToString()); StructureItems.Add(MacroscopeConstants.DescriptionLang, DescriptionLanguage); StructureItems.Add(MacroscopeConstants.Keywords, msDoc.GetKeywords()); StructureItems.Add(MacroscopeConstants.KeywordsLen, msDoc.GetKeywordsLength().ToString()); StructureItems.Add(MacroscopeConstants.KeywordsCount, msDoc.GetKeywordsCount().ToString()); StructureItems.Add(MacroscopeConstants.BodyTextLang, BodyTextLanguage); for (ushort HeadingLevel = 1; HeadingLevel <= MaxHeadingsDisplayed; HeadingLevel++) { List <string> HeadingList = msDoc.GetHeadings(HeadingLevel: HeadingLevel); string HeadingText = ""; if (HeadingList.Count > 0) { HeadingText = HeadingList[0]; } StructureItems.Add(string.Format(MacroscopeConstants.Hn, HeadingLevel), HeadingText); } StructureItems.Add(MacroscopeConstants.ErrorCondition, msDoc.GetErrorCondition()); // END: Columns ------------------------------------------------------// if (this.DisplayListView.Items.ContainsKey(PairKey)) { lvItem = this.DisplayListView.Items[PairKey]; } else { lvItem = new ListViewItem(PairKey); lvItem.UseItemStyleForSubItems = false; lvItem.Name = PairKey; for (int i = 0; i < this.DisplayListView.Columns.Count; i++) { lvItem.SubItems.Add(""); } ListViewItems.Add(lvItem); } if (lvItem != null) { lvItem.ForeColor = Color.Blue; int StatusCodeColIndex = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.StatusCode); int StatusColIndex = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.Status); foreach (string ItemsKey in StructureItems.Keys) { int ColIndex = this.DisplayListView.Columns.IndexOfKey(ItemsKey); string Text = StructureItems[ItemsKey]; if (!string.IsNullOrEmpty(StructureItems[ItemsKey])) { lvItem.SubItems[ColIndex].Text = Text; } else { lvItem.SubItems[ColIndex].Text = ""; } if (msDoc.GetIsInternal()) { lvItem.SubItems[ColIndex].ForeColor = Color.Green; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Gray; } if (ItemsKey.Equals(MacroscopeConstants.StatusCode)) { if ((StatusCode >= 200) && (StatusCode <= 299)) { lvItem.SubItems[ColIndex].ForeColor = Color.Green; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Green; lvItem.SubItems[StatusColIndex].ForeColor = Color.Green; } else if ((StatusCode >= 300) && (StatusCode <= 399)) { lvItem.SubItems[ColIndex].ForeColor = Color.Goldenrod; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Goldenrod; lvItem.SubItems[StatusColIndex].ForeColor = Color.Goldenrod; } else if ((StatusCode >= 400) && (StatusCode <= 599)) { lvItem.SubItems[ColIndex].ForeColor = Color.Red; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Red; lvItem.SubItems[StatusColIndex].ForeColor = Color.Red; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Blue; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Blue; lvItem.SubItems[StatusColIndex].ForeColor = Color.Blue; } if (StatusCode == 410) { lvItem.SubItems[ColIndex].ForeColor = Color.Purple; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Purple; lvItem.SubItems[StatusColIndex].ForeColor = Color.Purple; } } if (ItemsKey == MacroscopeConstants.RobotsRule) { if (Text.ToLower() == "disallowed") { lvItem.SubItems[ColIndex].ForeColor = Color.Red; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Green; } } if (ItemsKey == MacroscopeConstants.IsRedirect) { if (Text.ToLower() == "true") { lvItem.SubItems[ColIndex].ForeColor = Color.Red; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Gray; } } } } else { DebugMsg(string.Format("MacroscopeDisplayStructure: {0}", "lvItem is NULL")); } } }
/**************************************************************************/ protected override void RenderListView( List <ListViewItem> ListViewItems, MacroscopeDocumentCollection DocCollection, MacroscopeDocument msDoc, string Url ) { bool Proceed = false; if (msDoc.GetIsExternal()) { return; } if (msDoc.GetIsRedirect()) { return; } switch (msDoc.GetDocumentType()) { case MacroscopeConstants.DocumentType.HTML: Proceed = true; break; case MacroscopeConstants.DocumentType.PDF: Proceed = true; break; default: break; } if (Proceed) { string PageLocale = msDoc.GetLocale(); string PageLanguage = msDoc.GetIsoLanguageCode(); string DetectedLanguage = msDoc.GetDocumentTextLanguage(); int WordCount = msDoc.GetWordCount(); string ReadabilityGradeType = MacroscopeAnalyzeReadability.FormatAnalyzeReadabilityMethod( ReadabilityMethod: msDoc.GetReadabilityGradeMethod() ); string ReadabilityGrade = msDoc.GetReadabilityGrade().ToString("00.00"); string ReadabilityGradeDescription = msDoc.GetReadabilityGradeDescription(); string PairKey = string.Join("", Url); ListViewItem lvItem = null; if (string.IsNullOrEmpty(PageLocale)) { PageLocale = ""; } if (string.IsNullOrEmpty(PageLanguage)) { PageLanguage = ""; } if (string.IsNullOrEmpty(DetectedLanguage)) { DetectedLanguage = ""; } if (this.DisplayListView.Items.ContainsKey(PairKey)) { try { lvItem = this.DisplayListView.Items[PairKey]; lvItem.SubItems[ColUrl].Text = Url; lvItem.SubItems[ColLocale].Text = PageLocale; lvItem.SubItems[ColPageLanguage].Text = PageLanguage; lvItem.SubItems[ColDetectedLanguage].Text = DetectedLanguage; lvItem.SubItems[ColWordCount].Text = WordCount.ToString(); lvItem.SubItems[ColReadabilityGradeType].Text = ReadabilityGradeType; lvItem.SubItems[ColReadabilityGrade].Text = ReadabilityGrade; lvItem.SubItems[ColReadabilityGradeDescription].Text = ReadabilityGradeDescription; } catch (Exception ex) { DebugMsg(string.Format("MacroscopeDisplayPageText 1: {0}", ex.Message)); } } else { try { lvItem = new ListViewItem(PairKey); lvItem.UseItemStyleForSubItems = false; lvItem.Name = PairKey; lvItem.SubItems[ColUrl].Text = Url; lvItem.SubItems.Add(PageLocale); lvItem.SubItems.Add(PageLanguage); lvItem.SubItems.Add(DetectedLanguage); lvItem.SubItems.Add(WordCount.ToString()); lvItem.SubItems.Add(ReadabilityGradeType); lvItem.SubItems.Add(ReadabilityGrade); lvItem.SubItems.Add(ReadabilityGradeDescription); ListViewItems.Add(lvItem); } catch (Exception ex) { DebugMsg(string.Format("MacroscopeDisplayPageText 2: {0}", ex.Message)); } } if (lvItem != null) { lvItem.ForeColor = Color.Blue; // URL -------------------------------------------------------------// if (msDoc.GetIsInternal()) { lvItem.SubItems[ColUrl].ForeColor = Color.Green; } else { lvItem.SubItems[ColUrl].ForeColor = Color.Gray; } // Page Locale -----------------------------------------------------// if (msDoc.GetIsInternal()) { lvItem.SubItems[ColLocale].ForeColor = Color.Green; } else { lvItem.SubItems[ColLocale].ForeColor = Color.Gray; } // Page Language ---------------------------------------------------// if (msDoc.GetIsInternal()) { lvItem.SubItems[ColPageLanguage].ForeColor = Color.Green; lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Green; if (DetectedLanguage != PageLanguage) { lvItem.SubItems[ColPageLanguage].ForeColor = Color.Red; lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Red; } } else { lvItem.SubItems[ColPageLanguage].ForeColor = Color.Gray; lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Gray; } // Word Count ------------------------------------------------------// if (msDoc.GetIsInternal()) { if (WordCount > 0) { lvItem.SubItems[ColWordCount].ForeColor = Color.Green; } else { lvItem.SubItems[ColWordCount].ForeColor = Color.Red; } } else { lvItem.SubItems[ColWordCount].ForeColor = Color.Gray; } } } }
/**************************************************************************/ private void BuildWorksheetPageText( MacroscopeJobMaster JobMaster, XLWorkbook wb, string WorksheetLabel ) { var ws = wb.Worksheets.Add(WorksheetLabel); int iRow = 1; int iCol = 1; int iColMax = 1; MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection(); { ws.Cell(iRow, iCol).Value = "URL"; iCol++; ws.Cell(iRow, iCol).Value = "Page Locale"; iCol++; ws.Cell(iRow, iCol).Value = "Page Language"; iCol++; ws.Cell(iRow, iCol).Value = "Detected Language"; iCol++; ws.Cell(iRow, iCol).Value = "Word Count"; iCol++; ws.Cell(iRow, iCol).Value = "Readability Method"; iCol++; ws.Cell(iRow, iCol).Value = "Readability Grade"; iCol++; ws.Cell(iRow, iCol).Value = "Readability Grade Description"; } iColMax = iCol; iRow++; foreach (string Url in DocCollection.DocumentKeys()) { MacroscopeDocument msDoc = DocCollection.GetDocument(Url); Boolean Proceed = false; if (msDoc.GetIsExternal()) { continue; } if (msDoc.GetIsRedirect()) { continue; } if (msDoc.GetIsHtml()) { Proceed = true; } else if (msDoc.GetIsPdf()) { Proceed = true; } if (Proceed) { iCol = 1; string PageLocale = msDoc.GetLocale(); string PageLanguage = msDoc.GetIsoLanguageCode(); string DetectedLanguage = msDoc.GetDocumentTextLanguage(); int WordCount = msDoc.GetWordCount(); string ReadabilityGradeType = MacroscopeAnalyzeReadability.FormatAnalyzeReadabilityMethod( ReadabilityMethod: msDoc.GetReadabilityGradeMethod() ); string ReadabilityGrade = msDoc.GetReadabilityGrade().ToString("00.00"); string ReadabilityGradeDescription = msDoc.GetReadabilityGradeDescription(); if (string.IsNullOrEmpty(PageLocale)) { PageLocale = ""; } if (string.IsNullOrEmpty(PageLanguage)) { PageLanguage = ""; } if (string.IsNullOrEmpty(DetectedLanguage)) { DetectedLanguage = ""; } this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc); if (msDoc.GetIsInternal()) { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green); } else { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray); } iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(PageLocale)); if (msDoc.GetIsInternal()) { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green); } else { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray); } iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(PageLanguage)); if (PageLanguage != DetectedLanguage) { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red); } else { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green); } iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(DetectedLanguage)); if (PageLanguage != DetectedLanguage) { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red); } else { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green); } iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, WordCount); if (msDoc.GetIsInternal()) { if (WordCount > 0) { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green); } else { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red); } } else { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray); } iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, ReadabilityGradeType); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, ReadabilityGrade); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, ReadabilityGradeDescription); iRow++; } } { var rangeData = ws.Range(1, 1, iRow - 1, iColMax); var excelTable = rangeData.CreateTable(); } }
/**************************************************************************/ private void BuildWorksheetPageText( MacroscopeJobMaster JobMaster, CsvWriter ws ) { MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection(); { ws.WriteField("URL"); ws.WriteField("Page Locale"); ws.WriteField("Page Language"); ws.WriteField("Detected Language"); ws.WriteField("Word Count"); ws.WriteField("Readability Method"); ws.WriteField("Readability Grade"); ws.WriteField("Readability Grade Description"); ws.NextRecord(); } foreach (string Url in DocCollection.DocumentKeys()) { MacroscopeDocument msDoc = DocCollection.GetDocument(Url); Boolean Proceed = false; if (msDoc.GetIsExternal()) { continue; } if (msDoc.GetIsRedirect()) { continue; } if (msDoc.GetIsHtml()) { Proceed = true; } else if (msDoc.GetIsPdf()) { Proceed = true; } if (Proceed) { string PageLocale = msDoc.GetLocale(); string PageLanguage = msDoc.GetIsoLanguageCode(); string DetectedLanguage = msDoc.GetDocumentTextLanguage(); int WordCount = msDoc.GetWordCount(); string ReadabilityGradeType = MacroscopeAnalyzeReadability.FormatAnalyzeReadabilityMethod( ReadabilityMethod: msDoc.GetReadabilityGradeMethod() ); string ReadabilityGrade = msDoc.GetReadabilityGrade().ToString("00.00"); string ReadabilityGradeDescription = msDoc.GetReadabilityGradeDescription(); if (string.IsNullOrEmpty(PageLocale)) { PageLocale = ""; } if (string.IsNullOrEmpty(PageLanguage)) { PageLanguage = ""; } if (string.IsNullOrEmpty(DetectedLanguage)) { DetectedLanguage = ""; } this.InsertAndFormatUrlCell(ws, msDoc); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(PageLocale)); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(PageLanguage)); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(DetectedLanguage)); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(WordCount.ToString())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(ReadabilityGradeType)); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(ReadabilityGrade)); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(ReadabilityGradeDescription)); ws.NextRecord(); } } }