public void TestDetectLanguage()
        {
            List <string> UrlList = new List <string> ();

            UrlList.Add("https://nazuke.github.io/SEOMacroscope/");

            MacroscopePreferencesManager.SetDetectLanguage(Enabled: true);
            MacroscopePreferencesManager.SetRequestTimeout(Seconds: 10);

            for (int i = 0; i < 10; i++)
            {
                foreach (string Url in UrlList)
                {
                    MacroscopeDocument msDoc = new MacroscopeDocument(Url: Url);

                    Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));

                    Assert.IsTrue(msDoc.Execute(), string.Format("FAIL: {0}", "Execute()"));

                    Assert.IsTrue(msDoc.GetIsHtml(), string.Format("FAIL: {0}", Url));

                    Assert.IsNotNullOrEmpty(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));

                    string LanguageTitle       = msDoc.GetTitleLanguage();
                    string LanguageDescription = msDoc.GetDescriptionLanguage();
                    string LanguageBodyText    = msDoc.GetDocumentTextLanguage();

                    Assert.AreEqual("en", LanguageTitle, string.Format("FAIL: {0} :: {1}", "LanguageTitle", LanguageTitle));

                    Assert.AreEqual("en", LanguageDescription, string.Format("FAIL: {0} :: {1}", "LanguageDescription", LanguageDescription));

                    Assert.AreEqual("en", LanguageBodyText, string.Format("FAIL: {0} :: {1}", "LanguageBodyText", LanguageBodyText));
                }
            }
        }
Beispiel #2
0
        public async Task TestDetectLanguage()
        {
            MacroscopeJobMaster          JobMaster;
            MacroscopeDocumentCollection DocCollection;
            List <string> UrlList = new List <string>();

            UrlList.Add("https://nazuke.github.io/SEOMacroscope/");
            MacroscopePreferencesManager.SetDefaultValues();
            MacroscopePreferencesManager.SetDetectLanguage(Enabled: true);
            MacroscopePreferencesManager.SetRequestTimeout(Seconds: 10);

            JobMaster = new MacroscopeJobMaster(
                JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
                TaskController: this
                );

            DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);

            for (int i = 0; i < 10; i++)
            {
                foreach (string Url in UrlList)
                {
                    MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);

                    Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));

                    bool ExecuteResult = await msDoc.Execute();

                    Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));

                    Assert.IsTrue(msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML), string.Format("FAIL: {0}", Url));

                    Assert.IsNotNull(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));

                    Assert.IsNotEmpty(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));

                    string LanguageTitle       = msDoc.GetTitleLanguage();
                    string LanguageDescription = msDoc.GetDescriptionLanguage();
                    string LanguageBodyText    = msDoc.GetDocumentTextLanguage();

                    Assert.AreEqual("en", LanguageTitle, string.Format("FAIL: {0} :: {1}", "LanguageTitle", LanguageTitle));

                    Assert.AreEqual("en", LanguageDescription, string.Format("FAIL: {0} :: {1}", "LanguageDescription", LanguageDescription));

                    Assert.AreEqual("en", LanguageBodyText, string.Format("FAIL: {0} :: {1}", "LanguageBodyText", LanguageBodyText));
                }
            }
        }
Beispiel #3
0
        /** Render One ************************************************************/

        protected override void RenderListView(
            List <ListViewItem> ListViewItems,
            MacroscopeDocumentCollection DocCollection,
            MacroscopeDocument msDoc,
            string Url
            )
        {
            lock (this.DisplayListViewLock)
            {
                Dictionary <string, string> StructureItems = new Dictionary <string, string>();

                ListViewItem lvItem = null;

                string TitleLanguage       = msDoc.GetTitleLanguage();
                string DescriptionLanguage = msDoc.GetDescriptionLanguage();
                string BodyTextLanguage    = msDoc.GetDocumentTextLanguage();
                int    StatusCode          = (int)msDoc.GetStatusCode();
                string PairKey             = UrlToDigest(Url).ToString();

                if (string.IsNullOrEmpty(TitleLanguage))
                {
                    TitleLanguage = "";
                }

                if (string.IsNullOrEmpty(DescriptionLanguage))
                {
                    DescriptionLanguage = "";
                }

                if (string.IsNullOrEmpty(BodyTextLanguage))
                {
                    BodyTextLanguage = "";
                }

                // BEGIN: Columns ----------------------------------------------------//

                StructureItems.Add(MacroscopeConstants.Url, msDoc.GetUrl());

                StructureItems.Add(MacroscopeConstants.StatusCode, StatusCode.ToString());
                StructureItems.Add(MacroscopeConstants.Status, msDoc.GetStatusCode().ToString());
                StructureItems.Add(MacroscopeConstants.IsRedirect, msDoc.GetIsRedirect().ToString());

                StructureItems.Add(MacroscopeConstants.RobotsRule, msDoc.GetAllowedByRobotsAsString());

                StructureItems.Add(MacroscopeConstants.Duration, msDoc.GetDurationInSecondsFormatted());

                StructureItems.Add(MacroscopeConstants.ContentType, msDoc.GetMimeType());

                {
                    string Charset = msDoc.GetCharacterSet();
                    if (string.IsNullOrEmpty(Charset))
                    {
                        Charset = "";
                    }
                    StructureItems.Add(MacroscopeConstants.Charset, Charset);
                }

                {
                    string LocaleCode = msDoc.GetLocale();
                    if (string.IsNullOrEmpty(LocaleCode))
                    {
                        LocaleCode = "";
                    }
                    StructureItems.Add(MacroscopeConstants.Locale, LocaleCode);
                }

                {
                    string LanguageCode = msDoc.GetIsoLanguageCode();
                    if (string.IsNullOrEmpty(LanguageCode))
                    {
                        LanguageCode = "";
                    }
                    StructureItems.Add(MacroscopeConstants.Language, LanguageCode);
                }

                StructureItems.Add(MacroscopeConstants.DateCrawled, msDoc.GetCrawledDate());

                StructureItems.Add(MacroscopeConstants.DateServer, msDoc.GetDateServer());
                StructureItems.Add(MacroscopeConstants.DateModified, msDoc.GetDateModified());
                StructureItems.Add(MacroscopeConstants.DateExpires, msDoc.GetDateExpires());

                StructureItems.Add(MacroscopeConstants.Canonical, msDoc.GetCanonical());

                StructureItems.Add(MacroscopeConstants.PageDepth, msDoc.GetDepth().ToString());

                StructureItems.Add(MacroscopeConstants.Inlinks, msDoc.CountInlinks().ToString());
                StructureItems.Add(MacroscopeConstants.Outlinks, msDoc.CountOutlinks().ToString());

                StructureItems.Add(MacroscopeConstants.HyperlinksIn, msDoc.CountHyperlinksIn().ToString());
                StructureItems.Add(MacroscopeConstants.HyperlinksOut, msDoc.CountHyperlinksOut().ToString());

                // TODO: This is too slow:

                /*
                 * {
                 * List<decimal> HyperlinkRatio = DocCollection.GetDocumentHyperlinksRatio( Url: Url );
                 * StructureItems.Add( MacroscopeConstants.HyperlinksInRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 0 ] ) );
                 * StructureItems.Add( MacroscopeConstants.HyperlinksOutRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 1 ] ) );
                 * }
                 */

                StructureItems.Add(MacroscopeConstants.Author, msDoc.GetAuthor());

                StructureItems.Add(MacroscopeConstants.Title, msDoc.GetTitle());
                StructureItems.Add(MacroscopeConstants.TitleLen, msDoc.GetTitleLength().ToString());
                StructureItems.Add(MacroscopeConstants.TitleLang, TitleLanguage);

                StructureItems.Add(MacroscopeConstants.Description, msDoc.GetDescription());
                StructureItems.Add(MacroscopeConstants.DescriptionLen, msDoc.GetDescriptionLength().ToString());
                StructureItems.Add(MacroscopeConstants.DescriptionLang, DescriptionLanguage);

                StructureItems.Add(MacroscopeConstants.Keywords, msDoc.GetKeywords());
                StructureItems.Add(MacroscopeConstants.KeywordsLen, msDoc.GetKeywordsLength().ToString());
                StructureItems.Add(MacroscopeConstants.KeywordsCount, msDoc.GetKeywordsCount().ToString());

                StructureItems.Add(MacroscopeConstants.BodyTextLang, BodyTextLanguage);

                for (ushort HeadingLevel = 1; HeadingLevel <= MaxHeadingsDisplayed; HeadingLevel++)
                {
                    List <string> HeadingList = msDoc.GetHeadings(HeadingLevel: HeadingLevel);
                    string        HeadingText = "";
                    if (HeadingList.Count > 0)
                    {
                        HeadingText = HeadingList[0];
                    }
                    StructureItems.Add(string.Format(MacroscopeConstants.Hn, HeadingLevel), HeadingText);
                }

                StructureItems.Add(MacroscopeConstants.ErrorCondition, msDoc.GetErrorCondition());

                // END: Columns ------------------------------------------------------//

                if (this.DisplayListView.Items.ContainsKey(PairKey))
                {
                    lvItem = this.DisplayListView.Items[PairKey];
                }
                else
                {
                    lvItem = new ListViewItem(PairKey);
                    lvItem.UseItemStyleForSubItems = false;
                    lvItem.Name = PairKey;

                    for (int i = 0; i < this.DisplayListView.Columns.Count; i++)
                    {
                        lvItem.SubItems.Add("");
                    }

                    ListViewItems.Add(lvItem);
                }

                if (lvItem != null)
                {
                    lvItem.ForeColor = Color.Blue;

                    int StatusCodeColIndex = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.StatusCode);
                    int StatusColIndex     = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.Status);

                    foreach (string ItemsKey in StructureItems.Keys)
                    {
                        int    ColIndex = this.DisplayListView.Columns.IndexOfKey(ItemsKey);
                        string Text     = StructureItems[ItemsKey];

                        if (!string.IsNullOrEmpty(StructureItems[ItemsKey]))
                        {
                            lvItem.SubItems[ColIndex].Text = Text;
                        }
                        else
                        {
                            lvItem.SubItems[ColIndex].Text = "";
                        }

                        if (msDoc.GetIsInternal())
                        {
                            lvItem.SubItems[ColIndex].ForeColor = Color.Green;
                        }
                        else
                        {
                            lvItem.SubItems[ColIndex].ForeColor = Color.Gray;
                        }

                        if (ItemsKey.Equals(MacroscopeConstants.StatusCode))
                        {
                            if ((StatusCode >= 200) && (StatusCode <= 299))
                            {
                                lvItem.SubItems[ColIndex].ForeColor           = Color.Green;
                                lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Green;
                                lvItem.SubItems[StatusColIndex].ForeColor     = Color.Green;
                            }
                            else
                            if ((StatusCode >= 300) && (StatusCode <= 399))
                            {
                                lvItem.SubItems[ColIndex].ForeColor           = Color.Goldenrod;
                                lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Goldenrod;
                                lvItem.SubItems[StatusColIndex].ForeColor     = Color.Goldenrod;
                            }
                            else
                            if ((StatusCode >= 400) && (StatusCode <= 599))
                            {
                                lvItem.SubItems[ColIndex].ForeColor           = Color.Red;
                                lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Red;
                                lvItem.SubItems[StatusColIndex].ForeColor     = Color.Red;
                            }
                            else
                            {
                                lvItem.SubItems[ColIndex].ForeColor           = Color.Blue;
                                lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Blue;
                                lvItem.SubItems[StatusColIndex].ForeColor     = Color.Blue;
                            }

                            if (StatusCode == 410)
                            {
                                lvItem.SubItems[ColIndex].ForeColor           = Color.Purple;
                                lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Purple;
                                lvItem.SubItems[StatusColIndex].ForeColor     = Color.Purple;
                            }
                        }

                        if (ItemsKey == MacroscopeConstants.RobotsRule)
                        {
                            if (Text.ToLower() == "disallowed")
                            {
                                lvItem.SubItems[ColIndex].ForeColor = Color.Red;
                            }
                            else
                            {
                                lvItem.SubItems[ColIndex].ForeColor = Color.Green;
                            }
                        }

                        if (ItemsKey == MacroscopeConstants.IsRedirect)
                        {
                            if (Text.ToLower() == "true")
                            {
                                lvItem.SubItems[ColIndex].ForeColor = Color.Red;
                            }
                            else
                            {
                                lvItem.SubItems[ColIndex].ForeColor = Color.Gray;
                            }
                        }
                    }
                }
                else
                {
                    DebugMsg(string.Format("MacroscopeDisplayStructure: {0}", "lvItem is NULL"));
                }
            }
        }
        /**************************************************************************/

        protected override void RenderListView(
            List <ListViewItem> ListViewItems,
            MacroscopeDocumentCollection DocCollection,
            MacroscopeDocument msDoc,
            string Url
            )
        {
            bool Proceed = false;

            if (msDoc.GetIsExternal())
            {
                return;
            }

            if (msDoc.GetIsRedirect())
            {
                return;
            }

            switch (msDoc.GetDocumentType())
            {
            case MacroscopeConstants.DocumentType.HTML:
                Proceed = true;
                break;

            case MacroscopeConstants.DocumentType.PDF:
                Proceed = true;
                break;

            default:
                break;
            }

            if (Proceed)
            {
                string PageLocale           = msDoc.GetLocale();
                string PageLanguage         = msDoc.GetIsoLanguageCode();
                string DetectedLanguage     = msDoc.GetDocumentTextLanguage();
                int    WordCount            = msDoc.GetWordCount();
                string ReadabilityGradeType = MacroscopeAnalyzeReadability.FormatAnalyzeReadabilityMethod(
                    ReadabilityMethod: msDoc.GetReadabilityGradeMethod()
                    );
                string       ReadabilityGrade            = msDoc.GetReadabilityGrade().ToString("00.00");
                string       ReadabilityGradeDescription = msDoc.GetReadabilityGradeDescription();
                string       PairKey = string.Join("", Url);
                ListViewItem lvItem  = null;

                if (string.IsNullOrEmpty(PageLocale))
                {
                    PageLocale = "";
                }

                if (string.IsNullOrEmpty(PageLanguage))
                {
                    PageLanguage = "";
                }

                if (string.IsNullOrEmpty(DetectedLanguage))
                {
                    DetectedLanguage = "";
                }

                if (this.DisplayListView.Items.ContainsKey(PairKey))
                {
                    try
                    {
                        lvItem = this.DisplayListView.Items[PairKey];
                        lvItem.SubItems[ColUrl].Text                         = Url;
                        lvItem.SubItems[ColLocale].Text                      = PageLocale;
                        lvItem.SubItems[ColPageLanguage].Text                = PageLanguage;
                        lvItem.SubItems[ColDetectedLanguage].Text            = DetectedLanguage;
                        lvItem.SubItems[ColWordCount].Text                   = WordCount.ToString();
                        lvItem.SubItems[ColReadabilityGradeType].Text        = ReadabilityGradeType;
                        lvItem.SubItems[ColReadabilityGrade].Text            = ReadabilityGrade;
                        lvItem.SubItems[ColReadabilityGradeDescription].Text = ReadabilityGradeDescription;
                    }
                    catch (Exception ex)
                    {
                        DebugMsg(string.Format("MacroscopeDisplayPageText 1: {0}", ex.Message));
                    }
                }
                else
                {
                    try
                    {
                        lvItem = new ListViewItem(PairKey);
                        lvItem.UseItemStyleForSubItems = false;
                        lvItem.Name = PairKey;

                        lvItem.SubItems[ColUrl].Text = Url;
                        lvItem.SubItems.Add(PageLocale);
                        lvItem.SubItems.Add(PageLanguage);
                        lvItem.SubItems.Add(DetectedLanguage);
                        lvItem.SubItems.Add(WordCount.ToString());
                        lvItem.SubItems.Add(ReadabilityGradeType);
                        lvItem.SubItems.Add(ReadabilityGrade);
                        lvItem.SubItems.Add(ReadabilityGradeDescription);

                        ListViewItems.Add(lvItem);
                    }
                    catch (Exception ex)
                    {
                        DebugMsg(string.Format("MacroscopeDisplayPageText 2: {0}", ex.Message));
                    }
                }

                if (lvItem != null)
                {
                    lvItem.ForeColor = Color.Blue;

                    // URL -------------------------------------------------------------//

                    if (msDoc.GetIsInternal())
                    {
                        lvItem.SubItems[ColUrl].ForeColor = Color.Green;
                    }
                    else
                    {
                        lvItem.SubItems[ColUrl].ForeColor = Color.Gray;
                    }

                    // Page Locale -----------------------------------------------------//

                    if (msDoc.GetIsInternal())
                    {
                        lvItem.SubItems[ColLocale].ForeColor = Color.Green;
                    }
                    else
                    {
                        lvItem.SubItems[ColLocale].ForeColor = Color.Gray;
                    }

                    // Page Language ---------------------------------------------------//

                    if (msDoc.GetIsInternal())
                    {
                        lvItem.SubItems[ColPageLanguage].ForeColor     = Color.Green;
                        lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Green;

                        if (DetectedLanguage != PageLanguage)
                        {
                            lvItem.SubItems[ColPageLanguage].ForeColor     = Color.Red;
                            lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Red;
                        }
                    }
                    else
                    {
                        lvItem.SubItems[ColPageLanguage].ForeColor     = Color.Gray;
                        lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Gray;
                    }

                    // Word Count ------------------------------------------------------//

                    if (msDoc.GetIsInternal())
                    {
                        if (WordCount > 0)
                        {
                            lvItem.SubItems[ColWordCount].ForeColor = Color.Green;
                        }
                        else
                        {
                            lvItem.SubItems[ColWordCount].ForeColor = Color.Red;
                        }
                    }
                    else
                    {
                        lvItem.SubItems[ColWordCount].ForeColor = Color.Gray;
                    }
                }
            }
        }
        /**************************************************************************/

        private void BuildWorksheetPageText(
            MacroscopeJobMaster JobMaster,
            XLWorkbook wb,
            string WorksheetLabel
            )
        {
            var ws = wb.Worksheets.Add(WorksheetLabel);

            int iRow    = 1;
            int iCol    = 1;
            int iColMax = 1;

            MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();

            {
                ws.Cell(iRow, iCol).Value = "URL";
                iCol++;

                ws.Cell(iRow, iCol).Value = "Page Locale";
                iCol++;

                ws.Cell(iRow, iCol).Value = "Page Language";
                iCol++;

                ws.Cell(iRow, iCol).Value = "Detected Language";
                iCol++;

                ws.Cell(iRow, iCol).Value = "Word Count";
                iCol++;

                ws.Cell(iRow, iCol).Value = "Readability Method";
                iCol++;

                ws.Cell(iRow, iCol).Value = "Readability Grade";
                iCol++;

                ws.Cell(iRow, iCol).Value = "Readability Grade Description";
            }

            iColMax = iCol;

            iRow++;

            foreach (string Url in DocCollection.DocumentKeys())
            {
                MacroscopeDocument msDoc   = DocCollection.GetDocument(Url);
                Boolean            Proceed = false;

                if (msDoc.GetIsExternal())
                {
                    continue;
                }

                if (msDoc.GetIsRedirect())
                {
                    continue;
                }

                if (msDoc.GetIsHtml())
                {
                    Proceed = true;
                }
                else
                if (msDoc.GetIsPdf())
                {
                    Proceed = true;
                }

                if (Proceed)
                {
                    iCol = 1;

                    string PageLocale           = msDoc.GetLocale();
                    string PageLanguage         = msDoc.GetIsoLanguageCode();
                    string DetectedLanguage     = msDoc.GetDocumentTextLanguage();
                    int    WordCount            = msDoc.GetWordCount();
                    string ReadabilityGradeType = MacroscopeAnalyzeReadability.FormatAnalyzeReadabilityMethod(
                        ReadabilityMethod: msDoc.GetReadabilityGradeMethod()
                        );
                    string ReadabilityGrade            = msDoc.GetReadabilityGrade().ToString("00.00");
                    string ReadabilityGradeDescription = msDoc.GetReadabilityGradeDescription();

                    if (string.IsNullOrEmpty(PageLocale))
                    {
                        PageLocale = "";
                    }

                    if (string.IsNullOrEmpty(PageLanguage))
                    {
                        PageLanguage = "";
                    }

                    if (string.IsNullOrEmpty(DetectedLanguage))
                    {
                        DetectedLanguage = "";
                    }

                    this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);

                    if (msDoc.GetIsInternal())
                    {
                        ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
                    }
                    else
                    {
                        ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
                    }

                    iCol++;

                    this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(PageLocale));

                    if (msDoc.GetIsInternal())
                    {
                        ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
                    }
                    else
                    {
                        ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
                    }

                    iCol++;

                    this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(PageLanguage));

                    if (PageLanguage != DetectedLanguage)
                    {
                        ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
                    }
                    else
                    {
                        ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
                    }

                    iCol++;

                    this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(DetectedLanguage));

                    if (PageLanguage != DetectedLanguage)
                    {
                        ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
                    }
                    else
                    {
                        ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
                    }

                    iCol++;

                    this.InsertAndFormatContentCell(ws, iRow, iCol, WordCount);

                    if (msDoc.GetIsInternal())
                    {
                        if (WordCount > 0)
                        {
                            ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
                        }
                        else
                        {
                            ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
                        }
                    }
                    else
                    {
                        ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
                    }

                    iCol++;

                    this.InsertAndFormatContentCell(ws, iRow, iCol, ReadabilityGradeType);

                    iCol++;

                    this.InsertAndFormatContentCell(ws, iRow, iCol, ReadabilityGrade);

                    iCol++;

                    this.InsertAndFormatContentCell(ws, iRow, iCol, ReadabilityGradeDescription);

                    iRow++;
                }
            }

            {
                var rangeData  = ws.Range(1, 1, iRow - 1, iColMax);
                var excelTable = rangeData.CreateTable();
            }
        }
        /**************************************************************************/

        private void BuildWorksheetPageText(
            MacroscopeJobMaster JobMaster,
            CsvWriter ws
            )
        {
            MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();

            {
                ws.WriteField("URL");
                ws.WriteField("Page Locale");
                ws.WriteField("Page Language");
                ws.WriteField("Detected Language");
                ws.WriteField("Word Count");
                ws.WriteField("Readability Method");
                ws.WriteField("Readability Grade");
                ws.WriteField("Readability Grade Description");

                ws.NextRecord();
            }

            foreach (string Url in DocCollection.DocumentKeys())
            {
                MacroscopeDocument msDoc   = DocCollection.GetDocument(Url);
                Boolean            Proceed = false;

                if (msDoc.GetIsExternal())
                {
                    continue;
                }

                if (msDoc.GetIsRedirect())
                {
                    continue;
                }

                if (msDoc.GetIsHtml())
                {
                    Proceed = true;
                }
                else
                if (msDoc.GetIsPdf())
                {
                    Proceed = true;
                }

                if (Proceed)
                {
                    string PageLocale           = msDoc.GetLocale();
                    string PageLanguage         = msDoc.GetIsoLanguageCode();
                    string DetectedLanguage     = msDoc.GetDocumentTextLanguage();
                    int    WordCount            = msDoc.GetWordCount();
                    string ReadabilityGradeType = MacroscopeAnalyzeReadability.FormatAnalyzeReadabilityMethod(
                        ReadabilityMethod: msDoc.GetReadabilityGradeMethod()
                        );
                    string ReadabilityGrade            = msDoc.GetReadabilityGrade().ToString("00.00");
                    string ReadabilityGradeDescription = msDoc.GetReadabilityGradeDescription();

                    if (string.IsNullOrEmpty(PageLocale))
                    {
                        PageLocale = "";
                    }

                    if (string.IsNullOrEmpty(PageLanguage))
                    {
                        PageLanguage = "";
                    }

                    if (string.IsNullOrEmpty(DetectedLanguage))
                    {
                        DetectedLanguage = "";
                    }

                    this.InsertAndFormatUrlCell(ws, msDoc);

                    this.InsertAndFormatContentCell(ws, this.FormatIfMissing(PageLocale));

                    this.InsertAndFormatContentCell(ws, this.FormatIfMissing(PageLanguage));

                    this.InsertAndFormatContentCell(ws, this.FormatIfMissing(DetectedLanguage));

                    this.InsertAndFormatContentCell(ws, this.FormatIfMissing(WordCount.ToString()));

                    this.InsertAndFormatContentCell(ws, this.FormatIfMissing(ReadabilityGradeType));
                    this.InsertAndFormatContentCell(ws, this.FormatIfMissing(ReadabilityGrade));
                    this.InsertAndFormatContentCell(ws, this.FormatIfMissing(ReadabilityGradeDescription));

                    ws.NextRecord();
                }
            }
        }