/**************************************************************************/ protected override void RenderListView( List <ListViewItem> ListViewItems, MacroscopeDocumentCollection DocCollection, MacroscopeDocument msDoc, string Url ) { bool Proceed = false; if (msDoc.GetIsExternal()) { return; } if (msDoc.GetIsRedirect()) { return; } if (msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML)) { Proceed = true; } if (Proceed) { for (ushort HeadingLevel = 1; HeadingLevel <= MacroscopePreferencesManager.GetMaxHeadingDepth(); HeadingLevel++) { List <string> HeadingsList = msDoc.GetHeadings(HeadingLevel); for (int Order = 0; Order < HeadingsList.Count; Order++) { ListViewItem lvItem = null; string PairKey = string.Join(":", UrlToDigest(Url), UrlToDigest(HeadingLevel.ToString()), UrlToDigest(Order.ToString())).ToString(); int HeadingColIndex = HeadingLevel + ColH1Offset; string TextLabel = HeadingsList[Order]; int Occurences = DocCollection.GetStatsHeadingsCount(HeadingLevel: HeadingLevel, Text: TextLabel); if (this.DisplayListView.Items.ContainsKey(PairKey)) { try { lvItem = this.DisplayListView.Items[PairKey]; lvItem.SubItems[ColUrl].Text = Url; lvItem.SubItems[ColOccurences].Text = Occurences.ToString(); lvItem.SubItems[ColOrder].Text = (Order + 1).ToString(); lvItem.SubItems[HeadingColIndex].Text = TextLabel; } catch (Exception ex) { DebugMsg(string.Format("MacroscopeDisplayHeadings 1: {0}", ex.Message)); } } else { try { lvItem = new ListViewItem(PairKey); lvItem.UseItemStyleForSubItems = false; lvItem.Name = PairKey; lvItem.SubItems[ColUrl].Text = Url; lvItem.SubItems.Add(Occurences.ToString()); lvItem.SubItems.Add((Order + 1).ToString()); for (ushort k = 1; k <= 6; k++) { lvItem.SubItems.Add(""); } lvItem.SubItems[HeadingColIndex].Text = TextLabel; ListViewItems.Add(lvItem); } catch (Exception ex) { DebugMsg(string.Format("MacroscopeDisplayHeadings 2: {0}", ex.Message)); } } if (lvItem != null) { lvItem.ForeColor = Color.Blue; // URL -----------------------------------------------------------// if (msDoc.GetIsInternal()) { lvItem.SubItems[ColUrl].ForeColor = Color.Green; } else { lvItem.SubItems[ColUrl].ForeColor = Color.Gray; } // Occurences ----------------------------------------------------// if ((Occurences > 1) && (msDoc.GetIsInternal())) { lvItem.SubItems[ColOccurences].ForeColor = Color.Orange; } else if (msDoc.GetIsInternal()) { lvItem.SubItems[ColOccurences].ForeColor = Color.Green; } else { lvItem.SubItems[ColOccurences].ForeColor = Color.Gray; } // Check Missing H1 ----------------------------------------------// if ((HeadingLevel == 1) && string.IsNullOrEmpty(TextLabel)) { lvItem.SubItems[HeadingColIndex].Text = "MISSING"; lvItem.SubItems[HeadingColIndex].ForeColor = Color.Red; } else { lvItem.SubItems[HeadingColIndex].ForeColor = Color.Green; } } } } } }
/**************************************************************************/ private void BuildWorksheetPageHeadings( MacroscopeJobMaster JobMaster, CsvWriter ws ) { MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection(); { ws.WriteField("URL"); ws.WriteField("Occurences"); ws.WriteField("Order"); for (int i = 1; i <= 6; i++) { ws.WriteField(string.Format("H{0}", i)); } ws.NextRecord(); } foreach (string Url in DocCollection.DocumentKeys()) { MacroscopeDocument msDoc = DocCollection.GetDocument(Url); Boolean Proceed = false; if (msDoc.GetIsExternal()) { continue; } if (msDoc.GetIsRedirect()) { continue; } if (msDoc.GetIsHtml()) { Proceed = true; } if (Proceed) { for (ushort HeadingLevel = 1; HeadingLevel <= MacroscopePreferencesManager.GetMaxHeadingDepth(); HeadingLevel++) { List <string> HeadingsList = msDoc.GetHeadings(HeadingLevel); for (int Order = 0; Order < HeadingsList.Count; Order++) { int Occurences = DocCollection.GetStatsHeadingsCount(HeadingLevel: HeadingLevel, Text: HeadingsList[Order]); this.InsertAndFormatUrlCell(ws, msDoc); this.InsertAndFormatContentCell(ws, Occurences.ToString()); this.InsertAndFormatContentCell(ws, this.FormatIfMissing((Order + 1).ToString())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(HeadingsList[Order])); ws.NextRecord(); } } } } }
/** Render One ************************************************************/ protected override void RenderListView( List <ListViewItem> ListViewItems, MacroscopeDocumentCollection DocCollection, MacroscopeDocument msDoc, string Url ) { lock (this.DisplayListViewLock) { Dictionary <string, string> StructureItems = new Dictionary <string, string>(); ListViewItem lvItem = null; string TitleLanguage = msDoc.GetTitleLanguage(); string DescriptionLanguage = msDoc.GetDescriptionLanguage(); string BodyTextLanguage = msDoc.GetDocumentTextLanguage(); int StatusCode = (int)msDoc.GetStatusCode(); string PairKey = UrlToDigest(Url).ToString(); if (string.IsNullOrEmpty(TitleLanguage)) { TitleLanguage = ""; } if (string.IsNullOrEmpty(DescriptionLanguage)) { DescriptionLanguage = ""; } if (string.IsNullOrEmpty(BodyTextLanguage)) { BodyTextLanguage = ""; } // BEGIN: Columns ----------------------------------------------------// StructureItems.Add(MacroscopeConstants.Url, msDoc.GetUrl()); StructureItems.Add(MacroscopeConstants.StatusCode, StatusCode.ToString()); StructureItems.Add(MacroscopeConstants.Status, msDoc.GetStatusCode().ToString()); StructureItems.Add(MacroscopeConstants.IsRedirect, msDoc.GetIsRedirect().ToString()); StructureItems.Add(MacroscopeConstants.RobotsRule, msDoc.GetAllowedByRobotsAsString()); StructureItems.Add(MacroscopeConstants.Duration, msDoc.GetDurationInSecondsFormatted()); StructureItems.Add(MacroscopeConstants.ContentType, msDoc.GetMimeType()); { string Charset = msDoc.GetCharacterSet(); if (string.IsNullOrEmpty(Charset)) { Charset = ""; } StructureItems.Add(MacroscopeConstants.Charset, Charset); } { string LocaleCode = msDoc.GetLocale(); if (string.IsNullOrEmpty(LocaleCode)) { LocaleCode = ""; } StructureItems.Add(MacroscopeConstants.Locale, LocaleCode); } { string LanguageCode = msDoc.GetIsoLanguageCode(); if (string.IsNullOrEmpty(LanguageCode)) { LanguageCode = ""; } StructureItems.Add(MacroscopeConstants.Language, LanguageCode); } StructureItems.Add(MacroscopeConstants.DateCrawled, msDoc.GetCrawledDate()); StructureItems.Add(MacroscopeConstants.DateServer, msDoc.GetDateServer()); StructureItems.Add(MacroscopeConstants.DateModified, msDoc.GetDateModified()); StructureItems.Add(MacroscopeConstants.DateExpires, msDoc.GetDateExpires()); StructureItems.Add(MacroscopeConstants.Canonical, msDoc.GetCanonical()); StructureItems.Add(MacroscopeConstants.PageDepth, msDoc.GetDepth().ToString()); StructureItems.Add(MacroscopeConstants.Inlinks, msDoc.CountInlinks().ToString()); StructureItems.Add(MacroscopeConstants.Outlinks, msDoc.CountOutlinks().ToString()); StructureItems.Add(MacroscopeConstants.HyperlinksIn, msDoc.CountHyperlinksIn().ToString()); StructureItems.Add(MacroscopeConstants.HyperlinksOut, msDoc.CountHyperlinksOut().ToString()); // TODO: This is too slow: /* * { * List<decimal> HyperlinkRatio = DocCollection.GetDocumentHyperlinksRatio( Url: Url ); * StructureItems.Add( MacroscopeConstants.HyperlinksInRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 0 ] ) ); * StructureItems.Add( MacroscopeConstants.HyperlinksOutRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 1 ] ) ); * } */ StructureItems.Add(MacroscopeConstants.Author, msDoc.GetAuthor()); StructureItems.Add(MacroscopeConstants.Title, msDoc.GetTitle()); StructureItems.Add(MacroscopeConstants.TitleLen, msDoc.GetTitleLength().ToString()); StructureItems.Add(MacroscopeConstants.TitleLang, TitleLanguage); StructureItems.Add(MacroscopeConstants.Description, msDoc.GetDescription()); StructureItems.Add(MacroscopeConstants.DescriptionLen, msDoc.GetDescriptionLength().ToString()); StructureItems.Add(MacroscopeConstants.DescriptionLang, DescriptionLanguage); StructureItems.Add(MacroscopeConstants.Keywords, msDoc.GetKeywords()); StructureItems.Add(MacroscopeConstants.KeywordsLen, msDoc.GetKeywordsLength().ToString()); StructureItems.Add(MacroscopeConstants.KeywordsCount, msDoc.GetKeywordsCount().ToString()); StructureItems.Add(MacroscopeConstants.BodyTextLang, BodyTextLanguage); for (ushort HeadingLevel = 1; HeadingLevel <= MaxHeadingsDisplayed; HeadingLevel++) { List <string> HeadingList = msDoc.GetHeadings(HeadingLevel: HeadingLevel); string HeadingText = ""; if (HeadingList.Count > 0) { HeadingText = HeadingList[0]; } StructureItems.Add(string.Format(MacroscopeConstants.Hn, HeadingLevel), HeadingText); } StructureItems.Add(MacroscopeConstants.ErrorCondition, msDoc.GetErrorCondition()); // END: Columns ------------------------------------------------------// if (this.DisplayListView.Items.ContainsKey(PairKey)) { lvItem = this.DisplayListView.Items[PairKey]; } else { lvItem = new ListViewItem(PairKey); lvItem.UseItemStyleForSubItems = false; lvItem.Name = PairKey; for (int i = 0; i < this.DisplayListView.Columns.Count; i++) { lvItem.SubItems.Add(""); } ListViewItems.Add(lvItem); } if (lvItem != null) { lvItem.ForeColor = Color.Blue; int StatusCodeColIndex = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.StatusCode); int StatusColIndex = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.Status); foreach (string ItemsKey in StructureItems.Keys) { int ColIndex = this.DisplayListView.Columns.IndexOfKey(ItemsKey); string Text = StructureItems[ItemsKey]; if (!string.IsNullOrEmpty(StructureItems[ItemsKey])) { lvItem.SubItems[ColIndex].Text = Text; } else { lvItem.SubItems[ColIndex].Text = ""; } if (msDoc.GetIsInternal()) { lvItem.SubItems[ColIndex].ForeColor = Color.Green; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Gray; } if (ItemsKey.Equals(MacroscopeConstants.StatusCode)) { if ((StatusCode >= 200) && (StatusCode <= 299)) { lvItem.SubItems[ColIndex].ForeColor = Color.Green; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Green; lvItem.SubItems[StatusColIndex].ForeColor = Color.Green; } else if ((StatusCode >= 300) && (StatusCode <= 399)) { lvItem.SubItems[ColIndex].ForeColor = Color.Goldenrod; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Goldenrod; lvItem.SubItems[StatusColIndex].ForeColor = Color.Goldenrod; } else if ((StatusCode >= 400) && (StatusCode <= 599)) { lvItem.SubItems[ColIndex].ForeColor = Color.Red; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Red; lvItem.SubItems[StatusColIndex].ForeColor = Color.Red; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Blue; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Blue; lvItem.SubItems[StatusColIndex].ForeColor = Color.Blue; } if (StatusCode == 410) { lvItem.SubItems[ColIndex].ForeColor = Color.Purple; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Purple; lvItem.SubItems[StatusColIndex].ForeColor = Color.Purple; } } if (ItemsKey == MacroscopeConstants.RobotsRule) { if (Text.ToLower() == "disallowed") { lvItem.SubItems[ColIndex].ForeColor = Color.Red; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Green; } } if (ItemsKey == MacroscopeConstants.IsRedirect) { if (Text.ToLower() == "true") { lvItem.SubItems[ColIndex].ForeColor = Color.Red; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Gray; } } } } else { DebugMsg(string.Format("MacroscopeDisplayStructure: {0}", "lvItem is NULL")); } } }
/**************************************************************************/ private void BuildWorksheetPageHeadings( MacroscopeJobMaster JobMaster, XLWorkbook wb, string WorksheetLabel ) { var ws = wb.Worksheets.Add(WorksheetLabel); int iRow = 1; int iCol = 1; int iColMax = 1; MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection(); { ws.Cell(iRow, iCol).Value = "URL"; iCol++; ws.Cell(iRow, iCol).Value = "Occurences"; iCol++; ws.Cell(iRow, iCol).Value = "Order"; for (int i = 1; i <= 6; i++) { iCol++; ws.Cell(iRow, iCol).Value = string.Format("H{0}", i); } } iColMax = iCol; iRow++; foreach (string Url in DocCollection.DocumentKeys()) { MacroscopeDocument msDoc = DocCollection.GetDocument(Url); Boolean Proceed = false; if (msDoc.GetIsExternal()) { continue; } if (msDoc.GetIsRedirect()) { continue; } if (msDoc.GetIsHtml()) { Proceed = true; } if (Proceed) { for (ushort HeadingLevel = 1; HeadingLevel <= MacroscopePreferencesManager.GetMaxHeadingDepth(); HeadingLevel++) { List <string> HeadingsList = msDoc.GetHeadings(HeadingLevel); for (int Order = 0; Order < HeadingsList.Count; Order++) { int Occurences = DocCollection.GetStatsHeadingsCount(HeadingLevel: HeadingLevel, Text: HeadingsList[Order]); iCol = 1; this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc); if (msDoc.GetIsInternal()) { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green); } else { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray); } iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, Occurences); if ((Occurences > 1) && (msDoc.GetIsInternal())) { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Orange); } else if (msDoc.GetIsInternal()) { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green); } else { ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray); } iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing((Order + 1).ToString())); this.InsertAndFormatContentCell(ws, iRow, ( int )(HeadingLevel + iCol), this.FormatIfMissing(HeadingsList[Order])); iRow++; } } } } { var rangeData = ws.Range(1, 1, iRow - 1, iColMax); var excelTable = rangeData.CreateTable(); } }