/** Render One ************************************************************/ protected override void RenderListView( List <ListViewItem> ListViewItems, MacroscopeDocumentCollection DocCollection, MacroscopeDocument msDoc, string Url ) { lock (this.DisplayListViewLock) { ListViewItem lvItem = null; if (this.DisplayListView.Items.ContainsKey(Url)) { lvItem = this.DisplayListView.Items[Url]; } if (lvItem != null) { int ColIndexInlinks = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.Inlinks); int ColIndexOutlinks = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.Outlinks); int ColIndexInhyperlinks = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.HyperlinksIn); int ColIndexOuthyperlinks = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.HyperlinksOut); lvItem.SubItems[ColIndexInlinks].Text = msDoc.CountInlinks().ToString(); lvItem.SubItems[ColIndexOutlinks].Text = msDoc.CountOutlinks().ToString(); lvItem.SubItems[ColIndexInhyperlinks].Text = msDoc.CountHyperlinksIn().ToString(); lvItem.SubItems[ColIndexOuthyperlinks].Text = msDoc.CountHyperlinksOut().ToString(); } } }
/** Render One ************************************************************/ protected override void RenderListView( List <ListViewItem> ListViewItems, MacroscopeDocumentCollection DocCollection, MacroscopeDocument msDoc, string Url ) { lock (this.DisplayListViewLock) { Dictionary <string, string> StructureItems = new Dictionary <string, string>(); ListViewItem lvItem = null; string TitleLanguage = msDoc.GetTitleLanguage(); string DescriptionLanguage = msDoc.GetDescriptionLanguage(); string BodyTextLanguage = msDoc.GetDocumentTextLanguage(); int StatusCode = (int)msDoc.GetStatusCode(); string PairKey = UrlToDigest(Url).ToString(); if (string.IsNullOrEmpty(TitleLanguage)) { TitleLanguage = ""; } if (string.IsNullOrEmpty(DescriptionLanguage)) { DescriptionLanguage = ""; } if (string.IsNullOrEmpty(BodyTextLanguage)) { BodyTextLanguage = ""; } // BEGIN: Columns ----------------------------------------------------// StructureItems.Add(MacroscopeConstants.Url, msDoc.GetUrl()); StructureItems.Add(MacroscopeConstants.StatusCode, StatusCode.ToString()); StructureItems.Add(MacroscopeConstants.Status, msDoc.GetStatusCode().ToString()); StructureItems.Add(MacroscopeConstants.IsRedirect, msDoc.GetIsRedirect().ToString()); StructureItems.Add(MacroscopeConstants.RobotsRule, msDoc.GetAllowedByRobotsAsString()); StructureItems.Add(MacroscopeConstants.Duration, msDoc.GetDurationInSecondsFormatted()); StructureItems.Add(MacroscopeConstants.ContentType, msDoc.GetMimeType()); { string Charset = msDoc.GetCharacterSet(); if (string.IsNullOrEmpty(Charset)) { Charset = ""; } StructureItems.Add(MacroscopeConstants.Charset, Charset); } { string LocaleCode = msDoc.GetLocale(); if (string.IsNullOrEmpty(LocaleCode)) { LocaleCode = ""; } StructureItems.Add(MacroscopeConstants.Locale, LocaleCode); } { string LanguageCode = msDoc.GetIsoLanguageCode(); if (string.IsNullOrEmpty(LanguageCode)) { LanguageCode = ""; } StructureItems.Add(MacroscopeConstants.Language, LanguageCode); } StructureItems.Add(MacroscopeConstants.DateCrawled, msDoc.GetCrawledDate()); StructureItems.Add(MacroscopeConstants.DateServer, msDoc.GetDateServer()); StructureItems.Add(MacroscopeConstants.DateModified, msDoc.GetDateModified()); StructureItems.Add(MacroscopeConstants.DateExpires, msDoc.GetDateExpires()); StructureItems.Add(MacroscopeConstants.Canonical, msDoc.GetCanonical()); StructureItems.Add(MacroscopeConstants.PageDepth, msDoc.GetDepth().ToString()); StructureItems.Add(MacroscopeConstants.Inlinks, msDoc.CountInlinks().ToString()); StructureItems.Add(MacroscopeConstants.Outlinks, msDoc.CountOutlinks().ToString()); StructureItems.Add(MacroscopeConstants.HyperlinksIn, msDoc.CountHyperlinksIn().ToString()); StructureItems.Add(MacroscopeConstants.HyperlinksOut, msDoc.CountHyperlinksOut().ToString()); // TODO: This is too slow: /* * { * List<decimal> HyperlinkRatio = DocCollection.GetDocumentHyperlinksRatio( Url: Url ); * StructureItems.Add( MacroscopeConstants.HyperlinksInRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 0 ] ) ); * StructureItems.Add( MacroscopeConstants.HyperlinksOutRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 1 ] ) ); * } */ StructureItems.Add(MacroscopeConstants.Author, msDoc.GetAuthor()); StructureItems.Add(MacroscopeConstants.Title, msDoc.GetTitle()); StructureItems.Add(MacroscopeConstants.TitleLen, msDoc.GetTitleLength().ToString()); StructureItems.Add(MacroscopeConstants.TitleLang, TitleLanguage); StructureItems.Add(MacroscopeConstants.Description, msDoc.GetDescription()); StructureItems.Add(MacroscopeConstants.DescriptionLen, msDoc.GetDescriptionLength().ToString()); StructureItems.Add(MacroscopeConstants.DescriptionLang, DescriptionLanguage); StructureItems.Add(MacroscopeConstants.Keywords, msDoc.GetKeywords()); StructureItems.Add(MacroscopeConstants.KeywordsLen, msDoc.GetKeywordsLength().ToString()); StructureItems.Add(MacroscopeConstants.KeywordsCount, msDoc.GetKeywordsCount().ToString()); StructureItems.Add(MacroscopeConstants.BodyTextLang, BodyTextLanguage); for (ushort HeadingLevel = 1; HeadingLevel <= MaxHeadingsDisplayed; HeadingLevel++) { List <string> HeadingList = msDoc.GetHeadings(HeadingLevel: HeadingLevel); string HeadingText = ""; if (HeadingList.Count > 0) { HeadingText = HeadingList[0]; } StructureItems.Add(string.Format(MacroscopeConstants.Hn, HeadingLevel), HeadingText); } StructureItems.Add(MacroscopeConstants.ErrorCondition, msDoc.GetErrorCondition()); // END: Columns ------------------------------------------------------// if (this.DisplayListView.Items.ContainsKey(PairKey)) { lvItem = this.DisplayListView.Items[PairKey]; } else { lvItem = new ListViewItem(PairKey); lvItem.UseItemStyleForSubItems = false; lvItem.Name = PairKey; for (int i = 0; i < this.DisplayListView.Columns.Count; i++) { lvItem.SubItems.Add(""); } ListViewItems.Add(lvItem); } if (lvItem != null) { lvItem.ForeColor = Color.Blue; int StatusCodeColIndex = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.StatusCode); int StatusColIndex = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.Status); foreach (string ItemsKey in StructureItems.Keys) { int ColIndex = this.DisplayListView.Columns.IndexOfKey(ItemsKey); string Text = StructureItems[ItemsKey]; if (!string.IsNullOrEmpty(StructureItems[ItemsKey])) { lvItem.SubItems[ColIndex].Text = Text; } else { lvItem.SubItems[ColIndex].Text = ""; } if (msDoc.GetIsInternal()) { lvItem.SubItems[ColIndex].ForeColor = Color.Green; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Gray; } if (ItemsKey.Equals(MacroscopeConstants.StatusCode)) { if ((StatusCode >= 200) && (StatusCode <= 299)) { lvItem.SubItems[ColIndex].ForeColor = Color.Green; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Green; lvItem.SubItems[StatusColIndex].ForeColor = Color.Green; } else if ((StatusCode >= 300) && (StatusCode <= 399)) { lvItem.SubItems[ColIndex].ForeColor = Color.Goldenrod; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Goldenrod; lvItem.SubItems[StatusColIndex].ForeColor = Color.Goldenrod; } else if ((StatusCode >= 400) && (StatusCode <= 599)) { lvItem.SubItems[ColIndex].ForeColor = Color.Red; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Red; lvItem.SubItems[StatusColIndex].ForeColor = Color.Red; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Blue; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Blue; lvItem.SubItems[StatusColIndex].ForeColor = Color.Blue; } if (StatusCode == 410) { lvItem.SubItems[ColIndex].ForeColor = Color.Purple; lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Purple; lvItem.SubItems[StatusColIndex].ForeColor = Color.Purple; } } if (ItemsKey == MacroscopeConstants.RobotsRule) { if (Text.ToLower() == "disallowed") { lvItem.SubItems[ColIndex].ForeColor = Color.Red; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Green; } } if (ItemsKey == MacroscopeConstants.IsRedirect) { if (Text.ToLower() == "true") { lvItem.SubItems[ColIndex].ForeColor = Color.Red; } else { lvItem.SubItems[ColIndex].ForeColor = Color.Gray; } } } } else { DebugMsg(string.Format("MacroscopeDisplayStructure: {0}", "lvItem is NULL")); } } }
/**************************************************************************/ private void BuildWorksheetOverview( MacroscopeJobMaster JobMaster, XLWorkbook wb, string WorksheetLabel ) { var ws = wb.Worksheets.Add(WorksheetLabel); int iRow = 1; int iCol = 1; int iColMax = 1; MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection(); { ws.Cell(iRow, iCol).Value = "URL"; iCol++; ws.Cell(iRow, iCol).Value = "Status Code"; iCol++; ws.Cell(iRow, iCol).Value = "Status"; iCol++; ws.Cell(iRow, iCol).Value = "Redirect"; iCol++; ws.Cell(iRow, iCol).Value = "Duration"; iCol++; ws.Cell(iRow, iCol).Value = "Crawled Date"; iCol++; ws.Cell(iRow, iCol).Value = "Server Date"; iCol++; ws.Cell(iRow, iCol).Value = "Modified Date"; iCol++; ws.Cell(iRow, iCol).Value = "Expires Date"; iCol++; ws.Cell(iRow, iCol).Value = "Content-Type"; iCol++; ws.Cell(iRow, iCol).Value = "Locale"; iCol++; ws.Cell(iRow, iCol).Value = "Language"; iCol++; ws.Cell(iRow, iCol).Value = "Canonical"; iCol++; ws.Cell(iRow, iCol).Value = "Page Depth"; iCol++; ws.Cell(iRow, iCol).Value = "Links In"; iCol++; ws.Cell(iRow, iCol).Value = "Links Out"; iCol++; ws.Cell(iRow, iCol).Value = "Hyperlinks In"; iCol++; ws.Cell(iRow, iCol).Value = "Hyperlinks Out"; iCol++; ws.Cell(iRow, iCol).Value = "Title"; iCol++; ws.Cell(iRow, iCol).Value = "Title Length"; iCol++; ws.Cell(iRow, iCol).Value = "Description"; iCol++; ws.Cell(iRow, iCol).Value = "Description Length"; iCol++; ws.Cell(iRow, iCol).Value = "Error Condition"; for (int i = 1; i <= iCol; i++) { ws.Cell(iRow, i).Style.Font.SetBold(); } } iColMax = iCol; iRow++; foreach (string sKey in DocCollection.DocumentKeys()) { iCol = 1; MacroscopeDocument msDoc = DocCollection.GetDocument(sKey); this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc); iCol++; this.InsertAndFormatStatusCodeCell(ws, iRow, iCol, msDoc); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetStatusCode().ToString())); iCol++; this.InsertAndFormatRedirectCell(ws, iRow, iCol, msDoc); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.GetDurationInSecondsFormatted()); iCol++; this.InsertAndFormatDateCell(ws, iRow, iCol, msDoc.GetCrawledDate()); iCol++; this.InsertAndFormatDateCell(ws, iRow, iCol, msDoc.GetDateServer()); iCol++; this.InsertAndFormatDateCell(ws, iRow, iCol, msDoc.GetDateModified()); iCol++; this.InsertAndFormatDateCell(ws, iRow, iCol, msDoc.GetDateExpires()); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetMimeType())); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetLocale())); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetIsoLanguageCode())); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetCanonical())); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.GetDepth().ToString()); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.CountInlinks()); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.CountOutlinks()); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.CountHyperlinksIn()); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.CountHyperlinksOut()); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetTitle())); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.GetTitleLength().ToString()); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetDescription())); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.GetDescriptionLength()); iCol++; this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetErrorCondition())); iRow++; } { var rangeData = ws.Range(1, 1, iRow - 1, iColMax); var excelTable = rangeData.CreateTable(); } }
/**************************************************************************/ private void BuildWorksheetOverview( MacroscopeJobMaster JobMaster, CsvWriter ws ) { MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection(); { ws.WriteField("URL"); ws.WriteField("Status Code"); ws.WriteField("Status"); ws.WriteField("Redirect"); ws.WriteField("Duration"); ws.WriteField("Crawled Date"); ws.WriteField("Server Date"); ws.WriteField("Modified Date"); ws.WriteField("Expires Date"); ws.WriteField("Content-Type"); ws.WriteField("Locale"); ws.WriteField("Language"); ws.WriteField("Canonical"); ws.WriteField("Page Depth"); ws.WriteField("Links In"); ws.WriteField("Links Out"); ws.WriteField("Hyperlinks In"); ws.WriteField("Hyperlinks Out"); ws.WriteField("Title"); ws.WriteField("Title Length"); ws.WriteField("Description"); ws.WriteField("Description Length"); ws.WriteField("Error Condition"); ws.NextRecord(); } foreach (string Key in DocCollection.DocumentKeys()) { MacroscopeDocument msDoc = DocCollection.GetDocument(Key); this.InsertAndFormatUrlCell(ws, msDoc); this.InsertAndFormatStatusCodeCell(ws, msDoc); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetStatusCode().ToString())); this.InsertAndFormatRedirectCell(ws, msDoc); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDurationInSecondsFormatted())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetCrawledDate())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDateServer())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDateModified())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDateExpires())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetMimeType())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetLocale())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetIsoLanguageCode())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetCanonical())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDepth().ToString())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.CountInlinks().ToString())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.CountOutlinks().ToString())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.CountHyperlinksIn().ToString())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.CountHyperlinksOut().ToString())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetTitle())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetTitleLength().ToString())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDescription())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDescriptionLength().ToString())); this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetErrorCondition())); ws.NextRecord(); } }