/**************************************************************************/
protected override void RenderListView(
List <ListViewItem> ListViewItems,
MacroscopeDocumentCollection DocCollection,
MacroscopeDocument msDoc,
string Url
)
{
bool Proceed = false;
if (msDoc.GetIsExternal())
{
return;
}
if (msDoc.GetIsRedirect())
{
return;
}
if (msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML))
{
Proceed = true;
}
if (Proceed)
{
for (ushort HeadingLevel = 1; HeadingLevel <= MacroscopePreferencesManager.GetMaxHeadingDepth(); HeadingLevel++)
{
List <string> HeadingsList = msDoc.GetHeadings(HeadingLevel);
for (int Order = 0; Order < HeadingsList.Count; Order++)
{
ListViewItem lvItem = null;
string PairKey = string.Join(":", UrlToDigest(Url), UrlToDigest(HeadingLevel.ToString()), UrlToDigest(Order.ToString())).ToString();
int HeadingColIndex = HeadingLevel + ColH1Offset;
string TextLabel = HeadingsList[Order];
int Occurences = DocCollection.GetStatsHeadingsCount(HeadingLevel: HeadingLevel, Text: TextLabel);
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
try
{
lvItem = this.DisplayListView.Items[PairKey];
lvItem.SubItems[ColUrl].Text = Url;
lvItem.SubItems[ColOccurences].Text = Occurences.ToString();
lvItem.SubItems[ColOrder].Text = (Order + 1).ToString();
lvItem.SubItems[HeadingColIndex].Text = TextLabel;
}
catch (Exception ex)
{
DebugMsg(string.Format("MacroscopeDisplayHeadings 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
lvItem.SubItems[ColUrl].Text = Url;
lvItem.SubItems.Add(Occurences.ToString());
lvItem.SubItems.Add((Order + 1).ToString());
for (ushort k = 1; k <= 6; k++)
{
lvItem.SubItems.Add("");
}
lvItem.SubItems[HeadingColIndex].Text = TextLabel;
ListViewItems.Add(lvItem);
}
catch (Exception ex)
{
DebugMsg(string.Format("MacroscopeDisplayHeadings 2: {0}", ex.Message));
}
}
if (lvItem != null)
{
lvItem.ForeColor = Color.Blue;
// URL -----------------------------------------------------------//
if (msDoc.GetIsInternal())
{
lvItem.SubItems[ColUrl].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColUrl].ForeColor = Color.Gray;
}
// Occurences ----------------------------------------------------//
if ((Occurences > 1) && (msDoc.GetIsInternal()))
{
lvItem.SubItems[ColOccurences].ForeColor = Color.Orange;
}
else
if (msDoc.GetIsInternal())
{
lvItem.SubItems[ColOccurences].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColOccurences].ForeColor = Color.Gray;
}
// Check Missing H1 ----------------------------------------------//
if ((HeadingLevel == 1) && string.IsNullOrEmpty(TextLabel))
{
lvItem.SubItems[HeadingColIndex].Text = "MISSING";
lvItem.SubItems[HeadingColIndex].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[HeadingColIndex].ForeColor = Color.Green;
}
}
}
}
}
}
/**************************************************************************/
private void BuildWorksheetPageHeadings(
MacroscopeJobMaster JobMaster,
CsvWriter ws
)
{
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
{
ws.WriteField("URL");
ws.WriteField("Occurences");
ws.WriteField("Order");
for (int i = 1; i <= 6; i++)
{
ws.WriteField(string.Format("H{0}", i));
}
ws.NextRecord();
}
foreach (string Url in DocCollection.DocumentKeys())
{
MacroscopeDocument msDoc = DocCollection.GetDocument(Url);
Boolean Proceed = false;
if (msDoc.GetIsExternal())
{
continue;
}
if (msDoc.GetIsRedirect())
{
continue;
}
if (msDoc.GetIsHtml())
{
Proceed = true;
}
if (Proceed)
{
for (ushort HeadingLevel = 1; HeadingLevel <= MacroscopePreferencesManager.GetMaxHeadingDepth(); HeadingLevel++)
{
List <string> HeadingsList = msDoc.GetHeadings(HeadingLevel);
for (int Order = 0; Order < HeadingsList.Count; Order++)
{
int Occurences = DocCollection.GetStatsHeadingsCount(HeadingLevel: HeadingLevel, Text: HeadingsList[Order]);
this.InsertAndFormatUrlCell(ws, msDoc);
this.InsertAndFormatContentCell(ws, Occurences.ToString());
this.InsertAndFormatContentCell(ws, this.FormatIfMissing((Order + 1).ToString()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(HeadingsList[Order]));
ws.NextRecord();
}
}
}
}
}
/** Render One ************************************************************/
protected override void RenderListView(
List <ListViewItem> ListViewItems,
MacroscopeDocumentCollection DocCollection,
MacroscopeDocument msDoc,
string Url
)
{
lock (this.DisplayListViewLock)
{
Dictionary <string, string> StructureItems = new Dictionary <string, string>();
ListViewItem lvItem = null;
string TitleLanguage = msDoc.GetTitleLanguage();
string DescriptionLanguage = msDoc.GetDescriptionLanguage();
string BodyTextLanguage = msDoc.GetDocumentTextLanguage();
int StatusCode = (int)msDoc.GetStatusCode();
string PairKey = UrlToDigest(Url).ToString();
if (string.IsNullOrEmpty(TitleLanguage))
{
TitleLanguage = "";
}
if (string.IsNullOrEmpty(DescriptionLanguage))
{
DescriptionLanguage = "";
}
if (string.IsNullOrEmpty(BodyTextLanguage))
{
BodyTextLanguage = "";
}
// BEGIN: Columns ----------------------------------------------------//
StructureItems.Add(MacroscopeConstants.Url, msDoc.GetUrl());
StructureItems.Add(MacroscopeConstants.StatusCode, StatusCode.ToString());
StructureItems.Add(MacroscopeConstants.Status, msDoc.GetStatusCode().ToString());
StructureItems.Add(MacroscopeConstants.IsRedirect, msDoc.GetIsRedirect().ToString());
StructureItems.Add(MacroscopeConstants.RobotsRule, msDoc.GetAllowedByRobotsAsString());
StructureItems.Add(MacroscopeConstants.Duration, msDoc.GetDurationInSecondsFormatted());
StructureItems.Add(MacroscopeConstants.ContentType, msDoc.GetMimeType());
{
string Charset = msDoc.GetCharacterSet();
if (string.IsNullOrEmpty(Charset))
{
Charset = "";
}
StructureItems.Add(MacroscopeConstants.Charset, Charset);
}
{
string LocaleCode = msDoc.GetLocale();
if (string.IsNullOrEmpty(LocaleCode))
{
LocaleCode = "";
}
StructureItems.Add(MacroscopeConstants.Locale, LocaleCode);
}
{
string LanguageCode = msDoc.GetIsoLanguageCode();
if (string.IsNullOrEmpty(LanguageCode))
{
LanguageCode = "";
}
StructureItems.Add(MacroscopeConstants.Language, LanguageCode);
}
StructureItems.Add(MacroscopeConstants.DateCrawled, msDoc.GetCrawledDate());
StructureItems.Add(MacroscopeConstants.DateServer, msDoc.GetDateServer());
StructureItems.Add(MacroscopeConstants.DateModified, msDoc.GetDateModified());
StructureItems.Add(MacroscopeConstants.DateExpires, msDoc.GetDateExpires());
StructureItems.Add(MacroscopeConstants.Canonical, msDoc.GetCanonical());
StructureItems.Add(MacroscopeConstants.PageDepth, msDoc.GetDepth().ToString());
StructureItems.Add(MacroscopeConstants.Inlinks, msDoc.CountInlinks().ToString());
StructureItems.Add(MacroscopeConstants.Outlinks, msDoc.CountOutlinks().ToString());
StructureItems.Add(MacroscopeConstants.HyperlinksIn, msDoc.CountHyperlinksIn().ToString());
StructureItems.Add(MacroscopeConstants.HyperlinksOut, msDoc.CountHyperlinksOut().ToString());
// TODO: This is too slow:
/*
* {
* List<decimal> HyperlinkRatio = DocCollection.GetDocumentHyperlinksRatio( Url: Url );
* StructureItems.Add( MacroscopeConstants.HyperlinksInRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 0 ] ) );
* StructureItems.Add( MacroscopeConstants.HyperlinksOutRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 1 ] ) );
* }
*/
StructureItems.Add(MacroscopeConstants.Author, msDoc.GetAuthor());
StructureItems.Add(MacroscopeConstants.Title, msDoc.GetTitle());
StructureItems.Add(MacroscopeConstants.TitleLen, msDoc.GetTitleLength().ToString());
StructureItems.Add(MacroscopeConstants.TitleLang, TitleLanguage);
StructureItems.Add(MacroscopeConstants.Description, msDoc.GetDescription());
StructureItems.Add(MacroscopeConstants.DescriptionLen, msDoc.GetDescriptionLength().ToString());
StructureItems.Add(MacroscopeConstants.DescriptionLang, DescriptionLanguage);
StructureItems.Add(MacroscopeConstants.Keywords, msDoc.GetKeywords());
StructureItems.Add(MacroscopeConstants.KeywordsLen, msDoc.GetKeywordsLength().ToString());
StructureItems.Add(MacroscopeConstants.KeywordsCount, msDoc.GetKeywordsCount().ToString());
StructureItems.Add(MacroscopeConstants.BodyTextLang, BodyTextLanguage);
for (ushort HeadingLevel = 1; HeadingLevel <= MaxHeadingsDisplayed; HeadingLevel++)
{
List <string> HeadingList = msDoc.GetHeadings(HeadingLevel: HeadingLevel);
string HeadingText = "";
if (HeadingList.Count > 0)
{
HeadingText = HeadingList[0];
}
StructureItems.Add(string.Format(MacroscopeConstants.Hn, HeadingLevel), HeadingText);
}
StructureItems.Add(MacroscopeConstants.ErrorCondition, msDoc.GetErrorCondition());
// END: Columns ------------------------------------------------------//
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
lvItem = this.DisplayListView.Items[PairKey];
}
else
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
for (int i = 0; i < this.DisplayListView.Columns.Count; i++)
{
lvItem.SubItems.Add("");
}
ListViewItems.Add(lvItem);
}
if (lvItem != null)
{
lvItem.ForeColor = Color.Blue;
int StatusCodeColIndex = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.StatusCode);
int StatusColIndex = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.Status);
foreach (string ItemsKey in StructureItems.Keys)
{
int ColIndex = this.DisplayListView.Columns.IndexOfKey(ItemsKey);
string Text = StructureItems[ItemsKey];
if (!string.IsNullOrEmpty(StructureItems[ItemsKey]))
{
lvItem.SubItems[ColIndex].Text = Text;
}
else
{
lvItem.SubItems[ColIndex].Text = "";
}
if (msDoc.GetIsInternal())
{
lvItem.SubItems[ColIndex].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColIndex].ForeColor = Color.Gray;
}
if (ItemsKey.Equals(MacroscopeConstants.StatusCode))
{
if ((StatusCode >= 200) && (StatusCode <= 299))
{
lvItem.SubItems[ColIndex].ForeColor = Color.Green;
lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Green;
lvItem.SubItems[StatusColIndex].ForeColor = Color.Green;
}
else
if ((StatusCode >= 300) && (StatusCode <= 399))
{
lvItem.SubItems[ColIndex].ForeColor = Color.Goldenrod;
lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Goldenrod;
lvItem.SubItems[StatusColIndex].ForeColor = Color.Goldenrod;
}
else
if ((StatusCode >= 400) && (StatusCode <= 599))
{
lvItem.SubItems[ColIndex].ForeColor = Color.Red;
lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Red;
lvItem.SubItems[StatusColIndex].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[ColIndex].ForeColor = Color.Blue;
lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Blue;
lvItem.SubItems[StatusColIndex].ForeColor = Color.Blue;
}
if (StatusCode == 410)
{
lvItem.SubItems[ColIndex].ForeColor = Color.Purple;
lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Purple;
lvItem.SubItems[StatusColIndex].ForeColor = Color.Purple;
}
}
if (ItemsKey == MacroscopeConstants.RobotsRule)
{
if (Text.ToLower() == "disallowed")
{
lvItem.SubItems[ColIndex].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[ColIndex].ForeColor = Color.Green;
}
}
if (ItemsKey == MacroscopeConstants.IsRedirect)
{
if (Text.ToLower() == "true")
{
lvItem.SubItems[ColIndex].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[ColIndex].ForeColor = Color.Gray;
}
}
}
}
else
{
DebugMsg(string.Format("MacroscopeDisplayStructure: {0}", "lvItem is NULL"));
}
}
}
/**************************************************************************/
private void BuildWorksheetPageHeadings(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Occurences";
iCol++;
ws.Cell(iRow, iCol).Value = "Order";
for (int i = 1; i <= 6; i++)
{
iCol++;
ws.Cell(iRow, iCol).Value = string.Format("H{0}", i);
}
}
iColMax = iCol;
iRow++;
foreach (string Url in DocCollection.DocumentKeys())
{
MacroscopeDocument msDoc = DocCollection.GetDocument(Url);
Boolean Proceed = false;
if (msDoc.GetIsExternal())
{
continue;
}
if (msDoc.GetIsRedirect())
{
continue;
}
if (msDoc.GetIsHtml())
{
Proceed = true;
}
if (Proceed)
{
for (ushort HeadingLevel = 1; HeadingLevel <= MacroscopePreferencesManager.GetMaxHeadingDepth(); HeadingLevel++)
{
List <string> HeadingsList = msDoc.GetHeadings(HeadingLevel);
for (int Order = 0; Order < HeadingsList.Count; Order++)
{
int Occurences = DocCollection.GetStatsHeadingsCount(HeadingLevel: HeadingLevel, Text: HeadingsList[Order]);
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (msDoc.GetIsInternal())
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Occurences);
if ((Occurences > 1) && (msDoc.GetIsInternal()))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Orange);
}
else
if (msDoc.GetIsInternal())
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing((Order + 1).ToString()));
this.InsertAndFormatContentCell(ws, iRow, ( int )(HeadingLevel + iCol), this.FormatIfMissing(HeadingsList[Order]));
iRow++;
}
}
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
