/** Render One ************************************************************/
protected override void RenderListView(
List <ListViewItem> ListViewItems,
MacroscopeDocumentCollection DocCollection,
MacroscopeDocument msDoc,
string Url
)
{
lock (this.DisplayListViewLock)
{
ListViewItem lvItem = null;
if (this.DisplayListView.Items.ContainsKey(Url))
{
lvItem = this.DisplayListView.Items[Url];
}
if (lvItem != null)
{
int ColIndexInlinks = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.Inlinks);
int ColIndexOutlinks = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.Outlinks);
int ColIndexInhyperlinks = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.HyperlinksIn);
int ColIndexOuthyperlinks = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.HyperlinksOut);
lvItem.SubItems[ColIndexInlinks].Text = msDoc.CountInlinks().ToString();
lvItem.SubItems[ColIndexOutlinks].Text = msDoc.CountOutlinks().ToString();
lvItem.SubItems[ColIndexInhyperlinks].Text = msDoc.CountHyperlinksIn().ToString();
lvItem.SubItems[ColIndexOuthyperlinks].Text = msDoc.CountHyperlinksOut().ToString();
}
}
}
/** Render One ************************************************************/
protected override void RenderListView(
List <ListViewItem> ListViewItems,
MacroscopeDocumentCollection DocCollection,
MacroscopeDocument msDoc,
string Url
)
{
lock (this.DisplayListViewLock)
{
Dictionary <string, string> StructureItems = new Dictionary <string, string>();
ListViewItem lvItem = null;
string TitleLanguage = msDoc.GetTitleLanguage();
string DescriptionLanguage = msDoc.GetDescriptionLanguage();
string BodyTextLanguage = msDoc.GetDocumentTextLanguage();
int StatusCode = (int)msDoc.GetStatusCode();
string PairKey = UrlToDigest(Url).ToString();
if (string.IsNullOrEmpty(TitleLanguage))
{
TitleLanguage = "";
}
if (string.IsNullOrEmpty(DescriptionLanguage))
{
DescriptionLanguage = "";
}
if (string.IsNullOrEmpty(BodyTextLanguage))
{
BodyTextLanguage = "";
}
// BEGIN: Columns ----------------------------------------------------//
StructureItems.Add(MacroscopeConstants.Url, msDoc.GetUrl());
StructureItems.Add(MacroscopeConstants.StatusCode, StatusCode.ToString());
StructureItems.Add(MacroscopeConstants.Status, msDoc.GetStatusCode().ToString());
StructureItems.Add(MacroscopeConstants.IsRedirect, msDoc.GetIsRedirect().ToString());
StructureItems.Add(MacroscopeConstants.RobotsRule, msDoc.GetAllowedByRobotsAsString());
StructureItems.Add(MacroscopeConstants.Duration, msDoc.GetDurationInSecondsFormatted());
StructureItems.Add(MacroscopeConstants.ContentType, msDoc.GetMimeType());
{
string Charset = msDoc.GetCharacterSet();
if (string.IsNullOrEmpty(Charset))
{
Charset = "";
}
StructureItems.Add(MacroscopeConstants.Charset, Charset);
}
{
string LocaleCode = msDoc.GetLocale();
if (string.IsNullOrEmpty(LocaleCode))
{
LocaleCode = "";
}
StructureItems.Add(MacroscopeConstants.Locale, LocaleCode);
}
{
string LanguageCode = msDoc.GetIsoLanguageCode();
if (string.IsNullOrEmpty(LanguageCode))
{
LanguageCode = "";
}
StructureItems.Add(MacroscopeConstants.Language, LanguageCode);
}
StructureItems.Add(MacroscopeConstants.DateCrawled, msDoc.GetCrawledDate());
StructureItems.Add(MacroscopeConstants.DateServer, msDoc.GetDateServer());
StructureItems.Add(MacroscopeConstants.DateModified, msDoc.GetDateModified());
StructureItems.Add(MacroscopeConstants.DateExpires, msDoc.GetDateExpires());
StructureItems.Add(MacroscopeConstants.Canonical, msDoc.GetCanonical());
StructureItems.Add(MacroscopeConstants.PageDepth, msDoc.GetDepth().ToString());
StructureItems.Add(MacroscopeConstants.Inlinks, msDoc.CountInlinks().ToString());
StructureItems.Add(MacroscopeConstants.Outlinks, msDoc.CountOutlinks().ToString());
StructureItems.Add(MacroscopeConstants.HyperlinksIn, msDoc.CountHyperlinksIn().ToString());
StructureItems.Add(MacroscopeConstants.HyperlinksOut, msDoc.CountHyperlinksOut().ToString());
// TODO: This is too slow:
/*
* {
* List<decimal> HyperlinkRatio = DocCollection.GetDocumentHyperlinksRatio( Url: Url );
* StructureItems.Add( MacroscopeConstants.HyperlinksInRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 0 ] ) );
* StructureItems.Add( MacroscopeConstants.HyperlinksOutRatio, string.Format( "{0:0.00}%", HyperlinkRatio[ 1 ] ) );
* }
*/
StructureItems.Add(MacroscopeConstants.Author, msDoc.GetAuthor());
StructureItems.Add(MacroscopeConstants.Title, msDoc.GetTitle());
StructureItems.Add(MacroscopeConstants.TitleLen, msDoc.GetTitleLength().ToString());
StructureItems.Add(MacroscopeConstants.TitleLang, TitleLanguage);
StructureItems.Add(MacroscopeConstants.Description, msDoc.GetDescription());
StructureItems.Add(MacroscopeConstants.DescriptionLen, msDoc.GetDescriptionLength().ToString());
StructureItems.Add(MacroscopeConstants.DescriptionLang, DescriptionLanguage);
StructureItems.Add(MacroscopeConstants.Keywords, msDoc.GetKeywords());
StructureItems.Add(MacroscopeConstants.KeywordsLen, msDoc.GetKeywordsLength().ToString());
StructureItems.Add(MacroscopeConstants.KeywordsCount, msDoc.GetKeywordsCount().ToString());
StructureItems.Add(MacroscopeConstants.BodyTextLang, BodyTextLanguage);
for (ushort HeadingLevel = 1; HeadingLevel <= MaxHeadingsDisplayed; HeadingLevel++)
{
List <string> HeadingList = msDoc.GetHeadings(HeadingLevel: HeadingLevel);
string HeadingText = "";
if (HeadingList.Count > 0)
{
HeadingText = HeadingList[0];
}
StructureItems.Add(string.Format(MacroscopeConstants.Hn, HeadingLevel), HeadingText);
}
StructureItems.Add(MacroscopeConstants.ErrorCondition, msDoc.GetErrorCondition());
// END: Columns ------------------------------------------------------//
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
lvItem = this.DisplayListView.Items[PairKey];
}
else
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
for (int i = 0; i < this.DisplayListView.Columns.Count; i++)
{
lvItem.SubItems.Add("");
}
ListViewItems.Add(lvItem);
}
if (lvItem != null)
{
lvItem.ForeColor = Color.Blue;
int StatusCodeColIndex = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.StatusCode);
int StatusColIndex = this.DisplayListView.Columns.IndexOfKey(MacroscopeConstants.Status);
foreach (string ItemsKey in StructureItems.Keys)
{
int ColIndex = this.DisplayListView.Columns.IndexOfKey(ItemsKey);
string Text = StructureItems[ItemsKey];
if (!string.IsNullOrEmpty(StructureItems[ItemsKey]))
{
lvItem.SubItems[ColIndex].Text = Text;
}
else
{
lvItem.SubItems[ColIndex].Text = "";
}
if (msDoc.GetIsInternal())
{
lvItem.SubItems[ColIndex].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColIndex].ForeColor = Color.Gray;
}
if (ItemsKey.Equals(MacroscopeConstants.StatusCode))
{
if ((StatusCode >= 200) && (StatusCode <= 299))
{
lvItem.SubItems[ColIndex].ForeColor = Color.Green;
lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Green;
lvItem.SubItems[StatusColIndex].ForeColor = Color.Green;
}
else
if ((StatusCode >= 300) && (StatusCode <= 399))
{
lvItem.SubItems[ColIndex].ForeColor = Color.Goldenrod;
lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Goldenrod;
lvItem.SubItems[StatusColIndex].ForeColor = Color.Goldenrod;
}
else
if ((StatusCode >= 400) && (StatusCode <= 599))
{
lvItem.SubItems[ColIndex].ForeColor = Color.Red;
lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Red;
lvItem.SubItems[StatusColIndex].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[ColIndex].ForeColor = Color.Blue;
lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Blue;
lvItem.SubItems[StatusColIndex].ForeColor = Color.Blue;
}
if (StatusCode == 410)
{
lvItem.SubItems[ColIndex].ForeColor = Color.Purple;
lvItem.SubItems[StatusCodeColIndex].ForeColor = Color.Purple;
lvItem.SubItems[StatusColIndex].ForeColor = Color.Purple;
}
}
if (ItemsKey == MacroscopeConstants.RobotsRule)
{
if (Text.ToLower() == "disallowed")
{
lvItem.SubItems[ColIndex].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[ColIndex].ForeColor = Color.Green;
}
}
if (ItemsKey == MacroscopeConstants.IsRedirect)
{
if (Text.ToLower() == "true")
{
lvItem.SubItems[ColIndex].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[ColIndex].ForeColor = Color.Gray;
}
}
}
}
else
{
DebugMsg(string.Format("MacroscopeDisplayStructure: {0}", "lvItem is NULL"));
}
}
}
/**************************************************************************/
private void BuildWorksheetOverview(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Status Code";
iCol++;
ws.Cell(iRow, iCol).Value = "Status";
iCol++;
ws.Cell(iRow, iCol).Value = "Redirect";
iCol++;
ws.Cell(iRow, iCol).Value = "Duration";
iCol++;
ws.Cell(iRow, iCol).Value = "Crawled Date";
iCol++;
ws.Cell(iRow, iCol).Value = "Server Date";
iCol++;
ws.Cell(iRow, iCol).Value = "Modified Date";
iCol++;
ws.Cell(iRow, iCol).Value = "Expires Date";
iCol++;
ws.Cell(iRow, iCol).Value = "Content-Type";
iCol++;
ws.Cell(iRow, iCol).Value = "Locale";
iCol++;
ws.Cell(iRow, iCol).Value = "Language";
iCol++;
ws.Cell(iRow, iCol).Value = "Canonical";
iCol++;
ws.Cell(iRow, iCol).Value = "Page Depth";
iCol++;
ws.Cell(iRow, iCol).Value = "Links In";
iCol++;
ws.Cell(iRow, iCol).Value = "Links Out";
iCol++;
ws.Cell(iRow, iCol).Value = "Hyperlinks In";
iCol++;
ws.Cell(iRow, iCol).Value = "Hyperlinks Out";
iCol++;
ws.Cell(iRow, iCol).Value = "Title";
iCol++;
ws.Cell(iRow, iCol).Value = "Title Length";
iCol++;
ws.Cell(iRow, iCol).Value = "Description";
iCol++;
ws.Cell(iRow, iCol).Value = "Description Length";
iCol++;
ws.Cell(iRow, iCol).Value = "Error Condition";
for (int i = 1; i <= iCol; i++)
{
ws.Cell(iRow, i).Style.Font.SetBold();
}
}
iColMax = iCol;
iRow++;
foreach (string sKey in DocCollection.DocumentKeys())
{
iCol = 1;
MacroscopeDocument msDoc = DocCollection.GetDocument(sKey);
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatStatusCodeCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetStatusCode().ToString()));
iCol++;
this.InsertAndFormatRedirectCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.GetDurationInSecondsFormatted());
iCol++;
this.InsertAndFormatDateCell(ws, iRow, iCol, msDoc.GetCrawledDate());
iCol++;
this.InsertAndFormatDateCell(ws, iRow, iCol, msDoc.GetDateServer());
iCol++;
this.InsertAndFormatDateCell(ws, iRow, iCol, msDoc.GetDateModified());
iCol++;
this.InsertAndFormatDateCell(ws, iRow, iCol, msDoc.GetDateExpires());
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetMimeType()));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetLocale()));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetIsoLanguageCode()));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetCanonical()));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.GetDepth().ToString());
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.CountInlinks());
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.CountOutlinks());
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.CountHyperlinksIn());
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.CountHyperlinksOut());
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetTitle()));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.GetTitleLength().ToString());
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetDescription()));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, msDoc.GetDescriptionLength());
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(msDoc.GetErrorCondition()));
iRow++;
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetOverview(
MacroscopeJobMaster JobMaster,
CsvWriter ws
)
{
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
{
ws.WriteField("URL");
ws.WriteField("Status Code");
ws.WriteField("Status");
ws.WriteField("Redirect");
ws.WriteField("Duration");
ws.WriteField("Crawled Date");
ws.WriteField("Server Date");
ws.WriteField("Modified Date");
ws.WriteField("Expires Date");
ws.WriteField("Content-Type");
ws.WriteField("Locale");
ws.WriteField("Language");
ws.WriteField("Canonical");
ws.WriteField("Page Depth");
ws.WriteField("Links In");
ws.WriteField("Links Out");
ws.WriteField("Hyperlinks In");
ws.WriteField("Hyperlinks Out");
ws.WriteField("Title");
ws.WriteField("Title Length");
ws.WriteField("Description");
ws.WriteField("Description Length");
ws.WriteField("Error Condition");
ws.NextRecord();
}
foreach (string Key in DocCollection.DocumentKeys())
{
MacroscopeDocument msDoc = DocCollection.GetDocument(Key);
this.InsertAndFormatUrlCell(ws, msDoc);
this.InsertAndFormatStatusCodeCell(ws, msDoc);
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetStatusCode().ToString()));
this.InsertAndFormatRedirectCell(ws, msDoc);
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDurationInSecondsFormatted()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetCrawledDate()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDateServer()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDateModified()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDateExpires()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetMimeType()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetLocale()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetIsoLanguageCode()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetCanonical()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDepth().ToString()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.CountInlinks().ToString()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.CountOutlinks().ToString()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.CountHyperlinksIn().ToString()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.CountHyperlinksOut().ToString()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetTitle()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetTitleLength().ToString()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDescription()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetDescriptionLength().ToString()));
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(msDoc.GetErrorCondition()));
ws.NextRecord();
}
}
