/**************************************************************************/
private void BuildWorksheetBlockedByRobotsInternal(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Status";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
if (msDoc.GetIsInternal() && (!msDoc.GetAllowedByRobots()))
{
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (AllowedHosts.IsInternalUrl(Url: msDoc.GetUrl()))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatRobotsCell(ws, iRow, iCol, msDoc);
iRow++;
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetPageLinks(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Link Type";
iCol++;
ws.Cell(iRow, iCol).Value = "Source URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Target URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Follow";
iCol++;
ws.Cell(iRow, iCol).Value = "Alt Text";
iCol++;
ws.Cell(iRow, iCol).Value = "Raw Source URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Raw Target URL";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
foreach (MacroscopeLink Link in msDoc.IterateOutlinks())
{
string LinkType = Link.GetLinkType().ToString();
string SourceUrl = Link.GetSourceUrl();
string TargetUrl = Link.GetTargetUrl();
string AltText = Link.GetAltText();
string RawSourceUrl = Link.GetRawSourceUrl();
string RawTargetUrl = Link.GetRawTargetUrl();
string DoFollow = "No Follow";
if (Link.GetDoFollow())
{
DoFollow = "Follow";
}
if (string.IsNullOrEmpty(AltText))
{
AltText = "";
}
if (string.IsNullOrEmpty(RawSourceUrl))
{
RawSourceUrl = "";
}
if (string.IsNullOrEmpty(RawTargetUrl))
{
RawTargetUrl = "";
}
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (AllowedHosts.IsInternalUrl(Url: msDoc.GetUrl()))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(LinkType));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(SourceUrl));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(TargetUrl));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(DoFollow));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(AltText));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(RawSourceUrl));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(RawTargetUrl));
iRow++;
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetErrors(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "Status Code";
iCol++;
ws.Cell(iRow, iCol).Value = "Status";
iCol++;
ws.Cell(iRow, iCol).Value = "URL";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
MacroscopeHyperlinksIn HyperlinksIn = DocCollection.GetDocumentHyperlinksIn(msDoc.GetUrl());
int StatusCode = ( int )msDoc.GetStatusCode();
string Status = msDoc.GetStatusCode().ToString();
if (
(StatusCode >= 400) &&
(StatusCode <= 599))
{
iCol = 1;
this.InsertAndFormatContentCell(ws, iRow, iCol, StatusCode.ToString());
if ((StatusCode >= 400) && (StatusCode <= 599))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Blue);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Status);
if ((StatusCode >= 400) && (StatusCode <= 599))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Blue);
}
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc.GetUrl());
if (AllowedHosts.IsInternalUrl(Url: msDoc.GetUrl()))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iRow++;
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetPageRedirectsAudit(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "Origin URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Status Code";
iCol++;
ws.Cell(iRow, iCol).Value = "Status";
iCol++;
ws.Cell(iRow, iCol).Value = "Destination URL";
}
iColMax = iCol;
iRow++;
foreach (string Url in DocCollection.DocumentKeys())
{
MacroscopeDocument msDoc = DocCollection.GetDocument(Url);
if (!msDoc.GetIsRedirect())
{
continue;
}
string OriginURL = msDoc.GetUrlRedirectFrom();
string StatusCode = (( int )msDoc.GetStatusCode()).ToString();
string Status = msDoc.GetStatusCode().ToString();
string DestinationURL = msDoc.GetUrlRedirectTo();
if (OriginURL == null)
{
continue;
}
if (DestinationURL == null)
{
continue;
}
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, OriginURL);
if (AllowedHosts.IsInternalUrl(Url: OriginURL))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, StatusCode);
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Status);
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, DestinationURL);
if (AllowedHosts.IsInternalUrl(Url: DestinationURL))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iRow++;
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void RenderListView(
MacroscopeDocumentCollection DocCollection,
Dictionary <string, string> LocalesList
)
{
MacroscopeAllowedHosts AllowedHosts = this.MainForm.GetJobMaster().GetAllowedHosts();
SortedDictionary <string, int> LocaleColsTable = new SortedDictionary <string, int> ();
if (DocCollection.CountDocuments() == 0)
{
return;
}
List <ListViewItem> ListViewItems = new List <ListViewItem> ();
MacroscopeSinglePercentageProgressForm ProgressForm = new MacroscopeSinglePercentageProgressForm(this.MainForm);
decimal Count = 0;
decimal TotalDocs = ( decimal )DocCollection.CountDocuments();
decimal MajorPercentage = (( decimal )100 / TotalDocs) * Count;
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
ProgressForm.UpdatePercentages(
Title: "Preparing Display",
Message: "Processing document collection for display:",
MajorPercentage: MajorPercentage,
ProgressLabelMajor: string.Format("Document {0} / {1}", Count, TotalDocs)
);
}
this.DisplayListView.Items.Clear();
this.DisplayListView.Columns.Clear();
{
int LocaleColCount = 5;
this.DisplayListView.Columns.Add("URL", "URL");
this.DisplayListView.Columns.Add("Status Code", "Status Code");
this.DisplayListView.Columns.Add("Site Locale", "Site Locale");
this.DisplayListView.Columns.Add("HrefLang Present", "HrefLang Present");
this.DisplayListView.Columns.Add("Title", "Title");
foreach (string Locale in LocalesList.Keys)
{
string LocaleLabel = Locale.ToUpper();
string DateServerLabel = string.Format("{0} Date Server", Locale.ToUpper());
string DateModifiedLabel = string.Format("{0} Date Modified", Locale.ToUpper());
this.DisplayListView.Columns.Add(LocaleLabel, LocaleLabel);
this.DisplayListView.Columns.Add(DateServerLabel, DateServerLabel);
this.DisplayListView.Columns.Add(DateModifiedLabel, DateModifiedLabel);
LocaleColsTable[Locale] = LocaleColCount;
LocaleColCount++;
LocaleColsTable[DateServerLabel] = LocaleColCount;
LocaleColCount++;
LocaleColsTable[DateModifiedLabel] = LocaleColCount;
LocaleColCount++;
}
}
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
bool Proceed = false;
if (msDoc.GetIsInternal())
{
Proceed = true;
if (msDoc.GetIsRedirect())
{
Proceed = false;
}
if (!msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML))
{
Proceed = false;
}
}
if (Proceed)
{
Dictionary <string, MacroscopeHrefLang> HrefLangsTable = msDoc.GetHrefLangs();
string DocUrl = msDoc.GetUrl();
string PairKey = UrlToDigest(DocUrl).ToString();
HttpStatusCode StatusCode = msDoc.GetStatusCode();
int StatusCodeNum = ( int )StatusCode;
MacroscopeConstants.Specifiers HrefLangPresent = MacroscopeConstants.Specifiers.UNSPECIFIED;
string DocLocale = msDoc.GetLocale();
string DocTitle = msDoc.GetTitle();
ListViewItem lvItem = null;
if (
(HrefLangsTable != null) &&
(HrefLangsTable.Count > 1))
{
HrefLangPresent = MacroscopeConstants.Specifiers.SPECIFIED;
}
else
{
HrefLangPresent = MacroscopeConstants.Specifiers.UNSPECIFIED;
}
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
lvItem = this.DisplayListView.Items[PairKey];
}
else
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
lvItem.SubItems.Add("");
lvItem.SubItems.Add("");
lvItem.SubItems.Add("");
lvItem.SubItems.Add("");
lvItem.SubItems.Add("");
for (int i = 0; i < LocalesList.Keys.Count; i++)
{
lvItem.SubItems.Add("");
lvItem.SubItems.Add("");
lvItem.SubItems.Add("");
}
ListViewItems.Add(lvItem);
}
if (lvItem != null)
{
try
{
lvItem.SubItems[ColUrl].Text = DocUrl;
lvItem.SubItems[ColStatusCode].Text = StatusCode.ToString();
lvItem.SubItems[ColSiteLocale].Text = DocLocale;
lvItem.SubItems[ColHrefLangPresent].Text = "";
lvItem.SubItems[ColTitle].Text = DocTitle;
switch (HrefLangPresent)
{
case MacroscopeConstants.Specifiers.SPECIFIED:
lvItem.SubItems[ColHrefLangPresent].ForeColor = Color.Green;
lvItem.SubItems[ColHrefLangPresent].Text = "SPECIFIED";
break;
default:
lvItem.SubItems[ColHrefLangPresent].ForeColor = Color.Red;
lvItem.SubItems[ColHrefLangPresent].Text = "UNSPECIFIED";
break;
}
if (AllowedHosts.IsInternalUrl(DocUrl))
{
lvItem.SubItems[ColUrl].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColUrl].ForeColor = Color.Gray;
}
if ((StatusCodeNum >= 100) && (StatusCodeNum <= 299))
{
lvItem.SubItems[ColStatusCode].ForeColor = Color.Green;
}
else
if ((StatusCodeNum >= 300) && (StatusCodeNum <= 399))
{
lvItem.SubItems[ColStatusCode].ForeColor = Color.Orange;
}
else
if ((StatusCodeNum >= 400) && (StatusCodeNum <= 599))
{
lvItem.SubItems[ColStatusCode].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[ColSiteLocale].ForeColor = Color.Gray;
}
foreach (string Locale in LocalesList.Keys)
{
if (!string.IsNullOrEmpty(Locale))
{
string HrefLangUrl = null;
DateTime HrefLangDateServer = new DateTime();
DateTime HrefLangDateModified = new DateTime();
int LocaleCol = LocaleColsTable[Locale];
if (
(HrefLangsTable != null) &&
(HrefLangsTable.Count > 0))
{
if (HrefLangsTable.ContainsKey(Locale))
{
MacroscopeHrefLang HrefLangAlternate = HrefLangsTable[Locale];
if (HrefLangAlternate != null)
{
HrefLangUrl = HrefLangAlternate.GetUrl();
HrefLangDateServer = HrefLangAlternate.GetDateServer();
HrefLangDateModified = HrefLangAlternate.GetDateModified();
}
}
}
if (!string.IsNullOrEmpty(HrefLangUrl))
{
lvItem.SubItems[LocaleCol].ForeColor = Color.Blue;
lvItem.SubItems[LocaleCol].Text = HrefLangUrl;
lvItem.SubItems[LocaleCol + 1].Text = HrefLangDateServer.ToString();
lvItem.SubItems[LocaleCol + 2].Text = HrefLangDateModified.ToString();
}
else
{
lvItem.SubItems[LocaleCol].ForeColor = Color.Red;
lvItem.SubItems[LocaleCol].Text = "NOT SPECIFIED";
lvItem.SubItems[LocaleCol + 1].Text = "NOT SPECIFIED";
lvItem.SubItems[LocaleCol + 2].Text = "NOT SPECIFIED";
}
}
}
}
catch (Exception ex)
{
DebugMsg(string.Format("MacroscopeDisplayHrefLang: {0}", ex.Message));
DebugMsg(string.Format("MacroscopeDisplayHrefLang: {0}", ex.StackTrace));
}
}
else
{
DebugMsg(string.Format("MacroscopeDisplayHrefLang NOT SPECIFIED: {0}", PairKey));
}
}
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
Count++;
MajorPercentage = (( decimal )100 / TotalDocs) * Count;
ProgressForm.UpdatePercentages(
Title: null,
Message: null,
MajorPercentage: MajorPercentage,
ProgressLabelMajor: string.Format("Document {0} / {1}", Count, TotalDocs)
);
}
}
this.DisplayListView.Items.AddRange(ListViewItems.ToArray());
this.DisplayListView.AutoResizeColumns(ColumnHeaderAutoResizeStyle.ColumnContent);
this.DisplayListView.Columns[ColUrl].Width = 300;
this.DisplayListView.Columns[ColStatusCode].Width = 80;
this.DisplayListView.Columns[ColSiteLocale].Width = 100;
this.DisplayListView.Columns[ColTitle].Width = 100;
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
ProgressForm.DoClose();
}
if (ProgressForm != null)
{
ProgressForm.Dispose();
}
}
/**************************************************************************/
private void BuildWorksheetPageDuplicateTitles(
MacroscopeJobMaster JobMaster,
CsvWriter ws
)
{
decimal Count = 0;
decimal DocCount = 0;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
DocCount = ( decimal )DocCollection.CountDocuments();
{
ws.WriteField("URL");
ws.WriteField("Occurrences");
ws.WriteField("Title");
ws.NextRecord();
}
foreach (string Url in DocCollection.DocumentKeys())
{
MacroscopeDocument msDoc = DocCollection.GetDocument(Url);
Boolean Proceed = false;
if (DocCount > 0)
{
Count++;
this.ProgressForm.UpdatePercentages(
Title: null,
Message: null,
MajorPercentage: -1,
ProgressLabelMajor: string.Format("Documents Processed: {0}", Count),
MinorPercentage: (( decimal )100 / DocCount) * Count,
ProgressLabelMinor: Url,
SubMinorPercentage: -1,
ProgressLabelSubMinor: null
);
}
if (AllowedHosts.IsInternalUrl(Url: Url))
{
if (msDoc.GetIsHtml())
{
Proceed = true;
}
else
if (msDoc.GetIsPdf())
{
Proceed = true;
}
else
{
Proceed = false;
}
}
if (Proceed)
{
string Title = msDoc.GetTitle();
int Occurrences = DocCollection.GetStatsTitleCount(msDoc: msDoc);
if (Occurrences > 1)
{
this.InsertAndFormatUrlCell(ws, msDoc);
this.InsertAndFormatContentCell(ws, Occurrences);
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(Title));
ws.NextRecord();
}
}
}
}
/**************************************************************************/
protected override void RenderListView(
List <ListViewItem> ListViewItems,
MacroscopeDocumentCollection DocCollection,
MacroscopeDocument msDoc,
string Url
)
{
if (msDoc.GetIsRedirect())
{
return;
}
if (msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML))
{
MacroscopeAllowedHosts AllowedHosts = this.MainForm.GetJobMaster().GetAllowedHosts();
string Canonical = msDoc.GetCanonical();
HttpStatusCode StatusCode = msDoc.GetStatusCode();
string CanonicalLabel = Canonical;
ListViewItem lvItem = null;
string PairKey = UrlToDigest(Url: Url).ToString();
if (Canonical.Length == 0)
{
CanonicalLabel = "MISSING";
}
if (DisplayListView.Items.ContainsKey(PairKey))
{
try
{
lvItem = DisplayListView.Items[PairKey];
lvItem.SubItems[0].Text = Url;
lvItem.SubItems[1].Text = StatusCode.ToString();
lvItem.SubItems[2].Text = CanonicalLabel;
}
catch (Exception ex)
{
DebugMsg(string.Format("MacroscopeDisplayCanonical 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
lvItem.SubItems[0].Text = Url;
lvItem.SubItems.Add(StatusCode.ToString());
lvItem.SubItems.Add(CanonicalLabel);
DisplayListView.Items.Add(lvItem);
}
catch (Exception ex)
{
DebugMsg(string.Format("MacroscopeDisplayCanonical 2: {0}", ex.Message));
}
}
if (lvItem != null)
{
lvItem.ForeColor = Color.Gray;
if (AllowedHosts.IsInternalUrl(Url))
{
lvItem.SubItems[0].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[0].ForeColor = Color.Gray;
}
if ((( int )StatusCode >= 100) && (( int )StatusCode <= 299))
{
lvItem.SubItems[1].ForeColor = Color.Green;
}
else
if ((( int )StatusCode >= 300) && (( int )StatusCode <= 399))
{
lvItem.SubItems[1].ForeColor = Color.Orange;
}
else
if ((( int )StatusCode >= 400) && (( int )StatusCode <= 599))
{
lvItem.SubItems[1].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[2].ForeColor = Color.Gray;
}
if (Canonical.Length == 0)
{
if (AllowedHosts.IsInternalUrl(Url))
{
lvItem.SubItems[2].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[2].ForeColor = Color.Gray;
}
}
else
{
if (AllowedHosts.IsInternalUrl(Canonical))
{
lvItem.SubItems[2].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[2].ForeColor = Color.Red;
}
}
}
}
}
/**************************************************************************/
private void BuildWorksheetPageRedirectedLinks(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "Status Code";
iCol++;
ws.Cell(iRow, iCol).Value = "Status";
iCol++;
ws.Cell(iRow, iCol).Value = "Origin URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Destination URL";
}
iColMax = iCol;
iRow++;
foreach (string Url in DocCollection.DocumentKeys())
{
MacroscopeDocument msDoc = DocCollection.GetDocument(Url);
MacroscopeHyperlinksIn HyperlinksIn = DocCollection.GetDocumentHyperlinksIn(Url);
int StatusCode = ( int )msDoc.GetStatusCode();
string Status = msDoc.GetStatusCode().ToString();
if (
(StatusCode >= 300) &&
(StatusCode <= 399) &&
(HyperlinksIn != null))
{
foreach (MacroscopeHyperlinkIn HyperlinkIn in HyperlinksIn.IterateLinks())
{
string OriginUrl = HyperlinkIn.GetSourceUrl();
if (
(OriginUrl != null) &&
(OriginUrl.Length > 0))
{
iCol = 1;
this.InsertAndFormatContentCell(ws, iRow, iCol, StatusCode.ToString());
if ((StatusCode >= 400) && (StatusCode <= 599))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Blue);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Status);
if ((StatusCode >= 400) && (StatusCode <= 599))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Blue);
}
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, OriginUrl);
if (AllowedHosts.IsInternalUrl(Url: OriginUrl))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (AllowedHosts.IsInternalUrl(Url: Url))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iRow++;
}
}
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetPageDuplicateTitles(
MacroscopeJobMaster JobMaster,
CsvWriter ws
)
{
decimal Count = 0;
decimal DocCount = 0;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
DocCount = ( decimal )DocCollection.CountDocuments();
{
ws.WriteField("URL");
ws.WriteField("Occurrences");
ws.WriteField("Title");
ws.NextRecord();
}
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
bool Proceed = false;
if (DocCount > 0)
{
Count++;
this.ProgressForm.UpdatePercentages(
Title: null,
Message: null,
MajorPercentage: -1,
ProgressLabelMajor: string.Format("Documents Processed: {0}", Count),
MinorPercentage: (( decimal )100 / DocCount) * Count,
ProgressLabelMinor: msDoc.GetUrl(),
SubMinorPercentage: -1,
ProgressLabelSubMinor: null
);
}
if (AllowedHosts.IsInternalUrl(Url: msDoc.GetUrl()))
{
switch (msDoc.GetDocumentType())
{
case MacroscopeConstants.DocumentType.HTML:
Proceed = true;
break;
case MacroscopeConstants.DocumentType.PDF:
Proceed = true;
break;
default:
Proceed = false;
break;
}
}
if (Proceed)
{
string Title = msDoc.GetTitle();
int Occurrences = DocCollection.GetStatsTitleCount(msDoc: msDoc);
if (Occurrences > 1)
{
this.InsertAndFormatUrlCell(ws, msDoc);
this.InsertAndFormatContentCell(ws, Occurrences);
this.InsertAndFormatContentCell(ws, this.FormatIfMissing(Title));
ws.NextRecord();
}
}
}
}
/**************************************************************************/
private void BuildWorksheetPageHyperlinks(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "Source URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Target URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Follow";
iCol++;
ws.Cell(iRow, iCol).Value = "Target";
iCol++;
ws.Cell(iRow, iCol).Value = "Anchor Text";
iCol++;
ws.Cell(iRow, iCol).Value = "Title";
iCol++;
ws.Cell(iRow, iCol).Value = "Alt Text";
iCol++;
ws.Cell(iRow, iCol).Value = "Raw Target URL";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
MacroscopeHyperlinksOut HyperlinksOut = msDoc.GetHyperlinksOut();
foreach (MacroscopeHyperlinkOut HyperlinkOut in HyperlinksOut.IterateLinks())
{
string HyperlinkOutUrl = HyperlinkOut.GetTargetUrl();
string DoFollow = "No Follow";
string LinkTarget = HyperlinkOut.GetLinkTarget();
string AnchorText = HyperlinkOut.GetAnchorText();
string Title = HyperlinkOut.GetTitle();
string AltText = HyperlinkOut.GetAltText();
string RawTargetUrl = HyperlinkOut.GetRawTargetUrl();
if (HyperlinkOutUrl == null)
{
HyperlinkOutUrl = "";
}
if (HyperlinkOut.GetDoFollow())
{
DoFollow = "Follow";
}
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (AllowedHosts.IsInternalUrl(Url: msDoc.GetUrl()))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, HyperlinkOutUrl);
if ((HyperlinkOutUrl.Length > 0) && (AllowedHosts.IsInternalUrl(Url: HyperlinkOutUrl)))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
if ((HyperlinkOutUrl.Length > 0) && (AllowedHosts.IsExternalUrl(Url: HyperlinkOutUrl)))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
else
{
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(HyperlinkOutUrl));
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, DoFollow);
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, LinkTarget);
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(AnchorText));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(Title));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(AltText));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, RawTargetUrl);
iRow++;
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetSitemapXmlErrors(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "Sitemap URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Status Code";
iCol++;
ws.Cell(iRow, iCol).Value = "Robots";
iCol++;
ws.Cell(iRow, iCol).Value = "URL";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
if (msDoc.GetIsInternal() && msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.SITEMAPXML))
{
foreach (MacroscopeLink Outlink in msDoc.IterateOutlinks())
{
string TargetUrl = Outlink.GetTargetUrl();
MacroscopeDocument msDocLinked = DocCollection.GetDocumentByUrl(Url: TargetUrl);
bool InsertRow = false;
if (msDocLinked.GetIsInternal())
{
int StatusCode = (int)msDocLinked.GetStatusCode();
if ((StatusCode >= 400) && (StatusCode <= 599))
{
InsertRow = true;
}
if (!msDocLinked.GetAllowedByRobots())
{
InsertRow = true;
}
}
if (InsertRow)
{
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (AllowedHosts.IsInternalUrl(Url: msDoc.GetUrl()))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatStatusCodeCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatRobotsCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, TargetUrl);
if (AllowedHosts.IsInternalUrl(Url: TargetUrl))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iRow++;
}
}
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetPageDuplicatePages(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
decimal DocCount = 0;
decimal DocListCount = 0;
decimal CountOuter = 0;
decimal CountInner = 0;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
Dictionary <string, bool> CrossCheckList;
CrossCheckList = MacroscopeLevenshteinAnalysis.GetCrossCheckList(
Capacity: DocCollection.CountDocuments()
);
DocCount = ( decimal )DocCollection.CountDocuments();
{
ws.Cell(iRow, iCol).Value = "Status Code";
iCol++;
ws.Cell(iRow, iCol).Value = "Status";
iCol++;
ws.Cell(iRow, iCol).Value = "Origin URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Distance";
iCol++;
ws.Cell(iRow, iCol).Value = "Similar URL";
}
iColMax = iCol;
iRow++;
foreach (string UrlLeft in DocCollection.DocumentUrls())
{
MacroscopeDocument msDocLeft = DocCollection.GetDocumentByUrl(Url: UrlLeft);
MacroscopeLevenshteinAnalysis LevenshteinAnalysis = null;
CountOuter++;
CountInner = 0;
if (DocCount > 0)
{
this.ProgressForm.UpdatePercentages(
Title: null,
Message: null,
MajorPercentage: -1,
ProgressLabelMajor: string.Format("Documents Processed: {0}", CountOuter),
MinorPercentage: (( decimal )100 / DocCount) * CountOuter,
ProgressLabelMinor: UrlLeft,
SubMinorPercentage: 0,
ProgressLabelSubMinor: ""
);
}
if (msDocLeft.GetIsExternal())
{
continue;
}
if (!msDocLeft.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML))
{
continue;
}
LevenshteinAnalysis = new MacroscopeLevenshteinAnalysis(
msDoc: msDocLeft,
SizeDifference: MacroscopePreferencesManager.GetMaxLevenshteinSizeDifference(),
Threshold: MacroscopePreferencesManager.GetMaxLevenshteinDistance(),
CrossCheckList: CrossCheckList,
IPercentageDone: this
);
Dictionary <MacroscopeDocument, int> DocList;
DocList = LevenshteinAnalysis.AnalyzeDocCollection(
DocCollection: DocCollection
);
DocListCount = ( decimal )DocList.Count;
foreach (MacroscopeDocument msDocDuplicate in DocList.Keys)
{
int StatusCode = ( int )msDocLeft.GetStatusCode();
HttpStatusCode Status = msDocLeft.GetStatusCode();
string UrlDuplicate = msDocDuplicate.GetUrl();
int Distance = DocList[msDocDuplicate];
CountInner++;
iCol = 1;
if (DocCount > 0)
{
this.ProgressForm.UpdatePercentages(
Title: null,
Message: null,
MajorPercentage: -1,
ProgressLabelMajor: string.Format("Documents Processed: {0}", CountOuter),
MinorPercentage: (( decimal )100 / DocCount) * CountOuter,
ProgressLabelMinor: UrlLeft,
SubMinorPercentage: (( decimal )100 / DocListCount) * CountInner,
ProgressLabelSubMinor: UrlDuplicate
);
}
this.InsertAndFormatStatusCodeCell(ws, iRow, iCol, StatusCode);
iCol++;
this.InsertAndFormatStatusCodeCell(ws, iRow, iCol, Status);
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, UrlLeft);
if (AllowedHosts.IsInternalUrl(Url: UrlLeft))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Distance.ToString());
if (Distance <= MacroscopePreferencesManager.GetMaxLevenshteinDistance())
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, UrlDuplicate);
if (AllowedHosts.IsInternalUrl(Url: UrlDuplicate))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iRow++;
if (this.ProgressForm.Cancelled())
{
break;
}
}
if (this.ProgressForm.Cancelled())
{
break;
}
//Thread.Yield();
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void RenderListView(Dictionary <ulong, bool> History, MacroscopeDocumentCollection DocCollection)
{
if (History.Count == 0)
{
return;
}
List <ListViewItem> ListViewItems = new List <ListViewItem>(1);
MacroscopeAllowedHosts AllowedHosts = this.MainForm.GetJobMaster().GetAllowedHosts();
MacroscopeSinglePercentageProgressForm ProgressForm = new MacroscopeSinglePercentageProgressForm(this.MainForm);
decimal Count = 0;
decimal TotalDocs = (decimal)History.Count;
decimal MajorPercentage = ((decimal)100 / TotalDocs) * Count;
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
ProgressForm.UpdatePercentages(
Title: "Preparing Display",
Message: "Processing document collection for display:",
MajorPercentage: MajorPercentage,
ProgressLabelMajor: string.Format("Document {0} / {1}", Count, TotalDocs)
);
}
foreach (ulong DocKey in History.Keys)
{
ListViewItem lvItem = null;
MacroscopeDocument msDoc = DocCollection.GetDocumentByDocKey(DocKey: DocKey);
string PairKey = DocKey.ToString();
if (msDoc != null)
{
string Url = msDoc.GetUrl();
string Visited = "No";
string InDocCollection = "No";
if (History.ContainsKey(DocKey) && History[DocKey])
{
Visited = "Yes";
}
if (DocCollection.ContainsDocument(Url: Url))
{
InDocCollection = "Yes";
}
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
try
{
lvItem = this.DisplayListView.Items[PairKey];
lvItem.SubItems[ColUrl].Text = Url;
lvItem.SubItems[ColVisited].Text = Visited;
lvItem.SubItems[ColInDocCollection].Text = InDocCollection;
}
catch (Exception ex)
{
DebugMsg(string.Format("RenderListView 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
lvItem.SubItems[0].Text = Url;
lvItem.SubItems.Add(Visited);
lvItem.SubItems.Add(InDocCollection);
ListViewItems.Add(lvItem);
}
catch (Exception ex)
{
DebugMsg(string.Format("RenderListView 2: {0}", ex.Message));
}
}
if (lvItem != null)
{
lvItem.ForeColor = Color.Blue;
if (AllowedHosts.IsInternalUrl(Url))
{
lvItem.SubItems[ColUrl].ForeColor = Color.Green;
if (History.ContainsKey(DocKey) && History[DocKey])
{
lvItem.SubItems[ColVisited].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColVisited].ForeColor = Color.Red;
}
lvItem.SubItems[ColInDocCollection].ForeColor = Color.Blue;
}
else
{
lvItem.SubItems[ColUrl].ForeColor = Color.Gray;
lvItem.SubItems[ColVisited].ForeColor = Color.Gray;
lvItem.SubItems[ColInDocCollection].ForeColor = Color.Gray;
}
}
}
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
Count++;
TotalDocs = (decimal)History.Count;
MajorPercentage = ((decimal)100 / TotalDocs) * Count;
ProgressForm.UpdatePercentages(
Title: null,
Message: null,
MajorPercentage: MajorPercentage,
ProgressLabelMajor: string.Format("Document {0} / {1}", Count, TotalDocs)
);
}
}
this.DisplayListView.Items.AddRange(ListViewItems.ToArray());
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
ProgressForm.DoClose();
}
if (ProgressForm != null)
{
ProgressForm.Dispose();
}
}
/**************************************************************************/
private void RenderListView(Dictionary <string, Boolean> History)
{
if (History.Count == 0)
{
return;
}
List <ListViewItem> ListViewItems = new List <ListViewItem> (1);
MacroscopeAllowedHosts AllowedHosts = this.MainForm.GetJobMaster().GetAllowedHosts();
MacroscopeSinglePercentageProgressForm ProgressForm = new MacroscopeSinglePercentageProgressForm(this.MainForm);
decimal Count = 0;
decimal TotalDocs = ( decimal )History.Count;
decimal MajorPercentage = (( decimal )100 / TotalDocs) * Count;
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
ProgressForm.UpdatePercentages(
Title: "Preparing Display",
Message: "Processing document collection for display:",
MajorPercentage: MajorPercentage,
ProgressLabelMajor: string.Format("Document {0} / {1}", Count, TotalDocs)
);
}
foreach (string Url in History.Keys)
{
ListViewItem lvItem = null;
string Visited = "No";
if (History[Url])
{
Visited = "Yes";
}
if (this.DisplayListView.Items.ContainsKey(Url))
{
try
{
lvItem = this.DisplayListView.Items[Url];
lvItem.SubItems[1].Text = Visited;
}
catch (Exception ex)
{
DebugMsg(string.Format("RenderListView 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(Url);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = Url;
lvItem.SubItems.Add(Visited);
ListViewItems.Add(lvItem);
}
catch (Exception ex)
{
DebugMsg(string.Format("RenderListView 2: {0}", ex.Message));
}
}
if (lvItem != null)
{
lvItem.ForeColor = Color.Blue;
if (AllowedHosts.IsInternalUrl(Url))
{
lvItem.SubItems[0].ForeColor = Color.Green;
if (History[Url])
{
lvItem.SubItems[1].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[1].ForeColor = Color.Red;
}
}
else
{
lvItem.SubItems[0].ForeColor = Color.Gray;
lvItem.SubItems[1].ForeColor = Color.Gray;
}
}
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
Count++;
TotalDocs = ( decimal )History.Count;
MajorPercentage = (( decimal )100 / TotalDocs) * Count;
ProgressForm.UpdatePercentages(
Title: null,
Message: null,
MajorPercentage: MajorPercentage,
ProgressLabelMajor: string.Format("Document {0} / {1}", Count, TotalDocs)
);
}
}
this.DisplayListView.Items.AddRange(ListViewItems.ToArray());
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
ProgressForm.DoClose();
}
ProgressForm.Dispose();
}
/**************************************************************************/
private void RenderListView(MacroscopeJobItem [] UriQueue)
{
if (UriQueue.Length == 0)
{
return;
}
List <ListViewItem> ListViewItems = new List <ListViewItem> (1);
MacroscopeAllowedHosts AllowedHosts = this.MainForm.GetJobMaster().GetAllowedHosts();
MacroscopeSinglePercentageProgressForm ProgressForm = new MacroscopeSinglePercentageProgressForm(this.MainForm);
int Item = 1;
decimal Count = 0;
decimal TotalDocs = ( decimal )UriQueue.Length;
decimal MajorPercentage = (( decimal )100 / TotalDocs) * Count;
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
ProgressForm.UpdatePercentages(
Title: "Preparing Display",
Message: "Processing URI Queue for display:",
MajorPercentage: MajorPercentage,
ProgressLabelMajor: string.Format("URL {0} / {1}", Count, TotalDocs)
);
}
for (int i = 0; i < UriQueue.Length; i++)
{
ListViewItem lvItem = null;
string Url = UriQueue[i].GetItemUrl();
if (this.DisplayListView.Items.ContainsKey(Url))
{
try
{
lvItem = this.DisplayListView.Items[Url];
lvItem.SubItems[0].Text = Item.ToString();
}
catch (Exception ex)
{
DebugMsg(string.Format("RenderListView 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(Url);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = Url;
lvItem.SubItems[0].Text = Item.ToString();
lvItem.SubItems.Add(Url);
ListViewItems.Add(lvItem);
}
catch (Exception ex)
{
DebugMsg(string.Format("RenderListView 2: {0}", ex.Message));
}
}
if (lvItem != null)
{
lvItem.ForeColor = Color.Blue;
lvItem.SubItems[0].ForeColor = Color.Blue;
if (AllowedHosts.IsInternalUrl(Url))
{
lvItem.SubItems[1].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[1].ForeColor = Color.Gray;
}
}
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
Count++;
TotalDocs = ( decimal )UriQueue.Length;
MajorPercentage = (( decimal )100 / TotalDocs) * Count;
ProgressForm.UpdatePercentages(
Title: null,
Message: null,
MajorPercentage: MajorPercentage,
ProgressLabelMajor: string.Format("URL {0} / {1}", Count, TotalDocs)
);
}
Item++;
}
this.DisplayListView.Items.AddRange(ListViewItems.ToArray());
if (MacroscopePreferencesManager.GetShowProgressDialogues())
{
ProgressForm.DoClose();
}
ProgressForm.Dispose();
}
/**************************************************************************/
private void BuildWorksheetPageDuplicateTitles(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
decimal Count = 0;
decimal DocCount = 0;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
DocCount = ( decimal )DocCollection.CountDocuments();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Occurrences";
iCol++;
ws.Cell(iRow, iCol).Value = "Title";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
bool Proceed = false;
if (DocCount > 0)
{
Count++;
this.ProgressForm.UpdatePercentages(
Title: null,
Message: null,
MajorPercentage: -1,
ProgressLabelMajor: string.Format("Documents Processed: {0}", Count),
MinorPercentage: (( decimal )100 / DocCount) * Count,
ProgressLabelMinor: msDoc.GetUrl(),
SubMinorPercentage: -1,
ProgressLabelSubMinor: null
);
}
if (AllowedHosts.IsInternalUrl(Url: msDoc.GetUrl()))
{
switch (msDoc.GetDocumentType())
{
case MacroscopeConstants.DocumentType.HTML:
Proceed = true;
break;
case MacroscopeConstants.DocumentType.PDF:
Proceed = true;
break;
default:
Proceed = false;
break;
}
}
if (Proceed)
{
string Title = msDoc.GetTitle();
int Occurrences = DocCollection.GetStatsTitleCount(msDoc: msDoc);
if (Occurrences > 1)
{
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (msDoc.GetIsInternal())
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Occurrences);
if (Occurrences > 1)
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Orange);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(Title));
iRow++;
}
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
protected override void RenderListView(
List <ListViewItem> ListViewItems,
MacroscopeDocumentCollection DocCollection,
MacroscopeDocument msDoc,
string Url
)
{
if (!msDoc.GetIsRedirect())
{
return;
}
MacroscopeAllowedHosts AllowedHosts = this.MainForm.GetJobMaster().GetAllowedHosts();
int StatusCode = (int)msDoc.GetStatusCode();
string Status = msDoc.GetStatusCode().ToString();
string DestinationURL = msDoc.GetUrlRedirectTo();
string PairKey = string.Join("", Url);
if (
(!string.IsNullOrEmpty(Status)) &&
(!string.IsNullOrEmpty(DestinationURL)))
{
ListViewItem lvItem = null;
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
try
{
lvItem = this.DisplayListView.Items[PairKey];
lvItem.SubItems[COLURL].Text = Url;
lvItem.SubItems[COLSTATUSCODE].Text = StatusCode.ToString();
lvItem.SubItems[COLSTATUS].Text = Status;
lvItem.SubItems[COLDESTINATIONURL].Text = DestinationURL;
}
catch (Exception ex)
{
this.DebugMsg(string.Format("MacroscopeDisplayRedirectsAudit 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
lvItem.SubItems[COLURL].Text = Url;
lvItem.SubItems.Add(StatusCode.ToString());
lvItem.SubItems.Add(Status);
lvItem.SubItems.Add(DestinationURL);
ListViewItems.Add(lvItem);
}
catch (Exception ex)
{
this.DebugMsg(string.Format("MacroscopeDisplayRedirectsAudit 2: {0}", ex.Message));
}
}
if (lvItem != null)
{
if (msDoc.GetIsInternal())
{
for (int i = 0; i <= 3; i++)
{
lvItem.SubItems[i].ForeColor = Color.Blue;
}
if ((StatusCode >= 200) && (StatusCode <= 299))
{
for (int i = 0; i <= 3; i++)
{
lvItem.SubItems[i].ForeColor = Color.Green;
}
}
else
if ((StatusCode >= 300) && (StatusCode <= 399))
{
for (int i = 0; i <= 3; i++)
{
lvItem.SubItems[i].ForeColor = Color.Goldenrod;
}
}
else
if ((StatusCode >= 400) && (StatusCode <= 599))
{
for (int i = 0; i <= 3; i++)
{
lvItem.SubItems[i].ForeColor = Color.Red;
}
}
}
else
{
for (int i = 0; i <= 3; i++)
{
lvItem.SubItems[i].ForeColor = Color.Gray;
}
}
if (AllowedHosts.IsInternalUrl(DestinationURL))
{
lvItem.SubItems[COLDESTINATIONURL].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[COLDESTINATIONURL].ForeColor = Color.Gray;
}
}
}
}
/**************************************************************************/
private void BuildWorksheetPageRedirectChains(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
List <List <MacroscopeRedirectChainDocStruct> > RedirectChains = DocCollection.GetMacroscopeRedirectChains();
{
ws.Cell(iRow, iCol).Value = "Hop";
iCol++;
ws.Cell(iRow, iCol).Value = "Status";
}
iRow++;
foreach (List <MacroscopeRedirectChainDocStruct> DocList in RedirectChains)
{
int iHop = 1;
iCol = 1;
foreach (MacroscopeRedirectChainDocStruct RedirectChainDocStruct in DocList)
{
string Url = RedirectChainDocStruct.Url;
string StatusCode = RedirectChainDocStruct.StatusCode.ToString();
ws.Cell(1, iCol).Value = string.Format("Hop {0} URL", iHop);
this.InsertAndFormatUrlCell(ws, iRow, iCol, Url);
iCol++;
if (AllowedHosts.IsInternalUrl(Url: Url))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
ws.Cell(1, iCol).Value = string.Format("Hop {0} Status", iHop);
this.InsertAndFormatContentCell(ws, iRow, iCol, StatusCode);
iCol++;
iHop++;
}
if (iCol > iColMax)
{
iColMax = iCol;
}
iRow++;
}
if ((iRow > 1) && (iColMax > 2))
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax - 1);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetSitemapsAudit(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel,
MacroscopeDocumentList DocumentList,
bool InOut
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "In Sitemap";
iCol++;
ws.Cell(iRow, iCol).Value = "Status Code";
iCol++;
ws.Cell(iRow, iCol).Value = "Is Redirect";
iCol++;
ws.Cell(iRow, iCol).Value = "Robots";
iCol++;
ws.Cell(iRow, iCol).Value = "Sitemap";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocumentList.IterateDocuments())
{
string Url = null;
string Robots = null;
string SitemapUrl = null;
int StatusCode;
if (!msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML))
{
continue;
}
if (msDoc.GetIsExternal())
{
continue;
}
Url = msDoc.GetUrl();
StatusCode = (int)msDoc.GetStatusCode();
Robots = msDoc.GetAllowedByRobotsAsString();
SitemapUrl = DocumentList.GetDocumentNote(msDoc: msDoc);
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (msDoc.GetIsInternal())
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, InOut.ToString());
if (InOut)
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
iCol++;
this.InsertAndFormatStatusCodeCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatRedirectCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatRobotsCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, SitemapUrl);
if (AllowedHosts.IsInternalUrl(Url: SitemapUrl))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iRow++;
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetPageObservations(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Status Code";
iCol++;
ws.Cell(iRow, iCol).Value = "Status";
iCol++;
ws.Cell(iRow, iCol).Value = "Observation";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
string Url = msDoc.GetUrl();
string StatusCode = (( int )msDoc.GetStatusCode()).ToString();
string Status = msDoc.GetStatusCode().ToString();
foreach (KeyValuePair <string, string> RemarkPair in msDoc.IterateRemarks())
{
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (AllowedHosts.IsInternalUrl(Url: Url))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, StatusCode);
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Status);
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, RemarkPair.Value);
iRow++;
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetPageUriAnalysis(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Status Code";
iCol++;
ws.Cell(iRow, iCol).Value = "Status";
iCol++;
ws.Cell(iRow, iCol).Value = "Occurrences";
iCol++;
ws.Cell(iRow, iCol).Value = "Checksum";
}
iColMax = iCol;
iRow++;
foreach (string Url in DocCollection.DocumentKeys())
{
MacroscopeDocument msDoc = DocCollection.GetDocument(Url);
string StatusCode = (( int )msDoc.GetStatusCode()).ToString();
string Status = msDoc.GetStatusCode().ToString();
string Checksum = msDoc.GetChecksum();
int Count = DocCollection.GetStatsChecksumCount(Checksum: Checksum);
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (AllowedHosts.IsInternalUrl(Url: Url))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, StatusCode);
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Status);
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Count);
if (Count > 1)
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Blue);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Checksum);
if (Count > 1)
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Blue);
}
iRow++;
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
