/**************************************************************************/
private Dictionary <string, Dictionary <string, bool> > BuildSitemapUrlList(MacroscopeDocumentCollection DocCollection)
{
Dictionary <string, Dictionary <string, bool> > UrlMap = new Dictionary <string, Dictionary <string, bool> >();
try
{
MacroscopeDocumentList SitemapDocumentList = this.FindSitemaps(DocCollection: DocCollection);
foreach (MacroscopeDocument msDoc in SitemapDocumentList.IterateDocuments())
{
string SitemapUrl = msDoc.GetUrl();
if (!UrlMap.ContainsKey(SitemapUrl))
{
UrlMap.Add(SitemapUrl, new Dictionary <string, bool>());
}
foreach (MacroscopeLink Outlink in msDoc.IterateOutlinks())
{
string TargetUrl = Outlink.GetTargetUrl();
if (!UrlMap[SitemapUrl].ContainsKey(TargetUrl))
{
UrlMap[SitemapUrl].Add(TargetUrl, false);
}
}
}
}
catch (Exception ex)
{
this.DebugMsg(string.Format("BuildSitemapUrlList: {0}", ex.Message));
}
return(UrlMap);
}
/**************************************************************************/
public new void RenderListView(MacroscopeDocumentCollection DocCollection)
{
List <ListViewItem> ListViewItems = new List <ListViewItem>(1);
MacroscopeDocumentList OrphanedDocumentList = DocCollection.GetOrphanedDocumentList();
this.ClearData();
if (OrphanedDocumentList != null)
{
foreach (MacroscopeDocument msDoc in OrphanedDocumentList.IterateDocuments())
{
this.RenderListView(
ListViewItems: ListViewItems,
DocCollection: DocCollection,
msDoc: msDoc,
Url: msDoc.GetUrl()
);
}
}
this.DisplayListView.Items.AddRange(ListViewItems.ToArray());
}
/**************************************************************************/
private void BuildWorksheetPageOrphanedPages(
MacroscopeJobMaster JobMaster,
CsvWriter ws
)
{
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeDocumentList OrphanedDocumentList = DocCollection.GetOrphanedDocumentList();
{
ws.WriteField("URL");
ws.WriteField("Status Code");
ws.WriteField("Status");
ws.NextRecord();
}
if (OrphanedDocumentList != null)
{
foreach (MacroscopeDocument msDoc in OrphanedDocumentList.IterateDocuments())
{
string Url = msDoc.GetUrl();
string StatusCode = ((int)msDoc.GetStatusCode()).ToString();
string Status = msDoc.GetStatusCode().ToString();
string Checksum = msDoc.GetChecksum();
int Count = DocCollection.GetStatsChecksumCount(Checksum: Checksum);
this.InsertAndFormatUrlCell(ws, msDoc);
this.InsertAndFormatContentCell(ws, StatusCode);
this.InsertAndFormatContentCell(ws, Status);
ws.NextRecord();
}
}
}
/**************************************************************************/
private void BuildWorksheetKeywordTerms(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel,
Dictionary <string, int> DicTerms
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
decimal TermTotal = DicTerms.Count;
decimal TermCount = 0;
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "Occurrences";
iCol++;
ws.Cell(iRow, iCol).Value = "Term";
iCol++;
ws.Cell(iRow, iCol).Value = "URL";
}
iColMax = iCol;
iRow++;
foreach (string Term in DicTerms.Keys)
{
MacroscopeDocumentList DocumentList = DocCollection.GetDeepKeywordAnalysDocumentList(Term);
decimal DocTotal = ( decimal )DocumentList.CountDocuments();
decimal DocCount = 0;
TermCount++;
if (TermTotal > 0)
{
this.ProgressForm.UpdatePercentages(
Title: null,
Message: null,
MajorPercentage: -1,
ProgressLabelMajor: null,
MinorPercentage: (( decimal )100 / TermTotal) * TermCount,
ProgressLabelMinor: "Keywords Processed",
SubMinorPercentage: -1,
ProgressLabelSubMinor: null
);
}
foreach (MacroscopeDocument msDoc in DocumentList.IterateDocuments())
{
DocCount++;
if (DocTotal > 0)
{
this.ProgressForm.UpdatePercentages(
Title: null,
Message: null,
MajorPercentage: -1,
ProgressLabelMajor: null,
MinorPercentage: -1,
ProgressLabelMinor: null,
SubMinorPercentage: (( decimal )100 / DocTotal) * DocCount,
ProgressLabelSubMinor: "Documents Processed"
);
}
iCol = 1;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(DicTerms[Term].ToString()));
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(Term));
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc.GetUrl());
iRow++;
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetSitemapsAudit(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel,
MacroscopeDocumentList DocumentList,
bool InOut
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "In Sitemap";
iCol++;
ws.Cell(iRow, iCol).Value = "Status Code";
iCol++;
ws.Cell(iRow, iCol).Value = "Is Redirect";
iCol++;
ws.Cell(iRow, iCol).Value = "Robots";
iCol++;
ws.Cell(iRow, iCol).Value = "Sitemap";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocumentList.IterateDocuments())
{
string Url = null;
string Robots = null;
string SitemapUrl = null;
int StatusCode;
if (!msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML))
{
continue;
}
if (msDoc.GetIsExternal())
{
continue;
}
Url = msDoc.GetUrl();
StatusCode = (int)msDoc.GetStatusCode();
Robots = msDoc.GetAllowedByRobotsAsString();
SitemapUrl = DocumentList.GetDocumentNote(msDoc: msDoc);
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (msDoc.GetIsInternal())
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, InOut.ToString());
if (InOut)
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
iCol++;
this.InsertAndFormatStatusCodeCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatRedirectCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatRobotsCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, SitemapUrl);
if (AllowedHosts.IsInternalUrl(Url: SitemapUrl))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iRow++;
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetPageOrphanedPages(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
MacroscopeDocumentList OrphanedDocumentList = DocCollection.GetOrphanedDocumentList();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Status Code";
iCol++;
ws.Cell(iRow, iCol).Value = "Status";
}
iColMax = iCol;
iRow++;
if (OrphanedDocumentList != null)
{
foreach (MacroscopeDocument msDoc in OrphanedDocumentList.IterateDocuments())
{
string Url = msDoc.GetUrl();
string StatusCode = ((int)msDoc.GetStatusCode()).ToString();
string Status = msDoc.GetStatusCode().ToString();
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (AllowedHosts.IsInternalUrl(Url: Url))
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, StatusCode);
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Status);
iRow++;
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
}
/** -------------------------------------------------------------------- **/
private void _RenderListViewSitemapsAudit(
MacroscopeDocumentCollection DocCollection,
MacroscopeDocumentList DocumentList,
bool InOut
)
{
List <ListViewItem> ListViewItems = new List <ListViewItem>(1);
foreach (MacroscopeDocument msDoc in DocumentList.IterateDocuments())
{
string Url = null;
string Robots = null;
string SitemapUrl = null;
string PairKey = null;
ListViewItem lvItem = null;
int StatusCode;
if (!msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML))
{
continue;
}
if (msDoc.GetIsExternal())
{
continue;
}
Url = msDoc.GetUrl();
StatusCode = (int)msDoc.GetStatusCode();
Robots = msDoc.GetAllowedByRobotsAsString();
SitemapUrl = DocumentList.GetDocumentNote(msDoc: msDoc);
PairKey = string.Join("::::::::", Url);
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
try
{
lvItem = this.DisplayListView.Items[PairKey];
lvItem.SubItems[COL_URL].Text = Url;
lvItem.SubItems[COL_IN_SITEMAP].Text = InOut.ToString();
lvItem.SubItems[COL_STATUS_CODE].Text = msDoc.GetStatusCode().ToString();
lvItem.SubItems[COL_IS_REDIRECT].Text = msDoc.GetIsRedirect().ToString();
lvItem.SubItems[COL_ROBOTS].Text = Robots;
lvItem.SubItems[COL_SITEMAP].Text = SitemapUrl;
}
catch (Exception ex)
{
DebugMsg(string.Format("_RenderListViewSitemapsAudit 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
lvItem.SubItems[COL_URL].Text = Url;
lvItem.SubItems.Add(InOut.ToString());
lvItem.SubItems.Add(msDoc.GetStatusCode().ToString());
lvItem.SubItems.Add(msDoc.GetIsRedirect().ToString());
lvItem.SubItems.Add(Robots);
lvItem.SubItems.Add(SitemapUrl);
ListViewItems.Add(lvItem);
}
catch (Exception ex)
{
DebugMsg(string.Format("_RenderListViewSitemapsAudit 2: {0}", ex.Message));
}
}
try
{
if (lvItem != null)
{
lvItem.ForeColor = Color.Blue;
if (msDoc.GetIsInternal())
{
lvItem.SubItems[COL_URL].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[COL_URL].ForeColor = Color.Gray;
}
if (InOut)
{
lvItem.SubItems[COL_IN_SITEMAP].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[COL_IN_SITEMAP].ForeColor = Color.Red;
}
if ((StatusCode >= 200) && (StatusCode <= 299))
{
lvItem.SubItems[COL_STATUS_CODE].ForeColor = Color.Green;
}
else
if ((StatusCode >= 300) && (StatusCode <= 399))
{
lvItem.SubItems[COL_STATUS_CODE].ForeColor = Color.Goldenrod;
}
else
if ((StatusCode >= 400) && (StatusCode <= 599))
{
lvItem.SubItems[COL_STATUS_CODE].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[COL_STATUS_CODE].ForeColor = Color.Blue;
}
if (StatusCode == 410)
{
lvItem.SubItems[COL_STATUS_CODE].ForeColor = Color.Purple;
}
if (msDoc.GetIsRedirect())
{
lvItem.SubItems[COL_IS_REDIRECT].ForeColor = Color.Goldenrod;
}
else
{
lvItem.SubItems[COL_IS_REDIRECT].ForeColor = Color.Gray;
}
if (!msDoc.GetAllowedByRobots())
{
lvItem.SubItems[COL_ROBOTS].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[COL_ROBOTS].ForeColor = Color.Green;
}
if (msDoc.GetIsInternal())
{
lvItem.SubItems[COL_SITEMAP].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[COL_SITEMAP].ForeColor = Color.Gray;
}
}
else
{
lvItem.SubItems[3].ForeColor = Color.Gray;
}
}
catch (Exception ex)
{
DebugMsg(string.Format("_RenderListViewSitemapsAudit 3: {0}", ex.Message));
}
}
this.DisplayListView.Items.AddRange(ListViewItems.ToArray());
return;
}
