/**************************************************************************/
private MacroscopeLink AddSitemapTextOutlink(
string AbsoluteUrl,
MacroscopeConstants.InOutLinkType LinkType,
Boolean Follow
)
{
MacroscopeLink OutLink = null;
if (!MacroscopePreferencesManager.GetCheckExternalLinks())
{
MacroscopeAllowedHosts AllowedHosts = this.DocCollection.GetAllowedHosts();
if (AllowedHosts != null)
{
if (!AllowedHosts.IsAllowedFromUrl(Url: AbsoluteUrl))
{
return(OutLink);
}
}
}
OutLink = new MacroscopeLink(
SourceUrl: this.GetUrl(),
TargetUrl: AbsoluteUrl,
LinkType: LinkType,
Follow: Follow
);
this.Outlinks.Add(OutLink);
return(OutLink);
}
/**************************************************************************/
private MacroscopeLink AddSitemapXmlOutlink(
string AbsoluteUrl,
MacroscopeConstants.InOutLinkType LinkType,
Boolean Follow
)
{
MacroscopeLink OutLink = null;
Boolean Proceed = true;
if (!MacroscopePreferencesManager.GetCheckExternalLinks())
{
MacroscopeAllowedHosts AllowedHosts = this.DocCollection.GetAllowedHosts();
if (AllowedHosts != null)
{
if (!AllowedHosts.IsAllowedFromUrl(Url: AbsoluteUrl))
{
Proceed = false;
}
}
}
switch (LinkType)
{
case MacroscopeConstants.InOutLinkType.SITEMAPXML:
if (!MacroscopePreferencesManager.GetFetchXml())
{
Proceed = false;
}
break;
}
if (Proceed)
{
OutLink = new MacroscopeLink(
SourceUrl: this.GetUrl(),
TargetUrl: AbsoluteUrl,
LinkType: LinkType,
Follow: Follow
);
this.Outlinks.Add(OutLink);
}
return(OutLink);
}
/**************************************************************************/
private void RenderListViewSearchTargetUrls(
List <ListViewItem> ListViewItems,
MacroscopeDocument msDoc,
string Url,
string UrlFragment
)
{
MacroscopeAllowedHosts AllowedHosts = this.MainForm.GetJobMaster().GetAllowedHosts();
MacroscopeHyperlinksOut HyperlinksOut = msDoc.GetHyperlinksOut();
MacroscopeDocumentCollection DocCollection = this.MainForm.GetJobMaster().GetDocCollection();
foreach (MacroscopeHyperlinkOut HyperlinkOut in HyperlinksOut.IterateLinks())
{
string UrlTarget = HyperlinkOut.GetTargetUrl();
HttpStatusCode StatusCode = HttpStatusCode.NotFound;
string StatusCodeText = "Not crawled";
string StatusText = "Not crawled";
string PairKey = string.Join(":", UrlToDigest(Url: Url), UrlToDigest(Url: UrlTarget)).ToString();
string LinkTarget = HyperlinkOut.GetLinkTarget();
string LinkText = HyperlinkOut.GetAnchorText();
string LinkTitle = HyperlinkOut.GetTitle();
string AltText = HyperlinkOut.GetAltText();
string LinkTextLabel = LinkText;
string LinkTitleLabel = LinkTitle;
string AltTextLabel = AltText;
string DoFollow = "No Follow";
try
{
if (DocCollection.ContainsDocument(Url: HyperlinkOut.GetTargetUrl()))
{
StatusCode = DocCollection.GetDocumentByUrl(Url: HyperlinkOut.GetTargetUrl()).GetStatusCode();
StatusCodeText = ((int)StatusCode).ToString();
StatusText = StatusCode.ToString();
}
else
{
DebugMsg("Not in DocCollection");
}
}
catch (Exception ex)
{
this.DebugMsg(ex.Message);
}
if (HyperlinkOut.GetDoFollow())
{
DoFollow = "Follow";
}
if (LinkText.Length == 0)
{
LinkTextLabel = "MISSING";
}
if (LinkTitle.Length == 0)
{
LinkTitleLabel = "MISSING";
}
if (AltText.Length == 0)
{
AltTextLabel = "MISSING";
}
if (
(UrlTarget != null) &&
(UrlTarget.IndexOf(UrlFragment, StringComparison.CurrentCulture) >= 0))
{
ListViewItem lvItem = null;
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
try
{
lvItem = this.DisplayListView.Items[PairKey];
lvItem.SubItems[ColUrl].Text = Url;
lvItem.SubItems[ColUrlTarget].Text = UrlTarget;
lvItem.SubItems[ColStatusCode].Text = StatusCodeText;
lvItem.SubItems[ColStatus].Text = StatusText;
lvItem.SubItems[ColDoFollow].Text = DoFollow;
lvItem.SubItems[ColLinkTarget].Text = LinkTarget;
lvItem.SubItems[ColLinkAnchorTextLabel].Text = LinkTextLabel;
lvItem.SubItems[ColLinkTitleLabel].Text = LinkTitleLabel;
lvItem.SubItems[ColAltTextLabel].Text = AltTextLabel;
}
catch (Exception ex)
{
this.DebugMsg(string.Format("MacroscopeDisplayLinks 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
lvItem.SubItems[ColUrl].Text = Url;
lvItem.SubItems.Add(UrlTarget);
lvItem.SubItems.Add(StatusCodeText);
lvItem.SubItems.Add(StatusText);
lvItem.SubItems.Add(DoFollow);
lvItem.SubItems.Add(LinkTarget);
lvItem.SubItems.Add(LinkTextLabel);
lvItem.SubItems.Add(LinkTitleLabel);
lvItem.SubItems.Add(AltTextLabel);
ListViewItems.Add(lvItem);
}
catch (Exception ex)
{
this.DebugMsg(string.Format("MacroscopeDisplayLinks 2: {0}", ex.Message));
}
}
if (lvItem != null)
{
for (int i = 0; i < lvItem.SubItems.Count; i++)
{
lvItem.SubItems[i].ForeColor = Color.Blue;
}
if (AllowedHosts.IsAllowedFromUrl(Url))
{
lvItem.SubItems[ColUrl].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColUrl].ForeColor = Color.Gray;
}
if (AllowedHosts.IsAllowedFromUrl(UrlTarget))
{
lvItem.SubItems[ColUrlTarget].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColUrlTarget].ForeColor = Color.Gray;
}
if (AllowedHosts.IsAllowedFromUrl(Url))
{
if (HyperlinkOut.GetDoFollow())
{
lvItem.SubItems[ColDoFollow].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColDoFollow].ForeColor = Color.Red;
}
}
else
{
lvItem.SubItems[ColDoFollow].ForeColor = Color.Gray;
}
if (LinkText.Length == 0)
{
lvItem.SubItems[ColLinkAnchorTextLabel].ForeColor = Color.Gray;
}
if (LinkTitle.Length == 0)
{
lvItem.SubItems[ColLinkTitleLabel].ForeColor = Color.Gray;
}
if (AltText.Length == 0)
{
lvItem.SubItems[ColAltTextLabel].ForeColor = Color.Gray;
}
if (
(LinkText.Length == 0) &&
(LinkTitle.Length == 0) &&
(AltText.Length == 0))
{
lvItem.SubItems[ColLinkAnchorTextLabel].ForeColor = Color.Red;
lvItem.SubItems[ColLinkTitleLabel].ForeColor = Color.Red;
lvItem.SubItems[ColAltTextLabel].ForeColor = Color.Red;
}
}
}
}
}
/**************************************************************************/
private void BuildWorksheetMissingLanguageSpecifier(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Site Locale";
iCol++;
ws.Cell(iRow, iCol).Value = "Title";
for (int i = 1; i <= iCol; i++)
{
ws.Cell(iRow, i).Style.Font.SetBold();
}
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
string SiteLocale = msDoc.GetLocale();
if (
AllowedHosts.IsAllowedFromUrl(msDoc.GetUrl()) &&
string.IsNullOrEmpty(SiteLocale))
{
string SiteLocaleFormatted = this.FormatIfMissing(SiteLocale);
string Title = this.FormatIfMissing(msDoc.GetTitle());
iCol = 1;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, SiteLocaleFormatted);
if (SiteLocaleFormatted == "MISSING")
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Title);
if (Title == "MISSING")
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
iRow++;
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
protected override void RenderListView(
List <ListViewItem> ListViewItems,
MacroscopeDocumentCollection DocCollection,
MacroscopeDocument msDoc,
string Url
)
{
MacroscopeAllowedHosts AllowedHosts = this.MainForm.GetJobMaster().GetAllowedHosts();
MacroscopeHyperlinksOut HyperlinksOut = msDoc.GetHyperlinksOut();
foreach (MacroscopeHyperlinkOut HyperlinkOut in HyperlinksOut.IterateLinks())
{
ListViewItem lvItem = null;
string UrlTarget = HyperlinkOut.GetTargetUrl();
string PairKey = string.Join(":", UrlToDigest(Url), UrlToDigest(UrlTarget));
string LinkTarget = HyperlinkOut.GetLinkTarget();
string LinkText = HyperlinkOut.GetLinkText();
string LinkTitle = HyperlinkOut.GetLinkTitle();
string AltText = HyperlinkOut.GetAltText();
string LinkTextLabel = LinkText;
string LinkTitleLabel = LinkTitle;
string AltTextLabel = AltText;
string RawTargetUrl = HyperlinkOut.GetRawTargetUrl();
string DoFollow = "No Follow";
if (HyperlinkOut.GetDoFollow())
{
DoFollow = "Follow";
}
if (LinkText.Length == 0)
{
LinkTextLabel = "MISSING";
}
if (LinkTitle.Length == 0)
{
LinkTitleLabel = "MISSING";
}
if (AltText.Length == 0)
{
AltTextLabel = "MISSING";
}
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
try
{
lvItem = this.DisplayListView.Items[PairKey];
lvItem.SubItems[ColUrl].Text = Url;
lvItem.SubItems[ColUrlTarget].Text = UrlTarget;
lvItem.SubItems[ColDoFollow].Text = DoFollow;
lvItem.SubItems[ColLinkTarget].Text = LinkTarget;
lvItem.SubItems[ColLinkTextLabel].Text = LinkTextLabel;
lvItem.SubItems[ColLinkTitleLabel].Text = LinkTitleLabel;
lvItem.SubItems[ColAltTextLabel].Text = AltTextLabel;
lvItem.SubItems[ColRawTargetUrl].Text = RawTargetUrl;
}
catch (Exception ex)
{
this.DebugMsg(string.Format("MacroscopeDisplayLinks 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
lvItem.SubItems[ColUrl].Text = Url;
lvItem.SubItems.Add(UrlTarget);
lvItem.SubItems.Add(DoFollow);
lvItem.SubItems.Add(LinkTarget);
lvItem.SubItems.Add(LinkTextLabel);
lvItem.SubItems.Add(LinkTitleLabel);
lvItem.SubItems.Add(AltTextLabel);
lvItem.SubItems.Add(RawTargetUrl);
ListViewItems.Add(lvItem);
}
catch (Exception ex)
{
this.DebugMsg(string.Format("MacroscopeDisplayLinks 2: {0}", ex.Message));
}
}
if (lvItem != null)
{
for (int i = 0; i < lvItem.SubItems.Count; i++)
{
lvItem.SubItems[i].ForeColor = Color.Blue;
}
if (AllowedHosts.IsAllowedFromUrl(Url))
{
lvItem.SubItems[ColUrl].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColUrl].ForeColor = Color.Gray;
}
if (AllowedHosts.IsAllowedFromUrl(UrlTarget))
{
lvItem.SubItems[ColUrlTarget].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColUrlTarget].ForeColor = Color.Gray;
}
if (AllowedHosts.IsAllowedFromUrl(Url))
{
if (HyperlinkOut.GetDoFollow())
{
lvItem.SubItems[ColDoFollow].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColDoFollow].ForeColor = Color.Red;
}
}
else
{
lvItem.SubItems[ColDoFollow].ForeColor = Color.Gray;
}
if (LinkText.Length == 0)
{
lvItem.SubItems[ColLinkTextLabel].ForeColor = Color.Gray;
}
if (LinkTitle.Length == 0)
{
lvItem.SubItems[ColLinkTitleLabel].ForeColor = Color.Gray;
}
if (AltText.Length == 0)
{
lvItem.SubItems[ColAltTextLabel].ForeColor = Color.Gray;
}
if (
(LinkText.Length == 0) &&
(LinkTitle.Length == 0) &&
(AltText.Length == 0))
{
lvItem.SubItems[ColLinkTextLabel].ForeColor = Color.Red;
lvItem.SubItems[ColLinkTitleLabel].ForeColor = Color.Red;
lvItem.SubItems[ColAltTextLabel].ForeColor = Color.Red;
}
}
}
}
/**************************************************************************/
protected override void RenderListView(
List <ListViewItem> ListViewItems,
MacroscopeDocumentCollection DocCollection,
MacroscopeDocument msDoc,
string Url
)
{
MacroscopeAllowedHosts AllowedHosts = this.MainForm.GetJobMaster().GetAllowedHosts();
foreach (MacroscopeLink Link in msDoc.IterateOutlinks())
{
ListViewItem lvItem = null;
string LinkType = Link.GetLinkType().ToString();
string UrlTarget = Link.GetTargetUrl();
HttpStatusCode StatusCode = HttpStatusCode.NotFound;
string StatusCodeText = "Not crawled";
string StatusText = "Not crawled";
string PairKey = string.Join(":", UrlToDigest(Url: Url), UrlToDigest(Url: UrlTarget));
string DoFollow = "No Follow";
string AltText = Link.GetAltText();
string AltTextLabel = AltText;
string RawSourceUrl = Link.GetRawSourceUrl();
string RawTargetUrl = Link.GetRawTargetUrl();
try
{
if (DocCollection.ContainsDocument(Url: Link.GetTargetUrl()))
{
StatusCode = DocCollection.GetDocumentByUrl(Url: Link.GetTargetUrl()).GetStatusCode();
StatusCodeText = ((int)StatusCode).ToString();
StatusText = StatusCode.ToString();
}
}
catch (Exception ex)
{
this.DebugMsg(ex.Message);
}
if (Link.GetDoFollow())
{
DoFollow = "Follow";
}
if (string.IsNullOrEmpty(AltText))
{
AltTextLabel = "";
}
if (string.IsNullOrEmpty(RawSourceUrl))
{
RawSourceUrl = "";
}
if (string.IsNullOrEmpty(RawTargetUrl))
{
RawTargetUrl = "";
}
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
try
{
lvItem = this.DisplayListView.Items[PairKey];
lvItem.SubItems[ColType].Text = LinkType;
lvItem.SubItems[ColUrl].Text = Url;
lvItem.SubItems[ColUrlTarget].Text = UrlTarget;
lvItem.SubItems[ColStatusCode].Text = StatusCodeText;
lvItem.SubItems[ColStatus].Text = StatusText;
lvItem.SubItems[ColDoFollow].Text = DoFollow;
lvItem.SubItems[ColAltTextLabel].Text = AltTextLabel;
lvItem.SubItems[ColRawSourceUrl].Text = RawSourceUrl;
lvItem.SubItems[ColRawTargetUrl].Text = RawTargetUrl;
}
catch (Exception ex)
{
this.DebugMsg(string.Format("MacroscopeDisplayLinks 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
lvItem.SubItems[ColType].Text = LinkType;
lvItem.SubItems.Add(Url);
lvItem.SubItems.Add(UrlTarget);
lvItem.SubItems.Add(StatusCodeText);
lvItem.SubItems.Add(StatusText);
lvItem.SubItems.Add(DoFollow);
lvItem.SubItems.Add(AltTextLabel);
lvItem.SubItems.Add(RawSourceUrl);
lvItem.SubItems.Add(RawTargetUrl);
ListViewItems.Add(lvItem);
}
catch (Exception ex)
{
this.DebugMsg(string.Format("MacroscopeDisplayLinks 2: {0}", ex.Message));
}
}
if (lvItem != null)
{
for (int i = 0; i < lvItem.SubItems.Count; i++)
{
lvItem.SubItems[i].ForeColor = Color.Blue;
}
if (AllowedHosts.IsAllowedFromUrl(Url))
{
lvItem.SubItems[ColUrl].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColUrl].ForeColor = Color.Gray;
}
if (AllowedHosts.IsAllowedFromUrl(UrlTarget))
{
lvItem.SubItems[ColUrlTarget].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColUrlTarget].ForeColor = Color.Gray;
}
if (AllowedHosts.IsAllowedFromUrl(UrlTarget))
{
if (Link.GetDoFollow())
{
lvItem.SubItems[ColDoFollow].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColDoFollow].ForeColor = Color.Red;
}
}
else
{
lvItem.SubItems[ColDoFollow].ForeColor = Color.Gray;
}
}
}
}
/**************************************************************************/
private void RenderListViewSearchTargetUrls(
List <ListViewItem> ListViewItems,
MacroscopeDocument msDoc,
string Url,
string UrlFragment
)
{
MacroscopeAllowedHosts AllowedHosts = this.MainForm.GetJobMaster().GetAllowedHosts();
foreach (MacroscopeLink Link in msDoc.IterateOutlinks())
{
string LinkType = Link.GetLinkType().ToString();
string UrlTarget = Link.GetTargetUrl();
string PairKey = string.Join("::", Url, UrlTarget);
string DoFollow = "No Follow";
string AltText = Link.GetAltText();
string AltTextLabel = AltText;
string RawSourceUrl = Link.GetRawSourceUrl();
string RawTargetUrl = Link.GetRawTargetUrl();
if (Link.GetDoFollow())
{
DoFollow = "Follow";
}
if (string.IsNullOrEmpty(AltText))
{
AltTextLabel = "";
}
if (string.IsNullOrEmpty(RawSourceUrl))
{
RawSourceUrl = "";
}
if (string.IsNullOrEmpty(RawTargetUrl))
{
RawTargetUrl = "";
}
if (
(UrlTarget != null) &&
(UrlTarget.IndexOf(UrlFragment, StringComparison.CurrentCulture) >= 0))
{
ListViewItem lvItem = null;
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
try
{
lvItem = this.DisplayListView.Items[PairKey];
lvItem.SubItems[0].Text = LinkType;
lvItem.SubItems[1].Text = Url;
lvItem.SubItems[2].Text = UrlTarget;
lvItem.SubItems[3].Text = DoFollow;
lvItem.SubItems[4].Text = AltTextLabel;
lvItem.SubItems[5].Text = RawSourceUrl;
lvItem.SubItems[6].Text = RawTargetUrl;
}
catch (Exception ex)
{
this.DebugMsg(string.Format("MacroscopeDisplayLinks 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
lvItem.SubItems[0].Text = LinkType;
lvItem.SubItems.Add(Url);
lvItem.SubItems.Add(UrlTarget);
lvItem.SubItems.Add(DoFollow);
lvItem.SubItems.Add(AltTextLabel);
lvItem.SubItems.Add(RawSourceUrl);
lvItem.SubItems.Add(RawTargetUrl);
ListViewItems.Add(lvItem);
}
catch (Exception ex)
{
this.DebugMsg(string.Format("MacroscopeDisplayLinks 2: {0}", ex.Message));
}
}
if (lvItem != null)
{
for (int i = 0; i < lvItem.SubItems.Count; i++)
{
lvItem.SubItems[i].ForeColor = Color.Blue;
}
if (AllowedHosts.IsAllowedFromUrl(Url))
{
lvItem.SubItems[1].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[1].ForeColor = Color.Gray;
}
if (AllowedHosts.IsAllowedFromUrl(UrlTarget))
{
lvItem.SubItems[2].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[2].ForeColor = Color.Gray;
}
if (AllowedHosts.IsAllowedFromUrl(UrlTarget))
{
if (Link.GetDoFollow())
{
lvItem.SubItems[3].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[3].ForeColor = Color.Red;
}
}
else
{
lvItem.SubItems[3].ForeColor = Color.Gray;
}
}
}
}
}
