/**************************************************************************/
public void SetPrefsFormControlFields()
{
{ // Configure Display Options
this.checkBoxPauseDisplayDuringScan.Checked = MacroscopePreferencesManager.GetPauseDisplayDuringScan();
this.checkBoxShowProgressDialogues.Checked = MacroscopePreferencesManager.GetShowProgressDialogues();
}
{ //Configure Form Fields
/** Spidering Control ---------------------------------------------- **/
this.numericUpDownDepth.Minimum = -1;
this.numericUpDownDepth.Maximum = 10000;
this.numericUpDownPageLimit.Minimum = -1;
this.numericUpDownPageLimit.Maximum = 10000;
this.numericUpDownCrawlDelay.Minimum = 0;
this.numericUpDownCrawlDelay.Maximum = 60;
this.numericUpDownMaxRetries.Minimum = 0;
this.numericUpDownMaxRetries.Maximum = 10;
}
{
/** WebProxy Options ----------------------------------------------- **/
this.comboBoxProxyType.SelectedIndex = (int)MacroscopePreferencesManager.GetProxyType();
/** Server Certificate Options --------------------------------------- **/
this.checkBoxServerCertificateValidation.Checked = MacroscopePreferencesManager.GetServerCertificateValidation();
/** Spidering Control ---------------------------------------------- **/
this.numericUpDownMaxThreads.Value = MacroscopePreferencesManager.GetMaxThreads();
this.numericUpDownDepth.Value = MacroscopePreferencesManager.GetDepth();
this.numericUpDownPageLimit.Value = MacroscopePreferencesManager.GetPageLimit();
this.numericUpDownCrawlDelay.Value = MacroscopePreferencesManager.GetCrawlDelay();
this.numericUpDownRequestTimeout.Value = (Decimal)MacroscopePreferencesManager.GetRequestTimeout();
this.numericUpDownMaxRetries.Value = (Decimal)MacroscopePreferencesManager.GetMaxRetries();
this.checkBoxCrawlStrictUrlCheck.Checked = MacroscopePreferencesManager.GetCrawlStrictUrlCheck();
this.checkBoxCheckExternalLinks.Checked = MacroscopePreferencesManager.GetCheckExternalLinks();
this.checkBoxFetchExternalLinks.Checked = MacroscopePreferencesManager.GetFetchExternalLinks();
this.checkBoxFollowRobotsProtocol.Checked = MacroscopePreferencesManager.GetFollowRobotsProtocol();
this.checkBoxFollowSitemapLinks.Checked = MacroscopePreferencesManager.GetFollowSitemapLinks();
this.checkBoxProbeHumansText.Checked = MacroscopePreferencesManager.GetProbeHumansText();
this.checkBoxCheckRedirects.Checked = MacroscopePreferencesManager.GetCheckRedirects();
this.checkBoxFollowRedirects.Checked = MacroscopePreferencesManager.GetFollowRedirects();
this.checkBoxFollowNoFollow.Checked = MacroscopePreferencesManager.GetFollowNoFollow();
this.checkBoxIgnoreQueries.Checked = MacroscopePreferencesManager.GetIgnoreQueries();
this.checkBoxIgnoreHashFragments.Checked = MacroscopePreferencesManager.GetIgnoreHashFragments();
this.checkBoxFollowCanonicalLinks.Checked = MacroscopePreferencesManager.GetFollowCanonicalLinks();
this.checkBoxFollowAlternateLinks.Checked = MacroscopePreferencesManager.GetFollowAlternateLinks();
this.checkBoxFollowHrefLangLinks.Checked = MacroscopePreferencesManager.GetFollowHrefLangLinks();
this.checkBoxDowncaseLinks.Checked = MacroscopePreferencesManager.GetDowncaseLinks();
this.checkBoxFetchStylesheets.Checked = MacroscopePreferencesManager.GetFetchStylesheets();
this.checkBoxFetchJavascripts.Checked = MacroscopePreferencesManager.GetFetchJavascripts();
this.checkBoxFetchImages.Checked = MacroscopePreferencesManager.GetFetchImages();
this.checkBoxFetchAudio.Checked = MacroscopePreferencesManager.GetFetchAudio();
this.checkBoxFetchVideo.Checked = MacroscopePreferencesManager.GetFetchVideo();
this.checkBoxFetchXml.Checked = MacroscopePreferencesManager.GetFetchXml();
this.checkBoxFetchBinaries.Checked = MacroscopePreferencesManager.GetFetchBinaries();
this.checkBoxScanSitesInList.Checked = MacroscopePreferencesManager.GetScanSitesInList();
this.checkBoxProbeParentFolderUrls.Checked = MacroscopePreferencesManager.GetProbeParentFolderUrls();
this.checkBoxProbeHead404sWithGet.Checked = MacroscopePreferencesManager.GetProbeHead404sWithGet();
/** Analysis Options ----------------------------------------------- **/
this.checkBoxResolveAddresses.Checked = MacroscopePreferencesManager.GetResolveAddresses();
this.checkBoxCheckHreflangs.Checked = MacroscopePreferencesManager.GetCheckHreflangs();
this.checkBoxDetectLanguage.Checked = MacroscopePreferencesManager.GetDetectLanguage();
this.checkBoxProcessStylesheets.Checked = MacroscopePreferencesManager.GetProcessStylesheets();
this.checkBoxProcessJavascripts.Checked = MacroscopePreferencesManager.GetProcessJavascripts();
this.checkBoxProcessImages.Checked = MacroscopePreferencesManager.GetProcessImages();
this.checkBoxProcessPdfs.Checked = MacroscopePreferencesManager.GetProcessPdfs();
this.checkBoxProcessAudio.Checked = MacroscopePreferencesManager.GetProcessAudio();
this.checkBoxProcessVideo.Checked = MacroscopePreferencesManager.GetProcessVideo();
this.checkBoxProcessXml.Checked = MacroscopePreferencesManager.GetProcessXml();
this.checkBoxProcessBinaries.Checked = MacroscopePreferencesManager.GetProcessBinaries();
this.numericUpDownRedirectChainsMaxHops.Value = MacroscopePreferencesManager.GetRedirectChainsMaxHops();
this.checkBoxWarnAboutInsecureLinks.Checked = MacroscopePreferencesManager.GetWarnAboutInsecureLinks();
this.checkBoxEnableTextIndexing.Checked = MacroscopePreferencesManager.GetEnableTextIndexing();
this.checkBoxCaseSensitiveTextIndexing.Checked = MacroscopePreferencesManager.GetCaseSensitiveTextIndexing();
this.checkBoxDisregardHtml5ElementNav.Checked = MacroscopePreferencesManager.GetDisregardHtml5ElementNav();
this.checkBoxDisregardHtml5ElementHeader.Checked = MacroscopePreferencesManager.GetDisregardHtml5ElementHeader();
this.checkBoxDisregardHtml5ElementFooter.Checked = MacroscopePreferencesManager.GetDisregardHtml5ElementFooter();
this.checkBoxDetectQrCodeInImage.Checked = MacroscopePreferencesManager.GetDetectQrCodeInImage();
/** SEO Options ---------------------------------------------------- **/
this.numericUpDownTitleMinLen.Value = MacroscopePreferencesManager.GetTitleMinLen();
this.numericUpDownTitleMaxLen.Value = MacroscopePreferencesManager.GetTitleMaxLen();
this.numericUpDownTitleMinWords.Value = MacroscopePreferencesManager.GetTitleMinWords();
this.numericUpDownTitleMaxWords.Value = MacroscopePreferencesManager.GetTitleMaxWords();
this.numericUpDownTitleMaxPixelWidth.Value = MacroscopePreferencesManager.GetTitleMaxPixelWidth();
this.numericUpDownDescriptionMinLen.Value = MacroscopePreferencesManager.GetDescriptionMinLen();
this.numericUpDownDescriptionMaxLen.Value = MacroscopePreferencesManager.GetDescriptionMaxLen();
this.numericUpDownDescriptionMinWords.Value = MacroscopePreferencesManager.GetDescriptionMinWords();
this.numericUpDownDescriptionMaxWords.Value = MacroscopePreferencesManager.GetDescriptionMaxWords();
this.numericUpDownMaxHeadingDepth.Value = MacroscopePreferencesManager.GetMaxHeadingDepth();
this.checkBoxAnalyzeKeywordsInText.Checked = MacroscopePreferencesManager.GetAnalyzeKeywordsInText();
this.checkBoxAnalyzeTextReadability.Checked = MacroscopePreferencesManager.GetAnalyzeTextReadability();
this.comboBoxAnalyzeTextReadabilityEnglishAlgorithm.SelectedIndex = (int)MacroscopePreferencesManager.GetAnalyzeTextReadabilityEnglishAlgorithm();
this.checkBoxEnableLevenshteinDeduplication.Checked = MacroscopePreferencesManager.GetEnableLevenshteinDeduplication();
this.comboBoxLevenshteinAnalysisLevel.SelectedIndex = (int)MacroscopePreferencesManager.GetLevenshteinAnalysisLevel();
this.numericUpDownMaxLevenshteinSizeDifference.Value = MacroscopePreferencesManager.GetMaxLevenshteinSizeDifference();
this.numericUpDownMaxLevenshteinDistance.Value = MacroscopePreferencesManager.GetMaxLevenshteinDistance();
this.checkBoxAnalyzeClickPaths.Checked = MacroscopePreferencesManager.GetAnalyzeClickPaths();
// TODO: Finish implementing click path analysis:
#if DEBUG
this.groupBoxPageNavigationAnalysis.Visible = true;
#else
this.groupBoxPageNavigationAnalysis.Visible = false;
#endif
/** Custom Filter Options ------------------------------------------ **/
this.checkBoxCustomFiltersEnable.Checked = MacroscopePreferencesManager.GetCustomFiltersEnable();
this.numericUpDownCustomFiltersMaxItems.Value = MacroscopePreferencesManager.GetCustomFiltersMaxItems();
this.checkBoxCustomFiltersApplyToHtml.Checked = MacroscopePreferencesManager.GetCustomFiltersApplyToHtml();
this.checkBoxCustomFiltersApplyToCss.Checked = MacroscopePreferencesManager.GetCustomFiltersApplyToCss();
this.checkBoxCustomFiltersApplyToJavascripts.Checked = MacroscopePreferencesManager.GetCustomFiltersApplyToJavascripts();
this.checkBoxCustomFiltersApplyToText.Checked = MacroscopePreferencesManager.GetCustomFiltersApplyToText();
this.checkBoxCustomFiltersApplyToXml.Checked = MacroscopePreferencesManager.GetCustomFiltersApplyToXml();
/** Extractor Options ---------------------------------------------- **/
this.checkBoxDataExtractorsEnable.Checked = MacroscopePreferencesManager.GetDataExtractorsEnable();
this.checkBoxDataExtractorsCleanWhiteSpace.Checked = MacroscopePreferencesManager.GetDataExtractorsCleanWhiteSpace();
this.numericUpDownDataExtractorsMaxItemsCssSelectors.Value = MacroscopePreferencesManager.GetDataExtractorsMaxItemsCssSelectors();
this.numericUpDownDataExtractorsMaxItemsRegexes.Value = MacroscopePreferencesManager.GetDataExtractorsMaxItemsRegexes();
this.numericUpDownDataExtractorsMaxItemsXpaths.Value = MacroscopePreferencesManager.GetDataExtractorsMaxItemsXpaths();
this.checkBoxDataExtractorsApplyToHtml.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToHtml();
this.checkBoxDataExtractorsApplyToCss.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToCss();
this.checkBoxDataExtractorsApplyToJavascripts.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToJavascripts();
this.checkBoxDataExtractorsApplyToText.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToText();
this.checkBoxDataExtractorsApplyToPdf.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToPdf();
this.checkBoxDataExtractorsApplyToXml.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToXml();
/** Export Options ------------------------------------------------- **/
this.checkBoxSitemapIncludeLinkedPdfs.Checked = MacroscopePreferencesManager.GetSitemapIncludeLinkedPdfs();
/** Ignore Errors Settings ----------------------------------------- **/
this.checkBoxIgnoreErrors410.Checked = MacroscopePreferencesManager.GetIgnoreErrors410();
this.checkBoxIgnoreErrors451.Checked = MacroscopePreferencesManager.GetIgnoreErrors451();
}
}
/**************************************************************************/
protected override void RenderListView(
List <ListViewItem> ListViewItems,
MacroscopeDocumentCollection DocCollection,
MacroscopeDocument msDoc,
string Url
)
{
bool Proceed = false;
if (msDoc.GetIsExternal())
{
return;
}
if (msDoc.GetIsRedirect())
{
return;
}
switch (msDoc.GetDocumentType())
{
case MacroscopeConstants.DocumentType.HTML:
Proceed = true;
break;
case MacroscopeConstants.DocumentType.PDF:
Proceed = true;
break;
default:
break;
}
if (Proceed)
{
ListViewItem lvItem = null;
int Occurrences = 0;
string PageLanguage = msDoc.GetIsoLanguageCode();
string DetectedLanguage = msDoc.GetDescriptionLanguage();
string Description = msDoc.GetDescription();
int DescriptionLength = msDoc.GetDescriptionLength();
string PairKey = string.Join(":", UrlToDigest(Url), UrlToDigest(Description));
if (string.IsNullOrEmpty(PageLanguage))
{
PageLanguage = "";
}
if (string.IsNullOrEmpty(DetectedLanguage))
{
DetectedLanguage = "";
}
if (DescriptionLength > 0)
{
Occurrences = DocCollection.GetStatsDescriptionCount(msDoc: msDoc);
}
else
{
Description = "MISSING";
}
if (this.DisplayListView.Items.ContainsKey(PairKey))
{
try
{
lvItem = this.DisplayListView.Items[PairKey];
lvItem.SubItems[ColUrl].Text = Url;
lvItem.SubItems[ColPageLanguage].Text = PageLanguage;
lvItem.SubItems[ColDetectedLanguage].Text = DetectedLanguage;
lvItem.SubItems[ColOccurences].Text = Occurrences.ToString();
lvItem.SubItems[ColDescriptionText].Text = Description;
lvItem.SubItems[ColLength].Text = DescriptionLength.ToString();
}
catch (Exception ex)
{
DebugMsg(string.Format("MacroscopeDisplayDescriptions 1: {0}", ex.Message));
}
}
else
{
try
{
lvItem = new ListViewItem(PairKey);
lvItem.UseItemStyleForSubItems = false;
lvItem.Name = PairKey;
lvItem.SubItems[ColUrl].Text = Url;
lvItem.SubItems.Add(PageLanguage);
lvItem.SubItems.Add(DetectedLanguage);
lvItem.SubItems.Add(Occurrences.ToString());
lvItem.SubItems.Add(Description);
lvItem.SubItems.Add(DescriptionLength.ToString());
ListViewItems.Add(lvItem);
}
catch (Exception ex)
{
DebugMsg(string.Format("MacroscopeDisplayDescriptions 2: {0}", ex.Message));
}
}
if (lvItem != null)
{
lvItem.ForeColor = Color.Blue;
// URL -------------------------------------------------------------//
if (msDoc.GetIsInternal())
{
lvItem.SubItems[ColUrl].ForeColor = Color.Green;
}
else
{
lvItem.SubItems[ColUrl].ForeColor = Color.Gray;
}
// Description Language --------------------------------------------//
if (msDoc.GetIsInternal())
{
lvItem.SubItems[ColPageLanguage].ForeColor = Color.Green;
lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Green;
if (DetectedLanguage != PageLanguage)
{
lvItem.SubItems[ColPageLanguage].ForeColor = Color.Red;
lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Red;
}
}
else
{
lvItem.SubItems[ColPageLanguage].ForeColor = Color.Gray;
lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Gray;
}
// Check Description Length ----------------------------------------//
if (msDoc.GetIsInternal())
{
if (DescriptionLength < MacroscopePreferencesManager.GetDescriptionMinLen())
{
lvItem.SubItems[ColUrl].ForeColor = Color.Red;
lvItem.SubItems[ColOccurences].ForeColor = Color.Red;
lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Red;
lvItem.SubItems[ColDescriptionText].ForeColor = Color.Red;
lvItem.SubItems[ColLength].ForeColor = Color.Red;
}
else
if (DescriptionLength > MacroscopePreferencesManager.GetDescriptionMaxLen())
{
lvItem.SubItems[ColUrl].ForeColor = Color.Red;
lvItem.SubItems[ColOccurences].ForeColor = Color.Red;
lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Red;
lvItem.SubItems[ColDescriptionText].ForeColor = Color.Red;
lvItem.SubItems[ColLength].ForeColor = Color.Red;
}
else
{
lvItem.SubItems[ColOccurences].ForeColor = Color.Green;
lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Green;
lvItem.SubItems[ColDescriptionText].ForeColor = Color.Green;
lvItem.SubItems[ColLength].ForeColor = Color.Green;
}
}
else
{
lvItem.SubItems[ColOccurences].ForeColor = Color.Gray;
lvItem.SubItems[ColDetectedLanguage].ForeColor = Color.Gray;
lvItem.SubItems[ColDescriptionText].ForeColor = Color.Gray;
lvItem.SubItems[ColLength].ForeColor = Color.Gray;
}
}
}
}
/**************************************************************************/
private void BuildWorksheetPageDescriptions(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
{
ws.Cell(iRow, iCol).Value = "URL";
iCol++;
ws.Cell(iRow, iCol).Value = "Page Language";
iCol++;
ws.Cell(iRow, iCol).Value = "Detected Language";
iCol++;
ws.Cell(iRow, iCol).Value = "Occurrences";
iCol++;
ws.Cell(iRow, iCol).Value = "Description";
iCol++;
ws.Cell(iRow, iCol).Value = "Description Length";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
bool Proceed = false;
if (msDoc.GetIsExternal())
{
continue;
}
if (msDoc.GetIsRedirect())
{
continue;
}
switch (msDoc.GetDocumentType())
{
case MacroscopeConstants.DocumentType.HTML:
Proceed = true;
break;
case MacroscopeConstants.DocumentType.PDF:
Proceed = true;
break;
default:
break;
}
if (Proceed)
{
iCol = 1;
string PageLanguage = msDoc.GetIsoLanguageCode();
string DetectedLanguage = msDoc.GetTitleLanguage();
string Description = msDoc.GetDescription();
int Occurrences = 0;
int DescriptionLength = msDoc.GetDescriptionLength();
if (DescriptionLength > 0)
{
Occurrences = DocCollection.GetStatsDescriptionCount(msDoc: msDoc);
}
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
if (msDoc.GetIsInternal())
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Gray);
}
iCol++;
switch (msDoc.GetDocumentType())
{
case MacroscopeConstants.DocumentType.HTML:
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(PageLanguage));
break;
case MacroscopeConstants.DocumentType.PDF:
this.InsertAndFormatContentCell(ws, iRow, iCol, PageLanguage);
break;
default:
break;
}
if (PageLanguage != DetectedLanguage)
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(DetectedLanguage));
if (PageLanguage != DetectedLanguage)
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, Occurrences);
if (Occurrences > 1)
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Orange);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, this.FormatIfMissing(Description));
if (DescriptionLength <= 0)
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
ws.Cell(iRow, iCol).Value = "MISSING";
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, DescriptionLength);
if (DescriptionLength < MacroscopePreferencesManager.GetDescriptionMinLen())
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
else
if (DescriptionLength > MacroscopePreferencesManager.GetDescriptionMaxLen())
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Red);
}
else
{
ws.Cell(iRow, iCol).Style.Font.SetFontColor(XLColor.Green);
}
iRow++;
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
