/**************************************************************************/
public void SetPrefsFormControlFields()
{
{ // Configure Display Options
this.checkBoxPauseDisplayDuringScan.Checked = MacroscopePreferencesManager.GetPauseDisplayDuringScan();
this.checkBoxShowProgressDialogues.Checked = MacroscopePreferencesManager.GetShowProgressDialogues();
}
{ //Configure Form Fields
/** Spidering Control ---------------------------------------------- **/
this.numericUpDownDepth.Minimum = -1;
this.numericUpDownDepth.Maximum = 10000;
this.numericUpDownPageLimit.Minimum = -1;
this.numericUpDownPageLimit.Maximum = 10000;
this.numericUpDownCrawlDelay.Minimum = 0;
this.numericUpDownCrawlDelay.Maximum = 60;
this.numericUpDownMaxRetries.Minimum = 0;
this.numericUpDownMaxRetries.Maximum = 10;
}
{
/** WebProxy Options ----------------------------------------------- **/
this.comboBoxProxyType.SelectedIndex = (int)MacroscopePreferencesManager.GetProxyType();
/** Server Certificate Options --------------------------------------- **/
this.checkBoxServerCertificateValidation.Checked = MacroscopePreferencesManager.GetServerCertificateValidation();
/** Spidering Control ---------------------------------------------- **/
this.numericUpDownMaxThreads.Value = MacroscopePreferencesManager.GetMaxThreads();
this.numericUpDownDepth.Value = MacroscopePreferencesManager.GetDepth();
this.numericUpDownPageLimit.Value = MacroscopePreferencesManager.GetPageLimit();
this.numericUpDownCrawlDelay.Value = MacroscopePreferencesManager.GetCrawlDelay();
this.numericUpDownRequestTimeout.Value = (Decimal)MacroscopePreferencesManager.GetRequestTimeout();
this.numericUpDownMaxRetries.Value = (Decimal)MacroscopePreferencesManager.GetMaxRetries();
this.checkBoxCrawlStrictUrlCheck.Checked = MacroscopePreferencesManager.GetCrawlStrictUrlCheck();
this.checkBoxCheckExternalLinks.Checked = MacroscopePreferencesManager.GetCheckExternalLinks();
this.checkBoxFetchExternalLinks.Checked = MacroscopePreferencesManager.GetFetchExternalLinks();
this.checkBoxFollowRobotsProtocol.Checked = MacroscopePreferencesManager.GetFollowRobotsProtocol();
this.checkBoxFollowSitemapLinks.Checked = MacroscopePreferencesManager.GetFollowSitemapLinks();
this.checkBoxProbeHumansText.Checked = MacroscopePreferencesManager.GetProbeHumansText();
this.checkBoxCheckRedirects.Checked = MacroscopePreferencesManager.GetCheckRedirects();
this.checkBoxFollowRedirects.Checked = MacroscopePreferencesManager.GetFollowRedirects();
this.checkBoxFollowNoFollow.Checked = MacroscopePreferencesManager.GetFollowNoFollow();
this.checkBoxIgnoreQueries.Checked = MacroscopePreferencesManager.GetIgnoreQueries();
this.checkBoxIgnoreHashFragments.Checked = MacroscopePreferencesManager.GetIgnoreHashFragments();
this.checkBoxFollowCanonicalLinks.Checked = MacroscopePreferencesManager.GetFollowCanonicalLinks();
this.checkBoxFollowAlternateLinks.Checked = MacroscopePreferencesManager.GetFollowAlternateLinks();
this.checkBoxFollowHrefLangLinks.Checked = MacroscopePreferencesManager.GetFollowHrefLangLinks();
this.checkBoxDowncaseLinks.Checked = MacroscopePreferencesManager.GetDowncaseLinks();
this.checkBoxFetchStylesheets.Checked = MacroscopePreferencesManager.GetFetchStylesheets();
this.checkBoxFetchJavascripts.Checked = MacroscopePreferencesManager.GetFetchJavascripts();
this.checkBoxFetchImages.Checked = MacroscopePreferencesManager.GetFetchImages();
this.checkBoxFetchAudio.Checked = MacroscopePreferencesManager.GetFetchAudio();
this.checkBoxFetchVideo.Checked = MacroscopePreferencesManager.GetFetchVideo();
this.checkBoxFetchXml.Checked = MacroscopePreferencesManager.GetFetchXml();
this.checkBoxFetchBinaries.Checked = MacroscopePreferencesManager.GetFetchBinaries();
this.checkBoxScanSitesInList.Checked = MacroscopePreferencesManager.GetScanSitesInList();
this.checkBoxProbeParentFolderUrls.Checked = MacroscopePreferencesManager.GetProbeParentFolderUrls();
this.checkBoxProbeHead404sWithGet.Checked = MacroscopePreferencesManager.GetProbeHead404sWithGet();
/** Analysis Options ----------------------------------------------- **/
this.checkBoxResolveAddresses.Checked = MacroscopePreferencesManager.GetResolveAddresses();
this.checkBoxCheckHreflangs.Checked = MacroscopePreferencesManager.GetCheckHreflangs();
this.checkBoxDetectLanguage.Checked = MacroscopePreferencesManager.GetDetectLanguage();
this.checkBoxProcessStylesheets.Checked = MacroscopePreferencesManager.GetProcessStylesheets();
this.checkBoxProcessJavascripts.Checked = MacroscopePreferencesManager.GetProcessJavascripts();
this.checkBoxProcessImages.Checked = MacroscopePreferencesManager.GetProcessImages();
this.checkBoxProcessPdfs.Checked = MacroscopePreferencesManager.GetProcessPdfs();
this.checkBoxProcessAudio.Checked = MacroscopePreferencesManager.GetProcessAudio();
this.checkBoxProcessVideo.Checked = MacroscopePreferencesManager.GetProcessVideo();
this.checkBoxProcessXml.Checked = MacroscopePreferencesManager.GetProcessXml();
this.checkBoxProcessBinaries.Checked = MacroscopePreferencesManager.GetProcessBinaries();
this.numericUpDownRedirectChainsMaxHops.Value = MacroscopePreferencesManager.GetRedirectChainsMaxHops();
this.checkBoxWarnAboutInsecureLinks.Checked = MacroscopePreferencesManager.GetWarnAboutInsecureLinks();
this.checkBoxEnableTextIndexing.Checked = MacroscopePreferencesManager.GetEnableTextIndexing();
this.checkBoxCaseSensitiveTextIndexing.Checked = MacroscopePreferencesManager.GetCaseSensitiveTextIndexing();
this.checkBoxDisregardHtml5ElementNav.Checked = MacroscopePreferencesManager.GetDisregardHtml5ElementNav();
this.checkBoxDisregardHtml5ElementHeader.Checked = MacroscopePreferencesManager.GetDisregardHtml5ElementHeader();
this.checkBoxDisregardHtml5ElementFooter.Checked = MacroscopePreferencesManager.GetDisregardHtml5ElementFooter();
this.checkBoxDetectQrCodeInImage.Checked = MacroscopePreferencesManager.GetDetectQrCodeInImage();
/** SEO Options ---------------------------------------------------- **/
this.numericUpDownTitleMinLen.Value = MacroscopePreferencesManager.GetTitleMinLen();
this.numericUpDownTitleMaxLen.Value = MacroscopePreferencesManager.GetTitleMaxLen();
this.numericUpDownTitleMinWords.Value = MacroscopePreferencesManager.GetTitleMinWords();
this.numericUpDownTitleMaxWords.Value = MacroscopePreferencesManager.GetTitleMaxWords();
this.numericUpDownTitleMaxPixelWidth.Value = MacroscopePreferencesManager.GetTitleMaxPixelWidth();
this.numericUpDownDescriptionMinLen.Value = MacroscopePreferencesManager.GetDescriptionMinLen();
this.numericUpDownDescriptionMaxLen.Value = MacroscopePreferencesManager.GetDescriptionMaxLen();
this.numericUpDownDescriptionMinWords.Value = MacroscopePreferencesManager.GetDescriptionMinWords();
this.numericUpDownDescriptionMaxWords.Value = MacroscopePreferencesManager.GetDescriptionMaxWords();
this.numericUpDownMaxHeadingDepth.Value = MacroscopePreferencesManager.GetMaxHeadingDepth();
this.checkBoxAnalyzeKeywordsInText.Checked = MacroscopePreferencesManager.GetAnalyzeKeywordsInText();
this.checkBoxAnalyzeTextReadability.Checked = MacroscopePreferencesManager.GetAnalyzeTextReadability();
this.comboBoxAnalyzeTextReadabilityEnglishAlgorithm.SelectedIndex = (int)MacroscopePreferencesManager.GetAnalyzeTextReadabilityEnglishAlgorithm();
this.checkBoxEnableLevenshteinDeduplication.Checked = MacroscopePreferencesManager.GetEnableLevenshteinDeduplication();
this.comboBoxLevenshteinAnalysisLevel.SelectedIndex = (int)MacroscopePreferencesManager.GetLevenshteinAnalysisLevel();
this.numericUpDownMaxLevenshteinSizeDifference.Value = MacroscopePreferencesManager.GetMaxLevenshteinSizeDifference();
this.numericUpDownMaxLevenshteinDistance.Value = MacroscopePreferencesManager.GetMaxLevenshteinDistance();
this.checkBoxAnalyzeClickPaths.Checked = MacroscopePreferencesManager.GetAnalyzeClickPaths();
// TODO: Finish implementing click path analysis:
#if DEBUG
this.groupBoxPageNavigationAnalysis.Visible = true;
#else
this.groupBoxPageNavigationAnalysis.Visible = false;
#endif
/** Custom Filter Options ------------------------------------------ **/
this.checkBoxCustomFiltersEnable.Checked = MacroscopePreferencesManager.GetCustomFiltersEnable();
this.numericUpDownCustomFiltersMaxItems.Value = MacroscopePreferencesManager.GetCustomFiltersMaxItems();
this.checkBoxCustomFiltersApplyToHtml.Checked = MacroscopePreferencesManager.GetCustomFiltersApplyToHtml();
this.checkBoxCustomFiltersApplyToCss.Checked = MacroscopePreferencesManager.GetCustomFiltersApplyToCss();
this.checkBoxCustomFiltersApplyToJavascripts.Checked = MacroscopePreferencesManager.GetCustomFiltersApplyToJavascripts();
this.checkBoxCustomFiltersApplyToText.Checked = MacroscopePreferencesManager.GetCustomFiltersApplyToText();
this.checkBoxCustomFiltersApplyToXml.Checked = MacroscopePreferencesManager.GetCustomFiltersApplyToXml();
/** Extractor Options ---------------------------------------------- **/
this.checkBoxDataExtractorsEnable.Checked = MacroscopePreferencesManager.GetDataExtractorsEnable();
this.checkBoxDataExtractorsCleanWhiteSpace.Checked = MacroscopePreferencesManager.GetDataExtractorsCleanWhiteSpace();
this.numericUpDownDataExtractorsMaxItemsCssSelectors.Value = MacroscopePreferencesManager.GetDataExtractorsMaxItemsCssSelectors();
this.numericUpDownDataExtractorsMaxItemsRegexes.Value = MacroscopePreferencesManager.GetDataExtractorsMaxItemsRegexes();
this.numericUpDownDataExtractorsMaxItemsXpaths.Value = MacroscopePreferencesManager.GetDataExtractorsMaxItemsXpaths();
this.checkBoxDataExtractorsApplyToHtml.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToHtml();
this.checkBoxDataExtractorsApplyToCss.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToCss();
this.checkBoxDataExtractorsApplyToJavascripts.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToJavascripts();
this.checkBoxDataExtractorsApplyToText.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToText();
this.checkBoxDataExtractorsApplyToPdf.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToPdf();
this.checkBoxDataExtractorsApplyToXml.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToXml();
/** Export Options ------------------------------------------------- **/
this.checkBoxSitemapIncludeLinkedPdfs.Checked = MacroscopePreferencesManager.GetSitemapIncludeLinkedPdfs();
/** Ignore Errors Settings ----------------------------------------- **/
this.checkBoxIgnoreErrors410.Checked = MacroscopePreferencesManager.GetIgnoreErrors410();
this.checkBoxIgnoreErrors451.Checked = MacroscopePreferencesManager.GetIgnoreErrors451();
}
}
/** -------------------------------------------------------------------- **/
private async Task _ProcessXmlPage()
{
XmlDocument XmlDoc = null;
MacroscopeHttpTwoClient Client = this.DocCollection.GetJobMaster().GetHttpClient();
MacroscopeHttpTwoClientResponse Response = null;
string ResponseErrorCondition = null;
try
{
Response = await Client.Get(
this.GetUri(),
this.ConfigureXmlPageRequestHeadersCallback,
this.PostProcessRequestHttpHeadersCallback
);
}
catch (MacroscopeDocumentException ex)
{
this.DebugMsg(string.Format("_ProcessXmlPage :: MacroscopeDocumentException: {0}", ex.Message));
ResponseErrorCondition = ex.Message;
this.SetStatusCode(HttpStatusCode.BadRequest);
this.AddRemark("_ProcessXmlPage", ex.Message);
}
catch (Exception ex)
{
this.DebugMsg(string.Format("_ProcessXmlPage :: Exception: {0}", ex.Message));
ResponseErrorCondition = ex.Message;
this.SetStatusCode(HttpStatusCode.BadRequest);
this.AddRemark("_ProcessXmlPage", ex.Message);
}
if (Response != null)
{
string RawData = "";
this.ProcessResponseHttpHeaders(Response: Response);
// Get Response Body
try
{
DebugMsg(string.Format("MIME TYPE: {0}", this.MimeType));
RawData = Response.GetContentAsString();
this.SetContentLength(Length: RawData.Length); // May need to find bytes length
this.SetWasDownloaded(true);
this.SetChecksum(RawData);
}
catch (Exception ex)
{
DebugMsg(string.Format("Exception: {0}", ex.Message));
this.SetStatusCode(HttpStatusCode.BadRequest);
RawData = "";
this.SetContentLength(Length: 0);
}
if (!string.IsNullOrEmpty(RawData))
{
XmlDoc = new XmlDocument();
try
{
XmlDoc.LoadXml(RawData);
}
catch (XmlException ex)
{
DebugMsg(string.Format("XmlException: {0}", ex.Message));
}
catch (Exception ex)
{
DebugMsg(string.Format("Exception: {0}", ex.Message));
}
DebugMsg(string.Format("XmlDoc: {0}", XmlDoc));
}
else
{
DebugMsg(string.Format("RawData: {0}", "EMPTY"));
}
/** Custom Filters ------------------------------------------------- **/
if (!string.IsNullOrEmpty(RawData))
{
if (
MacroscopePreferencesManager.GetCustomFiltersEnable() &&
MacroscopePreferencesManager.GetCustomFiltersApplyToXml())
{
MacroscopeCustomFilters CustomFilter = this.DocCollection.GetJobMaster().GetCustomFilter();
if ((CustomFilter != null) && (CustomFilter.IsEnabled()))
{
this.ProcessGenericCustomFiltered(
CustomFilter: CustomFilter,
GenericText: RawData
);
}
}
}
/** Data Extractors ------------------------------------------------ **/
if (!string.IsNullOrEmpty(RawData))
{
if (
MacroscopePreferencesManager.GetDataExtractorsEnable() &&
MacroscopePreferencesManager.GetDataExtractorsApplyToXml())
{
this.ProcessGenericDataExtractors(GenericText: RawData);
}
}
/** ---------------------------------------------------------------- **/
if ((XmlDoc != null) && (XmlDoc.DocumentElement != null))
{
if (this.DetectSitemapXmlDocument(XmlDoc))
{
DebugMsg(string.Format("ProcessXmlPage: {0} :: {1}", "SITEMAP DETECTED", this.GetUrl()));
this.SetDocumentType(Type: MacroscopeConstants.DocumentType.SITEMAPXML);
this.ProcessSitemapXmlOutlinks(XmlDoc);
}
}
/** ---------------------------------------------------------------- **/
if (RawData != null)
{
this.SetDocumentText(Text: RawData);
}
/** ---------------------------------------------------------------- **/
}
if (ResponseErrorCondition != null)
{
this.ProcessErrorCondition(ResponseErrorCondition);
}
}
/**************************************************************************/
private void ProcessXmlPage()
{
XmlDocument XmlDoc = null;
HttpWebRequest req = null;
HttpWebResponse res = null;
string ResponseErrorCondition = null;
Boolean IsAuthenticating = false;
try
{
req = WebRequest.CreateHttp(this.DocUrl);
req.Method = "GET";
req.Timeout = this.Timeout;
req.KeepAlive = false;
req.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
this.PrepareRequestHttpHeaders(req: req);
IsAuthenticating = this.AuthenticateRequest(req);
MacroscopePreferencesManager.EnableHttpProxy(req);
res = ( HttpWebResponse )req.GetResponse();
}
catch (UriFormatException ex)
{
DebugMsg(string.Format("ProcessXmlPage :: UriFormatException: {0}", ex.Message));
ResponseErrorCondition = ex.Message;
}
catch (WebException ex)
{
DebugMsg(string.Format("ProcessXmlPage :: WebException: {0}", ex.Message));
DebugMsg(string.Format("ProcessXmlPage :: WebException: {0}", this.DocUrl));
DebugMsg(string.Format("ProcessXmlPage :: WebExceptionStatus: {0}", ex.Status));
ResponseErrorCondition = ex.Status.ToString();
}
if (res != null)
{
string RawData = "";
this.ProcessResponseHttpHeaders(req, res);
if (IsAuthenticating)
{
this.VerifyOrPurgeCredential();
}
// Get Response Body
try
{
DebugMsg(string.Format("MIME TYPE: {0}", this.MimeType));
Stream ResponseStream = res.GetResponseStream();
StreamReader ResponseStreamReader = new StreamReader(ResponseStream, Encoding.UTF8); // Assume UTF-8
RawData = ResponseStreamReader.ReadToEnd();
this.ContentLength = RawData.Length; // May need to find bytes length
this.SetWasDownloaded(true);
this.SetChecksum(RawData);
}
catch (WebException ex)
{
DebugMsg(string.Format("WebException: {0}", ex.Message));
if (ex.Response != null)
{
this.SetStatusCode((( HttpWebResponse )ex.Response).StatusCode);
}
else
{
this.SetStatusCode(( HttpStatusCode )ex.Status);
}
RawData = "";
this.ContentLength = 0;
}
catch (Exception ex)
{
DebugMsg(string.Format("Exception: {0}", ex.Message));
this.SetStatusCode(HttpStatusCode.BadRequest);
RawData = "";
this.ContentLength = 0;
}
if (!string.IsNullOrEmpty(RawData))
{
XmlDoc = new XmlDocument();
try
{
XmlDoc.LoadXml(RawData);
}
catch (XmlException ex)
{
DebugMsg(string.Format("XmlException: {0}", ex.Message));
}
catch (Exception ex)
{
DebugMsg(string.Format("Exception: {0}", ex.Message));
}
DebugMsg(string.Format("XmlDoc: {0}", XmlDoc));
}
else
{
DebugMsg(string.Format("RawData: {0}", "EMPTY"));
}
/** Custom Filters ------------------------------------------------- **/
if (!string.IsNullOrEmpty(RawData))
{
if (
MacroscopePreferencesManager.GetCustomFiltersEnable() &&
MacroscopePreferencesManager.GetCustomFiltersApplyToXml())
{
MacroscopeCustomFilters CustomFilter = this.DocCollection.GetJobMaster().GetCustomFilter();
if ((CustomFilter != null) && (CustomFilter.IsEnabled()))
{
this.ProcessGenericCustomFiltered(
CustomFilter: CustomFilter,
GenericText: RawData
);
}
}
}
/** Data Extractors ------------------------------------------------ **/
if (!string.IsNullOrEmpty(RawData))
{
if (
MacroscopePreferencesManager.GetDataExtractorsEnable() &&
MacroscopePreferencesManager.GetDataExtractorsApplyToXml())
{
this.ProcessGenericDataExtractors(GenericText: RawData);
}
}
/** ---------------------------------------------------------------- **/
if ((XmlDoc != null) & (XmlDoc.DocumentElement != null))
{
if (this.DetectSitemapXmlDocument(XmlDoc))
{
DebugMsg(string.Format("ProcessXmlPage: {0} :: {1}", "SITEMAP DETECTED", this.GetUrl()));
this.SetIsSitemapXml();
this.ProcessSitemapXmlOutlinks(XmlDoc);
}
}
res.Close();
res.Dispose();
}
if (ResponseErrorCondition != null)
{
this.ProcessErrorCondition(ResponseErrorCondition);
}
}
/**************************************************************************/
public Boolean CanApplyDataExtractorsToDocument(MacroscopeDocument msDoc)
{
Boolean CanApply = true;
if (
(msDoc == null) ||
(msDoc.GetIsRedirect()) ||
(msDoc.GetStatusCode() != HttpStatusCode.OK) ||
(!msDoc.GetIsInternal()))
{
CanApply = false;
}
else
{
if (
!(msDoc.GetIsHtml() ||
msDoc.GetIsCss() ||
msDoc.GetIsJavascript() ||
msDoc.GetIsText() ||
msDoc.GetIsXml()))
{
CanApply = false;
}
else
{
if (
msDoc.GetIsHtml() &&
(!MacroscopePreferencesManager.GetDataExtractorsApplyToHtml()))
{
CanApply = false;
}
else
if (
msDoc.GetIsCss() &&
(!MacroscopePreferencesManager.GetDataExtractorsApplyToCss()))
{
CanApply = false;
}
else
if (
msDoc.GetIsJavascript() &&
(!MacroscopePreferencesManager.GetDataExtractorsApplyToJavascripts()))
{
CanApply = false;
}
else
if (
msDoc.GetIsText() &&
(!MacroscopePreferencesManager.GetDataExtractorsApplyToText()))
{
CanApply = false;
}
else
if (
msDoc.GetIsXml() &&
(!MacroscopePreferencesManager.GetDataExtractorsApplyToXml()))
{
CanApply = false;
}
}
}
return(CanApply);
}
/**************************************************************************/
public bool CanApplyDataExtractorsToDocument ( MacroscopeDocument msDoc )
{
bool CanApply = true;
if (
( msDoc == null )
|| ( msDoc.GetIsRedirect() )
|| ( msDoc.GetStatusCode() != HttpStatusCode.OK )
|| ( !msDoc.GetIsInternal() ) )
{
CanApply = false;
}
else
{
if (
!( msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.HTML )
|| msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.CSS )
|| msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.JAVASCRIPT )
|| msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.PDF )
|| msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.TEXT )
|| msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.XML ) ) )
{
CanApply = false;
}
else
{
switch ( msDoc.GetDocumentType() )
{
case MacroscopeConstants.DocumentType.HTML:
if ( !MacroscopePreferencesManager.GetDataExtractorsApplyToHtml() )
{
CanApply = false;
}
break;
case MacroscopeConstants.DocumentType.CSS:
if ( !MacroscopePreferencesManager.GetDataExtractorsApplyToCss() )
{
CanApply = false;
}
break;
case MacroscopeConstants.DocumentType.JAVASCRIPT:
if( !MacroscopePreferencesManager.GetDataExtractorsApplyToJavascripts() )
{
CanApply = false;
}
break;
case MacroscopeConstants.DocumentType.PDF:
if( !MacroscopePreferencesManager.GetDataExtractorsApplyToPdf() )
{
CanApply = false;
}
break;
case MacroscopeConstants.DocumentType.TEXT:
if ( !MacroscopePreferencesManager.GetDataExtractorsApplyToText() )
{
CanApply = false;
}
break;
case MacroscopeConstants.DocumentType.XML:
if ( !MacroscopePreferencesManager.GetDataExtractorsApplyToXml() )
{
CanApply = false;
}
break;
default:
break;
}
}
}
return ( CanApply );
}
