/**************************************************************************/

        public void SetPrefsFormControlFields()
        {
            { // Configure Display Options
                this.checkBoxPauseDisplayDuringScan.Checked = MacroscopePreferencesManager.GetPauseDisplayDuringScan();
                this.checkBoxShowProgressDialogues.Checked  = MacroscopePreferencesManager.GetShowProgressDialogues();
            }

            { //Configure Form Fields
              /** Spidering Control ---------------------------------------------- **/

                this.numericUpDownDepth.Minimum = -1;
                this.numericUpDownDepth.Maximum = 10000;

                this.numericUpDownPageLimit.Minimum = -1;
                this.numericUpDownPageLimit.Maximum = 10000;

                this.numericUpDownCrawlDelay.Minimum = 0;
                this.numericUpDownCrawlDelay.Maximum = 60;

                this.numericUpDownMaxRetries.Minimum = 0;
                this.numericUpDownMaxRetries.Maximum = 10;
            }

            {
                /** WebProxy Options ----------------------------------------------- **/

                this.comboBoxProxyType.SelectedIndex = (int)MacroscopePreferencesManager.GetProxyType();

                /** Server Certificate Options --------------------------------------- **/

                this.checkBoxServerCertificateValidation.Checked = MacroscopePreferencesManager.GetServerCertificateValidation();

                /** Spidering Control ---------------------------------------------- **/

                this.numericUpDownMaxThreads.Value     = MacroscopePreferencesManager.GetMaxThreads();
                this.numericUpDownDepth.Value          = MacroscopePreferencesManager.GetDepth();
                this.numericUpDownPageLimit.Value      = MacroscopePreferencesManager.GetPageLimit();
                this.numericUpDownCrawlDelay.Value     = MacroscopePreferencesManager.GetCrawlDelay();
                this.numericUpDownRequestTimeout.Value = (Decimal)MacroscopePreferencesManager.GetRequestTimeout();
                this.numericUpDownMaxRetries.Value     = (Decimal)MacroscopePreferencesManager.GetMaxRetries();

                this.checkBoxCrawlStrictUrlCheck.Checked = MacroscopePreferencesManager.GetCrawlStrictUrlCheck();

                this.checkBoxCheckExternalLinks.Checked = MacroscopePreferencesManager.GetCheckExternalLinks();
                this.checkBoxFetchExternalLinks.Checked = MacroscopePreferencesManager.GetFetchExternalLinks();

                this.checkBoxFollowRobotsProtocol.Checked = MacroscopePreferencesManager.GetFollowRobotsProtocol();
                this.checkBoxFollowSitemapLinks.Checked   = MacroscopePreferencesManager.GetFollowSitemapLinks();
                this.checkBoxProbeHumansText.Checked      = MacroscopePreferencesManager.GetProbeHumansText();

                this.checkBoxCheckRedirects.Checked  = MacroscopePreferencesManager.GetCheckRedirects();
                this.checkBoxFollowRedirects.Checked = MacroscopePreferencesManager.GetFollowRedirects();

                this.checkBoxFollowNoFollow.Checked       = MacroscopePreferencesManager.GetFollowNoFollow();
                this.checkBoxIgnoreQueries.Checked        = MacroscopePreferencesManager.GetIgnoreQueries();
                this.checkBoxIgnoreHashFragments.Checked  = MacroscopePreferencesManager.GetIgnoreHashFragments();
                this.checkBoxFollowCanonicalLinks.Checked = MacroscopePreferencesManager.GetFollowCanonicalLinks();
                this.checkBoxFollowAlternateLinks.Checked = MacroscopePreferencesManager.GetFollowAlternateLinks();
                this.checkBoxFollowHrefLangLinks.Checked  = MacroscopePreferencesManager.GetFollowHrefLangLinks();
                this.checkBoxDowncaseLinks.Checked        = MacroscopePreferencesManager.GetDowncaseLinks();

                this.checkBoxFetchStylesheets.Checked = MacroscopePreferencesManager.GetFetchStylesheets();
                this.checkBoxFetchJavascripts.Checked = MacroscopePreferencesManager.GetFetchJavascripts();
                this.checkBoxFetchImages.Checked      = MacroscopePreferencesManager.GetFetchImages();
                this.checkBoxFetchAudio.Checked       = MacroscopePreferencesManager.GetFetchAudio();
                this.checkBoxFetchVideo.Checked       = MacroscopePreferencesManager.GetFetchVideo();
                this.checkBoxFetchXml.Checked         = MacroscopePreferencesManager.GetFetchXml();
                this.checkBoxFetchBinaries.Checked    = MacroscopePreferencesManager.GetFetchBinaries();

                this.checkBoxScanSitesInList.Checked = MacroscopePreferencesManager.GetScanSitesInList();

                this.checkBoxProbeParentFolderUrls.Checked = MacroscopePreferencesManager.GetProbeParentFolderUrls();

                this.checkBoxProbeHead404sWithGet.Checked = MacroscopePreferencesManager.GetProbeHead404sWithGet();

                /** Analysis Options ----------------------------------------------- **/

                this.checkBoxResolveAddresses.Checked = MacroscopePreferencesManager.GetResolveAddresses();

                this.checkBoxCheckHreflangs.Checked = MacroscopePreferencesManager.GetCheckHreflangs();
                this.checkBoxDetectLanguage.Checked = MacroscopePreferencesManager.GetDetectLanguage();

                this.checkBoxProcessStylesheets.Checked = MacroscopePreferencesManager.GetProcessStylesheets();
                this.checkBoxProcessJavascripts.Checked = MacroscopePreferencesManager.GetProcessJavascripts();
                this.checkBoxProcessImages.Checked      = MacroscopePreferencesManager.GetProcessImages();
                this.checkBoxProcessPdfs.Checked        = MacroscopePreferencesManager.GetProcessPdfs();
                this.checkBoxProcessAudio.Checked       = MacroscopePreferencesManager.GetProcessAudio();
                this.checkBoxProcessVideo.Checked       = MacroscopePreferencesManager.GetProcessVideo();
                this.checkBoxProcessXml.Checked         = MacroscopePreferencesManager.GetProcessXml();
                this.checkBoxProcessBinaries.Checked    = MacroscopePreferencesManager.GetProcessBinaries();

                this.numericUpDownRedirectChainsMaxHops.Value = MacroscopePreferencesManager.GetRedirectChainsMaxHops();

                this.checkBoxWarnAboutInsecureLinks.Checked = MacroscopePreferencesManager.GetWarnAboutInsecureLinks();

                this.checkBoxEnableTextIndexing.Checked        = MacroscopePreferencesManager.GetEnableTextIndexing();
                this.checkBoxCaseSensitiveTextIndexing.Checked = MacroscopePreferencesManager.GetCaseSensitiveTextIndexing();

                this.checkBoxDisregardHtml5ElementNav.Checked    = MacroscopePreferencesManager.GetDisregardHtml5ElementNav();
                this.checkBoxDisregardHtml5ElementHeader.Checked = MacroscopePreferencesManager.GetDisregardHtml5ElementHeader();
                this.checkBoxDisregardHtml5ElementFooter.Checked = MacroscopePreferencesManager.GetDisregardHtml5ElementFooter();

                this.checkBoxDetectQrCodeInImage.Checked = MacroscopePreferencesManager.GetDetectQrCodeInImage();

                /** SEO Options ---------------------------------------------------- **/

                this.numericUpDownTitleMinLen.Value        = MacroscopePreferencesManager.GetTitleMinLen();
                this.numericUpDownTitleMaxLen.Value        = MacroscopePreferencesManager.GetTitleMaxLen();
                this.numericUpDownTitleMinWords.Value      = MacroscopePreferencesManager.GetTitleMinWords();
                this.numericUpDownTitleMaxWords.Value      = MacroscopePreferencesManager.GetTitleMaxWords();
                this.numericUpDownTitleMaxPixelWidth.Value = MacroscopePreferencesManager.GetTitleMaxPixelWidth();

                this.numericUpDownDescriptionMinLen.Value   = MacroscopePreferencesManager.GetDescriptionMinLen();
                this.numericUpDownDescriptionMaxLen.Value   = MacroscopePreferencesManager.GetDescriptionMaxLen();
                this.numericUpDownDescriptionMinWords.Value = MacroscopePreferencesManager.GetDescriptionMinWords();
                this.numericUpDownDescriptionMaxWords.Value = MacroscopePreferencesManager.GetDescriptionMaxWords();

                this.numericUpDownMaxHeadingDepth.Value = MacroscopePreferencesManager.GetMaxHeadingDepth();

                this.checkBoxAnalyzeKeywordsInText.Checked  = MacroscopePreferencesManager.GetAnalyzeKeywordsInText();
                this.checkBoxAnalyzeTextReadability.Checked = MacroscopePreferencesManager.GetAnalyzeTextReadability();
                this.comboBoxAnalyzeTextReadabilityEnglishAlgorithm.SelectedIndex = (int)MacroscopePreferencesManager.GetAnalyzeTextReadabilityEnglishAlgorithm();

                this.checkBoxEnableLevenshteinDeduplication.Checked  = MacroscopePreferencesManager.GetEnableLevenshteinDeduplication();
                this.comboBoxLevenshteinAnalysisLevel.SelectedIndex  = (int)MacroscopePreferencesManager.GetLevenshteinAnalysisLevel();
                this.numericUpDownMaxLevenshteinSizeDifference.Value = MacroscopePreferencesManager.GetMaxLevenshteinSizeDifference();
                this.numericUpDownMaxLevenshteinDistance.Value       = MacroscopePreferencesManager.GetMaxLevenshteinDistance();

                this.checkBoxAnalyzeClickPaths.Checked = MacroscopePreferencesManager.GetAnalyzeClickPaths();

                // TODO: Finish implementing click path analysis:
#if DEBUG
                this.groupBoxPageNavigationAnalysis.Visible = true;
#else
                this.groupBoxPageNavigationAnalysis.Visible = false;
#endif

                /** Custom Filter Options ------------------------------------------ **/

                this.checkBoxCustomFiltersEnable.Checked      = MacroscopePreferencesManager.GetCustomFiltersEnable();
                this.numericUpDownCustomFiltersMaxItems.Value = MacroscopePreferencesManager.GetCustomFiltersMaxItems();

                this.checkBoxCustomFiltersApplyToHtml.Checked        = MacroscopePreferencesManager.GetCustomFiltersApplyToHtml();
                this.checkBoxCustomFiltersApplyToCss.Checked         = MacroscopePreferencesManager.GetCustomFiltersApplyToCss();
                this.checkBoxCustomFiltersApplyToJavascripts.Checked = MacroscopePreferencesManager.GetCustomFiltersApplyToJavascripts();
                this.checkBoxCustomFiltersApplyToText.Checked        = MacroscopePreferencesManager.GetCustomFiltersApplyToText();
                this.checkBoxCustomFiltersApplyToXml.Checked         = MacroscopePreferencesManager.GetCustomFiltersApplyToXml();

                /** Extractor Options ---------------------------------------------- **/

                this.checkBoxDataExtractorsEnable.Checked          = MacroscopePreferencesManager.GetDataExtractorsEnable();
                this.checkBoxDataExtractorsCleanWhiteSpace.Checked = MacroscopePreferencesManager.GetDataExtractorsCleanWhiteSpace();

                this.numericUpDownDataExtractorsMaxItemsCssSelectors.Value = MacroscopePreferencesManager.GetDataExtractorsMaxItemsCssSelectors();
                this.numericUpDownDataExtractorsMaxItemsRegexes.Value      = MacroscopePreferencesManager.GetDataExtractorsMaxItemsRegexes();
                this.numericUpDownDataExtractorsMaxItemsXpaths.Value       = MacroscopePreferencesManager.GetDataExtractorsMaxItemsXpaths();

                this.checkBoxDataExtractorsApplyToHtml.Checked        = MacroscopePreferencesManager.GetDataExtractorsApplyToHtml();
                this.checkBoxDataExtractorsApplyToCss.Checked         = MacroscopePreferencesManager.GetDataExtractorsApplyToCss();
                this.checkBoxDataExtractorsApplyToJavascripts.Checked = MacroscopePreferencesManager.GetDataExtractorsApplyToJavascripts();
                this.checkBoxDataExtractorsApplyToText.Checked        = MacroscopePreferencesManager.GetDataExtractorsApplyToText();
                this.checkBoxDataExtractorsApplyToPdf.Checked         = MacroscopePreferencesManager.GetDataExtractorsApplyToPdf();
                this.checkBoxDataExtractorsApplyToXml.Checked         = MacroscopePreferencesManager.GetDataExtractorsApplyToXml();

                /** Export Options ------------------------------------------------- **/

                this.checkBoxSitemapIncludeLinkedPdfs.Checked = MacroscopePreferencesManager.GetSitemapIncludeLinkedPdfs();

                /** Ignore Errors Settings ----------------------------------------- **/

                this.checkBoxIgnoreErrors410.Checked = MacroscopePreferencesManager.GetIgnoreErrors410();
                this.checkBoxIgnoreErrors451.Checked = MacroscopePreferencesManager.GetIgnoreErrors451();
            }
        }
Beispiel #2
0
        /**************************************************************************/

        public Boolean CanApplyDataExtractorsToDocument(MacroscopeDocument msDoc)
        {
            Boolean CanApply = true;

            if (
                (msDoc == null) ||
                (msDoc.GetIsRedirect()) ||
                (msDoc.GetStatusCode() != HttpStatusCode.OK) ||
                (!msDoc.GetIsInternal()))
            {
                CanApply = false;
            }
            else
            {
                if (
                    !(msDoc.GetIsHtml() ||
                      msDoc.GetIsCss() ||
                      msDoc.GetIsJavascript() ||
                      msDoc.GetIsText() ||
                      msDoc.GetIsXml()))
                {
                    CanApply = false;
                }
                else
                {
                    if (
                        msDoc.GetIsHtml() &&
                        (!MacroscopePreferencesManager.GetDataExtractorsApplyToHtml()))
                    {
                        CanApply = false;
                    }
                    else
                    if (
                        msDoc.GetIsCss() &&
                        (!MacroscopePreferencesManager.GetDataExtractorsApplyToCss()))
                    {
                        CanApply = false;
                    }
                    else
                    if (
                        msDoc.GetIsJavascript() &&
                        (!MacroscopePreferencesManager.GetDataExtractorsApplyToJavascripts()))
                    {
                        CanApply = false;
                    }
                    else
                    if (
                        msDoc.GetIsText() &&
                        (!MacroscopePreferencesManager.GetDataExtractorsApplyToText()))
                    {
                        CanApply = false;
                    }
                    else
                    if (
                        msDoc.GetIsXml() &&
                        (!MacroscopePreferencesManager.GetDataExtractorsApplyToXml()))
                    {
                        CanApply = false;
                    }
                }
            }

            return(CanApply);
        }
Beispiel #3
0
        /**************************************************************************/

        private void ProcessTextPage()
        {
            List <string>   TextDoc = new List <string> ();
            HttpWebRequest  req     = null;
            HttpWebResponse res     = null;
            string          ResponseErrorCondition = null;
            Boolean         IsAuthenticating       = false;

            try
            {
                req           = WebRequest.CreateHttp(this.DocUrl);
                req.Method    = "GET";
                req.Timeout   = this.Timeout;
                req.KeepAlive = false;
                req.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;

                this.PrepareRequestHttpHeaders(req: req);

                IsAuthenticating = this.AuthenticateRequest(req);

                MacroscopePreferencesManager.EnableHttpProxy(req);

                res = ( HttpWebResponse )req.GetResponse();
            }
            catch (UriFormatException ex)
            {
                DebugMsg(string.Format("ProcessTextPage :: UriFormatException: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
            }
            catch (WebException ex)
            {
                DebugMsg(string.Format("ProcessTextPage :: WebException: {0}", ex.Message));
                DebugMsg(string.Format("ProcessTextPage :: WebException: {0}", this.DocUrl));
                DebugMsg(string.Format("ProcessTextPage :: WebExceptionStatus: {0}", ex.Status));
                ResponseErrorCondition = ex.Status.ToString();
            }

            if (res != null)
            {
                string RawData = "";

                this.ProcessResponseHttpHeaders(req, res);

                if (IsAuthenticating)
                {
                    this.VerifyOrPurgeCredential();
                }

                // Get Response Body
                try
                {
                    DebugMsg(string.Format("MIME TYPE: {0}", this.MimeType));

                    Stream       ResponseStream       = res.GetResponseStream();
                    StreamReader ResponseStreamReader = new StreamReader(ResponseStream, Encoding.UTF8); // Assume UTF-8
                    RawData = ResponseStreamReader.ReadToEnd();

                    this.ContentLength = RawData.Length; // May need to find bytes length

                    this.SetWasDownloaded(true);

                    this.SetChecksum(RawData);
                }
                catch (WebException ex)
                {
                    DebugMsg(string.Format("WebException: {0}", ex.Message));

                    if (ex.Response != null)
                    {
                        this.SetStatusCode((( HttpWebResponse )ex.Response).StatusCode);
                    }
                    else
                    {
                        this.SetStatusCode(( HttpStatusCode )ex.Status);
                    }

                    RawData            = "";
                    this.ContentLength = 0;
                }
                catch (Exception ex)
                {
                    DebugMsg(string.Format("Exception: {0}", ex.Message));
                    this.SetStatusCode(HttpStatusCode.BadRequest);
                    RawData            = "";
                    this.ContentLength = 0;
                }

                /** ---------------------------------------------------------------- **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    string [] Lines = Regex.Split(RawData, @"[\r\n]+");
                    TextDoc = Lines.ToList();

                    DebugMsg(string.Format("TextDoc: {0}", TextDoc.Count));
                }
                else
                {
                    DebugMsg(string.Format("RawData: {0}", "EMPTY"));
                }

                /** Custom Filters ------------------------------------------------- **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetCustomFiltersEnable() &&
                        MacroscopePreferencesManager.GetCustomFiltersApplyToText())
                    {
                        MacroscopeCustomFilters CustomFilter = this.DocCollection.GetJobMaster().GetCustomFilter();

                        if ((CustomFilter != null) && (CustomFilter.IsEnabled()))
                        {
                            this.ProcessGenericCustomFiltered(
                                CustomFilter: CustomFilter,
                                GenericText: RawData
                                );
                        }
                    }
                }

                /** Data Extractors ------------------------------------------------ **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetDataExtractorsEnable() &&
                        MacroscopePreferencesManager.GetDataExtractorsApplyToText())
                    {
                        this.ProcessGenericDataExtractors(GenericText: RawData);
                    }
                }

                /** Process Text Document ------------------------------------------ **/

                if ((TextDoc != null) && (TextDoc.Count > 0))
                {
                    if (this.GetPath().EndsWith("robots.txt", StringComparison.InvariantCultureIgnoreCase))
                    {
                        this.ProcessRobotsTextOutlinks(TextDoc: TextDoc);
                    }

                    if (this.DetectSitemapTextDocument(TextDoc: TextDoc))
                    {
                        DebugMsg(string.Format("ProcessTextPage: {0} :: {1}", "SITEMAP DETECTED", this.GetUrl()));
                        this.SetIsSitemapText();
                        this.ProcessSitemapTextOutlinks(TextDoc);
                    }
                }

                /** ---------------------------------------------------------------- **/

                res.Close();

                res.Dispose();
            }

            if (ResponseErrorCondition != null)
            {
                this.ProcessErrorCondition(ResponseErrorCondition);
            }
        }
Beispiel #4
0
        /** -------------------------------------------------------------------- **/

        private async Task _ProcessTextPage()
        {
            List <string>                   TextDoc  = new List <string>();
            MacroscopeHttpTwoClient         Client   = this.DocCollection.GetJobMaster().GetHttpClient();
            MacroscopeHttpTwoClientResponse Response = null;
            string ResponseErrorCondition            = null;

            try
            {
                Response = await Client.Get(
                    this.GetUri(),
                    this.ConfigureTextPageRequestHeadersCallback,
                    this.PostProcessRequestHttpHeadersCallback
                    );
            }
            catch (MacroscopeDocumentException ex)
            {
                this.DebugMsg(string.Format("_ProcessTextPage :: MacroscopeDocumentException: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
                this.SetStatusCode(HttpStatusCode.BadRequest);
                this.AddRemark("_ProcessTextPage", ex.Message);
            }
            catch (Exception ex)
            {
                this.DebugMsg(string.Format("_ProcessTextPage :: Exception: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
                this.SetStatusCode(HttpStatusCode.BadRequest);
                this.AddRemark("_ProcessTextPage", ex.Message);
            }

            if (Response != null)
            {
                string RawData = "";

                this.ProcessResponseHttpHeaders(Response: Response);

                /** Get Response Body ---------------------------------------------- **/

                try
                {
                    DebugMsg(string.Format("MIME TYPE: {0}", this.MimeType));

                    RawData = Response.GetContentAsString();

                    this.SetContentLength(Length: RawData.Length); // May need to find bytes length

                    this.SetWasDownloaded(true);

                    this.SetChecksum(RawData);
                }
                catch (Exception ex)
                {
                    DebugMsg(string.Format("Exception: {0}", ex.Message));
                    this.SetStatusCode(HttpStatusCode.BadRequest);
                    RawData = "";
                    this.SetContentLength(Length: 0);
                }

                /** ---------------------------------------------------------------- **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    string[] Lines = Regex.Split(RawData, @"[\r\n]+");
                    TextDoc = Lines.ToList();

                    DebugMsg(string.Format("TextDoc: {0}", TextDoc.Count));
                }
                else
                {
                    DebugMsg(string.Format("RawData: {0}", "EMPTY"));
                }

                /** Custom Filters ------------------------------------------------- **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetCustomFiltersEnable() &&
                        MacroscopePreferencesManager.GetCustomFiltersApplyToText())
                    {
                        MacroscopeCustomFilters CustomFilter = this.DocCollection.GetJobMaster().GetCustomFilter();

                        if ((CustomFilter != null) && (CustomFilter.IsEnabled()))
                        {
                            this.ProcessGenericCustomFiltered(
                                CustomFilter: CustomFilter,
                                GenericText: RawData
                                );
                        }
                    }
                }

                /** Data Extractors ------------------------------------------------ **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetDataExtractorsEnable() &&
                        MacroscopePreferencesManager.GetDataExtractorsApplyToText())
                    {
                        this.ProcessGenericDataExtractors(GenericText: RawData);
                    }
                }

                /** Process Text Document ------------------------------------------ **/

                if ((TextDoc != null) && (TextDoc.Count > 0))
                {
                    this.SetDocumentText(Text: string.Join(Environment.NewLine, TextDoc));

                    if (this.GetPath().EndsWith("robots.txt", StringComparison.InvariantCultureIgnoreCase))
                    {
                        long?TextSize          = this.GetContentLength();
                        long?RobotsMaxTextSize = 1024 * 512;

                        this.ProcessRobotsTextOutlinks(TextDoc: TextDoc);

                        if (this.DetectSitemapTextDocument(TextDoc: TextDoc))
                        {
                            DebugMsg(string.Format("ProcessTextPage: {0} :: {1}", "SITEMAP DETECTED", this.GetUrl()));
                            this.SetDocumentType(Type: MacroscopeConstants.DocumentType.SITEMAPTEXT);
                            this.ProcessSitemapTextOutlinks(TextDoc: TextDoc);
                        }

                        if (TextSize > RobotsMaxTextSize)
                        {
                            this.AddRemark("ROBOTS_TOO_BIG", "Robots.txt is larger than 512KB");
                        }
                    }
                    else
                    {
                        if (this.GetIsInternal())
                        {
                            this.ProcessPureTextOutlinks(TextDoc: TextDoc, LinkType: MacroscopeConstants.InOutLinkType.PURETEXT);
                        }
                    }
                }
                else
                {
                    this.SetDocumentText(Text: "");
                }

                /** ---------------------------------------------------------------- **/
            }

            if (ResponseErrorCondition != null)
            {
                this.ProcessErrorCondition(ResponseErrorCondition);
            }
        }
    /**************************************************************************/

    public bool CanApplyDataExtractorsToDocument ( MacroscopeDocument msDoc )
    {

      bool CanApply = true;

      if (
        ( msDoc == null )
        || ( msDoc.GetIsRedirect() )
        || ( msDoc.GetStatusCode() != HttpStatusCode.OK )
        || ( !msDoc.GetIsInternal() ) )
      {
        CanApply = false;
      }
      else
      {

        if (
          !( msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.HTML )
          || msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.CSS )
          || msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.JAVASCRIPT )
          || msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.PDF )
          || msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.TEXT )
          || msDoc.IsDocumentType( Type: MacroscopeConstants.DocumentType.XML ) ) )
        {
          CanApply = false;
        }
        else
        {

          switch ( msDoc.GetDocumentType() )
          {
            case MacroscopeConstants.DocumentType.HTML:
              if ( !MacroscopePreferencesManager.GetDataExtractorsApplyToHtml() )
              {
                CanApply = false;
              }
              break;
            case MacroscopeConstants.DocumentType.CSS:
              if ( !MacroscopePreferencesManager.GetDataExtractorsApplyToCss() )
              {
                CanApply = false;
              }
              break;
            case MacroscopeConstants.DocumentType.JAVASCRIPT:
              if( !MacroscopePreferencesManager.GetDataExtractorsApplyToJavascripts() )
              {
                CanApply = false;
              }
              break;
            case MacroscopeConstants.DocumentType.PDF:
              if( !MacroscopePreferencesManager.GetDataExtractorsApplyToPdf() )
              {
                CanApply = false;
              }
              break;
            case MacroscopeConstants.DocumentType.TEXT:
              if ( !MacroscopePreferencesManager.GetDataExtractorsApplyToText() )
              {
                CanApply = false;
              }
              break;
            case MacroscopeConstants.DocumentType.XML:
              if ( !MacroscopePreferencesManager.GetDataExtractorsApplyToXml() )
              {
                CanApply = false;
              }
              break;
            default:
              break;
          }

        }

      }

      return ( CanApply );

    }