Esempio n. 1
0
        /**************************************************************************/

        public void ConfigureCustomFilterForm(
            MacroscopeCustomFilterForm NewContainerForm,
            MacroscopeCustomFilters NewCustomFilter
            )
        {
            this.ContainerForm = NewContainerForm;

            this.CustomFilter = NewCustomFilter;

            int Max = this.CustomFilter.GetSize();
            TableLayoutPanel Table = this.tableLayoutPanelControlsGrid;

            Table.ColumnCount = 3;
            Table.RowCount    = Max + 1;

            {
                List <string> ColumnLabels = new List <string> (3)
                {
                    "",
                    "Filter Action",
                    "Search String"
                };

                for (int i = 0; i < ColumnLabels.Count; i++)
                {
                    Label TextLabelCol = new Label();
                    TextLabelCol.Text      = ColumnLabels[i];
                    TextLabelCol.Dock      = DockStyle.Fill;
                    TextLabelCol.Margin    = new Padding(5, 5, 5, 5);
                    TextLabelCol.TextAlign = ContentAlignment.BottomLeft;
                    TextLabelCol.Height    = 20;
                    Table.Controls.Add(TextLabelCol);
                }
            }

            for (int Slot = 0; Slot < Max; Slot++)
            {
                Label    TextBoxLabel        = new Label();
                ComboBox StateComboBoxFilter = new ComboBox();
                TextBox  TextBoxFilter       = new TextBox();

                TextBoxLabel.Text      = string.Format("Custom Filter {0}", Slot + 1);
                TextBoxLabel.TextAlign = ContentAlignment.MiddleRight;
                TextBoxLabel.Dock      = DockStyle.Fill;
                TextBoxLabel.Margin    = new Padding(5, 5, 5, 5);
                TextBoxLabel.Width     = 50;

                StateComboBoxFilter.Name = string.Format("ComboBoxFilter{0}", Slot + 1);
                StateComboBoxFilter.Items.Add("No action");
                StateComboBoxFilter.Items.Add("Must have text");
                StateComboBoxFilter.Items.Add("Must not have text");
                StateComboBoxFilter.Items.Add("Must have regex");
                StateComboBoxFilter.Items.Add("Must not have regex");
                StateComboBoxFilter.DropDownStyle = ComboBoxStyle.DropDownList;
                StateComboBoxFilter.SelectedIndex = 0;
                StateComboBoxFilter.Margin        = new Padding(5, 5, 5, 5);
                StateComboBoxFilter.Width         = 100;

                TextBoxFilter.Name         = string.Format("TextBoxFilter{0}", Slot + 1);
                TextBoxFilter.Dock         = DockStyle.Fill;
                TextBoxFilter.Margin       = new Padding(5, 5, 5, 5);
                TextBoxFilter.Tag          = Slot.ToString();
                TextBoxFilter.KeyUp       += this.CallbackTextBoxKeyUp;
                TextBoxFilter.TextChanged += CallbackTextBoxExpressionTextChanged;

                Table.Controls.Add(TextBoxLabel);
                Table.Controls.Add(StateComboBoxFilter);
                Table.Controls.Add(TextBoxFilter);

                this.TextBoxLabels.Add(TextBoxLabel);
                this.StateComboBoxFilters.Add(StateComboBoxFilter);
                this.TextBoxFilters.Add(TextBoxFilter);
            }

            // Add empty last row for space adjustment
            for (int i = 0; i < Table.ColumnCount; i++)
            {
                Label TextLabelCol = new Label();
                TextLabelCol.Text = "";
                Table.Controls.Add(TextLabelCol);
            }
        }
        /** -------------------------------------------------------------------- **/

        private async Task _ProcessCssPage()
        {
            MacroscopeHttpTwoClient         Client   = this.DocCollection.GetJobMaster().GetHttpClient();
            MacroscopeHttpTwoClientResponse Response = null;
            string ResponseErrorCondition            = null;

            DebugMsg(string.Format("ProcessCssPage: {0}", ""));

            try
            {
                Response = await Client.Get(
                    this.GetUri(),
                    this.ConfigureCssPageRequestHeadersCallback,
                    this.PostProcessRequestHttpHeadersCallback
                    );
            }
            catch (MacroscopeDocumentException ex)
            {
                this.DebugMsg(string.Format("_ProcessCssPage :: MacroscopeDocumentException: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
                this.SetStatusCode(HttpStatusCode.BadRequest);
                this.AddRemark("_ProcessCssPage", ex.Message);
            }
            catch (Exception ex)
            {
                this.DebugMsg(string.Format("_ProcessCssPage :: Exception: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
                this.SetStatusCode(HttpStatusCode.BadRequest);
                this.AddRemark("_ProcessCssPage", ex.Message);
            }

            /** Set Base URL ----------------------------------------------------- **/
            if (Response != null)
            {
                Uri    CssUri       = this.GetUri();
                string CssLocalPath = CssUri.LocalPath;
                string CssPath      = System.IO.Path.GetDirectoryName(CssLocalPath);
                Uri    NewCssUri    = new Uri(new UriBuilder(scheme: CssUri.Scheme, host: CssUri.Host, port: CssUri.Port, pathValue: CssPath).ToString());
                this.SetBaseHref(NewCssUri.ToString());
            }

            /** Process The Response Body ---------------------------------------- **/

            if (Response != null)
            {
                string RawData = "";

                this.ProcessResponseHttpHeaders(Response: Response);

                /** Get Response Body ---------------------------------------------- **/

                try
                {
                    DebugMsg(string.Format("MIME TYPE: {0}", this.MimeType));

                    RawData = Response.GetContentAsString();

                    this.SetContentLength(Length: RawData.Length); // May need to find bytes length

                    this.SetWasDownloaded(true);
                }
                catch (Exception ex)
                {
                    DebugMsg(string.Format("Exception: {0}", ex.Message));
                    this.SetStatusCode(HttpStatusCode.Ambiguous);
                    this.SetContentLength(Length: 0);
                }

                if (!string.IsNullOrEmpty(RawData))
                {
                    try
                    {
                        CssLoader     Parser     = new CssLoader();
                        CssStyleSheet Stylesheet = Parser.ParseSheet(RawData, this.GetUri(), this.GetUri());
                        this.ProcessCssOutlinks(Stylesheet: Stylesheet);
                    }
                    catch (Exception ex)
                    {
                        this.DebugMsg(string.Format("ProcessHtmlAttributeCssLinks: {0}", ex.Message));
                        this.AddRemark("ProcessHtmlAttributeCssLinks", ex.Message);
                    }
                }
                else
                {
                    DebugMsg(string.Format("ProcessCssPage: ERROR: {0}", this.GetUrl()));
                }

                /** Custom Filters ------------------------------------------------- **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetCustomFiltersEnable() &&
                        MacroscopePreferencesManager.GetCustomFiltersApplyToCss())
                    {
                        MacroscopeCustomFilters CustomFilter = this.DocCollection.GetJobMaster().GetCustomFilter();

                        if ((CustomFilter != null) && (CustomFilter.IsEnabled()))
                        {
                            this.ProcessGenericCustomFiltered(
                                CustomFilter: CustomFilter,
                                GenericText: RawData
                                );
                        }
                    }
                }

                /** Data Extractors ------------------------------------------------ **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetDataExtractorsEnable() &&
                        MacroscopePreferencesManager.GetDataExtractorsApplyToCss())
                    {
                        this.ProcessGenericDataExtractors(GenericText: RawData);
                    }
                }

                /** Title ---------------------------------------------------------- **/

                {
                    MatchCollection reMatches     = Regex.Matches(this.DocUrl, "/([^/]+)$");
                    string          DocumentTitle = null;
                    foreach (Match match in reMatches)
                    {
                        if (match.Groups[1].Value.Length > 0)
                        {
                            DocumentTitle = match.Groups[1].Value.ToString();
                            break;
                        }
                    }
                    if (DocumentTitle != null)
                    {
                        this.SetTitle(DocumentTitle, MacroscopeConstants.TextProcessingMode.NO_PROCESSING);
                        DebugMsg(string.Format("TITLE: {0}", this.GetTitle()));
                    }
                    else
                    {
                        DebugMsg(string.Format("TITLE: {0}", "MISSING"));
                    }
                }
            }

            if (ResponseErrorCondition != null)
            {
                this.ProcessErrorCondition(ResponseErrorCondition);
            }
        }
Esempio n. 3
0
        /** -------------------------------------------------------------------- **/

        private async Task _ProcessXmlPage()
        {
            XmlDocument                     XmlDoc   = null;
            MacroscopeHttpTwoClient         Client   = this.DocCollection.GetJobMaster().GetHttpClient();
            MacroscopeHttpTwoClientResponse Response = null;
            string ResponseErrorCondition            = null;

            try
            {
                Response = await Client.Get(
                    this.GetUri(),
                    this.ConfigureXmlPageRequestHeadersCallback,
                    this.PostProcessRequestHttpHeadersCallback
                    );
            }
            catch (MacroscopeDocumentException ex)
            {
                this.DebugMsg(string.Format("_ProcessXmlPage :: MacroscopeDocumentException: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
                this.SetStatusCode(HttpStatusCode.BadRequest);
                this.AddRemark("_ProcessXmlPage", ex.Message);
            }
            catch (Exception ex)
            {
                this.DebugMsg(string.Format("_ProcessXmlPage :: Exception: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
                this.SetStatusCode(HttpStatusCode.BadRequest);
                this.AddRemark("_ProcessXmlPage", ex.Message);
            }

            if (Response != null)
            {
                string RawData = "";

                this.ProcessResponseHttpHeaders(Response: Response);

                // Get Response Body
                try
                {
                    DebugMsg(string.Format("MIME TYPE: {0}", this.MimeType));

                    RawData = Response.GetContentAsString();

                    this.SetContentLength(Length: RawData.Length); // May need to find bytes length

                    this.SetWasDownloaded(true);

                    this.SetChecksum(RawData);
                }
                catch (Exception ex)
                {
                    DebugMsg(string.Format("Exception: {0}", ex.Message));
                    this.SetStatusCode(HttpStatusCode.BadRequest);
                    RawData = "";
                    this.SetContentLength(Length: 0);
                }

                if (!string.IsNullOrEmpty(RawData))
                {
                    XmlDoc = new XmlDocument();

                    try
                    {
                        XmlDoc.LoadXml(RawData);
                    }
                    catch (XmlException ex)
                    {
                        DebugMsg(string.Format("XmlException: {0}", ex.Message));
                    }
                    catch (Exception ex)
                    {
                        DebugMsg(string.Format("Exception: {0}", ex.Message));
                    }

                    DebugMsg(string.Format("XmlDoc: {0}", XmlDoc));
                }
                else
                {
                    DebugMsg(string.Format("RawData: {0}", "EMPTY"));
                }

                /** Custom Filters ------------------------------------------------- **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetCustomFiltersEnable() &&
                        MacroscopePreferencesManager.GetCustomFiltersApplyToXml())
                    {
                        MacroscopeCustomFilters CustomFilter = this.DocCollection.GetJobMaster().GetCustomFilter();

                        if ((CustomFilter != null) && (CustomFilter.IsEnabled()))
                        {
                            this.ProcessGenericCustomFiltered(
                                CustomFilter: CustomFilter,
                                GenericText: RawData
                                );
                        }
                    }
                }

                /** Data Extractors ------------------------------------------------ **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetDataExtractorsEnable() &&
                        MacroscopePreferencesManager.GetDataExtractorsApplyToXml())
                    {
                        this.ProcessGenericDataExtractors(GenericText: RawData);
                    }
                }

                /** ---------------------------------------------------------------- **/

                if ((XmlDoc != null) && (XmlDoc.DocumentElement != null))
                {
                    if (this.DetectSitemapXmlDocument(XmlDoc))
                    {
                        DebugMsg(string.Format("ProcessXmlPage: {0} :: {1}", "SITEMAP DETECTED", this.GetUrl()));
                        this.SetDocumentType(Type: MacroscopeConstants.DocumentType.SITEMAPXML);
                        this.ProcessSitemapXmlOutlinks(XmlDoc);
                    }
                }

                /** ---------------------------------------------------------------- **/

                if (RawData != null)
                {
                    this.SetDocumentText(Text: RawData);
                }

                /** ---------------------------------------------------------------- **/
            }

            if (ResponseErrorCondition != null)
            {
                this.ProcessErrorCondition(ResponseErrorCondition);
            }
        }
        /** -------------------------------------------------------------------- **/

        private async Task _ProcessJavascriptPage()
        {
            MacroscopeHttpTwoClient         Client   = this.DocCollection.GetJobMaster().GetHttpClient();
            MacroscopeHttpTwoClientResponse Response = null;
            string ResponseErrorCondition            = null;

            try
            {
                Response = await Client.Get(
                    this.GetUri(),
                    this.ConfigureJavascriptPageRequestHeadersCallback,
                    this.PostProcessRequestHttpHeadersCallback
                    );
            }
            catch (MacroscopeDocumentException ex)
            {
                this.DebugMsg(string.Format("_ProcessJavascriptPage :: MacroscopeDocumentException: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
                this.SetStatusCode(HttpStatusCode.BadRequest);
                this.AddRemark("_ProcessJavascriptPage", ex.Message);
            }
            catch (Exception ex)
            {
                this.DebugMsg(string.Format("_ProcessJavascriptPage :: Exception: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
                this.SetStatusCode(HttpStatusCode.BadRequest);
                this.AddRemark("_ProcessJavascriptPage", ex.Message);
            }

            if (Response != null)
            {
                string RawData = "";

                this.ProcessResponseHttpHeaders(Response: Response);

                /** Get Response Body ---------------------------------------------- **/

                try
                {
                    DebugMsg(string.Format("MIME TYPE: {0}", this.MimeType));

                    /*
                     * Encoding encUseEncoding = Encoding.UTF8;
                     *
                     * if( this.GetCharacterEncoding() != null )
                     * {
                     * encUseEncoding = this.GetCharacterEncoding();
                     * }
                     * else
                     * {
                     * encUseEncoding = this.JavascriptSniffCharset();
                     * }
                     */

                    RawData = Response.GetContentAsString();
                    this.SetContentLength(Length: RawData.Length); // May need to find bytes length
                    this.SetChecksum(RawData);
                }
                catch (Exception ex)
                {
                    DebugMsg(string.Format("Exception: {0}", ex.Message));
                    this.SetStatusCode(HttpStatusCode.Ambiguous);
                    RawData = "";
                    this.SetContentLength(Length: 0);
                }

                /** Custom Filters ------------------------------------------------- **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetCustomFiltersEnable() &&
                        MacroscopePreferencesManager.GetCustomFiltersApplyToJavascripts())
                    {
                        MacroscopeCustomFilters CustomFilter = this.DocCollection.GetJobMaster().GetCustomFilter();

                        if ((CustomFilter != null) && (CustomFilter.IsEnabled()))
                        {
                            this.ProcessGenericCustomFiltered(
                                CustomFilter: CustomFilter,
                                GenericText: RawData
                                );
                        }
                    }
                }

                /** Data Extractors ------------------------------------------------ **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetDataExtractorsEnable() &&
                        MacroscopePreferencesManager.GetDataExtractorsApplyToJavascripts())
                    {
                        this.ProcessGenericDataExtractors(GenericText: RawData);
                    }
                }

                /** Title ---------------------------------------------------------- **/

                {
                    MatchCollection reMatches     = Regex.Matches(this.DocUrl, "/([^/]+)$");
                    string          DocumentTitle = null;
                    foreach (Match match in reMatches)
                    {
                        if (match.Groups[1].Value.Length > 0)
                        {
                            DocumentTitle = match.Groups[1].Value.ToString();
                            break;
                        }
                    }
                    if (DocumentTitle != null)
                    {
                        this.SetTitle(DocumentTitle, MacroscopeConstants.TextProcessingMode.NO_PROCESSING);
                        DebugMsg(string.Format("TITLE: {0}", this.GetTitle()));
                    }
                    else
                    {
                        DebugMsg(string.Format("TITLE: {0}", "MISSING"));
                    }
                }

                /** ---------------------------------------------------------------- **/
            }

            if (ResponseErrorCondition != null)
            {
                this.ProcessErrorCondition(ResponseErrorCondition);
            }
        }
Esempio n. 5
0
        /**************************************************************************/

        private void RenderListView(
            MacroscopeDocumentCollection DocCollection,
            List <string> UrlList,
            MacroscopeCustomFilters CustomFilter
            )
        {
            if (this.FilterColOffset == -1)
            {
                throw (new Exception("this.FilterColOffset invalid"));
            }

            if (DocCollection.CountDocuments() == 0)
            {
                return;
            }

            MacroscopeAllowedHosts   AllowedHosts    = this.MainForm.GetJobMaster().GetAllowedHosts();
            Dictionary <string, int> FilterColsTable = new Dictionary <string, int>(CustomFilter.GetSize());
            List <ListViewItem>      ListViewItems   = new List <ListViewItem>();

            MacroscopeSinglePercentageProgressForm ProgressForm = new MacroscopeSinglePercentageProgressForm(this.MainForm);
            decimal Count           = 0;
            decimal TotalDocs       = (decimal)DocCollection.CountDocuments();
            decimal MajorPercentage = ((decimal)100 / TotalDocs) * Count;

            if (MacroscopePreferencesManager.GetShowProgressDialogues())
            {
                ProgressForm.UpdatePercentages(
                    Title: "Preparing Display",
                    Message: "Processing document collection for display:",
                    MajorPercentage: MajorPercentage,
                    ProgressLabelMajor: string.Format("Document {0} / {1}", Count, TotalDocs)
                    );
            }

            for (int Slot = 0; Slot < CustomFilter.GetSize(); Slot++)
            {
                string FilterPattern = CustomFilter.GetPattern(Slot).Key;

                if (FilterColsTable.ContainsKey(FilterPattern))
                {
                    FilterColsTable.Add(string.Format("EMPTY{0}", Slot + 1), Slot + 1);
                }
                else
                {
                    FilterColsTable.Add(FilterPattern, Slot + 1);
                }
            }

            foreach (string Url in UrlList)
            {
                MacroscopeDocument msDoc  = DocCollection.GetDocumentByUrl(Url: Url);
                ListViewItem       lvItem = null;
                string             DocUrl;
                string             PairKey;
                string             StatusCode;
                string             Status;
                string             MimeType;

                if (msDoc == null)
                {
                    continue;
                }
                else
                {
                    DocUrl     = msDoc.GetUrl();
                    PairKey    = DocUrl;
                    StatusCode = ((int)msDoc.GetStatusCode()).ToString();
                    Status     = msDoc.GetStatusCode().ToString();
                    MimeType   = msDoc.GetMimeType();
                }

                if (!CustomFilter.CanApplyCustomFiltersToDocument(msDoc: msDoc))
                {
                    continue;
                }

                if (this.DisplayListView.Items.ContainsKey(PairKey))
                {
                    lvItem = this.DisplayListView.Items[PairKey];
                }
                else
                {
                    lvItem = new ListViewItem(PairKey);
                    lvItem.UseItemStyleForSubItems = false;
                    lvItem.Name = PairKey;

                    lvItem.SubItems.Add("");
                    lvItem.SubItems.Add("");
                    lvItem.SubItems.Add("");
                    lvItem.SubItems.Add("");

                    for (int Slot = 0; Slot < CustomFilter.GetSize(); Slot++)
                    {
                        lvItem.SubItems.Add("");
                    }

                    ListViewItems.Add(lvItem);
                }

                if (lvItem != null)
                {
                    try
                    {
                        lvItem.SubItems[ColUrl].Text        = DocUrl;
                        lvItem.SubItems[ColStatusCode].Text = StatusCode;
                        lvItem.SubItems[ColStatus].Text     = Status;
                        lvItem.SubItems[ColMimeType].Text   = MimeType;

                        for (int Slot = 0; Slot < CustomFilter.GetSize(); Slot++)
                        {
                            string FilterPattern = CustomFilter.GetPattern(Slot: Slot).Key;
                            KeyValuePair <string, MacroscopeConstants.TextPresence> Pair = msDoc.GetCustomFilteredItem(Text: FilterPattern);
                            int ColOffset = this.FilterColOffset + FilterColsTable[FilterPattern];

                            if ((Pair.Key != null) && (Pair.Value != MacroscopeConstants.TextPresence.UNDEFINED))
                            {
                                lvItem.SubItems[ColOffset].Text = MacroscopeConstants.TextPresenceLabels[Pair.Value];

                                switch (Pair.Value)
                                {
                                case MacroscopeConstants.TextPresence.CONTAINS_STRING:
                                    lvItem.SubItems[ColOffset].ForeColor = Color.Green;
                                    break;

                                case MacroscopeConstants.TextPresence.NOT_CONTAINS_STRING:
                                    lvItem.SubItems[ColOffset].ForeColor = Color.Green;
                                    break;

                                case MacroscopeConstants.TextPresence.MUST_CONTAIN_STRING:
                                    lvItem.SubItems[ColOffset].ForeColor = Color.Red;
                                    break;

                                case MacroscopeConstants.TextPresence.SHOULD_NOT_CONTAIN_STRING:
                                    lvItem.SubItems[ColOffset].ForeColor = Color.Red;
                                    break;

                                case MacroscopeConstants.TextPresence.CONTAINS_REGEX:
                                    lvItem.SubItems[ColOffset].ForeColor = Color.Green;
                                    break;

                                case MacroscopeConstants.TextPresence.NOT_CONTAINS_REGEX:
                                    lvItem.SubItems[ColOffset].ForeColor = Color.Green;
                                    break;

                                case MacroscopeConstants.TextPresence.MUST_CONTAIN_REGEX:
                                    lvItem.SubItems[ColOffset].ForeColor = Color.Red;
                                    break;

                                case MacroscopeConstants.TextPresence.SHOULD_NOT_CONTAIN_REGEX:
                                    lvItem.SubItems[ColOffset].ForeColor = Color.Red;
                                    break;

                                default:
                                    lvItem.SubItems[ColOffset].ForeColor = Color.Gray;
                                    break;
                                }
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        DebugMsg(string.Format("MacroscopeDisplayCustomFilters: {0}", ex.Message));
                        DebugMsg(string.Format("MacroscopeDisplayCustomFilters: {0}", ex.StackTrace));
                    }
                }
                else
                {
                    DebugMsg(string.Format("MacroscopeDisplayCustomFilters MISSING: {0}", PairKey));
                }

                if (msDoc.GetIsInternal())
                {
                    lvItem.SubItems[ColUrl].ForeColor = Color.Green;
                }
                else
                {
                    lvItem.SubItems[ColUrl].ForeColor = Color.Gray;
                }

                if (Regex.IsMatch(StatusCode, "^[2]"))
                {
                    lvItem.SubItems[ColStatusCode].ForeColor = Color.Green;
                    lvItem.SubItems[ColStatus].ForeColor     = Color.Green;
                }
                else
                if (Regex.IsMatch(StatusCode, "^[3]"))
                {
                    lvItem.SubItems[ColStatusCode].ForeColor = Color.Goldenrod;
                    lvItem.SubItems[ColStatus].ForeColor     = Color.Goldenrod;
                }
                else
                if (Regex.IsMatch(StatusCode, "^[45]"))
                {
                    lvItem.SubItems[ColStatusCode].ForeColor = Color.Red;
                    lvItem.SubItems[ColStatus].ForeColor     = Color.Red;
                }
                else
                {
                    lvItem.SubItems[ColStatusCode].ForeColor = Color.Blue;
                    lvItem.SubItems[ColStatus].ForeColor     = Color.Blue;
                }

                if (MacroscopePreferencesManager.GetShowProgressDialogues())
                {
                    Count++;

                    MajorPercentage = ((decimal)100 / TotalDocs) * Count;

                    ProgressForm.UpdatePercentages(
                        Title: null,
                        Message: null,
                        MajorPercentage: MajorPercentage,
                        ProgressLabelMajor: string.Format("Document {0} / {1}", Count, TotalDocs)
                        );
                }
            }

            this.DisplayListView.Items.AddRange(ListViewItems.ToArray());

            this.DisplayListView.AutoResizeColumns(ColumnHeaderAutoResizeStyle.HeaderSize);

            this.DisplayListView.Columns[ColUrl].Width        = 300;
            this.DisplayListView.Columns[ColStatusCode].Width = 100;
            this.DisplayListView.Columns[ColStatus].Width     = 100;
            this.DisplayListView.Columns[ColMimeType].Width   = 100;

            if (MacroscopePreferencesManager.GetShowProgressDialogues())
            {
                ProgressForm.DoClose();
            }

            if (ProgressForm != null)
            {
                ProgressForm.Dispose();
            }
        }
Esempio n. 6
0
        /**************************************************************************/

        public MacroscopeCsvCustomFilterReport(MacroscopeCustomFilters NewCustomFilter)
        {
            this.CustomFilter = NewCustomFilter;
        }
Esempio n. 7
0
        /** -------------------------------------------------------------------- **/

        private async Task _ProcessTextPage()
        {
            List <string>                   TextDoc  = new List <string>();
            MacroscopeHttpTwoClient         Client   = this.DocCollection.GetJobMaster().GetHttpClient();
            MacroscopeHttpTwoClientResponse Response = null;
            string ResponseErrorCondition            = null;

            try
            {
                Response = await Client.Get(
                    this.GetUri(),
                    this.ConfigureTextPageRequestHeadersCallback,
                    this.PostProcessRequestHttpHeadersCallback
                    );
            }
            catch (MacroscopeDocumentException ex)
            {
                this.DebugMsg(string.Format("_ProcessTextPage :: MacroscopeDocumentException: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
                this.SetStatusCode(HttpStatusCode.BadRequest);
                this.AddRemark("_ProcessTextPage", ex.Message);
            }
            catch (Exception ex)
            {
                this.DebugMsg(string.Format("_ProcessTextPage :: Exception: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
                this.SetStatusCode(HttpStatusCode.BadRequest);
                this.AddRemark("_ProcessTextPage", ex.Message);
            }

            if (Response != null)
            {
                string RawData = "";

                this.ProcessResponseHttpHeaders(Response: Response);

                /** Get Response Body ---------------------------------------------- **/

                try
                {
                    DebugMsg(string.Format("MIME TYPE: {0}", this.MimeType));

                    RawData = Response.GetContentAsString();

                    this.SetContentLength(Length: RawData.Length); // May need to find bytes length

                    this.SetWasDownloaded(true);

                    this.SetChecksum(RawData);
                }
                catch (Exception ex)
                {
                    DebugMsg(string.Format("Exception: {0}", ex.Message));
                    this.SetStatusCode(HttpStatusCode.BadRequest);
                    RawData = "";
                    this.SetContentLength(Length: 0);
                }

                /** ---------------------------------------------------------------- **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    string[] Lines = Regex.Split(RawData, @"[\r\n]+");
                    TextDoc = Lines.ToList();

                    DebugMsg(string.Format("TextDoc: {0}", TextDoc.Count));
                }
                else
                {
                    DebugMsg(string.Format("RawData: {0}", "EMPTY"));
                }

                /** Custom Filters ------------------------------------------------- **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetCustomFiltersEnable() &&
                        MacroscopePreferencesManager.GetCustomFiltersApplyToText())
                    {
                        MacroscopeCustomFilters CustomFilter = this.DocCollection.GetJobMaster().GetCustomFilter();

                        if ((CustomFilter != null) && (CustomFilter.IsEnabled()))
                        {
                            this.ProcessGenericCustomFiltered(
                                CustomFilter: CustomFilter,
                                GenericText: RawData
                                );
                        }
                    }
                }

                /** Data Extractors ------------------------------------------------ **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetDataExtractorsEnable() &&
                        MacroscopePreferencesManager.GetDataExtractorsApplyToText())
                    {
                        this.ProcessGenericDataExtractors(GenericText: RawData);
                    }
                }

                /** Process Text Document ------------------------------------------ **/

                if ((TextDoc != null) && (TextDoc.Count > 0))
                {
                    this.SetDocumentText(Text: string.Join(Environment.NewLine, TextDoc));

                    if (this.GetPath().EndsWith("robots.txt", StringComparison.InvariantCultureIgnoreCase))
                    {
                        long?TextSize          = this.GetContentLength();
                        long?RobotsMaxTextSize = 1024 * 512;

                        this.ProcessRobotsTextOutlinks(TextDoc: TextDoc);

                        if (this.DetectSitemapTextDocument(TextDoc: TextDoc))
                        {
                            DebugMsg(string.Format("ProcessTextPage: {0} :: {1}", "SITEMAP DETECTED", this.GetUrl()));
                            this.SetDocumentType(Type: MacroscopeConstants.DocumentType.SITEMAPTEXT);
                            this.ProcessSitemapTextOutlinks(TextDoc: TextDoc);
                        }

                        if (TextSize > RobotsMaxTextSize)
                        {
                            this.AddRemark("ROBOTS_TOO_BIG", "Robots.txt is larger than 512KB");
                        }
                    }
                    else
                    {
                        if (this.GetIsInternal())
                        {
                            this.ProcessPureTextOutlinks(TextDoc: TextDoc, LinkType: MacroscopeConstants.InOutLinkType.PURETEXT);
                        }
                    }
                }
                else
                {
                    this.SetDocumentText(Text: "");
                }

                /** ---------------------------------------------------------------- **/
            }

            if (ResponseErrorCondition != null)
            {
                this.ProcessErrorCondition(ResponseErrorCondition);
            }
        }
Esempio n. 8
0
        /**************************************************************************/

        private void ProcessCssPage()
        {
            HttpWebRequest  req = null;
            HttpWebResponse res = null;
            string          ResponseErrorCondition = null;
            Boolean         IsAuthenticating       = false;

            DebugMsg(string.Format("ProcessCssPage: {0}", ""));

            try
            {
                req           = WebRequest.CreateHttp(this.DocUrl);
                req.Method    = "GET";
                req.Timeout   = this.Timeout;
                req.KeepAlive = false;
                req.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;

                this.PrepareRequestHttpHeaders(req: req);

                IsAuthenticating = this.AuthenticateRequest(req);

                MacroscopePreferencesManager.EnableHttpProxy(req);

                res = ( HttpWebResponse )req.GetResponse();
            }
            catch (UriFormatException ex)
            {
                DebugMsg(string.Format("ProcessCssPage :: UriFormatException: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
            }
            catch (TimeoutException ex)
            {
                DebugMsg(string.Format("ProcessCssPage :: TimeoutException: {0}", ex.Message));
                ResponseErrorCondition = ex.Message;
            }
            catch (WebException ex)
            {
                DebugMsg(string.Format("ProcessCssPage :: WebException: {0}", ex.Message));
                DebugMsg(string.Format("ProcessCssPage :: WebException: {0}", ex.Status));
                DebugMsg(string.Format("ProcessCssPage :: WebException: {0}", ( int )ex.Status));

                ResponseErrorCondition = ex.Status.ToString();
            }

            if (res != null)
            {
                string RawData = "";

                this.ProcessResponseHttpHeaders(req, res);

                if (IsAuthenticating)
                {
                    this.VerifyOrPurgeCredential();
                }

                // Get Response Body
                try
                {
                    DebugMsg(string.Format("MIME TYPE: {0}", this.MimeType));

                    Stream       ResponseStream = res.GetResponseStream();
                    StreamReader ResponseStreamReader;

                    if (this.GetCharacterEncoding() != null)
                    {
                        ResponseStreamReader = new StreamReader(ResponseStream, this.GetCharacterEncoding());
                    }
                    else
                    {
                        ResponseStreamReader = new StreamReader(ResponseStream);
                    }

                    RawData = ResponseStreamReader.ReadToEnd();

                    this.ContentLength = RawData.Length; // May need to find bytes length

                    this.SetWasDownloaded(true);
                }
                catch (WebException ex)
                {
                    DebugMsg(string.Format("WebException: {0}", ex.Message));

                    if (ex.Response != null)
                    {
                        this.SetStatusCode((( HttpWebResponse )ex.Response).StatusCode);
                    }
                    else
                    {
                        this.SetStatusCode(( HttpStatusCode )ex.Status);
                    }

                    RawData            = "";
                    this.ContentLength = 0;
                }
                catch (Exception ex)
                {
                    DebugMsg(string.Format("Exception: {0}", ex.Message));
                    this.SetStatusCode(HttpStatusCode.BadRequest);
                    this.ContentLength = 0;
                }

                if (!string.IsNullOrEmpty(RawData))
                {
                    ExCSS.Parser     ExCssParser     = new ExCSS.Parser();
                    ExCSS.StyleSheet ExCssStylesheet = ExCssParser.Parse(RawData);

                    this.ProcessCssOutlinks(ExCssStylesheet);
                }
                else
                {
                    DebugMsg(string.Format("ProcessCssPage: ERROR: {0}", this.GetUrl()));
                }

                /** Custom Filters ------------------------------------------------- **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetCustomFiltersEnable() &&
                        MacroscopePreferencesManager.GetCustomFiltersApplyToCss())
                    {
                        MacroscopeCustomFilters CustomFilter = this.DocCollection.GetJobMaster().GetCustomFilter();

                        if ((CustomFilter != null) && (CustomFilter.IsEnabled()))
                        {
                            this.ProcessGenericCustomFiltered(
                                CustomFilter: CustomFilter,
                                GenericText: RawData
                                );
                        }
                    }
                }

                /** Data Extractors ------------------------------------------------ **/

                if (!string.IsNullOrEmpty(RawData))
                {
                    if (
                        MacroscopePreferencesManager.GetDataExtractorsEnable() &&
                        MacroscopePreferencesManager.GetDataExtractorsApplyToCss())
                    {
                        this.ProcessGenericDataExtractors(GenericText: RawData);
                    }
                }

                /** Title ---------------------------------------------------------- **/

                {
                    MatchCollection reMatches     = Regex.Matches(this.DocUrl, "/([^/]+)$");
                    string          DocumentTitle = null;
                    foreach (Match match in reMatches)
                    {
                        if (match.Groups[1].Value.Length > 0)
                        {
                            DocumentTitle = match.Groups[1].Value.ToString();
                            break;
                        }
                    }
                    if (DocumentTitle != null)
                    {
                        this.SetTitle(DocumentTitle, MacroscopeConstants.TextProcessingMode.NO_PROCESSING);
                        DebugMsg(string.Format("TITLE: {0}", this.GetTitle()));
                    }
                    else
                    {
                        DebugMsg(string.Format("TITLE: {0}", "MISSING"));
                    }
                }

                res.Close();

                res.Dispose();
            }

            if (ResponseErrorCondition != null)
            {
                this.ProcessErrorCondition(ResponseErrorCondition);
            }
        }