Exemple #1
0
        protected bool ReadPage(string url)
        {
            var client = GetWebClient();

            parameterList.Clear();
            httpHeaderList.Clear();
            BeforeTrees.Run(this, GetText());

            if (InputParameterTree != null)
            {
                RawlerBase.GetText(GetText(), InputParameterTree, this);
            }

            if (MethodType == Tool.MethodType.GET)
            {
                this.text = client.HttpGet(url, parameterList, httpHeaderList);
            }
            else if (MethodType == Tool.MethodType.POST)
            {
                this.text = client.HttpPost(url, parameterList, httpHeaderList);
            }

            this.currentUrl = url;
            this.pastUrl    = this.currentUrl;

            if (this.Text.Length > 0)
            {
                return(true);
            }
            else
            {
                if (client.ErrMessage != null && (client.ErrMessage.Contains("503") || client.ErrMessage.Contains("500")))
                {
                    ReportManage.ErrReport(this, $"{client.ErrMessage} {url}の読み込みに失敗しました。");
                }
                else
                {
                    if (visbleErr)
                    {
                        ReportManage.ErrReport(this, url + "の読み込みに失敗しました。");
                    }
                    if (ErrorEvent != null)
                    {
                        ErrorEvent(this, new EventArgs());
                    }
                    if (ErrEventTree != null)
                    {
                        ErrEventTree.SetParent();
                        Document d = new Document()
                        {
                            TextValue = client.ErrMessage
                        };
                        d.SetParent(this);
                        d.AddChildren(ErrEventTree);
                        d.Run();
                    }
                }
                return(false);
            }
        }
Exemple #2
0
        protected async Task <bool> ReadPage(string url)
        {
            var client = GetWebClient();

            if (MethodType == MethodType.GET)
            {
                this.text = await client.HttpGet(url, null, tmpReferer);
            }
            else if (MethodType == MethodType.POST)
            {
                parameterDic.Clear();
                if (InputParameterTree != null)
                {
                    RawlerBase.GetText(GetText(), InputParameterTree, this);
                }
                List <KeyValue> list = new List <KeyValue>();
                foreach (var item in parameterDic)
                {
                    list.Add(new KeyValue()
                    {
                        Key = item.Key, Value = item.Value
                    });
                }
                this.text = await client.HttpPost(url, list, tmpReferer);
            }

            this.currentUrl = url;
            this.pastUrl    = this.currentUrl;

            if (this.Text.Length > 0)
            {
                return(true);
            }
            else
            {
                if (client.ErrMessage.Contains("503") || client.ErrMessage.Contains("500"))
                {
                    ReportManage.Report(this, "待機します", true, true);
                    await Task.Delay(new TimeSpan(0, 0, 30));

                    urlStack.Push(url);
                }
                else
                {
                    if (visbleErr)
                    {
                        ReportManage.ErrReport(this, url + "の読み込みに失敗しました。");
                    }
                    if (ErrorEvent != null)
                    {
                        ErrorEvent(this, new EventArgs());
                    }
                    if (ErrEventTree != null)
                    {
                        ErrEventTree.SetParent();
                        Document d = new Document()
                        {
                            TextValue = client.ErrMessage
                        };
                        d.SetParent(this);
                        d.AddChildren(ErrEventTree);
                        d.Run();
                    }
                }
                return(false);
            }
        }