/// <summary> /// commentを出力する。 /// </summary> protected void ReportComment() { if (!string.IsNullOrEmpty(this.commnet)) { ReportManage.Report(this, this.Comment); } }
public void JsonFileSave() { string filename = this.FileName.Convert(this); if (this.FileNameTree != null) { filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this); } if (string.IsNullOrEmpty(filename) == false) { System.IO.StreamWriter sw = null; if (this.FileSaveMode == Tool.FileSaveMode.Create) { sw = System.IO.File.CreateText(filename); } else if (this.FileSaveMode == Tool.FileSaveMode.Append) { sw = System.IO.File.AppendText(filename); } if (saveFileType == FileType.Json) { sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.GetDataDicForJson()))); } sw.Close(); ReportManage.Report(this, filename + "作成完了", true, EndReport); } }
private void TsvFileSave() { if (doLastFileSave == false) { return; } string filename = this.FileName.Convert(this); if (this.FileNameTree != null) { filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this); } if (string.IsNullOrEmpty(filename) == false) { System.IO.StreamWriter sw = null; if (this.FileSaveMode == Tool.FileSaveMode.Create) { sw = System.IO.File.CreateText(filename); } else if (this.FileSaveMode == Tool.FileSaveMode.Append) { sw = System.IO.File.AppendText(filename); } sw.Write(ToTsv()); sw.Close(); ReportManage.Report(this, filename + "作成完了", true, EndReport); } }
public override void Run(bool runChildren) { var d = (IData)this.GetUpperInterface <IData>(); if (d != null) { ReportManage.Report(this, d.GetCurrentDataRow().ToString(), true, true); } }
public override void Run(bool runChildren) { Data data = null; IRawler current = this.Parent; while (current != null) { if (current is Data) { data = current as Data; break; } current = current.Parent; } if (data != null) { if (data.GetCurrentDataNull()) { if (ignoreDataNull == false) { ReportManage.ErrReport(this, "RowがNullです。Writeが動作していないようです。"); var list = this.GetAncestorRawler().Where(n => n is Page); if (DoPageReLoad) { if (list.Count() > 0) { var p = list.First() as Page; pageCount++; if (PageReLoadCount < pageCount) { ReportManage.Report(this, "再読み込み待機中。"); System.Threading.Thread.Sleep(1000 * pageCount * pageCount); p.Run(); } else { ReportManage.ErrReport(this, "書き込み先のData クラスが見つかりませんでした。"); } } } } } else { pageCount = 0; data.NextDataRow(); ReportManage.Report(this, "NextDataRow"); } } else { ReportManage.ErrReport(this, "書き込み先のData クラスが見つかりませんでした。"); } this.RunChildren(runChildren); }
/// <summary> /// 再びログインする。 /// </summary> public void ReLogin() { string html = this.HttpPost(loginPage, vals, false); if (this.GetCookieCount() > 0) { if (ErrString != null) { if (html.Contains(ErrString) == false) { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } else { hasLogin = false; ReportManage.ErrReport(this, "ログイン失敗"); } } else { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } } else { if (isNoCookieSite) { hasLogin = false; ReportManage.ErrReport(this, "ログイン失敗"); } else { if (ErrString != null) { if (html.Contains(ErrString) == false) { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } else { hasLogin = false; ReportManage.ErrReport(this, "ログイン失敗"); } } else { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } } } this.text = html; }
public override string HttpGet(string url, Encoding enc) { this.Sleep(); if (ReportUrl) { ReportManage.Report(this, "GET " + url, true, true); } System.Net.Http.HttpClient client = GetHttpClient(); var result = client.GetStringAsync(url).ConfigureAwait(false); return(result.GetAwaiter().GetResult()); }
public void NextDataRow(DataRowObject cDataRow) { ReportManage.Report(this, "NextDataRow"); if (Commited != null) { Commited(this, new EventDataRow(cDataRow)); } currentDataRow = new DataRowObject(); if (stock) { dataList.Add(cDataRow); } }
public override void Run(bool runChildren) { if (doSave) { if (FileName != null) { filename = FileName; } else { if (string.IsNullOrEmpty(filename)) { ReportManage.ErrReport(this, "FileNameが空です。"); return; } //Microsoft.Win32.SaveFileDialog saveDialog = new Microsoft.Win32.SaveFileDialog(); //saveDialog.Title = "保存ファイルの指定 CountData:" + this.Comment; //if (string.IsNullOrEmpty(ExtendFilter) == false) //{ // saveDialog.Filter = RawlerLib.Io.FilterStringCreate(ExtendFilter); //} //if (saveDialog.ShowDialog() == true) //{ // filename = saveDialog.FileName; //} //using (var file = System.IO.File.CreateText(filename)) //{ // file.WriteLine("Group\tKey\tCount"); //} } } base.Run(runChildren); if (doSave || FileName != null) { try { Write(filename); } catch (Exception e) { ReportManage.ErrReport(this, "CountDataでファイルの書き込みに失敗しました。" + e.Message); } } if (reportData) { ReportManage.Report(this, this.ToTsv(), true, true); } }
public override void Run(bool runChildren) { string text; if (viewParentText && string.IsNullOrEmpty(Message)) { text = Header.Convert(this) + GetText() + Footer.Convert(this); } else { text = Header.Convert(this) + Message.Convert(this) + this.Footer.Convert(this); } ReportManage.Report(this, System.Net.WebUtility.HtmlDecode(text), returncode, visible); this.RunChildren(runChildren); }
public string HttpGet(string url, List <KeyValue> parameterList, List <KeyValue> httpHeaderList) { this.Sleep(); if (ReportUrl) { ReportManage.Report(this, "GET " + url, true, true); } System.Net.Http.HttpClient client = GetHttpClient(); client.SetHeader(httpHeaderList); var result = client.GetStringAsync(url).ConfigureAwait(false).GetAwaiter().GetResult(); client.RemoveHeader(httpHeaderList); return(result); }
public string HttpPost(string url, List <KeyValue> parameterList, List <KeyValue> httpHeaderList) { this.Sleep(); System.Net.Http.HttpClient client = GetHttpClient(); client.SetHeader(httpHeaderList); if (ReportUrl) { ReportManage.Report(this, $"Post {url} [{string.Join("\t", parameterList.Select(n=>$"{n.Key}:{n.Value}"))}]", true, true); } var r = client.PostAsync(url, new FormUrlEncodedContent(parameterList.Select(n => new KeyValuePair <string, string>(n.Key, n.Value)))).ConfigureAwait(false).GetAwaiter().GetResult(); var r2 = r.Content.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult(); this.ErrMessage = r.StatusCode.ToString(); return(r2); }
/// <summary> /// ファイルで保存する /// </summary> protected void SaveFile() { string filename = this.FileName.Convert(this); if (this.FileNameTree != null) { filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this); } if (string.IsNullOrEmpty(filename) == false) { System.IO.StreamWriter sw = null; if (this.FileSaveMode == Tool.FileSaveMode.Create) { sw = System.IO.File.CreateText(filename); } else if (this.FileSaveMode == Tool.FileSaveMode.Append) { sw = System.IO.File.AppendText(filename); } if (saveFileType == FileType.Json) { sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.DataDic))); } else if (saveFileType == FileType.Tsv) { sw.WriteLine(this.ToTsv()); } else if (saveFileType == FileType.LTsv) { foreach (var item in this.GetDataRows()) { var d = item.DataDic.ToDictionary(n => n.Key, n => n.Value.JoinText(",")); sw.WriteLine(d.ToLtsvLine()); } } sw.Close(); ReportManage.Report(this, filename + "作成完了", true, EndReport); } }
public void Write(RawlerBase rawler, string text, bool viewTotal) { if (DoRun) { if (viewParent) { text = this.Parent.ToObjectString() + " " + text; } var t = sw.ElapsedMilliseconds + "\t" + text; total += sw.ElapsedMilliseconds; if (viewTotal) { ReportManage.Report(rawler, t, true, true); ReportManage.Report(rawler, "Total:" + total, true, true); } else { ReportManage.Report(rawler, t, true, true); } sw.Restart(); } }
/// <summary> /// HttpPost /// </summary> /// <param name="url"></param> /// <param name="vals"></param> /// <returns></returns> public virtual byte[] HttpPostByte(string url, List <KeyValue> vals) { if (ReportUrl) { ReportManage.Report(this, "POST " + url, true, true); } try { Sleep(); wc.Referer = referer; wc.UserAgent = userAgent; wc.BasicAuthorization = BasicAuthorization; System.Collections.Specialized.NameValueCollection list = new System.Collections.Specialized.NameValueCollection(); foreach (var item in vals) { list.Add(item.Key, item.Value); } var data2 = wc.UploadValues(url, list); return(data2); } catch (Exception e) { if (visbleErr) { ReportManage.ErrReport(this, "Url:" + url + " " + e.Message); } //if (e is System.UriFormatException) //{ // ReportManage.ErrReport(this,"["+url+"]は無効なURLです。"); //} ErrMessage = e.Message; // throw new Exception("HttpGet:"+url+"に失敗しました"); } return(null); }
public override void Run(bool runChildren) { var page = GetPage(); var u = GetText().Replace("&#", "&&&&").Split('#'); string url = string.Empty; if (u.Length > 0) { url = u[0].Replace("&&&&", "&#"); } if (this.Parent.Text != null && this.Parent.Text.Length > 0) { if (page != null) { //始まりがURLと違う場合初期化 if (page.GetStartUrl() != baseUrl) { baseUrl = page.GetStartUrl(); urlHash.Clear(); count = 0; } if (allowSameUrl == false) { if (urlHash.Contains(url) == false) { count++; urlHash.Add(url); if (count < maxCount) { // this.Text = this.Parent.Text; if (sleepTime > 0) { System.Threading.Thread.Sleep(new TimeSpan(0, 0, sleepTime)); } this.RunChildren(runChildren); ReportManage.Report(this, "NextPage:" + GetText()); page.PushUrl(url); } else { ReportManage.Report(this, "NextPage:指定ページ数を取得しました"); } } } else { if (count < maxCount) { // this.Text = this.Parent.Text; if (sleepTime > 0) { System.Threading.Thread.Sleep(new TimeSpan(0, 0, sleepTime)); } this.RunChildren(runChildren); ReportManage.Report(this, "NextPage:" + GetText()); page.PushUrl(url); } else { ReportManage.Report(this, "NextPage:指定ページ数を取得しました"); } } } } }
/// <summary> /// HttpPost /// </summary> /// <param name="url"></param> /// <param name="vals"></param> /// <returns></returns> public virtual string HttpPost(string url, List <KeyValue> vals) { if (ReportUrl) { var t = vals.Select(n => n.Key + ":" + n.Value.Trim().Replace("\n", "").Replace("\r", "")).JoinText("\n"); ReportManage.Report(this, "POST " + url + "\n" + t, true, true); } try { Sleep(); wc.Referer = referer; wc.UserAgent = userAgent; wc.BasicAuthorization = BasicAuthorization; System.Collections.Specialized.NameValueCollection list = new System.Collections.Specialized.NameValueCollection(); //foreach (var item in vals.GroupBy(n=>n.Key)) //{ // if (item.Count() == 1) // { // list.Add(item.Key, item.First().Value); // } // //同じ値が複数ある時用? // else // { // int i = 0; // foreach (var item2 in item) // { // list.Add(item2.Key + "[" + i + "]", item2.Value); // i++; // } // } //} foreach (var item in vals) { list.Add(item.Key, item.Value); } if (encoder == null) { encoder = GetEncoding(); } wc.Encoding = encoder; // wc.UploadValues var data2 = wc.UploadValues2(new Uri(url), vals); return(data2); } catch (Exception e) { if (visbleErr) { ReportManage.ErrReport(this, "Url:" + url + " " + e.Message); } //if (e is System.UriFormatException) //{ // ReportManage.ErrReport(this,"["+url+"]は無効なURLです。"); //} ErrMessage = e.Message; // throw new Exception("HttpGet:"+url+"に失敗しました"); } return(string.Empty); }
/// <summary> /// Image用 失敗時NULL /// </summary> /// <param name="url"></param> /// <returns></returns> public virtual byte[] HttpGetByte(string url) { Sleep(); if (ReportUrl) { ReportManage.Report(this, "GET " + url, true, true); } byte[] data; try { wc.Referer = referer; wc.UserAgent = userAgent; wc.BasicAuthorization = BasicAuthorization; data = wc.DownloadData(url); //// リクエストの作成 //HttpWebRequest req = (HttpWebRequest)WebRequest.Create(Uri.EscapeUriString(url)); //req.CookieContainer = cc; //if (addUserAgent) //{ // req.UserAgent = userAgent; //} //if (referer != string.Empty) //{ // req.Referer = referer; //} //// req.Referer = "http://www.pixiv.net/"; //WebResponse res = req.GetResponse(); //// レスポンスの読み取り //Stream resStream = res.GetResponseStream(); ////b = new StreamReader(resStream); ////StreamReader sr = new StreamReader(resStream); //while (true) //{ // int b = resStream.ReadByte(); // if (b == -1) // { // break; // } // data.Add((byte)b); //} ////sr.Close(); //resStream.Close(); } catch (Exception e) { if (visbleErr) { ReportManage.ErrReport(this, "Url:" + url + " " + e.Message); } data = null; } if (data != null) { return(data); } else { return(null); } }
/// <summary> /// HttpGet /// </summary> /// <param name="url"></param> /// <param name="enc"></param> /// <returns></returns> public virtual string HttpGet(string url, Encoding enc) { Sleep(); if (UseCache) { if (casheDic.ContainsKey(url)) { return(casheDic[url]); } } ErrMessage = string.Empty; string result = string.Empty; bool retry = false; if (ReportUrl) { ReportManage.Report(this, "GET " + url, true, true); } try { wc.Referer = referer; wc.UserAgent = userAgent; wc.BasicAuthorization = BasicAuthorization; if (enc != null) { wc.Encoding = enc; result = wc.DownloadString(url); } else { var data = wc.DownloadData(url); result = GetAutoEncoding(data, out encoder); //RawlerLib.Text.TxtEnc txtEnc = new RawlerLib.Text.TxtEnc(); //byte[] byteArray2 = wc.DownloadData(url); //txtEnc.SetFromByteArray(ref byteArray2); //txtEnc.Codec = "shift_jis"; //result = txtEnc.Text; } } catch (Exception e) { if (visbleErr) { ReportManage.ErrReport(this, "Url:" + url + " " + e.Message); } ErrMessage = e.Message; retry = false; } if (retry) { count++; if (count <= tryCount) { ReportManage.ErrReport(this, "HttpGet:" + url + "にリトライ待機中"); System.Threading.Thread.Sleep(new TimeSpan(0, 0, 10 * count * count)); result = this.HttpGet(url, enc); } else { ReportManage.ErrReport(this, "HttpGet:" + url + "に失敗しました"); result = string.Empty; } } if (UseCache) { casheDic.GetValueOrAdd(url, result); } return(result); }