protected bool ReadPage(string url) { var client = GetWebClient(); parameterList.Clear(); httpHeaderList.Clear(); BeforeTrees.Run(this, GetText()); if (InputParameterTree != null) { RawlerBase.GetText(GetText(), InputParameterTree, this); } if (MethodType == Tool.MethodType.GET) { this.text = client.HttpGet(url, parameterList, httpHeaderList); } else if (MethodType == Tool.MethodType.POST) { this.text = client.HttpPost(url, parameterList, httpHeaderList); } this.currentUrl = url; this.pastUrl = this.currentUrl; if (this.Text.Length > 0) { return(true); } else { if (client.ErrMessage != null && (client.ErrMessage.Contains("503") || client.ErrMessage.Contains("500"))) { ReportManage.ErrReport(this, $"{client.ErrMessage} {url}の読み込みに失敗しました。"); } else { if (visbleErr) { ReportManage.ErrReport(this, url + "の読み込みに失敗しました。"); } if (ErrorEvent != null) { ErrorEvent(this, new EventArgs()); } if (ErrEventTree != null) { ErrEventTree.SetParent(); Document d = new Document() { TextValue = client.ErrMessage }; d.SetParent(this); d.AddChildren(ErrEventTree); d.Run(); } } return(false); } }
public RawlerBase GetUpperInterface <T>() { bool flag = false; RawlerBase rawler = this.Parent; while (true) { if (rawler == null) { break; } if (rawler is T) { flag = true; break; } else if (rawler.Parent == null) { break; } else { rawler = rawler.Parent; } } if (flag) { return(rawler); } else { return(null); } }
private void TsvFileSave() { if (doLastFileSave == false) { return; } string filename = this.FileName.Convert(this); if (this.FileNameTree != null) { filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this); } if (string.IsNullOrEmpty(filename) == false) { System.IO.StreamWriter sw = null; if (this.FileSaveMode == Tool.FileSaveMode.Create) { sw = System.IO.File.CreateText(filename); } else if (this.FileSaveMode == Tool.FileSaveMode.Append) { sw = System.IO.File.AppendText(filename); } sw.Write(ToTsv()); sw.Close(); ReportManage.Report(this, filename + "作成完了", true, EndReport); } }
/// <summary> /// 指定したキーで上流のKeyValueStoreから値を取得する。 /// </summary> /// <param name="rawler"></param> /// <param name="key"></param> /// <returns></returns> public static string GetValueByKey(RawlerBase rawler, string key) { KeyValueStore[] r = null; if (ancestorKeyValueStoreDic.ContainsKey(rawler)) { ancestorKeyValueStoreDic[rawler].TryGetTarget(out r); } if (r == null) { r = rawler.GetAncestorRawler().OfType <KeyValueStore>().ToArray(); if (ancestorKeyValueStoreDic.ContainsKey(rawler)) { ancestorKeyValueStoreDic[rawler].SetTarget(r); } else { ancestorKeyValueStoreDic.Add(rawler, new WeakReference <KeyValueStore[]>(r)); } } string val = null; foreach (var item in r) { if (item.dic.ContainsKey(key)) { val = item.dic[key]; break; } } if (val == null) { ReportManage.ErrReport(rawler, "key:" + key + "が見つかりません"); } return(val); }
public void JsonFileSave() { string filename = this.FileName.Convert(this); if (this.FileNameTree != null) { filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this); } if (string.IsNullOrEmpty(filename) == false) { System.IO.StreamWriter sw = null; if (this.FileSaveMode == Tool.FileSaveMode.Create) { sw = System.IO.File.CreateText(filename); } else if (this.FileSaveMode == Tool.FileSaveMode.Append) { sw = System.IO.File.AppendText(filename); } if (saveFileType == FileType.Json) { sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.GetDataDicForJson()))); } sw.Close(); ReportManage.Report(this, filename + "作成完了", true, EndReport); } }
//public new void Run() //{ // Run(true); //} /// <summary> /// 実行 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { string containsText = this.ContainsText.Convert(this); string text = this.GetText(); if (ContainsTextTree != null) { containsText = RawlerBase.GetText(this.Parent.Text, ContainsTextTree, this); } if (TextTree != null) { text = RawlerBase.GetText(this.Parent.Text, TextTree, this); } if (text.Contains(containsText) == this.Result) { this.RunChildren(runChildren); } //if (ContainsTextTree == null) //{ // if (this.GetText().Contains(this.ContainsText) == this.Result) // { // this.RunChildren(runChildren); // } //} //else //{ // string t = RawlerBase.GetText(this.Parent.Text, ContainsTextTree, this); // if (this.GetText().Contains(t) == this.Result) // { // this.RunChildren(runChildren); // } //} }
public override void Run(bool runChildren) { string t = string.Empty; if (string.IsNullOrEmpty(Value)) { t = GetText(); } else { t = Value; } if (Header != null) { t = Header + t; } if (HeaderTree != null) { t = RawlerBase.GetText(this.Parent.Text, HeaderTree, this.Parent) + t; } if (Footer != null) { t = t + Footer; } if (FooterTree != null) { t = t + RawlerBase.GetText(this.Parent.Text, FooterTree, this.Parent); } SetText(t); base.Run(runChildren); }
public override void Run(bool runChildren) { string group = string.Empty; if (GroupTree != null && this.Parent != null) { GroupTree.SetParent(); group = RawlerBase.GetText(this.Parent.Text, GroupTree, this); } else { if (GroupName != null) { group = GroupName; } } int num = 1; if (AddNumTree != null && this.Parent != null) { int.TryParse(RawlerBase.GetText(this.Parent.Text, AddNumTree, this), out num); } var c = this.GetAncestorRawler().Where(n => n is CountData); if (c.Count() > 0) { ((CountData)c.First()).AddCount(group, GetText(), num); } else { ReportManage.ErrReport(this, "上流にCountDataがありません"); } base.Run(runChildren); }
/// <summary> /// クローンを作る /// </summary> /// <typeparam name="T"></typeparam> /// <param name="parent"></param> /// <returns></returns> public virtual T Clone <T>(RawlerBase parent) where T : RawlerBase, new() { T clone = new T(); RawlerLib.ObjectLib.FildCopy(this as T, clone); clone.SetParent(parent); CloneEvent(clone); clone.Children.Clear(); foreach (var item in clone.GetType().GetProperties()) { if (item.PropertyType.IsSubclassOf(typeof(RawlerBase))) { var r = item.GetValue(this as T, null) as RawlerBase; item.SetValue(clone, r.Clone(), null); } //else if(item.PropertyType == typeof(RawlerCollection)) //{ // var col = item.GetValue(clone, null) as RawlerCollection; // col.Clear(); // foreach (var item2 in item.GetValue(this as T,null) as RawlerCollection) // { // col.Add(item2); // } //} } foreach (var item in this.Children) { var child = item.Clone(clone); clone.AddChildren(child); } return(clone); }
/// <summary> /// 直近の上流に指定の型があったら取得する。 /// </summary> /// <typeparam name="T"></typeparam> /// <returns></returns> public T GetUpperRawler <T>() where T : RawlerBase { T result = null; RawlerBase rawler = this.Parent; if (rawler != null) { while (true) { if (rawler is T) { result = rawler as T; break; } else if (rawler.Parent == null) { break; } else { rawler = rawler.Parent; } } } return(result); }
public override void Run(bool runChildren) { parameterList.Clear(); if (BeforeTrees.Any()) { foreach (var item in BeforeTrees) { RawlerBase.GetText(GetText(), item, this); } } string url; if (string.IsNullOrEmpty(Url)) { url = GetText(); } else { url = Url; } url = url + "?" + parameterList.Select(n => $"{Uri.EscapeUriString( n.Key)}={Uri.EscapeUriString(n.Value)}").JoinText("&"); SetText(url); base.Run(runChildren); }
/// <summary> /// {key} という記法で上流のKeyValueの値を取得する。 /// </summary> /// <param name="text"></param> /// <param name="rawler"></param> /// <returns></returns> public static string Convert(this string text, RawlerBase rawler) { if (text == null) { return(null); } else { return(GetConvertStr(text, rawler)); // var text2 = text.Replace("[[", "&(").Replace("]]", "&)"); //// System.Text.RegularExpressions.Regex r = new System.Text.RegularExpressions.Regex(@"\[\w*\]"); // foreach (var item in r.Matches(text).OfType<System.Text.RegularExpressions.Match>()) // { // var key = item.Value.Replace("[", "").Replace("]", ""); // var key1 = HeaderConvert(key, rawler); // if(key1.Length==0) // { // var val = KeyValueStore.GetValueByKey(rawler, key); // if (val.Length > 0) // { // text2 = text2.Replace(item.Value, val); // } // } // else // { // text2 = text2.Replace(item.Value, key1); // } // } // return text2.Replace("&(", "[").Replace("&)", "]"); } }
private static void AddReportEventArgs(RawlerBase sender, ReportEvnetArgs args) { if (sender != null) { var reportStock = sender.GetUpperRawler <ReportStock>(); if (reportStock != null) { reportStock.AddReport(args); return; } } if (args.IsErr) { if (stockErrReport) { reportList.Add(args); } } else { if (stockReport) { reportList.Add(args); } } }
/// <summary> /// Dispose() /// </summary> public virtual void Dispose() { this.text = null; this.parent = null; this.children = null; this.commnet = null; this.preTree = null; }
protected string GetFileName() { if (FileNameTree != null) { return(RawlerBase.GetText(this.GetText(), FileNameTree, this)); } return(this.FileName.Convert(this)); }
static string GetConvertStr(string text, RawlerBase rawler) { if (keyDic.ContainsKey(text)) { return(GetBlockText(keyDic[text], rawler)); } var text2 = text.Replace("[[", "&(").Replace("]]", "&)"); // System.Text.RegularExpressions.Regex r = new System.Text.RegularExpressions.Regex(@"\[\w*\]"); int postion = 0; List <StrBlock> blockList = new List <StrBlock>(); foreach (var item in r.Matches(text2).OfType <System.Text.RegularExpressions.Match>()) { var key = item.Value.Replace("[", "").Replace("]", ""); if (item.Index != postion) { blockList.Add(new StrBlock() { Text = text2.Substring(postion, item.Index - postion).Replace("&(", "[").Replace("&)", "]") }); } postion = item.Index + item.Value.Length; var d = key.Split(':'); if (d.Length == 1) { blockList.Add(new StrBlock_Key() { Key = key }); } else { blockList.Add(new StrBlock_Key2() { Key = d.Last(), Header = d.First() }); } } if (postion < text2.Length) { blockList.Add(new StrBlock() { Text = text2.Substring(postion, text2.Length - postion).Replace("&(", "[").Replace("&)", "]") }); } if (keyDic.ContainsKey(text)) { keyDic[text] = blockList.ToArray(); } else { keyDic.Add(text, blockList.ToArray()); } return(GetBlockText(keyDic[text], rawler)); }
protected override void CloneEvent(RawlerBase rawler) { base.CloneEvent(rawler); if (rawler is CustomText) { var r = (CustomText)rawler; r.CreateText = this.CreateText; } }
protected override void CloneEvent(RawlerBase rawler) { base.CloneEvent(rawler); if (rawler is CustomFilter) { var r = (CustomFilter)rawler; r.Check = this.Check; } }
public void Run(RawlerBase parent) { try { var type = parent.GetType(); var field = type.GetProperty(PropertyName); var text = RawlerBase.GetText(string.Empty, Child, parent); if (field.PropertyType == typeof(string)) { field.SetValue(parent, text, null); } else if (field.PropertyType == typeof(int)) { int num; if (int.TryParse(text, out num)) { field.SetValue(parent, num, null); } else { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をint型に変換に失敗しました"); } } else if (field.PropertyType == typeof(double)) { double num; if (double.TryParse(text, out num)) { field.SetValue(parent, num, null); } else { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をdouble型に変換に失敗しました"); } } else if (field.PropertyType == typeof(bool)) { if (text.ToLower() == "true") { field.SetValue(parent, true, null); } else if (text.ToLower() == "false") { field.SetValue(parent, false, null); } else { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をbool型に変換に失敗しました。Valueは" + text); } } } catch (Exception ex) { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "でエラーが発生しました。" + ex.Message); } }
public ReportEvnetArgs(RawlerBase sender, string message, bool returncode, int code = 0) : base() { this.Message = message; this.IsErr = false; this.DateTime = DateTime.Now; this.Visible = false; this.ReturnCode = returncode; this.Code = code; }
public override T Clone <T>(RawlerBase parent) { var obj = base.Clone <T>(parent); if (obj is RawlerMultiBase) { (obj as RawlerMultiBase).Query = this.Query.Clone(); } return(obj); }
/// <summary> /// 指定したテキストをPreTreeにかける。 /// </summary> /// <param name="text"></param> /// <returns></returns> protected string GetText(string preText) { if (preTree != null) { return(RawlerBase.GetText(preText, PreTree)); } else { return(preText); } }
public static string GetText(string text, RawlerBase rawler) { Document doc = new Document(); doc.SetText(text); rawler.SetParent(doc); var last = rawler.GetDescendantRawler().Last(); rawler.Run(); return(last.Text); }
/// <summary> /// 初期化実行 /// </summary> /// <param name="root"></param> public void Run(RawlerBase root) { if (this.Count == 0) { return; } foreach (var item in this) { item.Run(root); } }
private static string GetTopComment(RawlerBase sender) { if (visbleTopObjectComment) { return(sender.GetAncestorRawler().Last().Comment + ":"); } else { return(string.Empty); } }
public ReportEvnetArgs(RawlerBase sender, string message, bool returncode, bool isErr, int code = 0) : base() { // this.Sender = sender; this.Message = message; this.IsErr = isErr; this.DateTime = DateTime.Now; this.Visible = true; this.ReturnCode = returncode; this.Code = code; }
public override IEnumerable<string> Query(IEnumerable<string> list) { if (KeyTree == null) { return list.OrderBy(n => n); } else { return list.OrderBy(n => RawlerBase.GetText(n, KeyTree)); } }
public void addDocument(string text, RawlerBase rawlerTree) { var d = new Document(); d.SetText(text); d.SetParent(this); var tree = rawlerTree.Clone(); d.AddChildren(tree); documents.Add(d); }
protected string GetSwitchValue() { if (SwitchValueTree != null) { SwitchValueTree.SetParent(); return(RawlerBase.GetText(this.Parent.Text, SwitchValueTree, this.Parent)); } else { return(GetText()); } }
public override void Run(bool runChildren) { try { RawlerBase rawler = (RawlerBase)System.Xaml.XamlServices.Load(FileName); rawler.SetParent(this); rawler.Run(); } catch (Exception ex) { ReportManage.ErrReport(this, ex.ToString()); } }