public override void Run(bool runChildren) { string t = string.Empty; if (string.IsNullOrEmpty(Value)) { t = GetText(); } else { t = Value; } if (Header != null) { t = Header + t; } if (HeaderTree != null) { t = RawlerBase.GetText(this.Parent.Text, HeaderTree, this.Parent) + t; } if (Footer != null) { t = t + Footer; } if (FooterTree != null) { t = t + RawlerBase.GetText(this.Parent.Text, FooterTree, this.Parent); } SetText(t); base.Run(runChildren); }
private void TsvFileSave() { if (doLastFileSave == false) { return; } string filename = this.FileName.Convert(this); if (this.FileNameTree != null) { filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this); } if (string.IsNullOrEmpty(filename) == false) { System.IO.StreamWriter sw = null; if (this.FileSaveMode == Tool.FileSaveMode.Create) { sw = System.IO.File.CreateText(filename); } else if (this.FileSaveMode == Tool.FileSaveMode.Append) { sw = System.IO.File.AppendText(filename); } sw.Write(ToTsv()); sw.Close(); ReportManage.Report(this, filename + "作成完了", true, EndReport); } }
protected bool ReadPage(string url) { var client = GetWebClient(); parameterList.Clear(); httpHeaderList.Clear(); BeforeTrees.Run(this, GetText()); if (InputParameterTree != null) { RawlerBase.GetText(GetText(), InputParameterTree, this); } if (MethodType == Tool.MethodType.GET) { this.text = client.HttpGet(url, parameterList, httpHeaderList); } else if (MethodType == Tool.MethodType.POST) { this.text = client.HttpPost(url, parameterList, httpHeaderList); } this.currentUrl = url; this.pastUrl = this.currentUrl; if (this.Text.Length > 0) { return(true); } else { if (client.ErrMessage != null && (client.ErrMessage.Contains("503") || client.ErrMessage.Contains("500"))) { ReportManage.ErrReport(this, $"{client.ErrMessage} {url}の読み込みに失敗しました。"); } else { if (visbleErr) { ReportManage.ErrReport(this, url + "の読み込みに失敗しました。"); } if (ErrorEvent != null) { ErrorEvent(this, new EventArgs()); } if (ErrEventTree != null) { ErrEventTree.SetParent(); Document d = new Document() { TextValue = client.ErrMessage }; d.SetParent(this); d.AddChildren(ErrEventTree); d.Run(); } } return(false); } }
public override void Run(bool runChildren) { parameterList.Clear(); if (BeforeTrees.Any()) { foreach (var item in BeforeTrees) { RawlerBase.GetText(GetText(), item, this); } } string url; if (string.IsNullOrEmpty(Url)) { url = GetText(); } else { url = Url; } url = url + "?" + parameterList.Select(n => $"{Uri.EscapeUriString( n.Key)}={Uri.EscapeUriString(n.Value)}").JoinText("&"); SetText(url); base.Run(runChildren); }
//public new void Run() //{ // Run(true); //} /// <summary> /// 実行 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { string containsText = this.ContainsText.Convert(this); string text = this.GetText(); if (ContainsTextTree != null) { containsText = RawlerBase.GetText(this.Parent.Text, ContainsTextTree, this); } if (TextTree != null) { text = RawlerBase.GetText(this.Parent.Text, TextTree, this); } if (text.Contains(containsText) == this.Result) { this.RunChildren(runChildren); } //if (ContainsTextTree == null) //{ // if (this.GetText().Contains(this.ContainsText) == this.Result) // { // this.RunChildren(runChildren); // } //} //else //{ // string t = RawlerBase.GetText(this.Parent.Text, ContainsTextTree, this); // if (this.GetText().Contains(t) == this.Result) // { // this.RunChildren(runChildren); // } //} }
public void JsonFileSave() { string filename = this.FileName.Convert(this); if (this.FileNameTree != null) { filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this); } if (string.IsNullOrEmpty(filename) == false) { System.IO.StreamWriter sw = null; if (this.FileSaveMode == Tool.FileSaveMode.Create) { sw = System.IO.File.CreateText(filename); } else if (this.FileSaveMode == Tool.FileSaveMode.Append) { sw = System.IO.File.AppendText(filename); } if (saveFileType == FileType.Json) { sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.GetDataDicForJson()))); } sw.Close(); ReportManage.Report(this, filename + "作成完了", true, EndReport); } }
public override void Run(bool runChildren) { string group = string.Empty; if (GroupTree != null && this.Parent != null) { GroupTree.SetParent(); group = RawlerBase.GetText(this.Parent.Text, GroupTree, this); } else { if (GroupName != null) { group = GroupName; } } int num = 1; if (AddNumTree != null && this.Parent != null) { int.TryParse(RawlerBase.GetText(this.Parent.Text, AddNumTree, this), out num); } var c = this.GetAncestorRawler().Where(n => n is CountData); if (c.Count() > 0) { ((CountData)c.First()).AddCount(group, GetText(), num); } else { ReportManage.ErrReport(this, "上流にCountDataがありません"); } base.Run(runChildren); }
protected string GetFileName() { if (FileNameTree != null) { return(RawlerBase.GetText(this.GetText(), FileNameTree, this)); } return(this.FileName.Convert(this)); }
public void Run(RawlerBase parent) { try { var type = parent.GetType(); var field = type.GetProperty(PropertyName); var text = RawlerBase.GetText(string.Empty, Child, parent); if (field.PropertyType == typeof(string)) { field.SetValue(parent, text, null); } else if (field.PropertyType == typeof(int)) { int num; if (int.TryParse(text, out num)) { field.SetValue(parent, num, null); } else { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をint型に変換に失敗しました"); } } else if (field.PropertyType == typeof(double)) { double num; if (double.TryParse(text, out num)) { field.SetValue(parent, num, null); } else { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をdouble型に変換に失敗しました"); } } else if (field.PropertyType == typeof(bool)) { if (text.ToLower() == "true") { field.SetValue(parent, true, null); } else if (text.ToLower() == "false") { field.SetValue(parent, false, null); } else { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をbool型に変換に失敗しました。Valueは" + text); } } } catch (Exception ex) { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "でエラーが発生しました。" + ex.Message); } }
/// <summary> /// 指定したテキストをPreTreeにかける。 /// </summary> /// <param name="text"></param> /// <returns></returns> protected string GetText(string preText) { if (preTree != null) { return(RawlerBase.GetText(preText, PreTree)); } else { return(preText); } }
public override IEnumerable<string> Query(IEnumerable<string> list) { if (KeyTree == null) { return list.OrderBy(n => n); } else { return list.OrderBy(n => RawlerBase.GetText(n, KeyTree)); } }
protected string GetSwitchValue() { if (SwitchValueTree != null) { SwitchValueTree.SetParent(); return(RawlerBase.GetText(this.Parent.Text, SwitchValueTree, this.Parent)); } else { return(GetText()); } }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { var page = (IInputParameter)this.GetUpperInterface <IInputParameter>(); if (page == null) { ReportManage.ErrUpperNotFound <IInputParameter>(this); return; } string key = this.Key; if (KeyTree != null) { key = RawlerBase.GetText(this.Parent.Text, KeyTree, this.Parent); } else { key = key.Convert(this); } if (Value == null) { if (AddType == AddInputParameterType.replece) { page.ReplaceParameter(key, GetText()); } else { page.AddParameter(key, GetText()); } } else { if (AddType == AddInputParameterType.replece) { page.ReplaceParameter(key, Value.Convert(this)); } else { page.AddParameter(key, Value.Convert(this)); } } base.Run(runChildren); }
private IEnumerable <string> Convert(IEnumerable <string> list) { if (ConvertTree != null) { ConvertTree.SetParent(this.Parent); foreach (var item in list) { yield return(RawlerBase.GetText(item, ConvertTree, this.Parent)); } } else { foreach (var item in list) { yield return(item); } } }
public override IEnumerable<string> Query(IEnumerable<string> list) { var root = this.GetRoot().Rawler; if(SelectTree !=null) { foreach (var item in list) { yield return RawlerBase.GetText(item, SelectTree, root); } } else { ReportManage.ErrReport(root, "SelectTreeがありません"); // yield return base.Query(list); } }
public override void Run(bool runChildren) { string t = string.Empty; if (string.IsNullOrEmpty(Value)) { t = GetText(); } else { t = Value; } try { var dir = System.IO.Path.GetDirectoryName(t) + "\\"; var file = System.IO.Path.GetFileNameWithoutExtension(t); var ext = System.IO.Path.GetExtension(t); t = file; if (Header != null) { t = Header + t; } if (HeaderTree != null) { t = RawlerBase.GetText(this.Parent.Text, HeaderTree, this.Parent) + t; } if (Footer != null) { t = t + Footer; } if (FooterTree != null) { t = t + RawlerBase.GetText(this.Parent.Text, FooterTree, this.Parent); } t = dir + t + ext; } catch { ReportManage.ErrReport(this, "File名として不正です。"); } SetText(t); base.Run(runChildren); }
public override void Run(bool runChildren) { var file = this.GetUpperRawler <TsvReadLines>(); if (file != null) { string columnName = this.ColumnName.Convert(this); if (this.ColumnNameTree != null) { columnName = RawlerBase.GetText(GetText(), ColumnNameTree, this); } if (string.IsNullOrEmpty(columnName) == false) { if (file.ColumnNameDic.ContainsKey(columnName)) { var i = file.ColumnNameDic[columnName]; var d = file.Text.Split('\t'); if (d.Length > i) { SetText(d[i]); } else { SetText(Default); } } else { ReportManage.ErrReport(this, "該当するキーがありません:" + columnName); } } else { ReportManage.ErrReport(this, "ColumnNameが空です。"); } } else { ReportManage.ErrReport(this, "GetTsvValue の上流にTsvReadLinesがありません"); } base.Run(runChildren); }
/// <summary> /// PreTreeを通したあとのText /// </summary> /// <returns></returns> protected string GetText() { if (this.Parent != null) { if (preTree != null) { preTree.SetParent(); return(RawlerBase.GetText(this.Parent.Text, PreTree, this.Parent)); } else { return(this.Parent.Text); } } else { return(this.text); } }
private IEnumerable <TextPair <T> > Convert <T>(IEnumerable <TextPair <T> > list, Func <T, string> textFunc) { if (ConvertTree != null) { ConvertTree.SetParent(this.Parent); foreach (var item in list) { item.Text = RawlerBase.GetText(item.Text, ConvertTree, this.Parent); yield return(item); //yield return RawlerBase.GetText(item, ConvertTree, this.Parent); } } else { foreach (var item in list) { yield return(item); } } }
public override void Run(bool runChildren) { try { string fileName = this.FileName.Convert(this); if (FileNameTree != null) { FileNameTree.SetParent(); FileNameTree.SetParent(this); fileName = RawlerBase.GetText(GetText(), FileNameTree, this); } if (string.IsNullOrEmpty(fileName)) { if (string.IsNullOrEmpty(fileName)) { ReportManage.ErrReport(this, "FileNameが空です。"); return; } } baseFileName = fileName; currentFileName = fileName; FileInit(fileName); base.Run(runChildren); endAction?.Invoke(); } catch (Exception e) { ReportManage.ErrReport(this, e.Message); } finally { if (streamWriter != null) { streamWriter.Close(); streamWriter.Dispose(); } } }
/// <summary> /// ファイルで保存する /// </summary> protected void SaveFile() { string filename = this.FileName.Convert(this); if (this.FileNameTree != null) { filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this); } if (string.IsNullOrEmpty(filename) == false) { System.IO.StreamWriter sw = null; if (this.FileSaveMode == Tool.FileSaveMode.Create) { sw = System.IO.File.CreateText(filename); } else if (this.FileSaveMode == Tool.FileSaveMode.Append) { sw = System.IO.File.AppendText(filename); } if (saveFileType == FileType.Json) { sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.DataDic))); } else if (saveFileType == FileType.Tsv) { sw.WriteLine(this.ToTsv()); } else if (saveFileType == FileType.LTsv) { foreach (var item in this.GetDataRows()) { var d = item.DataDic.ToDictionary(n => n.Key, n => n.Value.JoinText(",")); sw.WriteLine(d.ToLtsvLine()); } } sw.Close(); ReportManage.Report(this, filename + "作成完了", true, EndReport); } }
public override void Run(bool runChildren) { string path = string.Empty; if (workFolderType == Tool.SpecialFolder.MyDocuments) { path = System.Environment.GetFolderPath(Environment.SpecialFolder.Personal); } if (workFolderType == Tool.SpecialFolder.Desktop) { path = System.Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory); } string folder = null; if (FolderNameTree != null) { folder = RawlerBase.GetText(this.GetText(), FolderNameTree, this); } else { folder = FolderName; } if (folder == null || folder.Length == 0) { ReportManage.ErrReport(this, "CreateFolderのFolderNameがありません"); } else { path = System.IO.Path.Combine(path, folder); if (System.IO.Directory.Exists(path) == false) { System.IO.Directory.CreateDirectory(path); } } base.Run(runChildren); }
public override void Run(bool runChildren) { string file = string.Empty; if (this.FileName != null) { file = this.FileName; } else if (FileNameTree != null) { file = RawlerBase.GetText(GetText(), FileNameTree, this); } else { file = this.GetText(); } if (System.IO.File.Exists(file) == Result) { base.Run(runChildren); } }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { if (string.IsNullOrEmpty(Format)) { ReportManage.ErrReport(this, "Formatが空です"); } else if (Args == null) { ReportManage.ErrReport(this, "Argsが空です"); } else { List <string> list = new List <string>(); foreach (var item in Args) { item.SetParent(this.Parent); item.SetParent(); list.Add(RawlerBase.GetText(GetText(), item, this.Parent)); } SetText(string.Format(Format, list.ToArray())); } base.Run(runChildren); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { string path = string.Empty; if (workFolderType == Tool.SpecialFolder.MyDocuments) { path = System.Environment.GetFolderPath(Environment.SpecialFolder.Personal); } if (workFolderType == Tool.SpecialFolder.Desktop) { path = System.Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory); } if (FolderTree != null) { var f = RawlerBase.GetText(GetText(), FolderTree, this); path = System.IO.Path.Combine(path, f); } else if (Folder != null) { path = System.IO.Path.Combine(path, Folder); } try { if (System.IO.Directory.Exists(path) == false) { System.IO.Directory.CreateDirectory(path); } System.IO.Directory.SetCurrentDirectory(path); } catch (Exception e) { ReportManage.ErrReport(this, e.Message); } SetText(System.IO.Directory.GetCurrentDirectory()); base.Run(runChildren); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { var client = this.GetAncestorRawler().OfType <WebClient>().DefaultIfEmpty(new WebClient()).FirstOrDefault(); string url = Url; if (string.IsNullOrEmpty(url)) { url = GetText(); } var page = this.GetUpperRawler <Page>(); if (page != null) { client.Referer = page.GetCurrentUrl(); } var data = client.HttpGetByte(url); string path = string.Empty; if (FolderNameTree != null) { path = RawlerBase.GetText(this.Parent.Text, FolderNameTree, this.Parent); } else { path = Folder.Convert(this); } if (path != null) { if (System.IO.Path.IsPathRooted(path) == false) { path = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), path); } if (System.IO.Directory.Exists(path) == false) { try { System.IO.Directory.CreateDirectory(path); } catch (Exception e) { ReportManage.ErrReport(this, "ディレクトリーの作成に失敗しました" + e.Message); } } } else { path = System.IO.Directory.GetCurrentDirectory(); } string ex = System.IO.Path.GetExtension(url); if (ex.Split('?').Length > 0) { ex = ex.Split('?').First(); } string fileName = string.Empty; if (SaveNameTree != null) { fileName = RawlerBase.GetText(this.Parent.Text, SaveNameTree, this.Parent); } else if (string.IsNullOrEmpty(SaveName) == false) { fileName = SaveName.Convert(this); } else { fileName = System.IO.Path.GetFileNameWithoutExtension(url); } path = System.IO.Path.Combine(path, fileName + ex); try { using (var writer = System.IO.File.Create(path)) { writer.Write(data, 0, data.Length); SetText(path); } } catch (Exception e) { ReportManage.ErrReport(this, "ファイルの保存に失敗しました" + e.Message + " Path:" + path); } SetText(fileName + ex); base.Run(runChildren); }
//public new void Run() //{ // Run(true); //} /// <summary> /// 実行 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { string txt = string.Empty; if (Value == null || Value.Length == 0) { txt = GetText(); } else { txt = Value.Convert(this); } if (useHtmlDecode) { txt = System.Net.WebUtility.HtmlDecode(txt); } string attribute = string.Empty; if (this.Attribute == null && attributeTree != null) { if (this.Parent != null) { attribute = RawlerBase.GetText(this.Parent.Text, attributeTree, this); } } else if (this.Attribute != null) { attribute = this.Attribute.Convert(this); } Data.DataWrite(this, attribute, txt, writeType, AttributeType); //IData data = null; //IRawler current = this.Parent; //while (current != null) //{ // if (current is IData) // { // data = current as IData; // break; // } // current = current.Parent; //} //if (data != null) //{ // string txt = string.Empty; // if (Value == null || Value.Length == 0) // { // txt = GetText(); // } // else // { // txt = Value; // } // if (useHtmlDecode) // { // txt = System.Net.WebUtility.HtmlDecode(txt); // } // if (this.Attribute == null && attributeTree != null) // { // string tmpAttributeText = string.Empty; // if (this.Parent != null) // { // tmpAttributeText = RawlerBase.GetText(this.Parent.Text, attributeTree,this); // } // data.DataWrite(tmpAttributeText, txt, writeType,AttributeType); // } // else // { // if (this.Attribute != null) // { // data.DataWrite(this.Attribute, txt, writeType,AttributeType); // } // else // { // data.DataWrite(string.Empty, txt, writeType, AttributeType); // } // } // //if (this.AttributeObjectName != null && this.AttributeObjectName.Length>0) // //{ // // var list = this.GetConectAllRawler().Where(n => n.Name == this.AttributeObjectName); // // if (list.Count() > 0) // // { // // data.DataWrite(list.First().Text, txt,writeType); // // } // // else // // { // // ReportManage.ErrReport(this, "AttributeObjectNameの指定が不正です。オブジェクトが見つかりませんでした。"); // // } // //} //} //else //{ // ReportManage.ErrReport(this,"書き込み先のDataオブジェクトが見つかりません。"); //} this.RunChildren(runChildren); }