public override void Run(bool runChildren) { string group = string.Empty; if (GroupTree != null && this.Parent != null) { GroupTree.SetParent(); group = RawlerBase.GetText(this.Parent.Text, GroupTree, this); } else { if (GroupName != null) { group = GroupName; } } int num = 1; if (AddNumTree != null && this.Parent != null) { int.TryParse(RawlerBase.GetText(this.Parent.Text, AddNumTree, this), out num); } var c = this.GetAncestorRawler().Where(n => n is CountData); if (c.Count() > 0) { ((CountData)c.First()).AddCount(group, GetText(), num); } else { ReportManage.ErrReport(this, "上流にCountDataがありません"); } base.Run(runChildren); }
/// <summary> /// ElementAt を適応する。 /// </summary> /// <typeparam name="T"></typeparam> /// <param name="list"></param> /// <returns></returns> IEnumerable <T> DoElementAt <T>(IEnumerable <T> list) { if (list.Count() == 0) { return(list); } if (ElementAt.HasValue) { try { var l = list.ToArray(); if (ElementAt.Value > -1) { return(new T[] { l[ElementAt.Value] }); //return new List<T>() { list.ElementAt(ElementAt.Value) }; } else { return(new T[] { l[list.Count() + ElementAt.Value] }); // return new List<T>() { list.ElementAt(list.Count() + ElementAt.Value) }; } } catch { ReportManage.ErrReport(this, "ElementAtの値がレンジから外れました。ElementAt:" + ElementAt.Value); } } return(list); }
public void PushUrl(string url) { var tmp = url.Replace("&#", "&&&&"); var u = tmp.Split('#'); if (u.Length > 0) { url = u[0].Replace("&&&&", "&#"); } if (oncePageLoad) { if (urlHash.Add(url)) { urlStack.Push(url); } else { if (visbleErr) { ReportManage.ErrReport(this, "すでに読み込んだURLです。スルーします。 " + url); } } } else { urlStack.Push(url); } }
public override void Run(bool runChildren) { if (string.IsNullOrEmpty(ParameterName) == false) { Regex regex = new Regex(ParameterName + "[ ]*=[\"| ]*(.+?)[\"| |$]", RegexOptions.IgnoreCase); Regex regex1 = new Regex(ParameterName + "[ ]*=[\'| ]*(.+?)[\'| |$]", RegexOptions.IgnoreCase); var r = regex.Match(GetText()); var r2 = regex1.Match(GetText()); if (r.Success) { SetText(r.Groups[1].Value); base.Run(runChildren); } else if (r2.Success) { SetText(r2.Groups[1].Value); base.Run(runChildren); } else { ReportManage.ErrReport(this, "GetTagParameterで指定した" + ParameterName + "が見つかりませんでした"); } } else { ReportManage.ErrReport(this, "GetTagParameterで指定した" + ParameterName + "が空文字です"); } }
/// <summary> /// 指定したキーで上流のKeyValueStoreから値を取得する。 /// </summary> /// <param name="rawler"></param> /// <param name="key"></param> /// <returns></returns> public static string GetValueByKey(RawlerBase rawler, string key) { KeyValueStore[] r = null; if (ancestorKeyValueStoreDic.ContainsKey(rawler)) { ancestorKeyValueStoreDic[rawler].TryGetTarget(out r); } if (r == null) { r = rawler.GetAncestorRawler().OfType <KeyValueStore>().ToArray(); if (ancestorKeyValueStoreDic.ContainsKey(rawler)) { ancestorKeyValueStoreDic[rawler].SetTarget(r); } else { ancestorKeyValueStoreDic.Add(rawler, new WeakReference <KeyValueStore[]>(r)); } } string val = null; foreach (var item in r) { if (item.dic.ContainsKey(key)) { val = item.dic[key]; break; } } if (val == null) { ReportManage.ErrReport(rawler, "key:" + key + "が見つかりません"); } return(val); }
public string GetValue(string columnName) { var data = this.Text.Split('\t'); if (columnNameDic.ContainsKey(columnName)) { if (data.Length >= columnNameDic[columnName]) { try { return(data[columnNameDic[columnName]]); } catch (Exception e) { ReportManage.ErrReport(this, e.Message + " ColumnName:" + columnName + " Text:" + this.Text); } } else { ReportManage.ErrReport(this, "キーが圏外です。:" + columnName); } } else { ReportManage.ErrReport(this, "該当するキーがありません:" + columnName); } return(string.Empty); }
private void TsvFileSave() { if (doLastFileSave == false) { return; } string filename = this.FileName.Convert(this); if (this.FileNameTree != null) { filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this); } if (string.IsNullOrEmpty(filename) == false) { System.IO.StreamWriter sw = null; if (this.FileSaveMode == Tool.FileSaveMode.Create) { sw = System.IO.File.CreateText(filename); } else if (this.FileSaveMode == Tool.FileSaveMode.Append) { sw = System.IO.File.AppendText(filename); } sw.Write(ToTsv()); sw.Close(); ReportManage.Report(this, filename + "作成完了", true, EndReport); } }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { var paras = GetText().Split('?'); List <string> list = new List <string>(); if (paras.Length > 1) { foreach (var item in paras.Last().Split('&')) { if (string.IsNullOrEmpty(ParameterName) == false) { var d = item.Split('='); if (d.First() == ParameterName) { list.Add(d.Last()); } } else { list.Add(item); } } } else { ReportManage.ErrReport(this, "UrlParameter:?がありません。パラメータが見つかりませんでした"); } base.RunChildrenForArray(runChildren, list); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { System.Text.RegularExpressions.Regex r = new System.Text.RegularExpressions.Regex("<input [^>]*>"); System.Text.RegularExpressions.Regex r2 = new System.Text.RegularExpressions.Regex(@"(\w)*\s*=\s*" + "\"([^\"]*)\""); var page = (IInputParameter)this.GetUpperInterface <IInputParameter>(); if (page == null) { ReportManage.ErrUpperNotFound <IInputParameter>(this); return; } List <KeyValue> list = new List <KeyValue>(); foreach (System.Text.RegularExpressions.Match item in r.Matches(GetText())) { var dic = GetParameter(item.Value); if (dic.ContainsKey("type") && dic["type"] == "hidden") { if (dic.ContainsKey("name") && dic.ContainsKey("value")) { page.AddParameter(dic["name"], dic["value"]); list.Add(new KeyValue() { Key = dic["name"], Value = dic["value"] }); } } } base.Run(runChildren); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { DateTime dt; if (DateTime.TryParse(GetText(), out dt)) { if (Format != null) { this.SetText(dt.ToString(Format)); } else { this.SetText(dt.ToString()); } base.Run(runChildren); } else { if (ErrText != null) { SetText(ErrText); } else { SetText("失敗:" + GetText()); } ReportManage.ErrReport(this, GetText() + "はDateTime.TryParseに失敗ました。"); } }
public override void Run(bool runChildren) { string filename = FileName.Convert(this); if (string.IsNullOrEmpty(filename)) { filename = GetText(); } if (string.IsNullOrEmpty(filename)) { ReportManage.ErrEmptyPropertyName(this, nameof(FileName)); return; } if (System.IO.File.Exists(filename) == false) { ReportManage.ErrReport(this, "File「" + filename + "」は存在しません"); return; } try { SetText(System.IO.File.ReadAllText(filename)); } catch (Exception ex) { ReportManage.ErrReport(this, filename + "を開くのに失敗しました" + ex.Message); return; } base.Run(runChildren); }
public override void Run(bool runChildren) { List <string> list = new List <string>(); System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(pattern, regexOption); foreach (System.Text.RegularExpressions.Match match in regex.Matches(GetText())) { if (match.Groups.Count > groupNum) { list.Add(match.Groups[groupNum].Value); } } if (list.Count == 0) { if (emptyReport) { ReportManage.ErrReport(this, "対象が見つかりませんでした"); } } else { this.RunChildrenForArray(runChildren, texts); } }
/// <summary> /// commentを出力する。 /// </summary> protected void ReportComment() { if (!string.IsNullOrEmpty(this.commnet)) { ReportManage.Report(this, this.Comment); } }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { if (FileName != null) { try { string line = GetText(); if (string.IsNullOrEmpty(Line) == false) { line = Line.Convert(this); } System.IO.File.AppendAllText(FileName.Convert(this), line + "\n"); } catch (Exception e) { ReportManage.ErrReport(this, "ファイルの書き込みに失敗しました。" + e.Message); } } else { ReportManage.ErrReport(this, "ファイル名を指定してください"); } base.Run(runChildren); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { var page = this.GetAncestorRawler().OfType <Page>().FirstOrDefault(); if (page != null) { if (tmpUrl == page.GetCurrentUrl()) { tmpCount++; if (tmpCount >= reloadCount) { ReportManage.ErrReport(this, "ReloadCount:規定数のリロード回数を超えました。"); OverCountTree.SetParent(this); OverCountTree.Run(); return; } } else { tmpUrl = page.GetCurrentUrl(); tmpCount = 0; } page.Reload(); System.Threading.Thread.Sleep((int)(sleepSeconds * 1000)); } base.Run(runChildren); }
public void JsonFileSave() { string filename = this.FileName.Convert(this); if (this.FileNameTree != null) { filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this); } if (string.IsNullOrEmpty(filename) == false) { System.IO.StreamWriter sw = null; if (this.FileSaveMode == Tool.FileSaveMode.Create) { sw = System.IO.File.CreateText(filename); } else if (this.FileSaveMode == Tool.FileSaveMode.Append) { sw = System.IO.File.AppendText(filename); } if (saveFileType == FileType.Json) { sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.GetDataDicForJson()))); } sw.Close(); ReportManage.Report(this, filename + "作成完了", true, EndReport); } }
protected bool ReadPage(string url) { var client = GetWebClient(); parameterList.Clear(); httpHeaderList.Clear(); BeforeTrees.Run(this, GetText()); if (InputParameterTree != null) { RawlerBase.GetText(GetText(), InputParameterTree, this); } if (MethodType == Tool.MethodType.GET) { this.text = client.HttpGet(url, parameterList, httpHeaderList); } else if (MethodType == Tool.MethodType.POST) { this.text = client.HttpPost(url, parameterList, httpHeaderList); } this.currentUrl = url; this.pastUrl = this.currentUrl; if (this.Text.Length > 0) { return(true); } else { if (client.ErrMessage != null && (client.ErrMessage.Contains("503") || client.ErrMessage.Contains("500"))) { ReportManage.ErrReport(this, $"{client.ErrMessage} {url}の読み込みに失敗しました。"); } else { if (visbleErr) { ReportManage.ErrReport(this, url + "の読み込みに失敗しました。"); } if (ErrorEvent != null) { ErrorEvent(this, new EventArgs()); } if (ErrEventTree != null) { ErrEventTree.SetParent(); Document d = new Document() { TextValue = client.ErrMessage }; d.SetParent(this); d.AddChildren(ErrEventTree); d.Run(); } } return(false); } }
public override void Run(bool runChildren) { var data = (IData)this.GetUpperInterface <IData>(); if (data == null) { ReportManage.ErrUpperNotFound <IData>(this); } var currentRow = data.GetCurrentDataRow(); var attribute = Attribute.Convert(this); SetText(GetText()); if (currentRow.DataDic.TryGetValue(attribute, out List <string> list) == false) { base.Run(runChildren); } else { if (list.Any() == false) { base.Run(runChildren); } } }
public override void Run(bool runChildren) { var d = (IData)this.GetUpperInterface <IData>(); if (d != null) { ReportManage.Report(this, d.GetCurrentDataRow().ToString(), true, true); } }
public override void Run(bool runChildren) { Data data = null; IRawler current = this.Parent; while (current != null) { if (current is Data) { data = current as Data; break; } current = current.Parent; } if (data != null) { if (data.GetCurrentDataNull()) { if (ignoreDataNull == false) { ReportManage.ErrReport(this, "RowがNullです。Writeが動作していないようです。"); var list = this.GetAncestorRawler().Where(n => n is Page); if (DoPageReLoad) { if (list.Count() > 0) { var p = list.First() as Page; pageCount++; if (PageReLoadCount < pageCount) { ReportManage.Report(this, "再読み込み待機中。"); System.Threading.Thread.Sleep(1000 * pageCount * pageCount); p.Run(); } else { ReportManage.ErrReport(this, "書き込み先のData クラスが見つかりませんでした。"); } } } } } else { pageCount = 0; data.NextDataRow(); ReportManage.Report(this, "NextDataRow"); } } else { ReportManage.ErrReport(this, "書き込み先のData クラスが見つかりませんでした。"); } this.RunChildren(runChildren); }
public override bool Check(string txt) { DateTime dt; if (StartDate == null && EndDate == null) { return(false); } if (DateTime.TryParse(txt, out dt)) { if (StartDate == null && EndDate != null) { if (dt < EndDate) { return(true); } else { return(false); } } else if (StartDate != null && EndDate != null) { if (dt >= StartDate && dt < EndDate) { return(true); } else { return(false); } } else { if (dt >= StartDate) { return(true); } else { return(false); } } } else { if (ErrReport) { ReportManage.ErrReport(this, "DateTime型のキャストに失敗:" + txt); } return(false); } // return base.Check(txt); }
public void Run(RawlerBase parent) { try { var type = parent.GetType(); var field = type.GetProperty(PropertyName); var text = RawlerBase.GetText(string.Empty, Child, parent); if (field.PropertyType == typeof(string)) { field.SetValue(parent, text, null); } else if (field.PropertyType == typeof(int)) { int num; if (int.TryParse(text, out num)) { field.SetValue(parent, num, null); } else { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をint型に変換に失敗しました"); } } else if (field.PropertyType == typeof(double)) { double num; if (double.TryParse(text, out num)) { field.SetValue(parent, num, null); } else { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をdouble型に変換に失敗しました"); } } else if (field.PropertyType == typeof(bool)) { if (text.ToLower() == "true") { field.SetValue(parent, true, null); } else if (text.ToLower() == "false") { field.SetValue(parent, false, null); } else { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をbool型に変換に失敗しました。Valueは" + text); } } } catch (Exception ex) { ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "でエラーが発生しました。" + ex.Message); } }
public override bool Check(string txt) { int dt; if (Start == null && End == null) { return(false); } if (int.TryParse(txt, out dt)) { if (Start == null && End != null) { if (dt < End) { return(true); } else { return(false); } } else if (Start != null && End != null) { if (dt >= Start && dt < End) { return(true); } else { return(false); } } else { if (dt >= Start) { return(true); } else { return(false); } } } else { if (ErrReport) { ReportManage.ErrReport(this, "int型のキャストに失敗:" + txt); } return(false); } // return base.Check(txt); }
/// <summary> /// 再びログインする。 /// </summary> public void ReLogin() { string html = this.HttpPost(loginPage, vals, false); if (this.GetCookieCount() > 0) { if (ErrString != null) { if (html.Contains(ErrString) == false) { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } else { hasLogin = false; ReportManage.ErrReport(this, "ログイン失敗"); } } else { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } } else { if (isNoCookieSite) { hasLogin = false; ReportManage.ErrReport(this, "ログイン失敗"); } else { if (ErrString != null) { if (html.Contains(ErrString) == false) { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } else { hasLogin = false; ReportManage.ErrReport(this, "ログイン失敗"); } } else { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } } } this.text = html; }
public override void Run(bool runChildren) { var g = this.GetUpperRawler <GroupBy>(); if (g == null) { ReportManage.ErrUpperNotFound <GroupBy>(this); return; } RunChildrenForArray(runChildren, g.GetCurrentKeyValue().Value); }
/// <summary> /// 実行 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { bool flag = false; if (loginErrMessages.Count > 0) { foreach (var item in loginErrMessages) { if (this.text.Contains(item)) { flag = true; } } } else { flag = true; } if (flag) { LoginClient loginClient = null; IRawler current = this.Parent; while (current != null) { if (current is LoginClient) { loginClient = current as LoginClient; break; } current = current.Parent; } if (loginClient != null) { // breakFlag = true; loginClient.ReLogin(); var list = this.GetAncestorRawler().Where(n => n is Page); if (list.Count() > 0) { var p = list.First() as Page; p.GetCurrentPage(); } else { ReportManage.ErrReport(this, "Pageオブジェクトが上流に見つかりません。"); } } else { ReportManage.ErrReport(this, "LoginClientオブジェクトが上流に見つかりません。"); } this.RunChildren(runChildren); } }
public override void Run(bool runChildren) { var g = this.GetUpperRawler <GroupBy>(); if (g == null) { ReportManage.ErrUpperNotFound <GroupBy>(this); return; } SetText(g.GetCurrentKeyValue().Key); base.Run(runChildren); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { if (this.GetAncestorRawler().OfType <UseCounter>().Any() == false) { ReportManage.ErrReport(this, "上流にUseCounterがありません"); } else { SetText(this.GetAncestorRawler().OfType <UseCounter>().First().GetCount(Key).ToString()); } base.Run(runChildren); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { if (string.IsNullOrEmpty(Key)) { ReportManage.ErrReport(this, "GetTempVarのKeyが空です。"); } else { this.SetText(GlobalVar.GetVar(Key)); } base.Run(runChildren); }
public static string GetVar(string key) { if (dic.ContainsKey(key)) { return(dic[key]); } else { ReportManage.ErrReport(null, "TempVarクラスのGetVarで「" + key + "」がありませんでした。"); return(string.Empty); } }