/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { var page = this.GetAncestorRawler().OfType <Page>().FirstOrDefault(); if (page != null) { if (tmpUrl == page.GetCurrentUrl()) { tmpCount++; if (tmpCount >= reloadCount) { ReportManage.ErrReport(this, "ReloadCount:規定数のリロード回数を超えました。"); OverCountTree.SetParent(this); OverCountTree.Run(); return; } } else { tmpUrl = page.GetCurrentUrl(); tmpCount = 0; } page.Reload(); Task.Delay((int)(sleepSeconds * 1000)).Wait(); } base.Run(runChildren); }
public string GetValue(string columnName) { var data = this.Text.Split('\t'); if (columnNameDic.ContainsKey(columnName)) { if (data.Length >= columnNameDic[columnName]) { try { return(data[columnNameDic[columnName]]); } catch (Exception e) { ReportManage.ErrReport(this, e.Message + " ColumnName:" + columnName + " Text:" + this.Text); } } else { ReportManage.ErrReport(this, "キーが圏外です。:" + columnName); } } else { ReportManage.ErrReport(this, "該当するキーがありません:" + columnName); } return(string.Empty); }
/// <summary> /// ElementAt を適応する。 /// </summary> /// <typeparam name="T"></typeparam> /// <param name="list"></param> /// <returns></returns> IEnumerable <T> DoElementAt <T>(IEnumerable <T> list) { if (list.Count() == 0) { return(list); } if (ElementAt.HasValue) { try { var l = list.ToArray(); if (ElementAt.Value > -1) { return(new T[] { l[ElementAt.Value] }); //return new List<T>() { list.ElementAt(ElementAt.Value) }; } else { return(new T[] { l[list.Count() + ElementAt.Value] }); // return new List<T>() { list.ElementAt(list.Count() + ElementAt.Value) }; } } catch { ReportManage.ErrReport(this, "ElementAtの値がレンジから外れました。ElementAt:" + ElementAt.Value); } } return(list); }
public override void Run(bool runChildren) { List <string> list = new List <string>(); JToken json; if (this.Parent is GetJsonData) { json = (this.Parent as GetJsonData).GetCurrentJToken(); } else { json = JToken.Parse(GetText()); } try { if (json is JArray) { foreach (var item in json) { list.Add(item.ToString()); } } } catch (Exception ex) { ReportManage.ErrReport(this, "GetJsonAarryに失敗しました。文字列:" + GetText()); } RunChildrenForArray(runChildren, list); }
/// <summary> /// Image用 失敗時NULL /// </summary> /// <param name="url"></param> /// <returns></returns> public virtual async Task <byte[]> HttpGetByte(string url, string referer = default(string)) { await Sleep(); if (ReportUrl) { ReportManage.Report(this, "GET " + url, true, true); } byte[] data; try { System.Net.Http.HttpClient client = new System.Net.Http.HttpClient(handler, false); client.DefaultRequestHeaders.UserAgent.ParseAdd(this.UserAgent); client.DefaultRequestHeaders.Referrer = new Uri(referer); data = await client.GetByteArrayAsync(url); } catch (Exception e) { if (visbleErr) { ReportManage.ErrReport(this, "Url:" + url + " " + e.Message); } data = null; } if (data != null) { return(data); } else { return(null); } }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { DateTime dt; if (DateTime.TryParse(GetText(), out dt)) { if (Format != null) { this.SetText(dt.ToString(Format)); } else { this.SetText(dt.ToString()); } base.Run(runChildren); } else { if (ErrText != null) { SetText(ErrText); } else { SetText("失敗:" + GetText()); } ReportManage.ErrReport(this, GetText() + "はDateTime.TryParseに失敗ました。"); } }
/// <summary> /// HttpPost /// </summary> /// <param name="url"></param> /// <param name="vals"></param> /// <returns></returns> public virtual async Task <byte[]> HttpPostByte(string url, List <KeyValue> vals, string referer = default(string)) { if (ReportUrl) { ReportManage.Report(this, "POST " + url, true, true); } try { await Sleep(); System.Net.Http.HttpClient client = new System.Net.Http.HttpClient(handler, false); client.DefaultRequestHeaders.UserAgent.ParseAdd(this.UserAgent); client.DefaultRequestHeaders.Referrer = new Uri(referer); var r = await client.PostAsync(url, new FormUrlEncodedContent(vals.Select(n => new KeyValuePair <string, string>(n.Key, n.Value)))); var d = await r.Content.ReadAsByteArrayAsync(); return(d); } catch (Exception e) { if (visbleErr) { ReportManage.ErrReport(this, "Url:" + url + " " + e.Message); } ErrMessage = e.Message; } return(null); }
public override void Run(bool runChildren) { if (string.IsNullOrEmpty(ParameterName) == false) { Regex regex = new Regex(ParameterName + "[ ]*=[\"| ]*(.+?)[\"| |$]", RegexOptions.IgnoreCase); Regex regex1 = new Regex(ParameterName + "[ ]*=[\'| ]*(.+?)[\'| |$]", RegexOptions.IgnoreCase); var r = regex.Match(GetText()); var r2 = regex1.Match(GetText()); if (r.Success) { SetText(r.Groups[1].Value); base.Run(runChildren); } else if (r2.Success) { SetText(r2.Groups[1].Value); base.Run(runChildren); } else { ReportManage.ErrReport(this, "GetTagParameterで指定した" + ParameterName + "が見つかりませんでした"); } } else { ReportManage.ErrReport(this, "GetTagParameterで指定した" + ParameterName + "が空文字です"); } }
/// <summary> /// 指定したキーで上流のKeyValueStoreから値を取得する。 /// </summary> /// <param name="rawler"></param> /// <param name="key"></param> /// <returns></returns> public static string GetValueByKey(RawlerBase rawler, string key) { KeyValueStore[] r = null; if (ancestorKeyValueStoreDic.ContainsKey(rawler)) { ancestorKeyValueStoreDic[rawler].TryGetTarget(out r); } if (r == null) { r = rawler.GetAncestorRawler().OfType <KeyValueStore>().ToArray(); if (ancestorKeyValueStoreDic.ContainsKey(rawler)) { ancestorKeyValueStoreDic[rawler].SetTarget(r); } else { ancestorKeyValueStoreDic.Add(rawler, new WeakReference <KeyValueStore[]>(r)); } } string val = null; foreach (var item in r) { if (item.dic.ContainsKey(key)) { val = item.dic[key]; break; } } if (val == null) { ReportManage.ErrReport(rawler, "key:" + key + "が見つかりません"); } return(val); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { if (FileName != null) { try { string line = GetText(); if (string.IsNullOrEmpty(Line) == false) { line = Line.Convert(this); } System.IO.File.AppendAllText(FileName.Convert(this), line + "\n"); } catch (Exception e) { ReportManage.ErrReport(this, "ファイルの書き込みに失敗しました。" + e.Message); } } else { ReportManage.ErrReport(this, "ファイル名を指定してください"); } base.Run(runChildren); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { System.Text.RegularExpressions.Regex r = new System.Text.RegularExpressions.Regex("<input [^>]*>"); System.Text.RegularExpressions.Regex r2 = new System.Text.RegularExpressions.Regex(@"(\w)*\s*=\s*" + "\"([^\"]*)\""); var p = this.GetAncestorRawler().OfType <Page>(); if (p.Any() == false) { ReportManage.ErrReport(this, "上流にPageがありません"); return; } var page = p.First(); List <KeyValue> list = new List <KeyValue>(); foreach (System.Text.RegularExpressions.Match item in r.Matches(GetText())) { var dic = GetParameter(item.Value); if (dic.ContainsKey("type") && dic["type"] == "hidden") { if (dic.ContainsKey("name") && dic.ContainsKey("value")) { page.AddParameter(dic["name"], dic["value"]); list.Add(new KeyValue() { Key = dic["name"], Value = dic["value"] }); } } } base.Run(runChildren); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { string filename = this.GetFileName(); if (string.IsNullOrEmpty(filename)) { ReportManage.ErrEmptyPropertyName(this, nameof(FileName)); return; } try { var r = ReadLine(); r.Wait(); int i = 0; var topline = r.Result.First(); columnNameDic = new Dictionary <string, int>(); foreach (var item in topline.Split('\t')) { columnNameDic.Add(item, i); i++; } var lines = r.Result.Skip(1); if (skip > 0) { lines = lines.Skip(skip); } base.RunChildrenForArray(runChildren, lines); } catch (Exception ex) { ReportManage.ErrReport(this, FileName + "を開くのに失敗しました" + ex.Message); } }
public override void Run(bool runChildren) { string filename = FileName.Convert(this); if (string.IsNullOrEmpty(filename)) { filename = GetText(); } if (string.IsNullOrEmpty(filename)) { ReportManage.ErrEmptyPropertyName(this, nameof(FileName)); return; } try { var t = Read().WaitResult(); SetText(t); } catch (Exception ex) { ReportManage.ErrReport(this, filename + "を開くのに失敗しました" + ex.Message); return; } base.Run(runChildren); }
public override void Run(bool runChildren) { var d = (IData)this.GetUpperInterface <IData>(); if (d != null) { ReportManage.Report(this, d.GetCurrentDataRow().ToString(), true, true); } }
public override void Run(bool runChildren) { Data data = null; IRawler current = this.Parent; while (current != null) { if (current is Data) { data = current as Data; break; } current = current.Parent; } if (data != null) { if (data.GetCurrentDataNull()) { if (ignoreDataNull == false) { ReportManage.ErrReport(this, "RowがNullです。Writeが動作していないようです。"); var list = this.GetAncestorRawler().Where(n => n is Page); if (DoPageReLoad) { if (list.Count() > 0) { var p = list.First() as Page; pageCount++; if (PageReLoadCount < pageCount) { ReportManage.Report(this, "再読み込み待機中。"); var t = Task.Delay(1000 * pageCount * pageCount); t.Wait(); p.Run(); } else { ReportManage.ErrReport(this, "書き込み先のData クラスが見つかりませんでした。"); } } } } } else { pageCount = 0; data.NextDataRow(); ReportManage.Report(this, "NextDataRow"); } } else { ReportManage.ErrReport(this, "書き込み先のData クラスが見つかりませんでした。"); } this.RunChildren(runChildren); }
/// <summary> /// 再びログインする。 /// </summary> public void ReLogin() { string html = this.HttpPost(loginPage, vals, false); if (this.GetCookieCount() > 0) { if (ErrString != null) { if (html.Contains(ErrString) == false) { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } else { hasLogin = false; ReportManage.ErrReport(this, "ログイン失敗"); } } else { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } } else { if (isNoCookieSite) { hasLogin = false; ReportManage.ErrReport(this, "ログイン失敗"); } else { if (ErrString != null) { if (html.Contains(ErrString) == false) { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } else { hasLogin = false; ReportManage.ErrReport(this, "ログイン失敗"); } } else { hasLogin = true; ReportManage.Report(this, "ログイン成功"); } } } this.text = html; }
/// <summary> /// 実行 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { bool flag = false; if (loginErrMessages.Count > 0) { foreach (var item in loginErrMessages) { if (this.text.Contains(item)) { flag = true; } } } else { flag = true; } if (flag) { LoginClient loginClient = null; IRawler current = this.Parent; while (current != null) { if (current is LoginClient) { loginClient = current as LoginClient; break; } current = current.Parent; } if (loginClient != null) { // breakFlag = true; loginClient.ReLogin(); var list = this.GetAncestorRawler().Where(n => n is Page); if (list.Count() > 0) { var p = list.First() as Page; p.GetCurrentPage(); } else { ReportManage.ErrReport(this, "Pageオブジェクトが上流に見つかりません。"); } } else { ReportManage.ErrReport(this, "LoginClientオブジェクトが上流に見つかりません。"); } this.RunChildren(runChildren); } }
public override void Run(bool runChildren) { var g = this.GetUpperRawler <GroupBy>(); if (g == null) { ReportManage.ErrUpperNotFound <GroupBy>(this); return; } RunChildrenForArray(runChildren, g.GetCurrentKeyValue().Value); }
public static string GetVar(string key) { if (dic.ContainsKey(key)) { return(dic[key]); } else { ReportManage.ErrReport(null, "TempVarクラスのGetVarで「" + key + "」がありませんでした。"); return(string.Empty); } }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { if (string.IsNullOrEmpty(Key)) { ReportManage.ErrReport(this, "GetTempVarのKeyが空です。"); } else { this.SetText(TempVar.GetVar(Key)); } base.Run(runChildren); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { if (this.GetAncestorRawler().OfType <UseCounter>().Any() == false) { ReportManage.ErrReport(this, "上流にUseCounterがありません"); } else { SetText(this.GetAncestorRawler().OfType <UseCounter>().First().GetCount(Key).ToString()); } base.Run(runChildren); }
/// <summary> /// このクラスでの実行すること。 /// </summary> /// <param name="runChildren"></param> public override void Run(bool runChildren) { if (this.GetAncestorRawler().OfType <UseCounter>().Any() == false) { ReportManage.ErrReport(this, "上流にUseCounterがありません"); } else { this.GetAncestorRawler().OfType <UseCounter>().First().Add(this.Key, Count); } base.Run(runChildren); }
public int GetCountLines() { try { return(System.IO.File.ReadLines(FileName).Count()); } catch { ReportManage.ErrReport(this, "ファイルが存在しません"); } return(-1); }
public string GetKeyValue(string key) { if (dic.ContainsKey(key)) { return(dic[key]); } else { ReportManage.ErrReport(this, "Key(" + key + ")が見つかりません"); return(string.Empty); } }
public override IEnumerable <string> Query(IEnumerable <string> list) { if (Num >= 0) { return(list.Skip(Num)); } else { ReportManage.ErrReport(new RawlerBase(), "QuerySkipのNumの値がありません。"); return(list); } }
public override void Run(bool runChildren) { var g = this.GetUpperRawler <GroupBy>(); if (g == null) { ReportManage.ErrUpperNotFound <GroupBy>(this); return; } SetText(g.GetCurrentKeyValue().Key); base.Run(runChildren); }
public override void Run(bool runChildren) { try { RawlerBase rawler = (RawlerBase)XamlServices.Load(FileName.Convert(this)); rawler.SetParent(this); rawler.Run(); } catch (Exception ex) { ReportManage.ErrReport(this, ex.ToString()); } }
public override void Run(bool runChildren) { var d = this.GetUpperRawler <Data>(); if (d == null) { ReportManage.ErrUpperNotFound <Data>(this); } else { } base.Run(runChildren); }
/// <summary> /// AddDataRowの共通呼び出し。 /// </summary> /// <param name="rawler"></param> /// <param name="datarow"></param> public static void AddDataRow(RawlerBase rawler, DataRowObject datarow) { var r = (IDataRows)rawler.GetUpperInterface <IDataRows>(); if (r != null) { r.AddDataRow(datarow); } else { ReportManage.ErrUpperNotFound <IDataRows>(rawler); } }
/// <summary> /// DataWriteの共通呼び出し /// </summary> /// <param name="rawler"></param> /// <param name="attribute"></param> /// <param name="value"></param> /// <param name="type"></param> /// <param name="attributeType"></param> public static void DataWrite(RawlerBase rawler, string attribute, string value, DataWriteType type, DataAttributeType attributeType) { var d = (IData)rawler.GetUpperInterface <IData>(); if (d != null) { d.DataWrite(attribute, value, type, attributeType); } else { ReportManage.ErrUpperNotFound <IData>(rawler); } }