public override void Run(bool runChildren)
{
string group = string.Empty;
if (GroupTree != null && this.Parent != null)
{
GroupTree.SetParent();
group = RawlerBase.GetText(this.Parent.Text, GroupTree, this);
}
else
{
if (GroupName != null)
{
group = GroupName;
}
}
int num = 1;
if (AddNumTree != null && this.Parent != null)
{
int.TryParse(RawlerBase.GetText(this.Parent.Text, AddNumTree, this), out num);
}
var c = this.GetAncestorRawler().Where(n => n is CountData);
if (c.Count() > 0)
{
((CountData)c.First()).AddCount(group, GetText(), num);
}
else
{
ReportManage.ErrReport(this, "上流にCountDataがありません");
}
base.Run(runChildren);
}
/// <summary>
/// ElementAt を適応する。
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="list"></param>
/// <returns></returns>
IEnumerable <T> DoElementAt <T>(IEnumerable <T> list)
{
if (list.Count() == 0)
{
return(list);
}
if (ElementAt.HasValue)
{
try
{
var l = list.ToArray();
if (ElementAt.Value > -1)
{
return(new T[] { l[ElementAt.Value] });
//return new List<T>() { list.ElementAt(ElementAt.Value) };
}
else
{
return(new T[] { l[list.Count() + ElementAt.Value] });
// return new List<T>() { list.ElementAt(list.Count() + ElementAt.Value) };
}
}
catch
{
ReportManage.ErrReport(this, "ElementAtの値がレンジから外れました。ElementAt:" + ElementAt.Value);
}
}
return(list);
}
public void PushUrl(string url)
{
var tmp = url.Replace("&#", "&&&&");
var u = tmp.Split('#');
if (u.Length > 0)
{
url = u[0].Replace("&&&&", "&#");
}
if (oncePageLoad)
{
if (urlHash.Add(url))
{
urlStack.Push(url);
}
else
{
if (visbleErr)
{
ReportManage.ErrReport(this, "すでに読み込んだURLです。スルーします。 " + url);
}
}
}
else
{
urlStack.Push(url);
}
}
public override void Run(bool runChildren)
{
if (string.IsNullOrEmpty(ParameterName) == false)
{
Regex regex = new Regex(ParameterName + "[ ]*=[\"| ]*(.+?)[\"| |$]", RegexOptions.IgnoreCase);
Regex regex1 = new Regex(ParameterName + "[ ]*=[\'| ]*(.+?)[\'| |$]", RegexOptions.IgnoreCase);
var r = regex.Match(GetText());
var r2 = regex1.Match(GetText());
if (r.Success)
{
SetText(r.Groups[1].Value);
base.Run(runChildren);
}
else if (r2.Success)
{
SetText(r2.Groups[1].Value);
base.Run(runChildren);
}
else
{
ReportManage.ErrReport(this, "GetTagParameterで指定した" + ParameterName + "が見つかりませんでした");
}
}
else
{
ReportManage.ErrReport(this, "GetTagParameterで指定した" + ParameterName + "が空文字です");
}
}
/// <summary>
/// 指定したキーで上流のKeyValueStoreから値を取得する。
/// </summary>
/// <param name="rawler"></param>
/// <param name="key"></param>
/// <returns></returns>
public static string GetValueByKey(RawlerBase rawler, string key)
{
KeyValueStore[] r = null;
if (ancestorKeyValueStoreDic.ContainsKey(rawler))
{
ancestorKeyValueStoreDic[rawler].TryGetTarget(out r);
}
if (r == null)
{
r = rawler.GetAncestorRawler().OfType <KeyValueStore>().ToArray();
if (ancestorKeyValueStoreDic.ContainsKey(rawler))
{
ancestorKeyValueStoreDic[rawler].SetTarget(r);
}
else
{
ancestorKeyValueStoreDic.Add(rawler, new WeakReference <KeyValueStore[]>(r));
}
}
string val = null;
foreach (var item in r)
{
if (item.dic.ContainsKey(key))
{
val = item.dic[key];
break;
}
}
if (val == null)
{
ReportManage.ErrReport(rawler, "key:" + key + "が見つかりません");
}
return(val);
}
public string GetValue(string columnName)
{
var data = this.Text.Split('\t');
if (columnNameDic.ContainsKey(columnName))
{
if (data.Length >= columnNameDic[columnName])
{
try
{
return(data[columnNameDic[columnName]]);
}
catch (Exception e)
{
ReportManage.ErrReport(this, e.Message + " ColumnName:" + columnName + " Text:" + this.Text);
}
}
else
{
ReportManage.ErrReport(this, "キーが圏外です。:" + columnName);
}
}
else
{
ReportManage.ErrReport(this, "該当するキーがありません:" + columnName);
}
return(string.Empty);
}
private void TsvFileSave()
{
if (doLastFileSave == false)
{
return;
}
string filename = this.FileName.Convert(this);
if (this.FileNameTree != null)
{
filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this);
}
if (string.IsNullOrEmpty(filename) == false)
{
System.IO.StreamWriter sw = null;
if (this.FileSaveMode == Tool.FileSaveMode.Create)
{
sw = System.IO.File.CreateText(filename);
}
else if (this.FileSaveMode == Tool.FileSaveMode.Append)
{
sw = System.IO.File.AppendText(filename);
}
sw.Write(ToTsv());
sw.Close();
ReportManage.Report(this, filename + "作成完了", true, EndReport);
}
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
var paras = GetText().Split('?');
List <string> list = new List <string>();
if (paras.Length > 1)
{
foreach (var item in paras.Last().Split('&'))
{
if (string.IsNullOrEmpty(ParameterName) == false)
{
var d = item.Split('=');
if (d.First() == ParameterName)
{
list.Add(d.Last());
}
}
else
{
list.Add(item);
}
}
}
else
{
ReportManage.ErrReport(this, "UrlParameter:?がありません。パラメータが見つかりませんでした");
}
base.RunChildrenForArray(runChildren, list);
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
System.Text.RegularExpressions.Regex r = new System.Text.RegularExpressions.Regex("<input [^>]*>");
System.Text.RegularExpressions.Regex r2 = new System.Text.RegularExpressions.Regex(@"(\w)*\s*=\s*" + "\"([^\"]*)\"");
var page = (IInputParameter)this.GetUpperInterface <IInputParameter>();
if (page == null)
{
ReportManage.ErrUpperNotFound <IInputParameter>(this);
return;
}
List <KeyValue> list = new List <KeyValue>();
foreach (System.Text.RegularExpressions.Match item in r.Matches(GetText()))
{
var dic = GetParameter(item.Value);
if (dic.ContainsKey("type") && dic["type"] == "hidden")
{
if (dic.ContainsKey("name") && dic.ContainsKey("value"))
{
page.AddParameter(dic["name"], dic["value"]);
list.Add(new KeyValue()
{
Key = dic["name"], Value = dic["value"]
});
}
}
}
base.Run(runChildren);
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
DateTime dt;
if (DateTime.TryParse(GetText(), out dt))
{
if (Format != null)
{
this.SetText(dt.ToString(Format));
}
else
{
this.SetText(dt.ToString());
}
base.Run(runChildren);
}
else
{
if (ErrText != null)
{
SetText(ErrText);
}
else
{
SetText("失敗:" + GetText());
}
ReportManage.ErrReport(this, GetText() + "はDateTime.TryParseに失敗ました。");
}
}
public override void Run(bool runChildren)
{
string filename = FileName.Convert(this);
if (string.IsNullOrEmpty(filename))
{
filename = GetText();
}
if (string.IsNullOrEmpty(filename))
{
ReportManage.ErrEmptyPropertyName(this, nameof(FileName));
return;
}
if (System.IO.File.Exists(filename) == false)
{
ReportManage.ErrReport(this, "File「" + filename + "」は存在しません");
return;
}
try
{
SetText(System.IO.File.ReadAllText(filename));
}
catch (Exception ex)
{
ReportManage.ErrReport(this, filename + "を開くのに失敗しました" + ex.Message);
return;
}
base.Run(runChildren);
}
public override void Run(bool runChildren)
{
List <string> list = new List <string>();
System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(pattern, regexOption);
foreach (System.Text.RegularExpressions.Match match in regex.Matches(GetText()))
{
if (match.Groups.Count > groupNum)
{
list.Add(match.Groups[groupNum].Value);
}
}
if (list.Count == 0)
{
if (emptyReport)
{
ReportManage.ErrReport(this, "対象が見つかりませんでした");
}
}
else
{
this.RunChildrenForArray(runChildren, texts);
}
}
/// <summary>
/// commentを出力する。
/// </summary>
protected void ReportComment()
{
if (!string.IsNullOrEmpty(this.commnet))
{
ReportManage.Report(this, this.Comment);
}
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
if (FileName != null)
{
try
{
string line = GetText();
if (string.IsNullOrEmpty(Line) == false)
{
line = Line.Convert(this);
}
System.IO.File.AppendAllText(FileName.Convert(this), line + "\n");
}
catch (Exception e)
{
ReportManage.ErrReport(this, "ファイルの書き込みに失敗しました。" + e.Message);
}
}
else
{
ReportManage.ErrReport(this, "ファイル名を指定してください");
}
base.Run(runChildren);
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
var page = this.GetAncestorRawler().OfType <Page>().FirstOrDefault();
if (page != null)
{
if (tmpUrl == page.GetCurrentUrl())
{
tmpCount++;
if (tmpCount >= reloadCount)
{
ReportManage.ErrReport(this, "ReloadCount:規定数のリロード回数を超えました。");
OverCountTree.SetParent(this);
OverCountTree.Run();
return;
}
}
else
{
tmpUrl = page.GetCurrentUrl();
tmpCount = 0;
}
page.Reload();
System.Threading.Thread.Sleep((int)(sleepSeconds * 1000));
}
base.Run(runChildren);
}
public void JsonFileSave()
{
string filename = this.FileName.Convert(this);
if (this.FileNameTree != null)
{
filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this);
}
if (string.IsNullOrEmpty(filename) == false)
{
System.IO.StreamWriter sw = null;
if (this.FileSaveMode == Tool.FileSaveMode.Create)
{
sw = System.IO.File.CreateText(filename);
}
else if (this.FileSaveMode == Tool.FileSaveMode.Append)
{
sw = System.IO.File.AppendText(filename);
}
if (saveFileType == FileType.Json)
{
sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.GetDataDicForJson())));
}
sw.Close();
ReportManage.Report(this, filename + "作成完了", true, EndReport);
}
}
protected bool ReadPage(string url)
{
var client = GetWebClient();
parameterList.Clear();
httpHeaderList.Clear();
BeforeTrees.Run(this, GetText());
if (InputParameterTree != null)
{
RawlerBase.GetText(GetText(), InputParameterTree, this);
}
if (MethodType == Tool.MethodType.GET)
{
this.text = client.HttpGet(url, parameterList, httpHeaderList);
}
else if (MethodType == Tool.MethodType.POST)
{
this.text = client.HttpPost(url, parameterList, httpHeaderList);
}
this.currentUrl = url;
this.pastUrl = this.currentUrl;
if (this.Text.Length > 0)
{
return(true);
}
else
{
if (client.ErrMessage != null && (client.ErrMessage.Contains("503") || client.ErrMessage.Contains("500")))
{
ReportManage.ErrReport(this, $"{client.ErrMessage} {url}の読み込みに失敗しました。");
}
else
{
if (visbleErr)
{
ReportManage.ErrReport(this, url + "の読み込みに失敗しました。");
}
if (ErrorEvent != null)
{
ErrorEvent(this, new EventArgs());
}
if (ErrEventTree != null)
{
ErrEventTree.SetParent();
Document d = new Document()
{
TextValue = client.ErrMessage
};
d.SetParent(this);
d.AddChildren(ErrEventTree);
d.Run();
}
}
return(false);
}
}
public override void Run(bool runChildren)
{
var data = (IData)this.GetUpperInterface <IData>();
if (data == null)
{
ReportManage.ErrUpperNotFound <IData>(this);
}
var currentRow = data.GetCurrentDataRow();
var attribute = Attribute.Convert(this);
SetText(GetText());
if (currentRow.DataDic.TryGetValue(attribute, out List <string> list) == false)
{
base.Run(runChildren);
}
else
{
if (list.Any() == false)
{
base.Run(runChildren);
}
}
}
public override void Run(bool runChildren)
{
var d = (IData)this.GetUpperInterface <IData>();
if (d != null)
{
ReportManage.Report(this, d.GetCurrentDataRow().ToString(), true, true);
}
}
public override void Run(bool runChildren)
{
Data data = null;
IRawler current = this.Parent;
while (current != null)
{
if (current is Data)
{
data = current as Data;
break;
}
current = current.Parent;
}
if (data != null)
{
if (data.GetCurrentDataNull())
{
if (ignoreDataNull == false)
{
ReportManage.ErrReport(this, "RowがNullです。Writeが動作していないようです。");
var list = this.GetAncestorRawler().Where(n => n is Page);
if (DoPageReLoad)
{
if (list.Count() > 0)
{
var p = list.First() as Page;
pageCount++;
if (PageReLoadCount < pageCount)
{
ReportManage.Report(this, "再読み込み待機中。");
System.Threading.Thread.Sleep(1000 * pageCount * pageCount);
p.Run();
}
else
{
ReportManage.ErrReport(this, "書き込み先のData クラスが見つかりませんでした。");
}
}
}
}
}
else
{
pageCount = 0;
data.NextDataRow();
ReportManage.Report(this, "NextDataRow");
}
}
else
{
ReportManage.ErrReport(this, "書き込み先のData クラスが見つかりませんでした。");
}
this.RunChildren(runChildren);
}
public override bool Check(string txt)
{
DateTime dt;
if (StartDate == null && EndDate == null)
{
return(false);
}
if (DateTime.TryParse(txt, out dt))
{
if (StartDate == null && EndDate != null)
{
if (dt < EndDate)
{
return(true);
}
else
{
return(false);
}
}
else if (StartDate != null && EndDate != null)
{
if (dt >= StartDate && dt < EndDate)
{
return(true);
}
else
{
return(false);
}
}
else
{
if (dt >= StartDate)
{
return(true);
}
else
{
return(false);
}
}
}
else
{
if (ErrReport)
{
ReportManage.ErrReport(this, "DateTime型のキャストに失敗：" + txt);
}
return(false);
}
// return base.Check(txt);
}
public void Run(RawlerBase parent)
{
try
{
var type = parent.GetType();
var field = type.GetProperty(PropertyName);
var text = RawlerBase.GetText(string.Empty, Child, parent);
if (field.PropertyType == typeof(string))
{
field.SetValue(parent, text, null);
}
else if (field.PropertyType == typeof(int))
{
int num;
if (int.TryParse(text, out num))
{
field.SetValue(parent, num, null);
}
else
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をint型に変換に失敗しました");
}
}
else if (field.PropertyType == typeof(double))
{
double num;
if (double.TryParse(text, out num))
{
field.SetValue(parent, num, null);
}
else
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をdouble型に変換に失敗しました");
}
}
else if (field.PropertyType == typeof(bool))
{
if (text.ToLower() == "true")
{
field.SetValue(parent, true, null);
}
else if (text.ToLower() == "false")
{
field.SetValue(parent, false, null);
}
else
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をbool型に変換に失敗しました。Valueは" + text);
}
}
}
catch (Exception ex)
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "でエラーが発生しました。" + ex.Message);
}
}
public override bool Check(string txt)
{
int dt;
if (Start == null && End == null)
{
return(false);
}
if (int.TryParse(txt, out dt))
{
if (Start == null && End != null)
{
if (dt < End)
{
return(true);
}
else
{
return(false);
}
}
else if (Start != null && End != null)
{
if (dt >= Start && dt < End)
{
return(true);
}
else
{
return(false);
}
}
else
{
if (dt >= Start)
{
return(true);
}
else
{
return(false);
}
}
}
else
{
if (ErrReport)
{
ReportManage.ErrReport(this, "int型のキャストに失敗：" + txt);
}
return(false);
}
// return base.Check(txt);
}
/// <summary>
/// 再びログインする。
/// </summary>
public void ReLogin()
{
string html = this.HttpPost(loginPage, vals, false);
if (this.GetCookieCount() > 0)
{
if (ErrString != null)
{
if (html.Contains(ErrString) == false)
{
hasLogin = true;
ReportManage.Report(this, "ログイン成功");
}
else
{
hasLogin = false;
ReportManage.ErrReport(this, "ログイン失敗");
}
}
else
{
hasLogin = true;
ReportManage.Report(this, "ログイン成功");
}
}
else
{
if (isNoCookieSite)
{
hasLogin = false;
ReportManage.ErrReport(this, "ログイン失敗");
}
else
{
if (ErrString != null)
{
if (html.Contains(ErrString) == false)
{
hasLogin = true;
ReportManage.Report(this, "ログイン成功");
}
else
{
hasLogin = false;
ReportManage.ErrReport(this, "ログイン失敗");
}
}
else
{
hasLogin = true;
ReportManage.Report(this, "ログイン成功");
}
}
}
this.text = html;
}
public override void Run(bool runChildren)
{
var g = this.GetUpperRawler <GroupBy>();
if (g == null)
{
ReportManage.ErrUpperNotFound <GroupBy>(this);
return;
}
RunChildrenForArray(runChildren, g.GetCurrentKeyValue().Value);
}
/// <summary>
/// 実行
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
bool flag = false;
if (loginErrMessages.Count > 0)
{
foreach (var item in loginErrMessages)
{
if (this.text.Contains(item))
{
flag = true;
}
}
}
else
{
flag = true;
}
if (flag)
{
LoginClient loginClient = null;
IRawler current = this.Parent;
while (current != null)
{
if (current is LoginClient)
{
loginClient = current as LoginClient;
break;
}
current = current.Parent;
}
if (loginClient != null)
{
// breakFlag = true;
loginClient.ReLogin();
var list = this.GetAncestorRawler().Where(n => n is Page);
if (list.Count() > 0)
{
var p = list.First() as Page;
p.GetCurrentPage();
}
else
{
ReportManage.ErrReport(this, "Pageオブジェクトが上流に見つかりません。");
}
}
else
{
ReportManage.ErrReport(this, "LoginClientオブジェクトが上流に見つかりません。");
}
this.RunChildren(runChildren);
}
}
public override void Run(bool runChildren)
{
var g = this.GetUpperRawler <GroupBy>();
if (g == null)
{
ReportManage.ErrUpperNotFound <GroupBy>(this);
return;
}
SetText(g.GetCurrentKeyValue().Key);
base.Run(runChildren);
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
if (this.GetAncestorRawler().OfType <UseCounter>().Any() == false)
{
ReportManage.ErrReport(this, "上流にUseCounterがありません");
}
else
{
SetText(this.GetAncestorRawler().OfType <UseCounter>().First().GetCount(Key).ToString());
}
base.Run(runChildren);
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
if (string.IsNullOrEmpty(Key))
{
ReportManage.ErrReport(this, "GetTempVarのKeyが空です。");
}
else
{
this.SetText(GlobalVar.GetVar(Key));
}
base.Run(runChildren);
}
public static string GetVar(string key)
{
if (dic.ContainsKey(key))
{
return(dic[key]);
}
else
{
ReportManage.ErrReport(null, "TempVarクラスのGetVarで「" + key + "」がありませんでした。");
return(string.Empty);
}
}
