protected bool ReadPage(string url)
{
var client = GetWebClient();
parameterList.Clear();
httpHeaderList.Clear();
BeforeTrees.Run(this, GetText());
if (InputParameterTree != null)
{
RawlerBase.GetText(GetText(), InputParameterTree, this);
}
if (MethodType == Tool.MethodType.GET)
{
this.text = client.HttpGet(url, parameterList, httpHeaderList);
}
else if (MethodType == Tool.MethodType.POST)
{
this.text = client.HttpPost(url, parameterList, httpHeaderList);
}
this.currentUrl = url;
this.pastUrl = this.currentUrl;
if (this.Text.Length > 0)
{
return(true);
}
else
{
if (client.ErrMessage != null && (client.ErrMessage.Contains("503") || client.ErrMessage.Contains("500")))
{
ReportManage.ErrReport(this, $"{client.ErrMessage} {url}の読み込みに失敗しました。");
}
else
{
if (visbleErr)
{
ReportManage.ErrReport(this, url + "の読み込みに失敗しました。");
}
if (ErrorEvent != null)
{
ErrorEvent(this, new EventArgs());
}
if (ErrEventTree != null)
{
ErrEventTree.SetParent();
Document d = new Document()
{
TextValue = client.ErrMessage
};
d.SetParent(this);
d.AddChildren(ErrEventTree);
d.Run();
}
}
return(false);
}
}
public RawlerBase GetUpperInterface <T>()
{
bool flag = false;
RawlerBase rawler = this.Parent;
while (true)
{
if (rawler == null)
{
break;
}
if (rawler is T)
{
flag = true;
break;
}
else if (rawler.Parent == null)
{
break;
}
else
{
rawler = rawler.Parent;
}
}
if (flag)
{
return(rawler);
}
else
{
return(null);
}
}
private void TsvFileSave()
{
if (doLastFileSave == false)
{
return;
}
string filename = this.FileName.Convert(this);
if (this.FileNameTree != null)
{
filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this);
}
if (string.IsNullOrEmpty(filename) == false)
{
System.IO.StreamWriter sw = null;
if (this.FileSaveMode == Tool.FileSaveMode.Create)
{
sw = System.IO.File.CreateText(filename);
}
else if (this.FileSaveMode == Tool.FileSaveMode.Append)
{
sw = System.IO.File.AppendText(filename);
}
sw.Write(ToTsv());
sw.Close();
ReportManage.Report(this, filename + "作成完了", true, EndReport);
}
}
/// <summary>
/// 指定したキーで上流のKeyValueStoreから値を取得する。
/// </summary>
/// <param name="rawler"></param>
/// <param name="key"></param>
/// <returns></returns>
public static string GetValueByKey(RawlerBase rawler, string key)
{
KeyValueStore[] r = null;
if (ancestorKeyValueStoreDic.ContainsKey(rawler))
{
ancestorKeyValueStoreDic[rawler].TryGetTarget(out r);
}
if (r == null)
{
r = rawler.GetAncestorRawler().OfType <KeyValueStore>().ToArray();
if (ancestorKeyValueStoreDic.ContainsKey(rawler))
{
ancestorKeyValueStoreDic[rawler].SetTarget(r);
}
else
{
ancestorKeyValueStoreDic.Add(rawler, new WeakReference <KeyValueStore[]>(r));
}
}
string val = null;
foreach (var item in r)
{
if (item.dic.ContainsKey(key))
{
val = item.dic[key];
break;
}
}
if (val == null)
{
ReportManage.ErrReport(rawler, "key:" + key + "が見つかりません");
}
return(val);
}
public void JsonFileSave()
{
string filename = this.FileName.Convert(this);
if (this.FileNameTree != null)
{
filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this);
}
if (string.IsNullOrEmpty(filename) == false)
{
System.IO.StreamWriter sw = null;
if (this.FileSaveMode == Tool.FileSaveMode.Create)
{
sw = System.IO.File.CreateText(filename);
}
else if (this.FileSaveMode == Tool.FileSaveMode.Append)
{
sw = System.IO.File.AppendText(filename);
}
if (saveFileType == FileType.Json)
{
sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.GetDataDicForJson())));
}
sw.Close();
ReportManage.Report(this, filename + "作成完了", true, EndReport);
}
}
//public new void Run()
//{
// Run(true);
//}
/// <summary>
/// 実行
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
string containsText = this.ContainsText.Convert(this);
string text = this.GetText();
if (ContainsTextTree != null)
{
containsText = RawlerBase.GetText(this.Parent.Text, ContainsTextTree, this);
}
if (TextTree != null)
{
text = RawlerBase.GetText(this.Parent.Text, TextTree, this);
}
if (text.Contains(containsText) == this.Result)
{
this.RunChildren(runChildren);
}
//if (ContainsTextTree == null)
//{
// if (this.GetText().Contains(this.ContainsText) == this.Result)
// {
// this.RunChildren(runChildren);
// }
//}
//else
//{
// string t = RawlerBase.GetText(this.Parent.Text, ContainsTextTree, this);
// if (this.GetText().Contains(t) == this.Result)
// {
// this.RunChildren(runChildren);
// }
//}
}
public override void Run(bool runChildren)
{
string t = string.Empty;
if (string.IsNullOrEmpty(Value))
{
t = GetText();
}
else
{
t = Value;
}
if (Header != null)
{
t = Header + t;
}
if (HeaderTree != null)
{
t = RawlerBase.GetText(this.Parent.Text, HeaderTree, this.Parent) + t;
}
if (Footer != null)
{
t = t + Footer;
}
if (FooterTree != null)
{
t = t + RawlerBase.GetText(this.Parent.Text, FooterTree, this.Parent);
}
SetText(t);
base.Run(runChildren);
}
public override void Run(bool runChildren)
{
string group = string.Empty;
if (GroupTree != null && this.Parent != null)
{
GroupTree.SetParent();
group = RawlerBase.GetText(this.Parent.Text, GroupTree, this);
}
else
{
if (GroupName != null)
{
group = GroupName;
}
}
int num = 1;
if (AddNumTree != null && this.Parent != null)
{
int.TryParse(RawlerBase.GetText(this.Parent.Text, AddNumTree, this), out num);
}
var c = this.GetAncestorRawler().Where(n => n is CountData);
if (c.Count() > 0)
{
((CountData)c.First()).AddCount(group, GetText(), num);
}
else
{
ReportManage.ErrReport(this, "上流にCountDataがありません");
}
base.Run(runChildren);
}
/// <summary>
/// クローンを作る
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="parent"></param>
/// <returns></returns>
public virtual T Clone <T>(RawlerBase parent)
where T : RawlerBase, new()
{
T clone = new T();
RawlerLib.ObjectLib.FildCopy(this as T, clone);
clone.SetParent(parent);
CloneEvent(clone);
clone.Children.Clear();
foreach (var item in clone.GetType().GetProperties())
{
if (item.PropertyType.IsSubclassOf(typeof(RawlerBase)))
{
var r = item.GetValue(this as T, null) as RawlerBase;
item.SetValue(clone, r.Clone(), null);
}
//else if(item.PropertyType == typeof(RawlerCollection))
//{
// var col = item.GetValue(clone, null) as RawlerCollection;
// col.Clear();
// foreach (var item2 in item.GetValue(this as T,null) as RawlerCollection)
// {
// col.Add(item2);
// }
//}
}
foreach (var item in this.Children)
{
var child = item.Clone(clone);
clone.AddChildren(child);
}
return(clone);
}
/// <summary>
/// 直近の上流に指定の型があったら取得する。
/// </summary>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public T GetUpperRawler <T>()
where T : RawlerBase
{
T result = null;
RawlerBase rawler = this.Parent;
if (rawler != null)
{
while (true)
{
if (rawler is T)
{
result = rawler as T;
break;
}
else if (rawler.Parent == null)
{
break;
}
else
{
rawler = rawler.Parent;
}
}
}
return(result);
}
public override void Run(bool runChildren)
{
parameterList.Clear();
if (BeforeTrees.Any())
{
foreach (var item in BeforeTrees)
{
RawlerBase.GetText(GetText(), item, this);
}
}
string url;
if (string.IsNullOrEmpty(Url))
{
url = GetText();
}
else
{
url = Url;
}
url = url + "?" + parameterList.Select(n => $"{Uri.EscapeUriString( n.Key)}={Uri.EscapeUriString(n.Value)}").JoinText("&");
SetText(url);
base.Run(runChildren);
}
/// <summary>
/// {key} という記法で上流のKeyValueの値を取得する。
/// </summary>
/// <param name="text"></param>
/// <param name="rawler"></param>
/// <returns></returns>
public static string Convert(this string text, RawlerBase rawler)
{
if (text == null)
{
return(null);
}
else
{
return(GetConvertStr(text, rawler));
// var text2 = text.Replace("[[", "&(").Replace("]]", "&)");
//// System.Text.RegularExpressions.Regex r = new System.Text.RegularExpressions.Regex(@"\[\w*\]");
// foreach (var item in r.Matches(text).OfType<System.Text.RegularExpressions.Match>())
// {
// var key = item.Value.Replace("[", "").Replace("]", "");
// var key1 = HeaderConvert(key, rawler);
// if(key1.Length==0)
// {
// var val = KeyValueStore.GetValueByKey(rawler, key);
// if (val.Length > 0)
// {
// text2 = text2.Replace(item.Value, val);
// }
// }
// else
// {
// text2 = text2.Replace(item.Value, key1);
// }
// }
// return text2.Replace("&(", "[").Replace("&)", "]");
}
}
private static void AddReportEventArgs(RawlerBase sender, ReportEvnetArgs args)
{
if (sender != null)
{
var reportStock = sender.GetUpperRawler <ReportStock>();
if (reportStock != null)
{
reportStock.AddReport(args);
return;
}
}
if (args.IsErr)
{
if (stockErrReport)
{
reportList.Add(args);
}
}
else
{
if (stockReport)
{
reportList.Add(args);
}
}
}
/// <summary>
/// Dispose()
/// </summary>
public virtual void Dispose()
{
this.text = null;
this.parent = null;
this.children = null;
this.commnet = null;
this.preTree = null;
}
protected string GetFileName()
{
if (FileNameTree != null)
{
return(RawlerBase.GetText(this.GetText(), FileNameTree, this));
}
return(this.FileName.Convert(this));
}
static string GetConvertStr(string text, RawlerBase rawler)
{
if (keyDic.ContainsKey(text))
{
return(GetBlockText(keyDic[text], rawler));
}
var text2 = text.Replace("[[", "&(").Replace("]]", "&)");
// System.Text.RegularExpressions.Regex r = new System.Text.RegularExpressions.Regex(@"\[\w*\]");
int postion = 0;
List <StrBlock> blockList = new List <StrBlock>();
foreach (var item in r.Matches(text2).OfType <System.Text.RegularExpressions.Match>())
{
var key = item.Value.Replace("[", "").Replace("]", "");
if (item.Index != postion)
{
blockList.Add(new StrBlock()
{
Text = text2.Substring(postion, item.Index - postion).Replace("&(", "[").Replace("&)", "]")
});
}
postion = item.Index + item.Value.Length;
var d = key.Split(':');
if (d.Length == 1)
{
blockList.Add(new StrBlock_Key()
{
Key = key
});
}
else
{
blockList.Add(new StrBlock_Key2()
{
Key = d.Last(), Header = d.First()
});
}
}
if (postion < text2.Length)
{
blockList.Add(new StrBlock()
{
Text = text2.Substring(postion, text2.Length - postion).Replace("&(", "[").Replace("&)", "]")
});
}
if (keyDic.ContainsKey(text))
{
keyDic[text] = blockList.ToArray();
}
else
{
keyDic.Add(text, blockList.ToArray());
}
return(GetBlockText(keyDic[text], rawler));
}
protected override void CloneEvent(RawlerBase rawler)
{
base.CloneEvent(rawler);
if (rawler is CustomText)
{
var r = (CustomText)rawler;
r.CreateText = this.CreateText;
}
}
protected override void CloneEvent(RawlerBase rawler)
{
base.CloneEvent(rawler);
if (rawler is CustomFilter)
{
var r = (CustomFilter)rawler;
r.Check = this.Check;
}
}
public void Run(RawlerBase parent)
{
try
{
var type = parent.GetType();
var field = type.GetProperty(PropertyName);
var text = RawlerBase.GetText(string.Empty, Child, parent);
if (field.PropertyType == typeof(string))
{
field.SetValue(parent, text, null);
}
else if (field.PropertyType == typeof(int))
{
int num;
if (int.TryParse(text, out num))
{
field.SetValue(parent, num, null);
}
else
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をint型に変換に失敗しました");
}
}
else if (field.PropertyType == typeof(double))
{
double num;
if (double.TryParse(text, out num))
{
field.SetValue(parent, num, null);
}
else
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をdouble型に変換に失敗しました");
}
}
else if (field.PropertyType == typeof(bool))
{
if (text.ToLower() == "true")
{
field.SetValue(parent, true, null);
}
else if (text.ToLower() == "false")
{
field.SetValue(parent, false, null);
}
else
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をbool型に変換に失敗しました。Valueは" + text);
}
}
}
catch (Exception ex)
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "でエラーが発生しました。" + ex.Message);
}
}
public ReportEvnetArgs(RawlerBase sender, string message, bool returncode, int code = 0)
: base()
{
this.Message = message;
this.IsErr = false;
this.DateTime = DateTime.Now;
this.Visible = false;
this.ReturnCode = returncode;
this.Code = code;
}
public override T Clone <T>(RawlerBase parent)
{
var obj = base.Clone <T>(parent);
if (obj is RawlerMultiBase)
{
(obj as RawlerMultiBase).Query = this.Query.Clone();
}
return(obj);
}
/// <summary>
/// 指定したテキストをPreTreeにかける。
/// </summary>
/// <param name="text"></param>
/// <returns></returns>
protected string GetText(string preText)
{
if (preTree != null)
{
return(RawlerBase.GetText(preText, PreTree));
}
else
{
return(preText);
}
}
public static string GetText(string text, RawlerBase rawler)
{
Document doc = new Document();
doc.SetText(text);
rawler.SetParent(doc);
var last = rawler.GetDescendantRawler().Last();
rawler.Run();
return(last.Text);
}
/// <summary>
/// 初期化実行
/// </summary>
/// <param name="root"></param>
public void Run(RawlerBase root)
{
if (this.Count == 0)
{
return;
}
foreach (var item in this)
{
item.Run(root);
}
}
private static string GetTopComment(RawlerBase sender)
{
if (visbleTopObjectComment)
{
return(sender.GetAncestorRawler().Last().Comment + ":");
}
else
{
return(string.Empty);
}
}
public ReportEvnetArgs(RawlerBase sender, string message, bool returncode, bool isErr, int code = 0)
: base()
{
// this.Sender = sender;
this.Message = message;
this.IsErr = isErr;
this.DateTime = DateTime.Now;
this.Visible = true;
this.ReturnCode = returncode;
this.Code = code;
}
public override IEnumerable<string> Query(IEnumerable<string> list)
{
if (KeyTree == null)
{
return list.OrderBy(n => n);
}
else
{
return list.OrderBy(n => RawlerBase.GetText(n, KeyTree));
}
}
public void addDocument(string text, RawlerBase rawlerTree)
{
var d = new Document();
d.SetText(text);
d.SetParent(this);
var tree = rawlerTree.Clone();
d.AddChildren(tree);
documents.Add(d);
}
protected string GetSwitchValue()
{
if (SwitchValueTree != null)
{
SwitchValueTree.SetParent();
return(RawlerBase.GetText(this.Parent.Text, SwitchValueTree, this.Parent));
}
else
{
return(GetText());
}
}
public override void Run(bool runChildren)
{
try
{
RawlerBase rawler = (RawlerBase)System.Xaml.XamlServices.Load(FileName);
rawler.SetParent(this);
rawler.Run();
}
catch (Exception ex)
{
ReportManage.ErrReport(this, ex.ToString());
}
}
