public override void Run(bool runChildren)
{
string t = string.Empty;
if (string.IsNullOrEmpty(Value))
{
t = GetText();
}
else
{
t = Value;
}
if (Header != null)
{
t = Header + t;
}
if (HeaderTree != null)
{
t = RawlerBase.GetText(this.Parent.Text, HeaderTree, this.Parent) + t;
}
if (Footer != null)
{
t = t + Footer;
}
if (FooterTree != null)
{
t = t + RawlerBase.GetText(this.Parent.Text, FooterTree, this.Parent);
}
SetText(t);
base.Run(runChildren);
}
private void TsvFileSave()
{
if (doLastFileSave == false)
{
return;
}
string filename = this.FileName.Convert(this);
if (this.FileNameTree != null)
{
filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this);
}
if (string.IsNullOrEmpty(filename) == false)
{
System.IO.StreamWriter sw = null;
if (this.FileSaveMode == Tool.FileSaveMode.Create)
{
sw = System.IO.File.CreateText(filename);
}
else if (this.FileSaveMode == Tool.FileSaveMode.Append)
{
sw = System.IO.File.AppendText(filename);
}
sw.Write(ToTsv());
sw.Close();
ReportManage.Report(this, filename + "作成完了", true, EndReport);
}
}
protected bool ReadPage(string url)
{
var client = GetWebClient();
parameterList.Clear();
httpHeaderList.Clear();
BeforeTrees.Run(this, GetText());
if (InputParameterTree != null)
{
RawlerBase.GetText(GetText(), InputParameterTree, this);
}
if (MethodType == Tool.MethodType.GET)
{
this.text = client.HttpGet(url, parameterList, httpHeaderList);
}
else if (MethodType == Tool.MethodType.POST)
{
this.text = client.HttpPost(url, parameterList, httpHeaderList);
}
this.currentUrl = url;
this.pastUrl = this.currentUrl;
if (this.Text.Length > 0)
{
return(true);
}
else
{
if (client.ErrMessage != null && (client.ErrMessage.Contains("503") || client.ErrMessage.Contains("500")))
{
ReportManage.ErrReport(this, $"{client.ErrMessage} {url}の読み込みに失敗しました。");
}
else
{
if (visbleErr)
{
ReportManage.ErrReport(this, url + "の読み込みに失敗しました。");
}
if (ErrorEvent != null)
{
ErrorEvent(this, new EventArgs());
}
if (ErrEventTree != null)
{
ErrEventTree.SetParent();
Document d = new Document()
{
TextValue = client.ErrMessage
};
d.SetParent(this);
d.AddChildren(ErrEventTree);
d.Run();
}
}
return(false);
}
}
public override void Run(bool runChildren)
{
parameterList.Clear();
if (BeforeTrees.Any())
{
foreach (var item in BeforeTrees)
{
RawlerBase.GetText(GetText(), item, this);
}
}
string url;
if (string.IsNullOrEmpty(Url))
{
url = GetText();
}
else
{
url = Url;
}
url = url + "?" + parameterList.Select(n => $"{Uri.EscapeUriString( n.Key)}={Uri.EscapeUriString(n.Value)}").JoinText("&");
SetText(url);
base.Run(runChildren);
}
//public new void Run()
//{
// Run(true);
//}
/// <summary>
/// 実行
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
string containsText = this.ContainsText.Convert(this);
string text = this.GetText();
if (ContainsTextTree != null)
{
containsText = RawlerBase.GetText(this.Parent.Text, ContainsTextTree, this);
}
if (TextTree != null)
{
text = RawlerBase.GetText(this.Parent.Text, TextTree, this);
}
if (text.Contains(containsText) == this.Result)
{
this.RunChildren(runChildren);
}
//if (ContainsTextTree == null)
//{
// if (this.GetText().Contains(this.ContainsText) == this.Result)
// {
// this.RunChildren(runChildren);
// }
//}
//else
//{
// string t = RawlerBase.GetText(this.Parent.Text, ContainsTextTree, this);
// if (this.GetText().Contains(t) == this.Result)
// {
// this.RunChildren(runChildren);
// }
//}
}
public void JsonFileSave()
{
string filename = this.FileName.Convert(this);
if (this.FileNameTree != null)
{
filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this);
}
if (string.IsNullOrEmpty(filename) == false)
{
System.IO.StreamWriter sw = null;
if (this.FileSaveMode == Tool.FileSaveMode.Create)
{
sw = System.IO.File.CreateText(filename);
}
else if (this.FileSaveMode == Tool.FileSaveMode.Append)
{
sw = System.IO.File.AppendText(filename);
}
if (saveFileType == FileType.Json)
{
sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.GetDataDicForJson())));
}
sw.Close();
ReportManage.Report(this, filename + "作成完了", true, EndReport);
}
}
public override void Run(bool runChildren)
{
string group = string.Empty;
if (GroupTree != null && this.Parent != null)
{
GroupTree.SetParent();
group = RawlerBase.GetText(this.Parent.Text, GroupTree, this);
}
else
{
if (GroupName != null)
{
group = GroupName;
}
}
int num = 1;
if (AddNumTree != null && this.Parent != null)
{
int.TryParse(RawlerBase.GetText(this.Parent.Text, AddNumTree, this), out num);
}
var c = this.GetAncestorRawler().Where(n => n is CountData);
if (c.Count() > 0)
{
((CountData)c.First()).AddCount(group, GetText(), num);
}
else
{
ReportManage.ErrReport(this, "上流にCountDataがありません");
}
base.Run(runChildren);
}
protected string GetFileName()
{
if (FileNameTree != null)
{
return(RawlerBase.GetText(this.GetText(), FileNameTree, this));
}
return(this.FileName.Convert(this));
}
public void Run(RawlerBase parent)
{
try
{
var type = parent.GetType();
var field = type.GetProperty(PropertyName);
var text = RawlerBase.GetText(string.Empty, Child, parent);
if (field.PropertyType == typeof(string))
{
field.SetValue(parent, text, null);
}
else if (field.PropertyType == typeof(int))
{
int num;
if (int.TryParse(text, out num))
{
field.SetValue(parent, num, null);
}
else
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をint型に変換に失敗しました");
}
}
else if (field.PropertyType == typeof(double))
{
double num;
if (double.TryParse(text, out num))
{
field.SetValue(parent, num, null);
}
else
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をdouble型に変換に失敗しました");
}
}
else if (field.PropertyType == typeof(bool))
{
if (text.ToLower() == "true")
{
field.SetValue(parent, true, null);
}
else if (text.ToLower() == "false")
{
field.SetValue(parent, false, null);
}
else
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "の値をbool型に変換に失敗しました。Valueは" + text);
}
}
}
catch (Exception ex)
{
ReportManage.ErrReport(parent, "InitTreeで" + PropertyName + "でエラーが発生しました。" + ex.Message);
}
}
/// <summary>
/// 指定したテキストをPreTreeにかける。
/// </summary>
/// <param name="text"></param>
/// <returns></returns>
protected string GetText(string preText)
{
if (preTree != null)
{
return(RawlerBase.GetText(preText, PreTree));
}
else
{
return(preText);
}
}
public override IEnumerable<string> Query(IEnumerable<string> list)
{
if (KeyTree == null)
{
return list.OrderBy(n => n);
}
else
{
return list.OrderBy(n => RawlerBase.GetText(n, KeyTree));
}
}
protected string GetSwitchValue()
{
if (SwitchValueTree != null)
{
SwitchValueTree.SetParent();
return(RawlerBase.GetText(this.Parent.Text, SwitchValueTree, this.Parent));
}
else
{
return(GetText());
}
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
var page = (IInputParameter)this.GetUpperInterface <IInputParameter>();
if (page == null)
{
ReportManage.ErrUpperNotFound <IInputParameter>(this);
return;
}
string key = this.Key;
if (KeyTree != null)
{
key = RawlerBase.GetText(this.Parent.Text, KeyTree, this.Parent);
}
else
{
key = key.Convert(this);
}
if (Value == null)
{
if (AddType == AddInputParameterType.replece)
{
page.ReplaceParameter(key, GetText());
}
else
{
page.AddParameter(key, GetText());
}
}
else
{
if (AddType == AddInputParameterType.replece)
{
page.ReplaceParameter(key, Value.Convert(this));
}
else
{
page.AddParameter(key, Value.Convert(this));
}
}
base.Run(runChildren);
}
private IEnumerable <string> Convert(IEnumerable <string> list)
{
if (ConvertTree != null)
{
ConvertTree.SetParent(this.Parent);
foreach (var item in list)
{
yield return(RawlerBase.GetText(item, ConvertTree, this.Parent));
}
}
else
{
foreach (var item in list)
{
yield return(item);
}
}
}
public override IEnumerable<string> Query(IEnumerable<string> list)
{
var root = this.GetRoot().Rawler;
if(SelectTree !=null)
{
foreach (var item in list)
{
yield return RawlerBase.GetText(item, SelectTree, root);
}
}
else
{
ReportManage.ErrReport(root, "SelectTreeがありません");
// yield return base.Query(list);
}
}
public override void Run(bool runChildren)
{
string t = string.Empty;
if (string.IsNullOrEmpty(Value))
{
t = GetText();
}
else
{
t = Value;
}
try
{
var dir = System.IO.Path.GetDirectoryName(t) + "\\";
var file = System.IO.Path.GetFileNameWithoutExtension(t);
var ext = System.IO.Path.GetExtension(t);
t = file;
if (Header != null)
{
t = Header + t;
}
if (HeaderTree != null)
{
t = RawlerBase.GetText(this.Parent.Text, HeaderTree, this.Parent) + t;
}
if (Footer != null)
{
t = t + Footer;
}
if (FooterTree != null)
{
t = t + RawlerBase.GetText(this.Parent.Text, FooterTree, this.Parent);
}
t = dir + t + ext;
}
catch
{
ReportManage.ErrReport(this, "File名として不正です。");
}
SetText(t);
base.Run(runChildren);
}
public override void Run(bool runChildren)
{
var file = this.GetUpperRawler <TsvReadLines>();
if (file != null)
{
string columnName = this.ColumnName.Convert(this);
if (this.ColumnNameTree != null)
{
columnName = RawlerBase.GetText(GetText(), ColumnNameTree, this);
}
if (string.IsNullOrEmpty(columnName) == false)
{
if (file.ColumnNameDic.ContainsKey(columnName))
{
var i = file.ColumnNameDic[columnName];
var d = file.Text.Split('\t');
if (d.Length > i)
{
SetText(d[i]);
}
else
{
SetText(Default);
}
}
else
{
ReportManage.ErrReport(this, "該当するキーがありません:" + columnName);
}
}
else
{
ReportManage.ErrReport(this, "ColumnNameが空です。");
}
}
else
{
ReportManage.ErrReport(this, "GetTsvValue の上流にTsvReadLinesがありません");
}
base.Run(runChildren);
}
/// <summary>
/// PreTreeを通したあとのText
/// </summary>
/// <returns></returns>
protected string GetText()
{
if (this.Parent != null)
{
if (preTree != null)
{
preTree.SetParent();
return(RawlerBase.GetText(this.Parent.Text, PreTree, this.Parent));
}
else
{
return(this.Parent.Text);
}
}
else
{
return(this.text);
}
}
private IEnumerable <TextPair <T> > Convert <T>(IEnumerable <TextPair <T> > list, Func <T, string> textFunc)
{
if (ConvertTree != null)
{
ConvertTree.SetParent(this.Parent);
foreach (var item in list)
{
item.Text = RawlerBase.GetText(item.Text, ConvertTree, this.Parent);
yield return(item);
//yield return RawlerBase.GetText(item, ConvertTree, this.Parent);
}
}
else
{
foreach (var item in list)
{
yield return(item);
}
}
}
public override void Run(bool runChildren)
{
try
{
string fileName = this.FileName.Convert(this);
if (FileNameTree != null)
{
FileNameTree.SetParent();
FileNameTree.SetParent(this);
fileName = RawlerBase.GetText(GetText(), FileNameTree, this);
}
if (string.IsNullOrEmpty(fileName))
{
if (string.IsNullOrEmpty(fileName))
{
ReportManage.ErrReport(this, "FileNameが空です。");
return;
}
}
baseFileName = fileName;
currentFileName = fileName;
FileInit(fileName);
base.Run(runChildren);
endAction?.Invoke();
}
catch (Exception e)
{
ReportManage.ErrReport(this, e.Message);
}
finally
{
if (streamWriter != null)
{
streamWriter.Close();
streamWriter.Dispose();
}
}
}
/// <summary>
/// ファイルで保存する
/// </summary>
protected void SaveFile()
{
string filename = this.FileName.Convert(this);
if (this.FileNameTree != null)
{
filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this);
}
if (string.IsNullOrEmpty(filename) == false)
{
System.IO.StreamWriter sw = null;
if (this.FileSaveMode == Tool.FileSaveMode.Create)
{
sw = System.IO.File.CreateText(filename);
}
else if (this.FileSaveMode == Tool.FileSaveMode.Append)
{
sw = System.IO.File.AppendText(filename);
}
if (saveFileType == FileType.Json)
{
sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.DataDic)));
}
else if (saveFileType == FileType.Tsv)
{
sw.WriteLine(this.ToTsv());
}
else if (saveFileType == FileType.LTsv)
{
foreach (var item in this.GetDataRows())
{
var d = item.DataDic.ToDictionary(n => n.Key, n => n.Value.JoinText(","));
sw.WriteLine(d.ToLtsvLine());
}
}
sw.Close();
ReportManage.Report(this, filename + "作成完了", true, EndReport);
}
}
public override void Run(bool runChildren)
{
string path = string.Empty;
if (workFolderType == Tool.SpecialFolder.MyDocuments)
{
path = System.Environment.GetFolderPath(Environment.SpecialFolder.Personal);
}
if (workFolderType == Tool.SpecialFolder.Desktop)
{
path = System.Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory);
}
string folder = null;
if (FolderNameTree != null)
{
folder = RawlerBase.GetText(this.GetText(), FolderNameTree, this);
}
else
{
folder = FolderName;
}
if (folder == null || folder.Length == 0)
{
ReportManage.ErrReport(this, "CreateFolderのFolderNameがありません");
}
else
{
path = System.IO.Path.Combine(path, folder);
if (System.IO.Directory.Exists(path) == false)
{
System.IO.Directory.CreateDirectory(path);
}
}
base.Run(runChildren);
}
public override void Run(bool runChildren)
{
string file = string.Empty;
if (this.FileName != null)
{
file = this.FileName;
}
else if (FileNameTree != null)
{
file = RawlerBase.GetText(GetText(), FileNameTree, this);
}
else
{
file = this.GetText();
}
if (System.IO.File.Exists(file) == Result)
{
base.Run(runChildren);
}
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
if (string.IsNullOrEmpty(Format))
{
ReportManage.ErrReport(this, "Formatが空です");
}
else if (Args == null)
{
ReportManage.ErrReport(this, "Argsが空です");
}
else
{
List <string> list = new List <string>();
foreach (var item in Args)
{
item.SetParent(this.Parent);
item.SetParent();
list.Add(RawlerBase.GetText(GetText(), item, this.Parent));
}
SetText(string.Format(Format, list.ToArray()));
}
base.Run(runChildren);
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
string path = string.Empty;
if (workFolderType == Tool.SpecialFolder.MyDocuments)
{
path = System.Environment.GetFolderPath(Environment.SpecialFolder.Personal);
}
if (workFolderType == Tool.SpecialFolder.Desktop)
{
path = System.Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory);
}
if (FolderTree != null)
{
var f = RawlerBase.GetText(GetText(), FolderTree, this);
path = System.IO.Path.Combine(path, f);
}
else if (Folder != null)
{
path = System.IO.Path.Combine(path, Folder);
}
try
{
if (System.IO.Directory.Exists(path) == false)
{
System.IO.Directory.CreateDirectory(path);
}
System.IO.Directory.SetCurrentDirectory(path);
}
catch (Exception e)
{
ReportManage.ErrReport(this, e.Message);
}
SetText(System.IO.Directory.GetCurrentDirectory());
base.Run(runChildren);
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
var client = this.GetAncestorRawler().OfType <WebClient>().DefaultIfEmpty(new WebClient()).FirstOrDefault();
string url = Url;
if (string.IsNullOrEmpty(url))
{
url = GetText();
}
var page = this.GetUpperRawler <Page>();
if (page != null)
{
client.Referer = page.GetCurrentUrl();
}
var data = client.HttpGetByte(url);
string path = string.Empty;
if (FolderNameTree != null)
{
path = RawlerBase.GetText(this.Parent.Text, FolderNameTree, this.Parent);
}
else
{
path = Folder.Convert(this);
}
if (path != null)
{
if (System.IO.Path.IsPathRooted(path) == false)
{
path = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), path);
}
if (System.IO.Directory.Exists(path) == false)
{
try
{
System.IO.Directory.CreateDirectory(path);
}
catch (Exception e)
{
ReportManage.ErrReport(this, "ディレクトリーの作成に失敗しました" + e.Message);
}
}
}
else
{
path = System.IO.Directory.GetCurrentDirectory();
}
string ex = System.IO.Path.GetExtension(url);
if (ex.Split('?').Length > 0)
{
ex = ex.Split('?').First();
}
string fileName = string.Empty;
if (SaveNameTree != null)
{
fileName = RawlerBase.GetText(this.Parent.Text, SaveNameTree, this.Parent);
}
else if (string.IsNullOrEmpty(SaveName) == false)
{
fileName = SaveName.Convert(this);
}
else
{
fileName = System.IO.Path.GetFileNameWithoutExtension(url);
}
path = System.IO.Path.Combine(path, fileName + ex);
try
{
using (var writer = System.IO.File.Create(path))
{
writer.Write(data, 0, data.Length);
SetText(path);
}
}
catch (Exception e)
{
ReportManage.ErrReport(this, "ファイルの保存に失敗しました" + e.Message + " Path:" + path);
}
SetText(fileName + ex);
base.Run(runChildren);
}
//public new void Run()
//{
// Run(true);
//}
/// <summary>
/// 実行
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
string txt = string.Empty;
if (Value == null || Value.Length == 0)
{
txt = GetText();
}
else
{
txt = Value.Convert(this);
}
if (useHtmlDecode)
{
txt = System.Net.WebUtility.HtmlDecode(txt);
}
string attribute = string.Empty;
if (this.Attribute == null && attributeTree != null)
{
if (this.Parent != null)
{
attribute = RawlerBase.GetText(this.Parent.Text, attributeTree, this);
}
}
else if (this.Attribute != null)
{
attribute = this.Attribute.Convert(this);
}
Data.DataWrite(this, attribute, txt, writeType, AttributeType);
//IData data = null;
//IRawler current = this.Parent;
//while (current != null)
//{
// if (current is IData)
// {
// data = current as IData;
// break;
// }
// current = current.Parent;
//}
//if (data != null)
//{
// string txt = string.Empty;
// if (Value == null || Value.Length == 0)
// {
// txt = GetText();
// }
// else
// {
// txt = Value;
// }
// if (useHtmlDecode)
// {
// txt = System.Net.WebUtility.HtmlDecode(txt);
// }
// if (this.Attribute == null && attributeTree != null)
// {
// string tmpAttributeText = string.Empty;
// if (this.Parent != null)
// {
// tmpAttributeText = RawlerBase.GetText(this.Parent.Text, attributeTree,this);
// }
// data.DataWrite(tmpAttributeText, txt, writeType,AttributeType);
// }
// else
// {
// if (this.Attribute != null)
// {
// data.DataWrite(this.Attribute, txt, writeType,AttributeType);
// }
// else
// {
// data.DataWrite(string.Empty, txt, writeType, AttributeType);
// }
// }
// //if (this.AttributeObjectName != null && this.AttributeObjectName.Length>0)
// //{
// // var list = this.GetConectAllRawler().Where(n => n.Name == this.AttributeObjectName);
// // if (list.Count() > 0)
// // {
// // data.DataWrite(list.First().Text, txt,writeType);
// // }
// // else
// // {
// // ReportManage.ErrReport(this, "AttributeObjectNameの指定が不正です。オブジェクトが見つかりませんでした。");
// // }
// //}
//}
//else
//{
// ReportManage.ErrReport(this,"書き込み先のDataオブジェクトが見つかりません。");
//}
this.RunChildren(runChildren);
}
