/// <summary>
/// commentを出力する。
/// </summary>
protected void ReportComment()
{
if (!string.IsNullOrEmpty(this.commnet))
{
ReportManage.Report(this, this.Comment);
}
}
public void JsonFileSave()
{
string filename = this.FileName.Convert(this);
if (this.FileNameTree != null)
{
filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this);
}
if (string.IsNullOrEmpty(filename) == false)
{
System.IO.StreamWriter sw = null;
if (this.FileSaveMode == Tool.FileSaveMode.Create)
{
sw = System.IO.File.CreateText(filename);
}
else if (this.FileSaveMode == Tool.FileSaveMode.Append)
{
sw = System.IO.File.AppendText(filename);
}
if (saveFileType == FileType.Json)
{
sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.GetDataDicForJson())));
}
sw.Close();
ReportManage.Report(this, filename + "作成完了", true, EndReport);
}
}
private void TsvFileSave()
{
if (doLastFileSave == false)
{
return;
}
string filename = this.FileName.Convert(this);
if (this.FileNameTree != null)
{
filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this);
}
if (string.IsNullOrEmpty(filename) == false)
{
System.IO.StreamWriter sw = null;
if (this.FileSaveMode == Tool.FileSaveMode.Create)
{
sw = System.IO.File.CreateText(filename);
}
else if (this.FileSaveMode == Tool.FileSaveMode.Append)
{
sw = System.IO.File.AppendText(filename);
}
sw.Write(ToTsv());
sw.Close();
ReportManage.Report(this, filename + "作成完了", true, EndReport);
}
}
public override void Run(bool runChildren)
{
var d = (IData)this.GetUpperInterface <IData>();
if (d != null)
{
ReportManage.Report(this, d.GetCurrentDataRow().ToString(), true, true);
}
}
public override void Run(bool runChildren)
{
Data data = null;
IRawler current = this.Parent;
while (current != null)
{
if (current is Data)
{
data = current as Data;
break;
}
current = current.Parent;
}
if (data != null)
{
if (data.GetCurrentDataNull())
{
if (ignoreDataNull == false)
{
ReportManage.ErrReport(this, "RowがNullです。Writeが動作していないようです。");
var list = this.GetAncestorRawler().Where(n => n is Page);
if (DoPageReLoad)
{
if (list.Count() > 0)
{
var p = list.First() as Page;
pageCount++;
if (PageReLoadCount < pageCount)
{
ReportManage.Report(this, "再読み込み待機中。");
System.Threading.Thread.Sleep(1000 * pageCount * pageCount);
p.Run();
}
else
{
ReportManage.ErrReport(this, "書き込み先のData クラスが見つかりませんでした。");
}
}
}
}
}
else
{
pageCount = 0;
data.NextDataRow();
ReportManage.Report(this, "NextDataRow");
}
}
else
{
ReportManage.ErrReport(this, "書き込み先のData クラスが見つかりませんでした。");
}
this.RunChildren(runChildren);
}
/// <summary>
/// 再びログインする。
/// </summary>
public void ReLogin()
{
string html = this.HttpPost(loginPage, vals, false);
if (this.GetCookieCount() > 0)
{
if (ErrString != null)
{
if (html.Contains(ErrString) == false)
{
hasLogin = true;
ReportManage.Report(this, "ログイン成功");
}
else
{
hasLogin = false;
ReportManage.ErrReport(this, "ログイン失敗");
}
}
else
{
hasLogin = true;
ReportManage.Report(this, "ログイン成功");
}
}
else
{
if (isNoCookieSite)
{
hasLogin = false;
ReportManage.ErrReport(this, "ログイン失敗");
}
else
{
if (ErrString != null)
{
if (html.Contains(ErrString) == false)
{
hasLogin = true;
ReportManage.Report(this, "ログイン成功");
}
else
{
hasLogin = false;
ReportManage.ErrReport(this, "ログイン失敗");
}
}
else
{
hasLogin = true;
ReportManage.Report(this, "ログイン成功");
}
}
}
this.text = html;
}
public override string HttpGet(string url, Encoding enc)
{
this.Sleep();
if (ReportUrl)
{
ReportManage.Report(this, "GET " + url, true, true);
}
System.Net.Http.HttpClient client = GetHttpClient();
var result = client.GetStringAsync(url).ConfigureAwait(false);
return(result.GetAwaiter().GetResult());
}
public void NextDataRow(DataRowObject cDataRow)
{
ReportManage.Report(this, "NextDataRow");
if (Commited != null)
{
Commited(this, new EventDataRow(cDataRow));
}
currentDataRow = new DataRowObject();
if (stock)
{
dataList.Add(cDataRow);
}
}
public override void Run(bool runChildren)
{
if (doSave)
{
if (FileName != null)
{
filename = FileName;
}
else
{
if (string.IsNullOrEmpty(filename))
{
ReportManage.ErrReport(this, "FileNameが空です。");
return;
}
//Microsoft.Win32.SaveFileDialog saveDialog = new Microsoft.Win32.SaveFileDialog();
//saveDialog.Title = "保存ファイルの指定 CountData:" + this.Comment;
//if (string.IsNullOrEmpty(ExtendFilter) == false)
//{
// saveDialog.Filter = RawlerLib.Io.FilterStringCreate(ExtendFilter);
//}
//if (saveDialog.ShowDialog() == true)
//{
// filename = saveDialog.FileName;
//}
//using (var file = System.IO.File.CreateText(filename))
//{
// file.WriteLine("Group\tKey\tCount");
//}
}
}
base.Run(runChildren);
if (doSave || FileName != null)
{
try
{
Write(filename);
}
catch (Exception e)
{
ReportManage.ErrReport(this, "CountDataでファイルの書き込みに失敗しました。" + e.Message);
}
}
if (reportData)
{
ReportManage.Report(this, this.ToTsv(), true, true);
}
}
public override void Run(bool runChildren)
{
string text;
if (viewParentText && string.IsNullOrEmpty(Message))
{
text = Header.Convert(this) + GetText() + Footer.Convert(this);
}
else
{
text = Header.Convert(this) + Message.Convert(this) + this.Footer.Convert(this);
}
ReportManage.Report(this, System.Net.WebUtility.HtmlDecode(text), returncode, visible);
this.RunChildren(runChildren);
}
public string HttpGet(string url, List <KeyValue> parameterList, List <KeyValue> httpHeaderList)
{
this.Sleep();
if (ReportUrl)
{
ReportManage.Report(this, "GET " + url, true, true);
}
System.Net.Http.HttpClient client = GetHttpClient();
client.SetHeader(httpHeaderList);
var result = client.GetStringAsync(url).ConfigureAwait(false).GetAwaiter().GetResult();
client.RemoveHeader(httpHeaderList);
return(result);
}
public string HttpPost(string url, List <KeyValue> parameterList, List <KeyValue> httpHeaderList)
{
this.Sleep();
System.Net.Http.HttpClient client = GetHttpClient();
client.SetHeader(httpHeaderList);
if (ReportUrl)
{
ReportManage.Report(this, $"Post {url} [{string.Join("\t", parameterList.Select(n=>$"{n.Key}:{n.Value}"))}]", true, true);
}
var r = client.PostAsync(url, new FormUrlEncodedContent(parameterList.Select(n => new KeyValuePair <string, string>(n.Key, n.Value)))).ConfigureAwait(false).GetAwaiter().GetResult();
var r2 = r.Content.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult();
this.ErrMessage = r.StatusCode.ToString();
return(r2);
}
/// <summary>
/// ファイルで保存する
/// </summary>
protected void SaveFile()
{
string filename = this.FileName.Convert(this);
if (this.FileNameTree != null)
{
filename = RawlerBase.GetText(this.GetText(), this.FileNameTree, this);
}
if (string.IsNullOrEmpty(filename) == false)
{
System.IO.StreamWriter sw = null;
if (this.FileSaveMode == Tool.FileSaveMode.Create)
{
sw = System.IO.File.CreateText(filename);
}
else if (this.FileSaveMode == Tool.FileSaveMode.Append)
{
sw = System.IO.File.AppendText(filename);
}
if (saveFileType == FileType.Json)
{
sw.WriteLine(Codeplex.Data.DynamicJson.Serialize(this.GetDataRows().Select(n => n.DataDic)));
}
else if (saveFileType == FileType.Tsv)
{
sw.WriteLine(this.ToTsv());
}
else if (saveFileType == FileType.LTsv)
{
foreach (var item in this.GetDataRows())
{
var d = item.DataDic.ToDictionary(n => n.Key, n => n.Value.JoinText(","));
sw.WriteLine(d.ToLtsvLine());
}
}
sw.Close();
ReportManage.Report(this, filename + "作成完了", true, EndReport);
}
}
public void Write(RawlerBase rawler, string text, bool viewTotal)
{
if (DoRun)
{
if (viewParent)
{
text = this.Parent.ToObjectString() + " " + text;
}
var t = sw.ElapsedMilliseconds + "\t" + text;
total += sw.ElapsedMilliseconds;
if (viewTotal)
{
ReportManage.Report(rawler, t, true, true);
ReportManage.Report(rawler, "Total:" + total, true, true);
}
else
{
ReportManage.Report(rawler, t, true, true);
}
sw.Restart();
}
}
/// <summary>
/// HttpPost
/// </summary>
/// <param name="url"></param>
/// <param name="vals"></param>
/// <returns></returns>
public virtual byte[] HttpPostByte(string url, List <KeyValue> vals)
{
if (ReportUrl)
{
ReportManage.Report(this, "POST " + url, true, true);
}
try
{
Sleep();
wc.Referer = referer;
wc.UserAgent = userAgent;
wc.BasicAuthorization = BasicAuthorization;
System.Collections.Specialized.NameValueCollection list = new System.Collections.Specialized.NameValueCollection();
foreach (var item in vals)
{
list.Add(item.Key, item.Value);
}
var data2 = wc.UploadValues(url, list);
return(data2);
}
catch (Exception e)
{
if (visbleErr)
{
ReportManage.ErrReport(this, "Url:" + url + " " + e.Message);
}
//if (e is System.UriFormatException)
//{
// ReportManage.ErrReport(this,"["+url+"]は無効なURLです。");
//}
ErrMessage = e.Message;
// throw new Exception("HttpGet:"+url+"に失敗しました");
}
return(null);
}
public override void Run(bool runChildren)
{
var page = GetPage();
var u = GetText().Replace("&#", "&&&&").Split('#');
string url = string.Empty;
if (u.Length > 0)
{
url = u[0].Replace("&&&&", "&#");
}
if (this.Parent.Text != null && this.Parent.Text.Length > 0)
{
if (page != null)
{
//始まりがURLと違う場合初期化
if (page.GetStartUrl() != baseUrl)
{
baseUrl = page.GetStartUrl();
urlHash.Clear();
count = 0;
}
if (allowSameUrl == false)
{
if (urlHash.Contains(url) == false)
{
count++;
urlHash.Add(url);
if (count < maxCount)
{
// this.Text = this.Parent.Text;
if (sleepTime > 0)
{
System.Threading.Thread.Sleep(new TimeSpan(0, 0, sleepTime));
}
this.RunChildren(runChildren);
ReportManage.Report(this, "NextPage:" + GetText());
page.PushUrl(url);
}
else
{
ReportManage.Report(this, "NextPage:指定ページ数を取得しました");
}
}
}
else
{
if (count < maxCount)
{
// this.Text = this.Parent.Text;
if (sleepTime > 0)
{
System.Threading.Thread.Sleep(new TimeSpan(0, 0, sleepTime));
}
this.RunChildren(runChildren);
ReportManage.Report(this, "NextPage:" + GetText());
page.PushUrl(url);
}
else
{
ReportManage.Report(this, "NextPage:指定ページ数を取得しました");
}
}
}
}
}
/// <summary>
/// HttpPost
/// </summary>
/// <param name="url"></param>
/// <param name="vals"></param>
/// <returns></returns>
public virtual string HttpPost(string url, List <KeyValue> vals)
{
if (ReportUrl)
{
var t = vals.Select(n => n.Key + ":" + n.Value.Trim().Replace("\n", "").Replace("\r", "")).JoinText("\n");
ReportManage.Report(this, "POST " + url + "\n" + t, true, true);
}
try
{
Sleep();
wc.Referer = referer;
wc.UserAgent = userAgent;
wc.BasicAuthorization = BasicAuthorization;
System.Collections.Specialized.NameValueCollection list = new System.Collections.Specialized.NameValueCollection();
//foreach (var item in vals.GroupBy(n=>n.Key))
//{
// if (item.Count() == 1)
// {
// list.Add(item.Key, item.First().Value);
// }
// //同じ値が複数ある時用？
// else
// {
// int i = 0;
// foreach (var item2 in item)
// {
// list.Add(item2.Key + "[" + i + "]", item2.Value);
// i++;
// }
// }
//}
foreach (var item in vals)
{
list.Add(item.Key, item.Value);
}
if (encoder == null)
{
encoder = GetEncoding();
}
wc.Encoding = encoder;
// wc.UploadValues
var data2 = wc.UploadValues2(new Uri(url), vals);
return(data2);
}
catch (Exception e)
{
if (visbleErr)
{
ReportManage.ErrReport(this, "Url:" + url + " " + e.Message);
}
//if (e is System.UriFormatException)
//{
// ReportManage.ErrReport(this,"["+url+"]は無効なURLです。");
//}
ErrMessage = e.Message;
// throw new Exception("HttpGet:"+url+"に失敗しました");
}
return(string.Empty);
}
/// <summary>
/// Image用 失敗時NULL
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public virtual byte[] HttpGetByte(string url)
{
Sleep();
if (ReportUrl)
{
ReportManage.Report(this, "GET " + url, true, true);
}
byte[] data;
try
{
wc.Referer = referer;
wc.UserAgent = userAgent;
wc.BasicAuthorization = BasicAuthorization;
data = wc.DownloadData(url);
//// リクエストの作成
//HttpWebRequest req = (HttpWebRequest)WebRequest.Create(Uri.EscapeUriString(url));
//req.CookieContainer = cc;
//if (addUserAgent)
//{
// req.UserAgent = userAgent;
//}
//if (referer != string.Empty)
//{
// req.Referer = referer;
//}
//// req.Referer = "http://www.pixiv.net/";
//WebResponse res = req.GetResponse();
//// レスポンスの読み取り
//Stream resStream = res.GetResponseStream();
////b = new StreamReader(resStream);
////StreamReader sr = new StreamReader(resStream);
//while (true)
//{
// int b = resStream.ReadByte();
// if (b == -1)
// {
// break;
// }
// data.Add((byte)b);
//}
////sr.Close();
//resStream.Close();
}
catch (Exception e)
{
if (visbleErr)
{
ReportManage.ErrReport(this, "Url:" + url + " " + e.Message);
}
data = null;
}
if (data != null)
{
return(data);
}
else
{
return(null);
}
}
/// <summary>
/// HttpGet
/// </summary>
/// <param name="url"></param>
/// <param name="enc"></param>
/// <returns></returns>
public virtual string HttpGet(string url, Encoding enc)
{
Sleep();
if (UseCache)
{
if (casheDic.ContainsKey(url))
{
return(casheDic[url]);
}
}
ErrMessage = string.Empty;
string result = string.Empty;
bool retry = false;
if (ReportUrl)
{
ReportManage.Report(this, "GET " + url, true, true);
}
try
{
wc.Referer = referer;
wc.UserAgent = userAgent;
wc.BasicAuthorization = BasicAuthorization;
if (enc != null)
{
wc.Encoding = enc;
result = wc.DownloadString(url);
}
else
{
var data = wc.DownloadData(url);
result = GetAutoEncoding(data, out encoder);
//RawlerLib.Text.TxtEnc txtEnc = new RawlerLib.Text.TxtEnc();
//byte[] byteArray2 = wc.DownloadData(url);
//txtEnc.SetFromByteArray(ref byteArray2);
//txtEnc.Codec = "shift_jis";
//result = txtEnc.Text;
}
}
catch (Exception e)
{
if (visbleErr)
{
ReportManage.ErrReport(this, "Url:" + url + " " + e.Message);
}
ErrMessage = e.Message;
retry = false;
}
if (retry)
{
count++;
if (count <= tryCount)
{
ReportManage.ErrReport(this, "HttpGet:" + url + "にリトライ待機中");
System.Threading.Thread.Sleep(new TimeSpan(0, 0, 10 * count * count));
result = this.HttpGet(url, enc);
}
else
{
ReportManage.ErrReport(this, "HttpGet:" + url + "に失敗しました");
result = string.Empty;
}
}
if (UseCache)
{
casheDic.GetValueOrAdd(url, result);
}
return(result);
}
