public override void Run(bool runChildren)
{
currentFileName = FileName.Convert(this);
baseFileName = currentFileName;
if (currentFileName.Length == 0)
{
ReportManage.ErrReport(this, "FileNameがありません");
return;
}
var f = CheckFileSizeAndCreate(System.IO.File.AppendText(currentFileName));
int count = 0;
ActionBlock <string> actionBlock = new ActionBlock <string>((n) => {
if (count % 10000 == 0)
{
f = CheckFileSizeAndCreate(f);
}
f.WriteLine(n);
ReportManage.Report(this, "NextDataRow");
count++;
});
inputBlock = new BufferBlock <string>();
inputBlock.LinkTo(actionBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
base.Run(runChildren);
inputBlock.Complete();
actionBlock.Completion.Wait();
f.Close();
}
private static void StopWatch()
{
if (startDate != null)
{
var time = DateTime.Now - startDate;
ReportManage.Report(null, "経過時間：" + time.ToString(), true, true);
}
MessageBox.Show("Complete");
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
//App認証の場合
if (SetTwitterApiKeys.UseAppOnlyAutherentcation || AppOnlyAuthentication != null)
{
if (AppOnlyAuthentication == null)
{
AppOnlyAuthentication = new AppOnlyAuthentication();
}
AppOnlyAuthentication.SetUp(this);
token = OAuth2.GetToken(AppOnlyAuthentication.ConsumerKey, AppOnlyAuthentication.ConsumerSecret);
}
else
{
//普通の認証の場合
if (GetOAuthTokens() == null)
{
var session = OAuth.Authorize(ConsumerKey, ConsumerSecret);
ReportManage.Report(this, "PINがありません。ブラウザに表示されているPINの値を入力してください。PIN=\"(表示されているPIN)\" とTwitterLoginクラスに情報を追加してください");
System.Diagnostics.Process.Start(session.AuthorizeUri.ToString());
Task reportProgressTask = Task.Factory.StartNew(() =>
{
PinDialog pin = new PinDialog();
if (pin.ShowDialog() == true)
{
var tokens = session.GetTokens(pin.PIN);
TokenData td = new TokenData()
{
ConsumerKey = ConsumerKey,
ConsumerSecret = ConsumerSecret,
AccessToken = tokens.AccessToken,
AccessTokenSecret = tokens.AccessTokenSecret
};
this.Token = new Tokens(tokens);
this.TokenData = td;
System.Xaml.XamlServices.Save(tokenSettingFileName, td);
}
else
{
return;
}
},
CancellationToken.None,
TaskCreationOptions.None,
RawlerLib.UIData.UISyncContext);
reportProgressTask.Wait();
}
}
base.Run(runChildren);
}
private void PauseResumeButton_Click(object sender, RoutedEventArgs e)
{
pause = !pause;
if (pause)
{
ReportManage.Report(null, "---Pause---");
PauseResumeButton.Content = "Resume";
}
else
{
ReportManage.Report(null, "---Resume---");
PauseResumeButton.Content = "Pause";
}
}
protected void ReadCompletedFile()
{
if (UseCompletedFile)
{
if (System.IO.File.Exists(completedFileName))
{
ReportManage.Report(this, completedFileName + "を読み込んでいます。", true, true);
foreach (var item in System.IO.File.ReadAllLines(completedFileName))
{
if (IsSingle)
{
completedHash.Add(item);
}
}
}
}
}
public void SetUp(RawlerBase rawler)
{
if (string.IsNullOrEmpty(this.ConsumerKey) || string.IsNullOrEmpty(this.ConsumerSecret))
{
if (KeyValueStore.ContainsKey(rawler, "ConsumerKey", "ConsumerSecret"))
{
this.ConsumerKey = KeyValueStore.GetValueByKey(rawler, "ConsumerKey");
this.ConsumerSecret = KeyValueStore.GetValueByKey(rawler, "ConsumerSecret");
ReportManage.Report(rawler, "KeyValueStoreからのAPI Keyを使います", true, true);
}
else if (string.IsNullOrEmpty(SetTwitterApiKeys.consumerKey) || string.IsNullOrEmpty(SetTwitterApiKeys.consumerSecret))
{
this.ConsumerKey = "gHVupgapEXlTZdu7rf3oOg";
this.ConsumerSecret = "YOicLtW8utx3NJyy88wtzq8QN3ilXeQoEGCPIJNzo";
ReportManage.Report(rawler, "RawlerのAPI Keyを使います", true, true);
}
else
{
this.ConsumerKey = SetTwitterApiKeys.consumerKey;
this.ConsumerSecret = SetTwitterApiKeys.consumerSecret;
}
}
}
public override void Run(bool runChildren)
{
LoadSetting();
bool flag = true;
Properties.ForEach(n => n.SetParent(this));
Task reportProgressTask = Task.Factory.StartNew(() =>
{
FormWindow fw = new FormWindow();
fw.Title = Title.Convert(this);
fw.SetUp(Properties);
if (fw.ShowDialog() == true)
{
PropertiesUpdate();
SaveSetting();
}
else
{
flag = false;
}
},
CancellationToken.None,
TaskCreationOptions.None,
RawlerLib.UIData.UISyncContext);
reportProgressTask.Wait();
if (flag)
{
base.Run(runChildren);
}
else
{
ReportManage.Report(this, "Formがキャンセルされました。終了します。", true, true);
}
}
public IEnumerable <string> ReadData()
{
List <string> list = new List <string>();
var login = this.GetUpperRawler <TwitterLogin>();
int totalCount = 0;
if (login != null)
{
// long max_id = long.MaxValue;
while (true)
{
/// <para>- <c>long</c> user_id (optional)</para>
/// <para>- <c>string</c> screen_name (optional)</para>
/// <para>- <c>int</c> count (optional)</para>
/// <para>- <c>long</c> since_id(optional)</para>
/// <para>- <c>long</c> max_id (optional)</para>
/// <para>- <c>bool</c> trim_user (optional)</para>
/// <para>- <c>bool</c> contributor_details (optional)</para>
/// <para>- <c>bool</c> include_rts (optional)</para>
/// <para>- <c>bool</c> exclude_replies (optional)</para>
Dictionary <string, object> dic = new Dictionary <string, object>()
{
{ "include_rts", IncludeRetweets },
{ "exclude_replies", exclude_replies },
{ "count", 100 }
};
if (string.IsNullOrEmpty(screenName) == false)
{
dic.Add("screen_name", screenName);
}
else
{
if (ParentUserIdType == RawlerTwitter.ParentUserIdType.ScreenName)
{
dic.Add("screen_name", GetText());
}
else
{
long id;
if (long.TryParse(GetText(), out id))
{
dic.Add("user_id", long.Parse(GetText()));
}
else
{
ReportManage.ErrReport(this, "親テキストがLONG型ではありません。");
dic.Add("screen_name", GetText());
}
}
}
if (max_id < long.MaxValue)
{
dic.Add("max_id", max_id - 1);
}
int count = 0;
long tmp_max_id = max_id;
bool retry = false;
try
{
foreach (var item in login.Token.Statuses.UserTimeline(dic))
{
count++;
totalCount++;
max_id = Math.Min(max_id, item.Id);
list.Add(JObject.FromObject(item).ToString(Newtonsoft.Json.Formatting.None));
// yield return JObject.FromObject(item).ToString();
if (MaxCount > 0 && MaxCount <= totalCount)
{
break;
}
}
}
catch (Exception e)
{
ReportManage.ErrReport(this, GetText() + "\t" + e.Message);
if (e.Message == "Not authorized.")
{
break;
}
else if (e.Message == "Rate limit exceeded")
{
retry = true;
}
else if (e.Message == "Over capacity")
{
retry = true;
}
else
{
throw e;
}
}
if (retry)
{
ReportManage.Report(this, "3分Sleep", true, true);
// this.GetUpperRawler<TwitterLogin>().ReLogin();
System.Threading.Thread.Sleep(new TimeSpan(0, 3, 0));
max_id = tmp_max_id;
}
if (MaxCount > 0 && MaxCount <= totalCount)
{
break;
}
if (count < 10)
{
break;
}
if (SleepSecond > 0)
{
GC.Collect();
System.Threading.Thread.Sleep((int)(SleepSecond * 1000));
}
}
}
else
{
ReportManage.ErrReport(this, "TwitterLoginをTweetUserTimelineの上流に配置してください");
}
return(list);
}
protected IEnumerable <string> ReadList()
{
var login = this.GetAncestorRawler().OfType <TwitterLogin>().First();
bool flag = true;
var key = GetKey();
decimal maxId = -1;
decimal minId = decimal.MaxValue;
string list_id = GetText();
if (string.IsNullOrEmpty(list_id) == false)
{
list_id = this.list_id;
}
bool isFrist = true;
if (searchWordDic.ContainsKey(key))
{
isFrist = false;
}
int loop = 1;
while (flag)
{
ReportManage.Report(this, loop + "回目", true, true);
loop++;
Dictionary <string, object> dic = new Dictionary <string, object>()
{
{ "list_id", list_id },
{ "slug", this.slug },
{ "owner_id", this.owner_id },
{ "owner_screen_name", this.owner_screen_name },
{ "include_rts", this.include_rts },
{ "count", 100 },
};
if (minId != decimal.MaxValue)
{
dic.Add("max_id", minId - 1);
}
if (isFrist == false && searchWordDic.ContainsKey(key))
{
dic.Add("since_id", searchWordDic[key].Item1);
}
var result = login.Token.Lists.Statuses(dic.Where(n => n.Value != null));
int c = 0;
int d = 0;
if (result != null)
{
//Jsonを下流に流す
foreach (var item in result)
{
maxId = Math.Max(maxId, item.Id);
minId = Math.Min(minId, item.Id);
if (searchWordDic.ContainsKey(key))
{
if (searchWordDic[key].Item1 <item.Id || searchWordDic[key].Item2> item.Id)
{
c++;
yield return(Newtonsoft.Json.Linq.JObject.FromObject(item).ToString());
// yield return Codeplex.Data.DynamicJson.Serialize(item);
}
else
{
d++;
}
}
else
{
c++;
yield return(Newtonsoft.Json.Linq.JObject.FromObject(item).ToString());
// yield return Codeplex.Data.DynamicJson.Serialize(item);
// yield return Codeplex.Data.DynamicJson.Serialize(item);
}
}
//すでに取得したものを含むとき
if (d > 0 || c == 0)
{
flag = false;
}
}
else
{
flag = false;
}
if (searchWordDic.ContainsKey(key))
{
searchWordDic[key] = new Tuple <decimal, decimal>(maxId, minId);
}
else
{
searchWordDic.Add(key, new Tuple <decimal, decimal>(maxId, minId));
}
if (SleepSecond > 0)
{
System.Threading.Thread.Sleep((int)(SleepSecond * 1000));
}
}
}
public override void Run(bool runChildren)
{
input = new BufferBlock <KeyValuePair <string, int> >();
BatchBlock <KeyValuePair <string, int> > batch = new BatchBlock <KeyValuePair <string, int> >(BatchSize);
TransformBlock <KeyValuePair <string, int>[], Dictionary <string, int> > trans = new TransformBlock <KeyValuePair <string, int>[], Dictionary <string, int> >(l =>
{
return(l.ToCountDictionary(n => n.Key, n => n.Value));
});
BatchBlock <Dictionary <string, int> > dicBatch = new BatchBlock <Dictionary <string, int> >(3);
TransformBlock <Dictionary <string, int>[], Dictionary <string, int> > margeDic = new TransformBlock <Dictionary <string, int>[], Dictionary <string, int> >(n => {
return(n.Marge());
});
bool retrunFlag = true;
BufferBlock <Dictionary <string, int> > result = new BufferBlock <Dictionary <string, int> >();
ActionBlock <Dictionary <string, int> > returnAction = new ActionBlock <Dictionary <string, int> >(n => {
if (retrunFlag)
{
dicBatch.Post(n);
}
else
{
result.Post(n);
}
});
input.LinkTo(batch, new DataflowLinkOptions()
{
PropagateCompletion = true
});
batch.LinkTo(trans, new DataflowLinkOptions()
{
PropagateCompletion = true
});
trans.LinkTo(dicBatch, new DataflowLinkOptions()
{
PropagateCompletion = true
});
dicBatch.LinkTo(margeDic, new DataflowLinkOptions()
{
PropagateCompletion = true
});
margeDic.LinkTo(returnAction, new DataflowLinkOptions()
{
PropagateCompletion = true
});
SetText(GetText());
base.Run(runChildren);
retrunFlag = false;
input.Complete();
returnAction.Completion.Wait();
IList <Dictionary <string, int> > list;
if (result.TryReceiveAll(out list))
{
var r = list.Marge();
if (FileSave)
{
using (var f = System.IO.File.CreateText(FileName.Convert(this)))
{
foreach (var item in r.OrderByDescending(n => n.Value).Where(n => n.Value >= CutCount))
{
f.WriteLine(item.Key + "\t" + item.Value);
ReportManage.Report(this, item.Key + "\t" + item.Value, true, true);
}
ReportManage.Report(this, "FileWrite:" + FileName.Convert(this), true, true);
}
}
var reduce = this.GetUpperRawler <Reduce>();
if (reduce != null)
{
foreach (var item in r.Where(n => n.Value >= CutCount))
{
reduce.AddKeyValue(item);
}
}
}
else
{
ReportManage.ErrReport(this, "TryReceiveAll 失敗");
}
}
protected void ReadSearch()
{
var login = this.GetAncestorRawler().OfType <TwitterLogin>().First();
bool flag = true;
decimal maxId = -1;
decimal minId = decimal.MaxValue;
string searchWord = GetText();
if (string.IsNullOrEmpty(SearchWord) == false)
{
searchWord = SearchWord.Convert(this);
}
bool isFrist = true;
if (searchWordDic.ContainsKey(searchWord))
{
isFrist = false;
}
int loop = 1;
while (flag)
{
ReportManage.Report(this, loop + "回目", true, true);
loop++;
/// <para>- <c>string</c> q (required)</para>
/// <para>- <c>string</c> geocode (optional)</para>
/// <para>- <c>string</c> lang (optional)</para>
/// <para>- <c>string</c> locale (optional)</para>
/// <para>- <c>string</c> result_type (optional)</para>
/// <para>- <c>int</c> count (optional)</para>
/// <para>- <c>string</c> until (optional)</para>
/// <para>- <c>long</c> since_id (optional)</para>
/// <para>- <c>long</c> max_id (optional)</para>
/// <para>- <c>bool</c> include_entities (optional)</para>
Dictionary <string, object> dic = new Dictionary <string, object>()
{
{ "q", searchWord },
{ "geocode", this.GeoCode },
{ "lang", this.Language },
{ "locale", this.Locale },
{ "result_type", "recent" },
{ "count", 100 },
// {"until", },
// {"max_id",},
// {"include_entities", "paint.exe"},
};
if (minId != decimal.MaxValue)
{
dic.Add("max_id", minId - 1);
}
if (isFrist == false && searchWordDic.ContainsKey(searchWord))
{
dic.Add("since_id", searchWordDic[searchWord].Item1);
}
try
{
var result = login.Token.Search.Tweets(dic.Where(n => n.Value != null));
int c = 0;
int d = 0;
if (result != null)
{
List <string> list = new List <string>();
//Jsonを下流に流す
foreach (var item in result)
{
maxId = Math.Max(maxId, item.Id);
minId = Math.Min(minId, item.Id);
if (searchWordDic.ContainsKey(searchWord))
{
if (searchWordDic[searchWord].Item1 <item.Id || searchWordDic[searchWord].Item2> item.Id)
{
c++;
list.Add(Newtonsoft.Json.Linq.JObject.FromObject(item).ToString());
// yield return Codeplex.Data.DynamicJson.Serialize(item);
}
else
{
d++;
}
}
else
{
c++;
list.Add(Newtonsoft.Json.Linq.JObject.FromObject(item).ToString());
// yield return Codeplex.Data.DynamicJson.Serialize(item);
// yield return Codeplex.Data.DynamicJson.Serialize(item);
}
}
Post(list);
//すでに取得したものを含むとき
if (d > 0 || c == 0)
{
flag = false;
}
}
else
{
flag = false;
}
if (searchWordDic.ContainsKey(searchWord))
{
searchWordDic[searchWord] = new Tuple <decimal, decimal>(maxId, minId);
}
else
{
searchWordDic.Add(searchWord, new Tuple <decimal, decimal>(maxId, minId));
}
}
catch (Exception ex)
{
ReportManage.ErrReport(this, ex.ToString());
if (ex.Message.Contains("Rate limit exceeded"))
{
ReportManage.Report(this, "30秒停止");
System.Threading.Thread.Sleep((int)(30 * 1000));
}
else
{
flag = false;
}
}
if (MaxLoopCount > loop)
{
break;
}
if (SleepSecond > 0)
{
System.Threading.Thread.Sleep((int)(SleepSecond * 1000));
}
}
}
