public TwitterMonitor(string monitoringObjectJson, string queryJson, string jsonFilePrefix, string accessToken, string accessTokenSecret, string consumerKey, string consumerSecret, int savingInterval = 50000, int sleepTimeInSeconds = 6)
{
if (string.IsNullOrWhiteSpace(accessToken) || string.IsNullOrWhiteSpace(accessTokenSecret) || string.IsNullOrWhiteSpace(consumerKey) || string.IsNullOrWhiteSpace(consumerSecret))
{
throw new ArgumentNullException("Twitter access details not provided! AccessToken, AccessTokenSecret, ConsumerKey, or ConsumerSecret is/are not given!");
}
tweetSaveInterval = savingInterval;
sleepInterval = sleepTimeInSeconds;
tr = new TwitterRetriever(accessToken, accessTokenSecret, consumerKey, consumerSecret);
if (!string.IsNullOrWhiteSpace(monitoringObjectJson) && File.Exists(monitoringObjectJson))
{
monitoringObjects = MonitoringObject.Load(monitoringObjectJson);
log.Info("Found " + monitoringObjects.Count.ToString() + " objects to monitor.");
}
if (!string.IsNullOrWhiteSpace(queryJson) && File.Exists(queryJson))
{
queries = Query.Load(queryJson);
log.Info("Found " + queries.Count.ToString() + " queries for monitoring.");
}
jsonOutFilePrefix = jsonFilePrefix;
}
public void Run()
{
log4net.Config.XmlConfigurator.Configure();
log.Info("Starting Twitter Monitor.");
Dictionary <int, int> queryDefaultSleepTimes = new Dictionary <int, int>();
Dictionary <int, int> queryCurrentSleepTimes = new Dictionary <int, int>();
List <KeyValuePair <int, DateTime> > queryList = new List <KeyValuePair <int, DateTime> >();
for (int i = 0; i < queries.Count; i++)
{
queryDefaultSleepTimes.Add(i, queries[i].sleepTime);
queryCurrentSleepTimes.Add(i, queries[i].sleepTime);
queryList.Add(new KeyValuePair <int, DateTime>(i, DateTime.Now));
}
Dictionary <int, int> objectCurrentSleepTimes = new Dictionary <int, int>();
List <KeyValuePair <int, DateTime> > objectList = new List <KeyValuePair <int, DateTime> >();
for (int i = 0; i < monitoringObjects.Count; i++)
{
if (!string.IsNullOrWhiteSpace(monitoringObjects[i].twitterAccount))
{
objectCurrentSleepTimes.Add(i, 60);
objectList.Add(new KeyValuePair <int, DateTime>(i, DateTime.Now));
}
}
string stopFile = AppDomain.CurrentDomain.BaseDirectory;
TimeSpan sleep = new TimeSpan(0, 0, sleepInterval);
if (!stopFile.EndsWith(Path.DirectorySeparatorChar.ToString()))
{
stopFile += Path.DirectorySeparatorChar.ToString();
}
stopFile += "stop";
List <Tweet> currTweetList = new List <Tweet>();
int counter = 0;
bool firstSave = true;
while (!File.Exists(stopFile))//A whacky way to stop a programm, but a very useful one!
{
queryList.Sort((x, y) => x.Value.CompareTo(y.Value));
objectList.Sort((x, y) => x.Value.CompareTo(y.Value));
if (DateTime.Now > queryList[0].Value)
{
bool checkMessageBody = !queries[queryList[0].Key].query.StartsWith("@");
List <Tweet> tweets = tr.GetTweetsForQuery(queries[queryList[0].Key].query, queries[queryList[0].Key].lastId, checkMessageBody);
if (tweets.Count > 0)
{
log.Info("Found " + tweets.Count.ToString() + " tweets for query " + queries[queryList[0].Key].id.ToString() + ".");
decimal lastId = 0;
foreach (Tweet t in tweets)
{
t.monitoringObjectId1 = queries[queryList[0].Key].monitoringObjectId1;
t.monitoringObjectId2 = queries[queryList[0].Key].monitoringObjectId2;
t.queryId = queries[queryList[0].Key].id;
currTweetList.Add(t);
counter++;
if (t.tweetId > lastId)
{
lastId = t.tweetId;
}
}
if (lastId > 0)
{
queries[queryList[0].Key].lastId = lastId;
}
queryCurrentSleepTimes[queryList[0].Key] = queryDefaultSleepTimes[queryList[0].Key];
log.Info("Resetting sleep time for query " + queries[queryList[0].Key].id.ToString() + " to " + queryCurrentSleepTimes[queryList[0].Key] + ".");
}
else
{
if (queryCurrentSleepTimes[queryList[0].Key] < 2880) //48 hours are the maximum to wait!B
{
queryCurrentSleepTimes[queryList[0].Key] *= 2;
}
log.Info("Increasing sleep time for query " + queries[queryList[0].Key].id.ToString() + " to " + queryCurrentSleepTimes[queryList[0].Key] + ".");
}
queryList[0] = new KeyValuePair <int, DateTime>(queryList[0].Key, DateTime.Now.AddMinutes(queryCurrentSleepTimes[queryList[0].Key]));
}
System.Threading.Thread.Sleep(sleep);
if (DateTime.Now > objectList[0].Value)
{
List <Tweet> tweets = new List <Tweet>();
if (monitoringObjects[objectList[0].Key].twitterAccount.Contains(";"))
{
char[] sep = new char[] { ';' };
string[] accounts = monitoringObjects[objectList[0].Key].twitterAccount.Split(sep, StringSplitOptions.RemoveEmptyEntries);
foreach (string account in accounts)
{
tweets.AddRange(tr.GetTweetsForUser(account, monitoringObjects[objectList[0].Key].lastId));
}
}
else
{
tweets = tr.GetTweetsForUser(monitoringObjects[objectList[0].Key].twitterAccount, monitoringObjects[objectList[0].Key].lastId);
}
if (tweets.Count > 0)
{
log.Info("Found " + tweets.Count.ToString() + " tweets for user " + monitoringObjects[objectList[0].Key].twitterAccount + ".");
decimal lastId = 0;
foreach (Tweet t in tweets)
{
t.monitoringObjectId1 = monitoringObjects[objectList[0].Key].id;
t.monitoringObjectId2 = monitoringObjects[objectList[0].Key].parentId;
t.queryId = null;
currTweetList.Add(t);
counter++;
if (t.tweetId > lastId)
{
lastId = t.tweetId;
}
}
if (lastId > 0)
{
monitoringObjects[objectList[0].Key].lastId = lastId;
}
objectCurrentSleepTimes[objectList[0].Key] = 60;
log.Info("Resetting sleep time for user " + monitoringObjects[objectList[0].Key].twitterAccount + " to " + 60 + ".");
}
else
{
if (objectCurrentSleepTimes[objectList[0].Key] < 2880) //48 hours are the maximum to wait!
{
objectCurrentSleepTimes[objectList[0].Key] *= 2;
}
log.Info("Increasing sleep time for user " + monitoringObjects[objectList[0].Key].twitterAccount + " to " + objectCurrentSleepTimes[objectList[0].Key].ToString() + ".");
}
objectList[0] = new KeyValuePair <int, DateTime>(objectList[0].Key, DateTime.Now.AddMinutes(objectCurrentSleepTimes[objectList[0].Key]));
System.Threading.Thread.Sleep(sleep);
}
if (counter >= 1000 && firstSave && tweetSaveInterval > 1000)
{
firstSave = false;
DateTime time = DateTime.Now;
string date_suffix = time.Day.ToString() + "-" + time.Month.ToString() + "-" + time.Year.ToString() + "-" + time.Hour.ToString() + "-" + time.Minute.ToString() + "-" + time.Second.ToString();
Tweet.SaveTweets(currTweetList, jsonOutFilePrefix + "-" + date_suffix + ".json");
Query.SaveQueries(queries, jsonOutFilePrefix + "-queries-" + date_suffix + ".json");
MonitoringObject.SaveMonitoringObjects(monitoringObjects, jsonOutFilePrefix + "-monitoring-objects-" + date_suffix + ".json");
currTweetList.Clear();
}
else if (currTweetList.Count > tweetSaveInterval)
{
DateTime time = DateTime.Now;
string date_suffix = time.Day.ToString() + "-" + time.Month.ToString() + "-" + time.Year.ToString() + "-" + time.Hour.ToString() + "-" + time.Minute.ToString() + "-" + time.Second.ToString();
Tweet.SaveTweets(currTweetList, jsonOutFilePrefix + "-" + date_suffix + ".json");
Query.SaveQueries(queries, jsonOutFilePrefix + "-queries-" + date_suffix + ".json");
MonitoringObject.SaveMonitoringObjects(monitoringObjects, jsonOutFilePrefix + "-monitoring-objects-" + date_suffix + ".json");
currTweetList.Clear();
}
}
DateTime finalTime = DateTime.Now;
string finalDateSuffix = finalTime.Day.ToString() + "-" + finalTime.Month.ToString() + "-" + finalTime.Year.ToString() + "-" + finalTime.Hour.ToString() + "-" + finalTime.Minute.ToString() + "-" + finalTime.Second.ToString();
Tweet.SaveTweets(currTweetList, jsonOutFilePrefix + "-" + finalDateSuffix + ".json");
Query.SaveQueries(queries, jsonOutFilePrefix + "-queries-" + finalDateSuffix + ".json");
MonitoringObject.SaveMonitoringObjects(monitoringObjects, jsonOutFilePrefix + "-monitoring-objects-" + finalDateSuffix + ".json");
currTweetList.Clear();
log.Info("Stopping Twitter Monitor.");
}