//write to WebAPI service if this is to be used again
public override void Process(IDroneDataComponent component)
{
TwitterDataComponent twitterDataComponent = component as TwitterDataComponent;
//if (!Object.Equals(twitterDataComponent, null))
// SaveTrendData(twitterDataComponent);
}
public void TwitterFollower_GetData()
{
TwitterFollower t = new TwitterFollower(new TwitterFollowerDataSource());
t.GetData(t.Context);
TwitterDataComponent tdc = t.DroneDataComponent as TwitterDataComponent;
}
/// <summary>
/// Main method that gathers data
/// </summary>
/// <param name="context">iDroneContext</param>
public override void GetData(object context)
{
try
{
BaseContext cont = context as BaseContext;
if (!Object.Equals(cont, null))
{
SetContextStatus("processing", cont);
SetNextRunIntervalByNode(ProcessorName, cont);
if (XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName))
{
//do work
WriteToUsageLogFile("TwitterTrend.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetTwitterTrendsAllPlaces");
TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent;
_dataComponent.TrendRootList = GetTwitterTrendsAllPlaces();
//DroneDataSource.Process(_dataComponent);
//add via webapi call if this is needed again
WriteToUsageLogFile("TwitterTrend.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetTwitterTrendsAllPlaces");
}
SetContextStatus("waiting", cont);
}
RaiseProcessingCompletedEvent(new EventArgs());
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Drone.Twitter.Components.TwitterTrend.GetData()");
}
}
public override void Process(IDroneDataComponent component)
{
TwitterDataComponent twitterDataComponent = component as TwitterDataComponent;
if (!Object.Equals(twitterDataComponent.TwitterUserList, null))
{
SaveFollowerData(twitterDataComponent.TwitterUserList);
}
}
public void Twitter_GetData()
{
Drone.Twitter.Components.Twitter t = new Twitter.Components.Twitter(new TwitterDataSource());
t.GetData(t.Context);
TwitterDataComponent tdc = t.DroneDataComponent as TwitterDataComponent;
Assert.AreEqual(Drone.Twitter.Components.Twitter.ComponentTypeName, "TwitterComponent");
//Assert.IsNotNull(tdc.AllTwitterMentions);
}
public Twitter(IDroneDataSource datasource)
: base(datasource)
{
DroneDataSource = datasource;
DroneDataComponent = new TwitterDataComponent();
Context = new TwitterContext {
Status = "waiting", TimeOfStatus = DateTime.Now, DurationPreviousStatus = TimeSpan.FromSeconds(0)
};
}
public override void Process(IDroneDataComponent component)
{
TwitterDataComponent twitterDataComponent = component as TwitterDataComponent;
if (!Object.Equals(twitterDataComponent.KeywordStatus, null))
{
SendAllMentions(twitterDataComponent.KeywordStatus);
}
}
public void Twitter_GetTrendsForAllPlaces()
{
TwitterTrend tt = new TwitterTrend(new TwitterTestDataSource());
tt.GetData(tt.Context);
TwitterDataComponent tdc = tt.DroneDataComponent as TwitterDataComponent;
Assert.IsNotNull(tdc.TrendRootList);
Assert.AreNotEqual(0, tdc.TrendRootList.Count());
}
public void TwitterUser_GetData()
{
TwitterUser t = new TwitterUser(new TwitterUserDataSource());
t.GetData(t.Context);
TwitterDataComponent tdc = t.DroneDataComponent as TwitterDataComponent;
Assert.IsNull(tdc.TwitterUserList); //first time through will not run... don't need to collect data just set the next run interval
Assert.IsNotNull(t.Context.NextRun);
t.GetData(t.Context);
Assert.IsNotNull(tdc.TwitterUserList);
Assert.IsTrue(tdc.TwitterUserList[0].followers_count > 0);
}
public void Twitter_GetAvailablePlaces()
{
Drone.Twitter.Components.Twitter t = new Twitter.Components.Twitter(new TwitterTestDataSource());
TwitterDataComponent tdc = t.DroneDataComponent as TwitterDataComponent;
tdc.TwitterAvailablePlaces = Drone.Twitter.Components.Twitter.GetAvailablePlaces();
foreach (var item in tdc.TwitterAvailablePlaces.OrderBy(x => x.country))
{
Debug.WriteLine(string.Format("Country:{0}. Name:{1}. WoeID:{3} Type:{2}", item.country, item.name, item.placeType.name, item.woeid));
}
Assert.IsNotNull(tdc.TwitterAvailablePlaces);
}
private static void SaveFollowerData(TwitterDataComponent twitterDataComponent)
{
if (!Object.Equals(twitterDataComponent.TwitterUserList, null))
{
foreach (User user in twitterDataComponent.TwitterUserList)
{
try
{
if (user.id != "0" && user.followers_count != 0)
{
DataFactory.ExecuteNonQuery("TwitterFollowersInsert",
new KeyValuePair <string, object>("@HandleID", user.id),
new KeyValuePair <string, object>("@Followers", user.followers_count),
new KeyValuePair <string, object>("@ListCount", user.listed_count),
new KeyValuePair <string, object>("@CreateDate", DateTime.Today.AddDays(-1)));
}
else
{
ExceptionExtensions.LogWarning(new ArgumentNullException("twitter_user")
, "TwitterDataSource.SaveFollowerData"
, string.Format("user id: {0} - followers: {1} - list count: {2}"
, user.id
, user.followers_count
, user.listed_count));
}
}
catch (Exception e)
{
if (e.Message.Contains("deadlocked"))
{
SaveFollowerData(twitterDataComponent);
ExceptionExtensions.LogInformation("TwitterDataSource.SaveFollowerData", "Deadlock encountered, trying again");
}
else
{
ExceptionExtensions.LogError(e, "TwitterDataSource.SaveFollowerData", string.Format("user: {0}", user.id));
//if tempdb full or other critical db error, re-throw
if (Utility.IsCriticalDBError(e))
{
throw;
}
}
}
}
}
}
public override void ProcessMessage(object sender, MessageEventArgs args)
{
bool handled = false;
try
{
string msg = Encoding.UTF8.GetString(args.Message.BodyStream.ToByteArray());
KeywordStatus ks = Utility.DeserializeXMLString <KeywordStatus>(msg);
if (!Object.Equals(ks, null))
{
TwitterDataComponent tdc = new TwitterDataComponent();
tdc.KeywordStatus = ks;
handled = true;
FireMessageProcessingEvent();
QueueComponentDataSource.Process(tdc);
FireMessageProcessedEvent();
}
List <User> users = Utility.DeserializeXMLString <List <User> >(msg);
if (!Object.Equals(null, users))
{
TwitterDataComponent tdc = new TwitterDataComponent();
tdc.TwitterUserList = users;
handled = true;
FireMessageProcessingEvent();
QueueComponentDataSource.Process(tdc);
FireMessageProcessedEvent();
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "QueueTwitter.ProcessMessage");
if (Utility.IsCriticalDBError(e))
{
FireShuttingDownEvent();
}
if (handled)
{
FireMessageProcessedEvent();
}
}
}
private static void SaveTrendData(TwitterDataComponent twitterDataComponent)
{
if (!Object.Equals(twitterDataComponent.TrendRootList, null) && (twitterDataComponent.TrendRootList.Count > 0))
{
foreach (List <TrendRoot> trendRootList in twitterDataComponent.TrendRootList)
{
if (trendRootList.Count > 0)
{
TrendRoot trendRoot = trendRootList[0];
foreach (Trend trend in trendRoot.trends)
{
try
{
DataFactory.ExecuteNonQuery("TwitterTrendInsert",
new KeyValuePair <string, object>("@trendQuery", Uri.UnescapeDataString(trend.query)),
new KeyValuePair <string, object>("@WoeID", trendRoot.locations[0].woeid),
new KeyValuePair <string, object>("@CreateDate", DateTime.ParseExact(trendRoot.created_at, @"yyyy-MM-dd\THH:mm:ss\Z", CultureInfo.InvariantCulture).AddHours(-7)));
}
catch (Exception e)
{
if (e.Message.Contains("deadlocked"))
{
SaveTrendData(twitterDataComponent);
ExceptionExtensions.LogInformation("TwitterTrendDataSource.SaveTrendData", "Deadlock encountered, trying again");
}
else
{
ExceptionExtensions.LogError(e, "TwitterTrendDataSource.SaveTrendData", string.Format("woeid: {0}", trendRoot.locations.Count > 0 ? trendRoot.locations[0].woeid.ToString() : "no woeid"));
//if tempdb full or other critical db error, re-throw
if (Utility.IsCriticalDBError(e))
{
throw;
}
}
}
}
}
}
}
}
/// <summary>
/// Main method that gathers data
/// </summary>
/// <param name="context">iDroneContext</param>
public override void GetData(object context)
{
try
{
BaseContext cont = context as BaseContext;
if (!Object.Equals(cont, null))
{
SetContextStatus("processing", cont);
DateTime tempNextRun = cont.NextRun;
SetNextRunIntervalByNode(ProcessorName, cont);
if (XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName))
{
WriteToUsageLogFile("TwitterFollower.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetTwitterFollowers");
//wait 5 min for other twitter processes to finish so we don't get rate limited
Thread.Sleep(300000);
//do work
TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent;
UserManager um = new UserManager();
OAuthTokens oat = Utility.GetOAuthToken(Xml);
_dataComponent.TwitterUserList = um.GetAllFollowers(oat);
DroneDataSource.Process(_dataComponent);
WriteToUsageLogFile("TwitterFollower.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetTwitterFollowers");
}
SetContextStatus("waiting", cont);
}
RaiseProcessingCompletedEvent(new EventArgs());
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Drone.Twitter.Components.TwitterFollower.GetData()");
}
}
public override void Process(IDroneDataComponent component)
{
TwitterDataComponent twitterDataComponent = component as TwitterDataComponent;
try
{
if (!Object.Equals(twitterDataComponent.KeywordStatus, null))
{
SaveStatus(twitterDataComponent.KeywordStatus);
}
if (!Object.Equals(twitterDataComponent.TwitterUserList, null))
{
SaveUsers(twitterDataComponent.TwitterUserList); //followers
}
}
catch (Exception e)
{
twitterDataComponent.SaveFailure = true;
ExceptionExtensions.LogError(e, "TwitterDataSource_Process");
}
}
/// <summary>
/// Main method that gathers data
/// </summary>
/// <param name="context">iDroneContext</param>
public override void GetData(object context)
{
try
{
BaseContext cont = context as BaseContext;
if (!Object.Equals(cont, null))
{
SetContextStatus("processing", cont);
DateTime tempNextRun = cont.NextRun;
SetNextRunIntervalByNode(ProcessorName, cont);
if (DateTime.Compare(tempNextRun, DateTime.MinValue) != 0 && XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName))
{
WriteToUsageLogFile("TwitterUser.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetTwitterUsers Competitors");
//do work
TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent;
_dataComponent.TwitterUserList = GetTwitterUsers();
DroneDataSource.Process(_dataComponent);
WriteToUsageLogFile("TwitterUser.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetTwitterUsers Competitors");
}
SetContextStatus("waiting", cont);
}
RaiseProcessingCompletedEvent(new EventArgs());
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Drone.Twitter.Components.TwitterUser.GetData()");
}
}
public TwitterTrend()
: base()
{
DroneDataSource = new TwitterTrendDataSource();
DroneDataComponent = new TwitterDataComponent();
}
public TwitterFollower(IDroneDataSource datasource)
: base(datasource)
{
DroneDataSource = datasource;
DroneDataComponent = new TwitterDataComponent();
}
public TwitterFollower()
: base()
{
DroneDataSource = new TwitterFollowerDataSource();
DroneDataComponent = new TwitterDataComponent();
}
/// <summary>
/// Get all tweets with "GoDaddy" in them.
/// 100 per page, up to 15 pages if we loop and make seperate calls for each page
/// </summary>
internal Dictionary <int, List <Status> > GetAllMentionsByQuery(TwitterContext cont)
{
Dictionary <int, List <Status> > allTwitterMentions = new Dictionary <int, List <Status> >();
if (XMLUtility.IsComponentEnabled(Xml, ProcessorName))
{
int countPerPage; int pageCount;
TwitterDataSource data = new TwitterDataSource();
TweetManager tm = new TweetManager();
XMLUtility.GetPageResultCounts(Xml, ProcessorName, out countPerPage, out pageCount, 100, 3);
List <Keyword> queries = data.GetTweetQueries();
bool useSinceId = XMLUtility.UseSinceId(Xml, ProcessorName);
//create backup of current keys in case of failure at db level
lock (cont) cont.prevRunLatestTweetIDs = cont.LatestTweetIDs.ToDictionary(entry => entry.Key, entry => entry.Value);
foreach (Keyword item in queries)
{
try
{
long sinceId = 0;
if (useSinceId)
{
//get the last recorded id for this query and use it
if (!Object.Equals(cont, null))
{
lock (cont)
{
if (cont.LatestTweetIDs.ContainsKey(item.KeywordId))
{
cont.LatestTweetIDs.TryGetValue(item.KeywordId, out sinceId);
}
}
}
}
//call the mention object in the API wrapper
TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent;
KeywordStatus ks = new KeywordStatus();
ks.KeywordId = item.KeywordId;
ks.StatusList = tm.GetTweetsByQuery(countPerPage, item.KeywordValue + (sinceId > 0 ? "&since_id=" + sinceId : string.Empty), Utility.GetOAuthToken(Xml));
allTwitterMentions.Add(ks.KeywordId, ks.StatusList);
_dataComponent.KeywordStatus = ks;
DroneDataSource.Process(_dataComponent);
//if there was a failure saving to the db, reset the since id to gather and try again
if (_dataComponent.SaveFailure)
{
lock (cont) cont.LatestTweetIDs = cont.prevRunLatestTweetIDs.ToDictionary(entry => entry.Key, entry => entry.Value);
}
//get the last id for this query and store it
if (!Object.Equals(cont, null) && allTwitterMentions.ContainsKey(item.KeywordId) && allTwitterMentions[item.KeywordId].Count > 0)
{
long latestID;
long.TryParse(allTwitterMentions[item.KeywordId][0].id.ToString(), out latestID);
lock (cont)
{
if (cont.LatestTweetIDs.ContainsKey(item.KeywordId))
{
cont.LatestTweetIDs[item.KeywordId] = latestID;
}
else
{
cont.LatestTweetIDs.Add(item.KeywordId, latestID);
}
}
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Twitter.GetAllMentionsByQuery()", "Keyword name: " + item.KeywordName);
}
}
}
return(allTwitterMentions);
}
