//write to WebAPI service if this is to be used again public override void Process(IDroneDataComponent component) { TwitterDataComponent twitterDataComponent = component as TwitterDataComponent; //if (!Object.Equals(twitterDataComponent, null)) // SaveTrendData(twitterDataComponent); }
public void TwitterFollower_GetData() { TwitterFollower t = new TwitterFollower(new TwitterFollowerDataSource()); t.GetData(t.Context); TwitterDataComponent tdc = t.DroneDataComponent as TwitterDataComponent; }
/// <summary> /// Main method that gathers data /// </summary> /// <param name="context">iDroneContext</param> public override void GetData(object context) { try { BaseContext cont = context as BaseContext; if (!Object.Equals(cont, null)) { SetContextStatus("processing", cont); SetNextRunIntervalByNode(ProcessorName, cont); if (XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName)) { //do work WriteToUsageLogFile("TwitterTrend.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetTwitterTrendsAllPlaces"); TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent; _dataComponent.TrendRootList = GetTwitterTrendsAllPlaces(); //DroneDataSource.Process(_dataComponent); //add via webapi call if this is needed again WriteToUsageLogFile("TwitterTrend.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetTwitterTrendsAllPlaces"); } SetContextStatus("waiting", cont); } RaiseProcessingCompletedEvent(new EventArgs()); } catch (Exception e) { ExceptionExtensions.LogError(e, "Drone.Twitter.Components.TwitterTrend.GetData()"); } }
public override void Process(IDroneDataComponent component) { TwitterDataComponent twitterDataComponent = component as TwitterDataComponent; if (!Object.Equals(twitterDataComponent.TwitterUserList, null)) { SaveFollowerData(twitterDataComponent.TwitterUserList); } }
public void Twitter_GetData() { Drone.Twitter.Components.Twitter t = new Twitter.Components.Twitter(new TwitterDataSource()); t.GetData(t.Context); TwitterDataComponent tdc = t.DroneDataComponent as TwitterDataComponent; Assert.AreEqual(Drone.Twitter.Components.Twitter.ComponentTypeName, "TwitterComponent"); //Assert.IsNotNull(tdc.AllTwitterMentions); }
public Twitter(IDroneDataSource datasource) : base(datasource) { DroneDataSource = datasource; DroneDataComponent = new TwitterDataComponent(); Context = new TwitterContext { Status = "waiting", TimeOfStatus = DateTime.Now, DurationPreviousStatus = TimeSpan.FromSeconds(0) }; }
public override void Process(IDroneDataComponent component) { TwitterDataComponent twitterDataComponent = component as TwitterDataComponent; if (!Object.Equals(twitterDataComponent.KeywordStatus, null)) { SendAllMentions(twitterDataComponent.KeywordStatus); } }
public void Twitter_GetTrendsForAllPlaces() { TwitterTrend tt = new TwitterTrend(new TwitterTestDataSource()); tt.GetData(tt.Context); TwitterDataComponent tdc = tt.DroneDataComponent as TwitterDataComponent; Assert.IsNotNull(tdc.TrendRootList); Assert.AreNotEqual(0, tdc.TrendRootList.Count()); }
public void TwitterUser_GetData() { TwitterUser t = new TwitterUser(new TwitterUserDataSource()); t.GetData(t.Context); TwitterDataComponent tdc = t.DroneDataComponent as TwitterDataComponent; Assert.IsNull(tdc.TwitterUserList); //first time through will not run... don't need to collect data just set the next run interval Assert.IsNotNull(t.Context.NextRun); t.GetData(t.Context); Assert.IsNotNull(tdc.TwitterUserList); Assert.IsTrue(tdc.TwitterUserList[0].followers_count > 0); }
public void Twitter_GetAvailablePlaces() { Drone.Twitter.Components.Twitter t = new Twitter.Components.Twitter(new TwitterTestDataSource()); TwitterDataComponent tdc = t.DroneDataComponent as TwitterDataComponent; tdc.TwitterAvailablePlaces = Drone.Twitter.Components.Twitter.GetAvailablePlaces(); foreach (var item in tdc.TwitterAvailablePlaces.OrderBy(x => x.country)) { Debug.WriteLine(string.Format("Country:{0}. Name:{1}. WoeID:{3} Type:{2}", item.country, item.name, item.placeType.name, item.woeid)); } Assert.IsNotNull(tdc.TwitterAvailablePlaces); }
private static void SaveFollowerData(TwitterDataComponent twitterDataComponent) { if (!Object.Equals(twitterDataComponent.TwitterUserList, null)) { foreach (User user in twitterDataComponent.TwitterUserList) { try { if (user.id != "0" && user.followers_count != 0) { DataFactory.ExecuteNonQuery("TwitterFollowersInsert", new KeyValuePair <string, object>("@HandleID", user.id), new KeyValuePair <string, object>("@Followers", user.followers_count), new KeyValuePair <string, object>("@ListCount", user.listed_count), new KeyValuePair <string, object>("@CreateDate", DateTime.Today.AddDays(-1))); } else { ExceptionExtensions.LogWarning(new ArgumentNullException("twitter_user") , "TwitterDataSource.SaveFollowerData" , string.Format("user id: {0} - followers: {1} - list count: {2}" , user.id , user.followers_count , user.listed_count)); } } catch (Exception e) { if (e.Message.Contains("deadlocked")) { SaveFollowerData(twitterDataComponent); ExceptionExtensions.LogInformation("TwitterDataSource.SaveFollowerData", "Deadlock encountered, trying again"); } else { ExceptionExtensions.LogError(e, "TwitterDataSource.SaveFollowerData", string.Format("user: {0}", user.id)); //if tempdb full or other critical db error, re-throw if (Utility.IsCriticalDBError(e)) { throw; } } } } } }
public override void ProcessMessage(object sender, MessageEventArgs args) { bool handled = false; try { string msg = Encoding.UTF8.GetString(args.Message.BodyStream.ToByteArray()); KeywordStatus ks = Utility.DeserializeXMLString <KeywordStatus>(msg); if (!Object.Equals(ks, null)) { TwitterDataComponent tdc = new TwitterDataComponent(); tdc.KeywordStatus = ks; handled = true; FireMessageProcessingEvent(); QueueComponentDataSource.Process(tdc); FireMessageProcessedEvent(); } List <User> users = Utility.DeserializeXMLString <List <User> >(msg); if (!Object.Equals(null, users)) { TwitterDataComponent tdc = new TwitterDataComponent(); tdc.TwitterUserList = users; handled = true; FireMessageProcessingEvent(); QueueComponentDataSource.Process(tdc); FireMessageProcessedEvent(); } } catch (Exception e) { ExceptionExtensions.LogError(e, "QueueTwitter.ProcessMessage"); if (Utility.IsCriticalDBError(e)) { FireShuttingDownEvent(); } if (handled) { FireMessageProcessedEvent(); } } }
private static void SaveTrendData(TwitterDataComponent twitterDataComponent) { if (!Object.Equals(twitterDataComponent.TrendRootList, null) && (twitterDataComponent.TrendRootList.Count > 0)) { foreach (List <TrendRoot> trendRootList in twitterDataComponent.TrendRootList) { if (trendRootList.Count > 0) { TrendRoot trendRoot = trendRootList[0]; foreach (Trend trend in trendRoot.trends) { try { DataFactory.ExecuteNonQuery("TwitterTrendInsert", new KeyValuePair <string, object>("@trendQuery", Uri.UnescapeDataString(trend.query)), new KeyValuePair <string, object>("@WoeID", trendRoot.locations[0].woeid), new KeyValuePair <string, object>("@CreateDate", DateTime.ParseExact(trendRoot.created_at, @"yyyy-MM-dd\THH:mm:ss\Z", CultureInfo.InvariantCulture).AddHours(-7))); } catch (Exception e) { if (e.Message.Contains("deadlocked")) { SaveTrendData(twitterDataComponent); ExceptionExtensions.LogInformation("TwitterTrendDataSource.SaveTrendData", "Deadlock encountered, trying again"); } else { ExceptionExtensions.LogError(e, "TwitterTrendDataSource.SaveTrendData", string.Format("woeid: {0}", trendRoot.locations.Count > 0 ? trendRoot.locations[0].woeid.ToString() : "no woeid")); //if tempdb full or other critical db error, re-throw if (Utility.IsCriticalDBError(e)) { throw; } } } } } } } }
/// <summary> /// Main method that gathers data /// </summary> /// <param name="context">iDroneContext</param> public override void GetData(object context) { try { BaseContext cont = context as BaseContext; if (!Object.Equals(cont, null)) { SetContextStatus("processing", cont); DateTime tempNextRun = cont.NextRun; SetNextRunIntervalByNode(ProcessorName, cont); if (XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName)) { WriteToUsageLogFile("TwitterFollower.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetTwitterFollowers"); //wait 5 min for other twitter processes to finish so we don't get rate limited Thread.Sleep(300000); //do work TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent; UserManager um = new UserManager(); OAuthTokens oat = Utility.GetOAuthToken(Xml); _dataComponent.TwitterUserList = um.GetAllFollowers(oat); DroneDataSource.Process(_dataComponent); WriteToUsageLogFile("TwitterFollower.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetTwitterFollowers"); } SetContextStatus("waiting", cont); } RaiseProcessingCompletedEvent(new EventArgs()); } catch (Exception e) { ExceptionExtensions.LogError(e, "Drone.Twitter.Components.TwitterFollower.GetData()"); } }
public override void Process(IDroneDataComponent component) { TwitterDataComponent twitterDataComponent = component as TwitterDataComponent; try { if (!Object.Equals(twitterDataComponent.KeywordStatus, null)) { SaveStatus(twitterDataComponent.KeywordStatus); } if (!Object.Equals(twitterDataComponent.TwitterUserList, null)) { SaveUsers(twitterDataComponent.TwitterUserList); //followers } } catch (Exception e) { twitterDataComponent.SaveFailure = true; ExceptionExtensions.LogError(e, "TwitterDataSource_Process"); } }
/// <summary> /// Main method that gathers data /// </summary> /// <param name="context">iDroneContext</param> public override void GetData(object context) { try { BaseContext cont = context as BaseContext; if (!Object.Equals(cont, null)) { SetContextStatus("processing", cont); DateTime tempNextRun = cont.NextRun; SetNextRunIntervalByNode(ProcessorName, cont); if (DateTime.Compare(tempNextRun, DateTime.MinValue) != 0 && XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName)) { WriteToUsageLogFile("TwitterUser.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetTwitterUsers Competitors"); //do work TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent; _dataComponent.TwitterUserList = GetTwitterUsers(); DroneDataSource.Process(_dataComponent); WriteToUsageLogFile("TwitterUser.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetTwitterUsers Competitors"); } SetContextStatus("waiting", cont); } RaiseProcessingCompletedEvent(new EventArgs()); } catch (Exception e) { ExceptionExtensions.LogError(e, "Drone.Twitter.Components.TwitterUser.GetData()"); } }
public TwitterTrend() : base() { DroneDataSource = new TwitterTrendDataSource(); DroneDataComponent = new TwitterDataComponent(); }
public TwitterFollower(IDroneDataSource datasource) : base(datasource) { DroneDataSource = datasource; DroneDataComponent = new TwitterDataComponent(); }
public TwitterFollower() : base() { DroneDataSource = new TwitterFollowerDataSource(); DroneDataComponent = new TwitterDataComponent(); }
/// <summary> /// Get all tweets with "GoDaddy" in them. /// 100 per page, up to 15 pages if we loop and make seperate calls for each page /// </summary> internal Dictionary <int, List <Status> > GetAllMentionsByQuery(TwitterContext cont) { Dictionary <int, List <Status> > allTwitterMentions = new Dictionary <int, List <Status> >(); if (XMLUtility.IsComponentEnabled(Xml, ProcessorName)) { int countPerPage; int pageCount; TwitterDataSource data = new TwitterDataSource(); TweetManager tm = new TweetManager(); XMLUtility.GetPageResultCounts(Xml, ProcessorName, out countPerPage, out pageCount, 100, 3); List <Keyword> queries = data.GetTweetQueries(); bool useSinceId = XMLUtility.UseSinceId(Xml, ProcessorName); //create backup of current keys in case of failure at db level lock (cont) cont.prevRunLatestTweetIDs = cont.LatestTweetIDs.ToDictionary(entry => entry.Key, entry => entry.Value); foreach (Keyword item in queries) { try { long sinceId = 0; if (useSinceId) { //get the last recorded id for this query and use it if (!Object.Equals(cont, null)) { lock (cont) { if (cont.LatestTweetIDs.ContainsKey(item.KeywordId)) { cont.LatestTweetIDs.TryGetValue(item.KeywordId, out sinceId); } } } } //call the mention object in the API wrapper TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent; KeywordStatus ks = new KeywordStatus(); ks.KeywordId = item.KeywordId; ks.StatusList = tm.GetTweetsByQuery(countPerPage, item.KeywordValue + (sinceId > 0 ? "&since_id=" + sinceId : string.Empty), Utility.GetOAuthToken(Xml)); allTwitterMentions.Add(ks.KeywordId, ks.StatusList); _dataComponent.KeywordStatus = ks; DroneDataSource.Process(_dataComponent); //if there was a failure saving to the db, reset the since id to gather and try again if (_dataComponent.SaveFailure) { lock (cont) cont.LatestTweetIDs = cont.prevRunLatestTweetIDs.ToDictionary(entry => entry.Key, entry => entry.Value); } //get the last id for this query and store it if (!Object.Equals(cont, null) && allTwitterMentions.ContainsKey(item.KeywordId) && allTwitterMentions[item.KeywordId].Count > 0) { long latestID; long.TryParse(allTwitterMentions[item.KeywordId][0].id.ToString(), out latestID); lock (cont) { if (cont.LatestTweetIDs.ContainsKey(item.KeywordId)) { cont.LatestTweetIDs[item.KeywordId] = latestID; } else { cont.LatestTweetIDs.Add(item.KeywordId, latestID); } } } } catch (Exception e) { ExceptionExtensions.LogError(e, "Twitter.GetAllMentionsByQuery()", "Keyword name: " + item.KeywordName); } } } return(allTwitterMentions); }