public async Task <TwitterData> GetLatestTwitterData(int searchRequestID) { TwitterData lastTwitterData = null; try { using (var db = new BEMainDBContext()) { var execsBySreq = db.BEExecutions .Where(exec => (exec.SearchRequestID == searchRequestID)) .Include(exec => exec.TheMinerData) .Where(exec => exec.TheMinerData.Count != 0) .OrderBy(exec => exec.StartedOn) .ToList(); var lastMinerData = execsBySreq.Select(exec => exec.TheMinerData .Where(md => md.TheSource == SourceOption.Twitter) .Last()) .Where(md => md != null) .ToList() .Last(); lastTwitterData = db.TwitterData .Where(td => td.TheExecutionID == lastMinerData.TheExecutionID) .ToList() .FirstOrDefault(); } } catch { lastTwitterData = null; } return(lastTwitterData); }
public void TwitterUpdateSnapshotsInDb() { List <TwitterAccount> twitterAccounts = new List <TwitterAccount>(); twitterAccounts.Add(new TwitterAccount { Id = "MapleLeafs", FriendlyName = "Toronto Maple Leafs" }); twitterAccounts.Add(new TwitterAccount { Id = "phoenixcoyotes", FriendlyName = "Phoenix Coyotes" }); List <TwitterSnapshot> twitterAccountSnapshots = TwitterData.UpdateSnapshotsInDb(twitterAccounts); Assert.AreEqual(2, twitterAccountSnapshots.Count, "There are 2 snapshots"); // Call Update again to make sure dupes are added twitterAccountSnapshots = TwitterData.UpdateSnapshotsInDb(twitterAccounts); // Make sure we have the right items in the db using (SportsDataContext db = new SportsDataContext()) { List <TwitterSnapshot> snapshotsFromToday = (from s in db.TwitterSnapshot_DbSet.Include(x => x.TwitterAccount) where DbFunctions.TruncateTime(s.DateOfSnapshot) == DbFunctions.TruncateTime(DateTime.UtcNow) orderby s.TwitterAccount.Id select s).ToList(); Assert.AreEqual(2, snapshotsFromToday.Count, "There are 2 snapshots, not 4"); Assert.AreEqual(DateTime.UtcNow.Date, snapshotsFromToday[0].DateOfSnapshot.Date, "The snapshots are from today"); Assert.AreEqual(snapshotsFromToday[0].DateOfSnapshot.Date, snapshotsFromToday[1].DateOfSnapshot.Date, "The snapshots are equal"); Assert.AreEqual("MapleLeafs", snapshotsFromToday[0].TwitterAccount.Id, "The first snapshot is from MapleLeafs"); Assert.AreEqual("phoenixcoyotes", snapshotsFromToday[1].TwitterAccount.Id, "The first snapshot is from phoenixcoyotes"); } }
public async Task PostTwitterData(TwitterData newTwitterData) { using (var db = new BEMainDBContext()) { db.TwitterData.Add(newTwitterData); await db.SaveChangesAsync(); } }
private void TagHandler(object sender, IrcEventArgs e) { string sendto = (string.IsNullOrEmpty(e.Data.Channel)) ? e.Data.Nick : e.Data.Channel; if (e.Data.MessageArray[0].ToLower() == "!+tag") { var tag = new TwitterTag { Tag = e.Data.MessageArray[1] }; TwitterData.TwitterTags.InsertOnSubmit(tag); TwitterData.SubmitChanges(); BotMethods.SendMessage(SendType.Message, sendto, "Automatic Search activated: " + "http://search.twitter.com/search?q=" + HttpUtility.UrlEncode(e.Data.MessageArray[1])); } if (e.Data.MessageArray[0].ToLower() == "!-tag") { var tag = TwitterData.TwitterTags.Where(t => t.Tag == e.Data.MessageArray[1]).FirstOrDefault(); if (tag != null) { TwitterData.TwitterTags.DeleteOnSubmit(tag); TwitterData.SubmitChanges(); } } if (e.Data.MessageArray[0].ToLower() == "!tags") { foreach (string line in TwitterData.TwitterTags.Select(t => t.Tag).ToLines(350)) { BotMethods.SendMessage(SendType.Message, sendto, line); } } }
public async Task SaveTwitterDataAsync() { TwitterData data = new TwitterData(); data.AccountList = new List <TwitterAccount>(this.AccountList); data.TimelineTabList = new List <TimelineTab>(this.TimelineListTab); await SaveData.SaveTwitterDataAsync(data); }
public async Task <TwitterData> GetTwitterData(int executionID) { TwitterData twitterData = null; using (var db = new BEMainDBContext()) { twitterData = db.TwitterData .Where(td => td.TheExecutionID == executionID) .ToList() .FirstOrDefault(); } return(twitterData); }
public ActionResult Social() { Dictionary <string, Exception> results = new Dictionary <string, Exception>(); if (this.IsKeyValid()) { results.Add("FacebookData", this.Update(delegate() { FacebookData.UpdateAllSnapshotsInDb(); })); results.Add("TwitterData", this.Update(delegate() { TwitterData.UpdateAllSnapshotsInDb(); })); } return(Json(results, JsonRequestBehavior.AllowGet)); }
public List <TwitterObject> GetTwitter(string param) { string url = $"https://api.twitter.com/1.1/search/tweets.json?q={param}&result_type=recent&tweet_mode=extended&count=100"; Auth.SetUserCredentials(ConfigurationManager.AppSettings["CustomerKey"], ConfigurationManager.AppSettings["CustomerKeySecret"], ConfigurationManager.AppSettings["AcessToken"], ConfigurationManager.AppSettings["AcessTokenSecret"]); var results = TwitterAccessor.GetQueryableJsonObjectFromGETQuery(url).ToString(); TwitterData twitterData = JsonConvert.DeserializeObject <TwitterData>(results); return(twitterData.twitterData.FindAll(x => x.retweeted == null)); }
/*prepares a string from the CSV(scrub and split), turns it into object for mongo */ private TwitterData ConvertToTwitterData(string indata) { TwitterData twitterData = new TwitterData(); var lineFromFile = indata.Replace("\"", "").Split(','); twitterData.Standford_id = long.Parse((lineFromFile[1])); twitterData.Polarity = int.Parse(lineFromFile[0]); twitterData.LongDate = lineFromFile[2]; twitterData.Query = lineFromFile[3]; twitterData.UserName = lineFromFile[4]; for (int i = 5; i < lineFromFile.Length; i++) { twitterData.Text += lineFromFile[i]; } return(twitterData); }
/// <summary> /// キー作成 /// </summary> private void createKey() { var twitter = new TwitterData(); twitter.Open(); twitter.GetAllTimeLineData("irof"); localKeyA = twitter.GetRandomStatus(); localKeyB = twitter.GetRandomStatus(localKeyA.StatusID); //画面描画 SetWebBrowserA("https://twitter.com/irof/statuses/" + localKeyA.StatusID); SetWebBrowserB("https://twitter.com/irof/statuses/" + localKeyB.StatusID); //公開キー作成 this.createPublicKey(); }
public TwitterData GetMyTimeline(string url, OAuthData oAuthData) { timelineUrl = url; var authenticate = new Authenticate(); TwitAuthenticateResponse twitAuthResponse = authenticate.AuthenticateMe(oAuthData.OAuthConsumerKey, oAuthData.OAuthConsumerSecret, OAuthUrl); // Do the timeline var timeLineJson = new Utility().RequstJson(timelineUrl, twitAuthResponse.token_type, twitAuthResponse.access_token); TwitterData response = new TwitterData(); response.xmls = new List <XmlDocument>(); if (timeLineJson.StartsWith("Error:")) { response.error = timeLineJson; return(response); } //List<XmlDocument> xmls = new List<XmlDocument>(); if (timeLineJson == string.Empty || timeLineJson == "[]") { return(response); } timeLineJson = timeLineJson.Substring(3, timeLineJson.Length - 6); var timeLines = timeLineJson.Split(new [] { "\"},{\"" }, System.StringSplitOptions.None); try { foreach (string line in timeLines) { response.xmls.Add(JsonConvert.DeserializeXmlNode("{\"root\":[{\"" + line + "\"}]}")); } } catch (Exception e) { Console.WriteLine(e.Message); } return(response); }
public async Task <List <TwitterUser> > PrepareUserInsert(List <String> twitterUserList) { List <TwitterUser> twitterUserData = new List <TwitterUser>(); foreach (var user in twitterUserList) { TwitterData twitterData = await scrapeTwitterUser(user); TwitterUser newTwitterUser = new TwitterUser { Id_str = twitterData.User.Id_str, Screen_name = twitterData.User.Screen_name, Description = twitterData.User.Description, Followers_count = twitterData.User.Followers_count }; twitterUserData.Add(newTwitterUser); } ; return(twitterUserData); }
/******************** Actor Logic Implementation Methods ********************/ // Called before the Miner starts its job (mainMineAsync) protected override async Task onMineBeginAsync() { // Initialize the TwitterData for this Execution, starting from where the previous execution finished BESearchRequest theSearchRequest = await GetTheSearchRequest(); TwitterData theTwitterData = await dbHandlerService.GetLatestTwitterData(theSearchRequest.ID); if (theTwitterData == null) { miningForFirstTime = true; theTwitterData = new TwitterData(theSearchRequest.ActiveExecutionID); } else { miningForFirstTime = false; theTwitterData.TheTextsNum = 0; theTwitterData.TheExecutionID = theSearchRequest.ActiveExecutionID; } await SaveTheMinerData(theTwitterData); }
public async Task RepeatTwitterApiCall(String user, String max_id) { List <TwitterData> twitterData = await scrapeTwitterData(user, max_id); //Twitter API will return only one tweet (the max_id tweet) once reach limit (3200) of tweets available if (twitterData != null && twitterData.Count > 1) { //The max_id tweet is fetched on previous call so don't include. twitterData = twitterData.Where(x => x.Id_str != max_id).ToList(); max_id = twitterData.LastOrDefault().Id_str; foreach (var tweet in twitterData) { foreach (var item in _searchTerms) { if (tweet.Full_text.ToLower().Contains(item)) { TwitterData newTwitterData = new TwitterData { Id_str = tweet.Id_str, Created_at = tweet.Created_at, Full_text = tweet.Full_text, Retweet_count = tweet.Retweet_count, Favorite_count = tweet.Favorite_count, Screen_name = tweet.User.Screen_name }; _userTweetsAll.Add(newTwitterData); break; } } } await Task.Delay(5000); await RepeatTwitterApiCall(user, max_id); } else { Console.WriteLine("Done."); } }
public void ReadTweets(string screenName, bool isFirstTime) { bool multipleFlag = false; string maxId = ""; string sinceId = ""; while (!finishedToProcessScreenName && pageCounter < 16) { pageCounter++; if (isFirstTime) { string maxIdInDB = DataContext.GetMinMessageIdForUser(screenName); decimal maxIdInt = 0; try { maxIdInt = maxIdInDB == null ? 0 : decimal.Parse(maxIdInDB) - 1; } catch (Exception e) { Console.WriteLine(e.Message); } maxId = pageCounter == 1 ? "" : "&max_id=" + maxIdInt; timelineUrl = string.Format(TimelineFormatFirstTime, screenName, IncludeRts, ExcludeReplies, maxId, Count); } else { //if first 200 rows get sinceid from DB alse use the original and the id of the oldest new tweet as maxid string id = multipleFlag == false ? (sinceId = DataContext.GetMaxMessageIdForUser(screenName)) : sinceId + "&max_id=" + maxId; timelineUrl = string.Format(TimelineFormatNotFirstTime, screenName, IncludeRts, ExcludeReplies, id, Count); } OAuthTwitterWrapper.OAuthTwitterWrapper oAuthT = new OAuthTwitterWrapper.OAuthTwitterWrapper(); TwitterData tData = oAuthT.GetMyTimeline(timelineUrl, oAuthData); List <TwitterItem> items = new List <TwitterItem>(); // convert xml to twitter item foreach (XmlDocument status in tData.xmls) { XElement xml = XElement.Load(new XmlNodeReader(status)); items.Add(new TwitterItem(xml)); statusCounter++; } if (items.Count < 200) { finishedToProcessScreenName = true; //insert rows to DB if (items.Count > 0) { DataContext.InsertTwitterItems(items); } DataContext.UpdateScreenNames_LastUpdated(currentScreenName); } //insert rows to DB else { Console.WriteLine(String.Format("sinceid = {0, 2} maxid = {1, 2} ", sinceId, maxId)); DataContext.InsertTwitterItems(items); multipleFlag = true; //get the oldest new tweet's id (maxid) maxId = items[items.Count - 1].MessageId.ToString(); DataContext.UpdateScreenNames_LastUpdated(currentScreenName); } Console.WriteLine("stutus counter = " + items.Count); authenticateMessageCounter++; if (authenticateMessageCounter >= 120) { Console.WriteLine(" authenticate Message Counter = " + authenticateMessageCounter); //DataContext.UpdateScreenNames_LastUpdated(currentScreenName); DataContext.UpdateQueriesTracking(screenName, tData.error, runId, items.Count); done = true; break; } if (tData.error == null) { tData.error = "HTTP/1.1 200 OK"; } DataContext.UpdateQueriesTracking(screenName, tData.error, runId, items.Count); } finishedFlag = true; }
protected async Task SaveTheMinerData(TwitterData theTwitterData) { await StateManager.SetStateAsync(NewStateNames.TheMinerData, theTwitterData); await SaveStateAsync(); }
/*---------- TwitterData Management ----------*/ public async Task StoreTwitterData(TwitterData newTwitterData) { await TheTwitterDataContr.PostTwitterData(newTwitterData); }
public async Task InsertTwitterData(TwitterData twitterData, String tableName) { string sql = $@"insert into {tableName} (Id_str, Created_at, Full_text, Retweet_Count, Favorite_count, Screen_name) values (@Id_str, @Created_at, @Full_text, @Retweet_Count, @Favorite_count, @Screen_name)"; await _sqlAccess.SaveData(sql, twitterData); }
public async Task Start() { var client = new TwitterClient(_credentials); // Using the sample stream var stream = client.Streams.CreateSampleStream(); try { //setting stream filter levels stream.AddLanguageFilter(LanguageFilter.English); stream.FilterLevel = Tweetinvi.Streaming.Parameters.StreamFilterLevel.Low; DateTime streamingStartedOn = DateTime.UtcNow; TwitterData _data = new TwitterData(); //set file for save twitter data if (File.Exists(_filePath)) { File.Delete(_filePath); } File.Create(_filePath).Dispose(); stream.TweetReceived += (sender, t) => { //exclude retweets if (t.Tweet.IsRetweet) { return; } //setting total tweets _data.TotalTweets++; //setting average tweet per hour / minute / second var timeDiff = DateTime.UtcNow - streamingStartedOn; _data.AverageTweetPerHour = Convert.ToInt32(_data.TotalTweets / timeDiff.TotalHours); _data.AverageTweetPerMinute = Convert.ToInt32(_data.TotalTweets / timeDiff.TotalMinutes); _data.AverageTweetPerSecond = Convert.ToInt32(_data.TotalTweets / timeDiff.TotalSeconds); if (_data.TotalTweets == 1) { _data.TopDomains = new List <string>(); _data.TopHashtags = new List <string>(); _data.TopEmojis = new List <string>(); } //setting top domains and url percentage if (t.Tweet.Urls.Count > 0) { _data.TotalTweetWithUrl++; _data.TopDomains.AddRange(t.Tweet.Urls.Select(u => u.DisplayedURL.Split("/")[0]).Distinct().ToList()); } _data.UrlPercentage = Math.Round((decimal)(_data.TotalTweetWithUrl * 100 / _data.TotalTweets), 2); //setting photo percentage if (t.Tweet.Entities.Medias.Count > 0) { _data.TotalTweetWithPhoto++; } _data.PhotoPercentage = Math.Round((decimal)(_data.TotalTweetWithPhoto * 100 / _data.TotalTweets), 2); //setting top emoji and emoji percentage var emojis = t.Tweet.FullText.Where(c => c > 255).Select(emoji => Utils.Utf8Encoder.GetString(Utils.Utf8Encoder.GetBytes(emoji.ToString()))).Distinct().ToList(); if (emojis != null) { if (emojis.Count > 0) { _data.TotalTweetWithEmoji++; _data.TopEmojis.AddRange(emojis); //_data.TopEmojis.AddRange(t.Tweet.Entities.Symbols.Select(s => s.Text).Distinct().ToList()); } _data.EmojiPercentage = Math.Round((decimal)(_data.TotalTweetWithEmoji * 100 / _data.TotalTweets), 2); } //setting top hashtags if (t.Tweet.Hashtags.Count > 0) { _data.TopHashtags.AddRange(t.Tweet.Hashtags.Select(h => h.Text).Distinct().ToList()); } //group and order by numbers _data.TopDomains = _data.TopDomains.GroupBy(domain => domain).OrderByDescending(domain => domain.Count()).Select(domain => domain.Key).Take(5).ToList(); _data.TopHashtags = _data.TopHashtags.GroupBy(hash => hash).OrderByDescending(hash => hash.Count()).Select(hash => hash.Key).Take(5).ToList(); _data.TopEmojis = _data.TopEmojis.GroupBy(emoji => emoji).OrderByDescending(emoji => emoji.Count()).Select(emoji => emoji.Key).Take(5).ToList(); string json = JsonConvert.SerializeObject(_data); if (File.Exists(_filePath)) { //write data in file File.WriteAllText(_filePath, json); } else { //stop streaming service stream.Stop(); } }; // Start await stream.StartAsync(); } catch (Exception ex) { //stop streaming service on exception stream.Stop(); throw ex; } }
public async Task SaveTwitterDataAsync() { TwitterData data = new TwitterData(); data.AccountList = new List<TwitterAccount>(this.AccountList); data.TimelineTabList = new List<TimelineTab>(this.TimelineListTab); await SaveData.SaveTwitterDataAsync(data); }
public string GetTwitterData(string screenName) { TweetDAL tDAL = new TweetDAL(); string data = tDAL.GetData(screenName); var jsonObject = JArray.Parse(data); if (jsonObject.HasValues) { string temp = (string)jsonObject[0]["user"]["profile_image_url_https"]; ProfileInfo tProfileInfo = new ProfileInfo((string)jsonObject[0]["user"]["name"], (string)jsonObject[0]["user"]["screen_name"], temp.Substring(0, (temp.Length - ((temp.Substring(temp.IndexOf("_normal."))).Length))) + temp.Substring(temp.IndexOf("_normal.") + 7), (string)jsonObject[0]["user"]["description"], (string)jsonObject[0]["user"]["followers_count"], (string)jsonObject[0]["user"]["friends_count"]); Tweets[] tData = new Tweets[10]; int count = 0; foreach (var item in jsonObject) { string Const_TwitterDateTemplate = "ddd MMM dd HH:mm:ss +ffff yyyy"; DateTime createdAt = DateTime.ParseExact((string)item["created_at"], Const_TwitterDateTemplate, new System.Globalization.CultureInfo("en-US")); string mediaUrl = "", mediaType = "", retweet = ""; if (item.SelectToken("retweeted_status") != null) { retweet = (string)item["retweeted_status"]["full_text"]; } if (item["entities"].HasValues) { if (item.SelectToken("extended_entities") != null) { if (item["extended_entities"].SelectToken("media") != null) { if ((string)item["extended_entities"]["media"][0]["type"] == "photo") { mediaType = "photo"; mediaUrl = (string)item["extended_entities"]["media"][0]["media_url_https"]; } else if ((string)item["extended_entities"]["media"][0]["type"] == "video") { mediaType = "video"; string[] url = new string[4]; int rate = 0, j = 0; JArray tempArr = (JArray)item["extended_entities"]["media"][0]["video_info"]["variants"]; for (int i = 0; i < tempArr.Count; i++) { if ((string)item["extended_entities"]["media"][0]["video_info"]["variants"][i]["content_type"] == "video/mp4") { url[i] = (string)item["extended_entities"]["media"][0]["video_info"]["variants"][i]["url"]; if (rate == 0) { rate = (int)item["extended_entities"]["media"][0]["video_info"]["variants"][i]["bitrate"]; j = i; } else { if (rate > (int)item["extended_entities"]["media"][0]["video_info"]["variants"][i]["bitrate"]) { rate = (int)item["extended_entities"]["media"][0]["video_info"]["variants"][i]["bitrate"]; j = i; } } } } mediaUrl = (string)item["extended_entities"]["media"][0]["video_info"]["variants"][j]["url"]; } } } } if (retweet != "") { tData[count] = new Tweets(retweet, createdAt.ToString(), (string)item["retweet_count"], mediaType, mediaUrl); } else { tData[count] = new Tweets((string)item["full_text"], createdAt.ToString(), (string)item["retweet_count"], mediaType, mediaUrl); } count++; } TwitterData tProfile = new TwitterData(tProfileInfo, tData); var tProfileJson = new JavaScriptSerializer().Serialize(tProfile); return(tProfileJson); } else { return("{}"); } }