public void Execute() { int count = 0; WebResponseBuilder responseBuilder = new WebResponseBuilder(); string twitter_api_username = ConfigurationManager.AppSettings["twitter_user"]; string twitter_api_password = ConfigurationManager.AppSettings["twitter_pass"]; responseBuilder.UseCGICredentials(twitter_api_username, twitter_api_password); TwitterStreamStatusProvider provider = new TwitterStreamStatusProvider(responseBuilder); provider.YieldThisMany = NUM_STATUSES_TO_PULL; Console.Out.WriteLine("Parsing english model trigrams from source data in 'english_data' directory"); EnglishStatusProvider englishProvider = new EnglishStatusProvider(provider, "english_data"); englishProvider.Threshold = ENGLISH_THRESHOLD; Console.Out.WriteLine("About to start reading from twitter - up to " + NUM_STATUSES_TO_PULL + " statuses."); XtractDataContext db = new XtractDataContext(); foreach (TwitterStatus status in englishProvider.GetMessages()) { string screen_name = status.user.screen_name; var existing = db.Twusers.Where(u => u.screen_name == screen_name); if (existing.Count() == 0) { Twuser user = Twuser.From(status.user); user.english_similarity = status.english_similarity; db.Twusers.InsertOnSubmit(user); } else { Twuser user = existing.First(); if (user.english_similarity < status.english_similarity) { user.english_similarity = status.english_similarity; } } Tweet tweet = new Tweet(); tweet.english_similarity = status.english_similarity; tweet.screen_name = status.user.screen_name; tweet.text = status.text; tweet.twitter_id = status.id; tweet.sample_reason = SampleReason.sample_stream.ToString(); db.Tweets.InsertOnSubmit(tweet); db.SubmitChanges(); if (count++ > UPDATE_EVERY) { Console.Out.WriteLine("Wrote " + count + " tweets."); } } }
public void Execute() { int overallCount = 0; using (XtractDataContext dbRead = new XtractDataContext()) { dbRead.ObjectTrackingEnabled = false; foreach (Twuser user in from twuser in dbRead.Twusers where ((twuser.id % 10 == 0) && (twuser.last_parse_status==null)) orderby twuser.english_similarity descending select twuser) { string screen_name = user.screen_name; if (CheckForExisting(screen_name)) continue; OAuthTwitterResponseBuilder oAuthTwitter = new OAuthTwitterResponseBuilder(); UserStatusProvider provider = new UserStatusProvider(oAuthTwitter, user.screen_name); Console.Out.WriteLine("About to request data for @" + user.screen_name); DateTime nowish = DateTime.UtcNow; int count = 0; using (XtractDataContext dbWrite = new XtractDataContext()) { foreach (TwitterStatus status in provider.GetMessages()) { long twitter_id = status.id.Value; var existing = dbWrite.Tweets.Where(tw => tw.twitter_id == twitter_id); if (existing.Count() != 0) continue; Tweet tweet = new Tweet(); //tweet.english_similarity = status.english_similarity; tweet.screen_name = status.user.screen_name; tweet.text = status.text; tweet.twitter_id = status.id; DateTime createdAt = DateUtils.UTCDateTimeFromTwitterTimeStamp(status.created_at); tweet.date_tweeted = DateUtils.ISO8601TimeStampFromUTCDateTime(createdAt); tweet.date_scanned = DateUtils.ISO8601TimeStampFromUTCDateTime(nowish); tweet.sample_reason = SampleReason.user_data.ToString(); dbWrite.Tweets.InsertOnSubmit(tweet); count++; overallCount++; if (overallCount % 100 == 0) { dbWrite.SubmitChanges(); Console.Out.WriteLine(overallCount + " tweets saved so far"); } } } Console.Out.WriteLine(count + " tweets found for " + screen_name); using (XtractDataContext dbWrite = new XtractDataContext()) { var lastUser = (from twuser in dbWrite.Twusers where twuser.screen_name.Equals(screen_name) select twuser).First(); DownloadStatus lastStatus = (count==0)? DownloadStatus.NoDataFound : DownloadStatus.DataDownloaded; lastUser.last_parse_status = lastStatus.ToString(); dbWrite.SubmitChanges(); } } } }