public void Execute()
{
int overallCount = 0;
using (XtractDataContext dbRead = new XtractDataContext())
{
dbRead.ObjectTrackingEnabled = false;
foreach (Twuser user in from twuser in dbRead.Twusers
where ((twuser.id % 10 == 0) &&
(twuser.last_parse_status==null))
orderby twuser.english_similarity descending
select twuser)
{
string screen_name = user.screen_name;
if (CheckForExisting(screen_name)) continue;
OAuthTwitterResponseBuilder oAuthTwitter = new OAuthTwitterResponseBuilder();
UserStatusProvider provider = new UserStatusProvider(oAuthTwitter, user.screen_name);
Console.Out.WriteLine("About to request data for @" + user.screen_name);
DateTime nowish = DateTime.UtcNow;
int count = 0;
using (XtractDataContext dbWrite = new XtractDataContext())
{
foreach (TwitterStatus status in provider.GetMessages())
{
long twitter_id = status.id.Value;
var existing = dbWrite.Tweets.Where(tw => tw.twitter_id == twitter_id);
if (existing.Count() != 0) continue;
Tweet tweet = new Tweet();
//tweet.english_similarity = status.english_similarity;
tweet.screen_name = status.user.screen_name;
tweet.text = status.text;
tweet.twitter_id = status.id;
DateTime createdAt = DateUtils.UTCDateTimeFromTwitterTimeStamp(status.created_at);
tweet.date_tweeted = DateUtils.ISO8601TimeStampFromUTCDateTime(createdAt);
tweet.date_scanned = DateUtils.ISO8601TimeStampFromUTCDateTime(nowish);
tweet.sample_reason = SampleReason.user_data.ToString();
dbWrite.Tweets.InsertOnSubmit(tweet);
count++;
overallCount++;
if (overallCount % 100 == 0)
{
dbWrite.SubmitChanges();
Console.Out.WriteLine(overallCount + " tweets saved so far");
}
}
}
Console.Out.WriteLine(count + " tweets found for " + screen_name);
using (XtractDataContext dbWrite = new XtractDataContext())
{
var lastUser = (from twuser in dbWrite.Twusers
where twuser.screen_name.Equals(screen_name)
select twuser).First();
DownloadStatus lastStatus = (count==0)? DownloadStatus.NoDataFound : DownloadStatus.DataDownloaded;
lastUser.last_parse_status = lastStatus.ToString();
dbWrite.SubmitChanges();
}
}
}
}
public void OAuthAuthorizeSequence()
{
string REQUEST_TOKEN = "http://api.twitter.com/oauth/request_token";
string AUTHORIZE = "http://api.twitter.com/oauth/authorize";
string ACCESS_TOKEN = "http://api.twitter.com/oauth/access_token";
//1. The application uses oauth/request_token to obtain a request token from twitter.com.
OAuthTwitterResponseBuilder twitterAuth = new OAuthTwitterResponseBuilder();
//Redirect the user to Twitter for authorization.
//Using oauth_callback for local testing.
string oAuthToken;
string response = twitterAuth.GetResponse(WebMethod.GET, REQUEST_TOKEN, String.Empty);
oAuthToken = "ERROR OCCURED NO oAuthToken in response";
if (response.Length > 0)
{
//response contains token and token secret. We only need the token.
NameValueCollection qs = HttpUtility.ParseQueryString(response);
if (qs["oauth_token"] != null)
{
oAuthToken = qs["oauth_token"];
}
}
//2. The application directs the user to oauth/authorize on twitter.com.
string authURL= AUTHORIZE + "?oauth_token=" + oAuthToken;
Console.Out.WriteLine("Please visit following url to authorize:");
Console.Out.WriteLine(authURL);
Console.ReadKey();
//3. After obtaining approval from the user, a prompt on twitter.com will display a 7 digit PIN.
//4. The user is instructed to copy this PIN and return to the appliction.
//5. The application will prompt the user to enter the PIN from step 4.
string PIN = Console.ReadLine();
//6. The application uses the PIN as the value for the oauth_verifier parameter in a call to oauth/access_token which will verify the PIN and exchange a request_token for an access_token.
twitterAuth.Token = oAuthToken;
twitterAuth.OAuthVerifier = PIN;
response = twitterAuth.GetResponse(WebMethod.GET, ACCESS_TOKEN, String.Empty);
//7. Twitter will return an access_token for the application to generate subsequent OAuth signatures.
if (response.Length > 0)
{
//Store the Token and Token Secret
NameValueCollection qs = HttpUtility.ParseQueryString(response);
if (qs["oauth_token"] != null)
{
twitterAuth.Token = qs["oauth_token"];
}
if (qs["oauth_token_secret"] != null)
{
twitterAuth.TokenSecret = qs["oauth_token_secret"];
}
}
Console.Out.WriteLine("Old key :'" + oAuthToken + "'");
Console.Out.WriteLine("New? key :'" + twitterAuth.Token + "'");
Console.Out.WriteLine("TokenSecret :'" + twitterAuth.TokenSecret + "'");
Console.Out.WriteLine("Checking it worked");
string url = "http://twitter.com/account/verify_credentials.xml";
string xml = twitterAuth.GetResponse(WebMethod.GET, url, String.Empty);
Console.Out.WriteLine(xml);
Console.Out.WriteLine("Checking it worked again");
xml = twitterAuth.GetResponse(WebMethod.GET, url, String.Empty);
Console.Out.WriteLine(xml);
Console.ReadKey();
}
public void OAuthCheck()
{
OAuthTwitterResponseBuilder twitterAuth = new OAuthTwitterResponseBuilder();
string url = "http://twitter.com/account/verify_credentials.xml";
string xml = twitterAuth.GetResponse(WebMethod.GET, url, String.Empty);
Console.Out.WriteLine(xml);
}
public void OAuthCheckRateLimit()
{
OAuthTwitterResponseBuilder twitterAuth = new OAuthTwitterResponseBuilder();
TwitterRateLimitStatus status = RateLimitCheck.GetStatus(twitterAuth);
Console.Out.WriteLine(" remaining_hits:" + status.remaining_hits + " hourly_limit:" + status.hourly_limit + " reset_time:" + status.reset_time + " reset_time_in_seconds:" + status.reset_time_in_seconds);
Assert.LessOrEqual(status.remaining_hits, status.hourly_limit, "rate check bit nuts");
Assert.GreaterOrEqual(status.hourly_limit, 350, "expect to have 350 now");
}
