public RateLimitingStatusDto(TwitterRateLimitStatus status)
{
HourlyLimit = status.HourlyLimit;
RemainingHits = status.RemainingHits;
ResetsInMinutes = (int) (status.ResetTime - DateTime.Now).TotalMinutes;
ResetTime = status.ResetTime.ToUniversalTime();
}
public RateLimitingStatusDto(TwitterRateLimitStatus status)
{
HourlyLimit = status.HourlyLimit;
RemainingHits = status.RemainingHits;
ResetsInMinutes = (int)(status.ResetTime - DateTime.Now).TotalMinutes;
ResetTime = status.ResetTime.ToUniversalTime();
}
private void UpdateLimit(TwitterRateLimitStatus status)
{
_requestLimitStatus.HourlyLimit = status.HourlyLimit;
_requestLimitStatus.PeriodEndTime = status.ResetTime;
_requestLimitStatus.PeriodDuration = 1.Hour();
_requestLimitStatus.RemainingHits = status.RemainingHits;
}
public override async Task OnWork(DateTime time)
{
var atmms = await Twitter.ts.ListTweetsMentioningMeAsync(new TweetSharp.ListTweetsMentioningMeOptions {
Count = 5, IncludeEntities = true
});
TwitterRateLimitStatus rate = Twitter.ts.Response.RateLimitStatus;
Abbybot.print("You have used " + rate.RemainingHits + " out of your " + rate.HourlyLimit);
var tsmm = atmms.Value.ToArray();
foreach (var tmm in tsmm)
{
var test = await AbbybotMentionSql.GetLatestMentionIdsAsync((ulong)tmm.Id);
if (test.Count > 0)
{
continue;
}
await AbbybotMentionSql.AddLatestMentionIdAsync((ulong)tmm.Id);
var tmt = Regex.Replace(tmm.Text, @"http[^\s]+", "");
var t = Regex.Replace(tmt, @"@[^\s]+", "");
//var twitteruser = AbbybotUser.GetUserFromTwitterUser(tmm.Author.ScreenName);
//AbbybotTwitterCommandArgs atca = new AbbybotTwitterCommandArgs();
//atca.Message = t;
//atca.abbybotUser = (AbbybotUser)twitteruser;
Abbybot.print($"[\n{t}\n]");
}
}
protected static TimeSpan GetSleepTime(TwitterRateLimitStatus rls, IScheduler sched)
{
if (rls.HourlyLimit == -1)
return TimeSpan.Zero;
TimeSpan timeTillReset = rls.ResetTime - sched.Now;
int remainingHits = rls.RemainingHits == 0 ? 1 : rls.RemainingHits;
var secondsOfSleepTime = timeTillReset.TotalSeconds / remainingHits;
return TimeSpan.FromSeconds(secondsOfSleepTime);
}
static bool LimitExceeded(OAuthTokens tokens)
{
TwitterResponse <TwitterRateLimitStatus> response = TwitterRateLimitStatus.GetStatus(tokens);
if (response.RateLimiting.Remaining == 0)
{
Console.Out.WriteLine($"Twitter rate limit hit. Will resume pulling @ {response.RateLimiting.ResetDate.ToString("hh:mm:ss")}");
}
return(response.RateLimiting.Remaining == 0);
}
protected static TimeSpan GetSleepTime(TwitterRateLimitStatus rls, IScheduler sched)
{
if (rls.HourlyLimit == -1)
{
return(TimeSpan.Zero);
}
TimeSpan timeTillReset = rls.ResetTime - sched.Now;
int remainingHits = rls.RemainingHits == 0 ? 1 : rls.RemainingHits;
var secondsOfSleepTime = timeTillReset.TotalSeconds / remainingHits;
return(TimeSpan.FromSeconds(secondsOfSleepTime));
}
protected TimeSpan GetSleepTime(TwitterRateLimitStatus rateLimitStatus)
{
if (rateLimitStatus.HourlyLimit == -1)
{
return TimeSpan.FromMilliseconds(UserThrottleMs);
}
// let's send a message if the limit is not infinite
Messenger.Default.Send(new TwitterStatusMessage
{
HourlyLimit = rateLimitStatus.HourlyLimit,
RawSource = rateLimitStatus.RawSource,
RemainingHits = rateLimitStatus.RemainingHits,
ResetTime = rateLimitStatus.ResetTime,
ResetTimeInSeconds = rateLimitStatus.ResetTimeInSeconds
});
var timeTillReset = rateLimitStatus.ResetTime - Scheduler.Now;
var remainingHits = rateLimitStatus.RemainingHits == 0 ? 1 : rateLimitStatus.RemainingHits;
var secondsOfSleepTime = timeTillReset.TotalSeconds / remainingHits;
Console.WriteLine("sleep time: " + secondsOfSleepTime);
return TimeSpan.FromSeconds(secondsOfSleepTime).Add(TimeSpan.FromMilliseconds(UserThrottleMs));
}
protected TimeSpan GetSleepTime(TwitterRateLimitStatus rateLimitStatus)
{
if (rateLimitStatus.HourlyLimit == -1)
{
return(TimeSpan.FromMilliseconds(UserThrottleMs));
}
// let's send a message if the limit is not infinite
Messenger.Default.Send(new TwitterStatusMessage
{
HourlyLimit = rateLimitStatus.HourlyLimit,
RawSource = rateLimitStatus.RawSource,
RemainingHits = rateLimitStatus.RemainingHits,
ResetTime = rateLimitStatus.ResetTime,
ResetTimeInSeconds = rateLimitStatus.ResetTimeInSeconds
});
var timeTillReset = rateLimitStatus.ResetTime - Scheduler.Now;
var remainingHits = rateLimitStatus.RemainingHits == 0 ? 1 : rateLimitStatus.RemainingHits;
var secondsOfSleepTime = timeTillReset.TotalSeconds / remainingHits;
Console.WriteLine("sleep time: " + secondsOfSleepTime);
return(TimeSpan.FromSeconds(secondsOfSleepTime).Add(TimeSpan.FromMilliseconds(UserThrottleMs)));
}
private Dictionary<string, TwitterRateLimitStatus> GetBucketLimits(JObject limits)
{
if (limits == null || limits.Count <= 0)
return null;
var bucketLimits = new Dictionary<string, TwitterRateLimitStatus>();
foreach (var limit in limits) // /followers/list, /followers/ids or /account/verify_credentials, /account_settings
{
var bucketLimit = new TwitterRateLimitStatus(); // limit, remaining, reset
bucketLimits.Add(limit.Key, bucketLimit);
JObject limitValues = JObject.Load(limit.Value.CreateReader());
bucketLimit.RawSource = limitValues.ToString();
bucketLimit.HourlyLimit = limitValues.GetValue("limit").Value<int>();
bucketLimit.RemainingHits = limitValues.GetValue("remaining").Value<int>();
bucketLimit.ResetTime = limitValues.GetValue("reset").Value<long>().FromUnixTime();
var remainingSeconds = Convert.ToInt64((bucketLimit.ResetTime - DateTime.UtcNow).TotalSeconds);
bucketLimit.ResetTimeInSeconds = remainingSeconds < 0 ? -1 : remainingSeconds;
}
return bucketLimits;
}
private static List <CorpusDataRow> SearchTweets(string keyword, Sentiment sentiment, int count, TwitterService service, string outputPath)
{
List <CorpusDataRow> outputCorpus = new List <CorpusDataRow>();
long?lastId = null;
int index = 0;
int skipCount = 0;
keyword += sentiment == Sentiment.Positive ? " :)" : " :(";
while (skipCount == 0 && outputCorpus.Count < count)
{
// Fetch the tweet.
var statusList = service.Search(new SearchOptions()
{
Q = keyword, Lang = "en", IncludeEntities = false, Count = count, MaxId = lastId
});
lastId = statusList.Statuses.Last().Id;
foreach (var status in statusList.Statuses)
{
if (!status.Text.StartsWith("RT") && !status.Text.Contains(":P") &&
!((status.Text.Contains(":)") || status.Text.Contains(":-)") || status.Text.Contains(": )") || status.Text.Contains(":D") || status.Text.Contains("=)")) &&
(status.Text.Contains(":(") || status.Text.Contains(":-(") || status.Text.Contains(": ("))) &&
outputCorpus.Where(c => c.Tweet.Text == status.Text).Count() == 0)
{
status.Text = status.Text.Replace(",", " ");
status.Text = status.Text.Replace("\n", " ");
status.Text = status.Text.Replace("\r", " ");
status.Text = status.Text.Replace("\t", " ");
status.Text = status.Text.Replace(":)", " ");
status.Text = status.Text.Replace(":-)", " ");
status.Text = status.Text.Replace(": )", " ");
status.Text = status.Text.Replace(":D", " ");
status.Text = status.Text.Replace("=)", " ");
status.Text = status.Text.Replace(":(", " ");
status.Text = status.Text.Replace(":-(", " ");
status.Text = status.Text.Replace(": (", " ");
if (service.Response.StatusCode == System.Net.HttpStatusCode.OK)
{
// Convert the TwitterStatus to a Tweet DTO.
CorpusDataRow row = new CorpusDataRow();
row.Id = status.Id;
row.Keyword = keyword;
row.Tweet = Mapper.Map <TwitterStatus, Tweet>(status);
row.Sentiment = sentiment;
// Save the result to file.
SaveResult(row, outputPath);
outputCorpus.Add(row);
if ((index + 1) % 50 == 0)
{
Console.WriteLine("Processed " + (index + 1) + " tweets.");
}
}
else
{
// Check the rate limit.
TwitterRateLimitStatus rateSearch = service.Response.RateLimitStatus;
if (rateSearch.RemainingHits < 1)
{
DateTime resetTime = rateSearch.ResetTime + TimeSpan.FromMinutes(1);
Console.WriteLine("Rate Limit reached. Sleeping until " + resetTime);
Thread.Sleep(resetTime - DateTime.Now);
// Try this record again.
index--;
}
else
{
// Some other error. Maybe 404. Skip this record.
skipCount++;
Console.WriteLine("Skipped " + skipCount + " records. Got " + service.Response.StatusCode + ".");
}
}
}
}
}
Console.WriteLine("Saved " + outputCorpus.Count + ", Skipped " + skipCount + ".");
return(outputCorpus);
}
public static void TwitterUpdate()
{
Rust.TwitterAccount = "RustUpdates";
Warcraft.TwitterAccount = "Warcraft";
Overwatch.TwitterAccount = "PlayOverwatch";
Starcraft.TwitterAccount = "Starcraft";
Diablo.TwitterAccount = "Diablo";
Hearthstone.TwitterAccount = "PlayHearthstone";
Heroes.TwitterAccount = "BlizzHeroes";
if (hardreset)
{
if (resetTime >= 15)
{
hardreset = false;
resetTime = 0;
}
else
{
resetTime++;
return;
}
}
//gets channels to send tweets to
Channel TwitterRustCH = YDCbot.TwitterRustCH;
Channel TwitterBlizzCH = YDCbot.TwitterBlizzCH;
Channel TwitterFeedCh = YDCbot.TwitterFeedCh;
//starts a service with twitter to GET tweets
var service = new TwitterService(ConsumerKey, ConsumerSecret);
service.AuthenticateWith(token, tokenSecret);
//sets up a GET status request for each user, with given name and tweet amounts
//then posts those tweets in their correct channel
if (YDCbot.TwitterToFollow != "" && TwitterFeedCh != null)
{
List <TwitterStatus> TwitterStatuses = new List <TwitterStatus>();
var Tweetoptions = new TweetSharp.ListTweetsOnUserTimelineOptions();
Tweetoptions.ScreenName = YDCbot.TwitterToFollow;
if (FollowedTwitter.LastTweet != 0)
{
Tweetoptions.SinceId = FollowedTwitter.LastTweet;
}
else
{
Tweetoptions.Count = TweetAmount;
}
Tweetoptions.ExcludeReplies = true;
TwitterStatuses = service.ListTweetsOnUserTimeline(Tweetoptions).ToList();
if (TwitterStatuses.Count > 0)
{
foreach (TwitterStatus tweet in TwitterStatuses)
{
if (tweet.Id > FollowedTwitter.LastTweet)
{
FollowedTwitter.LastTweet = tweet.Id;
}
string TweetToDiscord = "``` " + "https://twitter.com/" + YDCbot.TwitterToFollow + "/status/" + tweet.Id + " ```";
TwitterFeedCh.SendMessage(TweetToDiscord);
}
}
}
//used as examples, will only work if user has correct text channels in their server
//YDCbot.cs has the void that sets everything up, check for correct channel names
#region RustUpdates
if (YDCbot.HasTwitterRust)
{
List <TwitterStatus> RustStatuses = new List <TwitterStatus>();
var Rustoptions = new TweetSharp.ListTweetsOnUserTimelineOptions();
Rustoptions.ScreenName = Rust.TwitterAccount;
if (Rust.LastTweet != 0)
{
Rustoptions.SinceId = Rust.LastTweet;
}
else
{
Rustoptions.Count = TweetAmount;
}
Rustoptions.ExcludeReplies = true;
RustStatuses = service.ListTweetsOnUserTimeline(Rustoptions).ToList();
if (RustStatuses.Count > 0)
{
foreach (TwitterStatus tweet in RustStatuses)
{
if (tweet.Id > Rust.LastTweet)
{
Rust.LastTweet = tweet.Id;
}
string TweetToDiscord = "``` " + "https://twitter.com/" + Rust.TwitterAccount + "/status/" + tweet.Id + " ```";
TwitterRustCH.SendMessage(TweetToDiscord);
}
}
}
#endregion
if (YDCbot.HasTwitterBlizz)
{
#region Diablo
List <TwitterStatus> DiabloStatuses = new List <TwitterStatus>();
var Diablooptions = new TweetSharp.ListTweetsOnUserTimelineOptions();
Diablooptions.ScreenName = Diablo.TwitterAccount;
if (Diablo.LastTweet != 0)
{
Diablooptions.SinceId = Diablo.LastTweet;
}
else
{
Diablooptions.Count = TweetAmount;
}
Diablooptions.ExcludeReplies = true;
DiabloStatuses = service.ListTweetsOnUserTimeline(Diablooptions).ToList();
if (DiabloStatuses.Count > 0)
{
foreach (TwitterStatus tweet in DiabloStatuses)
{
if (tweet.Id > Diablo.LastTweet)
{
Diablo.LastTweet = tweet.Id;
}
string TweetToDiscord = "``` " + "https://twitter.com/" + Diablo.TwitterAccount + "/status/" + tweet.Id + " ```";
TwitterBlizzCH.SendMessage(TweetToDiscord);
if (tweet.Entities.Media.Count > 0)
{
foreach (var link in tweet.Entities.Media)
{
TwitterBlizzCH.SendMessage(link.MediaUrlHttps);
}
}
}
}
#endregion
#region Warcraft
List <TwitterStatus> WarcraftStatuses = new List <TwitterStatus>();
var Warcraftoptions = new TweetSharp.ListTweetsOnUserTimelineOptions();
Warcraftoptions.ScreenName = Warcraft.TwitterAccount;
if (Warcraft.LastTweet != 0)
{
Warcraftoptions.SinceId = Warcraft.LastTweet;
}
else
{
Warcraftoptions.Count = TweetAmount;
}
Warcraftoptions.ExcludeReplies = true;
WarcraftStatuses = service.ListTweetsOnUserTimeline(Warcraftoptions).ToList();
if (WarcraftStatuses.Count > 0)
{
foreach (TwitterStatus tweet in WarcraftStatuses)
{
if (tweet.Id > Warcraft.LastTweet)
{
Warcraft.LastTweet = tweet.Id;
}
string TweetToDiscord = "``` " + "https://twitter.com/" + Warcraft.TwitterAccount + "/status/" + tweet.Id + " ```";
TwitterBlizzCH.SendMessage(TweetToDiscord);
if (tweet.Entities.Media.Count > 0)
{
foreach (var link in tweet.Entities.Media)
{
TwitterBlizzCH.SendMessage(link.MediaUrlHttps);
}
}
}
}
#endregion
#region Starcraft
List <TwitterStatus> StarcraftStatuses = new List <TwitterStatus>();
var Starcraftoptions = new TweetSharp.ListTweetsOnUserTimelineOptions();
Starcraftoptions.ScreenName = Starcraft.TwitterAccount;
if (Starcraft.LastTweet != 0)
{
Starcraftoptions.SinceId = Starcraft.LastTweet;
}
else
{
Starcraftoptions.Count = TweetAmount;
}
Starcraftoptions.ExcludeReplies = true;
StarcraftStatuses = service.ListTweetsOnUserTimeline(Starcraftoptions).ToList();
if (StarcraftStatuses.Count > 0)
{
foreach (TwitterStatus tweet in StarcraftStatuses)
{
if (tweet.Id > Starcraft.LastTweet)
{
Starcraft.LastTweet = tweet.Id;
}
string TweetToDiscord = "``` " + "https://twitter.com/" + Starcraft.TwitterAccount + "/status/" + tweet.Id + " ```";
TwitterBlizzCH.SendMessage(TweetToDiscord);
if (tweet.Entities.Media.Count > 0)
{
foreach (var link in tweet.Entities.Media)
{
TwitterBlizzCH.SendMessage(link.MediaUrlHttps);
}
}
}
}
#endregion
#region Heroes
List <TwitterStatus> HeroesStatuses = new List <TwitterStatus>();
var Heroesoptions = new TweetSharp.ListTweetsOnUserTimelineOptions();
Heroesoptions.ScreenName = Heroes.TwitterAccount;
if (Heroes.LastTweet != 0)
{
Heroesoptions.SinceId = Heroes.LastTweet;
}
else
{
Heroesoptions.Count = TweetAmount;
}
Heroesoptions.ExcludeReplies = true;
HeroesStatuses = service.ListTweetsOnUserTimeline(Heroesoptions).ToList();
if (HeroesStatuses.Count > 0)
{
foreach (TwitterStatus tweet in HeroesStatuses)
{
if (tweet.Id > Heroes.LastTweet)
{
Heroes.LastTweet = tweet.Id;
}
string TweetToDiscord = "``` " + "https://twitter.com/" + Heroes.TwitterAccount + "/status/" + tweet.Id + " ```";
TwitterBlizzCH.SendMessage(TweetToDiscord);
if (tweet.Entities.Media.Count > 0)
{
foreach (var link in tweet.Entities.Media)
{
TwitterBlizzCH.SendMessage(link.MediaUrlHttps);
}
}
}
}
#endregion
#region Hearthstone
List <TwitterStatus> HearthstoneStatuses = new List <TwitterStatus>();
var Hearthoptions = new TweetSharp.ListTweetsOnUserTimelineOptions();
Hearthoptions.ScreenName = Hearthstone.TwitterAccount;
if (Hearthstone.LastTweet != 0)
{
Hearthoptions.SinceId = Hearthstone.LastTweet;
}
else
{
Hearthoptions.Count = TweetAmount;
}
Hearthoptions.ExcludeReplies = true;
HearthstoneStatuses = service.ListTweetsOnUserTimeline(Hearthoptions).ToList();
if (HearthstoneStatuses.Count > 0)
{
foreach (TwitterStatus tweet in HearthstoneStatuses)
{
if (tweet.Id > Hearthstone.LastTweet)
{
Hearthstone.LastTweet = tweet.Id;
}
string TweetToDiscord = "``` " + "https://twitter.com/" + Hearthstone.TwitterAccount + "/status/" + tweet.Id + " ```";
TwitterBlizzCH.SendMessage(TweetToDiscord);
if (tweet.Entities.Media.Count > 0)
{
foreach (var link in tweet.Entities.Media)
{
TwitterBlizzCH.SendMessage(link.MediaUrlHttps);
}
}
}
}
#endregion
#region Overwatch
List <TwitterStatus> OverwatchStatuses = new List <TwitterStatus>();
var Overwatchoptions = new TweetSharp.ListTweetsOnUserTimelineOptions();
Overwatchoptions.ScreenName = Overwatch.TwitterAccount;
if (Overwatch.LastTweet != 0)
{
Overwatchoptions.SinceId = Overwatch.LastTweet;
}
else
{
Overwatchoptions.Count = TweetAmount;
}
Overwatchoptions.ExcludeReplies = true;
OverwatchStatuses = service.ListTweetsOnUserTimeline(Overwatchoptions).ToList();
if (OverwatchStatuses.Count > 0)
{
foreach (TwitterStatus tweet in OverwatchStatuses)
{
if (tweet.Id > Overwatch.LastTweet)
{
Overwatch.LastTweet = tweet.Id;
}
string TweetToDiscord = "``` " + "https://twitter.com/" + Overwatch.TwitterAccount + "/status/" + tweet.Id + " ```";
TwitterBlizzCH.SendMessage(TweetToDiscord);
if (tweet.Entities.Media.Count > 0)
{
foreach (var link in tweet.Entities.Media)
{
TwitterBlizzCH.SendMessage(link.MediaUrlHttps);
}
}
}
}
#endregion
}
TwitterRateLimitStatus rate = service.Response.RateLimitStatus; //checking rate limits to prevent twitter bot from being ip banned
Console.WriteLine("You have " + rate.RemainingHits + " remaining out of the " + rate.HourlyLimit + " limit. @" + DateTime.Now.ToString("h:mm:ss tt"));
if (rate.RemainingHits < 25)
{
hardreset = true;
return;
}
SaveTwitter();
}
private static TwitterRateLimitStatus DeserializeRateLimitStatusXml(string xml)
{
var document = new XmlDocument();
document.LoadXml(xml);
// easier to use linq for rate limit status
var hourlylimitNode = document.SelectSingleNode("/hash/hourly-limit");
var hourly = hourlylimitNode != null ? hourlylimitNode.FirstChild : null;
var remaininghitsNode = document.SelectSingleNode("/hash/remaining-hits");
var hits = remaininghitsNode != null ? remaininghitsNode.FirstChild : null;
var resettimeinsecondsNode = document.SelectSingleNode("/hash/reset-time-in-seconds");
var seconds = resettimeinsecondsNode != null ? resettimeinsecondsNode.FirstChild : null;
// var time = document.SelectSingleNode("/hash/reset-time").FirstChild;
var hourlyValue = hourly != null && hourly.Value != null
? Convert.ToInt32(hourly.Value, CultureInfo.InvariantCulture)
: 0;
var hitsValue = hits != null && hits.Value != null
? Convert.ToInt32(hits.Value, CultureInfo.InvariantCulture)
: 0;
var secondsValue = seconds != null && seconds.Value != null
? Convert.ToInt64(seconds.Value, CultureInfo.InvariantCulture)
: 0;
var resetTime = secondsValue.FromUnixTime();
var limit = new TwitterRateLimitStatus
{
HourlyLimit = hourlyValue,
RemainingHits = hitsValue,
ResetTimeInSeconds = secondsValue,
ResetTime = resetTime
};
return limit;
}
protected static TimeSpan GetSleepTime(TwitterService service, IScheduler sched)
{
TwitterRateLimitStatus rls = service.Response.RateLimitStatus;
return(GetSleepTime(rls, sched));
}
public static void PrintTwitterRateLimitStatus(TwitterService service)
{
TwitterRateLimitStatus rate = service.Response.RateLimitStatus;
ConsoleWrapper.WriteLine($"{DateTime.Now.ToLongTimeString()} Twitter rate limit status: {rate.RemainingHits}/{rate.HourlyLimit}");
}
/// <summary>
/// Executes the command.
/// </summary>
/// <returns>The results of the command.</returns>
public TwitterResponse <T> ExecuteCommand()
{
TwitterResponse <T> twitterResponse = new TwitterResponse <T>();
if (this.OptionalProperties.UseSSL)
{
this.Uri = new Uri(this.Uri.AbsoluteUri.Replace("http://", "https://"));
}
// Loop through all of the custom attributes assigned to the command class
foreach (Attribute attribute in this.GetType().GetCustomAttributes(false))
{
if (attribute is AuthorizedCommandAttribute)
{
if (this.Tokens == null)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Tokens are required for the \"{0}\" command.", this.GetType()));
}
if (string.IsNullOrEmpty(this.Tokens.ConsumerKey) ||
string.IsNullOrEmpty(this.Tokens.ConsumerSecret) ||
string.IsNullOrEmpty(this.Tokens.AccessToken) ||
string.IsNullOrEmpty(this.Tokens.AccessTokenSecret))
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Token values cannot be null when executing the \"{0}\" command.", this.GetType()));
}
}
else if (attribute is RateLimitedAttribute)
{
// Get the rate limiting status
if (TwitterRateLimitStatus.GetStatus(this.Tokens).ResponseObject.RemainingHits == 0)
{
throw new TwitterizerException("You are being rate limited.");
}
}
}
// Prepare the query parameters
Dictionary <string, object> queryParameters = new Dictionary <string, object>();
foreach (KeyValuePair <string, object> item in this.RequestParameters)
{
queryParameters.Add(item.Key, item.Value);
}
// Declare the variable to be returned
twitterResponse.ResponseObject = default(T);
twitterResponse.RequestUrl = this.Uri.AbsoluteUri;
RateLimiting rateLimiting;
AccessLevel accessLevel;
byte[] responseData;
try
{
WebRequestBuilder requestBuilder = new WebRequestBuilder(this.Uri, this.Verb, this.Tokens)
{
Multipart = this.Multipart
};
#if !SILVERLIGHT
if (this.OptionalProperties != null)
{
requestBuilder.Proxy = this.OptionalProperties.Proxy;
}
#endif
foreach (var item in queryParameters)
{
requestBuilder.Parameters.Add(item.Key, item.Value);
}
HttpWebResponse response = requestBuilder.ExecuteRequest();
if (response == null)
{
twitterResponse.Result = RequestResult.Unknown;
return(twitterResponse);
}
responseData = ConversionUtility.ReadStream(response.GetResponseStream());
twitterResponse.Content = Encoding.UTF8.GetString(responseData, 0, responseData.Length);
twitterResponse.RequestUrl = requestBuilder.RequestUri.AbsoluteUri;
#if !SILVERLIGHT
// Parse the rate limiting HTTP Headers
rateLimiting = ParseRateLimitHeaders(response.Headers);
// Parse Access Level
accessLevel = ParseAccessLevel(response.Headers);
#else
rateLimiting = null;
accessLevel = AccessLevel.Unknown;
#endif
// Lookup the status code and set the status accordingly
SetStatusCode(twitterResponse, response.StatusCode, rateLimiting);
twitterResponse.RateLimiting = rateLimiting;
twitterResponse.AccessLevel = accessLevel;
}
catch (WebException wex)
{
if (new[]
{
#if !SILVERLIGHT
WebExceptionStatus.Timeout,
WebExceptionStatus.ConnectionClosed,
#endif
WebExceptionStatus.ConnectFailure
}.Contains(wex.Status))
{
twitterResponse.Result = RequestResult.ConnectionFailure;
twitterResponse.ErrorMessage = wex.Message;
return(twitterResponse);
}
// The exception response should always be an HttpWebResponse, but we check for good measure.
HttpWebResponse exceptionResponse = wex.Response as HttpWebResponse;
if (exceptionResponse == null)
{
throw;
}
responseData = ConversionUtility.ReadStream(exceptionResponse.GetResponseStream());
twitterResponse.Content = Encoding.UTF8.GetString(responseData, 0, responseData.Length);
#if !SILVERLIGHT
rateLimiting = ParseRateLimitHeaders(exceptionResponse.Headers);
// Parse Access Level
accessLevel = ParseAccessLevel(exceptionResponse.Headers);
#else
rateLimiting = null;
accessLevel = AccessLevel.Unknown;
#endif
// Try to read the error message, if there is one.
try
{
TwitterErrorDetails errorDetails = SerializationHelper <TwitterErrorDetails> .Deserialize(responseData);
twitterResponse.ErrorMessage = errorDetails.ErrorMessage;
}
catch (Exception)
{
// Occasionally, Twitter responds with XML error data even though we asked for json.
// This is that scenario. We will deal with it by doing nothing. It's up to the developer to deal with it.
}
// Lookup the status code and set the status accordingly
SetStatusCode(twitterResponse, exceptionResponse.StatusCode, rateLimiting);
twitterResponse.RateLimiting = rateLimiting;
twitterResponse.AccessLevel = accessLevel;
if (wex.Status == WebExceptionStatus.UnknownError)
{
throw;
}
return(twitterResponse);
}
try
{
twitterResponse.ResponseObject = SerializationHelper <T> .Deserialize(responseData, this.DeserializationHandler);
}
catch (Newtonsoft.Json.JsonReaderException)
{
twitterResponse.ErrorMessage = "Unable to parse JSON";
twitterResponse.Result = RequestResult.Unknown;
return(twitterResponse);
}
catch (Newtonsoft.Json.JsonSerializationException)
{
twitterResponse.ErrorMessage = "Unable to parse JSON";
twitterResponse.Result = RequestResult.Unknown;
return(twitterResponse);
}
// Pass the current oauth tokens into the new object, so method calls from there will keep the authentication.
twitterResponse.Tokens = this.Tokens;
return(twitterResponse);
}
private void GetAllTweetsForUser(string username)
{
Dispatcher.BeginInvoke(new Action(delegate
{
tweetList.Items.Clear();
retweetList.Items.Clear();
replyList.Items.Clear();
mentionCounter.Clear();
}));
var currentTweets = service.ListTweetsOnUserTimeline(new ListTweetsOnUserTimelineOptions
{
ScreenName = username,
Count = 200
});
TwitterRateLimitStatus rateStatus = service.Response.RateLimitStatus;
if (currentTweets == null)
{
MessageBox.Show($"Couldn't find any tweets for this user. Rate limits remaining: {rateStatus.RemainingHits}");
return;
}
int tweetCount = currentTweets.Count();
Dispatcher.BeginInvoke(new Action(delegate
{
progressBar.Minimum = 0;
progressBar.Maximum = tweetCount;
progressBar.Value = 0;
}));
foreach (var tweet in currentTweets)
{
if (unitOfWork.Tweets.TweetAlreadyExists(tweet.Id))
{
continue;
}
// save the user
TwitterUser tUser = tweet.User;
User user;
if (unitOfWork.Users.UserAlreadyExists(tUser.Id, out user) == false)
{
user = CreateUser(tUser);
unitOfWork.Users.Add(user);
}
// save the tweet
Tweet t = CreateTweet(tweet);
unitOfWork.Tweets.Add(t);
// save any linked hashtags
if (tweet.Entities.HashTags.Any())
{
HandleHashtags(tweet);
}
if (tweet.Entities.Mentions.Any())
{
HandleMentions(tweet);
}
unitOfWork.Complete();
// DISPLAY
Dispatcher.BeginInvoke(new Action(delegate
{
if (tweet.Text.StartsWith("RT "))
{
OutputReTweet(tweet);
}
else if (tweet.InReplyToStatusId.HasValue)
{
OutputReplyTweet(tweet);
}
else
{
OutputNormalTweet(tweet);
}
progressBar.Value += 1;
}));
}
Dispatcher.BeginInvoke(new Action(delegate
{
UpdateMainUserList();
}));
}
/// <summary>
/// Loads the tweet text data for each id in the corpus.
/// </summary>
/// <param name="service">TwitterService</param>
/// <param name="corpus">List of CorpusDataRow</param>
/// <param name="outputPath">File path to output data file.</param>
/// <returns>List of CorpusDataRow (with Tweet DTO populated).</returns>
private static List <CorpusDataRow> LoadTweets(TwitterService service, List <CorpusDataRow> corpus, string outputPath)
{
List <CorpusDataRow> outputCorpus = new List <CorpusDataRow>();
int skipCount = 0;
for (int index = GetResumeIndex(corpus, outputPath); index < corpus.Count; index++)
{
CorpusDataRow row = corpus[index];
// Fetch the tweet.
var status = service.GetTweet(new GetTweetOptions()
{
Id = row.Id
});
status.Text = status.Text.Replace(",", " ");
status.Text = status.Text.Replace("\n", " ");
status.Text = status.Text.Replace("\r", " ");
status.Text = status.Text.Replace("\t", " ");
if (service.Response.StatusCode == System.Net.HttpStatusCode.OK)
{
// Convert the TwitterStatus to a Tweet DTO.
row.Tweet = Mapper.Map <TwitterStatus, Tweet>(status);
// Save the result to file.
SaveResult(row, outputPath);
outputCorpus.Add(row);
if ((index + 1) % 50 == 0)
{
Console.WriteLine("Processed " + (index + 1) + " tweets.");
}
}
else
{
// Check the rate limit.
TwitterRateLimitStatus rateSearch = service.Response.RateLimitStatus;
if (rateSearch.RemainingHits < 1)
{
DateTime resetTime = rateSearch.ResetTime + TimeSpan.FromMinutes(1);
Console.WriteLine("Rate Limit reached. Sleeping until " + resetTime);
Thread.Sleep(resetTime - DateTime.Now);
// Try this record again.
index--;
}
else
{
// Some other error. Maybe 404. Skip this record.
skipCount++;
Console.WriteLine("Skipped " + skipCount + " records. Got " + service.Response.StatusCode + ".");
}
}
}
Console.WriteLine("Saved " + outputCorpus.Count + ", Skipped " + skipCount + ".");
return(outputCorpus);
}
