public SearchResult Search(string query)
{
SearchResult searchResult = new SearchResult();
searchResult.Query = HttpUtility.UrlEncode(query);
Cenzor cenzor = new Cenzor();
if (query != cenzor.Cenz(query, dict))
{
return(searchResult);
}
InstagramSearch instagram = new InstagramSearch();
VKSearch vk = new VKSearch();
TwitterSearch twitter = new TwitterSearch();
instThread = new Thread(() => searchResult.InstPagination = instagram.Search(query, searchResult.Posts, "", dict));
vkThread = new Thread(() => searchResult.VKPagination = vk.Search(query, searchResult.Posts, "", dict));
twitterThread = new Thread(() => searchResult.TwitterPagination = twitter.Search(query, searchResult.Posts, "", dict));
instThread.Start();
vkThread.Start();
twitterThread.Start();
instThread.Join();
vkThread.Join();
twitterThread.Join();
searchResult.Posts = searchResult.Posts.OrderByDescending(p => p.Date).ToList();
return(searchResult);
}
public SearchResult More(string query, string vkPageInfo, string instPageInfo, string twitterPageInfo)
{
SearchResult searchResult = new SearchResult();
searchResult.Query = HttpUtility.UrlEncode(query);
InstagramSearch instagram = new InstagramSearch();
VKSearch vk = new VKSearch();
TwitterSearch twitter = new TwitterSearch();
instThread = new Thread(() => searchResult.InstPagination = string.IsNullOrEmpty(instPageInfo)
? instPageInfo
: instagram.Search(query, searchResult.Posts, instPageInfo, dict));
vkThread = new Thread(() => searchResult.VKPagination = string.IsNullOrEmpty(vkPageInfo)
? vkPageInfo
: vk.Search(query, searchResult.Posts, vkPageInfo, dict));
twitterThread = new Thread(() => searchResult.TwitterPagination = string.IsNullOrEmpty(twitterPageInfo)
? twitterPageInfo
: twitter.Search(query, searchResult.Posts, twitterPageInfo, dict));
instThread.Start();
vkThread.Start();
twitterThread.Start();
instThread.Join();
vkThread.Join();
twitterThread.Join();
searchResult.Posts = searchResult.Posts.OrderByDescending(p => p.Date).ToList();
return(searchResult);
}
public static void Search()
{
SearchOptions options = new SearchOptions();
options.NumberPerPage = 19;
TwitterResponse <TwitterSearchResultCollection> searchResponse = TwitterSearch.Search("twitter", options);
Assert.IsNotNull(searchResponse);
Assert.That(searchResponse.Result == RequestResult.Success, searchResponse.ErrorMessage);
Assert.IsNotNull(searchResponse.ResponseObject);
Assert.That(searchResponse.ResponseObject.Count == 19);
var request = Twitterizer.TwitterSearch.Search("twitter");
Assert.IsNotNull(request);
Assert.That(request.Result == RequestResult.Success, request.ErrorMessage);
Assert.IsNotNull(request.ResponseObject);
Assert.Greater(request.ResponseObject.MaxId, 0);
Assert.Greater(request.ResponseObject.CompletedIn, 0);
Assert.IsNotNullOrEmpty(request.ResponseObject.MaxIdStr);
Assert.IsNotNullOrEmpty(request.ResponseObject.NextPage);
Assert.Greater(request.ResponseObject.Page, 0);
Assert.AreEqual("twitter", request.ResponseObject.Query);
Assert.IsNotNullOrEmpty(request.ResponseObject.RefreshUrl);
}
public static void SearchLocal()
{
SearchOptions options = new SearchOptions();
options.GeoCode = "30.4413,-84.2809,20mi";
TwitterResponse <TwitterSearchResultCollection> searchResponse = TwitterSearch.Search("", options);
Assert.IsNotNull(searchResponse);
Assert.That(searchResponse.Result == RequestResult.Success, searchResponse.ErrorMessage);
Assert.IsNotNull(searchResponse.ResponseObject);
}
public static List <Tweets> GetTweetList()
{
LastTweetInfo lastTweetInfo = NoSqlDatabase.GetLastTweetId();
decimal TweetIdBefore = lastTweetInfo.TweetId;
decimal TweetIdAfter = 0;
OAuthTokens tokens = new OAuthTokens();
tokens.ConsumerKey = Constants.TwitterOAuthConsumerKey;
tokens.ConsumerSecret = Constants.TwitterOAuthConsumerSecret;
tokens.AccessToken = Constants.TwitterOAuthAccessToken;
tokens.AccessTokenSecret = Constants.TwitterOAuthAccessTokenSecret;
SearchOptions options = new SearchOptions();
options.SinceId = TweetIdBefore;
//options.SinceId = 932808858153639937;
options.ResultType = SearchOptionsResultType.Recent;
TwitterResponse <TwitterSearchResultCollection> result = TwitterSearch.Search(tokens, Constants.TwitterSearchQuery, options);
decimal userid = 0;
List <Tweets> resultTweets = new List <Tweets>();
foreach (var myresult in result.ResponseObject)
{
userid = myresult.User.Id;
double sentimentScore = AzureAPI.getSentimentScore(myresult.Text);
string tweetLink = string.Format(Constants.TwitterMessageLink, myresult.User.ScreenName, myresult.StringId);
resultTweets.Add(TwitterAPI.BuildTweet(myresult.Id, tweetLink, myresult.CreatedDate, myresult.Text, sentimentScore, Constants.DefaultTweetStatus));
}
//TweetIdAfter = result.ResponseObject.OrderByDescending(tweets => tweets.Id).First().Id;
TweetIdAfter = resultTweets.Count == 0 ? 0 : resultTweets.OrderByDescending(tweets => tweets.TweetId).First().TweetId;
if (TweetIdAfter != 0)
{
NoSqlDatabase.UpdateLastTweetId(lastTweetInfo.Id, TweetIdAfter);
}
return(resultTweets);
}
public static void Search()
{
SearchOptions options = new SearchOptions();
options.NumberPerPage = 19;
TwitterResponse <TwitterSearchResultCollection> searchResponse = TwitterSearch.Search("twitter", options);
Assert.IsNotNull(searchResponse);
Assert.That(searchResponse.Result == RequestResult.Success, searchResponse.ErrorMessage);
Assert.IsNotNull(searchResponse.ResponseObject);
Assert.That(searchResponse.ResponseObject.Count == 19);
var request = Twitterizer.TwitterSearch.Search("twitter");
Assert.IsNotNull(request);
Assert.That(request.Result == RequestResult.Success, request.ErrorMessage);
Assert.IsNotNull(request.ResponseObject);
}
public List <Tweet> GetTweetsForQuery(string query, decimal?sinceId, bool checkWhetherContainsInMessage = false)
{
try
{
if (sinceId == null)
{
so.SinceId = 0;
}
else
{
so.SinceId = sinceId.Value;
}
TwitterResponse <TwitterSearchResultCollection> twitterResponse = TwitterSearch.Search(tokens, "\"" + query + "\"", so); // TwitterAccount.VerifyCredentials(tokens);
List <Tweet> tweets = new List <Tweet>();
if (twitterResponse.Result == RequestResult.Success)
{
log.Info("Successfully retrieved results for query \"" + query + "\".");
foreach (TwitterStatus status in twitterResponse.ResponseObject)
{
Tweet t = GetTweetFromStatus(status);
if (t != null && t.language != null && (!checkWhetherContainsInMessage || t.message.ToLower().Contains(query.ToLower())))
{
tweets.Add(t);
}
}
}
else
{
log.Warn("Cannot retrieve results for query \"" + query + "\".");
}
return(tweets);
}
catch (Exception ex)
{
log.Warn("Communication error! Cannot retrieve results for query \"" + query + "\".", ex);
return(new List <Tweet>());
}
}
public static CommandResult SearchTweets(decimal[] ids)
{
if (!IsInitiated)
{
return(CommandResult.NotInitiated);
}
foreach (var id in ids)
{
if (id == 0)
{
continue;
}
var UserToProcess = (new TwitterContext()).Users.FirstOrDefault(u => u.Id == id);
if (UserToProcess == null)
{
Form.AppendLineToOutput(string.Format("User with id {0} does not exist in the database.", id), Color.Olive);
continue;
}
//else continue and search for tweets to the user and mentions of the user
string screenname = UserToProcess.ScreenName;
decimal idNumber = UserToProcess.Id;
TwitterContext db = null;
try
{
TwitterResponse <TwitterSearchResultCollection> searchResult;
int numTweets = 0;
int duplicatesFound = 0;
Form.AppendLineToOutput(string.Format("Attempt to search for tweets to and mentioning {0} in the last 4 months", screenname), Color.Olive);
//continue from SearchResult if there is tweet, otherwise use user's latest tweet id
//decimal latestTweetId = GetEarliestTweetIdFromSearchResult(id);
//if(latestTweetId == 0)
//{
// latestTweetId = GetLatestTweetId(id);
//}
//decimal earliestTweetId = GetEarliestTweetId(id);
decimal latestTweetId = 701646604403478528;
//701646604403478528 : id of a tweet dated 21 Feb 2016 9:56 p.m.
decimal earliestTweetId = 656323760102879232;
//656323760102879232 : id of a tweet dated 19 Oct 2015 9:19 p.m.
do
{
searchResult = TwitterSearch.Search(
Tokens,
string.Format("to%3A{0}%20%40{0}", screenname),
new SearchOptions
{
MaxId = latestTweetId - 1,
SinceId = earliestTweetId,
ResultType = SearchOptionsResultType.Recent,
Count = 100
});
if (searchResult.Result == RequestResult.Success)
{
TwitterSearchResultCollection tweets = searchResult.ResponseObject;
if (tweets == null || tweets.Count == 0)
{
break;
}
else
{
//refresh context
db = new TwitterContext();
db.Configuration.AutoDetectChangesEnabled = false;
decimal earliestIdFromSearch = latestTweetId;
foreach (TwitterStatus tweet in tweets)
{
SearchResult sr = new SearchResult();
sr.ForAirlineId = idNumber;
sr.ForAirlineScreenname = screenname;
sr.TweetId = tweet.Id;
sr.CreatedDate = tweet.CreatedDate;
sr.PosterUserId = tweet.User.Id;
sr.PosterScreenname = tweet.User.ScreenName;
sr.TweetText = tweet.Text;
sr.IsTweetToAirline = (tweet.InReplyToUserId == idNumber);
sr.IsRetweetOfAirline = (tweet.RetweetedStatus != null && tweet.RetweetedStatus.User.Id == idNumber);
sr.IsMentionAirline = (!sr.IsTweetToAirline && !sr.IsRetweetOfAirline);
if (tweet.Id < earliestIdFromSearch)
{
earliestIdFromSearch = tweet.Id;
}
numTweets++;
db.SearchResults.Add(sr);
}
try
{
db.SaveChanges();
}
catch (Exception ex)
{
Form.AppendLineToOutput(ex.Message, Color.Olive);
Form.AppendLineToOutput(ex.StackTrace, Color.Olive);
return(CommandResult.Failure);
}
latestTweetId = earliestIdFromSearch;
}
Form.AppendLineToOutput(string.Format("{0} tweets processed for {1} (id:{2}). {3} duplicate {4} not saved.", numTweets, screenname, idNumber, duplicatesFound, duplicatesFound == 1 ? "tweet" : "tweets"), Color.Olive);
}
else if (searchResult.Result == RequestResult.RateLimited)
{
WaitForRateLimitReset(searchResult);
Form.AppendLineToOutput("Resuming search tweets command", Color.Olive);
continue;
}
else
{
HandleTwitterizerError <TwitterSearchResultCollection>(searchResult);
//log that this user should be retried later
File.AppendAllText(@".\users-to-retry.txt", idNumber.ToString() + Environment.NewLine);
break; //give up with current user for now
}
} while (searchResult.ResponseObject != null && searchResult.ResponseObject.Count > 0);
}
catch (Exception e)
{
Exception current = e;
Form.AppendLineToOutput(string.Format("Unexpected exception : {0}", e.Message), Color.Olive);
while (current.InnerException != null)
{
Form.AppendLineToOutput(string.Format("Inner exception : {0}", current.InnerException.Message), Color.Olive);
current = current.InnerException;
}
//log that this user should be retried later
File.AppendAllText(@".\users-to-retry.txt", idNumber.ToString() + Environment.NewLine);
continue; //give up with current user
}
finally
{
if (db != null)
{
db.Dispose();
}
}
}
return(CommandResult.Success);
}
private void toolStripDropDownButton2_Click(object sender, EventArgs e)
{
toolStripStatusLabel1.Text = "Restarting...";
ts.EndStream();
timeline.Clear();
if (st == StartType.UserStream)
{
ts.StartUserStream(null, new StreamStoppedCallback((x) => { toolStripStatusLabel1.Text = "Stopped."; }), new StatusCreatedCallback(x => { Add(x); }), null, null, null, new EventCallback(x => { Event(x); }), null);
var tt = TwitterTimeline.HomeTimeline(ts.Tokens);
try
{
foreach (var tss in tt.ResponseObject)
{
timeline.Add(tss);
}
toolStripStatusLabel1.Text = "Restarted.";
}
catch
{
MessageBox.Show(tt.ErrorMessage);
}
}
else if (st == StartType.FilterStream)
{
string query = "";
if (ts.StreamOptions.Track.Count != 0)
{
foreach (string ss in ts.StreamOptions.Track)
{
query += ss + "+AND+";
}
query = query.Remove(query.Length - 5, 5);
}
ts.StartPublicStream(new StreamStoppedCallback((x) => { toolStripStatusLabel1.Text = "Stopped."; }), new StatusCreatedCallback(x => { Add(x); }), null, new EventCallback(x => { Event(x); }));
var tt = TwitterSearch.Search(ts.Tokens, query, new SearchOptions()
{
ResultType = SearchOptionsResultType.Popular
});
try
{
foreach (var tss in tt.ResponseObject)
{
timeline.Add(new TwitterStatus()
{
Text = tss.Text, User = new TwitterUser()
{
ScreenName = tss.FromUserScreenName
}
});
}
toolStripStatusLabel1.Text = "Restarted.";
}
catch
{
MessageBox.Show(tt.ErrorMessage);
}
}
else if (st == StartType.Mentions)
{
ts.StreamOptions.Track.Add("@" + ExtendedOAuthTokens.Tokens.First <ExtendedOAuthTokens>((x) => { return(x.OAuthTokens == ts.Tokens); }).UserName);
ts.StartPublicStream(new StreamStoppedCallback((x) => { toolStripStatusLabel1.Text = "Stopped."; }), new StatusCreatedCallback(x => { Add(x); }), null, new EventCallback(x => { Event(x); }));
var tt = TwitterTimeline.Mentions(ts.Tokens);
try
{
foreach (var tss in tt.ResponseObject)
{
timeline.Add(tss);
}
toolStripStatusLabel1.Text = "Restarted.";
}
catch
{
MessageBox.Show(tt.ErrorMessage);
}
}
ShowF();
}
public Column(TwitterStream stream, StartType s, string title)
{
if (back != null)
{
listView1.BackgroundImage = back;
}
InitializeComponent();
listView1.SmallImageList = new ImageList();
ts = stream;
if (s == StartType.UserStream)
{
ts.StartUserStream(null, new StreamStoppedCallback((x) => { toolStripStatusLabel1.Text = "Stopped."; }), new StatusCreatedCallback(x => { Add(x); }), null, null, null, new EventCallback(x => { Event(x); }), null);
var tt = TwitterTimeline.HomeTimeline(stream.Tokens);
try
{
foreach (var tss in tt.ResponseObject)
{
timeline.Add(tss);
}
}
catch
{
MessageBox.Show(tt.ErrorMessage);
}
}
else if (s == StartType.FilterStream)
{
string query = "";
if (ts.StreamOptions.Track.Count != 0)
{
foreach (string ss in ts.StreamOptions.Track)
{
query += ss + "+AND+";
}
query = query.Remove(query.Length - 5, 5);
}
ts.StartPublicStream(new StreamStoppedCallback((x) => { toolStripStatusLabel1.Text = "Stopped."; }), new StatusCreatedCallback(x => { Add(x); }), null, new EventCallback(x => { Event(x); }));
var tt = TwitterSearch.Search(ts.Tokens, query, new SearchOptions()
{
ResultType = SearchOptionsResultType.Popular
});
try
{
foreach (var tss in tt.ResponseObject)
{
timeline.Add(new TwitterStatus()
{
Text = tss.Text, User = new TwitterUser()
{
ScreenName = tss.FromUserScreenName
}
});
}
}
catch
{
MessageBox.Show(tt.ErrorMessage);
}
}
else if (s == StartType.Mentions)
{
ts.StreamOptions.Track.Add("@" + ExtendedOAuthTokens.Tokens.First <ExtendedOAuthTokens>((x) => { return(x.OAuthTokens == ts.Tokens); }).UserName);
ts.StartPublicStream(new StreamStoppedCallback((x) => { toolStripStatusLabel1.Text = "Stopped."; }), new StatusCreatedCallback(x => { Add(x); }), null, new EventCallback(x => { Event(x); }));
var tt = TwitterTimeline.Mentions(stream.Tokens);
try
{
foreach (var tss in tt.ResponseObject)
{
timeline.Add(tss);
}
}
catch
{
MessageBox.Show(tt.ErrorMessage);
}
}
st = s;
this.Text = title;
Columns.Add(this);
ShowF();
}
public void Renew()
{
if (st == StartType.UserStream)
{
timeline.Clear();
var tt = TwitterTimeline.HomeTimeline(ts.Tokens);
try
{
foreach (var tss in tt.ResponseObject)
{
timeline.Add(tss);
}
}
catch
{
MessageBox.Show(tt.ErrorMessage);
}
}
else if (st == StartType.FilterStream)
{
string query = "";
if (ts.StreamOptions.Track.Count != 0)
{
foreach (string ss in ts.StreamOptions.Track)
{
query += ss + "+AND+";
}
query = query.Remove(query.Length - 5, 5);
}
timeline.Clear();
var tt = TwitterSearch.Search(ts.Tokens, query, new SearchOptions()
{
ResultType = SearchOptionsResultType.Popular
});
try
{
foreach (var tss in tt.ResponseObject)
{
timeline.Add(new TwitterStatus()
{
Text = tss.Text, User = new TwitterUser()
{
ScreenName = tss.FromUserScreenName
}
});
}
}
catch
{
MessageBox.Show(tt.ErrorMessage);
}
}
else if (st == StartType.Mentions)
{
timeline.Clear();
var tt = TwitterTimeline.Mentions(ts.Tokens);
try
{
foreach (var tss in tt.ResponseObject)
{
timeline.Add(tss);
}
}
catch
{
MessageBox.Show(tt.ErrorMessage);
}
}
ShowF();
}