public SearchResult More(string query, string vkPageInfo, string instPageInfo, string twitterPageInfo)
{
SearchResult searchResult = new SearchResult();
searchResult.Query = HttpUtility.UrlEncode(query);
InstagramSearch instagram = new InstagramSearch();
VKSearch vk = new VKSearch();
TwitterSearch twitter = new TwitterSearch();
instThread = new Thread(() => searchResult.InstPagination = string.IsNullOrEmpty(instPageInfo)
? instPageInfo
: instagram.Search(query, searchResult.Posts, instPageInfo, dict));
vkThread = new Thread(() => searchResult.VKPagination = string.IsNullOrEmpty(vkPageInfo)
? vkPageInfo
: vk.Search(query, searchResult.Posts, vkPageInfo, dict));
twitterThread = new Thread(() => searchResult.TwitterPagination = string.IsNullOrEmpty(twitterPageInfo)
? twitterPageInfo
: twitter.Search(query, searchResult.Posts, twitterPageInfo, dict));
instThread.Start();
vkThread.Start();
twitterThread.Start();
instThread.Join();
vkThread.Join();
twitterThread.Join();
searchResult.Posts = searchResult.Posts.OrderByDescending(p => p.Date).ToList();
return(searchResult);
}
public ActionResult Index(TwitterSearch model)
{
//search hashtag on twitter
if (!string.IsNullOrWhiteSpace(model.HashTag))
{
var index = new RAMDirectory();
SessionHelper.SetIndex(index);
TwitterFlow flow = new TwitterFlow(SessionHelper.GetIndex());
var searchResult = flow.SearchHashTagOnTwitter(model.HashTag);
//for local testing only!
//var searchResult = flow.SearchHashTagOnTwitter();
model.TwitterResult = searchResult;
flow.IndexSearchResult(searchResult);
model.HashTag = null;
}
//filter search result
if (!string.IsNullOrWhiteSpace(model.Filter) && SessionHelper.GetIndex() != null && string.IsNullOrWhiteSpace(model.HashTag))
{
TwitterFlow flow = new TwitterFlow(SessionHelper.GetIndex());
var filterResult = flow.FilterSearchResult(model.Filter);
model.TwitterResult = filterResult;
model.Filter = null;
}
return(View(model));
}
public static void Search()
{
SearchOptions options = new SearchOptions();
options.NumberPerPage = 19;
TwitterResponse <TwitterSearchResultCollection> searchResponse = TwitterSearch.Search("twitter", options);
Assert.IsNotNull(searchResponse);
Assert.That(searchResponse.Result == RequestResult.Success, searchResponse.ErrorMessage);
Assert.IsNotNull(searchResponse.ResponseObject);
Assert.That(searchResponse.ResponseObject.Count == 19);
var request = Twitterizer.TwitterSearch.Search("twitter");
Assert.IsNotNull(request);
Assert.That(request.Result == RequestResult.Success, request.ErrorMessage);
Assert.IsNotNull(request.ResponseObject);
Assert.Greater(request.ResponseObject.MaxId, 0);
Assert.Greater(request.ResponseObject.CompletedIn, 0);
Assert.IsNotNullOrEmpty(request.ResponseObject.MaxIdStr);
Assert.IsNotNullOrEmpty(request.ResponseObject.NextPage);
Assert.Greater(request.ResponseObject.Page, 0);
Assert.AreEqual("twitter", request.ResponseObject.Query);
Assert.IsNotNullOrEmpty(request.ResponseObject.RefreshUrl);
}
public SearchResult Search(string query)
{
SearchResult searchResult = new SearchResult();
searchResult.Query = HttpUtility.UrlEncode(query);
Cenzor cenzor = new Cenzor();
if (query != cenzor.Cenz(query, dict))
{
return(searchResult);
}
InstagramSearch instagram = new InstagramSearch();
VKSearch vk = new VKSearch();
TwitterSearch twitter = new TwitterSearch();
instThread = new Thread(() => searchResult.InstPagination = instagram.Search(query, searchResult.Posts, "", dict));
vkThread = new Thread(() => searchResult.VKPagination = vk.Search(query, searchResult.Posts, "", dict));
twitterThread = new Thread(() => searchResult.TwitterPagination = twitter.Search(query, searchResult.Posts, "", dict));
instThread.Start();
vkThread.Start();
twitterThread.Start();
instThread.Join();
vkThread.Join();
twitterThread.Join();
searchResult.Posts = searchResult.Posts.OrderByDescending(p => p.Date).ToList();
return(searchResult);
}
private Dictionary <DateTime, int>[] GetSearchResults(string twit_keywor1, string twit_keywor2)
{
var keyword1 = TwitterSearch.SearchByHashTag(twit_keywor1, 15);
var keyword2 = TwitterSearch.SearchByHashTag(twit_keywor2, 15);
return(new Dictionary <DateTime, int>[] { keyword1, keyword2 });
}
static void Main(string[] args)
{
try
{
TwitterSearch.TwitterMain();
}
catch (Exception ex)
{
var st = new StackTrace(ex, true);
var frame = st.GetFrame(0);
var line = st.GetFrame(0).GetFileLineNumber();
MethodBase site = ex.TargetSite;
string sMethodName = site == null ? null : site.Name;
Console.WriteLine("------------------");
Console.WriteLine(DateTime.Now.ToString());
Console.WriteLine("Line: " + line.ToString());
Console.WriteLine("Method: " + sMethodName);
Console.WriteLine("Exception: " + ex.Message);
Console.Write(ex.StackTrace.ToString());
Console.WriteLine("");
}
catch { }
}
public void ProcessResults_WithErrors_PopulatesErrorList()
{
var searchProc = new TwitterSearchRequestProcessor <TwitterSearch> {
BaseUrl = "https://api.twitter.com/1.1/search/"
};
List <TwitterSearch> searches = searchProc.ProcessResults(SearchErrorJson);
Assert.IsNotNull(searches);
TwitterSearch search = searches.SingleOrDefault();
Assert.IsNotNull(search);
List <TwitterError> errors = search.Errors;
Assert.IsNotNull(errors);
Assert.AreEqual(2, errors.Count);
TwitterError error = errors.FirstOrDefault();
Assert.IsNotNull(error);
Assert.AreEqual("tweet", error.ResourceType);
Assert.AreEqual("non_public_metrics.impression_count", error.Field);
Assert.AreEqual("Field Authorization Error", error.Title);
Assert.AreEqual("data", error.Section);
Assert.AreEqual("Sorry, you are not authorized to access non_public_metrics.impression_count on a Tweet.", error.Detail);
Assert.AreEqual("https://api.twitter.com/2/problems/not-authorized-for-field", error.Type);
}
public async Task TwDelta()
{
Application.Current.Dispatcher.Invoke(() =>
{
TwitterSearch SearchDelta = new TwitterSearch();
SearchDelta.TweetSearch(mw.watchlist_list);
});
return;
}
public void TwitterBtn()
{
TwitterSearch c1 = new TwitterSearch();
string Count = c1.TweetSearch(mw.watchlist_list);
Count = "";
Logs log = new Logs();
log.LogMessageToFile("TWITTER GET DONE");
}
protected override async Task Crawl()
{
Search = await GetTwitterSearch();
await NodeJsCrawler();
await SendMail("");
Sources = ReadSource();
if (Sources?.Count <= 0)
{
throw new Exception("无采集数据");
}
}
public async Task <ActionResult> TwitterSearchAsync(string query)
{
TwitterAdapter.ConsumerKey = TwitterConfiguration.ConsumerKeyApiKey;
TwitterAdapter.ConsumerSecret = TwitterConfiguration.ConsumerSecretApiSecret;
var results = TwitterAdapter.SearchAsync(query);
var model = new TwitterSearch {
Query = query, TwitterResult = results
};
return(View(model));
}
public static void SearchLocal()
{
SearchOptions options = new SearchOptions();
options.GeoCode = "30.4413,-84.2809,20mi";
TwitterResponse <TwitterSearchResultCollection> searchResponse = TwitterSearch.Search("", options);
Assert.IsNotNull(searchResponse);
Assert.That(searchResponse.Result == RequestResult.Success, searchResponse.ErrorMessage);
Assert.IsNotNull(searchResponse.ResponseObject);
}
public async Task <ActionResult> TwitterSearchAsync(string query)
{
TwitterAdapter.ConsumerKey = System.Web.HttpContext.Current.Application["TwitterConsumerKey"].ToString();
TwitterAdapter.ConsumerSecret = System.Web.HttpContext.Current.Application["TwitterConsumerSecret"].ToString();
var results = await TwitterAdapter.SearchAsync(query);
var model = new TwitterSearch()
{
Query = query, ResultsXml = results
};
return(this.View(model));
}
internal static AlexaResponse PickAWinnerIntentHandler(Request request)
{
TwitterAdapter.ConsumerKey = TwitterConfiguration.ConsumerKeyApiKey;
TwitterAdapter.ConsumerSecret = TwitterConfiguration.ConsumerSecretApiSecret;
var query = TwitterConfiguration.DefaultQuery;
var results = TwitterAdapter.SearchAsync(query);
var model = new TwitterSearch {
Query = query, TwitterResult = results
};
var result = model.TwitterResult[new Random().Next(model.TwitterResult.Count)];
using (var db = new alexaskilldemoEntities())
{
var competitionWinner = db.CompetitionWinners.FirstOrDefault();
if (competitionWinner == null)
{
db.CompetitionWinners.Add(new CompetitionWinner
{
Name = result.ScreenNameResponse,
Tweet = result.Text,
CreatedDate = result.CreatedAt,
ProfileImageUrl = result.ProfileImageUrl,
UpdatedDate = DateTime.UtcNow
});
}
else
{
competitionWinner.Name = result.ScreenNameResponse;
competitionWinner.Tweet = result.Text;
competitionWinner.CreatedDate = result.CreatedAt;
competitionWinner.ProfileImageUrl = result.ProfileImageUrl;
competitionWinner.UpdatedDate = DateTime.UtcNow;
}
db.SaveChanges();
}
var text = "I have selected user <prosody volume=\"x-loud\">" + result.ScreenNameResponse + "</prosody><emphasis level=\"moderate\"> Tweet <break time=\"1s\"/>" + result.Text + "</emphasis>" +
"<audio src='https://s3.amazonaws.com/ask-soundlibrary/human/amzn_sfx_crowd_cheer_med_01.mp3'/> " +
"<audio src='https://s3.amazonaws.com/ask-soundlibrary/human/amzn_sfx_crowd_applause_05.mp3'/>";
var response = new AlexaResponse(text, new SimpleCard()
{
Title = "Sitecore storefront",
Content = text
}, true, text);
return(response);
}
public async Task <List <TwitterResultJson> > SearchHttp(TwitterSearch search)
{
using (var client = new HttpClient()
{
Timeout = TimeSpan.FromMinutes(1)
})
{
var resp = await client.PostAsync("http://localhost:3000/twitter/search/", new StringContent(JsonConvert.SerializeObject(search), Encoding.UTF8, "application/json"));
var result = await resp.Content.ReadAsStringAsync();
return(JsonConvert.DeserializeObject <List <TwitterResultJson> >(result));
}
}
public void TestTwitterSearch()
{
// Arrange
HomeController controller = new HomeController();
// Act
var model = new TwitterSearch
{
HashTag = "tayfun"
};
ViewResult result = controller.Index(model) as ViewResult;
// Assert
Assert.IsNotNull(result.Model as TwitterSearch);
}
private static void SpeakTweets(object terms)
{
lock (SyncRoot)
{
//Get list of tweets
var t = terms as List <string>;
if (t == null)
{
return;
}
var temp = new List <string>();
foreach (var term in t)
{
var list = TwitterSearch.GetSearchResults(term, 1);
temp.AddRange(list.Select(tweet => tweet.Title).ToList());
}
//only add unique tweets to the list
var tweets = new List <string>();
foreach (var tweet in temp)
{
if (!tweets.Contains(tweet))
{
tweets.Add(tweet);
}
}
tweets.Shuffle();
var random = new Random(DateTime.Now.Millisecond);
foreach (var tweet in tweets)
{
var ts = tweet;
//tweet clean
ts = ts.CleanTweet();
//TED clean
ts = ts.TedClean();
BearDuino.Bear.CloseEyes(false);
Thread.Sleep(300);
BearDuino.Bear.Speak(ts);
if (random.NextDouble() > 0.8)
{
BearDuino.Bear.CloseEyes(true);
}
Thread.Sleep(1000);
}
}
}
public static List <Tweets> GetTweetList()
{
LastTweetInfo lastTweetInfo = NoSqlDatabase.GetLastTweetId();
decimal TweetIdBefore = lastTweetInfo.TweetId;
decimal TweetIdAfter = 0;
OAuthTokens tokens = new OAuthTokens();
tokens.ConsumerKey = Constants.TwitterOAuthConsumerKey;
tokens.ConsumerSecret = Constants.TwitterOAuthConsumerSecret;
tokens.AccessToken = Constants.TwitterOAuthAccessToken;
tokens.AccessTokenSecret = Constants.TwitterOAuthAccessTokenSecret;
SearchOptions options = new SearchOptions();
options.SinceId = TweetIdBefore;
//options.SinceId = 932808858153639937;
options.ResultType = SearchOptionsResultType.Recent;
TwitterResponse <TwitterSearchResultCollection> result = TwitterSearch.Search(tokens, Constants.TwitterSearchQuery, options);
decimal userid = 0;
List <Tweets> resultTweets = new List <Tweets>();
foreach (var myresult in result.ResponseObject)
{
userid = myresult.User.Id;
double sentimentScore = AzureAPI.getSentimentScore(myresult.Text);
string tweetLink = string.Format(Constants.TwitterMessageLink, myresult.User.ScreenName, myresult.StringId);
resultTweets.Add(TwitterAPI.BuildTweet(myresult.Id, tweetLink, myresult.CreatedDate, myresult.Text, sentimentScore, Constants.DefaultTweetStatus));
}
//TweetIdAfter = result.ResponseObject.OrderByDescending(tweets => tweets.Id).First().Id;
TweetIdAfter = resultTweets.Count == 0 ? 0 : resultTweets.OrderByDescending(tweets => tweets.TweetId).First().TweetId;
if (TweetIdAfter != 0)
{
NoSqlDatabase.UpdateLastTweetId(lastTweetInfo.Id, TweetIdAfter);
}
return(resultTweets);
}
public void ProcessResults_Populates_Meta()
{
var searchProc = new TwitterSearchRequestProcessor <TwitterSearch> {
BaseUrl = BaseUrl2
};
List <TwitterSearch> results = searchProc.ProcessResults(SearchDefaultJson);
Assert.IsNotNull(results);
TwitterSearch search = results.SingleOrDefault();
Assert.IsNotNull(search);
TwitterSearchMeta meta = search.Meta;
Assert.IsNotNull(meta);
Assert.AreEqual("1317802724407316480", meta.NewestID);
Assert.AreEqual("1316030424850800640", meta.OldestID);
Assert.AreEqual(4, meta.Count);
}
public static void Search()
{
SearchOptions options = new SearchOptions();
options.NumberPerPage = 19;
TwitterResponse <TwitterSearchResultCollection> searchResponse = TwitterSearch.Search("twitter", options);
Assert.IsNotNull(searchResponse);
Assert.That(searchResponse.Result == RequestResult.Success, searchResponse.ErrorMessage);
Assert.IsNotNull(searchResponse.ResponseObject);
Assert.That(searchResponse.ResponseObject.Count == 19);
var request = Twitterizer.TwitterSearch.Search("twitter");
Assert.IsNotNull(request);
Assert.That(request.Result == RequestResult.Success, request.ErrorMessage);
Assert.IsNotNull(request.ResponseObject);
}
protected override void Tick()
{
var tweets = TwitterSearch.FromUsers(_since, Brain.Settings.Twitters.ToArray());
_since = tweets.Max_id_str;
foreach (var tweet in tweets.Results)
{
if (tweet.Entities.Urls.Count() > 0)
{
var tweet1 = tweet;
Brain.Pipe.ListenOnce((s, match, arg3) => Process.Start(tweet1.Entities.Urls.First().Url), "open|more|show");
foreach (var twitterEntityUrl in tweet.Entities.Urls)
{
tweet.Text = tweet.Text.Replace(twitterEntityUrl.Url, "");
}
}
TweetView.Create(tweet.Text, tweet.From_user);
Brain.ListenerManager.CurrentListener.Output("{0}: {1}".Template(tweet.From_user_name, tweet.Text));
}
}
public void TestFilterSearchResult()
{
// Arrange
HomeController controller = new HomeController();
//prepare mocked index
TwitterFlow flow = new TwitterFlow(SessionHelper.GetIndex());
var searchResult = flow.SearchHashTagOnTwitter();
flow.IndexSearchResult(searchResult);
// Act
var model = new TwitterSearch
{
Filter = "tazminat"
};
ViewResult result = controller.Index(model) as ViewResult;
// Assert
Assert.AreEqual(2, (result.Model as TwitterSearch).TwitterResult.Count);
}
public void ProcessResults_Populates_DefaultTweets()
{
var searchProc = new TwitterSearchRequestProcessor <TwitterSearch> {
BaseUrl = BaseUrl2
};
List <TwitterSearch> results = searchProc.ProcessResults(SearchDefaultJson);
Assert.IsNotNull(results);
TwitterSearch search = results.SingleOrDefault();
Assert.IsNotNull(search);
List <Tweet> tweets = search.Tweets;
Assert.IsNotNull(tweets);
Assert.AreEqual(4, tweets.Count);
Tweet firstTweet = tweets.FirstOrDefault();
Assert.IsNotNull(firstTweet);
Assert.AreEqual("1317802724407316480", firstTweet.ID);
Assert.AreEqual("Test Data 1", firstTweet.Text);
}
/// <summary>
/// 关键词转Twitter高级查询对象
/// </summary>
/// <returns></returns>
async Task <TwitterSearch> GetTwitterSearch()
{
var data = new TwitterSearch();
//data.proxy = "localhost:7777";
await BorrowProxy();
data.proxy = Proxy;
data.token = KeyWord.Trim();
data.count = CrawlCount;
data.sinceTime = DateTime.Now.AddMonths(-1).ToString("yyyy-MM-01");
data.untilTime = DateTime.Now.ToString("yyyy-MM-dd");
if (KeyWord.Contains("#"))
{
data.hasTitle = KeyWord;
}
else
{
data.keyWord = KeyWord;
}
return(data);
}
public List <Tweet> GetTweetsForQuery(string query, decimal?sinceId, bool checkWhetherContainsInMessage = false)
{
try
{
if (sinceId == null)
{
so.SinceId = 0;
}
else
{
so.SinceId = sinceId.Value;
}
TwitterResponse <TwitterSearchResultCollection> twitterResponse = TwitterSearch.Search(tokens, "\"" + query + "\"", so); // TwitterAccount.VerifyCredentials(tokens);
List <Tweet> tweets = new List <Tweet>();
if (twitterResponse.Result == RequestResult.Success)
{
log.Info("Successfully retrieved results for query \"" + query + "\".");
foreach (TwitterStatus status in twitterResponse.ResponseObject)
{
Tweet t = GetTweetFromStatus(status);
if (t != null && t.language != null && (!checkWhetherContainsInMessage || t.message.ToLower().Contains(query.ToLower())))
{
tweets.Add(t);
}
}
}
else
{
log.Warn("Cannot retrieve results for query \"" + query + "\".");
}
return(tweets);
}
catch (Exception ex)
{
log.Warn("Communication error! Cannot retrieve results for query \"" + query + "\".", ex);
return(new List <Tweet>());
}
}
private async void HttpBtn_Click(object sender, EventArgs e)
{
var fPath = @"C:\Users\www\Desktop\Twitter项目\Twitter查询关键字.txt";
var fLines = File.ReadAllLines(fPath).Distinct().ToArray();
var jsonList = new List <TwitterResultJson>();
foreach (var item in fLines)
{
var keyWord = item.Trim();
var data = new TwitterSearch();
data.proxy = "localhost:7777";
data.token = keyWord.Trim();
data.count = 20000;
data.sinceTime = DateTime.Now.ToString("yyyy-MM-01");
data.untilTime = DateTime.Now.ToString("yyyy-MM-dd");
if (keyWord.Contains("#"))
{
data.hasTitle = keyWord;
}
else
{
data.keyWord = keyWord;
}
try
{
var resultJsonList = await SearchHttp(data);
jsonList.AddRange(resultJsonList);
}
catch
{
continue;
}
}
Console.WriteLine("采集JSON结束！");
await DownloadMedia(jsonList);
}
public void ProcessResults_Handles_Response_With_No_Results()
{
var searchProc = new TwitterSearchRequestProcessor <TwitterSearch> {
BaseUrl = "https://api.twitter.com/1.1/search/"
};
List <TwitterSearch> searches = searchProc.ProcessResults(EmptyResponse);
Assert.IsNotNull(searches);
TwitterSearch search = searches.SingleOrDefault();
Assert.IsNotNull(search);
List <Tweet> results = search.Tweets;
Assert.IsNull(results);
TwitterSearchMeta meta = search.Meta;
Assert.IsNotNull(meta);
Assert.IsNull(meta.NewestID);
Assert.IsNull(meta.OldestID);
Assert.AreEqual(0, meta.Count);
}
public void TWITT_GET_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
Application.Current.Dispatcher.Invoke(() =>
{
{
TwitterSearch c1 = new TwitterSearch();
string Count = c1.TweetSearch(mw.watchlist_list);
Logs log = new Logs();
log.LogMessageToFile("TWITTER GET auto at: ");
}
if (timerTwittGet == 1)
{
System.Timers.Timer twGT = new System.Timers.Timer();
twGT.AutoReset = true;
twGT.Interval = 870000;
twGT.Elapsed += TWITT_GET_Elapsed;
twGT.Start();
timerTwittGet++;
}
});
}
protected void Page_Load(object sender, EventArgs e)
{
string query = (string)Session["query"];
if (query != null)
{
TextBoxSearch.Text = query;
TwitterSearch engine = new TwitterSearch();
List <TwitterEntry> result = new List <TwitterEntry>();
List <TwitterEntry> query1 = engine.BuscaEntradaTwitter(query);
List <TwitterEntry> query2 = engine.BuscaEntradaTwitter("ganhe " + query);
List <TwitterEntry> query3 = engine.BuscaEntradaTwitter("concorra " + query);
List <TwitterEntry> query4 = engine.BuscaEntradaTwitter("desconto " + query);
List <TwitterEntry> query5 = engine.BuscaEntradaTwitter("retuite " + query);
List <TwitterEntry> query6 = engine.BuscaEntradaTwitter("oferta " + query);
List <TwitterEntry> query7 = engine.BuscaEntradaTwitter("cadastre " + query);
result.AddRange(query1);
result.AddRange(query2);
result.AddRange(query3);
result.AddRange(query4);
result.AddRange(query5);
result.AddRange(query6);
result.AddRange(query7);
List <TwitterEntry> cadastro = new List <TwitterEntry>();
List <TwitterEntry> retuite = new List <TwitterEntry>();
List <TwitterEntry> desconto = new List <TwitterEntry>();
HashSet <int> keys = new HashSet <int>();
foreach (TwitterEntry entry in result)
{
if (!keys.Contains(entry.Tweet.GetHashCode()))
{
int classe = Classificador.Classificador.classificar(entry.Tweet);
if (classe == Classificador.Classificador.RT)
{
retuite.Add(entry);
}
else if (classe == Classificador.Classificador.DESCONTO)
{
desconto.Add(entry);
}
else if (classe == Classificador.Classificador.CADASTRO)
{
cadastro.Add(entry);
}
keys.Add(entry.Tweet.GetHashCode());
}
}
result.Clear();
divulgacao.Text = Util.loadTweets(cadastro);
retweets.Text = Util.loadTweets(retuite);
descontos.Text = Util.loadTweets(desconto);
//limpa dados da sessão
Session.Clear();
}
}
public TwitterTicker() : base(60000)
{
var tweets = TwitterSearch.FromUsers("0", Brain.Settings.Twitters.ToArray());
_since = tweets.Max_id_str;
}
