private void AddFiltersStream(TwitterFilterOptions filteredOptions)
{
var user = User.GetUserFromScreenName(filteredOptions.FollowUserName);
_stream.AddFollow(user);
_stream.AddTrack(filteredOptions.Track);
}
/*Constructor: takes the keys needed, last paramets are filters for the stream*/
public TwitterStreamProcessor(string consumerKey, string consumerSecret,
string accessToken, string accessTokenSecret, params string[] filters)
{
PictureDataQueue = new Queue <PictureData>();
PictureMetaDataList = new List <PictureMetaData>();
//The credentials for the twitter api
var cred = new TwitterCredentials(consumerKey, consumerSecret,
accessToken, accessTokenSecret);
stream = Tweetinvi.Stream.CreateFilteredStream(cred);
//insert the filters
foreach (var filter in filters)
{
stream.AddTrack(filter);
}
//Delegate the method responsible for the handling of the matches
stream.MatchingTweetReceived += handleMatchingTweet;
//start the stream as a task: will run "forever"
Task.Factory.StartNew(() => stream.StartStreamMatchingAllConditions());
StreamStart = DateTime.Now;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
Console.WriteLine("What topic would you like to get tweets about?");
string subject = Console.ReadLine();
while (!stoppingToken.IsCancellationRequested)
{
ITwitterCredentials demo = Auth.SetUserCredentials(_options.ConsumerKey, _options.ConsumerSecret, _options.AccessToken, _options.AccessTokenSecret);
IFilteredStream stream = Stream.CreateFilteredStream();
stream.AddTrack(subject);
stream.AddTweetLanguageFilter(LanguageFilter.Portuguese);
_logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
_logger.LogInformation("I'm listening to twitter");
stream.MatchingTweetReceived += async(sender, arguments) =>
{
await SendTweetsByKafka(new Tweet()
{
TweetedBy = arguments.Tweet.CreatedBy.ScreenName, TweetedAt = arguments.Tweet.CreatedAt, Text = arguments.Tweet.Text
});
};
stream.StartStreamMatchingAllConditions();
await Task.Delay(5000, stoppingToken);
}
}
public void Start(string track, MatchCondition condition, params Model.Location[] locations)
{
// Filters
if (!string.IsNullOrEmpty(track))
{
_stream.AddTrack(track);
}
if (null != locations && locations.Length > 0)
{
foreach (var location in locations)
{
_stream.AddLocation(
new Coordinates(location.Latitude1, location.Longitude1),
new Coordinates(location.Latitude2, location.Longitude2)
);
}
}
// Start
if (condition == MatchCondition.AnyCondition)
{
_stream.StartStreamMatchingAnyCondition();
}
else
{
_stream.StartStreamMatchingAllConditions();
}
}
private static void startTweetStream(string searchString)
{
stream.AddTrack(searchString);
stream.MatchingTweetReceived += (sender, arg) =>
{
try
{
var jsonData = JsonConvert.SerializeObject(arg.Tweet.TweetDTO);
groupedTweets.Add(arg.Tweet.Text.ToLower());
count++;
}
catch (Exception exception)
{
Debug.WriteLine("# CAUGHT AN EXCEPTION" + searchString);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("{0} > Exception: {1}", DateTime.Now, exception.Message);
Console.ResetColor();
}
};
//Prevent crashing at unexpected EOF or 0 bytes
stream.StreamStopped += (sender, args) =>
{
Debug.WriteLine("# STREAM STOPPED");
if (args.Exception != null)
{
stream.StartStreamMatchingAllConditions();
}
};
stream.StartStreamMatchingAllConditions();
}
public void StartStream()
{
FilteredStream = Stream.CreateFilteredStream();
FilteredStream.AddTrack(Hashtag);
FilteredStream.MatchingTweetReceived += (sender, args) => RaiseTweetAdded(args.Tweet);
FilteredStream.StartStreamMatchingAllConditionsAsync();
}
public void CreateStream()
{
TwitterCredentials.SetCredentials(
RoleEnvironment.GetConfigurationSettingValue("UserAccessToken"),
RoleEnvironment.GetConfigurationSettingValue("UserAccessSecret"),
RoleEnvironment.GetConfigurationSettingValue("ConsumerKey"),
RoleEnvironment.GetConfigurationSettingValue("ConsumerSecret"));
//rateLimit = new ProtectedTimer(TimeSpan.FromSeconds(60), new Action(OnCheckRateRequestLimit));
//rateLimit.Start();
string words = RoleEnvironment.GetConfigurationSettingValue("TwitterSearchTerms");
string[] items = words.Split(';');
foreach (var item in items)
{
if (!string.IsNullOrEmpty(item))
{
stream.AddTrack(item);
Trace.Write(string.Concat("Track item -> ", item), typeof(TwitterStreamingEngine).FullName);
}
}
stream.MatchingTweetReceived += OnMatchingTweetReceived;
stream.DisconnectMessageReceived += OnStreamDisconnectMessageReveived;
stream.WarningFallingBehindDetected += OnWarningFallingBehindDetected;
stream.LimitReached += OnLimitReached;
stream.StreamStarted += OnStreamStarted;
stream.StreamStopped += OnStreamStopped;
stream.StartStreamMatchingAllConditions();
}
private void SetStreamTrackers()
{
foreach (var t in _tickers)
{
_twitterStream.AddTrack($"${t.NasdaqSymbol}");
}
}
public async Task StartAsync()
{
_eventHubClient = EventHubClient.CreateFromConnectionString(_targetConfig.ConnectionString, _targetConfig.EventHubName);
Auth.SetUserCredentials(_sourceConfig.Credentials.ConsumerKey, _sourceConfig.Credentials.ConsumerSecret, _sourceConfig.Credentials.AccessToken, _sourceConfig.Credentials.AccessTokenSecret);
IFilteredStream stream = Stream.CreateFilteredStream();
foreach (var language in _sourceConfig.Languages)
{
stream.AddTweetLanguageFilter((LanguageFilter)language);
}
foreach (string track in _sourceConfig.Tracks)
{
stream.AddTrack(track);
}
stream.MatchingTweetReceived += OnMatchingTweetReceived;
stream.DisconnectMessageReceived += OnDisconnectMessageReceived;
stream.StreamStopped += OnStreamStopped;
stream.StallWarnings = true;
await stream.StartStreamMatchingAnyConditionAsync();
}
public void StartStream(string keyword)
{
Console.WriteLine(
$"[{DateTime.Now}] - Starting listening for tweets that contains the keyword '{keyword}'...");
_stream.AddTrack(keyword);
_stream.MatchingTweetReceived += (sender, args) => { _tweetProcessor.ProcessTweetAsync(keyword, args); };
_stream.StartStreamMatchingAllConditions();
}
public static async Task Init(IServiceProvider service)
{
_serviceProvider = service;
_stream = Stream.CreateFilteredStream();
_stream.AddTrack(_searchTerm);
_stream.MatchingTweetReceived += TweetRecieved;
await _stream.StartStreamMatchingAllConditionsAsync();
}
public void AddTrack(string hashtag)
{
if (filteredStream.TracksCount > 0)
{
filteredStream.StopStream();
Dictionary <string, Action <ITweet> > tracks = filteredStream.Tracks;
ReCreateFilteredStream();
tracks.Keys.ForEach(s => filteredStream.AddTrack(s));
}
filteredStream.AddTrack(hashtag);
filteredStream.StartStreamMatchingAnyConditionAsync();
Debug.WriteLine("ADDED HASHTAG - " + hashtag);
}
private OnGoingStudy StartStudy(TwitterStudy study)
{
var reportData = new TwitterSpyStudyData(study);
Trace.WriteLine(string.Format("Study {0} started", study.Id));
foreach (var topic in study.Topics)
{
_stream.AddTrack(topic, reportData.ProcessTweet);
}
var onGoingStudy = new OnGoingStudy {
Data = reportData
};
// the object is stored so that the timer isn't garbage collected
var timer = new Timer(o =>
{
var data = (TwitterSpyStudyData)o;
//
// Remove the study from the ongoing list
//
lock (_onGoingStudies)
{
_onGoingStudies.Remove(onGoingStudy);
}
CheckIfStreamNeedsToBeStopped();
//
// Remove topics
//
foreach (var topic in data.Study.Topics)
{
_stream.RemoveTrack(topic);
}
var report = data.GetReport();
//
// send report to the client via queue
//
_reportQueue.AddMessage(new CloudQueueMessage(JsonConvert.SerializeObject(report)));
},
reportData,
study.Duration,
TimeSpan.FromTicks(-1));
onGoingStudy.Timer = timer;
return(onGoingStudy);
}
public void Initialize()
{
_logger.Write("Configuring Twitter contact source...");
Auth.SetUserCredentials(_configuration.ConsumerKey, _configuration.ConsumerSecret, _configuration.UserToken, _configuration.UserSecret);
_stream = Stream.CreateFilteredStream();
_stream.AddTrack(_configuration.PhrasesToTrack);
_stream.MatchingTweetReceived += Stream_MatchingTweetReceived;
_logger.WriteLine("Initialization of Twitter Harvester complete");
}
public static void MineTwitter(string term)
{
TwitterCredentials.SetCredentials(
ConfigurationManager.AppSettings["AccessToken"],
ConfigurationManager.AppSettings["AccessSecret"],
ConfigurationManager.AppSettings["ConsumerKey"],
ConfigurationManager.AppSettings["ConsumerSecret"]);
IFilteredStream stream = Tweetinvi.Stream.CreateFilteredStream();
stream.MatchingTweetReceived += Stream_TweetReceived;
stream.AddTrack(term);
stream.StartStreamMatchingAllConditions();
}
public MainWindow()
{
InitializeComponent();
TimerClock.Interval = (int)TimeSpan.FromMinutes(15).TotalMilliseconds;
TimerClock.Tick += delegate { TweetSendedWithin15MinWindow = 0; };
TimerClock.Start();
Auth.SetCredentials(Credentials.RandomCredential);
Jarvis = User.GetAuthenticatedUser();
KeywordStream = Stream.CreateFilteredStream();
UserStream = Stream.CreateUserStream();
UserStream.FollowedByUser += FlowedByUser;
UserStream.LimitReached += LimitReached;
KeywordStream.LimitReached += LimitReached;
KeywordStream.AddTrack("reallifejarvis");
KeywordStream.AddTrack("real life jarvis");
KeywordStream.AddTrack("talktomejarvis");
KeywordStream.AddTrack("talk to me jarvis");
KeywordStream.AddTrack("#retweetjarvis");
KeywordStream.MatchingTweetReceived += NewTweetFound;
KeywordStream.StartStreamMatchingAnyConditionAsync();
UserStream.StartStreamAsync();
}
/// <summary>
/// Start a stream, filtering ony the keyword and only English language tweets.
/// </summary>
protected void StartStream(string keyword)
{
stream = Stream.CreateFilteredStream();
stream.AddTrack(keyword);
stream.MatchingTweetReceived += (sender, args) =>
{
if (args.Tweet.Language == Language.English)
{
UpdateFdg(args.Tweet.Text);
}
};
stream.StartStreamMatchingAllConditionsAsync();
}
public FilteredStream(string keyword, bool isFake = false)
{
_isFake = isFake;
_stop = true;
if (_isFake)
{
SetupFakeObservable(keyword);
}
else
{
_filteredStream = Stream.CreateFilteredStream();
_filteredStream.AddTrack(keyword);
}
}
/// <summary>
/// Initializes and starts this stream listener.
/// </summary>
/// <param name="filterValue">A value used to filter out specific messages from the stream.</param>
/// <returns>An awaitable object for this asynchronous operation.</returns>
public async Task InitializeAsync(string filterValue)
{
if (_stream != null)
{
return;
}
_stream = Tweetinvi.Stream.CreateFilteredStream(
new TwitterCredentials(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET));
_stream.AddTrack(filterValue);
_tweetObservable = Observable.FromEventPattern <MatchedTweetReceivedEventArgs>(_stream, nameof(_stream.MatchingTweetReceived)).
Select(pattern => pattern.EventArgs.Tweet);
await _stream.StartStreamMatchingAnyConditionAsync();
}
void HandleTracks(bool add, params string[] tracks)
{
StopTweetStream();
foreach (var track in tracks)
{
if (add)
{
_filteredStream.AddTrack(track);
}
else
{
_filteredStream.RemoveTrack(track);
}
}
}
public void Start()
{
_stream = Stream.CreateFilteredStream();
_stream.AddTrack("tweetinvi");
_stream.Credentials = _credentials;
_stream.MatchingTweetReceived += (sender, args) =>
{
ITweet tweet = args.Tweet;
_hubContext.Clients.All.broadcastTweet(tweet);
};
_backgroundThread = new Thread(_stream.StartStreamMatchingAllConditions);
}
public void Start()
{
_stream = Stream.CreateFilteredStream();
_stream.AddTrack("tweetinvi");
_stream.Credentials = _credentials;
_stream.MatchingTweetReceived += (sender, args) =>
{
ITweet tweet = args.Tweet;
_hubContext.Clients.All.broadcastTweet(tweet);
};
_backgroundThread = new Thread(_stream.StartStreamMatchingAllConditions);
}
public CashTags(IEnumerable <string> tags)
{
this.Tags = new Dictionary <string, Tag>();
FilteredStream = Stream.CreateFilteredStream();
foreach (var tag in tags)
{
if (!Tags.Keys.Contains(tag))
{
this.Tags.Add(tag, new Tag(tag));
FilteredStream.AddTrack(tag);
Thread.Sleep(100);
}
}
FilteredStream.MatchingTweetReceived += filteredStream_MatchingTweetReceived;
WriteConsole();
}
/// <summary>
/// Starts a Twitter Stream based on the specified geographic coordinates and the now playing hash tag
/// </summary>
/// <param name="latitude1">Latitude of user location (bottom_left)</param>
/// <param name="longitude1">Longitude of user location (bottom_left)</param>
/// <param name="latitude2">Latitude of user location (top_right)</param>
/// <param name="longitude2">Longitude of user location (top_right)</param>
public static async Task StartStream(double latitude1, double longitude1, double latitude2, double longitude2)
{
// Setup Twitter credentials
TweetinviUtilities.SetTwitterCredentials();
// If the stream does not exists...
if (_stream == null)
{
//...then it is started
// Create a filtered stream
_stream = Stream.CreateFilteredStream();
_stream.AddTrack(Constants.NOWPLAYING_HASHTAG); // Lookup for nowplaying hashtag
// OPTIONAL: if you want to see how the feed is updated really quick, just comment the following line of code.
// You will see the effect of "infinite scroll" in the client
_stream.AddLocation(
new Coordinates(latitude1, longitude1),
new Coordinates(latitude2, longitude2)); // Lookup in the specific geographic coordinates
// OPTIONAL: if you want to filter the stream just for a specific user, uncomment the following line of code
//_stream.AddFollow(2834545563);
// Event that handles a matching tweet
_stream.MatchingTweetReceived += async(sender, args) =>
{
// A OEmbed tweet is sent to the client
IOEmbedTweet embedTweet = Tweet.GetOEmbedTweet(args.Tweet);
await _context.Clients.All.updateFeed(embedTweet);
};
// Start the stream matching all conditions
await _stream.StartStreamMatchingAllConditionsAsync();
}
else
{
//... otherwise resume it
_stream.ResumeStream();
}
}
public void StartStream(string keyword)
{
Console.WriteLine(
$"[{DateTime.Now}] - Starting listening for tweets that contains the keyword '{keyword}'...");
try
{
// Disable the exception swallowing to allow exception to be thrown by Tweetinvi
ExceptionHandler.SwallowWebExceptions = false;
var authenticatedUser = User.GetAuthenticatedUser(null, new GetAuthenticatedUserParameters());
if (authenticatedUser != null) // Something went wrong but we don't know what
{
_stream.AddTrack(keyword);
_stream.MatchingTweetReceived += (sender, args) =>
{
// Exclude RTs
if (args.Tweet.IsRetweet)
{
return;
}
if (_tweetsPerMinute.HasValue && _tweetsPerMinute.Value != default(int))
{
Throttle();
}
_tweetProcessor.ProcessTweetAsync(keyword, args);
_lastProcessedTime = Environment.TickCount;
Console.WriteLine($"[{DateTime.Now}] - Last processed time {_lastProcessedTime} ms.");
};
_stream.StartStreamMatchingAllConditions();
}
}
catch (TwitterException ex)
{
Console.WriteLine($"TwitterException Message : {ex.Message} - description : {ex.TwitterDescription}");
throw ex;
}
}
public async Task StartAsync(CancellationToken cancellationToken)
{
var consumerKey = Environment.GetEnvironmentVariable("TwitterConsumerKey");
var consumerSecret = Environment.GetEnvironmentVariable("TwitterConsumerSecret");
var accessKey = Environment.GetEnvironmentVariable("TwitterAccessKey");
var accessSecret = Environment.GetEnvironmentVariable("TwitterAccessSecret");
var userClient = new TwitterClient(consumerKey, consumerSecret, accessKey, accessSecret);
_filteredStream = userClient.Streams.CreateFilteredStream();
if (!string.IsNullOrEmpty(_attribute.Filter))
{
_filteredStream.AddTrack(_attribute.Filter);
}
if (!string.IsNullOrWhiteSpace(_attribute.User))
{
var user = await userClient.Users.GetUserAsync(_attribute.User);
_filteredStream.AddFollow(user);
}
_filteredStream.MatchingTweetReceived += async(obj, tweetEvent) =>
{
var triggerData = new TriggeredFunctionData
{
TriggerValue = tweetEvent
};
await Executor.TryExecuteAsync(triggerData, CancellationToken.None);
};
_filteredStream.DisconnectMessageReceived += (obj, disconnectEvent) =>
{
_filteredStream.Stop();
};
await _filteredStream.StartMatchingAllConditionsAsync();
}
public void StreamTwitter(Action <TweetDto, string> _Update, string _Connection, List <string> tracks, bool enableLocation)
{
cancellationToken = new CancellationTokenSource();
Connection = _Connection;
regex = new Regex(@"(?<=#)\w+");
Login();
Update = _Update;
GetLocation = new OpenStreetMapHelper();
stream = Stream.CreateFilteredStream();
tracks.ForEach(x => stream.AddTrack(x));
if (enableLocation)
{
stream.AddLocation(new Tweetinvi.Models.Coordinates(49.246292, -123.116226), new Tweetinvi.Models.Coordinates(-33.865143, 151.209900));
}
Task.Factory.StartNew(async() => await TweetAnalysis(), cancellationToken.Token);
stream.MatchingTweetReceived += (sender, args) =>
{
Tweets.Push(args.Tweet);
};
stream.StartStreamMatchingAllConditions();
}
public void SetUpListener()
{
// Set up your credentials (https://apps.twitter.com)
Auth.SetUserCredentials(Environment.GetEnvironmentVariable("CONSUMER_KEY"), Environment.GetEnvironmentVariable("CONSUMER_SECRET"),
Environment.GetEnvironmentVariable("ACCESS_TOKEN"), Environment.GetEnvironmentVariable("ACCESS_TOKEN_SECRET"));
// Publish the Tweet "Hello World" on your Timeline
//Tweet.PublishTweet("Hello World!");
_stream = Tweetinvi.Stream.CreateFilteredStream();
var keyWordString = File.ReadAllText("KeyWords.txt");
var keyWords = keyWordString.Split(',');
foreach (var word in keyWords)
{
_stream.AddTrack(word);
}
if (!File.Exists("out.txt"))
{
File.Create("out.txt");
}
var outputFile = new StreamWriter("out.txt", true);
_stream.MatchingTweetReceived += (sender, arguments) =>
{
string message = "A tweet containing '" + arguments.MatchingTracks[0] + "' has been found; the tweet is '" + arguments.Tweet + "' created by " + arguments.Tweet.CreatedBy;
Console.WriteLine(message);
RawDataEntry entry = new RawDataEntry
{
Id = "",
Source = "Twitter",
Author = arguments.Tweet.CreatedBy.ToString(),
Content = arguments.Tweet.ToString(),
TimeStamp = arguments.Tweet.CreatedAt
};
entry.Commit();
};
}
private void StartStreaming(string sequenceToken, int sampleSize, string filterValue)
{
var counter = 0;
var logEventsBatch = new List <InputLogEvent>();
_stream.AddTrack(filterValue);
// _stream.AddLocation(new Coordinates(32.0, -114.42), new Coordinates(41.96, -124.21));
_stream.MatchingTweetReceived += (sender, eventArgs) => {
Console.WriteLine(counter + ": " + eventArgs.Tweet);
var json = eventArgs.Tweet.ToJson();
var tweetInfo = JsonConvert.DeserializeObject <JObject>(json);
logEventsBatch.Add(new InputLogEvent {
Message = GetLogText(tweetInfo),
Timestamp = DateTime.Now
});
if (++counter % 50 == 0)
{
sequenceToken = DispatchLogEvents(logEventsBatch, sequenceToken);
logEventsBatch = new List <InputLogEvent>();
}
if (--sampleSize == 0)
{
sequenceToken = DispatchLogEvents(logEventsBatch, sequenceToken);
Stop();
}
};
_stream.StreamStopped += (sender, args) => {
if (args.Exception != null)
{
Console.WriteLine($"Stream stopped with exception: {args.Exception}");
}
if (args.DisconnectMessage != null)
{
Console.WriteLine($"Disconnect message: {args.DisconnectMessage}");
}
};
_stream.StartStreamMatchingAllConditions();
}
public static async Task RegisterFilteredStreamEventsAsync(string[] users, string[] trackKeywords)
{
if (users == null)
{
log.Fatal("No Users Are Being Followed. Server is useless.");
return;
}
try
{
stream = Stream.CreateFilteredStream();
foreach (string userName in users)
{
IUserIdentifier myUser = Tweetinvi.User.GetUserFromScreenName(userName);
stream.AddFollow(myUser);
}
if (trackKeywords != null)
{
foreach (string keyword in trackKeywords)
{
stream.AddTrack(keyword);
}
}
stream.MatchingTweetReceived += StreamEventCallbackManager.FilteredStream_MatchingTweetReceived;
stream.StreamStopped += StreamEventCallbackManager.Stream_Stopped;
stream.DisconnectMessageReceived += StreamEventCallbackManager.Stream_Disconnected;
stream.StreamPaused += StreamEventCallbackManager.Stream_Paused;
stream.StreamResumed += StreamEventCallbackManager.Stream_Resumed;
await stream.StartStreamMatchingAllConditionsAsync();
log.Info("Now Listening To Stream Events.");
}
catch (Exception e)
{
log.Fatal("FAILED TO REGISTER TO STREAM EVENTS!", e);
}
}
public static async Task StartStream()
{
var consumerKey = Environment.GetEnvironmentVariable("TWITTER_CONSUMER_KEY", EnvironmentVariableTarget.User);
var consumerSecret = Environment.GetEnvironmentVariable("TWITTER_CONSUMER_SECRET", EnvironmentVariableTarget.User);
var accessToken = Environment.GetEnvironmentVariable("TWITTER_ACCESS_TOKEN", EnvironmentVariableTarget.User);
var accessTokenSecret = Environment.GetEnvironmentVariable("TWITTER_ACCESS_TOKEN_SECRET", EnvironmentVariableTarget.User);
var credentials = Auth.SetUserCredentials(consumerKey, consumerSecret, accessToken, accessTokenSecret);
if (_stream == null)
{
_stream = Stream.CreateFilteredStream(credentials);
_stream.AddTrack("#WorldCup");
_stream.MatchingTweetReceived += async(sender, args) =>
{
await HubContext.Clients.All.SendTweet(args.Tweet.FullText);
};
await _stream.StartStreamMatchingAllConditionsAsync();
}
else
{
_stream.ResumeStream();
}
}
private void StartTask(List<string> streamingQueries, List<string> streamingQueriesIDs, List<ArrayList<ArrayList<QueryToken>>> searchTokens, List<string> searchQueriesIDs, Dictionary<string, string> streamingQueryEntities, List<string> authData)
{
if (authData.Count != 4)
{
StreamLogger.writeErrorToConsoleAndExit(streamTaskId, String.Format("Twitter authentication data requires exactly 4 values, {0} given", authData.Count),
"TwitterStreamingDacq.StreamTask", "StartTask");
}
if (streamingQueries.Count != streamingQueriesIDs.Count)
{
StreamLogger.writeErrorToConsoleAndExit(streamTaskId, String.Format("Length of queries {0} does not mach the length of queries IDs {1}", streamingQueries.Count, streamingQueriesIDs.Count),
"TwitterStreamingDacq.StreamTask", "StartTask");
}
var credentials = new TwitterCredentials(
authData[0], // ConsumerKey
authData[1], // ConsumerSecret
authData[2], // WorkerToken
authData[3] // WorkerSecret
);
filteredStream = Stream.CreateFilteredStream(credentials);
// match queries only on the tweets text
filteredStream.MatchOn = MatchOn.TweetText;
for (int i = 0; i < streamingQueries.Count; ++i)
{
string track = streamingQueries[i];
string streamingQueryID = streamingQueriesIDs[i];
string queryIDbase = StreamConfigParser.ExtractQueryIDBase(streamingQueryID);
string entity = "";
if (streamingQueryEntities.ContainsKey(queryIDbase))
entity = streamingQueryEntities[queryIDbase];
filteredStream.AddTrack(track, tweet =>
{
// triggered when this query gets a tweet
StreamToSearchAssigner ssa = new StreamToSearchAssigner();
List<string> matchingSearchIds = ssa.AssignStreamToSearch(tweet, streamingQueryID, searchTokens, searchQueriesIDs);
if (matchingSearchIds.Count > 0)
{
// check if json tweet matches the retrieved tweet
var jsonConvert = TweetinviContainer.Resolve<IJsonObjectConverter>();
var tweetDTO = jsonConvert.DeserializeObject<ITweetDTO>(lastTweetJson);
var tweetFromJson = Tweet.GenerateTweetFromDTO(tweetDTO);
bool matchingId = tweetFromJson.IdStr == tweet.IdStr;
if (tweet.IsTweetDestroyed)
CreateTweetDeleteNotification(tweet);
// if the recieved json of the tweet does not match the tracked tweet skip it and create a warning
if (tweetFromJson.IdStr == tweet.IdStr)
{
// match entity
if (String.IsNullOrEmpty(entity))
{
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Entity for sreaming query {0} could not be found! Entity set to {1}.", streamingQueryID, queryIDbase), StreamLogger.msgType.Warning,
"TwitterStreamingDacq.StreamTask", "StartTask.AddTrack", true);
entity = queryIDbase;
}
// save to nest db
PutStreamedTweetToNest(entity, tweet, lastTweetJson);
}
else
{
StreamLogger.writeMsgToConsole(streamTaskId, "Retrived JSON tweet and matched tweet do not match. Skipping this tweet", StreamLogger.msgType.Warning,
"TwitterStreamingDacq.StreamTask", "StartTask.AddTrack", true);
}
}
});
}
filteredStream.JsonObjectReceived += (sender, args) =>
{
lastTweetJson = args.Json;
};
filteredStream.DisconnectMessageReceived += (senderTwt, argsTwt) =>
{
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("DisconnectMessageReceived! {0}", argsTwt.DisconnectMessage), StreamLogger.msgType.Warning,
"TwitterStreamingDacq.StreamTask", "StartTask.DisconnectMessageReceived", true);
};
// notifies that stream attempts to access more than 1% of the total tweets published
filteredStream.LimitReached += (sender, args) =>
{
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Streaming limit reached! {0} tweets were not received from the start of streaming.",
args.NumberOfTweetsNotReceived), StreamLogger.msgType.Warning,
"TwitterStreamingDacq.StreamTask", "StartTask.LimitReached", true);
};
// notifies that a tweet has been deleted
filteredStream.TweetDeleted += (sender, args) =>
{
PutDeletedNotificationToNest(args.TweetDeletedInfo);
};
filteredStream.StreamStarted += (sender, args) =>
{
StreamLogger.writeMsgToConsole(streamTaskId, "Streaming started", StreamLogger.msgType.Empty, "", "", true);
};
// let us manage the exceptions from Tweetinvi
ExceptionHandler.SwallowWebExceptions = false;
// streaming has stopped, check for exception, log it, restart streaming
filteredStream.StreamStopped += (sender, args) =>
{
var disconnectMessage = args.DisconnectMessage;
var exception = args.Exception;
StreamLogger.writeMsgToConsole(streamTaskId, "Streaming stopped", StreamLogger.msgType.Empty,
"", "", true);
if (disconnectMessage != null)
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Disconnect message: {0}", disconnectMessage), StreamLogger.msgType.Error,
"TwitterStreamingDacq.StreamTask", "StartTask.StreamStopped", true);
if (exception != null)
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Exception message: {0}", exception.Message), StreamLogger.msgType.Error,
"TwitterStreamingDacq.StreamTask", "StartTask.StreamStopped", true);
};
DateTime streamStartedTime = new DateTime();
int numOfStreamRestarts = 0;
int numOfMaxStreamRestarts = Convert.ToInt32(ConfigurationManager.AppSettings["StreamTimeoutRestarts"]);
if (numOfMaxStreamRestarts < 0)
numOfMaxStreamRestarts = Int32.MaxValue;
// start streaming
while (true)
{
streamStartedTime = DateTime.Now;
try
{
filteredStream.StartStreamMatchingAnyCondition();
}
catch (Exception exception)
{
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Exception message: {0}", exception.Message), StreamLogger.msgType.Error,
"TwitterStreamingDacq.StreamTask", "StartTask.StartStreamLoop", true);
Console.WriteLine(exception.StackTrace);
}
// twitter hates too many streaming restarts in a short period of time, so wait a little bit before restarting
Thread.Sleep(15000);
// if there are too many restarts in a short period shutdown this loop altogether
if (DateTime.Now.Subtract(streamStartedTime).TotalSeconds < 60)
numOfStreamRestarts++;
else
numOfStreamRestarts = 0;
if (numOfStreamRestarts >= numOfMaxStreamRestarts)
{
StreamLogger.writeErrorToConsoleAndExit(streamTaskId, "Stream stopped completely due to too many restarts",
"TwitterStreamingDacq.StreamTask", "StartTask.StartStreamLoop");
break;
}
}
}