/// <summary>
/// This method streams the tweets with given keyword
/// </summary>
/// <param name="keyword"></param>
/// <returns></returns>
public static List <Tweet> GetTweetsWithKeyword(string keyword)
{
var tweets = default(IEnumerable <ITweet>);
var downloadedTweets = new List <Tweet>();
// Search for all tweets published at a specific location
//var searchParameter = Search.GenerateTweetSearchParameter(-51.5072, 0.1275, 30, DistanceMeasure.Kilometers);
filteredStream = Stream.CreateFilteredStream();
try
{
tweets = Search.SearchTweets(keyword); //unable to stream more than 100 tweets
}
catch
{
Console.WriteLine(" Couldn't download tweets, not trying again...");
//Thread.Sleep(4000);
}
if (tweets != null && tweets.Count() > 0)
{
foreach (var tweet in tweets)
{
var atweet = new Tweet(tweet);
downloadedTweets.Add(atweet);
}
}
return(downloadedTweets);
}
public async Task StartAsync()
{
_eventHubClient = EventHubClient.CreateFromConnectionString(_targetConfig.ConnectionString, _targetConfig.EventHubName);
Auth.SetUserCredentials(_sourceConfig.Credentials.ConsumerKey, _sourceConfig.Credentials.ConsumerSecret, _sourceConfig.Credentials.AccessToken, _sourceConfig.Credentials.AccessTokenSecret);
IFilteredStream stream = Stream.CreateFilteredStream();
foreach (var language in _sourceConfig.Languages)
{
stream.AddTweetLanguageFilter((LanguageFilter)language);
}
foreach (string track in _sourceConfig.Tracks)
{
stream.AddTrack(track);
}
stream.MatchingTweetReceived += OnMatchingTweetReceived;
stream.DisconnectMessageReceived += OnDisconnectMessageReceived;
stream.StreamStopped += OnStreamStopped;
stream.StallWarnings = true;
await stream.StartStreamMatchingAnyConditionAsync();
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
Console.WriteLine("What topic would you like to get tweets about?");
string subject = Console.ReadLine();
while (!stoppingToken.IsCancellationRequested)
{
ITwitterCredentials demo = Auth.SetUserCredentials(_options.ConsumerKey, _options.ConsumerSecret, _options.AccessToken, _options.AccessTokenSecret);
IFilteredStream stream = Stream.CreateFilteredStream();
stream.AddTrack(subject);
stream.AddTweetLanguageFilter(LanguageFilter.Portuguese);
_logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
_logger.LogInformation("I'm listening to twitter");
stream.MatchingTweetReceived += async(sender, arguments) =>
{
await SendTweetsByKafka(new Tweet()
{
TweetedBy = arguments.Tweet.CreatedBy.ScreenName, TweetedAt = arguments.Tweet.CreatedAt, Text = arguments.Tweet.Text
});
};
stream.StartStreamMatchingAllConditions();
await Task.Delay(5000, stoppingToken);
}
}
public TwitterStreamingEngine()
{
stream = Stream.CreateFilteredStream();
twitterSender = new EventHubSender(RoleEnvironment.GetConfigurationSettingValue("EventHubName"));
twitterLike = new EventHubSender(RoleEnvironment.GetConfigurationSettingValue("EventHubTwitterLike"));
twitterDislike = new EventHubSender(RoleEnvironment.GetConfigurationSettingValue("EventHubTwitterDislike"));
}
public TwitterWorker(TweetStorage storage)
{
_storage = storage;
_stream = Stream.CreateFilteredStream();
_stream.MatchingTweetAndLocationReceived += _stream_MatchingTweetAndLocationReceived;
}
public StreamFactory(ITweetProcessor tweetProcessor, ITwitterCredentials credentials, IFilteredStream stream)
{
_tweetProcessor = tweetProcessor;
_stream = stream ?? Stream.CreateFilteredStream();
Auth.SetUserCredentials(credentials.ConsumerKey, credentials.ConsumerSecret,
credentials.AccessToken, credentials.AccessTokenSecret);
}
public void StartStream()
{
FilteredStream = Stream.CreateFilteredStream();
FilteredStream.AddTrack(Hashtag);
FilteredStream.MatchingTweetReceived += (sender, args) => RaiseTweetAdded(args.Tweet);
FilteredStream.StartStreamMatchingAllConditionsAsync();
}
/*Constructor: takes the keys needed, last paramets are filters for the stream*/
public TwitterStreamProcessor(string consumerKey, string consumerSecret,
string accessToken, string accessTokenSecret, params string[] filters)
{
PictureDataQueue = new Queue <PictureData>();
PictureMetaDataList = new List <PictureMetaData>();
//The credentials for the twitter api
var cred = new TwitterCredentials(consumerKey, consumerSecret,
accessToken, accessTokenSecret);
stream = Tweetinvi.Stream.CreateFilteredStream(cred);
//insert the filters
foreach (var filter in filters)
{
stream.AddTrack(filter);
}
//Delegate the method responsible for the handling of the matches
stream.MatchingTweetReceived += handleMatchingTweet;
//start the stream as a task: will run "forever"
Task.Factory.StartNew(() => stream.StartStreamMatchingAllConditions());
StreamStart = DateTime.Now;
}
/// <summary>
/// User wants to stop the stream.
/// </summary>
protected void OnStop(object sender, EventArgs e)
{
if (stream != null)
{
stream.StreamStopped += (s, args) => stream = null;
stream.StopStream();
}
}
private IFilteredStream CreateStream()
{
IFilteredStream stream = Stream.CreateFilteredStream();
stream.MatchingTweetReceived += (sender, args) => DispatchTweet(args.Tweet);
return(stream);
}
private static void InitializeStream()
{
stream = Stream.CreateFilteredStream();
stream.AddFollow(user);
stream.StreamStopped += StreamStopped;
stream.StreamStarted += StreamStarted;
stream.MatchingTweetReceived += MatchingTweetReceived;
}
public static void initializeStream()
{
Auth.ApplicationCredentials = new TwitterCredentials(Constants.TConsumerKey, Constants.TConsumerKeySecret, Constants.TAccessToken, Constants.TAcessTokenSecret);
stream = Stream.CreateFilteredStream();
Debug.WriteLine("***** INITIALIZED TWEETSTREAM (should happen only once) *****");
Task.Run(() => startTweetStream("olympics"));
Task.Run(() => startTimer());
}
public void Stop()
{
Stream?.PauseStream();
Stream?.StopStream();
if (Stream?.StreamState == StreamState.Stop)
{
Stream = null;
}
}
private async Task ConnectToTwitterStream()
{
var stream = Stream.CreateFilteredStream();
stream.AddTrack(HASHTAG);
stream.MatchingTweetReceived += async(s, e) => await ProcessIncomingTweet(e.Tweet);
_twitterStream = stream;
await stream.StartStreamMatchingAllConditionsAsync();
}
public static async Task Init(IServiceProvider service)
{
_serviceProvider = service;
_stream = Stream.CreateFilteredStream();
_stream.AddTrack(_searchTerm);
_stream.MatchingTweetReceived += TweetRecieved;
await _stream.StartStreamMatchingAllConditionsAsync();
}
public void Initialize()
{
_logger.Write("Configuring Twitter contact source...");
Auth.SetUserCredentials(_configuration.ConsumerKey, _configuration.ConsumerSecret, _configuration.UserToken, _configuration.UserSecret);
_stream = Stream.CreateFilteredStream();
_stream.AddTrack(_configuration.PhrasesToTrack);
_stream.MatchingTweetReceived += Stream_MatchingTweetReceived;
_logger.WriteLine("Initialization of Twitter Harvester complete");
}
private void ListenerThreadFunction()
{
TwitterCredentials.SetCredentials(
ConfigurationManager.AppSettings["token_AccessToken"],
ConfigurationManager.AppSettings["token_AccessTokenSecret"],
ConfigurationManager.AppSettings["token_ConsumerKey"],
ConfigurationManager.AppSettings["token_ConsumerSecret"]);
while (threadRunning)
{
try
{
//var hbase = new HBaseWriter();
stream = Stream.CreateFilteredStream();
var location = Geo.GenerateLocation(-180, -90, 180, 90);
stream.AddLocation(location);
var tweetCount = 0;
var timer = Stopwatch.StartNew();
stream.MatchingTweetReceived += (sender, args) =>
{
tweetCount++;
var tweet = args.Tweet;
// Store indexItem in the buffer
lock (queue)
{
queue.Enqueue(tweet);
}
if (timer.ElapsedMilliseconds > 1000)
{
if (tweet.Coordinates != null)
{
Context.Logger.Info("{0}: {1} {2}", tweet.Id, tweet.Language.ToString(), tweet.Text);
Context.Logger.Info("\tLocation: {0}, {1}", tweet.Coordinates.Longitude, tweet.Coordinates.Latitude);
}
timer.Restart();
Context.Logger.Info("===== Tweets/sec: {0} =====", tweetCount);
tweetCount = 0;
}
};
stream.StartStreamMatchingAllConditions();
}
catch (Exception ex)
{
Context.Logger.Fatal("Exception: {0}", ex.Message);
}
}
}
private void StopStreams()
{
if (threadStream != null && stream != null)
{
threadStream.Abort();
threadShow.Abort();
threadShow = null;
threadStream = null;
stream.PauseStream();
stream = null;
}
}
private void ReCreateFilteredStream()
{
if (filteredStream != null)
{
filteredStream.StopStream();
filteredStream.MatchingTweetReceived -= FilteredStreamOnMatchingTweetReceived;
}
filteredStream = Stream.CreateFilteredStream();
filteredStream.AddTweetLanguageFilter(Language.Spanish);
filteredStream.MatchingTweetReceived += FilteredStreamOnMatchingTweetReceived;
}
public static void MineTwitter(string term)
{
TwitterCredentials.SetCredentials(
ConfigurationManager.AppSettings["AccessToken"],
ConfigurationManager.AppSettings["AccessSecret"],
ConfigurationManager.AppSettings["ConsumerKey"],
ConfigurationManager.AppSettings["ConsumerSecret"]);
IFilteredStream stream = Tweetinvi.Stream.CreateFilteredStream();
stream.MatchingTweetReceived += Stream_TweetReceived;
stream.AddTrack(term);
stream.StartStreamMatchingAllConditions();
}
public TwitterService(
ILogger <TwitterService <T> > logger,
IHubContext <T, ITwitterClient> hubContext,
ISentimentService sentimentService,
IFilteredStream filteredStream)
{
_logger = logger;
_hubContext = hubContext;
_sentimentService = sentimentService;
_filteredStream = filteredStream;
InitializeStream();
}
//--- Constructors ---
public TwitterStream()
{
var region = RegionEndpoint.GetBySystemName(AWS_REGION);
var chain = new CredentialProfileStoreChain();
AWSCredentials awsCredentials;
if (!chain.TryGetAWSCredentials("lambdasharp", out awsCredentials))
{
throw new Exception("AWS Credentials not found!");
}
_client = new AmazonCloudWatchLogsClient(awsCredentials, region);
SetTwitterCredentials();
_stream = Stream.CreateFilteredStream();
}
/// <summary>
/// Start a stream, filtering ony the keyword and only English language tweets.
/// </summary>
protected void StartStream(string keyword)
{
stream = Stream.CreateFilteredStream();
stream.AddTrack(keyword);
stream.MatchingTweetReceived += (sender, args) =>
{
if (args.Tweet.Language == Language.English)
{
UpdateFdg(args.Tweet.Text);
}
};
stream.StartStreamMatchingAllConditionsAsync();
}
public TwitterDataManager(bool isTest = false)
{
_filteredStream = Stream.CreateFilteredStream();
var accessToken = Constants.AccessToken;
var accessTokenSecret = Constants.AccessTokenSecret;
var consumerKey = Constants.ConsumerKey;
var consumerSecret = Constants.ConsumerSecret;
Auth.ApplicationCredentials = new TwitterCredentials(consumerKey, consumerSecret, accessToken, accessTokenSecret);
Auth.SetUserCredentials(consumerKey, consumerSecret, accessToken, accessTokenSecret);
if (isTest)
{
_isTest = true;
}
}
public override void SetupEvent()
{
TwitterConnector twitterConnector = this.InParameters["twitter"].GetValue() as TwitterConnector;
this.stream = twitterConnector.Client.Streams.CreateFilteredStream();
stream.MatchingTweetReceived += Stream_MatchingTweetReceived;
var twitterUser = twitterConnector.Client.Users.GetUserAsync(this.InParameters["user"].GetValue().ToString());
twitterUser.Wait();
stream.AddFollow(twitterUser.Result);
stream.StartMatchingAllConditionsAsync();
}
private void StartTwitterStream()
{
TwitterStream = Tweetinvi.Stream.CreateFilteredStream();
TwitterStream.MatchingTweetReceived += OnTweetReceived;
TwitterStream.StallWarnings = true;
TwitterStream.WarningFallingBehindDetected += (sender, args) =>
{
Log.WriteWarn("Twitter", $"Stream falling behind: {args.WarningMessage.PercentFull} {args.WarningMessage.Code} {args.WarningMessage.Message}");
};
TwitterStream.StreamStopped += (sender, args) =>
{
var ex = args.Exception;
var twitterDisconnectMessage = args.DisconnectMessage;
if (ex != null)
{
Log.WriteError("Twitter", ex.ToString());
}
if (twitterDisconnectMessage != null)
{
Log.WriteError("Twitter", $"Stream stopped: {twitterDisconnectMessage.Code} {twitterDisconnectMessage.Reason}");
}
TwitterStream.StopStream();
Task.Factory.StartNew(() =>
{
Thread.Sleep(5000);
TwitterStream.StartStreamMatchingAnyConditionAsync();
});
};
var twitterUsers = Tweetinvi.User.GetUsersFromScreenNames(Config.Twitter.AccountsToFollow.Keys);
foreach (var user in twitterUsers)
{
var channels = Config.Twitter.AccountsToFollow.First(u => u.Key.Equals(user.ScreenName, StringComparison.InvariantCultureIgnoreCase));
Log.WriteInfo("Twitter", $"Following @{user.ScreenName}");
TwitterToChannels.Add(user.Id, channels.Value);
TwitterStream.AddFollow(user);
}
TwitterStream.StartStreamMatchingAnyConditionAsync();
}
public void CreateStream()
{
var credentials = new TwitterCredentials(twitterSettings.ConsumerKey, twitterSettings.ConsumerSecret,
twitterSettings.AccessToken, twitterSettings.AccessTokenSecret);
Auth.SetCredentials(credentials);
if (stream != null)
{
this.StopStream();
}
stream = Stream.CreateFilteredStream(credentials);
}
public FilteredStream(string keyword, bool isFake = false)
{
_isFake = isFake;
_stop = true;
if (_isFake)
{
SetupFakeObservable(keyword);
}
else
{
_filteredStream = Stream.CreateFilteredStream();
_filteredStream.AddTrack(keyword);
}
}
public async Task MonitorTweetsAsync()
{
_twitterStream = _twitterClient.Streams.CreateFilteredStream();
SetStreamTrackers();
_twitterStream.MatchingTweetReceived += C_ProcessTweet;
try
{
await _twitterStream.StartMatchingAnyConditionAsync();
}
catch
{
Thread.Sleep(5000);
await MonitorTweetsAsync();
}
}
public TwitterStream(string consumerKey, string consumerSecret, string accessTokenKey, string accessTokenSecret)
: base(consumerKey, consumerSecret, accessTokenKey, accessTokenSecret)
{
_stream = Tweetinvi.Stream.CreateFilteredStream();
_stream.FilterLevel = StreamFilterLevel.Low;
_stream.StallWarnings = true;
_stream.MatchingTweetReceived += _stream_MatchingTweetReceived;
_stream.StreamStarted += _stream_StreamStarted;
_stream.StreamPaused += _stream_StreamPaused;
_stream.StreamResumed += _stream_StreamResumed;
_stream.StreamStopped += _stream_StreamStopped;
_stream.LimitReached += _stream_LimitReached;
_stream.JsonObjectReceived += _stream_JsonObjectReceived;
}
public void Run()
{
if (Stream == null)
{
Stream = Tweetinvi.Stream.CreateFilteredStream();
}
RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly;
Stream.Credentials = Auth.Credentials;
for (var i = 5; i < 151; i += 5)
{
Stream.AddTrack($"Lv{i}");
}
Stream.AddTrack("I need backup!Battle ID:");
Start();
}
public void Start()
{
_stream = Stream.CreateFilteredStream();
_stream.AddTrack("tweetinvi");
_stream.Credentials = _credentials;
_stream.MatchingTweetReceived += (sender, args) =>
{
ITweet tweet = args.Tweet;
_hubContext.Clients.All.broadcastTweet(tweet);
};
_backgroundThread = new Thread(_stream.StartStreamMatchingAllConditions);
}
public void InitListen()
{
cantTweets = 0;
Console.Out.WriteLine("Tweets test");
TwitterCredentials.SetCredentials("4460502861-Q1jsfXwbNoQAqPGqZVOF0oQFiOg4mODzgodyzK1", "VPD8QHpv4hBWVt5GHh98HxXrfql0lHzmU5BjxhWM7wkET", "2iNRWcj8m046oF8L70TF7CmI5", "ASLWbyUZboUCBrfKDDKXpA43fhLln5B9vy7TlXb4l6F6YLDkYO");
myStream = Tweetinvi.Stream.CreateFilteredStream();
DbTopic cfk = new DbTopic(dbName, "CFK", new List<String>() { "cristina kirchner", "@cfkargentina", "cfk" }, null, Domain_nacion);
ListenTo(cfk);
// A presidente / vice
DbTopic sci = new DbTopic(dbName, "Sci", new List<String>() { "scioli", "@danielscioli", "daniel scioli" }, null, Domain_nacion);
ListenTo(sci);
DbTopic mac = new DbTopic(dbName, "Mac", new List<String>() { "macri", "@mauriciomacri", "mauricio macri" }, new List<String>() { "jorge" }, Domain_nacion);
ListenTo(mac);
DbTopic snz = new DbTopic(dbName, "Sanz", new List<String>() { "sanz", "@sanzernesto", "ernesto sanz" }, new List<String>() { "alejandro" }, Domain_nacion);
ListenTo(snz);
DbTopic car = new DbTopic(dbName, "Carr", new List<String>() { "carrió", "@elisacarrio", "carrio", "lilita", "elisa carrio", "elisa carrió" }, null, Domain_nacion);
ListenTo(car);
DbTopic mas = new DbTopic(dbName, "Mas", new List<String>() { "massa", "@sergiomassa", "sergio massa" }, null, Domain_nacion);
ListenTo(mas);
DbTopic dls = new DbTopic(dbName, "DlS", new List<String>() { "de la sota", "@delasotaok", "manuel de la sota" }, null, Domain_nacion);
ListenTo(dls);
DbTopic stol = new DbTopic(dbName, "Stol", new List<String>() { "stolbizer", "@stolbizer", "margarita stolbizer" }, null, Domain_nacion);
ListenTo(stol);
DbTopic alt = new DbTopic(dbName, "Alt", new List<String>() { "altamira", "@altamirajorge", "jorge altamira" }, null, Domain_nacion);
ListenTo(alt);
DbTopic dca = new DbTopic(dbName, "dCñ", new List<String>() { "del caño", "@nicolasdelcano", "nicolás del caño", "nicolas del caño" }, null, Domain_nacion);
ListenTo(dca);
// A gobernador de Buenos Aires / vice
DbTopic afz = new DbTopic(dbName, "AFz", new List<String>() { "anibal", "@fernandezanibal", "aníbal", "aníbal fernández", "anibal fernandez" }, null, Domain_provincia);
ListenTo(afz);
DbTopic jDz = new DbTopic(dbName, "Mas", new List<String>() { "julian dominguez", "@dominguezjul", "julián domínguez" }, null, Domain_provincia);
ListenTo(jDz);
DbTopic vid = new DbTopic(dbName, "Vid", new List<String>() { "maria eugenia vidal", "@mariuvidal", "maría eugenia vidal" }, null, Domain_provincia);
ListenTo(vid);
DbTopic chi = new DbTopic(dbName, "Chi", new List<String>() { "christian castillo", "@chipicastillo" }, null, Domain_provincia);
ListenTo(chi);
DbTopic pit = new DbTopic(dbName, "Pit", new List<String>() { "pitrola", "@nestorpitrola", "nestor pitrola", }, null, Domain_provincia);
ListenTo(pit);
DbTopic lin = new DbTopic(dbName, "Lin", new List<String>() { "jaime linares", "@linaresjaime" }, null, Domain_provincia);
ListenTo(lin);
DbTopic sol = new DbTopic(dbName, "Sol", new List<String>() { "felipe solá", "@felipe_sola", "felipe sola", }, null, Domain_provincia);
ListenTo(sol);
// Agregar apodos, que presupongan neg?
myStream.StreamStopped += (sender, args) =>
{
Console.Out.Write("(Stream stopped) ");
};
}
private bool configureStream()
{
bool result = false;
try
{
this._stream = Tweetinvi.Stream.CreateFilteredStream();
this._stream.AddTrack(this._model.Hashtag);
this._stream.MatchingTweetReceived += (sender, args) =>
{
this.Receive(sender, args);
};
result = true;
}
catch (Exception ex)
{
this._errorCount++;
Logger.Log("TweetPicTask: Stream (" + this._model.Hashtag + ") configureStream error: " + ex.ToString());
}
return result;
}
public static void Start()
{
ConfigureTwitterCredentials();
defaultStream = CreateFilteredStream();
}
private void ListenerThreadFunction()
{
TwitterCredentials.SetCredentials(
ConfigurationManager.AppSettings["token_AccessToken"],
ConfigurationManager.AppSettings["token_AccessTokenSecret"],
ConfigurationManager.AppSettings["token_ConsumerKey"],
ConfigurationManager.AppSettings["token_ConsumerSecret"]);
while (threadRunning)
{
try
{
//var hbase = new HBaseWriter();
stream = Stream.CreateFilteredStream();
var location = Geo.GenerateLocation(-180, -90, 180, 90);
stream.AddLocation(location);
var tweetCount = 0;
var timer = Stopwatch.StartNew();
stream.MatchingTweetReceived += (sender, args) =>
{
tweetCount++;
var tweet = args.Tweet;
// Store indexItem in the buffer
lock (queue)
{
queue.Enqueue(tweet);
}
if (timer.ElapsedMilliseconds > 1000)
{
if (tweet.Coordinates != null)
{
Context.Logger.Info("{0}: {1} {2}", tweet.Id, tweet.Language.ToString(), tweet.Text);
Context.Logger.Info("\tLocation: {0}, {1}", tweet.Coordinates.Longitude, tweet.Coordinates.Latitude);
}
timer.Restart();
Context.Logger.Info("===== Tweets/sec: {0} =====", tweetCount);
tweetCount = 0;
}
};
stream.StartStreamMatchingAllConditions();
}
catch (Exception ex)
{
Context.Logger.Fatal("Exception: {0}", ex.Message);
}
}
}
private void StartTask(List<string> streamingQueries, List<string> streamingQueriesIDs, List<ArrayList<ArrayList<QueryToken>>> searchTokens, List<string> searchQueriesIDs, Dictionary<string, string> streamingQueryEntities, List<string> authData)
{
if (authData.Count != 4)
{
StreamLogger.writeErrorToConsoleAndExit(streamTaskId, String.Format("Twitter authentication data requires exactly 4 values, {0} given", authData.Count),
"TwitterStreamingDacq.StreamTask", "StartTask");
}
if (streamingQueries.Count != streamingQueriesIDs.Count)
{
StreamLogger.writeErrorToConsoleAndExit(streamTaskId, String.Format("Length of queries {0} does not mach the length of queries IDs {1}", streamingQueries.Count, streamingQueriesIDs.Count),
"TwitterStreamingDacq.StreamTask", "StartTask");
}
var credentials = new TwitterCredentials(
authData[0], // ConsumerKey
authData[1], // ConsumerSecret
authData[2], // WorkerToken
authData[3] // WorkerSecret
);
filteredStream = Stream.CreateFilteredStream(credentials);
// match queries only on the tweets text
filteredStream.MatchOn = MatchOn.TweetText;
for (int i = 0; i < streamingQueries.Count; ++i)
{
string track = streamingQueries[i];
string streamingQueryID = streamingQueriesIDs[i];
string queryIDbase = StreamConfigParser.ExtractQueryIDBase(streamingQueryID);
string entity = "";
if (streamingQueryEntities.ContainsKey(queryIDbase))
entity = streamingQueryEntities[queryIDbase];
filteredStream.AddTrack(track, tweet =>
{
// triggered when this query gets a tweet
StreamToSearchAssigner ssa = new StreamToSearchAssigner();
List<string> matchingSearchIds = ssa.AssignStreamToSearch(tweet, streamingQueryID, searchTokens, searchQueriesIDs);
if (matchingSearchIds.Count > 0)
{
// check if json tweet matches the retrieved tweet
var jsonConvert = TweetinviContainer.Resolve<IJsonObjectConverter>();
var tweetDTO = jsonConvert.DeserializeObject<ITweetDTO>(lastTweetJson);
var tweetFromJson = Tweet.GenerateTweetFromDTO(tweetDTO);
bool matchingId = tweetFromJson.IdStr == tweet.IdStr;
if (tweet.IsTweetDestroyed)
CreateTweetDeleteNotification(tweet);
// if the recieved json of the tweet does not match the tracked tweet skip it and create a warning
if (tweetFromJson.IdStr == tweet.IdStr)
{
// match entity
if (String.IsNullOrEmpty(entity))
{
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Entity for sreaming query {0} could not be found! Entity set to {1}.", streamingQueryID, queryIDbase), StreamLogger.msgType.Warning,
"TwitterStreamingDacq.StreamTask", "StartTask.AddTrack", true);
entity = queryIDbase;
}
// save to nest db
PutStreamedTweetToNest(entity, tweet, lastTweetJson);
}
else
{
StreamLogger.writeMsgToConsole(streamTaskId, "Retrived JSON tweet and matched tweet do not match. Skipping this tweet", StreamLogger.msgType.Warning,
"TwitterStreamingDacq.StreamTask", "StartTask.AddTrack", true);
}
}
});
}
filteredStream.JsonObjectReceived += (sender, args) =>
{
lastTweetJson = args.Json;
};
filteredStream.DisconnectMessageReceived += (senderTwt, argsTwt) =>
{
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("DisconnectMessageReceived! {0}", argsTwt.DisconnectMessage), StreamLogger.msgType.Warning,
"TwitterStreamingDacq.StreamTask", "StartTask.DisconnectMessageReceived", true);
};
// notifies that stream attempts to access more than 1% of the total tweets published
filteredStream.LimitReached += (sender, args) =>
{
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Streaming limit reached! {0} tweets were not received from the start of streaming.",
args.NumberOfTweetsNotReceived), StreamLogger.msgType.Warning,
"TwitterStreamingDacq.StreamTask", "StartTask.LimitReached", true);
};
// notifies that a tweet has been deleted
filteredStream.TweetDeleted += (sender, args) =>
{
PutDeletedNotificationToNest(args.TweetDeletedInfo);
};
filteredStream.StreamStarted += (sender, args) =>
{
StreamLogger.writeMsgToConsole(streamTaskId, "Streaming started", StreamLogger.msgType.Empty, "", "", true);
};
// let us manage the exceptions from Tweetinvi
ExceptionHandler.SwallowWebExceptions = false;
// streaming has stopped, check for exception, log it, restart streaming
filteredStream.StreamStopped += (sender, args) =>
{
var disconnectMessage = args.DisconnectMessage;
var exception = args.Exception;
StreamLogger.writeMsgToConsole(streamTaskId, "Streaming stopped", StreamLogger.msgType.Empty,
"", "", true);
if (disconnectMessage != null)
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Disconnect message: {0}", disconnectMessage), StreamLogger.msgType.Error,
"TwitterStreamingDacq.StreamTask", "StartTask.StreamStopped", true);
if (exception != null)
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Exception message: {0}", exception.Message), StreamLogger.msgType.Error,
"TwitterStreamingDacq.StreamTask", "StartTask.StreamStopped", true);
};
DateTime streamStartedTime = new DateTime();
int numOfStreamRestarts = 0;
int numOfMaxStreamRestarts = Convert.ToInt32(ConfigurationManager.AppSettings["StreamTimeoutRestarts"]);
if (numOfMaxStreamRestarts < 0)
numOfMaxStreamRestarts = Int32.MaxValue;
// start streaming
while (true)
{
streamStartedTime = DateTime.Now;
try
{
filteredStream.StartStreamMatchingAnyCondition();
}
catch (Exception exception)
{
StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Exception message: {0}", exception.Message), StreamLogger.msgType.Error,
"TwitterStreamingDacq.StreamTask", "StartTask.StartStreamLoop", true);
Console.WriteLine(exception.StackTrace);
}
// twitter hates too many streaming restarts in a short period of time, so wait a little bit before restarting
Thread.Sleep(15000);
// if there are too many restarts in a short period shutdown this loop altogether
if (DateTime.Now.Subtract(streamStartedTime).TotalSeconds < 60)
numOfStreamRestarts++;
else
numOfStreamRestarts = 0;
if (numOfStreamRestarts >= numOfMaxStreamRestarts)
{
StreamLogger.writeErrorToConsoleAndExit(streamTaskId, "Stream stopped completely due to too many restarts",
"TwitterStreamingDacq.StreamTask", "StartTask.StartStreamLoop");
break;
}
}
}
