public void Start(string track, MatchCondition condition, params Model.Location[] locations) { // Filters if (!string.IsNullOrEmpty(track)) { _stream.AddTrack(track); } if (null != locations && locations.Length > 0) { foreach (var location in locations) { _stream.AddLocation( new Coordinates(location.Latitude1, location.Longitude1), new Coordinates(location.Latitude2, location.Longitude2) ); } } // Start if (condition == MatchCondition.AnyCondition) { _stream.StartStreamMatchingAnyCondition(); } else { _stream.StartStreamMatchingAllConditions(); } }
public void StartStream() { if (!StreamRunning) { _workerTask = Task.Run(() => { _stream.StartStreamMatchingAnyCondition(); }); } }
public void StartStream(object sndr, DoWorkEventArgs e) { cantTweets = 0; myStream.StartStreamMatchingAnyCondition(); shortBlockTimer.Start(); quickBlockTimer.Start(); mediumBlockTimer.Start(); longBlockTimer.Start(); }
static void Main(string[] args) { // get and parse json configuration file string jsonConfig = File.ReadAllText(Directory.GetCurrentDirectory() + "/config.json"); dynamic jsonValues = JObject.Parse(jsonConfig); string ingressServiceUrl = jsonValues.endpoint; string producerToken = jsonValues.producertoken; int delayInterval = jsonValues.interval; consumerKey = jsonValues.ConsumerKey; consumerSecret = jsonValues.ConsumerSecret; accessToken = jsonValues.AccessToken; accessTokenSecret = jsonValues.AccessTokenSecret; Auth.SetUserCredentials(consumerKey, consumerSecret, accessToken, accessTokenSecret); _stream = Tweetinvi.Stream.CreateFilteredStream(); _stream.AddTweetLanguageFilter(LanguageFilter.English); //_stream.AddTrack("california fire"); //_stream.AddTrack("green fish"); List <string> tracks = jsonValues.Tracks.ToObject <List <string> >(); int id = 1; foreach (string track in tracks) { _stats.Add(new TrackStat { Id = id++, TrackValue = track }); _stream.AddTrack(track); } _periodStart = QuantizeTime(DateTime.Now); _stream.MatchingTweetReceived += (sender, tweetArgs) => { //CheckTime(); // need to do this on a timer var sanitized = Sanitize(tweetArgs.Tweet.FullText); //var sentence = new Sentence(sanitized); //Console.WriteLine(tweetArgs.Tweet.CreatedBy.Name); Console.WriteLine(tweetArgs.Tweet.Text); Console.WriteLine(String.Join(";", tweetArgs.MatchingTracks)); TrackStat stat; foreach (string track in tweetArgs.MatchingTracks) { stat = _stats.Where(x => x.TrackValue == track).FirstOrDefault(); if (stat != null) { ++stat.TweetCount; } } //Console.WriteLine(sentence.Sentiment); }; _stream.UnmanagedEventReceived += (sender, eventArgs) => { Console.WriteLine("unmanaged event received"); }; _stream.DisconnectMessageReceived += (sender, disconnectedArgs) => { Console.WriteLine("received disconnected message"); //Console.WriteLine(disconnectedArgs.DisconnectMessage); }; _stream.JsonObjectReceived += (sender, jsonArgs) => { //Console.WriteLine(jsonArgs.Json); }; _stream.KeepAliveReceived += (sender, eventArgs) => { Console.WriteLine("received keep alive"); }; _stream.LimitReached += (sender, eventArgs) => { Console.WriteLine("limit reached"); }; _stream.WarningFallingBehindDetected += (sender, eventArgs) => { Console.WriteLine("fall behind warning"); }; // create Http client to send requests to ingress service _client = new IngressClient(ingressServiceUrl, producerToken); // Use compression when sending data. _client.UseCompression = jsonValues.compressionGzip; Setup(); /* * // Send Type messages (both Static and Dynamic) * _client.CreateTypes(new string[] { FirstStaticType.JsonSchema, SecondStaticType.JsonSchema } ); * _client.CreateTypes(new string[] { FirstDynamicType.JsonSchema, SecondDynamicType.JsonSchema, ThirdDynamicType.JsonSchema }); * * // Send Container messages * Container[] streams = { * new Container() { Id = "Container1", TypeId = "FirstDynamicType" }, * new Container() { Id = "Container2", TypeId = "FirstDynamicType" }, * new Container() { Id = "Container3", TypeId = "SecondDynamicType"}, * new Container() { Id = "Container4", TypeId = "ThirdDynamicType" } * }; * _client.CreateContainers(streams); * * // Send Assets in Data messages * AssetLinkValues<FirstStaticType> assetParent = new AssetLinkValues<FirstStaticType>() * { * typeid = "FirstStaticType", * Values = new List<FirstStaticType> { new FirstStaticType() * { index="Asset1", name="Parent element", StringProperty="Parent element attribute value" } * } * }; * _client.SendValuesAsync(new AssetLinkValues<FirstStaticType>[] { assetParent}).Wait(); * * AssetLinkValues<SecondStaticType> assetChild = new AssetLinkValues<SecondStaticType>() * { * typeid = "SecondStaticType", * Values = new List<SecondStaticType> { new SecondStaticType() * { index="Asset2", name="Child element", StringProperty="Child element attribute value" } * } * }; * _client.SendValuesAsync(new AssetLinkValues<SecondStaticType>[] { assetChild }).Wait(); * * // Send Asset-to-child-Asset Links * AssetLinkValues<AFLink<StaticElement,StaticElement>> dataLink = new AssetLinkValues<AFLink<StaticElement, StaticElement>>() * { * typeid = "__Link", * Values = new List<AFLink<StaticElement, StaticElement>> { * new AFLink<StaticElement,StaticElement>() {source = new StaticElement() {typeid = "FirstStaticType", index = "_ROOT" }, target = new StaticElement() {typeid= "FirstStaticType", index= "Asset1" } }, * new AFLink<StaticElement,StaticElement>() {source = new StaticElement() {typeid = "FirstStaticType", index = "Asset1" }, target = new StaticElement() {typeid= "SecondStaticType", index= "Asset2" } } * } * }; * _client.SendValuesAsync(new AssetLinkValues<AFLink<StaticElement, StaticElement>>[] { dataLink, }).Wait(); * * // Send Asset-to-Data (i.e. Dynamic Attribute) Links * AssetLinkValues<AFLink<StaticElement, DynamicElement>> dynamic_dataLink = new AssetLinkValues<AFLink<StaticElement, DynamicElement>>() * { * typeid = "__Link", * Values = new List<AFLink<StaticElement, DynamicElement>> { * new AFLink<StaticElement, DynamicElement>(){ source = new StaticElement() { typeid = "FirstStaticType", index= "Asset1" },target= new DynamicElement() { containerid ="Container1" } }, * new AFLink<StaticElement, DynamicElement>(){ source = new StaticElement() { typeid = "SecondStaticType", index= "Asset2" },target= new DynamicElement() { containerid ="Container2" } }, * new AFLink<StaticElement, DynamicElement>(){ source = new StaticElement() { typeid = "SecondStaticType", index= "Asset2" },target= new DynamicElement() { containerid ="Container3" } }, * new AFLink<StaticElement, DynamicElement>(){ source = new StaticElement() { typeid = "SecondStaticType", index= "Asset2" },target= new DynamicElement() { containerid ="Container4" } } * } * }; * _client.SendValuesAsync(new AssetLinkValues<AFLink<StaticElement, DynamicElement>>[] { dynamic_dataLink }).Wait(); */ // Setting handler for Ctrl+C to exit sending data loop bool continueRunning = true; AutoResetEvent exitEvent = new AutoResetEvent(false); Console.CancelKeyPress += (sender, eventArgs) => { continueRunning = false; Console.Write("Stopping... "); eventArgs.Cancel = true; _stream.StopStream(); exitEvent.Set(); }; _timer = new System.Timers.Timer(5000) { AutoReset = false }; _timer.Elapsed += (s, e) => CheckTime(); _timer.Start(); _stream.StartStreamMatchingAnyCondition(); exitEvent.WaitOne(); return; /* * // simulate realtime data * Random rint = new Random(); * Random rdouble = new Random(); * string string_boolean_value = "True"; * int integer_enum_value = 0; * * // Now send simulated relatime data continously * while (continueRunning) * { * // Create set of integers to send to streams * List<FirstDynamicType> values = new List<FirstDynamicType>(); * for(int i = 0; i < 3; i++) * { * values.Add(new FirstDynamicType() { timestamp = DateTime.UtcNow, IntegerProperty = rint.Next() }); * Thread.Sleep(10); // Offset the time-stamps by 10 ms * } * DataValues vals1 = new DataValues() { ContainerId = streams[0].Id, Values = values }; * DataValues vals2 = new DataValues() { ContainerId = streams[1].Id, Values = values }; * // Now send them * //client.SendValuesAsync(new DataValues[] { vals1, vals2 }).Wait(); * * // Create set of SecondDynamicType values to send to streams * List<SecondDynamicType> fnumbers = new List<SecondDynamicType>(); * for (int i = 0; i < 3; i++) * { * string_boolean_value = (string_boolean_value == "True") ? "False" : "True"; * fnumbers.Add(new SecondDynamicType() { timestamp = DateTime.UtcNow, NumberProperty1 = rdouble.NextDouble(), NumberProperty2= rdouble.NextDouble(), StringEnum = string_boolean_value }); * Thread.Sleep(10); // Offset the time-stamps by 10 ms * } * DataValues nums = new DataValues() { ContainerId = streams[2].Id, Values = fnumbers }; * //client.SendValuesAsync(new DataValues[] { nums }).Wait(); * * * // Create set of ThirdDynamicType values to send to streams * List<ThirdDynamicType> enumvalues = new List<ThirdDynamicType>(); * for (int i = 0; i < 3; i++) * { * integer_enum_value = (integer_enum_value == 0) ? 1 : 0; * enumvalues.Add(new ThirdDynamicType() { timestamp = DateTime.UtcNow, IntegerEnum = integer_enum_value }); * Thread.Sleep(10); // Offset the time-stamps by 10 ms * } * DataValues bvals = new DataValues() { ContainerId = streams[3].Id, Values = enumvalues }; * //client.SendValuesAsync(new DataValues[] { bvals }).Wait(); * * Thread.Sleep(delayInterval); * } */ }
private void StartTask(List<string> streamingQueries, List<string> streamingQueriesIDs, List<ArrayList<ArrayList<QueryToken>>> searchTokens, List<string> searchQueriesIDs, Dictionary<string, string> streamingQueryEntities, List<string> authData) { if (authData.Count != 4) { StreamLogger.writeErrorToConsoleAndExit(streamTaskId, String.Format("Twitter authentication data requires exactly 4 values, {0} given", authData.Count), "TwitterStreamingDacq.StreamTask", "StartTask"); } if (streamingQueries.Count != streamingQueriesIDs.Count) { StreamLogger.writeErrorToConsoleAndExit(streamTaskId, String.Format("Length of queries {0} does not mach the length of queries IDs {1}", streamingQueries.Count, streamingQueriesIDs.Count), "TwitterStreamingDacq.StreamTask", "StartTask"); } var credentials = new TwitterCredentials( authData[0], // ConsumerKey authData[1], // ConsumerSecret authData[2], // WorkerToken authData[3] // WorkerSecret ); filteredStream = Stream.CreateFilteredStream(credentials); // match queries only on the tweets text filteredStream.MatchOn = MatchOn.TweetText; for (int i = 0; i < streamingQueries.Count; ++i) { string track = streamingQueries[i]; string streamingQueryID = streamingQueriesIDs[i]; string queryIDbase = StreamConfigParser.ExtractQueryIDBase(streamingQueryID); string entity = ""; if (streamingQueryEntities.ContainsKey(queryIDbase)) entity = streamingQueryEntities[queryIDbase]; filteredStream.AddTrack(track, tweet => { // triggered when this query gets a tweet StreamToSearchAssigner ssa = new StreamToSearchAssigner(); List<string> matchingSearchIds = ssa.AssignStreamToSearch(tweet, streamingQueryID, searchTokens, searchQueriesIDs); if (matchingSearchIds.Count > 0) { // check if json tweet matches the retrieved tweet var jsonConvert = TweetinviContainer.Resolve<IJsonObjectConverter>(); var tweetDTO = jsonConvert.DeserializeObject<ITweetDTO>(lastTweetJson); var tweetFromJson = Tweet.GenerateTweetFromDTO(tweetDTO); bool matchingId = tweetFromJson.IdStr == tweet.IdStr; if (tweet.IsTweetDestroyed) CreateTweetDeleteNotification(tweet); // if the recieved json of the tweet does not match the tracked tweet skip it and create a warning if (tweetFromJson.IdStr == tweet.IdStr) { // match entity if (String.IsNullOrEmpty(entity)) { StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Entity for sreaming query {0} could not be found! Entity set to {1}.", streamingQueryID, queryIDbase), StreamLogger.msgType.Warning, "TwitterStreamingDacq.StreamTask", "StartTask.AddTrack", true); entity = queryIDbase; } // save to nest db PutStreamedTweetToNest(entity, tweet, lastTweetJson); } else { StreamLogger.writeMsgToConsole(streamTaskId, "Retrived JSON tweet and matched tweet do not match. Skipping this tweet", StreamLogger.msgType.Warning, "TwitterStreamingDacq.StreamTask", "StartTask.AddTrack", true); } } }); } filteredStream.JsonObjectReceived += (sender, args) => { lastTweetJson = args.Json; }; filteredStream.DisconnectMessageReceived += (senderTwt, argsTwt) => { StreamLogger.writeMsgToConsole(streamTaskId, String.Format("DisconnectMessageReceived! {0}", argsTwt.DisconnectMessage), StreamLogger.msgType.Warning, "TwitterStreamingDacq.StreamTask", "StartTask.DisconnectMessageReceived", true); }; // notifies that stream attempts to access more than 1% of the total tweets published filteredStream.LimitReached += (sender, args) => { StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Streaming limit reached! {0} tweets were not received from the start of streaming.", args.NumberOfTweetsNotReceived), StreamLogger.msgType.Warning, "TwitterStreamingDacq.StreamTask", "StartTask.LimitReached", true); }; // notifies that a tweet has been deleted filteredStream.TweetDeleted += (sender, args) => { PutDeletedNotificationToNest(args.TweetDeletedInfo); }; filteredStream.StreamStarted += (sender, args) => { StreamLogger.writeMsgToConsole(streamTaskId, "Streaming started", StreamLogger.msgType.Empty, "", "", true); }; // let us manage the exceptions from Tweetinvi ExceptionHandler.SwallowWebExceptions = false; // streaming has stopped, check for exception, log it, restart streaming filteredStream.StreamStopped += (sender, args) => { var disconnectMessage = args.DisconnectMessage; var exception = args.Exception; StreamLogger.writeMsgToConsole(streamTaskId, "Streaming stopped", StreamLogger.msgType.Empty, "", "", true); if (disconnectMessage != null) StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Disconnect message: {0}", disconnectMessage), StreamLogger.msgType.Error, "TwitterStreamingDacq.StreamTask", "StartTask.StreamStopped", true); if (exception != null) StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Exception message: {0}", exception.Message), StreamLogger.msgType.Error, "TwitterStreamingDacq.StreamTask", "StartTask.StreamStopped", true); }; DateTime streamStartedTime = new DateTime(); int numOfStreamRestarts = 0; int numOfMaxStreamRestarts = Convert.ToInt32(ConfigurationManager.AppSettings["StreamTimeoutRestarts"]); if (numOfMaxStreamRestarts < 0) numOfMaxStreamRestarts = Int32.MaxValue; // start streaming while (true) { streamStartedTime = DateTime.Now; try { filteredStream.StartStreamMatchingAnyCondition(); } catch (Exception exception) { StreamLogger.writeMsgToConsole(streamTaskId, String.Format("Exception message: {0}", exception.Message), StreamLogger.msgType.Error, "TwitterStreamingDacq.StreamTask", "StartTask.StartStreamLoop", true); Console.WriteLine(exception.StackTrace); } // twitter hates too many streaming restarts in a short period of time, so wait a little bit before restarting Thread.Sleep(15000); // if there are too many restarts in a short period shutdown this loop altogether if (DateTime.Now.Subtract(streamStartedTime).TotalSeconds < 60) numOfStreamRestarts++; else numOfStreamRestarts = 0; if (numOfStreamRestarts >= numOfMaxStreamRestarts) { StreamLogger.writeErrorToConsoleAndExit(streamTaskId, "Stream stopped completely due to too many restarts", "TwitterStreamingDacq.StreamTask", "StartTask.StartStreamLoop"); break; } } }
private void ExecuteTwitterFinder(List <MultiBattleDefine> selectedBattles) { ExceptionHandler.SwallowWebExceptions = false; try { string consumerKey = ConfigurationManager.AppSettings["ConsumerKey"]; string consumerSecret = ConfigurationManager.AppSettings["ConsumerSecret"]; string accessToken = ConfigurationManager.AppSettings["AccessToken"]; string accessTokenSecret = ConfigurationManager.AppSettings["AccessTokenSecret"]; Auth.SetUserCredentials(consumerKey, consumerSecret, accessToken, accessTokenSecret); TweetinviEvents.QueryBeforeExecute += (sender, args) => { //Console.WriteLine(args.QueryURL); }; _twitterStream = Stream.CreateFilteredStream(); //加入等級Filter selectedBattles.Select(s => s.Level).Distinct().ForEach(e => { _twitterStream.AddTrack("Lv" + e); }); //加入名字Filter 降低通知頻率(中間名字有符號的會捉不到,放棄) //selectedBattles.ForEach(e => //{ // _twitterStream.AddTrack("Lv" + e.BattleLevel + " " + e.Value); //}); _twitterStream.AddTweetLanguageFilter(LanguageFilter.Japanese); //組合name的pattern string namePatternString = $"(?<name>{string.Join("|", selectedBattles.Select(s => $"Lv{s.Level}\\s*{s.Value}"))})"; Regex pattern = new Regex(@"参戦ID:(?<matchid>([a-zA-Z0-9]){8})\s*" + namePatternString); //取得ID和名字 //Regex pattern = new Regex(@"参戦ID:(?<matchid>([a-zA-Z0-9]){8})\s*Lv\d{2,3}\s(?<name>\w*)"); _twitterStream.MatchingTweetReceived += (sender, args) => { var tweet = args.Tweet; var match = pattern.Match(tweet.FullText); if (match.Success) { OnBattleFound?.Invoke(match.Groups["matchid"].ToString(), match.Groups["name"].ToString(), string.Empty); } //else //{ // WriteLog("Not matched"); //} }; _twitterStream.StartStreamMatchingAnyCondition(); //var SStatus = _twitterStream.StreamState; if (_twitterStream.StreamState == StreamState.Stop) { WriteLog(" stopped!"); } } catch (TwitterException ex) { WriteLog(ex.ToString()); } finally { if (null != _twitterStream && _twitterStream.StreamState != StreamState.Stop) { _twitterStream.StopStream(); WriteLog(" Auto stopped!"); } } }
public void StartStream() { Tweetinvi.ExceptionHandler.SwallowWebExceptions = false; var credentials = Tweetinvi.Auth.SetUserCredentials( TwitterAppCredentials.ConsumerKey, TwitterAppCredentials.ConsumerSecret, TwitterAppCredentials.AccessToken, TwitterAppCredentials.AccessSecret ); var user = Tweetinvi.User.GetUserFromScreenName("AndrewYang"); _filteredStream = Tweetinvi.Stream.CreateFilteredStream(); _filteredStream.FilterLevel = Tweetinvi.Streaming.Parameters.StreamFilterLevel.None; _filteredStream.TweetMode = Tweetinvi.TweetMode.Extended; //_filteredStream.AddLocation(new Location(30.6266, 81.4609, 30.6319, 81.6065)); _filteredStream.AddFollow(user.Id); _filteredStream.AddTrack("YangGang"); _filteredStream.AddTrack("NevadaCaucuses"); _filteredStream.AddTrack("StillVotingYang"); _filteredStream.AddTrack("NevadaForYang"); //stream.AddTrack("SecureTheBag"); //stream.AddTrack("YangMediaBlackout"); //stream.AddTrack("YangMediaShoutout"); //stream.AddTrack("YangWillWin"); //stream.AddTrack("YangOrBust"); //stream.AddTrack("YangBeatsTrump"); //stream.AddTrack("NewHampshireForYang"); //stream.AddTrack("AmericaNeedsYang"); //stream.AddTrack("LetYangSpeak"); //stream.AddTrack("VoteYang"); _filteredStream.MatchingTweetReceived += (sender, args) => { try { ITweet fullTweet = null; ITweet childTweet = null; var tweet = args.Tweet; if (tweet.IsRetweet) { return; } TwitterUser savedUser; using (var db = new TwitterSearchModel()) { savedUser = db.TwitterUsers.Where(u => u.TwitterUserID == tweet.CreatedBy.IdStr).FirstOrDefault(); } if (savedUser == null) { var user = Tweetinvi.User.GetUserFromId(tweet.CreatedBy.Id); var userJson = Tweetinvi.JsonSerializer.ToJson(user.UserDTO); Console.WriteLine($"New User: {userJson}"); using (var db = new TwitterSearchModel()) { db.TwitterUsers.Add(new TwitterUser { TwitterUserID = user.IdStr, TwitterUserJson = userJson }); db.SaveChanges(); } } TweetRecord savedTweet; using (var db = new TwitterSearchModel()) { savedTweet = db.TweetRecords.Where(t => t.TweetID == tweet.IdStr).FirstOrDefault(); } if (savedTweet == null) { fullTweet = Tweetinvi.Tweet.GetTweet(tweet.Id); //var voteYang = Tweetinvi.Timeline.GetUserTimeline(.GetExistingList() var tweetJson = Tweetinvi.JsonSerializer.ToJson(fullTweet.TweetDTO); using (var db = new TwitterSearchModel()) { db.TweetRecords.Add(new TweetRecord { TweetID = tweet.IdStr, TweetJson = tweetJson }); db.SaveChanges(); } Console.WriteLine($"User: {fullTweet.CreatedBy.Name}"); Console.WriteLine($"Tweet: {fullTweet.FullText ?? fullTweet.Text}"); Console.WriteLine($"{fullTweet.Url}"); Console.WriteLine(); } //tweetPostQueue.Enqueue(tweet) //var fbPostResult = PostToFacebook(tweet).Result; string childTweetIDStr = tweet.InReplyToStatusIdStr ?? tweet.QuotedStatusIdStr; if (childTweetIDStr != null) { long childTweetID = tweet.InReplyToStatusId ?? tweet.QuotedStatusId ?? 0; childTweet = Tweetinvi.Tweet.GetTweet(childTweetID); TweetRecord savedChildTweet; using (var db = new TwitterSearchModel()) { savedChildTweet = db.TweetRecords.Where(t => t.TweetID == childTweetIDStr).FirstOrDefault(); } if (savedChildTweet == null) { if (childTweetID != 0) { if (childTweet != null) { using (var db = new TwitterSearchModel()) { db.TweetRecords.Add( new TweetRecord { TweetID = childTweetIDStr, TweetJson = Tweetinvi.JsonSerializer.ToJson(childTweet.TweetDTO) }); db.SaveChanges(); } } } } } if (fullTweet != null && childTweet == null) { _tweetHub.Clients.All.SendAsync("ReceiveTweet", fullTweet.TweetDTO.ToJson(), null); } if (fullTweet != null && childTweet != null) { _tweetHub.Clients.All.SendAsync("ReceiveTweet", fullTweet.TweetDTO.ToJson(), childTweet.TweetDTO.ToJson()); } } catch (Exception e) { Console.WriteLine($"ERROR: { e.Message}"); } finally { } }; _filteredStream.StartStreamMatchingAnyCondition(); }