private async Task saveTweet(MatchedTweetReceivedEventArgs e, string keyword) { var tweet = formatTweet(e); tweet.Keyword = keyword; //we use keyword hashcode as partition key, it is assumed to be sparse IPersistence persistence = ServiceExtensions.GetPersistenceService(keyword); try { if (e.Tweet.IsRetweet) { await persistence.SaveReTweetAsync(tweet); } else { await persistence.SaveTweetAsync(tweet); } } catch (Exception ex) { ServiceEventSource.Current.ServiceMessage(this, "Tweet store failed: {0}\r\n\t{1}", e.Tweet.Text, ex.Message); throw; } }
private Task saveTweet(MatchedTweetReceivedEventArgs e) { Gof.Twitter.Tweet tweet = formatTweet(e); IPersistence persistence = ServiceExtensions.GetPersistenceService(e.Tweet.CreatedBy.Name); return(persistence.SaveRawTweetAsync(tweet)); }
public async Task <IActionResult> Index() { IEnumerable <string> influencers = null; try { IPersistence persistence = ServiceExtensions.GetPersistenceService(); influencers = await persistence.RetrieveInfluencerListAsync(); } catch (Exception ex) { //TODO:log influencers = new List <string>(); Trace.TraceError("{0}: {1}", ex.Message, ex.StackTrace); } return(View(influencers)); }
// GET api/values public async Task <IEnumerable <string> > Get() { return(await ServiceExtensions.GetPersistenceService().RetrieveInfluencerListAsync()); }
/// <summary> /// This is the main entry point for your service replica. /// This method executes when this replica of your service becomes primary and has write status. /// </summary> /// <param name="cancellationToken">Canceled when Service Fabric needs to shut down this service replica.</param> protected override async Task RunAsync(CancellationToken cancellationToken) { // TODO: Replace the following sample code with your own logic // or remove this RunAsync override if it's not needed in your service. var influencers = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, DateTime> >("influencers"); var influencerList = await ServiceExtensions.GetPersistenceService(DateTime.Now.ToString()).RetrieveInfluencerListAsync(); //bug if (influencerList.Count() == 0) { var healthInformation = new HealthInformation( "TwitterReader", "influencers should not be 0", HealthState.Error); this.Partition.ReportReplicaHealth(healthInformation); } readNextInfluencers(cancellationToken, influencerList.ToArray()); while (true) { cancellationToken.ThrowIfCancellationRequested(); string influencer = null; using (var tx = this.StateManager.CreateTransaction()) { var list = await influencers.CreateEnumerableAsync(tx, EnumerationMode.Ordered); using (var enumerator = list.GetAsyncEnumerator()) { while (await enumerator.MoveNextAsync(cancellationToken)) { if (enumerator.Current.Value == DateTime.MinValue || enumerator.Current.Value < DateTime.Now.AddMinutes(-20)) { if (await influencers.TryUpdateAsync(tx, enumerator.Current.Key, DateTime.Now, enumerator.Current.Value)) { influencer = enumerator.Current.Key; break; } } } } if (influencer != null) { ServiceEventSource.Current.ServiceMessage(this, "Adding reader for influencer: {0}", influencer); readNextInfluencers(cancellationToken, influencerList.ToArray()); } else { //ServiceEventSource.Current.ServiceMessage(this, "No more influencers to read"); } //var result = await myDictionary.TryGetValueAsync(tx, "Counter"); //ServiceEventSource.Current.ServiceMessage(this, "Current Counter Value: {0}", // result.HasValue ? result.Value.ToString() : "Value does not exist."); //await myDictionary.AddOrUpdateAsync(tx, "Counter", 0, (key, value) => ++value); //// If an exception is thrown before calling CommitAsync, the transaction aborts, all changes are //// discarded, and nothing is saved to the secondary replicas. await tx.CommitAsync(); } await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); } }
private async void readNextInfluencers(CancellationToken cancellationToken, string[] influencers) { try { //var influencers = await this.StateManager.GetOrAddAsync<IReliableDictionary<string, DateTime>>("influencers"); Auth.SetUserCredentials( getKey("ApiKey"), getKey("ApiSecret"), getKey("TokenKey"), getKey("TokenSecret")); var keywords = await ServiceExtensions.GetPersistenceService(DateTime.Now.ToString()).RetrieveKeywordListAsync(); SemaphoreSlim semaphore = new SemaphoreSlim(1); var twitStream = Stream.CreateFilteredStream(); try { foreach (var influencer in influencers) { twitStream.AddFollow(User.GetUserFromScreenName(influencer)); } twitStream.MatchingTweetReceived += async(o, e) => { List <Task> taskList = new List <Task>(); taskList.Add(saveTweet(e)); var uppertweet = e.Tweet.Text.ToUpper(); foreach (var word in keywords) { if (uppertweet.Contains(word.ToUpper())) { taskList.Add(saveTweet(e, word)); } } foreach (var hashtag in e.Tweet.Hashtags.Distinct()) { if (keywords.FirstOrDefault((s) => s.ToUpper() == hashtag.Text.ToUpper()) == null) { taskList.Add(saveTweet(e, hashtag.Text)); } } await Task.WhenAll(taskList); ServiceEventSource.Current.ServiceMessage(this, "Current Tweet: {0}", e.Tweet.Text); }; await semaphore.WaitAsync(); twitStream.LimitReached += (o, e) => { semaphore.Release(); }; twitStream.DisconnectMessageReceived += (o, e) => { semaphore.Release(); }; await twitStream.StartStreamMatchingAllConditionsAsync(); await semaphore.WaitAsync(cancellationToken); } finally { twitStream.StopStream(); ServiceEventSource.Current.ServiceMessage(this, "Influencer stream stopped: {0}", influencers); } } catch (Exception ex) { ServiceEventSource.Current.ServiceMessage(this, "Fatal error with twitter API: {0}", ex.Message); HealthInformation healthInformation = new HealthInformation("TwitterAPI", ex.Message, HealthState.Error); this.Partition.ReportReplicaHealth(healthInformation); } }