public async Task <TweetStatisticsModel> GetStatisticsAsync() { //List<TweetModel> myTweets = new List<TweetModel>(); _statsModel = new TweetStatisticsModel(); try { using (SyncedStream) { StreamReader mySyncedReader = new StreamReader(SyncedStream, System.Text.Encoding.UTF8); var mySyncedTweetLine = mySyncedReader.ReadLineAsync(); while (!String.IsNullOrEmpty(mySyncedTweetLine.Result)) { var newTweet = GetTweetFromResponse(mySyncedTweetLine.Result); if (newTweet != null) { // TODO push tweets to buffered file or memory to not slow down stream // then process file to get stats //myTweets.Add(newTweet); UpdateStats(_statsModel, newTweet); } mySyncedTweetLine = mySyncedReader.ReadLineAsync(); } } } catch (Exception ex) { var msg = ex.Message; } return(_statsModel); }
public void UpdateStats(TweetStatisticsModel statsModel, TweetModel newTweet) { statsModel.CountTweetsReceived += 1; var hashTags = GetHashtags(newTweet); var emojis = GetEmojis(newTweet); var urlsDictionary = GetDomainsDictionary(newTweet); if (hashTags.Count > 0) { statsModel.UpdateHashtagsUsed(hashTags); } if (emojis.Count > 0) { statsModel.UpdateEmojisUsed(emojis); statsModel.CountTweetsWithEmojis += 1; } if (urlsDictionary.Count > 0) { statsModel.CountTweetsWithUrls += 1; if (newTweet.ContainsOneOrMorePhotoUrl) { statsModel.CountTweetsWithPhotoUrls += 1; } statsModel.UpdateDomainsUsed(urlsDictionary); } // will update end time for every tweet, and end with the time of the final one statsModel.EndTime = DateTimeOffset.Now; }
public async Task OnGetAsync() { //SampleTweets = _datastore.GetSampleTweets(10);//for testing //this method works but it processes each tweet as it is read, slowing down the thread TweetStatistics = await _dataservice.PopulateTweetsAndGetStats(); //need to do thread line reads and processing of lines on separate threads //_dataservice.PopulateTweets(); //await _dataservice.PopulateTweets(); //TweetStatistics = await _dataservice.GetStatisticsAsync(); //TweetStatistics = _dataservice.GetStatisticsAsync().Result; }
public async Task <TweetStatisticsModel> PopulateTweetsAndGetStats()//TODO move to repository { _statsModel = new TweetStatisticsModel(); string myUrl = _configuration.GetValue <string>("SampleUrl"); var httpClient = new HttpClient(); string bearerToken = _configuration.GetValue <string>("BearerToken"); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken); try { using (httpClient) { var myStream = await httpClient.GetStreamAsync(myUrl); SyncedStream = Stream.Synchronized(myStream); using (myStream) { StreamReader myReader = new StreamReader(myStream, System.Text.Encoding.UTF8); var myTweetLine = myReader.ReadLine();//does data have line breaks? - yes /* * {{"data":{"id":"1363700900435435523","text":"@nuckyzuu แงกอดกอดดด"}} * {"data":{ "id":"1363700900431204352","text":"道民だからこの時期の暑さが違和感で日差しが🌞上海暑いな。夏大丈夫か😅"} } * {"data":{"id":"1363704956327399424","text":"Grateful is an attitude that makes life easier."}} */ while (!String.IsNullOrEmpty(myTweetLine)) { var newTweet = GetTweetFromResponse(myTweetLine); if (newTweet != null) { // TODO push tweets to buffered file or memory to not slow down stream // then process file to get stats //myTweets.Add(newTweet); UpdateStats(_statsModel, newTweet); } myTweetLine = myReader.ReadLine(); } } } } catch (Exception ex) { var msg = ex.Message; } return(_statsModel); }
public async Task PopulateTweets()//TODO move to repository for EF version { //List<TweetModel> myTweets = new List<TweetModel>(); _statsModel = new TweetStatisticsModel(); string myUrl = _configuration.GetValue <string>("SampleUrl"); var httpClient = new HttpClient(); string bearerToken = _configuration.GetValue <string>("BearerToken"); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken); try { using (httpClient) { var myStream = await httpClient.GetStreamAsync(myUrl); SyncedStream = Stream.Synchronized(myStream); //StringBuilder jsonStringBuilder = new StringBuilder(); using (myStream) { StreamReader myReader = new StreamReader(myStream, System.Text.Encoding.UTF8); var myTweetLine = myReader.ReadLine();//does data have line breaks? - yes /* * {{"data":{"id":"1363700900435435523","text":"@nuckyzuu แงกอดกอดดด"}} * {"data":{ "id":"1363700900431204352","text":"道民だからこの時期の暑さが違和感で日差しが🌞上海暑いな。夏大丈夫か😅"} } * {"data":{"id":"1363704956327399424","text":"Grateful is an attitude that makes life easier."}} */ while (!String.IsNullOrEmpty(myTweetLine)) { myTweetLine = myReader.ReadLine(); } } } //_tweets = myTweets; } catch (Exception ex) { var msg = ex.Message; //_tweets = null; } //return _statsModel; }
public void GetTweetFromResponseTests() { //Arrange //Act var myTweetModel = InMemoryTweetService.GetTweetFromResponse(tweet); var myTweetModelBad = InMemoryTweetService.GetTweetFromResponse(tweetBad); var myTweetModel0 = InMemoryTweetService.GetTweetFromResponse(tweet0); var myTweetModel2 = InMemoryTweetService.GetTweetFromResponse(tweet2); var myTweetModel3 = InMemoryTweetService.GetTweetFromResponse(tweet3); var myTweetModel4 = InMemoryTweetService.GetTweetFromResponse(tweet4); var myTweetModel5 = InMemoryTweetService.GetTweetFromResponse(tweet5); var myTweetModel6 = InMemoryTweetService.GetTweetFromResponse(tweet6); //Assert Assert.IsNull(myTweetModelBad); Assert.IsNotNull(myTweetModel); Assert.AreEqual(0, myTweetModel.entities.hashtags.Count); Assert.AreEqual(0, myTweetModel.entities.urls.Count); Assert.IsNotNull(myTweetModel0); Assert.AreEqual(0, myTweetModel0.entities.hashtags.Count); Assert.AreEqual(1, myTweetModel0.entities.urls.Count); Assert.IsNotNull(myTweetModel2); Assert.AreEqual(0, myTweetModel2.entities.hashtags.Count); Assert.AreEqual(0, myTweetModel2.entities.urls.Count); Assert.IsNotNull(myTweetModel3); Assert.AreEqual(0, myTweetModel3.entities.hashtags.Count); Assert.AreEqual(0, myTweetModel3.entities.urls.Count); Assert.IsNotNull(myTweetModel4); Assert.AreEqual(2, myTweetModel4.entities.hashtags.Count); Assert.AreEqual(0, myTweetModel4.entities.urls.Count); Assert.IsNotNull(myTweetModel5); Assert.AreEqual(1, myTweetModel5.entities.hashtags.Count); Assert.AreEqual(1, myTweetModel5.entities.urls.Count); Assert.IsNotNull(myTweetModel6); Assert.AreEqual(2, myTweetModel6.entities.hashtags.Count); Assert.AreEqual(0, myTweetModel6.entities.urls.Count); //Act TweetStatisticsModel model = new TweetStatisticsModel(); service.UpdateStats(model, myTweetModel); //Asert Assert.AreEqual(1, model.CountTweetsReceived); Assert.AreEqual(0, model.CountTweetsWithEmojis); Assert.AreEqual(0, model.CountTweetsWithPhotoUrls); Assert.AreEqual(0, model.CountTweetsWithUrls); //Act service.UpdateStats(model, myTweetModel0); //Asert Assert.AreEqual(2, model.CountTweetsReceived); Assert.AreEqual(0, model.CountTweetsWithEmojis); Assert.AreEqual(1, model.CountTweetsWithPhotoUrls); Assert.AreEqual(1, model.CountTweetsWithUrls); //Act service.UpdateStats(model, myTweetModel2); //Asert Assert.AreEqual(3, model.CountTweetsReceived); Assert.AreEqual(1, model.CountTweetsWithEmojis); Assert.AreEqual(1, model.CountTweetsWithPhotoUrls); Assert.AreEqual(1, model.CountTweetsWithUrls); //Act service.UpdateStats(model, myTweetModel3); //Asert Assert.AreEqual(4, model.CountTweetsReceived); Assert.AreEqual(2, model.CountTweetsWithEmojis); Assert.AreEqual(1, model.CountTweetsWithPhotoUrls); Assert.AreEqual(1, model.CountTweetsWithUrls); //Act service.UpdateStats(model, myTweetModel4); //Asert Assert.AreEqual(5, model.CountTweetsReceived); Assert.AreEqual(2, model.CountTweetsWithEmojis); Assert.AreEqual(1, model.CountTweetsWithPhotoUrls); Assert.AreEqual(1, model.CountTweetsWithUrls); //Act service.UpdateStats(model, myTweetModel5); //Asert Assert.AreEqual(6, model.CountTweetsReceived); Assert.AreEqual(3, model.CountTweetsWithEmojis); Assert.AreEqual(2, model.CountTweetsWithPhotoUrls); Assert.AreEqual(2, model.CountTweetsWithUrls); //Act service.UpdateStats(model, myTweetModel6); //Asert Assert.AreEqual(7, model.CountTweetsReceived); Assert.AreEqual(3, model.CountTweetsWithEmojis); Assert.AreEqual(2, model.CountTweetsWithPhotoUrls); Assert.AreEqual(2, model.CountTweetsWithUrls); Assert.AreEqual(7, model.EmojisUsed.Count); Assert.AreEqual(5, model.HashtagsUsed.Count); Assert.AreEqual(1, model.UrlDomainsUsed.Count); }