public async Task <TweetStatisticsModel> GetStatisticsAsync()
{
//List<TweetModel> myTweets = new List<TweetModel>();
_statsModel = new TweetStatisticsModel();
try
{
using (SyncedStream)
{
StreamReader mySyncedReader = new StreamReader(SyncedStream, System.Text.Encoding.UTF8);
var mySyncedTweetLine = mySyncedReader.ReadLineAsync();
while (!String.IsNullOrEmpty(mySyncedTweetLine.Result))
{
var newTweet = GetTweetFromResponse(mySyncedTweetLine.Result);
if (newTweet != null)
{
// TODO push tweets to buffered file or memory to not slow down stream
// then process file to get stats
//myTweets.Add(newTweet);
UpdateStats(_statsModel, newTweet);
}
mySyncedTweetLine = mySyncedReader.ReadLineAsync();
}
}
}
catch (Exception ex)
{
var msg = ex.Message;
}
return(_statsModel);
}
public void UpdateStats(TweetStatisticsModel statsModel, TweetModel newTweet)
{
statsModel.CountTweetsReceived += 1;
var hashTags = GetHashtags(newTweet);
var emojis = GetEmojis(newTweet);
var urlsDictionary = GetDomainsDictionary(newTweet);
if (hashTags.Count > 0)
{
statsModel.UpdateHashtagsUsed(hashTags);
}
if (emojis.Count > 0)
{
statsModel.UpdateEmojisUsed(emojis);
statsModel.CountTweetsWithEmojis += 1;
}
if (urlsDictionary.Count > 0)
{
statsModel.CountTweetsWithUrls += 1;
if (newTweet.ContainsOneOrMorePhotoUrl)
{
statsModel.CountTweetsWithPhotoUrls += 1;
}
statsModel.UpdateDomainsUsed(urlsDictionary);
}
// will update end time for every tweet, and end with the time of the final one
statsModel.EndTime = DateTimeOffset.Now;
}
public async Task OnGetAsync()
{
//SampleTweets = _datastore.GetSampleTweets(10);//for testing
//this method works but it processes each tweet as it is read, slowing down the thread
TweetStatistics = await _dataservice.PopulateTweetsAndGetStats();
//need to do thread line reads and processing of lines on separate threads
//_dataservice.PopulateTweets();
//await _dataservice.PopulateTweets();
//TweetStatistics = await _dataservice.GetStatisticsAsync();
//TweetStatistics = _dataservice.GetStatisticsAsync().Result;
}
public async Task <TweetStatisticsModel> PopulateTweetsAndGetStats()//TODO move to repository
{
_statsModel = new TweetStatisticsModel();
string myUrl = _configuration.GetValue <string>("SampleUrl");
var httpClient = new HttpClient();
string bearerToken = _configuration.GetValue <string>("BearerToken");
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken);
try
{
using (httpClient)
{
var myStream = await httpClient.GetStreamAsync(myUrl);
SyncedStream = Stream.Synchronized(myStream);
using (myStream)
{
StreamReader myReader = new StreamReader(myStream, System.Text.Encoding.UTF8);
var myTweetLine = myReader.ReadLine();//does data have line breaks? - yes
/*
* {{"data":{"id":"1363700900435435523","text":"@nuckyzuu แงกอดกอดดด"}}
* {"data":{ "id":"1363700900431204352","text":"道民だからこの時期の暑さが違和感で日差しが🌞上海暑いな。夏大丈夫か😅"} }
* {"data":{"id":"1363704956327399424","text":"Grateful is an attitude that makes life easier."}}
*/
while (!String.IsNullOrEmpty(myTweetLine))
{
var newTweet = GetTweetFromResponse(myTweetLine);
if (newTweet != null)
{
// TODO push tweets to buffered file or memory to not slow down stream
// then process file to get stats
//myTweets.Add(newTweet);
UpdateStats(_statsModel, newTweet);
}
myTweetLine = myReader.ReadLine();
}
}
}
}
catch (Exception ex)
{
var msg = ex.Message;
}
return(_statsModel);
}
public async Task PopulateTweets()//TODO move to repository for EF version
{
//List<TweetModel> myTweets = new List<TweetModel>();
_statsModel = new TweetStatisticsModel();
string myUrl = _configuration.GetValue <string>("SampleUrl");
var httpClient = new HttpClient();
string bearerToken = _configuration.GetValue <string>("BearerToken");
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken);
try
{
using (httpClient)
{
var myStream = await httpClient.GetStreamAsync(myUrl);
SyncedStream = Stream.Synchronized(myStream);
//StringBuilder jsonStringBuilder = new StringBuilder();
using (myStream)
{
StreamReader myReader = new StreamReader(myStream, System.Text.Encoding.UTF8);
var myTweetLine = myReader.ReadLine();//does data have line breaks? - yes
/*
* {{"data":{"id":"1363700900435435523","text":"@nuckyzuu แงกอดกอดดด"}}
* {"data":{ "id":"1363700900431204352","text":"道民だからこの時期の暑さが違和感で日差しが🌞上海暑いな。夏大丈夫か😅"} }
* {"data":{"id":"1363704956327399424","text":"Grateful is an attitude that makes life easier."}}
*/
while (!String.IsNullOrEmpty(myTweetLine))
{
myTweetLine = myReader.ReadLine();
}
}
}
//_tweets = myTweets;
}
catch (Exception ex)
{
var msg = ex.Message;
//_tweets = null;
}
//return _statsModel;
}
public void GetTweetFromResponseTests()
{
//Arrange
//Act
var myTweetModel = InMemoryTweetService.GetTweetFromResponse(tweet);
var myTweetModelBad = InMemoryTweetService.GetTweetFromResponse(tweetBad);
var myTweetModel0 = InMemoryTweetService.GetTweetFromResponse(tweet0);
var myTweetModel2 = InMemoryTweetService.GetTweetFromResponse(tweet2);
var myTweetModel3 = InMemoryTweetService.GetTweetFromResponse(tweet3);
var myTweetModel4 = InMemoryTweetService.GetTweetFromResponse(tweet4);
var myTweetModel5 = InMemoryTweetService.GetTweetFromResponse(tweet5);
var myTweetModel6 = InMemoryTweetService.GetTweetFromResponse(tweet6);
//Assert
Assert.IsNull(myTweetModelBad);
Assert.IsNotNull(myTweetModel);
Assert.AreEqual(0, myTweetModel.entities.hashtags.Count);
Assert.AreEqual(0, myTweetModel.entities.urls.Count);
Assert.IsNotNull(myTweetModel0);
Assert.AreEqual(0, myTweetModel0.entities.hashtags.Count);
Assert.AreEqual(1, myTweetModel0.entities.urls.Count);
Assert.IsNotNull(myTweetModel2);
Assert.AreEqual(0, myTweetModel2.entities.hashtags.Count);
Assert.AreEqual(0, myTweetModel2.entities.urls.Count);
Assert.IsNotNull(myTweetModel3);
Assert.AreEqual(0, myTweetModel3.entities.hashtags.Count);
Assert.AreEqual(0, myTweetModel3.entities.urls.Count);
Assert.IsNotNull(myTweetModel4);
Assert.AreEqual(2, myTweetModel4.entities.hashtags.Count);
Assert.AreEqual(0, myTweetModel4.entities.urls.Count);
Assert.IsNotNull(myTweetModel5);
Assert.AreEqual(1, myTweetModel5.entities.hashtags.Count);
Assert.AreEqual(1, myTweetModel5.entities.urls.Count);
Assert.IsNotNull(myTweetModel6);
Assert.AreEqual(2, myTweetModel6.entities.hashtags.Count);
Assert.AreEqual(0, myTweetModel6.entities.urls.Count);
//Act
TweetStatisticsModel model = new TweetStatisticsModel();
service.UpdateStats(model, myTweetModel);
//Asert
Assert.AreEqual(1, model.CountTweetsReceived);
Assert.AreEqual(0, model.CountTweetsWithEmojis);
Assert.AreEqual(0, model.CountTweetsWithPhotoUrls);
Assert.AreEqual(0, model.CountTweetsWithUrls);
//Act
service.UpdateStats(model, myTweetModel0);
//Asert
Assert.AreEqual(2, model.CountTweetsReceived);
Assert.AreEqual(0, model.CountTweetsWithEmojis);
Assert.AreEqual(1, model.CountTweetsWithPhotoUrls);
Assert.AreEqual(1, model.CountTweetsWithUrls);
//Act
service.UpdateStats(model, myTweetModel2);
//Asert
Assert.AreEqual(3, model.CountTweetsReceived);
Assert.AreEqual(1, model.CountTweetsWithEmojis);
Assert.AreEqual(1, model.CountTweetsWithPhotoUrls);
Assert.AreEqual(1, model.CountTweetsWithUrls);
//Act
service.UpdateStats(model, myTweetModel3);
//Asert
Assert.AreEqual(4, model.CountTweetsReceived);
Assert.AreEqual(2, model.CountTweetsWithEmojis);
Assert.AreEqual(1, model.CountTweetsWithPhotoUrls);
Assert.AreEqual(1, model.CountTweetsWithUrls);
//Act
service.UpdateStats(model, myTweetModel4);
//Asert
Assert.AreEqual(5, model.CountTweetsReceived);
Assert.AreEqual(2, model.CountTweetsWithEmojis);
Assert.AreEqual(1, model.CountTweetsWithPhotoUrls);
Assert.AreEqual(1, model.CountTweetsWithUrls);
//Act
service.UpdateStats(model, myTweetModel5);
//Asert
Assert.AreEqual(6, model.CountTweetsReceived);
Assert.AreEqual(3, model.CountTweetsWithEmojis);
Assert.AreEqual(2, model.CountTweetsWithPhotoUrls);
Assert.AreEqual(2, model.CountTweetsWithUrls);
//Act
service.UpdateStats(model, myTweetModel6);
//Asert
Assert.AreEqual(7, model.CountTweetsReceived);
Assert.AreEqual(3, model.CountTweetsWithEmojis);
Assert.AreEqual(2, model.CountTweetsWithPhotoUrls);
Assert.AreEqual(2, model.CountTweetsWithUrls);
Assert.AreEqual(7, model.EmojisUsed.Count);
Assert.AreEqual(5, model.HashtagsUsed.Count);
Assert.AreEqual(1, model.UrlDomainsUsed.Count);
}
