public void SMPTwitterSampleRetrieve()
{
int countPrev = 0, countCurrent = 0;
try
{
ISocialMediaProvider twitterSample = new SmpTwitterSample(log);
IRawDataQueue rdq = container.Resolve <IRawDataQueue>();
using (CancellationTokenSource cancellationTokenSource = new CancellationTokenSource())
{
CancellationToken cancel = cancellationTokenSource.Token;
twitterSample.Retrieve(rdq, cancel);
for (int i = 0; i < 5; i++)
{
countPrev = countCurrent;
Thread.Sleep(1000);
countCurrent = rdq.Count();
if (countCurrent <= countPrev)
{
Assert.Fail(
"TwitterSample Retrieve did not add to queue.\nPrevious Count: {0}\nCurrent count: {1}",
countPrev, countCurrent);
}
}
cancellationTokenSource.Cancel();
}
}
catch (Exception ex)
{
Assert.Fail(
"An exception occurred in SMPTwitterSample Retrieve Test. See details below:({0})\n{1}\n{2}",
ex.HResult, ex.Message, ex.StackTrace);
}
Assert.Pass("TwitterSample retrieved {0} messages", countCurrent);
}
public void AnalyzerLocalTest()
{
try
{
IRawDataQueue rdq = container.Resolve <IRawDataQueue>();
IDataStore ds = container.Resolve <IDataStore>();
foreach (string rd in rdqTestStrings)
{
rdq.Add(rd);
}
IAnalyzer analyzer = new AnalyzerLocal(log);
using (CancellationTokenSource cancellationTokenSource = new CancellationTokenSource())
{
CancellationToken cancel = cancellationTokenSource.Token;
analyzer.Analyze(rdq, ds, cancel);
while (rdq.Count() > 0)
{
Console.WriteLine("Waiting for analyzer to finish");
Thread.Sleep(10);
}
cancellationTokenSource.Cancel();
}
List <TweetMetadata> tmds = ds.Retrieve();
foreach (TweetMetadata tmd in tmds)
{
TweetMetadata dstd = dataStoreTestData.FirstOrDefault(x => x.Id == tmd.Id);
if (dstd == null)
{
Assert.Fail("Analyzer created tweet with invalid id: {0}", tmd.Id);
}
if (!dstd.Equals(tmd))
{
Assert.Fail("Analyzer did not properly analyze tweet with id: {0}", tmd.Id);
}
Console.WriteLine("Analyzer test Matched tweet: {0}", tmd.Id);
}
if (tmds.Count() != rdqValidStrings)
{
Assert.Fail("Analyzer did not create correct number of elements.\nExpected: {0}\nRetrieved:{1}\n",
rdqValidStrings, tmds.Count());
}
}
catch (Exception ex)
{
Assert.Fail(
"An exception occurred in AnalyzerLocalTest. See details below:({0})\n{1}\n{2}",
ex.HResult, ex.Message, ex.StackTrace);
}
Assert.Pass();
}
public async Task Retrieve(IRawDataQueue rawData, CancellationToken cancel)
{
try
{
// Establish connection.
_log.Write("Calling Service");
using (HttpClient client = new HttpClient())
{
string token = GetBearerToken();
if (string.IsNullOrWhiteSpace(token))
{
throw new Exception("Could not retrieve bearer token.");
}
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
Task <Stream> response = client.GetStreamAsync(
"https://api.twitter.com/2/tweets/sample/stream?tweet.fields=attachments,created_at,entities&expansions=attachments.media_keys");
// Get the stream of the content.
_log.Write("Retrieving Data");
using (var reader = new StreamReader(response.Result))
{
//will stop sampling when token is cancelled
while (!cancel.IsCancellationRequested)
{
//if one call fails we want to continue on
try
{
//using async method allows cancellation easier.
string jsonData = await reader.ReadLineAsync();
//_log.Write(string.Format("Retrieved Data:")); //debug code
//for now just store the raw string, will process data in a seperate thread
rawData.Add(jsonData);
}
catch (Exception ex)
{
_log.HandleException(ex, "Retrieving sample data from twitter.");
}
}
_log.Write("Data retrieval stopped.");
}
}
}
catch (Exception ex)
{
_log.HandleException(ex, String.Format("SmpTwitterSample.Retrieve"));
}
}
public async Task Analyze(IRawDataQueue rdq, IDataStore ds, CancellationToken cancel)
{
try
{
int retVal = await Task.Factory.StartNew(() => StoreData(rdq, ds, cancel));
if (retVal != 0)
{
_log.Write(String.Format("Analyzer encountered an error. Report may not be accurate. Error code: {0}", retVal));
}
}
catch (Exception ex)
{
_log.HandleException(ex, "AnalyzerLocal.Analyze");
}
}
static void Main(string[] args)
{
try
{
UnityContainer container = new UnityContainer();
container.RegisterType <ILog, LogConsole>();
container.RegisterType <IRawDataQueue, RdqMemoryThreadSafe>();
container.RegisterType <ISocialMediaProvider, SmpTwitterSample>();
container.RegisterType <IDataStore, DsMemoryThreadSafe>();
container.RegisterType <IAnalyzer, AnalyzerLocal>();
container.RegisterType <IReporter, ReporterConsole>();
IRawDataQueue rdq = container.Resolve <IRawDataQueue>();
ISocialMediaProvider smp = container.Resolve <ISocialMediaProvider>();
IDataStore ds = container.Resolve <IDataStore>();
IAnalyzer analyzer = container.Resolve <IAnalyzer>();
IReporter reporter = container.Resolve <IReporter>();
//Set up a cancellation token to end the collection when appropriate
using (CancellationTokenSource cancellationTokenSource = new CancellationTokenSource())
{
CancellationToken cancel = cancellationTokenSource.Token;
smp.Retrieve(rdq, cancel);
analyzer.Analyze(rdq, ds, cancel);
reporter.Send(ds, cancel);
Console.WriteLine("Press 'Enter' to stop.");
Console.ReadLine();
cancellationTokenSource.Cancel();
}
Console.WriteLine("Cancellation request recieved. Press 'Enter' to exit program.");
Console.ReadLine();
}
catch (Exception ex)
{
Console.WriteLine("An exception occurred. See details below. Press 'Enter' to close program.\n{0}", ex.Message);
Console.ReadLine();
}
}
private int StoreData(IRawDataQueue rdq, IDataStore ds, CancellationToken cancel)
{
int status = 0;
try
{
_log.Write("Starting to Analyze.");
bool cancelMessageDisplayed = false;
while (!(cancel.IsCancellationRequested && rdq.Count() > 0))
{
//if queue is empty, wait then continue
if (rdq.Count() <= 0)
{
Thread.Sleep(100);
continue;
}
else if (!cancelMessageDisplayed && cancel.IsCancellationRequested && rdq.Count() > 0)
{
_log.Write(string.Format(
"Analyzer recieved cancel request, continuing to process {0} remaining queued items.", rdq.Count()));
cancelMessageDisplayed = true;
}
string rawData = rdq.Retrieve();
if (!string.IsNullOrWhiteSpace(rawData))
{
//If one tweet is problematic we want to carry on.
try
{
//The message object and it's children or more complex than they need to be.
//They are built to mimic the Json structure as closely as neccessary.
//This will make it easier to collect additonal statistics in the future.
Message message = JsonConvert.DeserializeObject <Message>(rawData);
ds.Store(new TweetMetadata()
{
TimeStamp = message.data.created_at,
Id = message.data.id,
Data = message.data.text,
Emojis = (message.data.text == null) ?
new List <EmojiData>() :
//for each emoji add it to the list if it appears in the text
_emojis.Where(x => !string.IsNullOrWhiteSpace(x.character) &&
message.data.text.Contains(x.character)).ToList(),
HashTags = (message.data.entities == null) ?
new List <String>() :
(message.data.entities.hashtags == null) ?
new List <String>() :
message.data.entities.hashtags.Select(x => x.tag).Distinct().ToList(),
Domains = (message.data.entities == null) ?
new List <String>() :
(message.data.entities.urls == null) ?
new List <String>() :
//for each url extract the domain and add it to the list
message.data.entities.urls.Select(x => new Uri(x.expanded_url).Host).Distinct().ToList(),
MediaTypes = (message.data.attachments == null) ?
new List <String>() :
(message.data.attachments.media_keys == null) ?
new List <String>() :
//for each media attachment find it's entry in the "includes" section
//and add it's type to the list if it is not already there
message.data.attachments.media_keys
.Select(x => message.includes.media.FirstOrDefault(m => m.media_key == x).type)
.Distinct().ToList()
});
}
catch (Exception ex)
{
_log.HandleException(ex, string.Format("Processing Tweet:\n{0}\n", rawData));
}
} //if message not empty
} // while analyzing
_log.Write("All messages analyzed.");
}
catch (Exception ex)
{
_log.HandleException(ex, "AnalyzerLocal.StoreData");
status = (ex.HResult == 0) ? 1 : ex.HResult;
}
return(status);
}
