public static void CheckApplicationEnvironment()
{
Console.WriteLine("Checking NewsAPI environment");
if (!NewsAPIMethods.IsNewsEnvironmentReady())
{
Console.WriteLine("News API Key has not been set up.\n");
}
else
{
Console.WriteLine("NewsAPI environment has been set up");
}
Console.WriteLine("Checking NLP environment");
if (!NLPMethods.IsNLPEnvironmentReady())
{
Console.WriteLine("Google NLP environment variable has not been set up.\n" +
"Setting environment variable for Google Application Credential...\n");
try
{
NLPMethods.SetNLPEnvironmentVariable();
Console.WriteLine("Successfully set up NLP environment variable");
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
else
{
Console.WriteLine("NLP environment has been set up");
}
}
/*
* @TODO:
*
* - Write News Results Into news_info - DONE! woo!
* - Write Sentiment Results Into sentiment_results - Done! yayyyyy!
* - Do Basic Company Lookup (search by sentiment: entity - type = organization) - Done!
* - Write Company to stock_info - DONE! :D
*
* For Demo/Testing:
* - Make sure 'live news updates' works, but for the actual demo/functionality, get all news by time period (Ex. last 24hrs)
*
* Additional Considerations:
* - Companies may trade under different names (ex. Google - Alphabet)
* - Logging the industry? Possibly (consider: adding NLP category to stock_listing) - will have to figure out update queries for Mongo
* - Associating other entity results with a stock (ex. tba)
*
*/
static void Main(string[] args)
{
try
{
/*
* Program Flow:
* =============
*
* 1) Grab News
* 2) For Each Article:
* 3) Write to Database (news_info)
* 4) Analyse Sentiment
* 5) Write Sentiment to Database (sentiment_results)
* 6) Lookup Company -
* 6.1) By Entity Where Type = Organization:
* For each entity,
* If type = organization
* conn.GetStocks("SecurityName", entityValue);
* If list.empty
* //write that it's empty into the db or ignore
*
* If !list.empty
* //write into stock_info
* 7) Write Company to Database (stock_info)
*
*/
//Get MongoDB Connection
var conn = new MongoConnection();
/*
* DB TESTING
*
* var testDoc = new BsonDocument
* {
* { "student_id", 10011 },
* { "scores", new BsonArray
* {
* new BsonDocument{ {"type", "exam"}, {"score", 88.12334193287023 } },
* new BsonDocument{ {"type", "quiz"}, {"score", 74.92381029342834 } },
* new BsonDocument{ {"type", "homework"}, {"score", 89.97929384290324 } },
* new BsonDocument{ {"type", "homework"}, {"score", 82.12931030513218 } },
* new BsonDocument{ {"timestamp", DateTime.Now.ToString() } }
* }
* },
* { "class_id", 480 }
* };
*
* WRITE - implemented below
* var result = conn.InsertDocument("SATbot", "news_info", testDoc);
* var result2 = conn.InsertDocument("stock_listing", testDoc);
* Console.WriteLine(result);
*
* READ
* var latestDoc = conn.GetLast("SATbot", "news_info", "_id");
* var latest2 = conn.GetLast("news_info"); //gets last entry by _id
* var latest3 = conn.GetLast("news_info", "Date Retrieved"); //get last entry by criteria
* Console.WriteLine(latestDoc);
*
* READ WITH FILTER
* conn.GetStocks("SecurityName", "Apple");
* conn.GetStocks("SecurityName", "Microsoft");
* conn.GetStocks("SecurityName", "Hathaway");
*
* END DB TESTING
*/
//Environment Check
Console.WriteLine();
CheckApplicationEnvironment();
Console.WriteLine();
Console.WriteLine("-------------------------------------------------------------------------------");
Console.WriteLine();
//Retrieve Articles
Console.WriteLine("Retrieving articles...");
var articles = NewsAPIMethods.RetrieveNewsAsync().Result;
Console.WriteLine();
Console.WriteLine("Processing articles...\n");
//Write Articles to DB
foreach (var article in articles)
{
// Check if the article is already inserted (skip the insertion if it is)
BsonDocument articleBson;
var articleId = new ObjectId();
//@QUESTION: How does putting a blank object id in the param check whether it's been inserted already?
bool isInserted = NewsAPIMethods.IsInserted(article, conn, out articleId);
if (!isInserted)
{
Console.WriteLine();
//Insert the article into DB
articleBson = NewsAPIMethods.InsertArticle(article, conn);
//Null check
if (articleBson == null)
{
throw new Exception("NewsAPIMethods.InsertArticle returned a null object for: "
+ article.Title + "\n");
}
articleId = new ObjectId(articleBson.GetValue("_id", "No id found??").ToString());
Console.WriteLine($"Article: {article.Title} is successfully inserted at _id: {articleId}");
// Perform sentiment analysis
Console.WriteLine("Analyzing article sentiment...");
var sentimentResponse = NLPMethods.AnalyzeSentimentAsync(articleBson, article);
var sentimentResult = sentimentResponse.Result;
var sentimentDoc = sentimentResult.SentimentBsonDocument;
// Insert sentiment results to DB
var sentimentResultId = NLPMethods.InsertSentimentResult(sentimentDoc, conn);
Console.WriteLine($"Successfully inserted sentiment result of article (_id: {articleId}) at _id: {sentimentResultId}");
//Correlate entities and stocks
Console.WriteLine("Correlating entities and stocks...");
var entities = sentimentResult.Entities;
//Run stock correlation
var stocks = StockCorrelation.RunCorrelation(conn, entities);
if (stocks.Count > 0)
{
foreach (var stock in stocks)
{
Console.WriteLine($"Symbol: {stock.Symbol} | " +
$"Security Name: {stock.SecurityName} | " +
$"Company Name: {stock.CompanyName}");
//Insert to db
StockCorrelation.InsertToStockInfo(conn, sentimentDoc, stock.ToBsonDocument(),
"", "", "", "");
}
}
else
{
Console.WriteLine($"Could not correlate entities to any stocks.");
}
Console.WriteLine();
}
else
{
Console.WriteLine($"Article: {article.Title} already exists with _id: {articleId}");
}
}
Console.WriteLine();
Console.WriteLine("-------------------------------------------------------------------------------");
Console.WriteLine();
Console.WriteLine("Demo Entity Sentiment: ");
/*
* Test/Sample Article:
* Title: Google and Microsoft are working to make web forms more touch-friendly
* Description: Google and Microsoft have redesigned native form controls -- buttons and various input elements you see on web forms -- to look more harmonious and be more touch-friendly. They spent the past year working together to design a new theme and make built-in form .
*/
var entitySentimentTest = NLPMethods.AnalyzeEntitySentimentAsync("Google and Microsoft have redesigned native form controls -- buttons and various input elements you see on web forms -- to look more harmonious and be more touch-friendly. They spent the past year working together to design a new theme and make built-in form.").Result;
Console.WriteLine("Google and Microsoft are working to make web forms more touch-friendly");
Console.WriteLine(entitySentimentTest);
Console.WriteLine();
}
catch (Exception e)
{
Console.WriteLine(e.StackTrace);
Console.WriteLine(e.Message);
}
}