private static void saveData(newsItems headlines)
{
//return if no headlines to process
if (headlines.NewsItems.Count == 0)
{
return;
}
CloudTable table = getTableStorage("newsItems");
// Create the table if it doesn't exist.
table.CreateIfNotExists();
// Create the batch operation.
TableBatchOperation batchOperation = new TableBatchOperation();
// Create the TableOperation object that inserts the customer entity.
foreach (newsItem item in headlines.NewsItems)
{
batchOperation.InsertOrReplace(item);
}
// Execute the insert operation.
table.ExecuteBatch(batchOperation);
}
static async Task <newsItems> GetSentiment(newsItems headlines)
{
var client = new HttpClient();
var queryString = HttpUtility.ParseQueryString(string.Empty);
Console.WriteLine("{0} stories for sentiment calculation", headlines.NewsItems.Count);
//return if no headlines to process
if (headlines.NewsItems.Count == 0)
{
return(headlines);
}
// Request headers
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", TEXTANALYTICS_KEY);
var uri = "https://westus.api.cognitive.microsoft.com/text/analytics/v2.0/sentiment?" + queryString;
HttpResponseMessage response;
// Request body. Insert your text data here in JSON format.
// thank you https://text-analytics-demo.azurewebsites.net/Home/SampleCode
string mybody1 = "{\"documents\":[";
string mybody2 = "]}";
string headlineBody = "";
foreach (newsItem headline in headlines.NewsItems)
{
//add json escape characters
string tempTitle = HttpUtility.JavaScriptStringEncode(headline.Title);
string tempContent = HttpUtility.JavaScriptStringEncode(headline.Content);
headlineBody += "{\"id\":\"" + headline.ID + "\",\"text\":\"" + tempTitle + " " + tempContent + "\"},";
}
headlineBody = headlineBody.TrimEnd(',');
string finalBody = mybody1 + headlineBody + mybody2;
//finalBody = HttpUtility.JavaScriptStringEncode(finalBody,true);
#if DEBUG
Console.WriteLine(finalBody);
#endif
byte[] byteData = Encoding.UTF8.GetBytes(finalBody);
using (var content = new ByteArrayContent(byteData))
{
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
response = await client.PostAsync(uri, content);
}
string output = await response.Content.ReadAsStringAsync();
dynamic jsonObj = JsonConvert.DeserializeObject <dynamic>(output);
var value = jsonObj.documents;
foreach (var data in value)
{
foreach (newsItem item in headlines.NewsItems)
{
if (data.id == item.ID)
{
item.Sentiment = data.score;
}
}
}
return(headlines);
}
static void Main(string[] args)
{
Console.WriteLine("*****NEWS FEELING IN PROGRESS******");
newsItems headlines = new newsItems();
string thisStoryCount = MAXSTORYCOUNT;
try
{
if (args[0] != "")
{
thisStoryCount = args[0];
}
}
catch { }
Task <newsItems> t;
try
{
//get news headlines
Console.WriteLine("Getting news headlines...");
t = GetNews("Top News", thisStoryCount);
t.Wait();
headlines = t.Result;
}
catch (Exception e)
{
Console.Write("ERROR: {0}, {1}", e.Message, e.StackTrace);
}
try
{
//get sentimate - happy, meh, sucky
Console.WriteLine("Detecting feels...");
t = GetSentiment(headlines);
t.Wait();
headlines = t.Result;
}
catch (Exception e)
{
Console.Write("ERROR: {0}, {1}", e.Message, e.StackTrace);
}
try
{
//save data
Console.WriteLine("Save feeling for later...");
saveData(headlines);
}
catch (Exception e)
{
Console.Write("ERROR: {0}, {1}", e.Message, e.StackTrace);
}
try
{
//list headlines and sentimate
//Console.WriteLine("Show feels...");
//showData();
}
catch (Exception e)
{
Console.Write("ERROR: {0}, {1}", e.Message, e.StackTrace);
}
try
{
//do meta data work
Console.WriteLine("Calculating...");
calcAverageSentiment();
}
catch (Exception e)
{
Console.Write("ERROR: {0}, {1}", e.Message, e.StackTrace);
}
#if DEBUG
//Console.ReadLine();
#endif
}
private static async Task <newsItems> GetNews(string topic, string storyCount)
{
//List<string> headlines = new List<string>();
newsItems headlines = new newsItems();
var client = new HttpClient();
var queryString = HttpUtility.ParseQueryString(string.Empty);
// Request headers
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", NEWSSEARCH_KEY);
// Request parameters
queryString["q"] = topic;
queryString["count"] = storyCount;
queryString["offset"] = "0";
queryString["mkt"] = "en-us";
queryString["safeSearch"] = "Off";
queryString["freshness"] = "Day";
var uri = "https://api.cognitive.microsoft.com/bing/v5.0/news/search?" + queryString;
dynamic response = await client.GetAsync(uri);
dynamic content = await response.Content.ReadAsStringAsync();
searchResult mySearchResult = JsonConvert.DeserializeObject <searchResult>(content);
foreach (Value value in mySearchResult.value)
{
Image image = value.image;
Thumbnail thumb = image.thumbnail;
List <Provider> provider = value.provider;
#if DEBUG
Console.WriteLine("Story: {0}: {1}, {2}, {3}", value.name.GetHashCode().ToString(), provider[0].name.ToString(), value.category, value.name);
#endif
headlines.Add(new newsItem(value.name, value.url, thumb.contentUrl, thumb.height, thumb.width, value.description, Convert.ToDateTime(value.datePublished), provider[0].name.ToString(), value.category));
}
//check to see if we already recorded this headline
//flag items to remove
List <newsItem> duplicates = new List <newsItem>();
CloudTable table = getTableStorage("newsItems");
foreach (newsItem item in headlines.NewsItems)
{
TableOperation retrieveOperation = TableOperation.Retrieve <newsItem>("newsFeels", item.Title.GetHashCode().ToString());
TableResult query = table.Execute(retrieveOperation);
//if we have result and result has sentiment > 0 then remove that story from processing
if (query.Result != null && ((newsItem)query.Result).Sentiment > 0)
{
duplicates.Add(item);
}
}
//remove duplicates
foreach (newsItem item in duplicates)
{
headlines.NewsItems.Remove(item);
}
Console.WriteLine("{0} new stories detected", headlines.NewsItems.Count);
return(headlines);
}
