예제 #1
0
        private static void saveData(newsItems headlines)
        {
            //return if no headlines to process
            if (headlines.NewsItems.Count == 0)
            {
                return;
            }

            CloudTable table = getTableStorage("newsItems");

            // Create the table if it doesn't exist.
            table.CreateIfNotExists();

            // Create the batch operation.
            TableBatchOperation batchOperation = new TableBatchOperation();

            // Create the TableOperation object that inserts the customer entity.
            foreach (newsItem item in headlines.NewsItems)
            {
                batchOperation.InsertOrReplace(item);
            }
            // Execute the insert operation.
            table.ExecuteBatch(batchOperation);
        }
예제 #2
0
        static async Task <newsItems> GetSentiment(newsItems headlines)
        {
            var client      = new HttpClient();
            var queryString = HttpUtility.ParseQueryString(string.Empty);

            Console.WriteLine("{0} stories for sentiment calculation", headlines.NewsItems.Count);

            //return if no headlines to process
            if (headlines.NewsItems.Count == 0)
            {
                return(headlines);
            }

            // Request headers
            client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", TEXTANALYTICS_KEY);

            var uri = "https://westus.api.cognitive.microsoft.com/text/analytics/v2.0/sentiment?" + queryString;

            HttpResponseMessage response;

            // Request body. Insert your text data here in JSON format.
            // thank you https://text-analytics-demo.azurewebsites.net/Home/SampleCode
            string mybody1      = "{\"documents\":[";
            string mybody2      = "]}";
            string headlineBody = "";

            foreach (newsItem headline in headlines.NewsItems)
            {
                //add json escape characters
                string tempTitle   = HttpUtility.JavaScriptStringEncode(headline.Title);
                string tempContent = HttpUtility.JavaScriptStringEncode(headline.Content);

                headlineBody += "{\"id\":\"" + headline.ID + "\",\"text\":\"" + tempTitle + " " + tempContent + "\"},";
            }
            headlineBody = headlineBody.TrimEnd(',');
            string finalBody = mybody1 + headlineBody + mybody2;

            //finalBody = HttpUtility.JavaScriptStringEncode(finalBody,true);
#if DEBUG
            Console.WriteLine(finalBody);
#endif
            byte[] byteData = Encoding.UTF8.GetBytes(finalBody);

            using (var content = new ByteArrayContent(byteData))
            {
                content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
                response = await client.PostAsync(uri, content);
            }
            string output = await response.Content.ReadAsStringAsync();

            dynamic jsonObj = JsonConvert.DeserializeObject <dynamic>(output);
            var     value   = jsonObj.documents;

            foreach (var data in value)
            {
                foreach (newsItem item in headlines.NewsItems)
                {
                    if (data.id == item.ID)
                    {
                        item.Sentiment = data.score;
                    }
                }
            }
            return(headlines);
        }
예제 #3
0
        static void Main(string[] args)
        {
            Console.WriteLine("*****NEWS FEELING IN PROGRESS******");
            newsItems headlines      = new newsItems();
            string    thisStoryCount = MAXSTORYCOUNT;

            try
            {
                if (args[0] != "")
                {
                    thisStoryCount = args[0];
                }
            }
            catch { }
            Task <newsItems> t;

            try
            {
                //get news headlines
                Console.WriteLine("Getting news headlines...");
                t = GetNews("Top News", thisStoryCount);
                t.Wait();
                headlines = t.Result;
            }
            catch (Exception e)
            {
                Console.Write("ERROR: {0}, {1}", e.Message, e.StackTrace);
            }

            try
            {
                //get sentimate - happy, meh, sucky
                Console.WriteLine("Detecting feels...");
                t = GetSentiment(headlines);
                t.Wait();
                headlines = t.Result;
            }
            catch (Exception e)
            {
                Console.Write("ERROR: {0}, {1}", e.Message, e.StackTrace);
            }

            try
            {
                //save data
                Console.WriteLine("Save feeling for later...");
                saveData(headlines);
            }
            catch (Exception e)
            {
                Console.Write("ERROR: {0}, {1}", e.Message, e.StackTrace);
            }

            try
            {
                //list headlines and sentimate
                //Console.WriteLine("Show feels...");
                //showData();
            }
            catch (Exception e)
            {
                Console.Write("ERROR: {0}, {1}", e.Message, e.StackTrace);
            }

            try
            {
                //do meta data work
                Console.WriteLine("Calculating...");
                calcAverageSentiment();
            }
            catch (Exception e)
            {
                Console.Write("ERROR: {0}, {1}", e.Message, e.StackTrace);
            }


#if DEBUG
            //Console.ReadLine();
#endif
        }
예제 #4
0
        private static async Task <newsItems> GetNews(string topic, string storyCount)
        {
            //List<string> headlines = new List<string>();
            newsItems headlines   = new newsItems();
            var       client      = new HttpClient();
            var       queryString = HttpUtility.ParseQueryString(string.Empty);

            // Request headers
            client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", NEWSSEARCH_KEY);

            // Request parameters
            queryString["q"]          = topic;
            queryString["count"]      = storyCount;
            queryString["offset"]     = "0";
            queryString["mkt"]        = "en-us";
            queryString["safeSearch"] = "Off";
            queryString["freshness"]  = "Day";
            var uri = "https://api.cognitive.microsoft.com/bing/v5.0/news/search?" + queryString;

            dynamic response = await client.GetAsync(uri);

            dynamic content = await response.Content.ReadAsStringAsync();

            searchResult mySearchResult = JsonConvert.DeserializeObject <searchResult>(content);

            foreach (Value value in mySearchResult.value)
            {
                Image           image    = value.image;
                Thumbnail       thumb    = image.thumbnail;
                List <Provider> provider = value.provider;

#if DEBUG
                Console.WriteLine("Story: {0}: {1}, {2}, {3}", value.name.GetHashCode().ToString(), provider[0].name.ToString(), value.category, value.name);
#endif
                headlines.Add(new newsItem(value.name, value.url, thumb.contentUrl, thumb.height, thumb.width, value.description, Convert.ToDateTime(value.datePublished), provider[0].name.ToString(), value.category));
            }

            //check to see if we already recorded this headline
            //flag items to remove
            List <newsItem> duplicates = new List <newsItem>();
            CloudTable      table      = getTableStorage("newsItems");
            foreach (newsItem item in headlines.NewsItems)
            {
                TableOperation retrieveOperation = TableOperation.Retrieve <newsItem>("newsFeels", item.Title.GetHashCode().ToString());

                TableResult query = table.Execute(retrieveOperation);

                //if we have result and result has sentiment > 0 then remove that story from processing
                if (query.Result != null && ((newsItem)query.Result).Sentiment > 0)
                {
                    duplicates.Add(item);
                }
            }
            //remove duplicates
            foreach (newsItem item in duplicates)
            {
                headlines.NewsItems.Remove(item);
            }
            Console.WriteLine("{0} new stories detected", headlines.NewsItems.Count);
            return(headlines);
        }