//Getting reviews and rating of the products
public static void GetReviewPage(Regex reg_str, string resp_str, int prodId)
{
Match m = reg_str.Match(resp_str);
string temp = m.Groups[1].Value;
HttpWebRequest req1 = (HttpWebRequest)HttpWebRequest.Create(temp);
req1.Method = "POST";
HttpWebResponse res1 = (HttpWebResponse)req1.GetResponse();
StreamReader stream1 = new StreamReader(res1.GetResponseStream());
string final_response1 = stream1.ReadToEnd();
Regex r4 = new Regex("<span class=\"arp-rating-out-of-text\">\\s*(.+?)\\s*</span>");
Match m1 = r4.Match(final_response1);
string temp1 = m1.Groups[1].Value;
Console.WriteLine("Rating: " + temp1);
DbWrappers db = new DbWrappers();
Product prod = db.GetSpecificProduct(prodId);
string[] token = temp1.Split(' ');
prod.Rating = float.Parse(token[0]);;
hasReviews = true;
db.UpdateProduct(prod);
ExtractingReviews(temp, prodId);
}
public static void ExtractingReviews(string url, int prodId)
{
url = url + "&pageNumber=1";
string final_response = "";
// int count = 1;
for (int i = 1; i < 10; i++)
{
HttpWebRequest req1 = (HttpWebRequest)HttpWebRequest.Create(url);
req1.Method = "POST";
HttpWebResponse res1 = (HttpWebResponse)req1.GetResponse();
StreamReader stream1 = new StreamReader(res1.GetResponseStream());
final_response = stream1.ReadToEnd();
MatchCollection collectedReviews = Regex.Matches(final_response, "<span class=\"a-size-base review-text\">\\s*(.+?)\\s*</span>");
//FileStream fs = new FileStream("review.txt", FileMode.Append);
//StreamWriter sw = new StreamWriter(fs);
DbWrappers wrapper = new DbWrappers();
foreach (Match singleReview in collectedReviews)
{
string review = singleReview.Groups[1].Value;
ProductReview prodReview = new ProductReview();
prodReview.ProductId = prodId;
prodReview.Review = review;
wrapper.AddProductReviews(prodReview);
//sw.WriteLine(count);
//sw.WriteLine(review);
//count++;
}
//sw.Close();
//fs.Close();
url = url.Replace("pageNumber=" + i, "pageNumber=" + (i + 1));
}
}
public static void GetSentimentandNouns()
{
positiveFeatures.Columns.Add("pNouns", typeof(string));
negativeFeatures.Columns.Add("nNouns", typeof(string));
props.setProperty("annotators", "tokenize, ssplit, pos, parse, sentiment");
//props.setProperty("annotators", "tokenize, ssplit, parse");
props.setProperty("ner.useSUTime", "0");
List <string> nouns = new List <string>();
// List<string> adjectives = new List<string>();
string line = "";
int sentiment;
// We should change current directory, so StanfordCoreNLP could find all the model files automatically
var curDir = Environment.CurrentDirectory;
Directory.SetCurrentDirectory(jarRoot);
var pipeline = new StanfordCoreNLP(props);
Directory.SetCurrentDirectory(curDir);
DbWrappers wrap = new DbWrappers();
List <Product> products = wrap.GetAllReviewsAgainstProduct();
foreach (var prod in products)
{
foreach (var review in prod.ProductReviews)
{
Annotation annotation = pipeline.process(review.Review);
var Sentence = annotation.get(typeof(CoreAnnotations.SentencesAnnotation)) as ArrayList;
foreach (CoreMap sen in Sentence)
{
sentiment = GetSentiment(sen.ToString(), pipeline);
if (sentiment == 3 || sentiment == 4)
{
java.lang.Class treeClass = new edu.stanford.nlp.trees.TreeCoreAnnotations.TreeAnnotation().getClass();
edu.stanford.nlp.trees.Tree tree1 = (edu.stanford.nlp.trees.Tree)sen.get(treeClass);
string treeString = tree1.toString();
MatchCollection mc = Regex.Matches(treeString, @"(([N][N]\s\w*)|([N][N][P]\s\w*) |([N][P]\s\w*))");
foreach (Match m in mc)
{
string opinionPhrase = "";
string opinionString = m.ToString();
var phrase = opinionString.Split(' ');
foreach (var subPhrase in phrase)
{
if (!(subPhrase.Contains("NP")) && !(subPhrase.Contains("NNP")) && !(subPhrase.Contains("NNS")) && !(subPhrase.Contains("NN")))
{
String refinedSubPharse1 = subPhrase.Replace(")", string.Empty);
//Console.WriteLine(refinedSubPharse1);
string refinedSubPharse = refinedSubPharse1.ToLower();
if (refinedSubPharse.Contains("harddrive") || refinedSubPharse.Contains("memory") || refinedSubPharse.Contains("hdd") || refinedSubPharse.Contains("keyboard") || refinedSubPharse.Contains("processor") || refinedSubPharse.Contains("speed") || refinedSubPharse.Contains("operating system") || refinedSubPharse.Contains("display") || refinedSubPharse.Contains("graphics") || refinedSubPharse.Contains("price") || refinedSubPharse.Contains("money") || refinedSubPharse.Contains("weight") || refinedSubPharse.Contains("camera") || refinedSubPharse.Contains("dimension") || refinedSubPharse.Contains("screen") || refinedSubPharse.Contains("ram") || refinedSubPharse.Contains("hard disk") || refinedSubPharse.Contains("os") || refinedSubPharse.Contains("battery") || refinedSubPharse.Contains("performance") || refinedSubPharse.Contains("looks") || refinedSubPharse.Contains("hardware") || refinedSubPharse.Contains("speed") || refinedSubPharse.Contains("design"))
{
opinionPhrases.Add(refinedSubPharse);
DataRow newRow = positiveFeatures.NewRow();
newRow["pNouns"] = refinedSubPharse.ToString();
positiveFeatures.Rows.Add(newRow);
}
//opinionPhrase += refinedSubPharse + " ";
}
}
}
}//end sentiment if
else if (sentiment == 1 || sentiment == 0)
{
java.lang.Class treeClass = new edu.stanford.nlp.trees.TreeCoreAnnotations.TreeAnnotation().getClass();
edu.stanford.nlp.trees.Tree tree1 = (edu.stanford.nlp.trees.Tree)sen.get(treeClass);
string treeString = tree1.toString();
MatchCollection mc = Regex.Matches(treeString, @"(([N][N]\s\w*)|([N][N][P]\s\w*) |([N][P]\s\w*))");
foreach (Match m in mc)
{
string opinionPhrase = "";
string opinionString = m.ToString();
var phrase = opinionString.Split(' ');
foreach (var subPhrase in phrase)
{
if (!(subPhrase.Contains("NP")) && !(subPhrase.Contains("NNP")) && !(subPhrase.Contains("NNS")) && !(subPhrase.Contains("NN")))
{
String refinedSubPharse1 = subPhrase.Replace(")", string.Empty);
//Console.WriteLine(refinedSubPharse1);
string refinedSubPharse = refinedSubPharse1.ToLower();
if (refinedSubPharse.Contains("harddrive") || refinedSubPharse.Contains("memory") || refinedSubPharse.Contains("hdd") || refinedSubPharse.Contains("keyboard") || refinedSubPharse.Contains("processor") || refinedSubPharse.Contains("speed") || refinedSubPharse.Contains("operating system") || refinedSubPharse.Contains("display") || refinedSubPharse.Contains("graphics") || refinedSubPharse.Contains("price") || refinedSubPharse.Contains("money") || refinedSubPharse.Contains("weight") || refinedSubPharse.Contains("camera") || refinedSubPharse.Contains("dimension") || refinedSubPharse.Contains("screen") || refinedSubPharse.Contains("ram") || refinedSubPharse.Contains("hard disk") || refinedSubPharse.Contains("os") || refinedSubPharse.Contains("battery") || refinedSubPharse.Contains("performance") || refinedSubPharse.Contains("looks") || refinedSubPharse.Contains("hardware") || refinedSubPharse.Contains("speed") || refinedSubPharse.Contains("design") || refinedSubPharse.Contains("keys") || refinedSubPharse.Contains("ram") || refinedSubPharse.Contains("hard disk") || refinedSubPharse.Contains("os") || refinedSubPharse.Contains("battery") || refinedSubPharse.Contains("performance") || refinedSubPharse.Contains("looks") || refinedSubPharse.Contains("hardware") || refinedSubPharse.Contains("speed") || refinedSubPharse.Contains("design") || refinedSubPharse.Contains("quick"))
{
opinionPhrases.Add(refinedSubPharse);
DataRow newRow = negativeFeatures.NewRow();
newRow["nNouns"] = refinedSubPharse.ToString();
negativeFeatures.Rows.Add(newRow);
}
//opinionPhrase += refinedSubPharse + " ";
}
}
}
} //end sentiment if
}
} //end of review loop
Dictionary <string, int> featureDictionary = new Dictionary <string, int>();
var pFeatures = from row in positiveFeatures.AsEnumerable()
group row by row.Field <string>("pNouns") into grp
select new { key = grp.Key, cnt = grp.Count() };
foreach (var obj in pFeatures)
{
if (obj.key.Equals("mouse") || obj.key.Equals("pad") || obj.key.Equals("touchpad"))
{
featureDictionary["touchpad"] += obj.cnt;
}
else
{
featureDictionary[obj.key] = obj.cnt;
}
FeatureSentiment fs = new FeatureSentiment();
fs.Feature = obj.key;
fs.PId = prod.Id;
fs.Sentiment = 4;
fs.Count = obj.cnt;
wrap.AddFeatureSentiment(fs);
}
var nFeatures = from row in negativeFeatures.AsEnumerable()
group row by row.Field <string>("nNouns") into grp
select new { key = grp.Key, cnt = grp.Count() };
foreach (var obj in nFeatures)
{
FeatureSentiment fs = new FeatureSentiment();
fs.Feature = obj.key;
fs.PId = prod.Id;
fs.Sentiment = 1;
fs.Count = obj.cnt;
wrap.AddFeatureSentiment(fs);
}
}
}
public void GetReviews()
{
// Instantiate Amazon ProductAdvertisingAPI client
AWSECommerceServicePortTypeClient amazonClient = new AWSECommerceServicePortTypeClient();
DbWrappers wrapper = new DbWrappers();
List <Product> list = wrapper.GetAllProducts();
foreach (var prod in list)
{
if (hasReviews == true)
{
hasReviews = false;
}
// prepare an ItemSearch request
ItemSearchRequest request = new ItemSearchRequest();
request.SearchIndex = "Electronics";
//request.Title = "WCF";
request.Keywords = prod.Title;
request.ResponseGroup = new string[] { "ItemAttributes", "Offers", "OfferSummary", "Reviews" };
ItemSearch itemSearch = new ItemSearch();
itemSearch.Request = new ItemSearchRequest[] { request };
itemSearch.AWSAccessKeyId = ConfigurationManager.AppSettings["accessKeyId"];
itemSearch.AssociateTag = "newmobiles0d-20";
// send the ItemSearch request
try
{
ItemSearchResponse response = amazonClient.ItemSearch(itemSearch);
foreach (var item in response.Items[0].Item)
{
if (!hasReviews)
{
// Getting title
Console.WriteLine("Title: " + item.ItemAttributes.Title);
//Getting manufacturer of the item
Console.WriteLine("Manufacturer: " + item.ItemAttributes.Manufacturer);
//Getting model
Console.WriteLine("Model:" + item.ItemAttributes.Model);
if ((item.CustomerReviews.HasReviews) == true)
{
string str = item.CustomerReviews.IFrameURL;
string final_response = "";
try
{
//Getting html from IFrame URL
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(str);
req.Method = "POST";
HttpWebResponse res = (HttpWebResponse)req.GetResponse();
StreamReader stream = new StreamReader(res.GetResponseStream());
final_response = stream.ReadToEnd();
GetMainPageHtml(final_response, prod.Id);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
else
{
break;
}
}
}
catch (Exception e1)
{
}
}
Console.WriteLine("done...enter any key to continue>");
Console.ReadLine();
}
