public ReviewEntity PopulateReviewDetail(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
string rating = string.Empty;
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("Body is empty");
}
else
{
// Reviewer Name
// Reviewer Rating
// Review Text
var headerNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "ndmv-celeb-detail-bread ndmv-review-breadcrumb");
HtmlNode node = headerNode.SelectSingleNode("div");
var header = node == null ? string.Empty : node.InnerText;
var reviewerName = helper.GetElementWithAttribute(node, "a", "class", "fn");
var reviewName = reviewerName == null ? string.Empty : reviewerName.InnerText;
var reviewContentNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "row ndmv-celeb-detail-info ndmv-review-detail");
var contentNode = helper.GetElementWithAttribute(reviewContentNode, "div", "class", "col-md-16");
var textNode = contentNode.Elements("p");
var reviews = contentNode == null ? string.Empty : textNode.FirstOrDefault().InnerText.Replace("SPOILERS ALERT", string.Empty);
var reviewerRating = string.Empty;
var reviewRating = helper.GetElementWithAttribute(bodyNode, "div", "class", "ndmv-movie-rating");
if (reviewRating != null)
{
reviewerRating = PrepareRatingValue(reviewRating);
}
else
{
reviewerRating = string.Empty;
}
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = reviews.Trim();
re.ReviewerName = reviewName.Trim();
re.ReviewerRating = reviewerRating;
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
public static bool UpdateReviewById(this IStore store, ReviewEntity review)
{
Debug.Assert(review != null);
var list = new List<ReviewEntity> { review };
var retList = store.UpdateReviewsById(list);
Debug.Assert(retList.Count == 1);
return retList[retList.Keys.FirstOrDefault()];
}
public ReviewEntity(ReviewEntity review)
: base(review.PartitionKey, review.RowKey)
{
ReviewId = review.ReviewId;
ReviewerName = review.ReviewerName;
Review = review.Review;
ReviewerRating = review.ReviewerRating;
SystemRating = review.SystemRating;
}
public ReviewEntity PopulateReviewDetail(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
string rating = string.Empty;
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("Body is empty");
}
else
{
var headerNode = helper.GetElementWithAttribute(bodyNode, "h4", "class", "book_page_title");
//HtmlNode head = headerNode.SelectSingleNode("h1");
var header = headerNode == null ? string.Empty : headerNode.InnerText;
var reviewContentNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "book_para");
HtmlNodeCollection nodes = reviewContentNode.SelectNodes("p");
var review = string.Empty;
foreach (var ratingNode in nodes)
{
review += ratingNode.InnerText;
}
var reviewerRating = helper.GetElementWithAttribute(bodyNode, "article", "class", "floatl w591px");
//HtmlNodeCollection rateImg = reviewerRating.SelectNodes("figure");
if (reviewerRating != null)
{
try
{
rating = PrepareRatingValue(reviewerRating);
}
catch (Exception)
{
}
}
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = review.Trim();
re.ReviewerName = "Anupama Chopra";
re.ReviewerRating = rating;
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
public static ReviewEntity CreateReviewEntity(string reviewrName, string review, int reviewerRating = 0, int systemRating = 0)
{
var reviewId = Guid.NewGuid().ToString();
var reviewEntity = new ReviewEntity(reviewId);
reviewEntity.ReviewId = reviewId;
reviewEntity.ReviewerName = reviewrName;
reviewEntity.Review = review;
reviewEntity.ReviewerRating = reviewerRating;
reviewEntity.SystemRating = systemRating;
return reviewEntity;
}
public ReviewEntity PopulateReviewDetail(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
string rating = string.Empty;
string reviewerName = string.Empty;
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("Body is empty");
}
else
{
// Review Text
// Rate is not present
// Reviewer Name
var topNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "artTps");//artTps
var reviewerNode = helper.GetElementWithAttribute(topNode, "span", "class", "by");//artTps topNode.Element("span");
var reviewer = reviewerNode.Element("a");
if (reviewer != null)
{
reviewerName = reviewer.InnerText;
}
var reviewContentNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "fullCont");
HtmlNodeCollection nodes = reviewContentNode.SelectNodes("p");
var review = string.Empty;
foreach (var ratingNode in nodes)
{
review += ratingNode.InnerText;
}
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = review.Trim();
re.ReviewerName = reviewerName.Trim();
re.ReviewerRating = string.Empty;
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
public ReviewEntity PopulateReviewDetail(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
string rating = string.Empty;
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("Body is empty");
}
else
{
var headerNode = helper.GetElementWithAttribute(bodyNode, "span", "class", "arttle");
HtmlNode head = headerNode.SelectSingleNode("h1");
var header = head == null ? string.Empty : head.InnerText;
/*
var reviewerName = helper.GetElementWithAttribute(bodyNode, "span", "class", "grey1");
var reviewName = reviewerName == null ? string.Empty : reviewerName.InnerText;
*/
var reviewContentNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "Normal");
var review = reviewContentNode == null ? string.Empty : reviewContentNode.InnerText;
var reviewRatingNode = helper.GetElementWithAttribute(bodyNode, "div", "id", "sshow");
var ratingNode = helper.GetElementWithAttribute(reviewRatingNode, "td", "class", "flmcast");
var rates = helper.GetElementWithAttribute(ratingNode, "span", "id", "stp");
var rate = rates == null ? string.Empty : rates.InnerText;
int found = rate.IndexOf(",");
rate = rate.Substring(0, found);
rate = rate.Substring(rate.Length - 3);
if(!rate.Contains(".")){
rate = rate.Substring(rate.Length - 1);
}
rate = (Double.Parse(rate) * 2).ToString();
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = review.Trim();
re.ReviewerName = "Gaurav Malani";
re.ReviewerRating = rate;
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
public ReviewEntity(ReviewEntity review)
: base(review.PartitionKey, review.RowKey)
{
ReviewId = review.ReviewId;
ReviewerName = review.ReviewerName;
Review = review.Review;
ReviewerRating = review.ReviewerRating;
SystemRating = review.SystemRating;
MovieId = review.MovieId;
Hot = review.Hot;
OutLink = review.OutLink;
Affiliation = review.Affiliation;
}
public ReviewEntity PopulateReviewDetail(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
string rating = string.Empty;
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("Body is empty");
}
else
{
var headerNode = helper.GetElementWithAttribute(bodyNode, "td", "class", "articleheader");
HtmlNode head = headerNode.SelectSingleNode("h1");
var header = head == null ? head.InnerHtml : head.InnerText;
// this is for reviewer inside the p tag
var reviewerName1 = helper.GetElementWithAttribute(bodyNode, "p", "class", "articleauthor");
// this is for reviewer isnide the span
var reviewerName2 = helper.GetElementWithAttribute(bodyNode, "span", "class", "articleauthor");
var reviewName = reviewerName1 == null ? reviewerName2.InnerText : reviewerName1.InnerText;
reviewName = reviewName.Trim();
var reviewContentNode = helper.GetElementWithAttribute(bodyNode, "td", "class", "story");
HtmlNodeCollection nodes = reviewContentNode.SelectNodes("p");
var review = string.Empty;
foreach (var ratingNode in nodes)
{
review += ratingNode.InnerText;
}
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = review.Trim();
re.ReviewerName = reviewName.Trim();
re.ReviewerRating = string.Empty;
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
public ReviewEntity PopulateReviewDetail(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
string rating = string.Empty;
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("Body is empty");
}
else
{
var headerNode = helper.GetElementWithAttribute(bodyNode, "h1", "class", "arti_heading");
var header = headerNode == null ? headerNode.InnerHtml : headerNode.InnerText;
var reviewerName = helper.GetElementWithAttribute(bodyNode, "span", "class", "grey1");
var reviewName = reviewerName == null ? string.Empty : reviewerName.InnerText;
var reviewContentNode = helper.GetElementWithAttribute(bodyNode, "div", "itemprop", "articleBody");
HtmlNodeCollection nodes = reviewContentNode.SelectNodes("p");
var review = string.Empty;
var reviewerRating = string.Empty;
foreach (var ratingNode in nodes)
{
review += ratingNode.InnerText;
if (ratingNode.InnerText.ToLower().Contains("rating"))
{
reviewerRating = PrepareRatingValue(ratingNode);
}
}
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = review.Replace("'", "'").Trim();
re.ReviewerName = reviewName.Trim();
re.ReviewerRating = reviewerRating;
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
public ReviewEntity PopulateReviewDetail(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
string rating = string.Empty;
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("Body is empty");
}
else
{
var headerNode = helper.GetElementWithAttribute(bodyNode, "h1", "class", "entry-title");
//HtmlNode head = headerNode.SelectSingleNode("h1");
var header = headerNode == null ? headerNode.InnerHtml : headerNode.InnerText;
var reviewerName = helper.GetElementWithAttribute(bodyNode, "span", "class", "author vcard");
var reviewName = reviewerName == null ? string.Empty : reviewerName.InnerText;
var reviewContentNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "entry-content");
HtmlNodeCollection nodes = reviewContentNode.SelectNodes("p");
var review = string.Empty;
foreach (var ratingNode in nodes)
{
review += ratingNode.InnerText;
}
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = review.Trim();
re.ReviewerName = "Vajir Singh (" + reviewName.Trim() + ")";
re.ReviewerRating = string.Empty;
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
public ReviewEntity(ReviewEntity review)
: base(review.PartitionKey, review.RowKey)
{
ReviewId = review.ReviewId;
ReviewerName = review.ReviewerName;
Review = review.Review;
ReviewerRating = review.ReviewerRating;
SystemRating = review.SystemRating;
MovieId = review.MovieId;
Hot = review.Hot;
OutLink = review.OutLink;
Affiliation = review.Affiliation;
Summary = review.Summary;
MyScore = review.MyScore;
JsonString = review.JsonString;
Tags = review.Tags;
AlgoLogUrl = review.AlgoLogUrl;
}
public static ReviewEntity CreateReviewEntity(string reviewrName, string review, string movieId, string reviewerId, bool hot, string outLink, string affiliation, string summary, string myScore, string jsonString, int reviewerRating = 0, int systemRating = 0, string tags = null)
{
var reviewId = Guid.NewGuid().ToString();
var reviewEntity = new ReviewEntity(reviewId);
reviewEntity.ReviewId = reviewId;
reviewEntity.ReviewerName = reviewrName;
reviewEntity.Review = review;
reviewEntity.ReviewerRating = reviewerRating.ToString();
reviewEntity.SystemRating = systemRating;
reviewEntity.MovieId = movieId;
reviewEntity.ReviewerId = reviewerId;
reviewEntity.Hot = hot;
reviewEntity.OutLink = outLink;
reviewEntity.Affiliation = affiliation;
reviewEntity.Summary = summary;
reviewEntity.MyScore = myScore;
reviewEntity.JsonString = jsonString;
reviewEntity.Tags = tags ?? string.Empty;
reviewEntity.AlgoLogUrl = string.Empty;
return(reviewEntity);
}
private ReviewEntity PopulateReviewDetails(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("body node is null");
}
else
{
var headerNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "story_wid");
var reviewerName = helper.GetElementWithAttribute(headerNode, "span", "class", "sty_agn");
HtmlNode node = reviewerName.Element("a");
var reviewName = node == null ? reviewerName.InnerHtml : node.InnerText;
var reviewContent = helper.GetElementWithAttribute(headerNode, "div", "class", "sty_txt");
var review = reviewContent.InnerText;
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = review.Trim();
re.ReviewerName = reviewName.Trim();
re.ReviewerRating = string.Empty;
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
private ReviewEntity PopulateReviewDetails(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("body node is null");
}
else
{
var headerNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "Normal");
var reviewerNode = helper.GetElementWithAttribute(headerNode, "span", "id", "advenueINTEXT");
string reviewerName = string.Empty;
if (reviewerNode != null)
{
reviewerName = reviewerNode.InnerText;
}
// Clean up the review name - It has lots of scrape values along with name
if (!string.IsNullOrEmpty(reviewerName))
{
reviewerName = reviewerName.Replace(" ", " ").Replace("By", "").Trim();
}
HtmlNodeCollection nodes = headerNode.SelectNodes("strong");
var rating = string.Empty;
foreach (var node in nodes)
{
if (node.InnerText.ToLower().Trim().Contains("rating:"))
{
rating = node.InnerText.Replace(" ", "").Replace("Rating:", "").Length.ToString();
}
if (string.IsNullOrEmpty(reviewerName) && node.InnerText.ToLower().Trim().Contains("by:"))
{
reviewerName = rating = node.InnerText.Replace(" ", "").Replace("By:", "");
}
else if (string.IsNullOrEmpty(reviewerName))
{
reviewerName = "mumbaimirror";
}
}
float multipliedRating = 0;
float.TryParse(rating, out multipliedRating);
if (multipliedRating > 0)
{
// All other rating are based out of 10 where as Filmfare is out of 5.
rating = (multipliedRating * 2).ToString();
}
var review = string.Empty;
if (!string.IsNullOrEmpty(rating))
review = headerNode.InnerText.Substring(headerNode.InnerText.LastIndexOf("Rating:") + rating.Length + 1);
else if (!string.IsNullOrEmpty(reviewerName) && reviewerName != "mumbaimirror")
review = headerNode.InnerText.Substring(headerNode.InnerText.LastIndexOf(reviewerName) + rating.Length + 1);
else
review = headerNode.InnerText;
re.Affiliation = affiliation;
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Review = review.Trim();
re.ReviewerName = reviewerName.Trim();
re.ReviewerRating = rating.ToString();
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
public ReviewEntity PopulateReviewDetail(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("Body is empty");
}
else
{
try
{
// Rating
var ratingNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "story-rating");
var imageContainer = ratingNode.Element("span");
var ratingImages = imageContainer.Elements("img");
int rate = 0;
foreach (var rateImage in ratingImages)
{
HtmlAttribute src = rateImage.Attributes["src"];
if (src != null)
{
if (src.Value.Contains("star-one-1"))
{
rate += 1;
}
#region Commented Code
/*else if (src.Value.Contains("star-no-1"))
{
// We don't need to add any rating because its 0
}
else
{
// This case could be 0.5 but not sure how it appears on review page.
// Need to wait for same for other movie reviews
}*/
#endregion
}
}
rate = rate * 2;
// Reviewer
var reviewerNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "editor");
var reviewerName = reviewerNode.Element("a");//, "class", "fn");
var reviewName = reviewerName == null ? string.Empty : reviewerName.InnerText;
// Review Text
var reviewBody = helper.GetElementWithAttribute(bodyNode, "div", "class", "main-body-content");
var reviewText = reviewBody == null ? string.Empty : reviewBody.InnerText;
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = reviewText.Replace("'", "'").Trim();
re.ReviewerName = reviewName.Trim();
re.ReviewerRating = rate.ToString();
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
catch (Exception)
{
// Log an exception
}
}
}
return null;
}
private void CrawlfromXML(string xmlData, string movieName)
{
if (string.IsNullOrEmpty(xmlData)) return;
Crawler.MovieCrawler movieCrawler = new Crawler.MovieCrawler();
JavaScriptSerializer json = new JavaScriptSerializer();
try
{
XmlDocument xdoc = new XmlDocument();
#region Movie Crawler
xdoc.LoadXml(xmlData);
var movies = xdoc.SelectNodes("Movies/Month/Movie");
if (movies == null) return;
foreach (XmlNode movie in movies)
{
// Check movie name, we just need to crawl single movie and not all the movies present in XML file for current month
if (movie.Attributes["name"].Value.ToLower() != movieName.ToLower())
{
continue;
}
if (movie.Attributes["link"] != null && !string.IsNullOrEmpty(movie.Attributes["link"].Value))
{
try
{
List<string> critics = new List<string>();
#region Crawl Movie
MovieEntity mov = movieCrawler.Crawl(movie.Attributes["link"].Value);
TableManager tblMgr = new TableManager();
// Save the crawled content because in case of new movies, it fails
tblMgr.UpdateMovieById(mov);
string posterUrl = string.Empty;
if (movie.Attributes["santaposterlink"] != null && !string.IsNullOrEmpty(movie.Attributes["santaposterlink"].Value))
{
XMLMovieProperties prop = new XMLMovieProperties();
prop.SantaPosterLink = movie.Attributes["santaposterlink"].Value;
prop.MovieName = mov.UniqueName;
CrawlPosters(json.Serialize(prop));
}
// Crawl Songs from Saavn
if (string.IsNullOrEmpty(mov.RowKey) || string.IsNullOrEmpty(mov.MovieId)) continue;
tblMgr.UpdateMovieById(mov);
#endregion
#region Crawl Movie Reviews
#region Crawler
try
{
BollywoodHungamaReviews bh = new BollywoodHungamaReviews();
HindustanTimesReviews ht = new HindustanTimesReviews();
FilmfareReviews ff = new FilmfareReviews();
CnnIbn cibn = new CnnIbn();
BoxOfficeIndia boi = new BoxOfficeIndia();
Dna dna = new Dna();
FirstPost fp = new FirstPost();
IndianExpress ie = new IndianExpress();
KomalNahta kn = new KomalNahta();
MidDay md = new MidDay();
Ndtv ndtv = new Ndtv();
Rajasen rs = new Rajasen();
Rediff rdf = new Rediff();
Telegraph tg = new Telegraph();
TheHindu th = new TheHindu();
TimesOfIndia toi = new TimesOfIndia();
AnupamaChopra ac = new AnupamaChopra();
MumbaiMirror mm = new MumbaiMirror();
var reviews = movie.SelectNodes("Review");
List<ReviewEntity> reviewList = tblMgr.GetReviewByMovieId(mov.MovieId);
foreach (XmlNode review in reviews)
{
ReviewEntity duplicateRE = reviewList.Find(r => r.Affiliation == review.Attributes["name"].Value);
if (duplicateRE != null)
{
// We found the duplicate, skip this review to crawl
continue;
}
ReviewEntity re = new ReviewEntity();
string reviewLink = review.Attributes["link"].Value;
switch (review.Attributes["name"].Value.Trim())
{
case "BollywoodHungama":
case "Bollywood Hungama":
re = bh.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "Hindustan Times":
re = ht.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "Filmfare":
re = ff.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "CNN IBN":
case "CNNIBN":
re = cibn.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "Box Office India":
re = boi.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "DNA":
re = dna.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "FirstPost":
re = fp.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "Indian Express":
re = ie.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "Komal Nahta's Blog":
re = kn.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "Mid Day":
case "MidDay":
re = md.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "NDTV":
re = ndtv.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "rajasen.com":
re = rs.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "Rediff":
re = rdf.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "Telegraph":
re = tg.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "The Hindu":
re = th.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "Times of India":
re = toi.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "anupamachopra.com":
re = ac.Crawl(reviewLink, review.Attributes["name"].Value);
break;
case "Mumbai Mirror":
re = mm.Crawl(reviewLink, review.Attributes["name"].Value);
break;
}
if (re == null)
continue;
critics.Add(re.ReviewerName);
// update the IDs - Movie Id, Reviewer Id etc.
string reviewerId = ReviewCrawler.SetReviewer(re.ReviewerName, review.Attributes["name"].Value);
//re.RowKey = re.ReviewId = new Guid().ToString();
re.ReviewerId = reviewerId;
re.MovieId = mov.MovieId;
re.OutLink = reviewLink;
tblMgr.UpdateReviewById(re);
}
}
catch (Exception)
{
}
#endregion
#endregion
#region Lucene Search Index
List<APIRole.UDT.Cast> casts = json.Deserialize(mov.Cast, typeof(List<APIRole.UDT.Cast>)) as List<APIRole.UDT.Cast>;
List<String> posters = json.Deserialize(mov.Posters, typeof(List<String>)) as List<String>;
List<String> actors = new List<string>();
if (casts != null)
{
foreach (var actor in casts)
{
// actor, director, music, producer
string role = actor.role.ToLower();
string characterName = string.IsNullOrEmpty(actor.charactername) ? string.Empty : actor.charactername;
// Check if artist is already present in the list for some other role.
// If yes, skip it. Also if the actor name is missing then skip the artist
if (actors.Contains(actor.name) || string.IsNullOrEmpty(actor.name) || actor.name == "null")
continue;
// If we want to showcase main artists and not all, keep the following switch... case.
switch (role)
{
case "actor":
actors.Add(actor.name);
break;
case "producer":
// some times producer are listed as line producer etc.
// We are not interested in those artists as of now?! Hence skipping it
if (characterName == role)
{
actors.Add(actor.name);
}
break;
case "music":
case "director":
// Main music director and movie director does not have associated character name.
// Where as other side directors have associated character name as associate director, assitant director.
// Skipping such cases.
if (string.IsNullOrEmpty(characterName))
{
actors.Add(actor.name);
}
break;
}
// If we want to showcase all the technicians
//actors.Add(actor.name);
}
}
if (posters != null && posters.Count > 0)
{
posterUrl = posters[posters.Count - 1];
}
// include reviewer & their affiliation in index file
MovieSearchData movieSearchIndex = new MovieSearchData();
movieSearchIndex.Id = mov.RowKey;
movieSearchIndex.Title = mov.Name;
movieSearchIndex.Type = mov.Genre;
movieSearchIndex.TitleImageURL = posterUrl;
movieSearchIndex.UniqueName = mov.UniqueName;
movieSearchIndex.Description = json.Serialize(actors);
movieSearchIndex.Critics = json.Serialize(critics);
movieSearchIndex.Link = mov.UniqueName;
LuceneSearch.AddUpdateLuceneIndex(movieSearchIndex);
#endregion
}
catch (Exception)
{
Debug.WriteLine("Error while crawling movie - " + movie.Attributes["link"].Value);
}
}
}
#endregion
}
catch (Exception ex)
{
Debug.WriteLine("Exception: {0}", ex);
throw;
}
}
private ReviewEntity PopulateReviewDetails(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("body node is null");
}
else
{
var headerNode = helper.GetElementWithAttribute(bodyNode, "div", "id", "celeb_article_postview_tab");
var reviewrName = helper.GetElementWithAttribute(headerNode, "div", "class", "m9090");
var reviewName = reviewrName.InnerText;
// Clean up the review name - It has lots of scrape values along with name
if (!string.IsNullOrEmpty(reviewName))
{
reviewName = reviewName.Replace(" ", " ").Replace("By", "").Trim();
int nameLength = reviewName.IndexOf(",");
if (nameLength > 1)
{
reviewName = reviewName.Substring(0, nameLength);
}
}
var ratingNode = helper.GetElementWithAttribute(reviewrName, "img", "width", "93");
var rating = ratingNode.Attributes["title"] != null ? ratingNode.Attributes["title"].Value : string.Empty;
float multipliedRating = 0;
float.TryParse(rating, out multipliedRating);
if (multipliedRating > 0)
{
// All other rating are based out of 10 where as Filmfare is out of 5.
rating = (multipliedRating * 2).ToString();
}
var reviewContent = helper.GetElementWithAttribute(headerNode, "div", "class", " mfl mmb31 mfnt12 minline malignjus mmr18");
var review = reviewContent.InnerText;
re.Affiliation = affiliation;
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Review = review.Trim();
re.ReviewerName = reviewName.Trim();
re.ReviewerRating = rating.ToString();
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
public ReviewEntity GetRandomReview(string id)
{
var review = new ReviewEntity(id);
var rand = new Random();
review.ReviewId = id;
review.Review = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas libero erat, elementum at dictum quis, commodo eget augue. Suspendisse potenti. Sed a tempor magna. Maecenas quis metus ac sapien faucibus eleifend. Mauris id elementum augue, ac suscipit nunc. Sed non lorem at turpis pellentesque tempus gravida vitae lectus. Nullam quis blandit augue. Proin at pretium magna, quis rutrum arcu. Praesent consectetur aliquam magna, fermentum sodales dolor.";
review.ReviewerName = GetRandomElementsFromList(Reviewers, rand);
review.ReviewId = Guid.NewGuid().ToString();
review.ReviewerRating = rand.Next(10);
review.SystemRating = rand.Next(10);
return review;
}
public ReviewEntity PopulateReviewDetail(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
string rating = string.Empty;
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("Body is empty");
}
else
{
var headerNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "fleft");
var reviewerNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "blbox");
// var reviewerMiddleNode = helper.GetElementWithAttribute(reviewerNode, "div", "class", "blbox");
var reviewerName = helper.GetElementWithAttribute(reviewerNode, "span", "class", "fwb");
HtmlNode node = reviewerName.Element("a");
var reviewName = node == null ? reviewerName.InnerHtml : node.InnerText;
var reviewContentNode = helper.GetElementWithAttribute(bodyNode, "div", "id", "atxt_box");
HtmlNodeCollection nodes = reviewContentNode.SelectNodes("p");
var review = string.Empty;
var reviewerRating = string.Empty;
foreach (var ratingNode in nodes)
{
review += ratingNode.InnerText;
if (ratingNode.InnerText.ToLower().Contains("rating"))
{
try
{
rating = ratingNode.InnerText.Replace("Rating:", "").Replace("/", "").Trim();
rating = rating.Remove(rating.Length - 1);
rating = (Decimal.Parse(rating) * 2).ToString();
}
catch (Exception)
{
}
}
}
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = review.Trim();
re.ReviewerName = reviewName.Trim();
re.ReviewerRating = rating;
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
public ReviewEntity PopulateReviewDetail(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
string rating = string.Empty;
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("Body is empty");
}
else
{
var headerNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "article_detail");
HtmlNode head = headerNode.SelectSingleNode("h1");
var header = head == null ? head.InnerHtml : head.InnerText;
var reviewerName = helper.GetElementWithAttribute(headerNode, "div", "class", "metalink");
var reviewName = reviewerName == null ? string.Empty : reviewerName.InnerText;
reviewName = reviewName.Trim();
int rn = reviewName.IndexOf("|");
reviewName = reviewName.Substring(0, rn);
// reviewName = reviewName.Substring(0, reviewName.Length - 20);
var reviewContentNode = helper.GetElementWithAttribute(bodyNode, "span", "itemprop", "articleBody");
HtmlNodeCollection nodes = reviewContentNode.SelectNodes("p");
var review = string.Empty;
foreach (var ratingNode in nodes)
{
review += ratingNode.InnerText;
}
var reviewerRating = helper.GetElementWithAttribute(bodyNode, "h2", "class", "footer_style");
if (reviewerRating != null)
{
try
{
rating = PrepareRatingValue(reviewerRating);
}
catch (Exception)
{
}
}
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = review.Trim();
re.ReviewerName = reviewName.Trim();
re.ReviewerRating = rating;
re.MyScore = string.Empty;
re.JsonString = string.Empty;
return re;
}
}
return null;
}
public string TestUpdate()
{
SetConnectionString();
MovieEntity entity = new MovieEntity();
var rand = new Random((int)DateTimeOffset.UtcNow.Ticks);
entity.RowKey = entity.MovieId = Guid.NewGuid().ToString();
entity.ReviewIds = string.Format("{0},{1}", Math.Abs(rand.Next()), Math.Abs(rand.Next()));
entity.AggregateRating = Math.Abs(rand.Next(10)).ToString();
entity.Directors = string.Format("Gabbar_{0}", rand.Next());
entity.HotOrNot = (Math.Abs(rand.Next())%2) == 1 ? true : false;
entity.MusicDirectors = string.Format("Rahman_{0}", Math.Abs(rand.Next()));
entity.Name = string.Format("aashique {0}", rand.Next());
entity.Producers = string.Format("sippy_{0}", rand.Next());
entity.Actors = string.Format("sahruuk_{0}", rand.Next());
var reviewIds = entity.GetReviewIds();
var reviewList = new List<ReviewEntity>();
foreach (var reviewId in reviewIds)
{
var reviewEntity = new ReviewEntity();
reviewEntity.ReviewId = reviewEntity.RowKey = reviewId;
reviewEntity.Review = string.Format("This is review number {0}", reviewId);
reviewEntity.ReviewerName = string.Format("khan_{0}", rand.Next());
reviewEntity.ReviewerRating = rand.Next(10);
reviewEntity.SystemRating = rand.Next(10);
reviewList.Add(reviewEntity);
}
var tableMgr = new TableManager();
tableMgr.UpdateMovieById(entity);
tableMgr.UpdateReviewsById(reviewList);
return string.Format("Created movie id {0}", entity.MovieId);
}
public string TestUpdate()
{
SetConnectionString();
MovieEntity entity = new MovieEntity();
var rand = new Random((int)DateTimeOffset.UtcNow.Ticks);
#region commented code
/*entity.RowKey = entity.MovieId = Guid.NewGuid().ToString();
entity.Stats = string.Format("{0},{1}", Math.Abs(rand.Next()), Math.Abs(rand.Next()));
entity.Songs = Math.Abs(rand.Next(10)).ToString();
entity.Ratings = string.Format("Gabbar_{0}", rand.Next());
entity.Trailers = string.Format("Traler_{0}", rand.Next());
entity.Casts = string.Format("Rahman_{0}", Math.Abs(rand.Next()));
entity.Name = string.Format("aashique {0}", rand.Next());
entity.Synopsis = string.Format("sippy_{0}", rand.Next());
entity.Posters = @"{""height"" : 300,""width"" : 200,""url"" : ""test""}";
entity.Month = "March";
entity.Year = "2014";*/
#endregion
entity.RowKey = entity.MovieId = Guid.NewGuid().ToString();
entity.Stats = @"[{""budget"" : ""30,000"",""boxoffice"": ""50000""}]";;
entity.Songs = @"[{""name"" : ""chaiyya chaiyya"",""url"" : ""songtest""}, {""name"" : ""chaiyya chaiyya"",""url"" : ""songtest""}]";
entity.Ratings = @"[{""system"" : 5,""critic"" : 6,""hot"" : ""no""}]";
entity.Trailers = @"[{""name"" : ""best movie"",""url"" : ""songtest""}, {""name"" : ""chaiyya chaiyya"",""url"" : ""songtest""}]";
entity.Casts = @"[{""name"" : ""ben affleck"",""charactername"" : ""mickey"",""image"" : {""height"" : 300,""width"": 200,""url"" : ""test""},""role"" : ""producer""},
{""name"" : ""jerry afflect"",""charactername"" : ""mouse"",""image"" : {""height"" : 300,""width"" : 200,""url"" : ""test""},""role"" : ""actor""}]";
entity.Pictures = @"[{""caption"" : ""test caption"",""image"" : {""height"" : 300,""width"" : 200,""url"" : ""test""}}]";
entity.Name = string.Format("aashique {0}", rand.Next());
entity.Synopsis = "this is a brilliant scary movie";
entity.Posters = @"[{""height"" : 300,""width"" : 200,""url"" : ""test""}]";
entity.Genre = "Action";
entity.Month = "March";
entity.Year = "2014";
//var reviewIds = entity.GetReviewIds();
var reviewIds = new List<string>() { Math.Abs(rand.Next()).ToString(), Math.Abs(rand.Next()).ToString() };
var reviewList = new List<ReviewEntity>();
foreach (var reviewId in reviewIds)
{
var reviewEntity = new ReviewEntity();
reviewEntity.ReviewId = reviewEntity.RowKey = reviewId;
reviewEntity.MovieId = entity.MovieId;
reviewEntity.Review = string.Format("This is review number {0}", reviewId);
reviewEntity.ReviewerName = string.Format("khan_{0}", rand.Next());
reviewEntity.ReviewerRating = rand.Next(10);
reviewEntity.SystemRating = rand.Next(10);
reviewEntity.Hot = false;
reviewEntity.OutLink = "this is a link";
reviewEntity.Affiliation= @"[{""name"" : ""Yahoo"", ""link"" : ""http://in.yahoo.com/?p=us"", ""reviewlink"" : ""http://in.movies.yahoo.com/blogs/movie-reviews/yahoo-movies-review-gunday-124034093.html"", ""logoimage"" : ""Images/Yahoo_Logo.png""},{""name"" : ""Hidustan Times"", ""link"" : ""http://www.hindustantimes.com"", ""reviewlink"" : ""http://www.hindustantimes.com/entertainment/reviews/movie-review-by-rashid-irani-dallas-buyers-club-is-a-must-watch/article1-1189520.aspx"", ""logoimage"" : ""Images/hindustan-times.jpg""}]";
reviewList.Add(reviewEntity);
}
var tableMgr = new TableManager();
tableMgr.UpdateMovieById(entity);
tableMgr.UpdateReviewsById(reviewList);
return string.Format("Created movie id {0}", entity.MovieId);
}
public static ReviewEntity CreateReviewEntity(string reviewrName, string review, string movieId, bool hot, string outLink, string affiliation, int reviewerRating = 0, int systemRating = 0)
{
var reviewId = Guid.NewGuid().ToString();
var reviewEntity = new ReviewEntity(reviewId);
reviewEntity.ReviewId = reviewId;
reviewEntity.ReviewerName = reviewrName;
reviewEntity.Review = review;
reviewEntity.ReviewerRating = reviewerRating;
reviewEntity.SystemRating = systemRating;
reviewEntity.MovieId = movieId;
reviewEntity.Hot = hot;
reviewEntity.OutLink = outLink;
reviewEntity.Affiliation = affiliation;
return reviewEntity;
}
public static ReviewEntity CreateReviewEntity(string reviewrName, string review, string movieId, string reviewerId, bool hot, string outLink, string affiliation, string summary, string myScore, string jsonString, int reviewerRating = 0, int systemRating = 0, string tags = null)
{
var reviewId = Guid.NewGuid().ToString();
var reviewEntity = new ReviewEntity(reviewId);
reviewEntity.ReviewId = reviewId;
reviewEntity.ReviewerName = reviewrName;
reviewEntity.Review = review;
reviewEntity.ReviewerRating = reviewerRating.ToString();
reviewEntity.SystemRating = systemRating;
reviewEntity.MovieId = movieId;
reviewEntity.ReviewerId = reviewerId;
reviewEntity.Hot = hot;
reviewEntity.OutLink = outLink;
reviewEntity.Affiliation = affiliation;
reviewEntity.Summary = summary;
reviewEntity.MyScore = myScore;
reviewEntity.JsonString = jsonString;
reviewEntity.Tags = tags ?? string.Empty;
reviewEntity.AlgoLogUrl = string.Empty;
return reviewEntity;
}
/// <summary>
/// Update the review by its id
/// </summary>
/// <param name="store"></param>
/// <param name="review"></param>
/// <returns></returns>
public static bool UpdateReviewById(this IStore store, ReviewEntity review)
{
Debug.Assert(review != null);
var list = new ReviewEntity[] { review };
var retList = store.UpdateReviewsById(list);
Debug.Assert(retList.Count == 1);
var key = retList.Keys.FirstOrDefault();
return (key != null) ? retList[key] : false;
}
private ReviewEntity PopulateReviewDetails(string html, string affiliation)
{
ReviewEntity re = new ReviewEntity();
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(html);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode == null)
{
Console.WriteLine("body node is null");
}
else
{
var headerNode = helper.GetElementWithAttribute(bodyNode, "div", "class", "pageContent");
#region Get Reviewer
var reviewerNameContainer = helper.GetElementWithAttribute(headerNode, "span", "class", "written");
var reviewerList = reviewerNameContainer.Elements("a");
string reviewerName = string.Empty;
if (reviewerList != null)
{
try
{
reviewerName = reviewerList.FirstOrDefault().InnerText;
}
catch (Exception)
{
}
}
#endregion
#region Get Rating
var ratingNode = helper.GetElementWithAttribute(bodyNode, "span", "id", "rate_val_change");
var rating = ratingNode.Attributes["class"] != null ? ratingNode.Attributes["class"].Value : string.Empty;
rating = rating.Replace("rate", "");
float multipliedRating = 0;
float.TryParse(rating, out multipliedRating);
if (multipliedRating > 0)
{
// All other rating are based out of 10 where as Filmfare is out of 5.
rating = (multipliedRating * 2).ToString();
}
#endregion
#region Get Review Content
var reviewContent = helper.GetElementWithAttribute(bodyNode, "div", "class", "upperBlk");
var reviews = reviewContent.Element("figure");
string review = string.Empty;
if (reviews != null)
{
var reviewElements = reviews.Elements("p");
foreach (var r in reviewElements)
{
if (!string.IsNullOrEmpty(r.InnerText) && r.InnerText.Length > 300)
{
review = r.InnerText;
break;
}
}
}
#endregion
re.RowKey = re.ReviewId = Guid.NewGuid().ToString();
re.Affiliation = affiliation.Trim();
re.Review = review.Trim();
re.ReviewerName = reviewerName.Trim();
re.ReviewerRating = rating.ToString();
re.JsonString = string.Empty;
re.MyScore = string.Empty;
return re;
}
}
return null;
}