public static MovieEntity CreateMovieEntity(string name,
string posters,
string rating,
string synopsis,
string cast,
string stats,
string songs,
string trailers,
string pictures,
string genre,
string month,
string year)
{
var movieId = Guid.NewGuid().ToString();
var entity = new MovieEntity(movieId);
entity.MovieId = movieId;
entity.Name = name;
entity.Posters = posters;
entity.Ratings = rating;
entity.Synopsis = synopsis;
entity.Casts = cast;
entity.Stats = stats;
entity.Songs = songs;
entity.Trailers = trailers;
entity.Pictures = pictures;
entity.Genre = genre;
entity.Month = month;
entity.Year = year;
return(entity);
}
public static bool UpdateMovieById(this IStore store, MovieEntity movie)
{
Debug.Assert( movie != null );
var list = new List<MovieEntity> { movie };
var retList = store.UpdateMoviesById(list);
Debug.Assert(retList.Count == 1);
return retList[retList.Keys.FirstOrDefault()];
}
/// <summary>
/// Entry point for the crawler. Pass the URL from source file (XML)
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public MovieEntity Crawl(string url)
{
MovieEntity movie = new MovieEntity();
TableManager tableMgr = new TableManager();
thumbnailPath = string.Empty;
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
#region Get Movie Page Content
Stream receiveStream = response.GetResponseStream();
StreamReader readStream = null;
if (response.CharacterSet == null)
readStream = new StreamReader(receiveStream);
else
readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
moviePageContent = readStream.ReadToEnd();
response.Close();
readStream.Close();
#endregion
movie = PopulateMovieDetails(moviePageContent);
bool crawlPosters = true;
TableManager tblMgr = new TableManager();
MovieEntity me = tblMgr.GetMovieByUniqueName(movie.UniqueName);
if (me != null && !string.IsNullOrEmpty(me.RowKey))
{
movie.RowKey = me.RowKey;
movie.MovieId = me.MovieId;
movie.Popularity = Util.DEFAULT_POPULARITY;
movie.Posters = me.Posters;
movie.Songs = me.Songs;
movie.Trailers = me.Trailers;
movie.State = me.State;
crawlPosters = false;
}
PopulateMovieDetails(ref movie, url, crawlPosters);
return movie;
////tableMgr.UpdateMovieById(movie);
}
}
catch (Exception ex)
{
Debug.WriteLine("An error occurred while populating the movie details. Url = " + url + ". Error=" + ex.Message);
}
return movie;
}
public MovieEntity(MovieEntity entity)
: base(PARTITION_KEY, entity.RowKey)
{
MovieId = entity.MovieId;
Name = entity.Name;
AltNames = entity.AltNames;
Actors = entity.Actors;
Directors = entity.Directors;
Producers = entity.Producers;
MusicDirectors = entity.MusicDirectors;
ReviewIds = entity.ReviewIds;
AggregateRating = entity.AggregateRating;
HotOrNot = entity.HotOrNot;
}
public MovieEntity(MovieEntity entity)
: base(PARTITION_KEY, entity.RowKey)
{
MovieId = entity.MovieId;
Name = entity.Name;
AltNames = entity.AltNames;
Actors = entity.Actors;
Directors = entity.Directors;
Producers = entity.Producers;
MusicDirectors = entity.MusicDirectors;
ReviewIds = entity.ReviewIds;
AggregateRating = entity.AggregateRating;
HotOrNot = entity.HotOrNot;
}
public static MovieEntity CreateMovieEntity(string name,
string posters,
string rating,
string synopsis,
string cast,
string stats,
string songs,
string trailers,
string pictures,
string genre,
string month,
string year,
string uniqueName,
string state,
string myScore,
string jsonString,
string popularity,
string twitterHandle
)
{
var movieId = Guid.NewGuid().ToString();
var entity = new MovieEntity(movieId);
entity.MovieId = movieId;
entity.Name = name;
entity.Posters = posters;
entity.Rating = rating;
entity.Synopsis = synopsis;
entity.Cast = cast;
entity.Stats = stats;
entity.Songs = songs;
entity.Trailers = trailers;
entity.Pictures = pictures;
entity.Genre = genre;
entity.Month = month;
entity.Year = year;
entity.UniqueName = uniqueName;
entity.State = state;
entity.MyScore = myScore;
entity.JsonString = jsonString;
entity.Popularity = popularity;
entity.TwitterHandle = twitterHandle;
return(entity);
}
public MovieEntity GetRandomMovieEntity(string id)
{
var entity = new MovieEntity(id);
var rand = new Random();
entity.Posters = GetRandomElementsFromList(Actors, rand);
entity.Songs = rand.Next(10).ToString();
entity.Ratings = GetRandomElementsFromList(Directors, rand);
entity.Trailers = "Trailers";
entity.MovieId = id;
entity.Casts = GetRandomElementsFromList(MusicDirecotrs, rand);
entity.Name = GetRandomMovieName(Name, rand);
entity.Synopsis = GetRandomElementsFromList(Producers, rand);
entity.Stats = GetRandomElementsFromList(ReviewIds, rand);
return entity;
}
public MovieEntity(MovieEntity entity)
: base(PARTITION_KEY, entity.RowKey)
{
MovieId = entity.MovieId;
Name = entity.Name;
AltNames = entity.AltNames;
Posters = entity.Posters;
Ratings = entity.Ratings;
Synopsis = entity.Synopsis;
Casts = entity.Casts;
Stats = entity.Stats;
Songs = entity.Songs;
Trailers = entity.Trailers;
Pictures = entity.Pictures;
Genre = entity.Genre;
Month = entity.Month;
Year = entity.Year;
}
public MovieEntity GetRandomMovieEntity(string id)
{
var entity = new MovieEntity(id);
var rand = new Random();
entity.Actors = GetRandomElementsFromList(Actors, rand);
entity.AggregateRating = rand.Next(10).ToString();
entity.Directors = GetRandomElementsFromList(Directors, rand);
entity.HotOrNot = rand.Next(2) == 1 ? true : false;
entity.MovieId = id;
entity.MusicDirectors = GetRandomElementsFromList(MusicDirecotrs, rand);
entity.Name = GetRandomMovieName(Name, rand);
entity.Producers = GetRandomElementsFromList(Producers, rand);
entity.ReviewIds = GetRandomElementsFromList(ReviewIds, rand);
return entity;
}
public MovieEntity(MovieEntity entity)
: base(PARTITION_KEY, entity.RowKey)
{
MovieId = entity.MovieId;
Name = entity.Name;
AltNames = entity.AltNames;
Posters = entity.Posters;
Ratings = entity.Ratings;
Synopsis = entity.Synopsis;
Casts = entity.Casts;
Stats = entity.Stats;
Songs = entity.Songs;
Trailers = entity.Trailers;
Pictures = entity.Pictures;
Genre = entity.Genre;
Month = entity.Month;
Year = entity.Year;
}
private static void UpdateCache(MovieEntity movie)
{
// Remove movie from Cache
var movieKey = CacheConstants.MovieInfoJson + movie.UniqueName;
var isPreviouslyCached = CacheManager.Exists(movieKey);
CacheManager.Remove(movieKey);
var tableMgr = new TableManager();
// Cache if previously cached or movie is upcoming/current
Task.Run(() =>
{
if (isPreviouslyCached || movie.State == "upcoming" || movie.State == "now playing")
{
MovieInfoController.GetMovieInfo(movie.UniqueName);
}
});
// Update more Cache
Task.Run(() =>
{
// Update cache for AllMovies
// Note: We are not updating CacheConstants.AllMovieEntitiesSortedByName here
// because typically the name of the movie does not changes as often
CacheManager.Remove(CacheConstants.AllMovieEntities);
tableMgr.GetAllMovies();
// Update current movies
Task.Run(() =>
{
CacheManager.Remove(CacheConstants.UpcomingMovieEntities);
var movies = tableMgr.GetCurrentMovies();
});
// Update upcoming movies
Task.Run(() =>
{
CacheManager.Remove(CacheConstants.NowPlayingMovieEntities);
var movies = tableMgr.GetUpcomingMovies();
});
});
}
public static MovieEntity CreateMovieEntity(string name,
string actors,
string directors,
string producers,
string musicDirecotrs,
string reviewIds,
string aggregateRating,
bool hotOrNot)
{
var movieId = Guid.NewGuid().ToString();
var entity = new MovieEntity(movieId);
entity.MovieId = movieId;
entity.Name = name;
entity.Actors = actors;
entity.Directors = directors;
entity.Producers = producers;
entity.MusicDirectors = musicDirecotrs;
entity.ReviewIds = reviewIds;
entity.AggregateRating = aggregateRating;
entity.HotOrNot = hotOrNot;
return(entity);
}
public MovieEntity(MovieEntity entity)
: base(PARTITION_KEY, entity.RowKey)
{
MovieId = entity.MovieId;
Name = entity.Name;
AltNames = entity.AltNames;
Posters = entity.Posters;
Rating = entity.Rating;
Synopsis = entity.Synopsis;
Cast = entity.Cast;
Stats = entity.Stats;
Songs = entity.Songs;
Trailers = entity.Trailers;
Pictures = entity.Pictures;
Genre = entity.Genre;
Month = entity.Month;
Year = entity.Year;
UniqueName = entity.UniqueName;
State = entity.State;
MyScore = entity.MyScore;
JsonString = entity.JsonString;
Popularity = entity.Popularity;
TwitterHandle = entity.TwitterHandle;
}
public MovieEntity(MovieEntity entity)
: base(PARTITION_KEY, entity.RowKey)
{
MovieId = entity.MovieId;
Name = entity.Name;
AltNames = entity.AltNames;
Posters = entity.Posters;
Rating = entity.Rating;
Synopsis = entity.Synopsis;
Cast = entity.Cast;
Stats = entity.Stats;
Songs = entity.Songs;
Trailers = entity.Trailers;
Pictures = entity.Pictures;
Genre = entity.Genre;
Month = entity.Month;
Year = entity.Year;
UniqueName = entity.UniqueName;
State = entity.State;
MyScore = entity.MyScore;
JsonString = entity.JsonString;
Popularity = entity.Popularity;
TwitterHandle = entity.TwitterHandle;
}
/// <summary>
/// Entry point for the crawler. Pass the URL from source file (XML)
/// </summary>
/// <param name="url"></param>
public List<Songs> Crawl(string url)
{
MovieEntity movie = new MovieEntity();
TableManager tableMgr = new TableManager();
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
#region Get Movie Page Content
Stream receiveStream = response.GetResponseStream();
StreamReader readStream = null;
if (response.CharacterSet == null)
readStream = new StreamReader(receiveStream);
else
readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
songPageContent = readStream.ReadToEnd();
response.Close();
readStream.Close();
#endregion
}
return PopulateSongDetails(songPageContent);
}
catch (Exception ex)
{
Debug.WriteLine("An error occurred while populating the movie details. Url = " + url + ". Error=" + ex.Message);
}
return null;
}
public string TestUpdate()
{
SetConnectionString();
MovieEntity entity = new MovieEntity();
var rand = new Random((int)DateTimeOffset.UtcNow.Ticks);
entity.RowKey = entity.MovieId = Guid.NewGuid().ToString();
entity.ReviewIds = string.Format("{0},{1}", Math.Abs(rand.Next()), Math.Abs(rand.Next()));
entity.AggregateRating = Math.Abs(rand.Next(10)).ToString();
entity.Directors = string.Format("Gabbar_{0}", rand.Next());
entity.HotOrNot = (Math.Abs(rand.Next())%2) == 1 ? true : false;
entity.MusicDirectors = string.Format("Rahman_{0}", Math.Abs(rand.Next()));
entity.Name = string.Format("aashique {0}", rand.Next());
entity.Producers = string.Format("sippy_{0}", rand.Next());
entity.Actors = string.Format("sahruuk_{0}", rand.Next());
var reviewIds = entity.GetReviewIds();
var reviewList = new List<ReviewEntity>();
foreach (var reviewId in reviewIds)
{
var reviewEntity = new ReviewEntity();
reviewEntity.ReviewId = reviewEntity.RowKey = reviewId;
reviewEntity.Review = string.Format("This is review number {0}", reviewId);
reviewEntity.ReviewerName = string.Format("khan_{0}", rand.Next());
reviewEntity.ReviewerRating = rand.Next(10);
reviewEntity.SystemRating = rand.Next(10);
reviewList.Add(reviewEntity);
}
var tableMgr = new TableManager();
tableMgr.UpdateMovieById(entity);
tableMgr.UpdateReviewsById(reviewList);
return string.Format("Created movie id {0}", entity.MovieId);
}
private bool PopulateMovieDetails(ref MovieEntity movie, string url, bool isCrawlPosters)
{
try
{
List<string> poster = new List<string>();
List<Cast> cast = CrawlCast(url + "fullcredits");
List<Songs> songs = CrawlSongs(url + "soundtrack");
if (isCrawlPosters)
{
poster = CrawlPosters(url + "mediaindex", movie.Name, ref thumbnailPath);
// Call Bollywood Hungama poster crawler here
//poster = CrawlBollywoodHungamaPosters(ref poster, url, movie.Name, ref thumbnailPath);
}
else
{
poster = JsonConvert.DeserializeObject<List<string>>(movie.Posters) as List<string>;
}
if (movie.Posters != null && (movie.Posters.Trim() == "[]" || movie.Posters.Trim() == ""))
poster = CrawlPosters(url + "mediaindex", movie.Name, ref thumbnailPath);
movie.Cast = JsonConvert.SerializeObject(cast);
if (isCrawlPosters)
movie.Songs = JsonConvert.SerializeObject(songs);
if (poster != null && poster.Count > 0)
{
movie.Posters = JsonConvert.SerializeObject(poster);
}
else
{
movie.Posters = "[]";
}
return true;
}
catch (Exception e)
{
Debug.WriteLine("An error occurred while populating other bulk movie details. Url = " + url + ". Error=" + e.Message);
return false;
}
}
public MovieEntity GetMovieEntity()
{
// TODO: Add error/edge-case handling as appropriate
MovieEntity movie = new MovieEntity();
movie.MovieId = this.MovieId;
movie.Name = this.Name;
movie.AltNames = this.AltNames;
movie.Posters = this.Posters;
movie.Rating = this.Ratings;
movie.Synopsis = this.Synopsis;
movie.Cast = this.Casts ?? string.Empty;
movie.Stats = this.Stats ?? string.Empty;
movie.Songs = this.Songs;
movie.Trailers = this.Trailers;
movie.Pictures = this.Pictures;
movie.Genre = this.Genre;
movie.Month = this.Month;
movie.Year = this.Year;
movie.UniqueName = this.UniqueName;
movie.State = this.State;
movie.MyScore = this.MyScore;
movie.JsonString = this.JsonString;
movie.Popularity = this.Popularity;
return movie;
}
private static void UpdateLuceneIndex(MovieEntity movie)
{
var tableMgr = new TableManager();
// Update Lucene
Task.Run(() =>
{
//delete Entry in lucene search index
// Fix following method call - What shall be other param?
LuceneSearch.ClearLuceneIndexRecord(movie.MovieId, "Id");
LuceneSearch.ClearLuceneIndexRecord(movie.UniqueName, "UniqueName");
string posterUrl = "default-movie.jpg";
string critics = string.Empty;
if (!string.IsNullOrEmpty(movie.Posters))
{
List<string> pList = jsonSerializer.Value.Deserialize(movie.Posters, typeof(List<string>)) as List<string>;
if (pList != null && pList.Count > 0)
{
posterUrl = pList.Last();
}
}
var reviewDic = tableMgr.GetReviewsByMovieId(movie.MovieId);
if (reviewDic != null && reviewDic.Values != null && reviewDic.Values.Count > 0)
{
critics = jsonSerializer.Value.Serialize(reviewDic.Values.Select(re => re.ReviewerName));
}
// add updated entry in lucene search index
MovieSearchData movieSearchIndex = new MovieSearchData();
movieSearchIndex.Id = movie.RowKey;
movieSearchIndex.Title = movie.Name;
movieSearchIndex.Type = movie.Genre;
movieSearchIndex.TitleImageURL = posterUrl;
movieSearchIndex.UniqueName = movie.UniqueName;
movieSearchIndex.Description = movie.Cast;
movieSearchIndex.Critics = critics;
movieSearchIndex.Link = movie.UniqueName;
LuceneSearch.AddUpdateLuceneIndex(movieSearchIndex);
});
}
// Parent Node/Body
public MovieEntity GetMovieDetails(HtmlNode body)
{
MovieEntity movie = new MovieEntity();
movie.RowKey = movie.MovieId = Guid.NewGuid().ToString();
movie.Name = GetMovieName(body);
movie.AltNames = GetMovieByAltName(body);
movie.UniqueName = GetMovieUniqueName(movie.Name);
movie.Rating = GetMovieRating(body);
movie.Synopsis = GetMovieStory(body);
movie.Genre = GetMovieGenre(body);
movie.Month = GetMovieMonth(body);
movie.Year = GetMovieYear(body);
movie.Stats = GetMovieStats(body);
movie.TwitterHandle = string.Empty;
movie.Trailers = string.Empty;
movie.Pictures = string.Empty;
movie.State = string.Empty;
movie.MyScore = Util.DEFAULT_SCORE;
movie.JsonString = "[]";
movie.Name = movie.Name.Replace(":", string.Empty);
movie.AltNames = movie.AltNames.Replace(":", string.Empty);
movie.UniqueName = movie.UniqueName.Replace(":", string.Empty);
return movie;
}
public string TestUpdate()
{
SetConnectionString();
MovieEntity entity = new MovieEntity();
var rand = new Random((int)DateTimeOffset.UtcNow.Ticks);
#region commented code
/*entity.RowKey = entity.MovieId = Guid.NewGuid().ToString();
entity.Stats = string.Format("{0},{1}", Math.Abs(rand.Next()), Math.Abs(rand.Next()));
entity.Songs = Math.Abs(rand.Next(10)).ToString();
entity.Ratings = string.Format("Gabbar_{0}", rand.Next());
entity.Trailers = string.Format("Traler_{0}", rand.Next());
entity.Casts = string.Format("Rahman_{0}", Math.Abs(rand.Next()));
entity.Name = string.Format("aashique {0}", rand.Next());
entity.Synopsis = string.Format("sippy_{0}", rand.Next());
entity.Posters = @"{""height"" : 300,""width"" : 200,""url"" : ""test""}";
entity.Month = "March";
entity.Year = "2014";*/
#endregion
entity.RowKey = entity.MovieId = Guid.NewGuid().ToString();
entity.Stats = @"[{""budget"" : ""30,000"",""boxoffice"": ""50000""}]";;
entity.Songs = @"[{""name"" : ""chaiyya chaiyya"",""url"" : ""songtest""}, {""name"" : ""chaiyya chaiyya"",""url"" : ""songtest""}]";
entity.Ratings = @"[{""system"" : 5,""critic"" : 6,""hot"" : ""no""}]";
entity.Trailers = @"[{""name"" : ""best movie"",""url"" : ""songtest""}, {""name"" : ""chaiyya chaiyya"",""url"" : ""songtest""}]";
entity.Casts = @"[{""name"" : ""ben affleck"",""charactername"" : ""mickey"",""image"" : {""height"" : 300,""width"": 200,""url"" : ""test""},""role"" : ""producer""},
{""name"" : ""jerry afflect"",""charactername"" : ""mouse"",""image"" : {""height"" : 300,""width"" : 200,""url"" : ""test""},""role"" : ""actor""}]";
entity.Pictures = @"[{""caption"" : ""test caption"",""image"" : {""height"" : 300,""width"" : 200,""url"" : ""test""}}]";
entity.Name = string.Format("aashique {0}", rand.Next());
entity.Synopsis = "this is a brilliant scary movie";
entity.Posters = @"[{""height"" : 300,""width"" : 200,""url"" : ""test""}]";
entity.Genre = "Action";
entity.Month = "March";
entity.Year = "2014";
//var reviewIds = entity.GetReviewIds();
var reviewIds = new List<string>() { Math.Abs(rand.Next()).ToString(), Math.Abs(rand.Next()).ToString() };
var reviewList = new List<ReviewEntity>();
foreach (var reviewId in reviewIds)
{
var reviewEntity = new ReviewEntity();
reviewEntity.ReviewId = reviewEntity.RowKey = reviewId;
reviewEntity.MovieId = entity.MovieId;
reviewEntity.Review = string.Format("This is review number {0}", reviewId);
reviewEntity.ReviewerName = string.Format("khan_{0}", rand.Next());
reviewEntity.ReviewerRating = rand.Next(10);
reviewEntity.SystemRating = rand.Next(10);
reviewEntity.Hot = false;
reviewEntity.OutLink = "this is a link";
reviewEntity.Affiliation= @"[{""name"" : ""Yahoo"", ""link"" : ""http://in.yahoo.com/?p=us"", ""reviewlink"" : ""http://in.movies.yahoo.com/blogs/movie-reviews/yahoo-movies-review-gunday-124034093.html"", ""logoimage"" : ""Images/Yahoo_Logo.png""},{""name"" : ""Hidustan Times"", ""link"" : ""http://www.hindustantimes.com"", ""reviewlink"" : ""http://www.hindustantimes.com/entertainment/reviews/movie-review-by-rashid-irani-dallas-buyers-club-is-a-must-watch/article1-1189520.aspx"", ""logoimage"" : ""Images/hindustan-times.jpg""}]";
reviewList.Add(reviewEntity);
}
var tableMgr = new TableManager();
tableMgr.UpdateMovieById(entity);
tableMgr.UpdateReviewsById(reviewList);
return string.Format("Created movie id {0}", entity.MovieId);
}
public static MovieEntity CreateMovieEntity(string name,
string posters,
string rating,
string synopsis,
string cast,
string stats,
string songs,
string trailers,
string pictures,
string genre,
string month,
string year,
string uniqueName,
string state,
string myScore,
string jsonString,
string popularity,
string twitterHandle
)
{
var movieId = Guid.NewGuid().ToString();
var entity = new MovieEntity(movieId);
entity.MovieId = movieId;
entity.Name = name;
entity.Posters = posters;
entity.Rating = rating;
entity.Synopsis = synopsis;
entity.Cast = cast;
entity.Stats = stats;
entity.Songs = songs;
entity.Trailers = trailers;
entity.Pictures = pictures;
entity.Genre = genre;
entity.Month = month;
entity.Year = year;
entity.UniqueName = uniqueName;
entity.State = state;
entity.MyScore = myScore;
entity.JsonString = jsonString;
entity.Popularity = popularity;
entity.TwitterHandle = twitterHandle;
return entity;
}
public static MovieEntity CreateMovieEntity(string name,
string posters,
string rating,
string synopsis,
string cast,
string stats,
string songs,
string trailers,
string pictures,
string genre,
string month,
string year)
{
var movieId = Guid.NewGuid().ToString();
var entity = new MovieEntity(movieId);
entity.MovieId = movieId;
entity.Name = name;
entity.Posters = posters;
entity.Ratings = rating;
entity.Synopsis = synopsis;
entity.Casts = cast;
entity.Stats = stats;
entity.Songs = songs;
entity.Trailers = trailers;
entity.Pictures = pictures;
entity.Genre = genre;
entity.Month = month;
entity.Year = year;
return entity;
}
public static MovieEntity CreateMovieEntity(string name,
string actors,
string directors,
string producers,
string musicDirecotrs,
string reviewIds,
string aggregateRating,
bool hotOrNot)
{
var movieId = Guid.NewGuid().ToString();
var entity = new MovieEntity(movieId);
entity.MovieId = movieId;
entity.Name = name;
entity.Actors = actors;
entity.Directors = directors;
entity.Producers = producers;
entity.MusicDirectors = musicDirecotrs;
entity.ReviewIds = reviewIds;
entity.AggregateRating = aggregateRating;
entity.HotOrNot = hotOrNot;
return entity;
}
