public void Test_Search_Parse_Last_Page()
{
var dir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
UrlResponsePair pair = new UrlResponsePair(@"/search/movie/love/results?page=3",
File.ReadAllText(dir + @"\TestData\search_love_3.txt"), 4, 15);
SearchRequestItem item = new SearchRequestItem("1", "love", "movie", "offset=11&limit=65");
item.Urls = new List <string>();
item.Urls.Add(@"/search/movie/love/results?page=0");
item.Urls.Add(@"/search/movie/love/results?page=1");
item.Urls.Add(@"/search/movie/love/results?page=2");
item.Urls.Add(@"/search/movie/love/results?page=3");
IMetacriticData resp = item.Parse(pair);
CollectionAssert.AllItemsAreInstancesOfType(((SearchData)resp).SearchItems,
typeof(SearchData.SearchItem));
Assert.IsNull(((SearchData)resp).TotalResultCount);
Assert.AreEqual(((SearchData)resp).SearchItems.Count, 15);
Assert.AreEqual(((SearchData)resp).SearchItems[2].Title, "A Lot Like Love");
Assert.AreEqual(((SearchData)resp).SearchItems[6].Rating.CriticRating, 33);
Assert.AreEqual(((SearchData)resp).SearchItems[8].ReleaseDate, "October 19, 2007");
Assert.AreEqual(((SearchData)resp).SearchItems[13].Id, "/movie/my-summer-of-love");
}
public void Test_Search_Parse_Middle_Page2()
{
var dir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
UrlResponsePair pair = new UrlResponsePair(@"/search/movie/love/results?page=2",
File.ReadAllText(dir + @"\TestData\search_love_2.txt"), 3, 20);
SearchRequestItem item = new SearchRequestItem("1", "love", "movie", "offset=11&limit=65");
item.Urls = new List <string>();
item.Urls.Add(@"/search/movie/love/results?page=0");
item.Urls.Add(@"/search/movie/love/results?page=1");
item.Urls.Add(@"/search/movie/love/results?page=2");
item.Urls.Add(@"/search/movie/love/results?page=3");
IMetacriticData resp = item.Parse(pair);
CollectionAssert.AllItemsAreInstancesOfType(((SearchData)resp).SearchItems,
typeof(SearchData.SearchItem));
Assert.IsNull(((SearchData)resp).TotalResultCount);
Assert.AreEqual(((SearchData)resp).SearchItems.Count, 20);
Assert.AreEqual(((SearchData)resp).SearchItems[1].Title, "To Rome with Love");
Assert.AreEqual(((SearchData)resp).SearchItems[2].Rating.CriticRating, 0);
Assert.AreEqual(((SearchData)resp).SearchItems[9].ReleaseDate, "August 9, 2013");
Assert.AreEqual(((SearchData)resp).SearchItems[17].Id, "/movie/love-sex");
}
public void Test_Search_Parse_First_Page()
{
var dir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
UrlResponsePair pair = new UrlResponsePair(@"/search/movie/love/results?page=0",
File.ReadAllText(dir + @"\TestData\search_love_0.txt"), 1, 10);
SearchRequestItem item = new SearchRequestItem("1", "love", "movie", "offset=11&limit=65");
item.Urls = new List <string>();
item.Urls.Add(@"/search/movie/love/results?page=0");
item.Urls.Add(@"/search/movie/love/results?page=1");
item.Urls.Add(@"/search/movie/love/results?page=2");
item.Urls.Add(@"/search/movie/love/results?page=3");
IMetacriticData resp = item.Parse(pair);
CollectionAssert.AllItemsAreInstancesOfType(((SearchData)resp).SearchItems,
typeof(SearchData.SearchItem));
Assert.AreEqual(((SearchData)resp).TotalResultCount, 238);
Assert.AreEqual(((SearchData)resp).SearchItems.Count, 10);
Assert.AreEqual(((SearchData)resp).SearchItems[0].Title, "After Love");
Assert.AreEqual(((SearchData)resp).SearchItems[5].Id, "/movie/love-rosie");
Assert.AreEqual(((SearchData)resp).SearchItems[8].ReleaseDate, "April 11, 2014");
Assert.AreEqual(((SearchData)resp).SearchItems[9].Rating.CriticRating, 39);
}
public void Test_Search_Parse_Album()
{
var dir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
UrlResponsePair pair = new UrlResponsePair(@"/search/album/futire/results?page=0",
File.ReadAllText(dir + @"\TestData\search_album_future.txt"), 1, 20);
SearchRequestItem item = new SearchRequestItem("1", "future", "album", "");
item.Urls = new List <string>();
item.Urls.Add(@"/search/album/future/results?page=0");
IMetacriticData resp = item.Parse(pair);
CollectionAssert.AllItemsAreInstancesOfType(((SearchData)resp).SearchItems,
typeof(SearchData.SearchItem));
Assert.AreEqual(((SearchData)resp).TotalResultCount, 58);
Assert.AreEqual(((SearchData)resp).SearchItems.Count, 20);
Assert.AreEqual(((SearchData)resp).SearchItems[2].Id, "/album/future-present-past-ep");
Assert.AreEqual(((SearchData)resp).SearchItems[6].Id, "/album/barbara-barbara-we-face-a-shining-future");
Assert.AreEqual(((SearchData)resp).SearchItems[8].Id, "/album/love-in-the-future");
Assert.AreEqual(((SearchData)resp).SearchItems[13].Id, "/album/in-the-future");
Assert.AreEqual(((SearchData)resp).SearchItems[17].Id, "/album/future-standards");
}
public override IMetacriticData Parse(UrlResponsePair urlResponsePair)
{
string html = urlResponsePair.Response;
if (String.IsNullOrEmpty(m_thirdLevelRequest))
{
TVShow tvShow = new TVShow();
tvShow.Title = ParseItem(ref html, @"<h1>", @"</h1>");
tvShow.Season = Int32.Parse(ParseItem(ref html, @"Season ", @"</h2>"));
tvShow.Studio = ParseItem(ref html, @"<span itemprop=""name"">", @"</span>");
short?criticRating = null;
short?criticRatingCount = null;
if (short.TryParse(ParseItem(ref html, @"<span itemprop=""ratingValue"">", @"</span>"), out short tempCriticRating))
{
criticRating = tempCriticRating;
criticRatingCount = Int16.Parse(ParseItem(ref html, @"<span itemprop=""reviewCount"">", @"</span>"));
}
float?userRating = null;
short?userRatingCount = null;
int userRatingIdx = html.IndexOf("metascore_w user large");
if (userRatingIdx != -1)
{
html = html.Substring(userRatingIdx);
if (float.TryParse(ParseItem(ref html, @""">", @"</div>"), out float tempUserRating))
{
userRating = tempUserRating;
userRatingCount = Int16.Parse(ParseItem(ref html, @"user-reviews"">", @" Ratings"));
}
}
tvShow.Rating = new Rating(criticRating, userRating, criticRatingCount, userRatingCount);
string releaseDateStr = ParseItem(ref html, @"<span class=""data"" itemprop=""startDate"">", @"</span>");
DateTime releaseDate;
if (DateTime.TryParse(releaseDateStr, out releaseDate))
{
tvShow.ReleaseDate = releaseDate.ToString("MM/dd/yyyy");
}
if (UrlImagePath != null)
{
string key = UrlImagePath.Keys.FirstOrDefault(k => urlResponsePair.Url.Contains(k));
if (key != null)
{
string imgPath;
if (UrlImagePath.TryGetValue(key, out imgPath))
{
tvShow.ImageUrl = imgPath;
}
}
}
return(tvShow);
}
else if (m_thirdLevelRequest == "details")
{
MediaDetail mediaDetails = new MediaDetail();
while (html.Contains(@"<th scope=""row"">"))
{
string desc = ParseItem(ref html, @"<th scope=""row"">", @":</th>");
string value = ParseItem(ref html, @"<td>", @"</td>");
if (value.Contains("</a>"))
{
value = ParseItem(ref value, @""">", @"</a>");
}
if (desc == "Seasons")
{
value = value.Replace(" ", String.Empty);
}
if (desc.Contains("Genre"))
{
Regex rgx = new Regex("\\s+");
value = rgx.Replace(value, " ");
}
DetailItem detail = new DetailItem(desc, value);
mediaDetails.Details.Add(detail);
}
while (html.Contains(@"<td class=""person"">"))
{
html = html.Substring(html.IndexOf(@"<td class=""person"">") +
@"<td class=""person"">".Length);
string name = ParseItem(ref html, @""">", @"</a>");
string role = ParseItem(ref html, @"<td class=""role"">", @"</td>");
MediaCredit credit = new MediaCredit(name, role);
mediaDetails.Credits.Add(credit);
}
return(mediaDetails);
}
return(null);
}
public override IMetacriticData Parse(UrlResponsePair urlResponsePair)
{
string html = urlResponsePair.Response;
if (String.IsNullOrEmpty(m_thirdLevelRequest))
{
Album album = new Album();
album.Title = ParseItem(ref html, @"<h1>", @"</h1>");
album.PrimaryArtist = ParseItem(ref html, @"<span class=""band_name"" itemprop=""name"">", @"</span>");
string releaseDateStr = ParseItem(ref html, @"<span class=""data"" itemprop=""datePublished"">", @"</span>");
DateTime releaseDate;
if (DateTime.TryParse(releaseDateStr, out releaseDate))
{
album.ReleaseDate = releaseDate.ToString("MM/dd/yyyy");
}
short?criticRating = null;
short?criticRatingCount = null;
if (short.TryParse(ParseItem(ref html, @"<span itemprop=""ratingValue"">", @"</span>"), out short tempCriticRating))
{
criticRating = tempCriticRating;
criticRatingCount = Int16.Parse(ParseItem(ref html, @"<span itemprop=""reviewCount"">", @"</span>"));
}
float?userRating = null;
short?userRatingCount = null;
int userRatingIdx = html.IndexOf("metascore_w user large");
if (userRatingIdx != -1)
{
html = html.Substring(userRatingIdx);
if (float.TryParse(ParseItem(ref html, @""">", @"</div>"), out float tempUserRating))
{
userRating = tempUserRating;
userRatingCount = Int16.Parse(ParseItem(ref html, @"user-reviews"">", @" Ratings"));
}
}
album.Rating = new Rating(criticRating, userRating, criticRatingCount, userRatingCount);
if (UrlImagePath != null)
{
string imgPath;
if (UrlImagePath.TryGetValue(urlResponsePair.Url, out imgPath))
{
album.ImageUrl = imgPath;
}
}
return(album);
}
else if (m_thirdLevelRequest == "details")
{
MediaDetail mediaDetail = new MediaDetail();
html = html.Substring(html.IndexOf(@"""new_details"""));
while (html.Contains(@"span class=""label"">"))
{
string desc = ParseItem(ref html, @"span class=""label"">", @":</span>");
string value = string.Empty;
int nextIdx;
int valueIdx;
do
{
if (string.IsNullOrEmpty(value))
{
value = ParseItem(ref html, @"span class=""data"">", @"</span>");
}
else
{
value = string.Format("{0}, {1}", value,
ParseItem(ref html, @"span class=""data"">", @"</span>"));
}
nextIdx = html.IndexOf(@"<span class=""label"">");
valueIdx = html.IndexOf(@"span class=""data"">");
} while (nextIdx != -1 && valueIdx < nextIdx);
DetailItem detail = new DetailItem(desc, value);
mediaDetail.Details.Add(detail);
}
return(mediaDetail);
}
return(null);
}
public override IMetacriticData Parse(UrlResponsePair urlResponsePair)
{
string html = urlResponsePair.Response;
string name = ParseItem(ref html, @"""og:title"" content=""", @""">");
Person person = new Person(name);
if (IsMediaTypeAvailable(html))
{
html = html.Substring(html.IndexOf(@"class=""credits person_credits"""));
List <Person.CreditMediaItemPair> pairs = new List <Person.CreditMediaItemPair>();
while (html.Contains(@"class=""title brief_metascore"""))
{
html = html.Substring(html.IndexOf(@"class=""title brief_metascore""") +
@"class=""title brief_metascore""".Length);
short?criticScore = null;
if (short.TryParse(ParseItem(ref html, @""">", "</span>"), out short tempCriticScore))
{
criticScore = tempCriticScore;
}
string id = ParseItem(ref html, @"href=""", @""">");
// Album url has different format
string[] parts = id.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries);
if (parts.Length > 0 && parts[0] == "music")
{
id = TrimAlbumUrl(id);
}
else if (parts.Length > 0 && parts[0] == "tv")
{
id = id.Replace("tv/", "tvshow/").Replace(@"/season-", @"/");
}
string title = ParseItem(ref html, @""">", "</a>");
DateTime releaseDate;
if (!DateTime.TryParse(ParseItem(ref html, @"class=""year"">", "</td>"),
out releaseDate))
{
releaseDate = DateTime.MinValue;
}
string credit = ParseItem(ref html, @"class=""role"">", "</td>");
html = html.Substring(html.IndexOf(@"class=""score"">") +
@"class=""score"">".Length);
float?userScore = null;
if (float.TryParse(ParseItem(ref html, @""">", "</span>"), out float tempUserScore))
{
userScore = tempUserScore;
}
Rating rating = new Rating(criticScore, userScore);
MediaItem item = new MediaItem()
{
Id = id,
Title = title,
ReleaseDate = releaseDate.ToString("MM/dd/yyyy"),
Rating = rating
};
pairs.Add(new Person.CreditMediaItemPair(credit, item));
}
person.CreditMediaPairItems = pairs;
}
return(person);
}
public abstract IMetacriticData Parse(UrlResponsePair urlResponsePair);
public override IMetacriticData Parse(UrlResponsePair urlResponsePair)
{
SearchData data = new SearchData();
string response = urlResponsePair.Response;
if (!string.IsNullOrEmpty(response))
{
int lowerBound = 0;
int upperBound = 20;
if (Urls.Count > 1)
{
// First in sequence
if (urlResponsePair.SequenceNo == 1)
{
lowerBound = 20 - urlResponsePair.SearchItemCount;
}
// Last in sequence
else if (urlResponsePair.SequenceNo == Urls.Count)
{
upperBound = urlResponsePair.SearchItemCount;
}
}
else
{
string offsetStr = m_parameterData.GetParameterValue("offset");
int.TryParse(offsetStr, out int offset);
int pageAdjustedOffset = 0;
if (offset > 0)
{
pageAdjustedOffset = (offset % 20) - 1;
}
lowerBound = pageAdjustedOffset;
upperBound = pageAdjustedOffset + urlResponsePair.SearchItemCount;
}
if (urlResponsePair.SequenceNo == 1)
{
int idx = response.IndexOf(@"class=""query_results""");
if (idx >= 0)
{
response = response.Substring(idx);
if (response.Contains(@"Showing 1 to 20 of "))
{
data.TotalResultCount = int.Parse(ParseItem(ref response, @" of ", @" results"));
}
}
// No result
else
{
return(data);
}
}
int startIdx = response.IndexOf(@"class=""search_results");
if (startIdx != -1)
{
response = response.Substring(startIdx);
int idx = 1;
while (response.Contains(@"class=""result_wrap""") && idx <= upperBound)
{
SearchData.SearchItem item = new SearchData.SearchItem();
string id = ParseItem(ref response, @"href=""", @""">");
// Album url has different format
string[] parts = id.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries);
if (parts.Length > 0 && parts[0] == "music")
{
id = TrimAlbumUrl(id);
}
else if (parts.Length > 0 && parts[0] == "tv")
{
id = id.Replace("tv/", "tvshow/");
}
item.Id = id;
item.Title = ParseItem(ref response, @""">", "</a>");
string criticScoreStr = ParseItem(ref response, @""">", "</span>");
short.TryParse(criticScoreStr, out short criticScore);
item.Rating = new Rating(criticScore);
item.ReleaseDate = ParseItem(ref response, @"<span class=""data"">", "</span>");
if (response.Contains(@"class=""stat genre"""))
{
response = response.Substring(response.IndexOf(@"class=""stat genre"""));
string genre = ParseItem(ref response, @"<span class=""data"">", "</span>");
Regex rgx = new Regex("\\s+");
item.Genre = rgx.Replace(genre, " ");
}
if (idx > lowerBound)
{
data.AddItem(item);
}
idx++;
}
}
}
return(data);
}
public override IMetacriticData Parse(UrlResponsePair urlResponsePair)
{
string html = urlResponsePair.Response;
if (String.IsNullOrEmpty(m_thirdLevelRequest))
{
Movie movie = new Movie();
int startIndex = html.IndexOf(@"<script type=""application/ld+json"">");
if (startIndex != -1)
{
string infoString = html.Substring(startIndex);
int endIndex = infoString.IndexOf(@"</script>");
if (endIndex != -1)
{
infoString = infoString.Substring(0, endIndex);
movie.Title = ParseItem(ref infoString, @"""name"" : """, @"""");
string releaseDateStr = ParseItem(ref infoString, @"""datePublished"" : """, @"""");
DateTime releaseDate;
if (DateTime.TryParse(releaseDateStr, out releaseDate))
{
movie.ReleaseDate = releaseDate.ToString("MM/dd/yyyy");;
}
short?criticRating = null;
short?criticRatingCount = null;
if (infoString.Contains(@"""ratingValue"" : """))
{
if (short.TryParse(ParseItem(ref infoString, @"""ratingValue"" : """, @""""), out short tempCriticRating))
{
criticRating = tempCriticRating;
criticRatingCount = Int16.Parse(ParseItem(ref infoString, @"""ratingCount"" : """, @""""));
}
// Critic
html = html.Substring(html.IndexOf("Critics</span>"));
}
// User
float?userRating = null;
short?userRatingCount = null;
if (html.Contains(@">based on "))
{
html = html.Substring(html.IndexOf(">based on "));
if (short.TryParse(ParseItem(ref html, @">based on ", " Ratings"), out short tempUserRatingCount))
{
userRatingCount = tempUserRatingCount;
html = html.Substring(html.IndexOf("metascore_w user"));
userRating = float.Parse(ParseItem(ref html, @">", @"</span>"));
}
}
movie.Rating = new Rating(criticRating, userRating, criticRatingCount, userRatingCount);
infoString = infoString.Substring(infoString.IndexOf(@"""director"""));
movie.Director = ParseItem(ref infoString, @"""name"": """, @"""");
}
}
if (UrlImagePath != null)
{
string imgPath;
if (UrlImagePath.TryGetValue(urlResponsePair.Url, out imgPath))
{
movie.ImageUrl = imgPath;
}
}
return(movie);
}
else if (m_thirdLevelRequest == "details")
{
MediaDetail mediaDetail = new MediaDetail();
while (html.Contains(@"<td class=""label"">"))
{
string desc = ParseItem(ref html, @"<td class=""label"">", @":</td>");
string value = ParseItem(ref html, @"<td class=""data"">", @"</td>");
if (value.Contains("</a>"))
{
value = ParseItem(ref value, @""">", @"</a>");
}
if (value.Contains("<span>"))
{
value = value.Replace("<span>", "").Replace("</span>", "");
}
if (desc == "Genres" || desc == "Languages" || desc == "Countries")
{
Regex rgx = new Regex("\\s+");
value = rgx.Replace(value, " ");
}
DetailItem detail = new DetailItem(desc, value);
mediaDetail.Details.Add(detail);
}
while (html.Contains(@"<td class=""person"">"))
{
string name = ParseItem(ref html, @"<td class=""person"">", @"</td>");
if (name.Contains("</a>"))
{
name = ParseItem(ref name, @""">", @"</a>");
}
string role = ParseItem(ref html, @"<td class=""role"">", @"</td>");
MediaCredit credit = new MediaCredit(name, role);
mediaDetail.Credits.Add(credit);
}
return(mediaDetail);
}
return(null);
}
