//----==== PRIVATE ====-------------------------------------------------------------------- private void AddMovie(string nodeText, DateTime?articleDate, List <IMovie> result) { int index = nodeText.IndexOf(DELIMITER); var movieName = nodeText.Substring(0, index); // Might switch this to RegEx... var valueInMillions = (nodeText.Substring(index, nodeText.Length - index)?.Contains("million") ?? false) || (nodeText.Substring(index, nodeText.Length - index)?.Contains("milllion") ?? false); var estimatedBoxOffice = nodeText.Substring(index, nodeText.Length - index)?.Replace(DELIMITER, string.Empty).Replace("million", string.Empty).Replace("milllion", string.Empty); var parenIndex = estimatedBoxOffice.IndexOf("("); if (parenIndex > 0) { // Trim out the FML bux. estimatedBoxOffice = estimatedBoxOffice.Substring(0, parenIndex - 1); } if (!string.IsNullOrEmpty(movieName)) { var name = RemovePunctuation(HttpUtility.HtmlDecode(movieName)); var dayOfWeek = ParseDayOfWeek(name); var movie = new Movie { MovieName = MapName(ParseName(name, dayOfWeek)), Day = ParseDayOfWeek(name), Earnings = decimal.Parse(estimatedBoxOffice) * (valueInMillions ? 1000000 : 1) }; if (articleDate.HasValue) { movie.WeekendEnding = MovieDateUtil.NextSunday(articleDate); } result.Add(movie); } }
public override List <IMovie> Mine() { var result = new List <IMovie>(); var web = new HtmlWeb(); var doc = web.Load($"{Url}/?s=weekend+box+office+predictions"); // Lookup XPATH to get the right node that matches. // Select all of the <script> nodes that are children of <body> with an attribute of "src" // REF: https://www.w3schools.com/xml/xpath_syntax.asp var node = doc.DocumentNode.SelectSingleNode("//body//a[contains(@href, 'weekend-box-office-predictions-')]"); Error = string.Empty; if (node != null) { var href = node.GetAttributeValue("href", null); if (href != null) { DateTime?articleDate = null; UrlSource = href; // Now retrieve the article page. doc = web.Load(UrlSource); // Get the date of the article node = doc.DocumentNode.SelectSingleNode("//body//time[@class='post-published updated']"); if (node != null) { string articleText = node.InnerText.Replace("On ", string.Empty); DateTime parsedDateTime; if (DateTime.TryParse(articleText, out parsedDateTime)) { articleDate = parsedDateTime; } } // Get the data in the article. // Rank might be important because the movies below the main ones are ranked as well var tableRows = doc.DocumentNode.SelectNodes("//body//article//h2[@style='text-align: center;'] | //body//article//p[contains(.,'Prediction:')]"); if (tableRows != null) { string currentName = null; string currentValue = null; Movie currentMovie = null; foreach (var row in tableRows) { if (row.Name.ToLower() == "h2") { currentName = HttpUtility.HtmlDecode(row.InnerText); } else if (row.InnerText.StartsWith("Prediction:")) { currentValue = row.InnerText.Replace("Prediction:", string.Empty); } if (currentName != null && currentValue != null) { decimal earnings; var index = currentValue.IndexOf("million"); if (index > 0) { currentValue = currentValue.Substring(0, index).Replace("$", string.Empty).Trim(); } if (decimal.TryParse(currentValue, out earnings)) { currentName = RemovePunctuation(RemoveStudio(currentName)); currentMovie = new Movie { Name = MapName(currentName), Earnings = earnings * 1000000 }; if (articleDate.HasValue) { currentMovie.WeekendEnding = MovieDateUtil.NextSunday(articleDate.Value); } } currentName = null; currentValue = null; } if (currentMovie != null) { result.Add(currentMovie); currentMovie = null; } } } // TODO: Exclude movies that are already in the list. (or seek out movies from a list that was passed in) tableRows = doc.DocumentNode.SelectNodes("//body//article//h2[@style='text-align: center;'] | //body//article//p[contains(.,'million')]"); if (tableRows != null) { bool foundHoldovers = false; foreach (var row in tableRows) { if (foundHoldovers) { var index = row.InnerText.IndexOf("million", row.InnerText.IndexOf("million") + 1); // Only want the row (paragraph) with "million" in it multiple times. if (index > 0) { // Match one or more digits, followed by a period and a space. // Gobble up (non-greedy using the ?) to 'cume' (meaning cumulative) var matches = Regex.Matches(row.InnerText, @"\d+\.\s.*?cume"); foreach (Match match in matches) { var movie = new Movie(); // \s - Match up to white space var titleMatch = Regex.Match(match.Value, @"\s(.*?)\$"); var earningsMatch = Regex.Match(match.Value, @"\$\d+.*?\("); var fourDayMatch = Regex.Match(match.Value, @",\s\$\d+\.*\d*.*day"); if (!string.IsNullOrEmpty(fourDayMatch.Value)) { Error = FOUR_DAY; movie.Earnings = ParseEarnings(fourDayMatch.Value.Replace(",", string.Empty).Replace("(4-day", string.Empty)); } else if (!string.IsNullOrEmpty(earningsMatch.Value)) { movie.Earnings = ParseEarnings(earningsMatch.Value.Replace("(", string.Empty)); } if (!string.IsNullOrEmpty(titleMatch.Value)) { movie.Name = titleMatch.Value.Trim().Replace(" $", string.Empty); index = movie.Name.LastIndexOf("("); if (index > 0) { movie.Name = RemovePunctuation(HttpUtility.HtmlDecode(movie.Name.Substring(0, index))); } } if (movie.Name != null && movie.Earnings > 0) { if (articleDate.HasValue) { movie.WeekendEnding = MovieDateUtil.NextSunday(articleDate.Value); } result.Add(movie); } } break; } } if (row.Name.ToLower() == "h2" && row.InnerText.ToLower().Contains("holdover")) { foundHoldovers = true; } } } } } return(result); }
private List <IMovie> MineData(int articleNumber = 1) { var result = new List <IMovie>(); var web = new HtmlWeb(); var doc = web.Load($"{Url}category/tracking-forecasts"); // Can't instanciate an Active-X control within a web application. //var doc = web.LoadFromBrowser(Url); // Lookup XPATH to get the right node that matches. // Select all of the <script> nodes that are children of <body> with an attribute of "src" // REF: https://www.w3schools.com/xml/xpath_syntax.asp //var node = doc.DocumentNode.SelectSingleNode("//body//a[contains(@href, 'estimates-weekend')]"); //var node = doc.DocumentNode.SelectSingleNode("//body//a[contains(@href, 'weekend-forecast')]"); HtmlNode node = null; const string XPATH = "//body//a[contains(@class, 'single-item__link')]"; if (articleNumber == 1) { node = doc.DocumentNode.SelectSingleNode(XPATH); } else { var nodes = doc.DocumentNode.SelectNodes(XPATH); if (nodes != null && articleNumber <= nodes.Count) { node = nodes[articleNumber - 1]; } } if (node != null) { var href = node.GetAttributeValue("href", null); if (href != null) { DateTime?articleDate = null; UrlSource = href; // Now retrieve the article page. doc = web.Load(UrlSource); // Get the date of the article node = doc.DocumentNode.SelectSingleNode("//body//div[@class='entry-meta']/span[@class='entry-meta__post-date']"); if (node != null) { string articleText = node.InnerText.Replace(" • ", string.Empty); DateTime parsedDateTime; if (DateTime.TryParse(articleText, out parsedDateTime)) { articleDate = parsedDateTime; } } // Get the data in the table. //node = doc.DocumentNode.SelectSingleNode("//body//div[@class='post-container']/table"); //node = doc.DocumentNode.SelectSingleNode("//body//table[@class='wp-block-table aligncenter']"); //node = doc.DocumentNode.SelectSingleNode("//body//table[@width='716']"); node = doc.DocumentNode.SelectSingleNode("//body//table"); // TODO: Parse the header for column titles for mapping. var tableRows = node?.SelectNodes("tbody/tr[position()>1]"); // Skips the table header row (row 0) if (tableRows != null) { foreach (var row in tableRows) { Movie movie = null; var rowColumns = row.SelectNodes("td"); if (rowColumns != null) { int columnCount = 0; foreach (var column in rowColumns) { if (columnCount == 0) { movie = new Movie { Name = MapName(RemovePunctuation(HttpUtility.HtmlDecode(column.InnerText))) }; if (articleDate.HasValue) { movie.WeekendEnding = MovieDateUtil.NextSunday(articleDate); } } else if (columnCount == 2) { //movie.Earnings = decimal.Parse(column.InnerText?.Replace("$", string.Empty)); var rawText = RemovePunctuation(HttpUtility.HtmlDecode(column.InnerText)); if (rawText.Contains("4day")) { var tokens = rawText.Split(); if (tokens.Length > 2) { rawText = tokens[tokens.Length - 2]; } } var idx = rawText.IndexOf('\n'); if (idx > 0) { rawText = rawText.Substring(0, idx); } movie.Earnings = ParseEarnings(rawText); } columnCount++; } } if (movie != null) { result.Add(movie); } } } } } if (!result.Any() || result.Sum(movie => movie.EarningsBase) <= 0) { Error = NO_DATA; } return(result); }
public override List <IMovie> Mine() { var result = new List <IMovie>(); var web = new HtmlWeb(); var doc = web.Load($"{Url}/chatter/searchmessages?boardId=fml-main-chatter&query=bonus%20bar"); // Lookup XPATH to get the right node that matches. // REF: https://www.w3schools.com/xml/xpath_syntax.asp var node = doc.DocumentNode.SelectSingleNode($"//body//div/h3[@class='topic-item__title']"); if (node != null) { // Traverse up to the <div> node = node.ParentNode; } if (node != null) { var href = node.GetAttributeValue("data-href", null); if (href != null) { DateTime?articleDate = null; // Now retrieve the article page. UrlSource = $"{Url}/{href}"; doc = web.Load(UrlSource); // Get the date of the article node = doc.DocumentNode.SelectSingleNode("//body//span[@class='topic-item__attribution']/span[@class='time-date']"); if (node != null) { // Remove the first child span. if (node.HasChildNodes) { var articleText = HttpUtility.HtmlDecode(node.FirstChild.InnerText.Replace(",", string.Empty)).Trim(); // Remove the text after the year. var year = DateTime.Now.Year; var index = articleText.IndexOf(year.ToString()); if (index > 0) { articleText = articleText.Substring(0, index + year.ToString().Length); } DateTime parsedDateTime; if (DateTime.TryParse(articleText, out parsedDateTime)) { articleDate = parsedDateTime; } } } // Get the data node = doc.DocumentNode.SelectSingleNode("//body//div[@class='topic-item__body']"); if (node != null) { var movieNodes = node.SelectNodes($"//p/strong"); // Find all of the estimate paragraphs (are in BOLD) foreach (var movieNode in movieNodes) { var nodeText = HttpUtility.HtmlDecode(movieNode.InnerHtml); var index = nodeText.IndexOf(DELIMITER); var index2 = nodeText.IndexOf(DELIMITER, index + 1); if (index > 0 && index2 > 0) { var movieName = nodeText.Substring(0, index); // Might switch this to RegEx... var estimatedBoxOffice = nodeText.Substring(index + 1, index2 - (index + 1))?.Replace("$", string.Empty); var trimIndex = estimatedBoxOffice.IndexOf("("); if (trimIndex > 0) { // Trim out the drop percentage (and everything after). estimatedBoxOffice = estimatedBoxOffice.Substring(0, trimIndex - 1); } if (!string.IsNullOrEmpty(movieName)) { var name = RemovePunctuation(HttpUtility.HtmlDecode(movieName)); Movie movie = null; try { movie = new Movie { MovieName = MapName(ParseName(name)), Day = ParseDayOfWeek(name), Earnings = ParseEarnings(estimatedBoxOffice) }; if (movie.Day.HasValue) { CompoundLoaded = true; } } catch (Exception exception) { Error = "Some bad data"; ErrorDetail = $"The movie did not parse correctly \"{name}\" - {exception.Message}"; movie = null; } if (movie != null) { if (!result.Contains(movie)) { if (articleDate.HasValue) { movie.WeekendEnding = MovieDateUtil.NextSunday(articleDate); } result.Add(movie); } else if (movie.Day.HasValue) { if (articleDate.HasValue) { movie.WeekendEnding = MovieDateUtil.NextSunday(articleDate); } result.Add(movie); // Remove the movie that does NOT have a day. var toRemove = result.FirstOrDefault(item => item.Equals(movie) && !item.Day.HasValue); if (toRemove != null) { result.Remove(toRemove); } } } } } } } } } return(result); }
public override List <IMovie> Mine() { var result = new List <IMovie>(); var web = new HtmlWeb(); var doc = web.Load($"{Url}/news"); // Lookup XPATH to get the right node that matches. // REF: https://www.w3schools.com/xml/xpath_syntax.asp // Select the first <a> node that contains the title attribute. var node = doc.DocumentNode.SelectSingleNode($"//body//a[contains(@title, '{_articleTitle}')]"); if (node == null) { node = doc.DocumentNode.SelectSingleNode("//body//a[contains(@title, 'Box Office Estimates') and not(contains(@title, 'Perri'))]|//body//a[contains(@title, 'Box Office Predictions') and not(contains(@title, 'Perri'))]"); } if (node != null) { var href = node.GetAttributeValue("href", null); if (href != null) { DateTime?articleDate = null; // Now retrieve the article page. UrlSource = $"{Url}/{href}"; doc = web.Load(UrlSource); // Get the date of the article //node = doc.DocumentNode.SelectSingleNode("//body//div[@class='credits']/span[@class='date']"); node = doc.DocumentNode.SelectSingleNode("//body//div[@class='post__credits']/div[@class='post__date-time']/div"); if (node != null) { // Remove the first child span. if (node.HasChildNodes) { string articleText = HttpUtility.HtmlDecode(node.FirstChild.InnerText).Trim(); DateTime parsedDateTime; if (DateTime.TryParse(articleText, out parsedDateTime)) { articleDate = parsedDateTime; } } } // Get the data node = doc.DocumentNode.SelectSingleNode("//body//div[@class='post__content']"); if (node != null) { var movieNodes = node.SelectNodes($"//p[contains(., '{DELIMITER}')]|//p[contains(., '{DELIMITER2}')]"); // Find all of the estimate paragraphs // As of 11/2/2017 Todd is separating things with <br /> now. // As of 04/17/2019 Todd is putting the estimates all inside a single paragraph if (movieNodes.Count <= 5) { var innerHtml = HttpUtility.HtmlDecode(movieNodes.Last().InnerHtml); var delimiters = new string[] { "\n", "<br>", "<br>\n", "<br><br>" }; var tokens = innerHtml.Split(delimiters, StringSplitOptions.RemoveEmptyEntries); foreach (var token in tokens) { if (token != null && token.StartsWith("\"")) // && token.EndsWith("million")) { AddMovie(token.Replace("<br>", string.Empty), articleDate, result); } } } else { foreach (var movieNode in movieNodes) { int index = movieNode.InnerText.IndexOf(DELIMITER); if (index < 0) { index = movieNode.InnerText.IndexOf(DELIMITER2); } if (index > 0) { var nodeText = movieNode.InnerText; var movieName = nodeText.Substring(0, index); // Might switch this to RegEx... var multiplier = Multiplier(nodeText.Substring(index, nodeText.Length - index)); var estimatedBoxOffice = nodeText.Substring(index, nodeText.Length - index)?.Replace(DELIMITER, string.Empty).Replace(DELIMITER2, string.Empty).Replace("million", string.Empty).Replace("k", string.Empty); var parenIndex = estimatedBoxOffice.IndexOf("("); if (parenIndex > 0) { // Trim out the FML bux. estimatedBoxOffice = estimatedBoxOffice.Substring(0, parenIndex - 1); } if (!string.IsNullOrEmpty(movieName)) { var name = RemovePunctuation(HttpUtility.HtmlDecode(movieName)); Movie movie = null; try { var dayOfWeek = ParseDayOfWeek(name); movie = new Movie { MovieName = MapName(ParseName(name, dayOfWeek)), Day = dayOfWeek, Earnings = decimal.Parse(estimatedBoxOffice) * multiplier }; if (movie.Day.HasValue) { CompoundLoaded = true; } } catch (Exception exception) { Error = "Some bad data"; ErrorDetail = $"The movie did not parse correctly \"{name}\" - {exception.Message}"; movie = null; } if (movie != null) { if (articleDate.HasValue) { movie.WeekendEnding = MovieDateUtil.NextSunday(articleDate); } result.Add(movie); } } } } } } } } return(result); }
public override List <IMovie> Mine() { var result = new List <IMovie>(); var web = new HtmlWeb(); var doc = web.Load(Url); // Lookup XPATH to get the right node that matches. // Select all of the <script> nodes that are children of <body> with an attribute of "src" // REF: https://www.w3schools.com/xml/xpath_syntax.asp //var node = doc.DocumentNode.SelectSingleNode("//body//a[contains(@href, 'http://www.boxofficeprophets.com/column') and //strong='Weekend Forecast']"); HtmlNode node = null; var nodes = doc.DocumentNode.SelectNodes("//body//a[contains(@href, 'http://www.boxofficeprophets.com/column')]"); if (nodes != null) { foreach (var aNode in nodes) { if (aNode.InnerText.Contains("Weekend Forecast")) { node = aNode; break; } } } if (node != null) { var href = node.GetAttributeValue("href", null); if (href != null) { DateTime?articleDate = null; UrlSource = href; // Now retrieve the article page. doc = web.Load(UrlSource); // Get the date of the article node = doc.DocumentNode.SelectSingleNode("//body//div[@id='EchoTopic']//h4"); if (node != null) { string articleText = node.InnerText; DateTime parsedDateTime; if (DateTime.TryParse(articleText, out parsedDateTime)) { articleDate = parsedDateTime; } } // Get the data in the table. node = doc.DocumentNode.SelectSingleNode("//body//div[@id='EchoTopic']//table[@width='100%']"); // TODO: Parse the header for column titles for mapping. //var tableRows = node?.SelectNodes("//tr[position()>2]"); var tableRows = node?.SelectNodes("//tr[@bgcolor='eeeeee']"); if (tableRows == null) { // Try page 2 UrlSource += "&columnpage=2"; doc = web.Load(UrlSource); // Get the data in the table. node = doc.DocumentNode.SelectSingleNode("//body//div[@id='EchoTopic']//table[@width='100%']"); tableRows = node?.SelectNodes("//tr[@bgcolor='eeeeee']"); } if (tableRows != null) { foreach (var row in tableRows) { Movie movie = null; var rowColumns = row.SelectNodes("td"); if (rowColumns != null) { int columnCount = 0; foreach (var column in rowColumns) { if (columnCount == 1) { var movieName = MapName(RemovePunctuation(HttpUtility.HtmlDecode(column.InnerText))); // Don't create the movie is the name is hosed up. if (!string.IsNullOrEmpty(movieName)) { movie = new Movie { Name = movieName }; if (articleDate.HasValue) { movie.WeekendEnding = MovieDateUtil.NextSunday(articleDate); } } } else if (columnCount == 4 && movie != null) { movie.Earnings = decimal.Parse(column.InnerText) * 1000000; } columnCount++; } } if (movie != null) { result.Add(movie); } } } } } return(result); }
public override List <IMovie> Mine() { var result = new List <IMovie>(); var web = new HtmlWeb(); var doc = web.Load($"{Url}/chatter/searchmessages?boardId=fml-main-chatter&query=coupe"); // Lookup XPATH to get the right node that matches. // REF: https://www.w3schools.com/xml/xpath_syntax.asp var node = doc.DocumentNode.SelectSingleNode($"//body//div/h3[@class='topic-item__title']"); if (node != null) { // Traverse up to the <div> node = node.ParentNode; } if (node != null) { var href = node.GetAttributeValue("data-href", null); if (href != null) { DateTime?articleDate = null; // Now retrieve the article page. UrlSource = $"{Url}/{href}"; doc = web.Load(UrlSource); // Get the date of the article node = doc.DocumentNode.SelectSingleNode("//body//span[@class='topic-item__attribution']/span[@class='time-date']"); if (node != null) { // Remove the first child span. if (node.HasChildNodes) { var articleText = HttpUtility.HtmlDecode(node.FirstChild.InnerText.Replace(",", string.Empty)).Trim(); // Remove the text after the year. var year = DateTime.Now.Year; var index = articleText.IndexOf(year.ToString()); if (index > 0) { articleText = articleText.Substring(0, index + year.ToString().Length); } DateTime parsedDateTime; if (DateTime.TryParse(articleText, out parsedDateTime)) { articleDate = parsedDateTime; } } } // Get the data node = doc.DocumentNode.SelectSingleNode("//body//div[@class='topic-item__body']"); if (node != null) { var movieNodes = node.SelectNodes($"//p[contains(., '{DELIMITER}')]|//p[contains(., '{DELIMITER2}')]|//li[contains(., '{DELIMITER}')]"); // Find all of the estimate paragraphs // As of 11/2/2017 Todd is separating things with <br /> now. if (movieNodes.Count == 1) { var innerHtml = HttpUtility.HtmlDecode(movieNodes.First().InnerHtml); var delimiters = new string[] { "\n", "<br>\n", "<br><br>" }; var tokens = innerHtml.Split(delimiters, StringSplitOptions.RemoveEmptyEntries); foreach (var token in tokens) { if (token.StartsWith("\"")) // && token.EndsWith("million")) { AddMovie(token.Replace("<br>", string.Empty), articleDate, result); } } } else { foreach (var movieNode in movieNodes) { var nodeText = HttpUtility.HtmlDecode(movieNode.InnerHtml); int index = nodeText.IndexOf(DELIMITER); if (index < 0) { index = nodeText.IndexOf(DELIMITER2); } if (index > 0) { var movieName = nodeText.Substring(0, index).Replace("<br>", string.Empty); // Might switch this to RegEx... //var multiplier = Multiplier(nodeText.Substring(index, nodeText.Length - index)); var estimatedBoxOffice = nodeText.Substring(index, nodeText.Length - index)?.Replace(DELIMITER, string.Empty).Replace(DELIMITER2, string.Empty).Replace("million", string.Empty); var trimIndex = estimatedBoxOffice.IndexOf("("); if (trimIndex > 0) { // Trim out the drop percentage (and everything after). estimatedBoxOffice = estimatedBoxOffice.Substring(0, trimIndex - 1); } trimIndex = estimatedBoxOffice.IndexOf("<br>"); if (trimIndex > 0) { // Trim out the HTML break (and everything after). estimatedBoxOffice = estimatedBoxOffice.Substring(0, trimIndex); } trimIndex = estimatedBoxOffice.IndexOf("|"); if (trimIndex > 0) { // Trim out the COUPE label (and everything after). estimatedBoxOffice = estimatedBoxOffice.Substring(0, trimIndex - 1); } if (!string.IsNullOrEmpty(movieName)) { var name = RemovePunctuation(HttpUtility.HtmlDecode(movieName)); Movie movie = null; try { movie = new Movie { MovieName = MapName(ParseName(name)), Day = ParseDayOfWeek(name), Earnings = ParseEarnings(estimatedBoxOffice) }; if (movie.Day.HasValue) { CompoundLoaded = true; } } catch (Exception exception) { Error = "Some bad data"; ErrorDetail = $"The movie did not parse correctly \"{name}\" - {exception.Message}"; movie = null; } if (movie != null) { if (!result.Contains(movie)) { if (articleDate.HasValue) { movie.WeekendEnding = MovieDateUtil.NextSunday(articleDate); } result.Add(movie); } else if (movie.Day.HasValue) { if (articleDate.HasValue) { movie.WeekendEnding = MovieDateUtil.NextSunday(articleDate); } result.Add(movie); // Remove the movie that does NOT have a day. var toRemove = result.FirstOrDefault(item => item.Equals(movie) && !item.Day.HasValue); if (toRemove != null) { result.Remove(toRemove); } } } } } } } } } } return(result); }
private List <IMovie> MineForecast(int articleNumber = 1) { var result = new List <IMovie>(); string url = Url + "news/"; var web = new HtmlWeb(); var doc = web.Load(url); // Load main page. // Lookup XPATH to get the right node that matches. // Select all of the <script> nodes that are children of <body> with an attribute of "src" // REF: https://www.w3schools.com/xml/xpath_syntax.asp HtmlNode node = null; if (articleNumber == 1) { //node = doc.DocumentNode.SelectSingleNode("//body//a[contains(@href, '/news/?id=')]"); node = doc.DocumentNode.SelectSingleNode("//body//a[contains(@href, '/article/')]"); } else { //var nodes = doc.DocumentNode.SelectNodes("//body//a[contains(@href, '/news/?id=')]"); var nodes = doc.DocumentNode.SelectNodes("//body//a[contains(@href, '/article/')]"); if (nodes != null && articleNumber <= nodes.Count) { node = nodes[articleNumber - 1]; } } if (node != null) { var href = node.GetAttributeValue("href", null); if (href != null) { DateTime?articleDate = null; // Now retrieve the article page. UrlSource = $"{Url}/{href}"; doc = web.Load(UrlSource); // Get the date of the article (hoping that the date is the ONLY thing in such a small font) //node = doc.DocumentNode.SelectSingleNode("//body//font[@size='1']"); node = doc.DocumentNode.SelectSingleNode("//body//div[@class='mojo-news-byline']"); if (node != null) { // Remove the first child span. if (node.ChildNodes.Count > 1) { string articleText = HttpUtility.HtmlDecode(node.ChildNodes[1].InnerText).Trim(); var tokens = articleText.Split(new char[] { '-' }); DateTime parsedDateTime; if (tokens.Length > 0 && DateTime.TryParse(tokens[0].Replace("PDT", string.Empty).Replace("PST", string.Empty), out parsedDateTime)) { articleDate = parsedDateTime.Date; } } } // Need to scan for the <p> tag that contains "This weekend's forecast is directly below." // The movies are just in a <ul> tag (unsorted list) var movieNodes = doc.DocumentNode?.SelectNodes("//body//ul/li/span[@class='a-list-item']"); if (movieNodes == null) { Error = NO_DATA; } else { foreach (var movieNode in movieNodes) { int index = movieNode.InnerText.IndexOf(DELIMITER); if (index > 0) { var nodeText = movieNode.InnerText; var movieName = nodeText.Substring(0, index); // Might switch this to RegEx... var valueInMillions = nodeText.Substring(index, nodeText.Length - index)?.Contains("M"); var estimatedBoxOffice = nodeText.Substring(index, nodeText.Length - index)?.Replace(DELIMITER, string.Empty).Replace("M", string.Empty); var parenIndex = movieName.IndexOf("("); if (parenIndex > 0) { // Trim out the THEATERS (for now). movieName = movieName.Substring(0, parenIndex - 1).Trim(); } parenIndex = estimatedBoxOffice.IndexOf("("); if (parenIndex > 0) { // Trim out the multi-day value. estimatedBoxOffice = estimatedBoxOffice.Substring(0, parenIndex - 1).Trim(); } decimal estBoxOffice; if (!string.IsNullOrEmpty(movieName) && decimal.TryParse(estimatedBoxOffice, out estBoxOffice)) { var name = MapName(RemovePunctuation(HttpUtility.HtmlDecode(movieName))); var movie = new Movie { MovieName = name, Earnings = estBoxOffice * (valueInMillions.Value ? 1000000 : 1) }; if (articleDate.HasValue) { movie.WeekendEnding = MovieDateUtil.NextSunday(articleDate); } if (movie != null) { if (!result.Contains(movie)) { result.Add(movie); } else if (GameDays > 3) { // It's OK to override the BO value if the game days is MORE than the default. // Need to use "fuzzy" logic here because the names may have dates as suffixes and those should match. var found = result.Find(item => item.Equals(movie)); if (found != null && found.EarningsBase < movie.EarningsBase) { // Replace the movie if a larger value was found. (4 day weekend versus 3 day) result.Remove(found); result.Add(movie); Error = FOUR_DAY; } } } } } } } } } return(result); }