private List <DuelLink> GetDuelsLinks(MatchLink match) { return(_driver.FindElements(By.CssSelector(".table-participant > a")).Where(x => x.GetAttribute("href").Contains("javascript:") == false).Select(x => new DuelLink() { Name = x.Text, Url = x.GetAttribute("href"), MatchLink = match }).ToList()); }
private async Task <IEnumerable <MatchLink> > GetMatchLinks(string url) { var htmlWeb = new HtmlWeb(); var htmlDoc = await htmlWeb.LoadFromWebAsync(url); var divNodes = htmlDoc.DocumentNode.Descendants("div"); if (divNodes == null || !divNodes.Any()) { return(null); } var divMainContent = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "td-ss-main-content"); if (divMainContent == null) { divMainContent = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "td_block_inner"); } if (divMainContent == null) { return(null); } var results = new List <MatchLink>(); foreach (var nodeBlock in divMainContent.ChildNodes) { if (!(nodeBlock.Attributes.Contains("class") && nodeBlock.Attributes["class"].Value == "td-block-row")) { continue; } foreach (var node in nodeBlock.ChildNodes) { var timeNode = node.Descendants("time"); if (!timeNode.Any()) { continue; } var matchLink = new MatchLink(); matchLink.Date = timeNode.First().InnerHtml; var aNodes = node.Descendants("a"); foreach (var aNode in aNodes) { if (aNode.Descendants("img").Any()) { matchLink.ImageLink = aNode.ChildNodes[0].Attributes["data-img-url"].Value; } else { matchLink.Name = aNode.InnerHtml.HtmlDecode(); matchLink.Link = aNode.Attributes["href"].Value; } } results.Add(matchLink); } } return(results); }
private bool SaveMatch(MatchLink matchLink) { var htmlDoc = _htmlWeb.Load(matchLink.Link); var articleNode = htmlDoc.DocumentNode.Descendants("article").FirstOrDefault(); var iframeNode = articleNode?.Descendants("iframe"); var ulNodes = articleNode?.Descendants("ul"); var divNodes = articleNode?.Descendants("div"); var inputNodes = articleNode?.Descendants("input"); var divVcRow = divNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("vc_row") && x.Attributes["class"].Value.Contains("wpb_row") && x.Attributes["class"].Value.Contains("td-pb-row")); var divWpbWrapper = divVcRow?.Descendants("div")?.LastOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "wpb_wrapper"); if (divWpbWrapper == null || !iframeNode.Any()) { return(false); } var acpPost = inputNodes?.FirstOrDefault( x => x.Attributes.Contains("id") && x.Attributes.Contains("value") && x.Attributes["id"].Value == "acp_post")?.Attributes["value"]?.Value; var acpShortcode = inputNodes?.FirstOrDefault( x => x.Attributes.Contains("id") && x.Attributes.Contains("value") && x.Attributes["id"].Value == "acp_shortcode")?.Attributes["value"]?.Value; var clips = GetMatchClip(acpPost, acpShortcode); if (clips == null || !clips.Any()) { clips = iframeNode.Where(x => x.Attributes.Contains("data-lazy-src") && !x.Attributes["data-lazy-src"].Value.Contains("facebook.com")) .Select((x, i) => new Clip { Url = x.Attributes["data-lazy-src"].Value, ClipType = ClipType.PostMatch, LinkType = LinkType.Embed, Name = "Full show" }).ToList(); } var match = new Match(); match.Slug = matchLink.Slug; match.Title = matchLink.Name; //var ftpResult = await _ftpHelper.RemoteFiles(matchLink.ImageLink, match.Slug); //if (ftpResult != null) //{ // match.ImageName = ftpResult.FileName; // match.ImageServerId = ftpResult.ServerId; //} foreach (var childNode in divWpbWrapper.ChildNodes) { if (childNode.PreviousSibling == null) { continue; } if (childNode.PreviousSibling.InnerText.Contains("Competition")) { match.Competition = childNode.InnerText; } if (childNode.PreviousSibling.InnerText.Contains("Date")) { match.MatchDate = childNode.InnerText.ToDate(); } if (childNode.PreviousSibling.InnerText.Contains("Stadium")) { match.Stadium = childNode.InnerText; } if (childNode.PreviousSibling.InnerText.Contains("Referee")) { match.Referee = childNode.InnerText; } } if (match.MatchDate == null) { match.MatchDate = matchLink.RDateTime; } var divHeaderTeam1 = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "headerteam1"); if (divHeaderTeam1 != null) { match.Home = string.Empty; foreach (var h2 in divHeaderTeam1.Descendants("h2")) { match.Home += " " + h2.InnerText; } match.Home = match.Home.Trim(); foreach (var node in divHeaderTeam1.Descendants("div")) { match.HomePersonScored += string.IsNullOrWhiteSpace(match.HomePersonScored) ? node.InnerText : ("|" + node.InnerText); } } else { //td-tags td-post-small-box clearfix var tagNode = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("td-tags") && x.Attributes["class"].Value.Contains("td-post-small-box")); if (tagNode != null) { match.Home = tagNode.ChildNodes[1].ChildNodes[0].InnerText; } } var divHeaderTeam2 = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "headerteam2"); if (divHeaderTeam2 != null) { match.Away = string.Empty; foreach (var h2 in divHeaderTeam2.Descendants("h2")) { match.Away += " " + h2.InnerText; } match.Away = match.Away.Trim(); if (divHeaderTeam2.ChildNodes.Count > 1) { foreach (var node in divHeaderTeam2.Descendants("div")) { match.AwayPersonScored += string.IsNullOrWhiteSpace(match.AwayPersonScored) ? node.InnerText : ("|" + node.InnerText); } } } else { //td-tags td-post-small-box clearfix var tagNode = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("td-tags") && x.Attributes["class"].Value.Contains("td-post-small-box")); if (tagNode != null) { match.Away = tagNode.ChildNodes[2].ChildNodes[0].InnerText; } } var divScore = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "score"); if (divScore != null) { match.Score = $"{divScore.ChildNodes[0].InnerText} : {divScore.ChildNodes[2].InnerText}"; } var divTeam1Roster = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1roster"); var formations = new List <Formation>(); if (divTeam1Roster != null) { match.HomeManager = divTeam1Roster.ChildNodes[1].InnerText; formations.AddRange(divTeam1Roster.Descendants("li") .Select(liNode => new Formation { Type = FormationType.Home, Number = liNode.ChildNodes[0].InnerText.ToInt(), Name = liNode.ChildNodes[1].InnerText, IsSubstitution = liNode.Attributes.Contains("class") && liNode.Attributes["class"].Value == "issub" })); } var divTeam2Roster = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2roster"); if (divTeam2Roster != null) { match.AwayManager = divTeam2Roster.ChildNodes[1].InnerText; formations.AddRange(divTeam2Roster.Descendants("li") .Select(liNode => new Formation { Type = FormationType.Away, Number = liNode.ChildNodes[0].InnerText.ToInt(), Name = liNode.ChildNodes[1].InnerText, IsSubstitution = liNode.Attributes.Contains("class") && liNode.Attributes["class"].Value == "issub" })); } var substitutions = new List <Substitution>(); var divTeam1Subs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1subs"); if (divTeam1Subs != null) { substitutions.AddRange(divTeam1Subs.Descendants("li") .Select(liNode => new Substitution { Type = FormationType.Home, Number = liNode.ChildNodes[0].InnerText.ToInt(), Name = liNode.ChildNodes[1].InnerText })); } var divTeam2Subs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2subs"); if (divTeam2Subs != null) { substitutions.AddRange(divTeam2Subs.Descendants("li") .Select(liNode => new Substitution { Type = FormationType.Away, Number = liNode.ChildNodes[0].InnerText.ToInt(), Name = liNode.ChildNodes[1].InnerText })); } var actionSubstitutions = new List <ActionSubstitution>(); var divteam1Actualsubs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1actualsubs"); if (divteam1Actualsubs != null) { actionSubstitutions.AddRange(divteam1Actualsubs.Descendants("li") .Select(x => new ActionSubstitution { Min = x.ChildNodes[0].InnerText, In = x.ChildNodes[1].ChildNodes[0].InnerText, Out = x.ChildNodes[1].ChildNodes[2].InnerText, Type = FormationType.Home })); } var divteam2Actualsubs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2actualsubs"); if (divteam2Actualsubs != null) { actionSubstitutions.AddRange(divteam2Actualsubs.Descendants("li") .Select(x => new ActionSubstitution { Min = x.ChildNodes[0].InnerText, In = x.ChildNodes[1].ChildNodes[0].InnerText, Out = x.ChildNodes[1].ChildNodes[2].InnerText, Type = FormationType.Away })); } //await _matchRepository.Add(match, clips, formations, substitutions, actionSubstitutions); return(true); }
private async Task <bool> SaveMatch(MatchLink matchLink) { try { var htmlDoc = _htmlWeb.Load(matchLink.Link); var articleNode = htmlDoc.DocumentNode.Descendants("article").FirstOrDefault(); var iframeNode = articleNode?.Descendants("iframe"); var ulNodes = articleNode?.Descendants("ul"); var divNodes = articleNode?.Descendants("div"); var inputNodes = articleNode?.Descendants("input"); var divVcRow = divNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("vc_row") && x.Attributes["class"].Value.Contains("wpb_row") && x.Attributes["class"].Value.Contains("td-pb-row")); var divWpbWrapper = divVcRow?.Descendants("div")?.FirstOrDefault( x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "wpb_wrapper" && x.InnerText.Contains("Competition") && x.InnerText.Contains("Date") && x.InnerText.Contains("Stadium")); if (divWpbWrapper == null) { divWpbWrapper = divNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "td-post-content" && x.InnerText.Contains("Competition") && x.InnerText.Contains("Date") && x.InnerText.Contains("Stadium")); } if (divWpbWrapper == null || !iframeNode.Any()) { return(false); } var acpPost = inputNodes?.FirstOrDefault( x => x.Attributes.Contains("id") && x.Attributes.Contains("value") && x.Attributes["id"].Value == "acp_post")?.Attributes["value"]?.Value; var acpShortcode = inputNodes?.FirstOrDefault( x => x.Attributes.Contains("id") && x.Attributes.Contains("value") && x.Attributes["id"].Value == "acp_shortcode")?.Attributes["value"]?.Value; var clips = GetMatchClip(acpPost, acpShortcode); if (clips == null || !clips.Any()) { clips = iframeNode.Where(x => x.Attributes.Contains("src") && !x.Attributes["src"].Value.Contains("facebook.com")) .Select((x, i) => { var clip = new Clip { Url = x.Attributes["src"].Value, ClipType = ClipType.PreMatch, LinkType = LinkType.Embed, Name = "Full show" }; if (clip.Url.Contains("veuclips.com")) { var uriLink = new Uri(clip.Url); clip.Url = $"https://yfl.veuclips.com/embed/{uriLink.Segments[uriLink.Segments.Length - 1]}?autoplay=1&htmlplayer=1"; } else if (clip.Url.Contains("viuclips.net")) { var uriLink = new Uri(clip.Url); clip.Url = $"https://yfl.viuclips.net/embed/{uriLink.Segments[uriLink.Segments.Length - 1]}?autoplay=1&htmlplayer=1"; } else if (clip.Url.Contains("evideohat.com")) { var uriLink = new Uri(clip.Url); clip.Url = $"https://yfl.evideohat.com/embed00/{uriLink.Segments[uriLink.Segments.Length - 1]}"; } return(clip); }).ToList(); if (!clips.Any()) { var iframe = divNodes ?.FirstOrDefault(x => x.Attributes.Contains("id") && x.Attributes["id"].Value == "acp_content") ?.Descendants("noscript").FirstOrDefault() ?.Descendants("iframe").FirstOrDefault(x => x.Attributes.Contains("src")); if (iframe != null) { var link = iframe.Attributes["src"].Value; var clip = new Clip { Url = link, ClipType = ClipType.PreMatch, LinkType = LinkType.Embed, Name = "Full show" }; if (clip.Url.Contains("veuclips.com")) { var uriLink = new Uri(clip.Url); clip.Url = $"https://yfl.veuclips.com/embed/{uriLink.Segments[uriLink.Segments.Length - 1]}?autoplay=1&htmlplayer=1"; } else if (clip.Url.Contains("viuclips.net")) { var uriLink = new Uri(clip.Url); clip.Url = $"https://yfl.viuclips.net/embed/{uriLink.Segments[uriLink.Segments.Length - 1]}?autoplay=1&htmlplayer=1"; } else if (clip.Url.Contains("evideohat.com")) { var uriLink = new Uri(clip.Url); clip.Url = $"https://yfl.evideohat.com/embed00/{uriLink.Segments[uriLink.Segments.Length - 1]}"; } clips.Add(clip); } } var liItem = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("id") && x.Attributes["id"].Value == "acp_paging_menu") ?.Descendants("li").Where(x => x.Attributes.Contains("id") && x.Attributes["id"].Value.StartsWith("item")); if (liItem != null) { foreach (var li in liItem) { var aNode = li?.Descendants("a")?.FirstOrDefault(x => x.Attributes.Contains("href")); if (aNode != null) { var link = aNode.Attributes["href"].Value; var clip = GetMatchClip(link, Convert.ToInt32(li.Attributes["id"].Value.Replace("item", ""))); if (clip != null) { clips.Add(clip); } } else if (clips.Any()) { clips.First().ClipType = (ClipType)Convert.ToInt32(li.Attributes["id"].Value.Replace("item", "")); } } } } if (clips != null) { var ulAcpPaging = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("id") && x.Attributes["id"].Value == "acp_paging_menu"); if (ulAcpPaging != null) { foreach (var clip in clips) { var li = ulAcpPaging.Descendants("li").FirstOrDefault(x => x.Attributes.Contains("id") && x.Attributes["id"].Value == $"item{(int)clip.ClipType}"); if (li != null) { clip.Name = li.ChildNodes[0].ChildNodes[0].InnerText; } } } } var match = new Match(); match.Slug = matchLink.Slug; match.Title = matchLink.Name; var ftpResult = await _ftpHelper.RemoteFiles(matchLink.ImageLink, match.Slug); if (ftpResult != null) { match.ImageName = ftpResult.FileName; match.ImageServerId = ftpResult.ServerId; } foreach (var childNode in divWpbWrapper.ChildNodes) { if (childNode.PreviousSibling == null) { continue; } if (childNode.PreviousSibling.InnerText.Contains("Competition")) { match.Competition = childNode.InnerText.HtmlDecode(); } if (childNode.PreviousSibling.InnerText.Contains("Date")) { match.MatchDate = childNode.InnerText.ToDate(); } if (childNode.PreviousSibling.InnerText.Contains("Stadium") && string.IsNullOrWhiteSpace(match.Stadium)) { match.Stadium = childNode.InnerText.HtmlDecode(); } if (childNode.PreviousSibling.InnerText.Contains("Referee") && !childNode.InnerText.Contains("jQuery")) { match.Referee = childNode.InnerText.HtmlDecode(); break; } } if (match.MatchDate == null) { match.MatchDate = matchLink.RDateTime; } var divHeaderTeam1 = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "headerteam1"); if (divHeaderTeam1 != null) { match.Home = string.Empty; foreach (var h2 in divHeaderTeam1.Descendants("h2")) { match.Home += " " + h2.InnerText; } match.Home = match.Home.HtmlDecode().Trim(); foreach (var node in divHeaderTeam1.Descendants("div")) { match.HomePersonScored += string.IsNullOrWhiteSpace(match.HomePersonScored) ? node.InnerText : ("|" + node.InnerText); } } else { //td-tags td-post-small-box clearfix var tagNode = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("td-tags") && x.Attributes["class"].Value.Contains("td-post-small-box")); if (tagNode != null) { match.Home = tagNode.ChildNodes[1].ChildNodes[0].InnerText; } } var divHeaderTeam2 = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "headerteam2"); if (divHeaderTeam2 != null) { match.Away = string.Empty; foreach (var h2 in divHeaderTeam2.Descendants("h2")) { match.Away += " " + h2.InnerText; } match.Away = match.Away.HtmlDecode().Trim(); if (divHeaderTeam2.ChildNodes.Count > 1) { foreach (var node in divHeaderTeam2.Descendants("div")) { match.AwayPersonScored += string.IsNullOrWhiteSpace(match.AwayPersonScored) ? node.InnerText : ("|" + node.InnerText); } } } else { //td-tags td-post-small-box clearfix var tagNode = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("td-tags") && x.Attributes["class"].Value.Contains("td-post-small-box")); if (tagNode != null) { match.Away = tagNode.ChildNodes[2].ChildNodes[0].InnerText; } } var divScore = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "score"); if (divScore != null) { match.Score = $"{divScore.ChildNodes[0].InnerText} : {divScore.ChildNodes[2].InnerText}"; } var divTeam1Roster = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1roster"); var formations = new List <Formation>(); if (divTeam1Roster != null) { match.HomeManager = divTeam1Roster.ChildNodes[1].InnerText; formations.AddRange(divTeam1Roster.Descendants("li") .OrderBy(x => x.InnerStartIndex) .Select(liNode => new Formation { Type = FormationType.Home, Number = liNode.ChildNodes[0].InnerText.ToInt(), Name = liNode.ChildNodes[1].InnerText, IsSubstitution = liNode.Attributes.Contains("class") && liNode.Attributes["class"].Value == "issub", Scores = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && c.Attributes["class"].Value.Contains("list-goal")).Count(), YellowCard = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && (c.Attributes["class"].Value.Contains("list-yellowcard") || c.Attributes["class"].Value.Contains("list-yellowredcard"))).Count(), RedCard = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && c.Attributes["class"].Value.Contains("list-redcard")).Count(), })); } var divTeam2Roster = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2roster"); if (divTeam2Roster != null) { match.AwayManager = divTeam2Roster.ChildNodes[1].InnerText; formations.AddRange(divTeam2Roster.Descendants("li") .OrderBy(x => x.InnerStartIndex) .Select(liNode => new Formation { Type = FormationType.Away, Number = liNode.ChildNodes[0].InnerText.ToInt(), Name = liNode.ChildNodes[1].InnerText, IsSubstitution = liNode.Attributes.Contains("class") && liNode.Attributes["class"].Value == "issub", Scores = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && c.Attributes["class"].Value.Contains("list-goal")).Count(), YellowCard = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && (c.Attributes["class"].Value.Contains("list-yellowcard") || c.Attributes["class"].Value.Contains("list-yellowredcard"))).Count(), RedCard = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && c.Attributes["class"].Value.Contains("list-redcard")).Count(), })); } var substitutions = new List <Substitution>(); var divTeam1Subs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1subs"); if (divTeam1Subs != null) { substitutions.AddRange(divTeam1Subs.Descendants("li") .OrderBy(x => x.InnerStartIndex) .Select(liNode => new Substitution { Type = FormationType.Home, Number = liNode.ChildNodes[0].InnerText.ToInt(), Name = liNode.ChildNodes[1].InnerText })); } var divTeam2Subs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2subs"); if (divTeam2Subs != null) { substitutions.AddRange(divTeam2Subs.Descendants("li") .OrderBy(x => x.InnerStartIndex) .Select(liNode => new Substitution { Type = FormationType.Away, Number = liNode.ChildNodes[0].InnerText.ToInt(), Name = liNode.ChildNodes[1].InnerText })); } var actionSubstitutions = new List <ActionSubstitution>(); var divteam1Actualsubs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1actualsubs"); if (divteam1Actualsubs != null) { actionSubstitutions.AddRange(divteam1Actualsubs.Descendants("li") .OrderBy(x => x.InnerStartIndex) .Select(x => new ActionSubstitution { Min = x.ChildNodes[0].InnerText, In = x.ChildNodes[1].ChildNodes[0].InnerText, Out = x.ChildNodes[1].ChildNodes[2].InnerText, Type = FormationType.Home })); } var divteam2Actualsubs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2actualsubs"); if (divteam2Actualsubs != null) { actionSubstitutions.AddRange(divteam2Actualsubs.Descendants("li") .OrderBy(x => x.InnerStartIndex) .Select(x => new ActionSubstitution { Min = x.ChildNodes[0].InnerText, In = x.ChildNodes[1].ChildNodes[0].InnerText, Out = x.ChildNodes[1].ChildNodes[2].InnerText, Type = FormationType.Away })); } await _matchRepository.Add(match, clips, formations, substitutions, actionSubstitutions); return(true); } catch (Exception ex) { Console.WriteLine(ex.Message); return(false); } }