private async Task <bool> SaveMatch(MatchLink matchLink)
{
try
{
var htmlDoc = _htmlWeb.Load(matchLink.Link);
var articleNode = htmlDoc.DocumentNode.Descendants("article").FirstOrDefault();
var iframeNode = articleNode?.Descendants("iframe");
var ulNodes = articleNode?.Descendants("ul");
var divNodes = articleNode?.Descendants("div");
var inputNodes = articleNode?.Descendants("input");
var divVcRow = divNodes?.FirstOrDefault(x =>
x.Attributes.Contains("class") &&
x.Attributes["class"].Value.Contains("vc_row") &&
x.Attributes["class"].Value.Contains("wpb_row") &&
x.Attributes["class"].Value.Contains("td-pb-row"));
var divWpbWrapper =
divVcRow?.Descendants("div")?.FirstOrDefault(
x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "wpb_wrapper" &&
x.InnerText.Contains("Competition") &&
x.InnerText.Contains("Date") &&
x.InnerText.Contains("Stadium"));
if (divWpbWrapper == null)
{
divWpbWrapper = divNodes?.FirstOrDefault(x =>
x.Attributes.Contains("class") &&
x.Attributes["class"].Value == "td-post-content" &&
x.InnerText.Contains("Competition") &&
x.InnerText.Contains("Date") &&
x.InnerText.Contains("Stadium"));
}
if (divWpbWrapper == null || !iframeNode.Any())
{
return(false);
}
var acpPost = inputNodes?.FirstOrDefault(
x => x.Attributes.Contains("id") &&
x.Attributes.Contains("value") &&
x.Attributes["id"].Value == "acp_post")?.Attributes["value"]?.Value;
var acpShortcode = inputNodes?.FirstOrDefault(
x => x.Attributes.Contains("id") &&
x.Attributes.Contains("value") &&
x.Attributes["id"].Value == "acp_shortcode")?.Attributes["value"]?.Value;
var clips = GetMatchClip(acpPost, acpShortcode);
if (clips == null || !clips.Any())
{
clips = iframeNode.Where(x =>
x.Attributes.Contains("src") &&
!x.Attributes["src"].Value.Contains("facebook.com"))
.Select((x, i) =>
{
var clip = new Clip
{
Url = x.Attributes["src"].Value,
ClipType = ClipType.PreMatch,
LinkType = LinkType.Embed,
Name = "Full show"
};
if (clip.Url.Contains("veuclips.com"))
{
var uriLink = new Uri(clip.Url);
clip.Url = $"https://yfl.veuclips.com/embed/{uriLink.Segments[uriLink.Segments.Length - 1]}?autoplay=1&htmlplayer=1";
}
else if (clip.Url.Contains("viuclips.net"))
{
var uriLink = new Uri(clip.Url);
clip.Url = $"https://yfl.viuclips.net/embed/{uriLink.Segments[uriLink.Segments.Length - 1]}?autoplay=1&htmlplayer=1";
}
else if (clip.Url.Contains("evideohat.com"))
{
var uriLink = new Uri(clip.Url);
clip.Url = $"https://yfl.evideohat.com/embed00/{uriLink.Segments[uriLink.Segments.Length - 1]}";
}
return(clip);
}).ToList();
if (!clips.Any())
{
var iframe = divNodes
?.FirstOrDefault(x => x.Attributes.Contains("id") && x.Attributes["id"].Value == "acp_content")
?.Descendants("noscript").FirstOrDefault()
?.Descendants("iframe").FirstOrDefault(x => x.Attributes.Contains("src"));
if (iframe != null)
{
var link = iframe.Attributes["src"].Value;
var clip = new Clip
{
Url = link,
ClipType = ClipType.PreMatch,
LinkType = LinkType.Embed,
Name = "Full show"
};
if (clip.Url.Contains("veuclips.com"))
{
var uriLink = new Uri(clip.Url);
clip.Url = $"https://yfl.veuclips.com/embed/{uriLink.Segments[uriLink.Segments.Length - 1]}?autoplay=1&htmlplayer=1";
}
else if (clip.Url.Contains("viuclips.net"))
{
var uriLink = new Uri(clip.Url);
clip.Url = $"https://yfl.viuclips.net/embed/{uriLink.Segments[uriLink.Segments.Length - 1]}?autoplay=1&htmlplayer=1";
}
else if (clip.Url.Contains("evideohat.com"))
{
var uriLink = new Uri(clip.Url);
clip.Url = $"https://yfl.evideohat.com/embed00/{uriLink.Segments[uriLink.Segments.Length - 1]}";
}
clips.Add(clip);
}
}
var liItem = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("id") && x.Attributes["id"].Value == "acp_paging_menu")
?.Descendants("li").Where(x => x.Attributes.Contains("id") && x.Attributes["id"].Value.StartsWith("item"));
if (liItem != null)
{
foreach (var li in liItem)
{
var aNode = li?.Descendants("a")?.FirstOrDefault(x => x.Attributes.Contains("href"));
if (aNode != null)
{
var link = aNode.Attributes["href"].Value;
var clip = GetMatchClip(link, Convert.ToInt32(li.Attributes["id"].Value.Replace("item", "")));
if (clip != null)
{
clips.Add(clip);
}
}
else if (clips.Any())
{
clips.First().ClipType = (ClipType)Convert.ToInt32(li.Attributes["id"].Value.Replace("item", ""));
}
}
}
}
if (clips != null)
{
var ulAcpPaging = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("id") && x.Attributes["id"].Value == "acp_paging_menu");
if (ulAcpPaging != null)
{
foreach (var clip in clips)
{
var li = ulAcpPaging.Descendants("li").FirstOrDefault(x => x.Attributes.Contains("id") && x.Attributes["id"].Value == $"item{(int)clip.ClipType}");
if (li != null)
{
clip.Name = li.ChildNodes[0].ChildNodes[0].InnerText;
}
}
}
}
var match = new Match();
match.Slug = matchLink.Slug;
match.Title = matchLink.Name;
var ftpResult = await _ftpHelper.RemoteFiles(matchLink.ImageLink, match.Slug);
if (ftpResult != null)
{
match.ImageName = ftpResult.FileName;
match.ImageServerId = ftpResult.ServerId;
}
foreach (var childNode in divWpbWrapper.ChildNodes)
{
if (childNode.PreviousSibling == null)
{
continue;
}
if (childNode.PreviousSibling.InnerText.Contains("Competition"))
{
match.Competition = childNode.InnerText.HtmlDecode();
}
if (childNode.PreviousSibling.InnerText.Contains("Date"))
{
match.MatchDate = childNode.InnerText.ToDate();
}
if (childNode.PreviousSibling.InnerText.Contains("Stadium") && string.IsNullOrWhiteSpace(match.Stadium))
{
match.Stadium = childNode.InnerText.HtmlDecode();
}
if (childNode.PreviousSibling.InnerText.Contains("Referee") && !childNode.InnerText.Contains("jQuery"))
{
match.Referee = childNode.InnerText.HtmlDecode();
break;
}
}
if (match.MatchDate == null)
{
match.MatchDate = matchLink.RDateTime;
}
var divHeaderTeam1 = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "headerteam1");
if (divHeaderTeam1 != null)
{
match.Home = string.Empty;
foreach (var h2 in divHeaderTeam1.Descendants("h2"))
{
match.Home += " " + h2.InnerText;
}
match.Home = match.Home.HtmlDecode().Trim();
foreach (var node in divHeaderTeam1.Descendants("div"))
{
match.HomePersonScored += string.IsNullOrWhiteSpace(match.HomePersonScored)
? node.InnerText
: ("|" + node.InnerText);
}
}
else
{
//td-tags td-post-small-box clearfix
var tagNode = ulNodes?.FirstOrDefault(x =>
x.Attributes.Contains("class") &&
x.Attributes["class"].Value.Contains("td-tags") &&
x.Attributes["class"].Value.Contains("td-post-small-box"));
if (tagNode != null)
{
match.Home = tagNode.ChildNodes[1].ChildNodes[0].InnerText;
}
}
var divHeaderTeam2 = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "headerteam2");
if (divHeaderTeam2 != null)
{
match.Away = string.Empty;
foreach (var h2 in divHeaderTeam2.Descendants("h2"))
{
match.Away += " " + h2.InnerText;
}
match.Away = match.Away.HtmlDecode().Trim();
if (divHeaderTeam2.ChildNodes.Count > 1)
{
foreach (var node in divHeaderTeam2.Descendants("div"))
{
match.AwayPersonScored += string.IsNullOrWhiteSpace(match.AwayPersonScored) ? node.InnerText : ("|" + node.InnerText);
}
}
}
else
{
//td-tags td-post-small-box clearfix
var tagNode = ulNodes?.FirstOrDefault(x =>
x.Attributes.Contains("class") &&
x.Attributes["class"].Value.Contains("td-tags") &&
x.Attributes["class"].Value.Contains("td-post-small-box"));
if (tagNode != null)
{
match.Away = tagNode.ChildNodes[2].ChildNodes[0].InnerText;
}
}
var divScore = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "score");
if (divScore != null)
{
match.Score = $"{divScore.ChildNodes[0].InnerText} : {divScore.ChildNodes[2].InnerText}";
}
var divTeam1Roster = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1roster");
var formations = new List <Formation>();
if (divTeam1Roster != null)
{
match.HomeManager = divTeam1Roster.ChildNodes[1].InnerText;
formations.AddRange(divTeam1Roster.Descendants("li")
.OrderBy(x => x.InnerStartIndex)
.Select(liNode => new Formation
{
Type = FormationType.Home,
Number = liNode.ChildNodes[0].InnerText.ToInt(),
Name = liNode.ChildNodes[1].InnerText,
IsSubstitution = liNode.Attributes.Contains("class") && liNode.Attributes["class"].Value == "issub",
Scores = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && c.Attributes["class"].Value.Contains("list-goal")).Count(),
YellowCard = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && (c.Attributes["class"].Value.Contains("list-yellowcard") || c.Attributes["class"].Value.Contains("list-yellowredcard"))).Count(),
RedCard = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && c.Attributes["class"].Value.Contains("list-redcard")).Count(),
}));
}
var divTeam2Roster = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2roster");
if (divTeam2Roster != null)
{
match.AwayManager = divTeam2Roster.ChildNodes[1].InnerText;
formations.AddRange(divTeam2Roster.Descendants("li")
.OrderBy(x => x.InnerStartIndex)
.Select(liNode => new Formation
{
Type = FormationType.Away,
Number = liNode.ChildNodes[0].InnerText.ToInt(),
Name = liNode.ChildNodes[1].InnerText,
IsSubstitution = liNode.Attributes.Contains("class") && liNode.Attributes["class"].Value == "issub",
Scores = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && c.Attributes["class"].Value.Contains("list-goal")).Count(),
YellowCard = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && (c.Attributes["class"].Value.Contains("list-yellowcard") || c.Attributes["class"].Value.Contains("list-yellowredcard"))).Count(),
RedCard = liNode.ChildNodes.Where(c => c.Attributes.Contains("class") && c.Attributes["class"].Value.Contains("list-redcard")).Count(),
}));
}
var substitutions = new List <Substitution>();
var divTeam1Subs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1subs");
if (divTeam1Subs != null)
{
substitutions.AddRange(divTeam1Subs.Descendants("li")
.OrderBy(x => x.InnerStartIndex)
.Select(liNode => new Substitution
{
Type = FormationType.Home,
Number = liNode.ChildNodes[0].InnerText.ToInt(),
Name = liNode.ChildNodes[1].InnerText
}));
}
var divTeam2Subs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2subs");
if (divTeam2Subs != null)
{
substitutions.AddRange(divTeam2Subs.Descendants("li")
.OrderBy(x => x.InnerStartIndex)
.Select(liNode => new Substitution
{
Type = FormationType.Away,
Number = liNode.ChildNodes[0].InnerText.ToInt(),
Name = liNode.ChildNodes[1].InnerText
}));
}
var actionSubstitutions = new List <ActionSubstitution>();
var divteam1Actualsubs = ulNodes?.FirstOrDefault(x =>
x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1actualsubs");
if (divteam1Actualsubs != null)
{
actionSubstitutions.AddRange(divteam1Actualsubs.Descendants("li")
.OrderBy(x => x.InnerStartIndex)
.Select(x => new ActionSubstitution
{
Min = x.ChildNodes[0].InnerText,
In = x.ChildNodes[1].ChildNodes[0].InnerText,
Out = x.ChildNodes[1].ChildNodes[2].InnerText,
Type = FormationType.Home
}));
}
var divteam2Actualsubs = ulNodes?.FirstOrDefault(x =>
x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2actualsubs");
if (divteam2Actualsubs != null)
{
actionSubstitutions.AddRange(divteam2Actualsubs.Descendants("li")
.OrderBy(x => x.InnerStartIndex)
.Select(x => new ActionSubstitution
{
Min = x.ChildNodes[0].InnerText,
In = x.ChildNodes[1].ChildNodes[0].InnerText,
Out = x.ChildNodes[1].ChildNodes[2].InnerText,
Type = FormationType.Away
}));
}
await _matchRepository.Add(match, clips, formations, substitutions, actionSubstitutions);
return(true);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
return(false);
}
}
public async Task <bool> UpdateImage()
{
try
{
//var jobs = _dbContext.JobListCrawl.Where(x => x.Id == 89 && x.Complete > 0);
//foreach (var job in jobs)
//{
var movies = await _htmlHelper.GetJavMovies("http://jav789.com/movie?q=yuka+minami+temptation+to+a+dirty+housewife&ajax=1");
if (movies == null || movies.movies == null || movies.movies.Count == 0)
{
return(true);
}
foreach (var movie in movies.movies)
{
if (!_dbContext.Films.Any(x => x.Slug == movie.url))
{
continue;
}
string thumb_cover = null;
string thumb_name = null;
int? ftpId = null;
if (!string.IsNullOrWhiteSpace(movie.image_small))
{
var ftpUpload = await _ftpHelper.RemoteFiles(movie.image_small, movie.url);
if (ftpUpload.IsSuccessful)
{
thumb_name = ftpUpload.FileName;
ftpId = ftpUpload.ServerId;
}
}
if (movie.image == movie.image_small)
{
thumb_cover = thumb_name;
}
else if (!string.IsNullOrWhiteSpace(movie.image))
{
var ftpUpload = await _ftpHelper.RemoteFiles(movie.image, movie.url + "-cover");
if (ftpUpload.IsSuccessful)
{
thumb_cover = ftpUpload.FileName;
ftpId = ftpUpload.ServerId;
}
}
foreach (var star in movie.pornstars)
{
var newStar = await _htmlHelper.GetJavHiHiStar(star, movie.fromsite);
if (!string.IsNullOrWhiteSpace(newStar.ThumbName))
{
var uploadResults = await _ftpHelper.RemoteFiles(newStar.ThumbName, star.ToLower().Trim().Replace(" ", "-"));
}
}
}
//}
return(true);
}
catch
{
return(false);
}
}
private async Task <bool> SaveMatch(MatchLink matchLink)
{
var htmlWeb = new HtmlWeb();
var htmlDoc = await htmlWeb.LoadFromWebAsync(matchLink.Link);
var articleNode = htmlDoc.DocumentNode.Descendants("article").FirstOrDefault();
var iframeNode = articleNode?.Descendants("iframe");
var ulNodes = articleNode?.Descendants("ul");
var divNodes = articleNode?.Descendants("div");
var divVcRow = divNodes?.FirstOrDefault(x =>
x.Attributes.Contains("class") && x.Attributes["class"].Value == "vc_row wpb_row td-pb-row");
var divWpbWrapper =
divVcRow?.Descendants("div")?.LastOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "wpb_wrapper");
if (divWpbWrapper == null || !iframeNode.Any())
{
return(false);
}
var clips = new List <Clip>();
clips.AddRange(iframeNode.Where(x => x.Attributes.Contains("data-lazy-src")).Select((x, i) => new Clip
{
Url = x.Attributes["data-lazy-src"].Value,
ClipType = ClipType.HighLight,
LinkType = LinkType.Embed,
Name = "Server " + (i + 1)
}));
if (!clips.Any())
{
return(false);
}
var match = new Match();
match.Slug = matchLink.Slug;
match.Title = matchLink.Name;
var ftpResult = await _ftpHelper.RemoteFiles(matchLink.ImageLink, match.Slug);
if (ftpResult != null)
{
match.ImageName = ftpResult.FileName;
match.ImageServerId = ftpResult.ServerId;
}
foreach (var childNode in divWpbWrapper.ChildNodes)
{
if (childNode.PreviousSibling == null)
{
continue;
}
if (childNode.PreviousSibling.InnerText.Contains("Competition"))
{
match.Competition = childNode.InnerText;
}
if (childNode.PreviousSibling.InnerText.Contains("Date"))
{
match.MatchDate = childNode.InnerText.ToDate();
}
if (childNode.PreviousSibling.InnerText.Contains("Stadium"))
{
match.Stadium = childNode.InnerText;
}
if (childNode.PreviousSibling.InnerText.Contains("Referee"))
{
match.Referee = childNode.InnerText;
}
}
if (match.MatchDate == null)
{
match.MatchDate = matchLink.RDateTime;
}
var divHeaderTeam1 = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "headerteam1");
if (divHeaderTeam1 != null)
{
match.Home = string.Empty;
foreach (var h2 in divHeaderTeam1.Descendants("h2"))
{
match.Home += " " + h2.InnerText;
}
match.Home = match.Home.Trim();
foreach (var node in divHeaderTeam1.Descendants("div"))
{
match.HomePersonScored = string.IsNullOrWhiteSpace(match.HomePersonScored)
? node.InnerText
: ("|" + node.InnerText);
}
}
else
{
//td-tags td-post-small-box clearfix
var tagNode = ulNodes?.FirstOrDefault(x =>
x.Attributes.Contains("class") &&
x.Attributes["class"].Value.Contains("td-tags") &&
x.Attributes["class"].Value.Contains("td-post-small-box"));
if (tagNode != null)
{
match.Home = tagNode.ChildNodes[1].ChildNodes[0].InnerText;
}
}
var divHeaderTeam2 = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "headerteam2");
if (divHeaderTeam2 != null)
{
match.Away = string.Empty;
foreach (var h2 in divHeaderTeam2.Descendants("h2"))
{
match.Away += " " + h2.InnerText;
}
match.Away = match.Away.Trim();
if (divHeaderTeam2.ChildNodes.Count > 1)
{
foreach (var node in divHeaderTeam2.Descendants("div"))
{
match.AwayPersonScored = string.IsNullOrWhiteSpace(match.AwayPersonScored) ? node.InnerText : ("|" + node.InnerText);
}
}
}
else
{
//td-tags td-post-small-box clearfix
var tagNode = ulNodes?.FirstOrDefault(x =>
x.Attributes.Contains("class") &&
x.Attributes["class"].Value.Contains("td-tags") &&
x.Attributes["class"].Value.Contains("td-post-small-box"));
if (tagNode != null)
{
match.Away = tagNode.ChildNodes[2].ChildNodes[0].InnerText;
}
}
var divScore = divNodes.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "score");
if (divScore != null)
{
match.Score = $"{divScore.ChildNodes[0].InnerText} : {divScore.ChildNodes[2].InnerText}";
}
var divTeam1Roster = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1roster");
var formations = new List <Formation>();
if (divTeam1Roster != null)
{
match.HomeManager = divTeam1Roster.ChildNodes[1].InnerText;
formations.AddRange(divTeam1Roster.Descendants("li")
.Select(liNode => new Formation
{
Type = FormationType.Home,
Number = liNode.ChildNodes[0].InnerText.ToInt(),
Name = liNode.ChildNodes[1].InnerText,
IsSubstitution = liNode.Attributes.Contains("class") && liNode.Attributes["class"].Value == "issub"
}));
}
var divTeam2Roster = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2roster");
if (divTeam2Roster != null)
{
match.HomeManager = divTeam2Roster.ChildNodes[1].InnerText;
formations.AddRange(divTeam2Roster.Descendants("li")
.Select(liNode => new Formation
{
Type = FormationType.Home,
Number = liNode.ChildNodes[0].InnerText.ToInt(),
Name = liNode.ChildNodes[1].InnerText,
IsSubstitution = liNode.Attributes.Contains("class") && liNode.Attributes["class"].Value == "issub"
}));
}
var substitutions = new List <Substitution>();
var divTeam1Subs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1subs");
if (divTeam1Subs != null)
{
substitutions.AddRange(divTeam1Subs.Descendants("li")
.Select(liNode => new Substitution
{
Type = FormationType.Home,
Number = liNode.ChildNodes[0].InnerText.ToInt(),
Name = liNode.ChildNodes[1].InnerText
}));
}
var divTeam2Subs = ulNodes?.FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2subs");
if (divTeam2Subs != null)
{
substitutions.AddRange(divTeam2Subs.Descendants("li")
.Select(liNode => new Substitution
{
Type = FormationType.Home,
Number = liNode.ChildNodes[0].InnerText.ToInt(),
Name = liNode.ChildNodes[1].InnerText
}));
}
var actionSubstitutions = new List <ActionSubstitution>();
var divteam1Actualsubs = ulNodes?.FirstOrDefault(x =>
x.Attributes.Contains("class") && x.Attributes["class"].Value == "team1actualsubs");
if (divteam1Actualsubs != null)
{
actionSubstitutions.AddRange(divteam1Actualsubs.Descendants("li")
.Select(x => new ActionSubstitution
{
Min = x.ChildNodes[0].InnerText,
In = x.ChildNodes[1].ChildNodes[0].InnerText,
Out = x.ChildNodes[1].ChildNodes[2].InnerText,
Type = FormationType.Home
}));
}
var divteam2Actualsubs = ulNodes?.FirstOrDefault(x =>
x.Attributes.Contains("class") && x.Attributes["class"].Value == "team2actualsubs");
if (divteam2Actualsubs != null)
{
actionSubstitutions.AddRange(divteam2Actualsubs.Descendants("li")
.Select(x => new ActionSubstitution
{
Min = x.ChildNodes[0].InnerText,
In = x.ChildNodes[1].ChildNodes[0].InnerText,
Out = x.ChildNodes[1].ChildNodes[2].InnerText,
Type = FormationType.Away
}));
}
await _matchRepository.Add(match, clips, formations, substitutions, actionSubstitutions);
return(true);
}
