public bool DownloadAllMovies(bool redoIfFound, int numberOfPages = 0)
{
var videoEntities = new VideosEntities();
var downLoadProgressRepository = new DownLoadProgressRepository(videoEntities);
var currentPage = 0;
var lastPage = 0;
var result = false;
if (downLoadProgressRepository.Items.Any())
{
lastPage = (int)downLoadProgressRepository.Items.FirstOrDefault().LastPage;
}
try
{
var totalpages = (numberOfPages == 0) ? GetTotalPages("http://www.primewire.ag/index.php?sort=featured&page=" + lastPage) : numberOfPages;
for (int i = lastPage; i <= totalpages; i++)
{
currentPage = i;
if (numberOfPages > 0)
{
numberOfPages -= 1;
}
Debug.WriteLine("SCRAPE PAGE {0} - {1}", i, DateTime.Now);
var nextpage = "http://www.primewire.ag/index.php?sort=featured&page=" + i;
var task = new Task(() => ScrapePageAsync(nextpage, redoIfFound));
TaskDelay();
task.Start();
downLoadProgressRepository.UpdateLastPage(i + 1);
downLoadProgressRepository.SaveChanges();
}
result = true;
}
catch (Exception e)
{
//Skip Page and reTry
Debug.WriteLine(
"________________START OVER______________________{0}", e);
downLoadProgressRepository.AddSkipedPage(currentPage);
downLoadProgressRepository.UpdateLastPage(lastPage + 1);
downLoadProgressRepository.SaveChanges();
DownloadAllMovies(redoIfFound, numberOfPages);
}
return(result);
}
public VideoController()
{
_videoEntities = new VideosEntities();
_videoRepository = new Repository <Video, VideosEntities>(_videoEntities);
_actorRepository = new Repository <Actor, VideosEntities>(_videoEntities);
_genreRepository = new Repository <Genre, VideosEntities>(_videoEntities);
_linkRepository = new Repository <Link, VideosEntities>(_videoEntities);
_countryRepository = new Repository <Country, VideosEntities>(_videoEntities);
_hostRepository = new Repository <Host, VideosEntities>(_videoEntities);
_tvEpisodeRepository = new Repository <TvEpisode, VideosEntities>(_videoEntities);
_downLoadProgressRepository = new DownLoadProgressRepository(_videoEntities);
}
public void RefetchNoLinkVideos(int maxVideos = 0)
{
var videoEntities = new VideosEntities();
var videoRepository = new Repository <Video, VideosEntities>(videoEntities);
var allTask = new List <Task>();
var allVideos = videoRepository.Items.Where(x => !x.Links.Any());
if (maxVideos > 0)
{
allVideos.Take(maxVideos);
}
foreach (var video in allVideos)
{
var task = new Task(() => RefreshVideoLinks(video));
TaskDelay();
task.Start();
allTask.Add(task);
}
Task.WaitAll(allTask.ToArray());
}
private Video ParseVideoPage(string url, bool isTvShow, bool redoIfFound)
{
var videoEntities = new VideosEntities();
var videoRepository = new Repository <Video, VideosEntities>(videoEntities);
var actorRepository = new Repository <Actor, VideosEntities>(videoEntities);
var genreRepository = new Repository <Genre, VideosEntities>(videoEntities);
var countryRepository = new Repository <Country, VideosEntities>(videoEntities);
var document = new MyWebClient().GetHtmlDocument(url);
var movieInfo = document.DocumentNode.SelectSingleNode("//div[@class='index_container']");
var imdb = movieInfo.SelectSingleNode("//div[@class='mlink_imdb']/a").AttributeValue("href");
var name = movieInfo.SelectSingleNode("//div[@class='stage_navigation movie_navigation']").InnerTextValue().Trim();
var actors = movieInfo.SelectNodes("//span[@class='movie_info_actors']/a") ?? new HtmlNodeCollection(null);
var countries = movieInfo.SelectNodes("//span[@class='movie_info_country']/a") ?? new HtmlNodeCollection(null);
var generes = movieInfo.SelectNodes("//span[@class='movie_info_genres']/a") ?? new HtmlNodeCollection(null);
var newVideo = new Video();
var existingVideo = videoRepository.Items.FirstOrDefault(x => x.ImdbLink == imdb && x.Name == name);
if (existingVideo != null)
{
if (redoIfFound)
{
newVideo = existingVideo;
}
else
{
return(null);
}
}
newVideo.Image = new MyWebClient().UrlToImageByteArray(movieInfo.SelectSingleNode("//div[@class='movie_thumb']/img").AttributeValue("src"));
newVideo.Name = name;
newVideo.Description = movieInfo.SelectSingleNode("//table/tr[1]/td[1]/p").InnerTextValue().Trim();
newVideo.DateAdded = DateTime.Now;
newVideo.ReleaseDate = DateTime.Parse(movieInfo.SelectSingleNode("//table/tr[2]/td[2]").InnerTextValue());
newVideo.Runtime = movieInfo.SelectSingleNode("//table/tr[3]/td[2]").InnerTextValue().ToIntIgnoreStrings();
newVideo.Director = movieInfo.SelectSingleNode("//table/tr[6]/td[2]").InnerTextValue();
newVideo.Link = url;
newVideo.ImdbLink = imdb;
newVideo.BuyLink = movieInfo.SelectSingleNode("//div[@class='mlink_buydvd']/a").AttributeValue("href");
newVideo.TrailerLink = movieInfo.SelectSingleNode("//span[@class='movie_version_link']/a[2]").AttributeValue("href");
newVideo.Genres = new List <Genre>();
newVideo.Countries = new List <Country>();
newVideo.Actors = new List <Actor>();
foreach (HtmlNode genre in generes)
{
var newGenre = genreRepository.Items.FirstOrDefault(x => x.Name == genre.InnerText);
if (newGenre == null)
{
newGenre = new Genre {
Name = genre.InnerTextValue()
};
genreRepository.AddItem(newGenre);
genreRepository.SaveChanges();
}
newVideo.Genres.Add(newGenre);
}
foreach (HtmlNode country in countries)
{
var newCountry = countryRepository.Items.FirstOrDefault(x => x.Name == country.InnerText);
if (newCountry == null)
{
newCountry = new Country {
Name = country.InnerTextValue()
};
countryRepository.AddItem(newCountry);
countryRepository.SaveChanges();
}
newVideo.Countries.Add(newCountry);
}
foreach (HtmlNode actor in actors)
{
var newActor = actorRepository.Items.FirstOrDefault(x => x.Name == actor.InnerText);
if (newActor == null)
{
newActor = new Actor {
Name = actor.InnerTextValue()
};
actorRepository.AddItem(newActor);
actorRepository.SaveChanges();
}
newVideo.Actors.Add(newActor);
}
if (existingVideo != null)
{
videoRepository.UpdateItem(newVideo);
}
else
{
videoRepository.AddItem(newVideo);
}
videoRepository.SaveChanges();
Debug.WriteLine("PARSE: {0}- {1}", newVideo.Name, DateTime.Now);
try
{
RefreshVideoLinks(newVideo);
}
catch (Exception e)
{
Debug.WriteLine("!!!!!!REFRESH FAILED----{0}----because {1}", name, e);
}
return(newVideo);
}
public void RefreshVideoLinks(Video video)
{
if (video == null)
{
return;
}
var videoEntities = new VideosEntities();
var linkRepository = new Repository <Link, VideosEntities>(videoEntities);
var hostRepository = new Repository <Host, VideosEntities>(videoEntities);
var document = new MyWebClient().GetHtmlDocument(video.Link);
var xPath = "//table[contains(@class,'movie_version')]/tbody/tr/td/span[contains(@class,'quality_')]";
var movieLinks = document.DocumentNode.SelectNodes(xPath);
//delete all existing Links
if (video.Links != null)
{
foreach (Link link in video.Links)
{
linkRepository.Remove(link);
}
}
try
{
foreach (HtmlNode link in movieLinks)
{
var newLink = new Link();
var linkTableRow = link.ParentNode.ParentNode;
var linkUrl = linkTableRow.SelectSingleNode("td[2]/span/a").AttributeValue("href");
var quality = linkTableRow.SelectSingleNode("td[1]/span").AttributeValue("class");
var host = linkTableRow.SelectSingleNode("td[3]/span").InnerTextValue();
//Load Link URL and retrieve redirected url
var urlDoc = new MyWebClient().GetHtmlDocument("http://www.primewire.ag/" + linkUrl);
var redirectedUrl = urlDoc.ResponseUri;
//Extract HostName from javascript statment
Match match = Regex.Match(host, @"'([^']*)");
if (match.Success)
{
host = match.Groups[1].Value;
}
var newHost = hostRepository.Items.FirstOrDefault(x => x.Name == host);
if (newHost == null)
{
newHost = new Host {
Name = host
};
hostRepository.AddItem(newHost);
hostRepository.SaveChanges();
}
newLink.Quality = quality.Replace("quality_", "");
newLink.Host = newHost;
newLink.URL = redirectedUrl;
newLink.Video = video;
linkRepository.AddItem(newLink);
linkRepository.SaveChanges();
Debug.WriteLine("LINK:{0} - {1})", video.Name, DateTime.Now);
TaskDelay();
}
}
catch (Exception e)
{
Debug.WriteLine("!!!!!!URL FAILED----{0}----because {1}", video.Link, e);
}
}
