public void ProcessDetailsSingle(int torrentId) { _logger.Info("Detailing torrent #{0}", torrentId); Torrent torrent; using (var db = new DbTorronto()) { torrent = db.Torrent.First(t => t.ID == torrentId); } if (torrent == null) { _logger.Warn("No such torrent #{0}", torrentId); return; } using (var client = GetClient()) { var response = client.GetAsync("torrent/" + torrent.SiteID).Result; if (response.IsSuccessStatusCode) { var content = response.Content.ReadAsStringAsync().Result; var dom = CQ.Create(content, Encoding.UTF8); var links = dom["#details a"]; var title = (dom["title"].Select(x => x.InnerText).FirstOrDefault() ?? string.Empty).ToLower(); int?imdbId, kinopoiskId; FillExternalMovieIds(links, out imdbId, out kinopoiskId); var quality = _qualityDetector.Detect(dom); if (torrent.Category == TorrentCategory.Russian) { quality.TranslationQuality = Translation.Dub; } torrent.ImdbID = imdbId; torrent.KinopoiskID = kinopoiskId; torrent.VideoQuality = quality.VideoQuality; torrent.AudioQuality = quality.AudioQuality; torrent.Translation = quality.TranslationQuality; var isRemoved = title.Contains("не существует"); _torrentService.SaveDetails(torrent, isRemoved); QueueService.AddTorrentForReindex(torrentId); QueueService.AddTorrentForMatch(torrentId); File.WriteAllText(Path.Combine("html", "torrents", torrentId + ".html"), content); } } }
public void UpdateRecommendations() { List <int> unprocessed; using (var db = new DbTorronto()) { unprocessed = db.Movie .Where(x => x.LastRecommendation == null || x.LastRecommendation < DateTime.UtcNow.AddMonths(-6)) .Take(_undetailedMoviesLimit) .Select(x => x.ID.GetValueOrDefault()) .ToList(); } foreach (var movieId in unprocessed) { QueueService.AddMovieForRecommendation(movieId); } }
public void UpdateRatings() { List <int> unprocessed; using (var db = new DbTorronto()) { unprocessed = db.Movie .Where(x => x.RatingLastGather == null || x.RatingLastGather < DateTime.UtcNow.AddDays(-14)) .Take(500) .Select(x => x.ID.GetValueOrDefault()) .ToList(); } foreach (var movieId in unprocessed) { QueueService.AddMovieForRating(movieId); } }
public void SaveMovies(List <Movie> movies, int?userId) { using (var db = new DbTorronto()) { foreach (var movie in movies.Where(m => m.KinopoiskID != null)) { var existing = db.Movie .FirstOrDefault(x => x.KinopoiskID == movie.KinopoiskID); int movieId; if (existing == null) { movieId = CreateMovie(db, movie); if (movieId == 0) { continue; } QueueService.AddMovieForMatch(movieId); } else { movieId = existing.ID.GetValueOrDefault(); db.Movie .Where(x => x.ID == existing.ID) .Set(f => f.Status, movie.Status) .Set(f => f.Updated, DateTime.UtcNow) .Update(); } movie.ID = movieId; if (userId != null && !db.MovieUser.Any(x => x.MovieID == movieId && x.UserID == userId)) { db.InsertMovieUser(movieId, userId.GetValueOrDefault()); } } } }
public void ProcessDetails() { List <int> unprocessed; using (var db = new DbTorronto()) { unprocessed = db.Movie .Where(x => !x.IsDetailed) .OrderByDescending(x => x.Created) .Take(_undetailedMoviesLimit) .Select(x => x.ID.GetValueOrDefault()) .ToList(); } foreach (var movieId in unprocessed) { QueueService.AddMovieForDetails(movieId); QueueService.AddMovieForRating(movieId); } }
private void ProcessDetails() { List <int> unprocessed; using (var db = new DbTorronto()) { //todo move this to proper place db.RefreshTopWeekMovies(); unprocessed = db.Torrent .Where(x => !x.IsDetailed) .OrderByDescending(x => x.Created) .Take(_undetailedTorrentsLimit) .Select(x => x.ID.GetValueOrDefault()) .ToList(); } foreach (var torrentId in unprocessed) { QueueService.AddTorrentForDetails(torrentId); } }
public void GetRecommendations(int movieId) { _logger.Info("GetRecommendations for movie #{0}", movieId); Movie movie; using (var db = new DbTorronto()) { movie = db.Movie.FirstOrDefault(m => m.ID == movieId); } if (movie == null) { _logger.Warn("No such movie #{0}", movieId); return; } if (movie.LastRecommendation != null && movie.LastRecommendation > DateTime.UtcNow.AddMonths(-6)) { return; } using (var client = GetClient()) { client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:33.0) Gecko/20100101 Firefox/33.0"); client.DefaultRequestHeaders.Referrer = new Uri("http://www.kinopoisk.ru/film/" + movie.KinopoiskID); var response = client.GetAsync("film/" + movie.KinopoiskID + "/like").Result; if (response.IsSuccessStatusCode) { var content = response.Content.ReadAsStringAsync().Result; var dom = CQ.Create(content, Encoding.UTF8); var movieItems = dom["td.news a.all"]; var otherMovies = movieItems .Select(item => { var hrefMatch = _movieIdHrefRegex.Match(item.GetAttribute("href")); var title = item.InnerText; return(new Movie { KinopoiskID = hrefMatch.Success ? Convert.ToInt32(hrefMatch.Groups[1].Value) : (int?)null, Title = title ?? "Unknown yet" }); }) .Where(m => m.KinopoiskID != null) .Take(10); using (var db = new DbTorronto()) { var otherMoviePosition = 0; foreach (var otherMovie in otherMovies) { var existingMovie = db.Movie .FirstOrDefault(m => m.KinopoiskID == otherMovie.KinopoiskID); if (existingMovie == null) { existingMovie = new Movie { KinopoiskID = otherMovie.KinopoiskID, Title = otherMovie.Title }; existingMovie.ID = _movieService.CreateMovie(db, existingMovie); QueueService.AddMovieForDetails(existingMovie.ID.GetValueOrDefault()); } db.MovieRecommendation.InsertOrUpdate( () => new MovieRecommendation { MovieID = movie.ID.GetValueOrDefault(), OtherMovieID = existingMovie.ID.GetValueOrDefault(), Position = otherMoviePosition }, old => new MovieRecommendation { Position = otherMoviePosition }); otherMoviePosition++; } db.Movie .Where(m => m.ID == movie.ID) .Set(f => f.LastRecommendation, DateTime.UtcNow) .Update(); } } } }
public void ProcessDetailsSingle(int movieId) { _logger.Info("Detailing movie #{0}", movieId); Movie movie; using (var db = new DbTorronto()) { movie = db.Movie.FirstOrDefault(m => m.ID == movieId); } if (movie == null) { _logger.Warn("No such movie #{0}", movieId); return; } if (movie.IsDetailed) { _logger.Trace("Already detailed movie #{0}", movieId); return; } using (var client = GetClient()) { client.DefaultRequestHeaders.Referrer = new Uri("http://www.kinopoisk.ru/film/" + movie.KinopoiskID); var response = client.GetAsync("film/" + movie.KinopoiskID).Result; if (response.IsSuccessStatusCode) { var content = response.Content.ReadAsStringAsync().Result; File.WriteAllText(Path.Combine("html", "kinopoisk", movieId + ".html"), content); if (content.Contains("www.google.com/recaptcha/api/challenge")) { throw new Exception("Captcha found"); } var dom = CQ.Create(content, Encoding.UTF8); var title = dom .Find(@"#headerFilm h1[itemprop=name]") .Select(x => x.InnerText) .FirstOrDefault(); var originalTitle = dom .Find(@"#headerFilm span[itemprop=alternativeHeadline]") .Select(x => x.InnerText) .FirstOrDefault(); var release = dom .Find(@"#div_world_prem_td2 .prem_ical") .Select(x => x.GetAttribute("data-date-premier-start-link")) .FirstOrDefault(); var description = dom .Find(@".brand_words[itemprop=description]") .Select(x => x.InnerHTML) .FirstOrDefault(); var actors = dom .Find(@"#actorList a") .Select(x => { var attribute = x.GetAttribute("href") ?? string.Empty; var rr = _actorSiteIDRegex.Match(attribute); return(new Person { Name = x.InnerText, SiteID = rr.Success ? Convert.ToInt32(rr.Groups[1].Value) : 0 }); }) .TakeWhile(x => x.Name != "...") .Where(x => x.SiteID != 0); var genres = dom .Find(@"span[itemprop=genre] a") .Select(x => { var attribute = x.GetAttribute("href") ?? string.Empty; var rr = _genreSiteIDRegex.Match(attribute); return(new Genre { Name = x.InnerText, SiteID = rr.Success ? Convert.ToInt32(rr.Groups[1].Value) : 0 }); }) .Where(x => x.SiteID != 0); var duration = dom .Find(@"#runtime") .Select(x => { var r = _durationMinutesRegex.Match(x.InnerHTML ?? string.Empty); return(r.Success ? Convert.ToInt32(r.Groups[1].Value) : 0); }) .FirstOrDefault(); DateTime?releaseDate = null; if (release != null) { var year = Convert.ToInt32(release.Substring(0, 4)); var month = Convert.ToInt32(release.Substring(4, 2)); var day = Convert.ToInt32(release.Substring(6, 2)); if (year > 0 && month > 0 && day > 0) { releaseDate = new DateTime(year, month, day); } } int?imdbId = null; if (movie.ImdbID == null) { imdbId = TryGetImdbId(originalTitle, releaseDate); if (imdbId != null) { _logger.Info("movie #{0}, imdb = {1}", movie.ID, imdbId); } } movie.IsDetailed = true; movie.Title = title; movie.OriginalTitle = originalTitle; movie.ReleaseDate = releaseDate; movie.Description = description; movie.ImdbID = imdbId; movie.DurationMinutes = duration; Movie.AddRelated(movie, actors); Movie.AddRelated(movie, genres); _movieService.SaveDetails(movie); QueueService.AddMovieForReindex(movieId); QueueService.AddMovieForRecommendation(movieId); } } }