public void ProcessDetailsSingle(int torrentId)
{
_logger.Info("Detailing torrent #{0}", torrentId);
Torrent torrent;
using (var db = new DbTorronto())
{
torrent = db.Torrent.First(t => t.ID == torrentId);
}
if (torrent == null)
{
_logger.Warn("No such torrent #{0}", torrentId);
return;
}
using (var client = GetClient())
{
var response = client.GetAsync("torrent/" + torrent.SiteID).Result;
if (response.IsSuccessStatusCode)
{
var content = response.Content.ReadAsStringAsync().Result;
var dom = CQ.Create(content, Encoding.UTF8);
var links = dom["#details a"];
var title = (dom["title"].Select(x => x.InnerText).FirstOrDefault() ?? string.Empty).ToLower();
int?imdbId, kinopoiskId;
FillExternalMovieIds(links, out imdbId, out kinopoiskId);
var quality = _qualityDetector.Detect(dom);
if (torrent.Category == TorrentCategory.Russian)
{
quality.TranslationQuality = Translation.Dub;
}
torrent.ImdbID = imdbId;
torrent.KinopoiskID = kinopoiskId;
torrent.VideoQuality = quality.VideoQuality;
torrent.AudioQuality = quality.AudioQuality;
torrent.Translation = quality.TranslationQuality;
var isRemoved = title.Contains("не существует");
_torrentService.SaveDetails(torrent, isRemoved);
QueueService.AddTorrentForReindex(torrentId);
QueueService.AddTorrentForMatch(torrentId);
File.WriteAllText(Path.Combine("html", "torrents", torrentId + ".html"), content);
}
}
}
public void UpdateRecommendations()
{
List <int> unprocessed;
using (var db = new DbTorronto())
{
unprocessed = db.Movie
.Where(x => x.LastRecommendation == null || x.LastRecommendation < DateTime.UtcNow.AddMonths(-6))
.Take(_undetailedMoviesLimit)
.Select(x => x.ID.GetValueOrDefault())
.ToList();
}
foreach (var movieId in unprocessed)
{
QueueService.AddMovieForRecommendation(movieId);
}
}
public void UpdateRatings()
{
List <int> unprocessed;
using (var db = new DbTorronto())
{
unprocessed = db.Movie
.Where(x => x.RatingLastGather == null || x.RatingLastGather < DateTime.UtcNow.AddDays(-14))
.Take(500)
.Select(x => x.ID.GetValueOrDefault())
.ToList();
}
foreach (var movieId in unprocessed)
{
QueueService.AddMovieForRating(movieId);
}
}
public void SaveMovies(List <Movie> movies, int?userId)
{
using (var db = new DbTorronto())
{
foreach (var movie in movies.Where(m => m.KinopoiskID != null))
{
var existing = db.Movie
.FirstOrDefault(x => x.KinopoiskID == movie.KinopoiskID);
int movieId;
if (existing == null)
{
movieId = CreateMovie(db, movie);
if (movieId == 0)
{
continue;
}
QueueService.AddMovieForMatch(movieId);
}
else
{
movieId = existing.ID.GetValueOrDefault();
db.Movie
.Where(x => x.ID == existing.ID)
.Set(f => f.Status, movie.Status)
.Set(f => f.Updated, DateTime.UtcNow)
.Update();
}
movie.ID = movieId;
if (userId != null && !db.MovieUser.Any(x => x.MovieID == movieId && x.UserID == userId))
{
db.InsertMovieUser(movieId, userId.GetValueOrDefault());
}
}
}
}
public void ProcessDetails()
{
List <int> unprocessed;
using (var db = new DbTorronto())
{
unprocessed = db.Movie
.Where(x => !x.IsDetailed)
.OrderByDescending(x => x.Created)
.Take(_undetailedMoviesLimit)
.Select(x => x.ID.GetValueOrDefault())
.ToList();
}
foreach (var movieId in unprocessed)
{
QueueService.AddMovieForDetails(movieId);
QueueService.AddMovieForRating(movieId);
}
}
private void ProcessDetails()
{
List <int> unprocessed;
using (var db = new DbTorronto())
{
//todo move this to proper place
db.RefreshTopWeekMovies();
unprocessed = db.Torrent
.Where(x => !x.IsDetailed)
.OrderByDescending(x => x.Created)
.Take(_undetailedTorrentsLimit)
.Select(x => x.ID.GetValueOrDefault())
.ToList();
}
foreach (var torrentId in unprocessed)
{
QueueService.AddTorrentForDetails(torrentId);
}
}
public void GetRecommendations(int movieId)
{
_logger.Info("GetRecommendations for movie #{0}", movieId);
Movie movie;
using (var db = new DbTorronto())
{
movie = db.Movie.FirstOrDefault(m => m.ID == movieId);
}
if (movie == null)
{
_logger.Warn("No such movie #{0}", movieId);
return;
}
if (movie.LastRecommendation != null &&
movie.LastRecommendation > DateTime.UtcNow.AddMonths(-6))
{
return;
}
using (var client = GetClient())
{
client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:33.0) Gecko/20100101 Firefox/33.0");
client.DefaultRequestHeaders.Referrer = new Uri("http://www.kinopoisk.ru/film/" + movie.KinopoiskID);
var response = client.GetAsync("film/" + movie.KinopoiskID + "/like").Result;
if (response.IsSuccessStatusCode)
{
var content = response.Content.ReadAsStringAsync().Result;
var dom = CQ.Create(content, Encoding.UTF8);
var movieItems = dom["td.news a.all"];
var otherMovies = movieItems
.Select(item =>
{
var hrefMatch = _movieIdHrefRegex.Match(item.GetAttribute("href"));
var title = item.InnerText;
return(new Movie
{
KinopoiskID = hrefMatch.Success ? Convert.ToInt32(hrefMatch.Groups[1].Value) : (int?)null,
Title = title ?? "Unknown yet"
});
})
.Where(m => m.KinopoiskID != null)
.Take(10);
using (var db = new DbTorronto())
{
var otherMoviePosition = 0;
foreach (var otherMovie in otherMovies)
{
var existingMovie = db.Movie
.FirstOrDefault(m => m.KinopoiskID == otherMovie.KinopoiskID);
if (existingMovie == null)
{
existingMovie = new Movie
{
KinopoiskID = otherMovie.KinopoiskID,
Title = otherMovie.Title
};
existingMovie.ID = _movieService.CreateMovie(db, existingMovie);
QueueService.AddMovieForDetails(existingMovie.ID.GetValueOrDefault());
}
db.MovieRecommendation.InsertOrUpdate(
() => new MovieRecommendation
{
MovieID = movie.ID.GetValueOrDefault(),
OtherMovieID = existingMovie.ID.GetValueOrDefault(),
Position = otherMoviePosition
},
old => new MovieRecommendation
{
Position = otherMoviePosition
});
otherMoviePosition++;
}
db.Movie
.Where(m => m.ID == movie.ID)
.Set(f => f.LastRecommendation, DateTime.UtcNow)
.Update();
}
}
}
}
public void ProcessDetailsSingle(int movieId)
{
_logger.Info("Detailing movie #{0}", movieId);
Movie movie;
using (var db = new DbTorronto())
{
movie = db.Movie.FirstOrDefault(m => m.ID == movieId);
}
if (movie == null)
{
_logger.Warn("No such movie #{0}", movieId);
return;
}
if (movie.IsDetailed)
{
_logger.Trace("Already detailed movie #{0}", movieId);
return;
}
using (var client = GetClient())
{
client.DefaultRequestHeaders.Referrer = new Uri("http://www.kinopoisk.ru/film/" + movie.KinopoiskID);
var response = client.GetAsync("film/" + movie.KinopoiskID).Result;
if (response.IsSuccessStatusCode)
{
var content = response.Content.ReadAsStringAsync().Result;
File.WriteAllText(Path.Combine("html", "kinopoisk", movieId + ".html"), content);
if (content.Contains("www.google.com/recaptcha/api/challenge"))
{
throw new Exception("Captcha found");
}
var dom = CQ.Create(content, Encoding.UTF8);
var title = dom
.Find(@"#headerFilm h1[itemprop=name]")
.Select(x => x.InnerText)
.FirstOrDefault();
var originalTitle = dom
.Find(@"#headerFilm span[itemprop=alternativeHeadline]")
.Select(x => x.InnerText)
.FirstOrDefault();
var release = dom
.Find(@"#div_world_prem_td2 .prem_ical")
.Select(x => x.GetAttribute("data-date-premier-start-link"))
.FirstOrDefault();
var description = dom
.Find(@".brand_words[itemprop=description]")
.Select(x => x.InnerHTML)
.FirstOrDefault();
var actors = dom
.Find(@"#actorList a")
.Select(x =>
{
var attribute = x.GetAttribute("href") ?? string.Empty;
var rr = _actorSiteIDRegex.Match(attribute);
return(new Person
{
Name = x.InnerText,
SiteID = rr.Success ? Convert.ToInt32(rr.Groups[1].Value) : 0
});
})
.TakeWhile(x => x.Name != "...")
.Where(x => x.SiteID != 0);
var genres = dom
.Find(@"span[itemprop=genre] a")
.Select(x =>
{
var attribute = x.GetAttribute("href") ?? string.Empty;
var rr = _genreSiteIDRegex.Match(attribute);
return(new Genre
{
Name = x.InnerText,
SiteID = rr.Success ? Convert.ToInt32(rr.Groups[1].Value) : 0
});
})
.Where(x => x.SiteID != 0);
var duration =
dom
.Find(@"#runtime")
.Select(x =>
{
var r = _durationMinutesRegex.Match(x.InnerHTML ?? string.Empty);
return(r.Success ? Convert.ToInt32(r.Groups[1].Value) : 0);
})
.FirstOrDefault();
DateTime?releaseDate = null;
if (release != null)
{
var year = Convert.ToInt32(release.Substring(0, 4));
var month = Convert.ToInt32(release.Substring(4, 2));
var day = Convert.ToInt32(release.Substring(6, 2));
if (year > 0 && month > 0 && day > 0)
{
releaseDate = new DateTime(year, month, day);
}
}
int?imdbId = null;
if (movie.ImdbID == null)
{
imdbId = TryGetImdbId(originalTitle, releaseDate);
if (imdbId != null)
{
_logger.Info("movie #{0}, imdb = {1}", movie.ID, imdbId);
}
}
movie.IsDetailed = true;
movie.Title = title;
movie.OriginalTitle = originalTitle;
movie.ReleaseDate = releaseDate;
movie.Description = description;
movie.ImdbID = imdbId;
movie.DurationMinutes = duration;
Movie.AddRelated(movie, actors);
Movie.AddRelated(movie, genres);
_movieService.SaveDetails(movie);
QueueService.AddMovieForReindex(movieId);
QueueService.AddMovieForRecommendation(movieId);
}
}
}