private async Task <Movie> GetMovieAsync(IStudioClient studioClient, Movie movie) { var content = await _scrapperClient.GetAsync(movie.Uri); if (content.Contains("<h1>PAGE NOT FOUND</h1>")) { return(null); } var details = await studioClient.ParseDetailsAsync(content, movie.Uri); movie.Description = details.Description; movie.Categories = _mapper.Map <IEnumerable <Category> >(details.Categories); movie.Duration = details.Duration; if (details.Attachments != null) { movie.Attachments = _mapper.Map <IEnumerable <Attachment> >(details.Attachments); } if (details.Models != null) { movie.Models = _mapper.Map <IEnumerable <Model> >(details.Models); } return(movie); }
public async Task SyncMovieDetailsAsync(IStudioClient studioClient) { var stopwatch = Stopwatch.StartNew(); Console.Write($"'{studioClient.StudioName}': retrieving studio details. "); var studio = await _studioRepository.FindAsync(studioClient.StudioName); Console.WriteLine($"Studio Id: {studio.StudioId}"); Console.WriteLine("Retrieving movies..."); var movies = (await _movieRepository.FindMoviesWithoutDetailsAsync(studio.StudioId)).ToList(); Console.WriteLine($"Retrieved {movies.Count} movies"); Console.WriteLine("Getting movie details\n"); var buffer = new ConcurrentBag <Movie>(); total += movies.Count; var errorsCount = 0; await _concurrentActionHandler.ForeachAsync(movies, async movie => { try { Interlocked.Increment(ref counter); var movieDetails = await GetMovieAsync(studioClient, movie); if (movieDetails != null) { buffer.Add(movieDetails); if (errorsCount > 0) { Interlocked.Decrement(ref errorsCount); } } } catch (Exception e) { Interlocked.Increment(ref errorsCount); Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("SCRAPPING ERROR:"); Console.WriteLine(e.Message); Console.ForegroundColor = ConsoleColor.White; if (errorsCount == 5) { throw; } } finally { stopwatch.Stop(); } }, 4, () => IterationCompleted(buffer)); }
private async Task SyncMoviesAsync(IStudioClient studioClient, int startFrom, int studioId, SyncDetails syncDetails) { var buffer = new ConcurrentDictionary <int, IEnumerable <Studios.StudioMovie> >(); //await _concurrentActionHandler.ForAsync(async pageIndex => //{ // Console.WriteLine($"Retrieving {studioClient.StudioName}. Page {pageIndex}."); // var movies = await studioClient.GetMoviesAsync(pageIndex); // buffer.TryAdd(pageIndex, movies); //}, startFrom, 0, _syncConfiguration.MaxDegreeOfParallelism, async () => { syncDetails = await SaveAsync(buffer, syncDetails, studioId); }); }
private async Task StartGettingMoviesAsync(IStudioClient studioClient, int start, int end, Func <IEnumerable <Movie>, Task> pagesScraped, IEnumerable <Movie> latest, WebApp.Domain.Entities.SyncDetails syncDetails) { var completed = 0; var cts = new CancellationTokenSource(); await _concurrentActionHandler.ForAsync( async pageIndex => { Console.WriteLine($"Getting {studioClient.StudioName} - {pageIndex}"); var movies = await studioClient.GetMoviesAsync(pageIndex); var studioMovies = movies as StudioMovie[] ?? movies.ToArray(); syncDetails.LastSyncPage = pageIndex; if (latest != null && studioMovies.Any(e => latest.Any(l => l.Uri == e.Uri || l.Date > e.Date))) { Interlocked.Increment(ref completed); Console.WriteLine($"Need to cancel: {studioClient.StudioName}"); var enumerable = latest as Movie[] ?? latest.ToArray(); var date = enumerable.OrderByDescending(e => e.Date).LastOrDefault()?.Date; var result = studioMovies.Where(e => enumerable.All(l => !UrlsEqual(e.Uri, l.Uri)) && e.Date >= date); return(result); } return(studioMovies); }, start, end, _syncConfiguration.MaxDegreeOfParallelism, async (result) => { await semaphore.WaitAsync(); if (completed > 0) { cts.Cancel(); } var movies = result.SelectMany(e => e); var moviesToSave = _mapper.Map <IEnumerable <Movie> >(movies).ToList(); foreach (var movie in moviesToSave) { movie.Studio = syncDetails.Studio; } await pagesScraped(moviesToSave); syncDetails.LastSyncDate = DateTime.Now; await _syncDetailsRepository.UpdateAsync(syncDetails); semaphore.Release(); }, cts.Token); }
private async Task <Domain.Entities.Studio> GetStudioAsync(IStudioClient studioClient) { var studio = await _studioRepository.FindAsync(studioClient.StudioName); if (studio == null) { studio = new Domain.Entities.Studio { Name = studioClient.StudioName }; studio = await _studioRepository.AddAsync(studio); } return(studio); }
private async Task <SyncDetails> GetSyncDetailsAsync(IStudioClient studioClient) { var studio = await GetStudioAsync(studioClient); var syncDetails = await _syncDetailsRepository.FindByStudioAsync(studio.StudioId); if (syncDetails == null) { syncDetails = await _syncDetailsRepository.AddAsync(new SyncDetails { Studio = studio }); } return(syncDetails); }
public async Task SyncAsync(IStudioClient studioClient) { var studio = await GetStudioAsync(studioClient); var syncDetails = await _syncDetailsRepository.FindByStudioAsync(studio.StudioId); if (syncDetails == null) { var startFrom = await studioClient.GetPagesCountAsync(); await SyncMoviesAsync(studioClient, startFrom, studio.StudioId, null); } else if (syncDetails.LastSyncPage - 1 >= 1) { var startFrom = (int)syncDetails.LastSyncPage - 1; await SyncMoviesAsync(studioClient, startFrom, studio.StudioId, syncDetails); } else { await AppendMoviesAsync(studioClient, studio.StudioId); } }
private async Task AppendMoviesAsync(IStudioClient studioClient, int studioId) { var existingMovies = await _movieRepository.FindLatestAsync(studioId); var buffer = new ConcurrentDictionary <int, IEnumerable <Studios.StudioMovie> >(); var cts = new CancellationTokenSource(); //await _concurrentActionHandler.ForAsync(async pageIndex => //{ // var movies = await studioClient.GetMoviesAsync(pageIndex); // buffer.TryAdd(pageIndex, movies); //}, 1, 100, _syncConfiguration.MaxDegreeOfParallelism, async () => //{ // var pages = buffer.OrderBy(e => e.Key).ToList(); // buffer.Clear(); // if (pages.Any()) // { // var studioMovies = pages.SelectMany(e => e.Value); // studioMovies = studioMovies.Where(studioMovie => existingMovies.All(existingMovie => !string.Equals(existingMovie.Uri, studioMovie.Uri, StringComparison.CurrentCultureIgnoreCase) && existingMovie.Date <= studioMovie.Date)); // if (!studioMovies.Any()) // { // cts.Cancel(); // Console.WriteLine($"Append Completed;\n\n"); // } // else // { // var moviesToSave = MapMovies(studioMovies, studioId); // var saved = await _movieRepository.AddRangeAsync(moviesToSave); // Console.WriteLine($"Append success ({saved.Count()} movies);\n\n"); // } // } //}, cts.Token); }
public SessionsCollabService(IStudioClient client, int pageSize) { Client = client; _PageSize = pageSize; }
public ProjectsCollabService(IStudioClient client, int pageSize) { Client = client; _PageSize = pageSize; }