private async Task <Movie> GetMovieAsync(IStudioClient studioClient, Movie movie)
{
var content = await _scrapperClient.GetAsync(movie.Uri);
if (content.Contains("<h1>PAGE NOT FOUND</h1>"))
{
return(null);
}
var details = await studioClient.ParseDetailsAsync(content, movie.Uri);
movie.Description = details.Description;
movie.Categories = _mapper.Map <IEnumerable <Category> >(details.Categories);
movie.Duration = details.Duration;
if (details.Attachments != null)
{
movie.Attachments = _mapper.Map <IEnumerable <Attachment> >(details.Attachments);
}
if (details.Models != null)
{
movie.Models = _mapper.Map <IEnumerable <Model> >(details.Models);
}
return(movie);
}
public async Task SyncMovieDetailsAsync(IStudioClient studioClient)
{
var stopwatch = Stopwatch.StartNew();
Console.Write($"'{studioClient.StudioName}': retrieving studio details. ");
var studio = await _studioRepository.FindAsync(studioClient.StudioName);
Console.WriteLine($"Studio Id: {studio.StudioId}");
Console.WriteLine("Retrieving movies...");
var movies = (await _movieRepository.FindMoviesWithoutDetailsAsync(studio.StudioId)).ToList();
Console.WriteLine($"Retrieved {movies.Count} movies");
Console.WriteLine("Getting movie details\n");
var buffer = new ConcurrentBag <Movie>();
total += movies.Count;
var errorsCount = 0;
await _concurrentActionHandler.ForeachAsync(movies, async movie =>
{
try
{
Interlocked.Increment(ref counter);
var movieDetails = await GetMovieAsync(studioClient, movie);
if (movieDetails != null)
{
buffer.Add(movieDetails);
if (errorsCount > 0)
{
Interlocked.Decrement(ref errorsCount);
}
}
}
catch (Exception e)
{
Interlocked.Increment(ref errorsCount);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("SCRAPPING ERROR:");
Console.WriteLine(e.Message);
Console.ForegroundColor = ConsoleColor.White;
if (errorsCount == 5)
{
throw;
}
}
finally
{
stopwatch.Stop();
}
}, 4, () => IterationCompleted(buffer));
}
private async Task SyncMoviesAsync(IStudioClient studioClient, int startFrom, int studioId, SyncDetails syncDetails)
{
var buffer = new ConcurrentDictionary <int, IEnumerable <Studios.StudioMovie> >();
//await _concurrentActionHandler.ForAsync(async pageIndex =>
//{
// Console.WriteLine($"Retrieving {studioClient.StudioName}. Page {pageIndex}.");
// var movies = await studioClient.GetMoviesAsync(pageIndex);
// buffer.TryAdd(pageIndex, movies);
//}, startFrom, 0, _syncConfiguration.MaxDegreeOfParallelism, async () => { syncDetails = await SaveAsync(buffer, syncDetails, studioId); });
}
private async Task StartGettingMoviesAsync(IStudioClient studioClient, int start, int end, Func <IEnumerable <Movie>, Task> pagesScraped, IEnumerable <Movie> latest, WebApp.Domain.Entities.SyncDetails syncDetails)
{
var completed = 0;
var cts = new CancellationTokenSource();
await _concurrentActionHandler.ForAsync(
async pageIndex =>
{
Console.WriteLine($"Getting {studioClient.StudioName} - {pageIndex}");
var movies = await studioClient.GetMoviesAsync(pageIndex);
var studioMovies = movies as StudioMovie[] ?? movies.ToArray();
syncDetails.LastSyncPage = pageIndex;
if (latest != null && studioMovies.Any(e => latest.Any(l => l.Uri == e.Uri || l.Date > e.Date)))
{
Interlocked.Increment(ref completed);
Console.WriteLine($"Need to cancel: {studioClient.StudioName}");
var enumerable = latest as Movie[] ?? latest.ToArray();
var date = enumerable.OrderByDescending(e => e.Date).LastOrDefault()?.Date;
var result = studioMovies.Where(e => enumerable.All(l => !UrlsEqual(e.Uri, l.Uri)) && e.Date >= date);
return(result);
}
return(studioMovies);
}, start, end, _syncConfiguration.MaxDegreeOfParallelism, async (result) =>
{
await semaphore.WaitAsync();
if (completed > 0)
{
cts.Cancel();
}
var movies = result.SelectMany(e => e);
var moviesToSave = _mapper.Map <IEnumerable <Movie> >(movies).ToList();
foreach (var movie in moviesToSave)
{
movie.Studio = syncDetails.Studio;
}
await pagesScraped(moviesToSave);
syncDetails.LastSyncDate = DateTime.Now;
await _syncDetailsRepository.UpdateAsync(syncDetails);
semaphore.Release();
}, cts.Token);
}
private async Task <Domain.Entities.Studio> GetStudioAsync(IStudioClient studioClient)
{
var studio = await _studioRepository.FindAsync(studioClient.StudioName);
if (studio == null)
{
studio = new Domain.Entities.Studio
{
Name = studioClient.StudioName
};
studio = await _studioRepository.AddAsync(studio);
}
return(studio);
}
private async Task <SyncDetails> GetSyncDetailsAsync(IStudioClient studioClient)
{
var studio = await GetStudioAsync(studioClient);
var syncDetails = await _syncDetailsRepository.FindByStudioAsync(studio.StudioId);
if (syncDetails == null)
{
syncDetails = await _syncDetailsRepository.AddAsync(new SyncDetails
{
Studio = studio
});
}
return(syncDetails);
}
public async Task SyncAsync(IStudioClient studioClient)
{
var studio = await GetStudioAsync(studioClient);
var syncDetails = await _syncDetailsRepository.FindByStudioAsync(studio.StudioId);
if (syncDetails == null)
{
var startFrom = await studioClient.GetPagesCountAsync();
await SyncMoviesAsync(studioClient, startFrom, studio.StudioId, null);
}
else if (syncDetails.LastSyncPage - 1 >= 1)
{
var startFrom = (int)syncDetails.LastSyncPage - 1;
await SyncMoviesAsync(studioClient, startFrom, studio.StudioId, syncDetails);
}
else
{
await AppendMoviesAsync(studioClient, studio.StudioId);
}
}
private async Task AppendMoviesAsync(IStudioClient studioClient, int studioId)
{
var existingMovies = await _movieRepository.FindLatestAsync(studioId);
var buffer = new ConcurrentDictionary <int, IEnumerable <Studios.StudioMovie> >();
var cts = new CancellationTokenSource();
//await _concurrentActionHandler.ForAsync(async pageIndex =>
//{
// var movies = await studioClient.GetMoviesAsync(pageIndex);
// buffer.TryAdd(pageIndex, movies);
//}, 1, 100, _syncConfiguration.MaxDegreeOfParallelism, async () =>
//{
// var pages = buffer.OrderBy(e => e.Key).ToList();
// buffer.Clear();
// if (pages.Any())
// {
// var studioMovies = pages.SelectMany(e => e.Value);
// studioMovies = studioMovies.Where(studioMovie => existingMovies.All(existingMovie => !string.Equals(existingMovie.Uri, studioMovie.Uri, StringComparison.CurrentCultureIgnoreCase) && existingMovie.Date <= studioMovie.Date));
// if (!studioMovies.Any())
// {
// cts.Cancel();
// Console.WriteLine($"Append Completed;\n\n");
// }
// else
// {
// var moviesToSave = MapMovies(studioMovies, studioId);
// var saved = await _movieRepository.AddRangeAsync(moviesToSave);
// Console.WriteLine($"Append success ({saved.Count()} movies);\n\n");
// }
// }
//}, cts.Token);
}
public SessionsCollabService(IStudioClient client, int pageSize)
{
Client = client;
_PageSize = pageSize;
}
public ProjectsCollabService(IStudioClient client, int pageSize)
{
Client = client;
_PageSize = pageSize;
}
