private async Task <SyncDetails> UpdateSyncDetailsAsync(SyncDetails syncDetails, int studioId, int pageIndex)
{
if (syncDetails == null)
{
syncDetails = new SyncDetails
{
Studio = new Domain.Entities.Studio
{
StudioId = studioId
}
};
}
syncDetails.LastSyncDate = DateTime.UtcNow;
syncDetails.LastSyncPage = pageIndex;
if (syncDetails.SyncDetailsId == 0)
{
var saved = await _syncDetailsRepository.AddAsync(syncDetails);
syncDetails.SyncDetailsId = saved.SyncDetailsId;
}
else
{
await _syncDetailsRepository.UpdateAsync(syncDetails);
}
return(syncDetails);
}
private async Task StartGettingMoviesAsync(IStudioClient studioClient, int start, int end, Func <IEnumerable <Movie>, Task> pagesScraped, IEnumerable <Movie> latest, WebApp.Domain.Entities.SyncDetails syncDetails)
{
var completed = 0;
var cts = new CancellationTokenSource();
await _concurrentActionHandler.ForAsync(
async pageIndex =>
{
Console.WriteLine($"Getting {studioClient.StudioName} - {pageIndex}");
var movies = await studioClient.GetMoviesAsync(pageIndex);
var studioMovies = movies as StudioMovie[] ?? movies.ToArray();
syncDetails.LastSyncPage = pageIndex;
if (latest != null && studioMovies.Any(e => latest.Any(l => l.Uri == e.Uri || l.Date > e.Date)))
{
Interlocked.Increment(ref completed);
Console.WriteLine($"Need to cancel: {studioClient.StudioName}");
var enumerable = latest as Movie[] ?? latest.ToArray();
var date = enumerable.OrderByDescending(e => e.Date).LastOrDefault()?.Date;
var result = studioMovies.Where(e => enumerable.All(l => !UrlsEqual(e.Uri, l.Uri)) && e.Date >= date);
return(result);
}
return(studioMovies);
}, start, end, _syncConfiguration.MaxDegreeOfParallelism, async (result) =>
{
await semaphore.WaitAsync();
if (completed > 0)
{
cts.Cancel();
}
var movies = result.SelectMany(e => e);
var moviesToSave = _mapper.Map <IEnumerable <Movie> >(movies).ToList();
foreach (var movie in moviesToSave)
{
movie.Studio = syncDetails.Studio;
}
await pagesScraped(moviesToSave);
syncDetails.LastSyncDate = DateTime.Now;
await _syncDetailsRepository.UpdateAsync(syncDetails);
semaphore.Release();
}, cts.Token);
}