public async Task StartPhotosSearchQueueAsync(PhotoSearchOptions photoSearchOptions)
{
if (IsSearchQueueRunning)
{
return;
}
try
{
IsSearchQueueRunning = true;
while (!PhotosQueue.IsEmpty)
{
PhotosQueue.TryDequeue(out _);
}
var defaultPageSize = photoSearchOptions.PerPage != 0 ? photoSearchOptions.Page : DefaultPageSize;
var x = await _flickr.PhotosSearchAsync(photoSearchOptions);
var total = x.Total;
var count = PagingUtil.CalculateNumberOfPages(total, defaultPageSize);
var pages = new List <int>();
for (int i = 1; i <= count; i++)
{
pages.Add(i);
}
await AsyncHelper.RunWithMaxDegreeOfConcurrency(30, pages, async page =>
{
//copy options
var o = new PhotoSearchOptions();
foreach (var property in typeof(PhotoSearchOptions).GetProperties().Where(a => a.CanWrite && a.SetMethod != null && a.SetMethod.IsPublic))
{
property.SetValue(o, property.GetValue(photoSearchOptions));
}
o.Page = page;
var photoCollection = await _flickr.PhotosSearchAsync(o);
foreach (var photo in photoCollection)
{
PhotosQueue.Enqueue(photo);
}
});
}
finally
{
IsSearchQueueRunning = false;
}
}
public async Task StartPhotosSearchQueueAsync(PhotoSearchOptions photoSearchOptions, CancellationToken cancellation)
{
if (IsSearchQueueRunning)
{
return;
}
ThreadsCount = 0;
_flickrQueriesCount = 0;
try
{
IsSearchQueueRunning = true;
while (!PhotosQueue.IsEmpty)
{
if (cancellation.IsCancellationRequested)
{
return;
}
PhotosQueue.TryDequeue(out _);
}
var policy = Policy
.Handle <HttpRequestException>()
.WaitAndRetryAsync(
retryCount: RetriesCount,
sleepDurationProvider: attempt => TimeSpan.FromMilliseconds(200), // Wait 200ms between each try.
onRetry: (exception, calculatedWaitDuration) => // Capture some info for logging!
{
Log.Error("{@Status} ImageGalleryAPI Web Query Error! Retrying after {ex}", "ERROR", exception.InnerException?.Message ?? exception.Message);
});
var defaultPageSize = photoSearchOptions.PerPage != 0 ? photoSearchOptions.Page : DefaultPageSize;
var x = await policy.ExecuteAsync(async() => await _flickr.PhotosSearchAsync(photoSearchOptions));
_flickrQueriesCount++;
var total = x.Total;
var count = PagingUtil.CalculateNumberOfPages(total, defaultPageSize);
var pages = new List <int>();
for (int i = 1; i <= count; i++)
{
pages.Add(i);
}
await AsyncHelper.RunWithMaxDegreeOfConcurrency(cancellation, 30, pages, async page =>
{
if (cancellation.IsCancellationRequested)
{
return;
}
ThreadsCount++;
try
{
//copy options
var o = new PhotoSearchOptions();
foreach (var property in typeof(PhotoSearchOptions).GetProperties().Where(a => a.CanWrite && a.SetMethod != null && a.SetMethod.IsPublic))
{
property.SetValue(o, property.GetValue(photoSearchOptions));
}
o.Page = page;
var photoCollection = await policy.ExecuteAsync(async token => await _flickr.PhotosSearchAsync(o), cancellation);
_flickrQueriesCount++;
foreach (var photo in photoCollection)
{
if (cancellation.IsCancellationRequested)
{
return;
}
PhotosQueue.Enqueue(photo);
// photo.PrintPhoto();
Log.Information("{@Page} Flickr Enqueue Image Metadata Complete {@PhotoId}|{@Title}|{@LastUpdated}", photoCollection.Page, photo.PhotoId, photo.Title,
photo.LastUpdated);
}
}
catch (Exception ex)
{
//TODO logging
}
finally
{
ThreadsCount--;
Debug.WriteLine($"SS: {ThreadsCount}");
}
});
}
finally
{
IsSearchQueueRunning = false;
}
}