public async Task StartPhotosSearchQueueAsync(PhotoSearchOptions photoSearchOptions)
        {
            if (IsSearchQueueRunning)
            {
                return;
            }
            try
            {
                IsSearchQueueRunning = true;
                while (!PhotosQueue.IsEmpty)
                {
                    PhotosQueue.TryDequeue(out _);
                }

                var defaultPageSize = photoSearchOptions.PerPage != 0 ? photoSearchOptions.Page : DefaultPageSize;
                var x = await _flickr.PhotosSearchAsync(photoSearchOptions);

                var total = x.Total;

                var count = PagingUtil.CalculateNumberOfPages(total, defaultPageSize);
                var pages = new List <int>();
                for (int i = 1; i <= count; i++)
                {
                    pages.Add(i);
                }

                await AsyncHelper.RunWithMaxDegreeOfConcurrency(30, pages, async page =>
                {
                    //copy options
                    var o = new PhotoSearchOptions();
                    foreach (var property in typeof(PhotoSearchOptions).GetProperties().Where(a => a.CanWrite && a.SetMethod != null && a.SetMethod.IsPublic))
                    {
                        property.SetValue(o, property.GetValue(photoSearchOptions));
                    }

                    o.Page = page;
                    var photoCollection = await _flickr.PhotosSearchAsync(o);
                    foreach (var photo in photoCollection)
                    {
                        PhotosQueue.Enqueue(photo);
                    }
                });
            }
            finally
            {
                IsSearchQueueRunning = false;
            }
        }
Esempio n. 2
0
        public async Task StartPhotosSearchQueueAsync(PhotoSearchOptions photoSearchOptions, CancellationToken cancellation)
        {
            if (IsSearchQueueRunning)
            {
                return;
            }
            ThreadsCount        = 0;
            _flickrQueriesCount = 0;
            try
            {
                IsSearchQueueRunning = true;
                while (!PhotosQueue.IsEmpty)
                {
                    if (cancellation.IsCancellationRequested)
                    {
                        return;
                    }
                    PhotosQueue.TryDequeue(out _);
                }

                var policy = Policy
                             .Handle <HttpRequestException>()
                             .WaitAndRetryAsync(
                    retryCount: RetriesCount,
                    sleepDurationProvider: attempt => TimeSpan.FromMilliseconds(200), // Wait 200ms between each try.
                    onRetry: (exception, calculatedWaitDuration) =>                   // Capture some info for logging!
                {
                    Log.Error("{@Status} ImageGalleryAPI Web Query Error! Retrying after {ex}", "ERROR", exception.InnerException?.Message ?? exception.Message);
                });

                var defaultPageSize = photoSearchOptions.PerPage != 0 ? photoSearchOptions.Page : DefaultPageSize;
                var x = await policy.ExecuteAsync(async() => await _flickr.PhotosSearchAsync(photoSearchOptions));

                _flickrQueriesCount++;
                var total = x.Total;

                var count = PagingUtil.CalculateNumberOfPages(total, defaultPageSize);
                var pages = new List <int>();
                for (int i = 1; i <= count; i++)
                {
                    pages.Add(i);
                }

                await AsyncHelper.RunWithMaxDegreeOfConcurrency(cancellation, 30, pages, async page =>
                {
                    if (cancellation.IsCancellationRequested)
                    {
                        return;
                    }
                    ThreadsCount++;
                    try
                    {
                        //copy options
                        var o = new PhotoSearchOptions();
                        foreach (var property in typeof(PhotoSearchOptions).GetProperties().Where(a => a.CanWrite && a.SetMethod != null && a.SetMethod.IsPublic))
                        {
                            property.SetValue(o, property.GetValue(photoSearchOptions));
                        }

                        o.Page = page;
                        var photoCollection = await policy.ExecuteAsync(async token => await _flickr.PhotosSearchAsync(o), cancellation);
                        _flickrQueriesCount++;
                        foreach (var photo in photoCollection)
                        {
                            if (cancellation.IsCancellationRequested)
                            {
                                return;
                            }
                            PhotosQueue.Enqueue(photo);
                            // photo.PrintPhoto();
                            Log.Information("{@Page} Flickr Enqueue Image Metadata Complete {@PhotoId}|{@Title}|{@LastUpdated}", photoCollection.Page, photo.PhotoId, photo.Title,
                                            photo.LastUpdated);
                        }
                    }
                    catch (Exception ex)
                    {
                        //TODO logging
                    }
                    finally
                    {
                        ThreadsCount--;
                        Debug.WriteLine($"SS: {ThreadsCount}");
                    }
                });
            }
            finally
            {
                IsSearchQueueRunning = false;
            }
        }