Esempio n. 1
0
        private async Task <bool> ProcessLeafAsync(CatalogLeafItem leafItem, CancellationToken cancellationToken)
        {
            bool success;

            try
            {
                await _throttle.WaitAsync(cancellationToken);

                if (cancellationToken.IsCancellationRequested)
                {
                    return(false);
                }

                switch (leafItem.Type)
                {
                case CatalogLeafType.PackageDelete:
                    var packageDelete = await _client.GetPackageDeleteLeafAsync(leafItem.Url);

                    success = await _leafProcessor.ProcessPackageDeleteAsync(packageDelete);

                    break;

                case CatalogLeafType.PackageDetails:
                    var packageDetails = await _client.GetPackageDetailsLeafAsync(leafItem.Url);

                    success = await _leafProcessor.ProcessPackageDetailsAsync(packageDetails);

                    break;

                default:
                    throw new NotSupportedException($"The catalog leaf type '{leafItem.Type}' is not supported.");
                }
            }
            catch (Exception exception)
            {
                _logger.LogError(
                    0,
                    exception,
                    "An exception was thrown while processing leaf {leafUrl}.",
                    leafItem.Url);
                success = false;
            }
            finally
            {
                _throttle.Release();
            }

            if (!success)
            {
                _logger.LogWarning(
                    "Failed to process leaf {leafUrl} ({packageId} {packageVersion}, {leafType}).",
                    leafItem.Url,
                    leafItem.PackageId,
                    leafItem.PackageVersion,
                    leafItem.Type);
            }

            return(success);
        }
Esempio n. 2
0
        private async Task ProcessPageAsync(DateTimeOffset minCommitTimestamp, CatalogPageItem pageItem, CancellationToken token)
        {
            var page = await _client.GetPageAsync(pageItem.Url, token);

            var leafItems = page.GetLeavesInBounds(
                minCommitTimestamp,
                _settings.MaxCommitTimestamp,
                _settings.ExcludeRedundantLeaves);

            _logger.LogInformation(
                "On page {page}, {leaves} out of {totalLeaves} were in the time bounds.",
                pageItem.Url,
                leafItems.Count,
                page.Items.Count);

            DateTimeOffset?newCursor = null;

            var tasks = new List <Task <CatalogLeaf> >();

            foreach (var batch in leafItems
                     .Select((v, i) => new { Index = i, Value = v })
                     .GroupBy(v => v.Index / 25)
                     .Select(v => v.Select(p => p.Value)))
            {
                foreach (var leafItem in batch)
                {
                    newCursor = leafItem.CommitTimestamp;

                    tasks.Add(ProcessLeafAsync(leafItem, token));
                }

                await Task.WhenAll(tasks);

                foreach (var task in tasks)
                {
                    if (task.Result is PackageDeleteCatalogLeaf del)
                    {
                        await _leafProcessor.ProcessPackageDeleteAsync(del, token);
                    }
                    else if (task.Result is PackageDetailsCatalogLeaf detail)
                    {
                        await _leafProcessor.ProcessPackageDetailsAsync(detail, token);
                    }
                    else
                    {
                        _logger.LogError("Unsupported leaf type: {type}.", task.Result?.GetType());
                    }
                }

                tasks.Clear();
            }

            if (newCursor.HasValue)
            {
                await _cursor.SetAsync(newCursor.Value, token);
            }
        }
        private async Task <bool> ProcessLeafAsync(CatalogLeafItem leafItem, CancellationToken cancellationToken)
        {
            bool success;

            try
            {
                if (leafItem.IsPackageDelete())
                {
                    var packageDelete = await _client.GetPackageDeleteLeafAsync(leafItem.CatalogLeafUrl);

                    success = await _leafProcessor.ProcessPackageDeleteAsync(packageDelete, cancellationToken);
                }
                else if (leafItem.IsPackageDetails())
                {
                    var packageDetails = await _client.GetPackageDetailsLeafAsync(leafItem.CatalogLeafUrl);

                    success = await _leafProcessor.ProcessPackageDetailsAsync(packageDetails, cancellationToken);
                }
                else
                {
                    throw new NotSupportedException($"The catalog leaf type '{leafItem.Type}' is not supported.");
                }
            }
            catch (Exception exception)
            {
                _logger.LogError(
                    0,
                    exception,
                    "An exception was thrown while processing leaf {leafUrl}.",
                    leafItem.CatalogLeafUrl);
                success = false;
            }

            if (!success)
            {
                _logger.LogWarning(
                    "Failed to process leaf {leafUrl} ({packageId} {packageVersion}, {leafType}).",
                    leafItem.CatalogLeafUrl,
                    leafItem.PackageId,
                    leafItem.PackageVersion,
                    leafItem.Type);
            }

            return(success);
        }