Exemple #1
0
        private async Task <List <ILogMessage> > ValidateWithFeedIndexAsync(ISet <PackageIdentity> packages, ISet <PackageIdentity> symbolsPackages)
        {
            var messages = new List <ILogMessage>();

            var feedIndex = new PackageIndex(_context);

            if (await feedIndex.File.ExistsWithFetch(_context.Log, _context.Token))
            {
                var feedPackages = await feedIndex.GetPackagesAsync();

                var feedSymbolsPackages = await feedIndex.GetSymbolsPackagesAsync();

                var extraPackages        = packages.Except(feedPackages);
                var extraSymbolsPackages = symbolsPackages.Except(feedSymbolsPackages);

                var feedDiff = new PackageDiff(Enumerable.Empty <PackageIdentity>(), extraPackages);

                if (feedDiff.HasErrors)
                {
                    var sb = new StringBuilder();
                    sb.AppendLine($"Checking packages in {PackageIndex.File.EntityUri.AbsoluteUri}");
                    sb.Append(feedDiff.ToString());
                    messages.Add(new LogMessage(LogLevel.Error, sb.ToString()));
                }

                var feedSymbolsDiff = new PackageDiff(Enumerable.Empty <PackageIdentity>(), extraSymbolsPackages);

                if (feedSymbolsDiff.HasErrors)
                {
                    var sb = new StringBuilder();
                    sb.AppendLine($"Checking symbols packages in {PackageIndex.File.EntityUri.AbsoluteUri}");
                    sb.Append(feedSymbolsDiff.ToString());
                    messages.Add(new LogMessage(LogLevel.Error, sb.ToString()));
                }
            }

            return(messages);
        }
Exemple #2
0
        /// <summary>
        /// Download packages. This method does not lock the feed or verify the client version.
        /// </summary>
        public static async Task <bool> DownloadPackages(LocalSettings settings, ISleetFileSystem source, string outputPath, bool ignoreErrors, bool skipExisting, ILogger log, CancellationToken token)
        {
            if (string.IsNullOrEmpty(outputPath))
            {
                throw new ArgumentException("Missing output path parameter!");
            }

            var success = true;

            // Get sleet.settings.json
            var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);

            // Settings context used for all operations
            var context = new SleetContext()
            {
                LocalSettings  = settings,
                SourceSettings = sourceSettings,
                Log            = log,
                Source         = source,
                Token          = token
            };

            log.LogMinimal($"Reading feed {source.BaseURI.AbsoluteUri}");

            // Find all packages
            var packageIndex  = new PackageIndex(context);
            var flatContainer = new FlatContainer(context);
            var symbols       = new Symbols(context);

            // Discover all packages
            var packages = new List <KeyValuePair <PackageIdentity, ISleetFile> >();

            packages.AddRange((await packageIndex.GetPackagesAsync()).Select(e =>
                                                                             new KeyValuePair <PackageIdentity, ISleetFile>(e, context.Source.Get(flatContainer.GetNupkgPath(e)))));
            packages.AddRange((await packageIndex.GetSymbolsPackagesAsync()).Select(e =>
                                                                                    new KeyValuePair <PackageIdentity, ISleetFile>(e, symbols.GetSymbolsNupkgFile(e))));

            log.LogMinimal($"Downloading nupkgs to {outputPath}");

            // Run downloads
            var tasks   = packages.Select(e => new Func <Task <bool> >(() => DownloadPackageAsync(outputPath, skipExisting, log, e, token)));
            var results = await TaskUtils.RunAsync(tasks, useTaskRun : true, token : CancellationToken.None);

            var downloadSuccess = results.All(e => e);

            success &= downloadSuccess;

            if (packages.Count < 1)
            {
                log.LogWarning("The feed does not contain any packages.");
            }

            if (downloadSuccess)
            {
                if (packages.Count > 0)
                {
                    log.LogMinimal("Successfully downloaded packages.");
                }
            }
            else
            {
                var message = $"Failed to download all packages!";

                if (ignoreErrors)
                {
                    log.LogWarning(message);
                }
                else
                {
                    log.LogError(message);
                }
            }

            return(success);
        }
Exemple #3
0
        /// <summary>
        /// Validate packages. This does not lock or verify the version of the feed.
        /// </summary>
        public static async Task <bool> Validate(LocalSettings settings, ISleetFileSystem source, ILogger log, CancellationToken token)
        {
            var success = true;

            // Get sleet.settings.json
            var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);

            // Settings context used for all operations
            var context = new SleetContext()
            {
                LocalSettings  = settings,
                SourceSettings = sourceSettings,
                Log            = log,
                Source         = source,
                Token          = token
            };

            // Create all services
            var services = new List <ISleetService>();

            var registrations = new Registrations(context);
            var flatContainer = new FlatContainer(context);
            var search        = new Search(context);
            var autoComplete  = new AutoComplete(context);
            var packageIndex  = new PackageIndex(context);

            if (context.SourceSettings.CatalogEnabled)
            {
                // Add the catalog only if it is enabled
                var catalog = new Catalog(context);
                services.Add(catalog);
            }

            services.Add(registrations);
            services.Add(flatContainer);
            services.Add(search);

            if (context.SourceSettings.SymbolsEnabled)
            {
                var symbols = new Symbols(context);
                services.Add(symbols);
            }

            // Verify against the package index
            var indexedPackages = await packageIndex.GetPackagesAsync();

            var allIndexIds = indexedPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList();

            // Get symbols packages from index
            var indexedSymbolsPackages = await packageIndex.GetSymbolsPackagesAsync();

            var allIndexSymbolsIds = indexedSymbolsPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList();

            // Verify auto complete
            log.LogMinimal($"Validating {autoComplete.Name}");
            var autoCompleteIds = await autoComplete.GetPackageIds();

            var missingACIds = allIndexIds.Except(autoCompleteIds).ToList();
            var extraACIds   = autoCompleteIds.Except(allIndexIds).ToList();

            if (missingACIds.Count() > 0 || extraACIds.Count() > 0)
            {
                log.LogError("Missing autocomplete packages: " + string.Join(", ", missingACIds));
                log.LogError("Extra autocomplete packages: " + string.Join(", ", extraACIds));
                success = false;
            }
            else
            {
                log.LogMinimal("Autocomplete packages valid");
            }

            // Verify everything else
            foreach (var service in services)
            {
                log.LogMinimal($"Validating {service.Name}");

                var validatableService = service as IValidatableService;
                if (validatableService != null)
                {
                    // Run internal validations if the service supports it.
                    var messages = await validatableService.ValidateAsync();

                    success &= messages.All(e => e.Level != LogLevel.Error);

                    foreach (var message in messages)
                    {
                        await log.LogAsync(message);
                    }
                }
                else
                {
                    var allPackagesService = service as IPackagesLookup;
                    var byIdService        = service as IPackageIdLookup;

                    var allSymbolsPackagesService = service as ISymbolsPackagesLookup;
                    var symbolsByIdService        = service as ISymbolsPackageIdLookup;

                    var servicePackages        = new HashSet <PackageIdentity>();
                    var serviceSymbolsPackages = new HashSet <PackageIdentity>();

                    // Non-Symbols packages
                    if (allPackagesService != null)
                    {
                        // Use get all if possible
                        servicePackages.UnionWith(await allPackagesService.GetPackagesAsync());
                    }
                    else if (byIdService != null)
                    {
                        foreach (var id in allIndexIds)
                        {
                            servicePackages.UnionWith(await byIdService.GetPackagesByIdAsync(id));
                        }
                    }
                    else
                    {
                        log.LogError($"Unable to get packages for {service.Name}");
                        continue;
                    }

                    var diff = new PackageDiff(indexedPackages, servicePackages);

                    if (diff.HasErrors)
                    {
                        log.LogError(diff.ToString());

                        success = false;
                    }
                    else
                    {
                        log.LogMinimal(diff.ToString());
                        log.LogMinimal($"{service.Name} packages valid");
                    }

                    // Symbols packages
                    if (allSymbolsPackagesService != null)
                    {
                        // Use get all if possible
                        serviceSymbolsPackages.UnionWith(await allSymbolsPackagesService.GetSymbolsPackagesAsync());
                    }
                    else if (symbolsByIdService != null)
                    {
                        foreach (var id in allIndexSymbolsIds)
                        {
                            serviceSymbolsPackages.UnionWith(await symbolsByIdService.GetSymbolsPackagesByIdAsync(id));
                        }
                    }
                    else
                    {
                        // Symbols are not supported by this service
                        continue;
                    }

                    var symbolsDiff = new PackageDiff(indexedSymbolsPackages, serviceSymbolsPackages);

                    if (symbolsDiff.HasErrors)
                    {
                        log.LogError(symbolsDiff.ToString());
                        success = false;
                    }
                    else
                    {
                        log.LogMinimal(symbolsDiff.ToString());
                        log.LogMinimal($"{service.Name} symbols packages valid");
                    }
                }
            }

            if (success)
            {
                log.LogMinimal($"Feed valid");
            }
            else
            {
                log.LogError($"Feed invalid!");
            }

            return(success);
        }
Exemple #4
0
        /// <summary>
        /// Download packages. This method does not lock the feed or verify the client version.
        /// </summary>
        public static async Task <bool> DownloadPackages(LocalSettings settings, ISleetFileSystem source, string outputPath, bool ignoreErrors, ILogger log, CancellationToken token)
        {
            if (string.IsNullOrEmpty(outputPath))
            {
                throw new ArgumentException("Missing output path parameter!");
            }

            var success = true;

            // Get sleet.settings.json
            var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);

            // Settings context used for all operations
            var context = new SleetContext()
            {
                LocalSettings  = settings,
                SourceSettings = sourceSettings,
                Log            = log,
                Source         = source,
                Token          = token
            };

            log.LogMinimal($"Reading feed {source.BaseURI.AbsoluteUri}");

            // Find all packages
            var packageIndex  = new PackageIndex(context);
            var flatContainer = new FlatContainer(context);
            var symbols       = new Symbols(context);

            // Discover all packages
            var packages = new List <KeyValuePair <PackageIdentity, ISleetFile> >();

            packages.AddRange((await packageIndex.GetPackagesAsync()).Select(e =>
                                                                             new KeyValuePair <PackageIdentity, ISleetFile>(e, context.Source.Get(flatContainer.GetNupkgPath(e)))));
            packages.AddRange((await packageIndex.GetSymbolsPackagesAsync()).Select(e =>
                                                                                    new KeyValuePair <PackageIdentity, ISleetFile>(e, symbols.GetSymbolsNupkgFile(e))));

            var tasks           = new List <Task <bool> >(MaxThreads);
            var downloadSuccess = true;

            log.LogMinimal($"Downloading nupkgs to {outputPath}");

            foreach (var pair in packages)
            {
                if (tasks.Count >= MaxThreads)
                {
                    downloadSuccess &= await CompleteTask(tasks);
                }

                var package   = pair.Key;
                var nupkgFile = pair.Value;

                var fileName = UriUtility.GetFileName(nupkgFile.EntityUri);

                // id/id.version.nupkg or id/id.version.symbols.nupkg
                var outputNupkgPath = Path.Combine(outputPath,
                                                   package.Id.ToLowerInvariant(),
                                                   fileName.ToLowerInvariant());

                log.LogInformation($"Downloading {outputNupkgPath}");

                tasks.Add(nupkgFile.CopyTo(outputNupkgPath, overwrite: true, log: log, token: token));
            }

            while (tasks.Count > 0)
            {
                downloadSuccess &= await CompleteTask(tasks);
            }

            success &= downloadSuccess;

            if (packages.Count < 1)
            {
                log.LogWarning("The feed does not contain any packages.");
            }

            if (downloadSuccess)
            {
                if (packages.Count > 0)
                {
                    log.LogMinimal("Successfully downloaded packages.");
                }
            }
            else
            {
                var message = $"Failed to download all packages!";

                if (ignoreErrors)
                {
                    log.LogWarning(message);
                }
                else
                {
                    log.LogError(message);
                }
            }

            return(success);
        }