示例#1
0
        private async Task <List <ILogMessage> > ValidateWithFeedIndexAsync(ISet <PackageIdentity> packages, ISet <PackageIdentity> symbolsPackages)
        {
            var messages = new List <ILogMessage>();

            var feedIndex = new PackageIndex(_context);

            if (await feedIndex.File.ExistsWithFetch(_context.Log, _context.Token))
            {
                var feedPackages = await feedIndex.GetPackagesAsync();

                var feedSymbolsPackages = await feedIndex.GetSymbolsPackagesAsync();

                var extraPackages        = packages.Except(feedPackages);
                var extraSymbolsPackages = symbolsPackages.Except(feedSymbolsPackages);

                var feedDiff = new PackageDiff(Enumerable.Empty <PackageIdentity>(), extraPackages);

                if (feedDiff.HasErrors)
                {
                    var sb = new StringBuilder();
                    sb.AppendLine($"Checking packages in {PackageIndex.File.EntityUri.AbsoluteUri}");
                    sb.Append(feedDiff.ToString());
                    messages.Add(new LogMessage(LogLevel.Error, sb.ToString()));
                }

                var feedSymbolsDiff = new PackageDiff(Enumerable.Empty <PackageIdentity>(), extraSymbolsPackages);

                if (feedSymbolsDiff.HasErrors)
                {
                    var sb = new StringBuilder();
                    sb.AppendLine($"Checking symbols packages in {PackageIndex.File.EntityUri.AbsoluteUri}");
                    sb.Append(feedSymbolsDiff.ToString());
                    messages.Add(new LogMessage(LogLevel.Error, sb.ToString()));
                }
            }

            return(messages);
        }
示例#2
0
        /// <summary>
        /// Validate packages. This does not lock or verify the version of the feed.
        /// </summary>
        public static async Task <bool> Validate(LocalSettings settings, ISleetFileSystem source, ILogger log, CancellationToken token)
        {
            var success = true;

            // Get sleet.settings.json
            var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);

            // Settings context used for all operations
            var context = new SleetContext()
            {
                LocalSettings  = settings,
                SourceSettings = sourceSettings,
                Log            = log,
                Source         = source,
                Token          = token
            };

            // Create all services
            var services = new List <ISleetService>();

            var registrations = new Registrations(context);
            var flatContainer = new FlatContainer(context);
            var search        = new Search(context);
            var autoComplete  = new AutoComplete(context);
            var packageIndex  = new PackageIndex(context);

            if (context.SourceSettings.CatalogEnabled)
            {
                // Add the catalog only if it is enabled
                var catalog = new Catalog(context);
                services.Add(catalog);
            }

            services.Add(registrations);
            services.Add(flatContainer);
            services.Add(search);

            if (context.SourceSettings.SymbolsEnabled)
            {
                var symbols = new Symbols(context);
                services.Add(symbols);
            }

            // Verify against the package index
            var indexedPackages = await packageIndex.GetPackagesAsync();

            var allIndexIds = indexedPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList();

            // Get symbols packages from index
            var indexedSymbolsPackages = await packageIndex.GetSymbolsPackagesAsync();

            var allIndexSymbolsIds = indexedSymbolsPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList();

            // Verify auto complete
            log.LogMinimal($"Validating {autoComplete.Name}");
            var autoCompleteIds = await autoComplete.GetPackageIds();

            var missingACIds = allIndexIds.Except(autoCompleteIds).ToList();
            var extraACIds   = autoCompleteIds.Except(allIndexIds).ToList();

            if (missingACIds.Count() > 0 || extraACIds.Count() > 0)
            {
                log.LogError("Missing autocomplete packages: " + string.Join(", ", missingACIds));
                log.LogError("Extra autocomplete packages: " + string.Join(", ", extraACIds));
                success = false;
            }
            else
            {
                log.LogMinimal("Autocomplete packages valid");
            }

            // Verify everything else
            foreach (var service in services)
            {
                log.LogMinimal($"Validating {service.Name}");

                var validatableService = service as IValidatableService;
                if (validatableService != null)
                {
                    // Run internal validations if the service supports it.
                    var messages = await validatableService.ValidateAsync();

                    success &= messages.All(e => e.Level != LogLevel.Error);

                    foreach (var message in messages)
                    {
                        await log.LogAsync(message);
                    }
                }
                else
                {
                    var allPackagesService = service as IPackagesLookup;
                    var byIdService        = service as IPackageIdLookup;

                    var allSymbolsPackagesService = service as ISymbolsPackagesLookup;
                    var symbolsByIdService        = service as ISymbolsPackageIdLookup;

                    var servicePackages        = new HashSet <PackageIdentity>();
                    var serviceSymbolsPackages = new HashSet <PackageIdentity>();

                    // Non-Symbols packages
                    if (allPackagesService != null)
                    {
                        // Use get all if possible
                        servicePackages.UnionWith(await allPackagesService.GetPackagesAsync());
                    }
                    else if (byIdService != null)
                    {
                        foreach (var id in allIndexIds)
                        {
                            servicePackages.UnionWith(await byIdService.GetPackagesByIdAsync(id));
                        }
                    }
                    else
                    {
                        log.LogError($"Unable to get packages for {service.Name}");
                        continue;
                    }

                    var diff = new PackageDiff(indexedPackages, servicePackages);

                    if (diff.HasErrors)
                    {
                        log.LogError(diff.ToString());

                        success = false;
                    }
                    else
                    {
                        log.LogMinimal(diff.ToString());
                        log.LogMinimal($"{service.Name} packages valid");
                    }

                    // Symbols packages
                    if (allSymbolsPackagesService != null)
                    {
                        // Use get all if possible
                        serviceSymbolsPackages.UnionWith(await allSymbolsPackagesService.GetSymbolsPackagesAsync());
                    }
                    else if (symbolsByIdService != null)
                    {
                        foreach (var id in allIndexSymbolsIds)
                        {
                            serviceSymbolsPackages.UnionWith(await symbolsByIdService.GetSymbolsPackagesByIdAsync(id));
                        }
                    }
                    else
                    {
                        // Symbols are not supported by this service
                        continue;
                    }

                    var symbolsDiff = new PackageDiff(indexedSymbolsPackages, serviceSymbolsPackages);

                    if (symbolsDiff.HasErrors)
                    {
                        log.LogError(symbolsDiff.ToString());
                        success = false;
                    }
                    else
                    {
                        log.LogMinimal(symbolsDiff.ToString());
                        log.LogMinimal($"{service.Name} symbols packages valid");
                    }
                }
            }

            if (success)
            {
                log.LogMinimal($"Feed valid");
            }
            else
            {
                log.LogError($"Feed invalid!");
            }

            return(success);
        }
示例#3
0
        public async Task <IReadOnlyList <ILogMessage> > ValidateAsync()
        {
            var messages      = new List <ILogMessage>();
            var expectedFiles = new HashSet <ISleetFile>
            {
                PackageIndex.File
            };

            var packages = await GetPackagesAsync();

            var symbolsPackages = await GetSymbolsPackagesAsync();

            // Verify no additional packages exist in the index
            messages.AddRange(await ValidateWithFeedIndexAsync(packages, symbolsPackages));

            // De-dupe index files between packages to avoid threading conflicts
            // 1. Find all assemblies
            var assetIndexFiles = new Dictionary <PackageIdentity, AssetIndexFile>();

            foreach (var package in packages.Concat(symbolsPackages))
            {
                if (!assetIndexFiles.ContainsKey(package))
                {
                    assetIndexFiles.Add(package, GetPackageToAssetIndex(package));
                }
            }

            // Retrieve all indexes in parallel
            await Task.WhenAll(assetIndexFiles.Values.Select(e => e.File.FetchAsync(_context.Log, _context.Token)));

            expectedFiles.UnionWith(assetIndexFiles.Select(e => e.Value.File));

            // 2. Build a mapping for every assembly of the parents (symbols and non-symbols).
            var assemblyIndexFiles      = new Dictionary <AssetIndexEntry, PackageIndexFile>();
            var packageAssemblyFiles    = new Dictionary <PackageIdentity, ISet <AssetIndexEntry> >();
            var packageAssemblyFilesRev = new Dictionary <AssetIndexEntry, ISet <PackageIdentity> >();
            var symbolsAssemblyFiles    = new Dictionary <PackageIdentity, ISet <AssetIndexEntry> >();
            var symbolsAssemblyFilesRev = new Dictionary <AssetIndexEntry, ISet <PackageIdentity> >();

            foreach (var package in packages)
            {
                var assetIndex = assetIndexFiles[package];
                var assets     = await assetIndex.GetAssetsAsync();

                packageAssemblyFiles.Add(package, assets);

                foreach (var asset in assets)
                {
                    if (!assemblyIndexFiles.ContainsKey(asset))
                    {
                        var packageIndex = GetPackageIndexFile(asset);
                        assemblyIndexFiles.Add(asset, packageIndex);
                    }

                    if (!packageAssemblyFilesRev.TryGetValue(asset, out var packageSet))
                    {
                        packageSet = new HashSet <PackageIdentity>();
                        packageAssemblyFilesRev.Add(asset, packageSet);
                    }

                    packageSet.Add(package);
                }
            }

            foreach (var package in symbolsPackages)
            {
                var assetIndex = assetIndexFiles[package];
                var assets     = await assetIndex.GetSymbolsAssetsAsync();

                symbolsAssemblyFiles.Add(package, assets);

                foreach (var asset in assets)
                {
                    if (!assemblyIndexFiles.ContainsKey(asset))
                    {
                        var packageIndex = GetPackageIndexFile(asset);
                        assemblyIndexFiles.Add(asset, packageIndex);
                    }

                    if (!symbolsAssemblyFilesRev.TryGetValue(asset, out var packageSet))
                    {
                        packageSet = new HashSet <PackageIdentity>();
                        symbolsAssemblyFilesRev.Add(asset, packageSet);
                    }

                    packageSet.Add(package);
                }
            }

            // Retrieve all indexes in parallel
            await Task.WhenAll(assemblyIndexFiles.Values.Select(e => e.File.FetchAsync(_context.Log, _context.Token)));

            // Get all referenced files
            expectedFiles.UnionWith(assemblyIndexFiles
                                    .SelectMany(e => new[] { e.Key.Asset, e.Key.PackageIndex })
                                    .Select(e => _context.Source.Get(e)));

            // 3. Verify that the assembly -> package index contains the same identities.
            foreach (var asset in assemblyIndexFiles.Keys)
            {
                if (!packageAssemblyFilesRev.TryGetValue(asset, out var toAssembly))
                {
                    toAssembly = new HashSet <PackageIdentity>();
                }

                var fromAssembly = await assemblyIndexFiles[asset].GetPackagesAsync();

                var diff = new PackageDiff(toAssembly, fromAssembly);

                if (diff.HasErrors)
                {
                    var sb = new StringBuilder();
                    sb.AppendLine($"Checking package indexes for {asset.Asset.AbsoluteUri}");
                    sb.Append(diff.ToString());

                    messages.Add(new LogMessage(LogLevel.Error, sb.ToString()));
                }
                else
                {
                    messages.Add(new LogMessage(LogLevel.Verbose, $"Package indexes for {asset.Asset.AbsoluteUri} are valid."));
                }
            }

            foreach (var asset in assemblyIndexFiles.Keys)
            {
                if (!symbolsAssemblyFilesRev.TryGetValue(asset, out var toAssembly))
                {
                    toAssembly = new HashSet <PackageIdentity>();
                }

                var fromAssembly = await assemblyIndexFiles[asset].GetSymbolsPackagesAsync();

                var diff = new PackageDiff(toAssembly, fromAssembly);

                if (diff.HasErrors)
                {
                    var sb = new StringBuilder();
                    sb.AppendLine($"Checking symbols package indexes for {asset.Asset.AbsoluteUri}");
                    sb.Append(diff.ToString());

                    messages.Add(new LogMessage(LogLevel.Error, sb.ToString()));
                }
                else
                {
                    messages.Add(new LogMessage(LogLevel.Verbose, $"Symbols package indexes for {asset.Asset.AbsoluteUri} are valid."));
                }
            }

            // Check that all expected files exist
            var existsTasks = expectedFiles
                              .OrderBy(e => e.EntityUri.AbsoluteUri, StringComparer.Ordinal)
                              .Select(e => new KeyValuePair <ISleetFile, Task <bool> >(e, e.Exists(_context.Log, _context.Token)))
                              .ToList();

            foreach (var existsTask in existsTasks)
            {
                if (await existsTask.Value)
                {
                    messages.Add(new LogMessage(LogLevel.Verbose, $"Found {existsTask.Key.EntityUri.AbsoluteUri}"));
                }
                else
                {
                    messages.Add(new LogMessage(LogLevel.Error, $"Unable to find {existsTask.Key.EntityUri.AbsoluteUri}"));
                }
            }

            return(messages);
        }
示例#4
0
        public static async Task <int> RunCore(LocalSettings settings, ISleetFileSystem source, ILogger log)
        {
            var exitCode = 0;

            var token = CancellationToken.None;

            // Check if already initialized
            using (var feedLock = await SourceUtility.VerifyInitAndLock(source, log, token))
            {
                // Validate source
                await UpgradeUtility.UpgradeIfNeeded(source, log, token);

                // Get sleet.settings.json
                var sourceSettings = new SourceSettings();

                // Settings context used for all operations
                var context = new SleetContext()
                {
                    LocalSettings  = settings,
                    SourceSettings = sourceSettings,
                    Log            = log,
                    Source         = source,
                    Token          = token
                };

                // Create all services
                var catalog       = new Catalog(context);
                var registrations = new Registrations(context);
                var flatContainer = new FlatContainer(context);
                var search        = new Search(context);
                var autoComplete  = new AutoComplete(context);
                var packageIndex  = new PackageIndex(context);

                var services = new List <ISleetService>();
                services.Add(catalog);
                services.Add(registrations);
                services.Add(flatContainer);
                services.Add(search);

                // Verify against the package index
                var indexedPackages = await packageIndex.GetPackages();

                var allIndexIds = indexedPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList();

                // Verify auto complete
                log.LogMinimal($"Validating {autoComplete.Name}");
                var autoCompleteIds = await autoComplete.GetPackageIds();

                var missingACIds = allIndexIds.Except(autoCompleteIds).ToList();
                var extraACIds   = autoCompleteIds.Except(allIndexIds).ToList();

                if (missingACIds.Count() > 0 || extraACIds.Count() > 0)
                {
                    log.LogError("Missing autocomplete packages: " + string.Join(", ", missingACIds));
                    log.LogError("Extra autocomplete packages: " + string.Join(", ", extraACIds));
                    exitCode = 1;
                }
                else
                {
                    log.LogMinimal("Autocomplete packages valid");
                }

                // Verify everything else
                foreach (var service in services)
                {
                    log.LogMinimal($"Validating {service.Name}");

                    var allPackagesService = service as IPackagesLookup;
                    var byIdService        = service as IPackageIdLookup;

                    var servicePackages = new HashSet <PackageIdentity>();

                    // Use get all if possible
                    if (allPackagesService != null)
                    {
                        servicePackages.UnionWith(await allPackagesService.GetPackages());
                    }
                    else if (byIdService != null)
                    {
                        foreach (var id in allIndexIds)
                        {
                            servicePackages.UnionWith(await byIdService.GetPackagesById(id));
                        }
                    }
                    else
                    {
                        log.LogError($"Unable to get packages for {service.Name}");
                        continue;
                    }

                    var diff = new PackageDiff(indexedPackages, servicePackages);

                    if (diff.HasErrors)
                    {
                        log.LogError(diff.ToString());

                        exitCode = 1;
                    }
                    else
                    {
                        log.LogMinimal(diff.ToString());
                        log.LogMinimal($"{service.Name} packages valid");
                    }
                }

                if (exitCode != 0)
                {
                    log.LogError($"Feed invalid!");
                }
                else
                {
                    log.LogMinimal($"Feed valid");
                }
            }

            return(exitCode);
        }