public static async Task <int> RunCore(LocalSettings settings, ISleetFileSystem source, ILogger log) { var exitCode = 0; var token = CancellationToken.None; // Check if already initialized using (var feedLock = await SourceUtility.VerifyInitAndLock(source, log, token)) { // Validate source await UpgradeUtility.UpgradeIfNeeded(source, log, token); // Get sleet.settings.json var sourceSettings = new SourceSettings(); // Settings context used for all operations var context = new SleetContext() { LocalSettings = settings, SourceSettings = sourceSettings, Log = log, Source = source, Token = token }; // Create all services var catalog = new Catalog(context); var registrations = new Registrations(context); var flatContainer = new FlatContainer(context); var search = new Search(context); var autoComplete = new AutoComplete(context); var packageIndex = new PackageIndex(context); var services = new List <ISleetService>(); services.Add(catalog); services.Add(registrations); services.Add(flatContainer); services.Add(search); // Verify against the package index var indexedPackages = await packageIndex.GetPackages(); var allIndexIds = indexedPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); // Verify auto complete log.LogMinimal($"Validating {autoComplete.Name}"); var autoCompleteIds = await autoComplete.GetPackageIds(); var missingACIds = allIndexIds.Except(autoCompleteIds).ToList(); var extraACIds = autoCompleteIds.Except(allIndexIds).ToList(); if (missingACIds.Count() > 0 || extraACIds.Count() > 0) { log.LogError("Missing autocomplete packages: " + string.Join(", ", missingACIds)); log.LogError("Extra autocomplete packages: " + string.Join(", ", extraACIds)); exitCode = 1; } else { log.LogMinimal("Autocomplete packages valid"); } // Verify everything else foreach (var service in services) { log.LogMinimal($"Validating {service.Name}"); var allPackagesService = service as IPackagesLookup; var byIdService = service as IPackageIdLookup; var servicePackages = new HashSet <PackageIdentity>(); // Use get all if possible if (allPackagesService != null) { servicePackages.UnionWith(await allPackagesService.GetPackages()); } else if (byIdService != null) { foreach (var id in allIndexIds) { servicePackages.UnionWith(await byIdService.GetPackagesById(id)); } } else { log.LogError($"Unable to get packages for {service.Name}"); continue; } var diff = new PackageDiff(indexedPackages, servicePackages); if (diff.HasErrors) { log.LogError(diff.ToString()); exitCode = 1; } else { log.LogMinimal(diff.ToString()); log.LogMinimal($"{service.Name} packages valid"); } } if (exitCode != 0) { log.LogError($"Feed invalid!"); } else { log.LogMinimal($"Feed valid"); } } return(exitCode); }
/// <summary> /// Validate packages. This does not lock or verify the version of the feed. /// </summary> public static async Task <bool> Validate(LocalSettings settings, ISleetFileSystem source, ILogger log, CancellationToken token) { var success = true; // Get sleet.settings.json var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token); // Settings context used for all operations var context = new SleetContext() { LocalSettings = settings, SourceSettings = sourceSettings, Log = log, Source = source, Token = token }; // Create all services var services = new List <ISleetService>(); var registrations = new Registrations(context); var flatContainer = new FlatContainer(context); var search = new Search(context); var autoComplete = new AutoComplete(context); var packageIndex = new PackageIndex(context); if (context.SourceSettings.CatalogEnabled) { // Add the catalog only if it is enabled var catalog = new Catalog(context); services.Add(catalog); } services.Add(registrations); services.Add(flatContainer); services.Add(search); if (context.SourceSettings.SymbolsEnabled) { var symbols = new Symbols(context); services.Add(symbols); } // Verify against the package index var indexedPackages = await packageIndex.GetPackagesAsync(); var allIndexIds = indexedPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); // Get symbols packages from index var indexedSymbolsPackages = await packageIndex.GetSymbolsPackagesAsync(); var allIndexSymbolsIds = indexedSymbolsPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); // Verify auto complete log.LogMinimal($"Validating {autoComplete.Name}"); var autoCompleteIds = await autoComplete.GetPackageIds(); var missingACIds = allIndexIds.Except(autoCompleteIds).ToList(); var extraACIds = autoCompleteIds.Except(allIndexIds).ToList(); if (missingACIds.Count() > 0 || extraACIds.Count() > 0) { log.LogError("Missing autocomplete packages: " + string.Join(", ", missingACIds)); log.LogError("Extra autocomplete packages: " + string.Join(", ", extraACIds)); success = false; } else { log.LogMinimal("Autocomplete packages valid"); } // Verify everything else foreach (var service in services) { log.LogMinimal($"Validating {service.Name}"); var validatableService = service as IValidatableService; if (validatableService != null) { // Run internal validations if the service supports it. var messages = await validatableService.ValidateAsync(); success &= messages.All(e => e.Level != LogLevel.Error); foreach (var message in messages) { await log.LogAsync(message); } } else { var allPackagesService = service as IPackagesLookup; var byIdService = service as IPackageIdLookup; var allSymbolsPackagesService = service as ISymbolsPackagesLookup; var symbolsByIdService = service as ISymbolsPackageIdLookup; var servicePackages = new HashSet <PackageIdentity>(); var serviceSymbolsPackages = new HashSet <PackageIdentity>(); // Non-Symbols packages if (allPackagesService != null) { // Use get all if possible servicePackages.UnionWith(await allPackagesService.GetPackagesAsync()); } else if (byIdService != null) { foreach (var id in allIndexIds) { servicePackages.UnionWith(await byIdService.GetPackagesByIdAsync(id)); } } else { log.LogError($"Unable to get packages for {service.Name}"); continue; } var diff = new PackageDiff(indexedPackages, servicePackages); if (diff.HasErrors) { log.LogError(diff.ToString()); success = false; } else { log.LogMinimal(diff.ToString()); log.LogMinimal($"{service.Name} packages valid"); } // Symbols packages if (allSymbolsPackagesService != null) { // Use get all if possible serviceSymbolsPackages.UnionWith(await allSymbolsPackagesService.GetSymbolsPackagesAsync()); } else if (symbolsByIdService != null) { foreach (var id in allIndexSymbolsIds) { serviceSymbolsPackages.UnionWith(await symbolsByIdService.GetSymbolsPackagesByIdAsync(id)); } } else { // Symbols are not supported by this service continue; } var symbolsDiff = new PackageDiff(indexedSymbolsPackages, serviceSymbolsPackages); if (symbolsDiff.HasErrors) { log.LogError(symbolsDiff.ToString()); success = false; } else { log.LogMinimal(symbolsDiff.ToString()); log.LogMinimal($"{service.Name} symbols packages valid"); } } } if (success) { log.LogMinimal($"Feed valid"); } else { log.LogError($"Feed invalid!"); } return(success); }
/// <summary> /// Create a result containing all versions of the package. The passed in identity /// may or may not be the latest one that is shown. /// </summary> private async Task <JObject> CreatePackageEntry(PackageIdentity package, bool add) { var packageIndex = new PackageIndex(_context); var versions = await packageIndex.GetPackageVersions(package.Id); if (add) { versions.Add(package.Version); } else { versions.Remove(package.Version); } var latest = versions.Max(); var latestIdentity = new PackageIdentity(package.Id, latest); var packageUri = Registrations.GetPackageUri(_context.Source.BaseURI, latestIdentity); var packageEntry = JsonUtility.Create(packageUri, "Package"); var registrationUri = Registrations.GetIndexUri(_context.Source.BaseURI, package.Id); // Read the catalog entry from the package blob. The catalog may not be enabled. var registrations = new Registrations(_context); var catalogEntry = await registrations.GetCatalogEntryFromPackageBlob(latestIdentity); Debug.Assert(catalogEntry != null); packageEntry.Add("registration", registrationUri.AbsoluteUri); var copyProperties = new[] { "id", "version", "description", "summary", "title", "iconUrl", "licenseUrl", "projectUrl", "tags" }; JsonUtility.CopyProperties(catalogEntry, packageEntry, copyProperties, skipEmpty: false); var copyPropertiesDelimited = new[] { "authors", "owners" }; JsonUtility.CopyDelimitedProperties(catalogEntry, packageEntry, copyPropertiesDelimited, ','); JsonUtility.RequireArrayWithEmptyString(packageEntry, new[] { "tags", "authors" }); packageEntry.Add("totalDownloads", 0); var versionsArray = new JArray(); packageEntry.Add("versions", versionsArray); foreach (var version in versions.OrderBy(v => v)) { var versionIdentity = new PackageIdentity(package.Id, version); var versionUri = Registrations.GetPackageUri(_context.Source.BaseURI, versionIdentity); var versionEntry = JsonUtility.Create(versionUri, "Package"); versionEntry.Add("downloads", 0); versionEntry.Add("version", version.ToFullVersionString()); versionsArray.Add(versionEntry); } return(JsonLDTokenComparer.Format(packageEntry)); }