private async Task <List <ILogMessage> > ValidateWithFeedIndexAsync(ISet <PackageIdentity> packages, ISet <PackageIdentity> symbolsPackages)
{
var messages = new List <ILogMessage>();
var feedIndex = new PackageIndex(_context);
if (await feedIndex.File.ExistsWithFetch(_context.Log, _context.Token))
{
var feedPackages = await feedIndex.GetPackagesAsync();
var feedSymbolsPackages = await feedIndex.GetSymbolsPackagesAsync();
var extraPackages = packages.Except(feedPackages);
var extraSymbolsPackages = symbolsPackages.Except(feedSymbolsPackages);
var feedDiff = new PackageDiff(Enumerable.Empty <PackageIdentity>(), extraPackages);
if (feedDiff.HasErrors)
{
var sb = new StringBuilder();
sb.AppendLine($"Checking packages in {PackageIndex.File.EntityUri.AbsoluteUri}");
sb.Append(feedDiff.ToString());
messages.Add(new LogMessage(LogLevel.Error, sb.ToString()));
}
var feedSymbolsDiff = new PackageDiff(Enumerable.Empty <PackageIdentity>(), extraSymbolsPackages);
if (feedSymbolsDiff.HasErrors)
{
var sb = new StringBuilder();
sb.AppendLine($"Checking symbols packages in {PackageIndex.File.EntityUri.AbsoluteUri}");
sb.Append(feedSymbolsDiff.ToString());
messages.Add(new LogMessage(LogLevel.Error, sb.ToString()));
}
}
return(messages);
}
/// <summary>
/// Validate packages. This does not lock or verify the version of the feed.
/// </summary>
public static async Task <bool> Validate(LocalSettings settings, ISleetFileSystem source, ILogger log, CancellationToken token)
{
var success = true;
// Get sleet.settings.json
var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
// Create all services
var services = new List <ISleetService>();
var registrations = new Registrations(context);
var flatContainer = new FlatContainer(context);
var search = new Search(context);
var autoComplete = new AutoComplete(context);
var packageIndex = new PackageIndex(context);
if (context.SourceSettings.CatalogEnabled)
{
// Add the catalog only if it is enabled
var catalog = new Catalog(context);
services.Add(catalog);
}
services.Add(registrations);
services.Add(flatContainer);
services.Add(search);
if (context.SourceSettings.SymbolsEnabled)
{
var symbols = new Symbols(context);
services.Add(symbols);
}
// Verify against the package index
var indexedPackages = await packageIndex.GetPackagesAsync();
var allIndexIds = indexedPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
// Get symbols packages from index
var indexedSymbolsPackages = await packageIndex.GetSymbolsPackagesAsync();
var allIndexSymbolsIds = indexedSymbolsPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
// Verify auto complete
log.LogMinimal($"Validating {autoComplete.Name}");
var autoCompleteIds = await autoComplete.GetPackageIds();
var missingACIds = allIndexIds.Except(autoCompleteIds).ToList();
var extraACIds = autoCompleteIds.Except(allIndexIds).ToList();
if (missingACIds.Count() > 0 || extraACIds.Count() > 0)
{
log.LogError("Missing autocomplete packages: " + string.Join(", ", missingACIds));
log.LogError("Extra autocomplete packages: " + string.Join(", ", extraACIds));
success = false;
}
else
{
log.LogMinimal("Autocomplete packages valid");
}
// Verify everything else
foreach (var service in services)
{
log.LogMinimal($"Validating {service.Name}");
var validatableService = service as IValidatableService;
if (validatableService != null)
{
// Run internal validations if the service supports it.
var messages = await validatableService.ValidateAsync();
success &= messages.All(e => e.Level != LogLevel.Error);
foreach (var message in messages)
{
await log.LogAsync(message);
}
}
else
{
var allPackagesService = service as IPackagesLookup;
var byIdService = service as IPackageIdLookup;
var allSymbolsPackagesService = service as ISymbolsPackagesLookup;
var symbolsByIdService = service as ISymbolsPackageIdLookup;
var servicePackages = new HashSet <PackageIdentity>();
var serviceSymbolsPackages = new HashSet <PackageIdentity>();
// Non-Symbols packages
if (allPackagesService != null)
{
// Use get all if possible
servicePackages.UnionWith(await allPackagesService.GetPackagesAsync());
}
else if (byIdService != null)
{
foreach (var id in allIndexIds)
{
servicePackages.UnionWith(await byIdService.GetPackagesByIdAsync(id));
}
}
else
{
log.LogError($"Unable to get packages for {service.Name}");
continue;
}
var diff = new PackageDiff(indexedPackages, servicePackages);
if (diff.HasErrors)
{
log.LogError(diff.ToString());
success = false;
}
else
{
log.LogMinimal(diff.ToString());
log.LogMinimal($"{service.Name} packages valid");
}
// Symbols packages
if (allSymbolsPackagesService != null)
{
// Use get all if possible
serviceSymbolsPackages.UnionWith(await allSymbolsPackagesService.GetSymbolsPackagesAsync());
}
else if (symbolsByIdService != null)
{
foreach (var id in allIndexSymbolsIds)
{
serviceSymbolsPackages.UnionWith(await symbolsByIdService.GetSymbolsPackagesByIdAsync(id));
}
}
else
{
// Symbols are not supported by this service
continue;
}
var symbolsDiff = new PackageDiff(indexedSymbolsPackages, serviceSymbolsPackages);
if (symbolsDiff.HasErrors)
{
log.LogError(symbolsDiff.ToString());
success = false;
}
else
{
log.LogMinimal(symbolsDiff.ToString());
log.LogMinimal($"{service.Name} symbols packages valid");
}
}
}
if (success)
{
log.LogMinimal($"Feed valid");
}
else
{
log.LogError($"Feed invalid!");
}
return(success);
}
public async Task <IReadOnlyList <ILogMessage> > ValidateAsync()
{
var messages = new List <ILogMessage>();
var expectedFiles = new HashSet <ISleetFile>
{
PackageIndex.File
};
var packages = await GetPackagesAsync();
var symbolsPackages = await GetSymbolsPackagesAsync();
// Verify no additional packages exist in the index
messages.AddRange(await ValidateWithFeedIndexAsync(packages, symbolsPackages));
// De-dupe index files between packages to avoid threading conflicts
// 1. Find all assemblies
var assetIndexFiles = new Dictionary <PackageIdentity, AssetIndexFile>();
foreach (var package in packages.Concat(symbolsPackages))
{
if (!assetIndexFiles.ContainsKey(package))
{
assetIndexFiles.Add(package, GetPackageToAssetIndex(package));
}
}
// Retrieve all indexes in parallel
await Task.WhenAll(assetIndexFiles.Values.Select(e => e.File.FetchAsync(_context.Log, _context.Token)));
expectedFiles.UnionWith(assetIndexFiles.Select(e => e.Value.File));
// 2. Build a mapping for every assembly of the parents (symbols and non-symbols).
var assemblyIndexFiles = new Dictionary <AssetIndexEntry, PackageIndexFile>();
var packageAssemblyFiles = new Dictionary <PackageIdentity, ISet <AssetIndexEntry> >();
var packageAssemblyFilesRev = new Dictionary <AssetIndexEntry, ISet <PackageIdentity> >();
var symbolsAssemblyFiles = new Dictionary <PackageIdentity, ISet <AssetIndexEntry> >();
var symbolsAssemblyFilesRev = new Dictionary <AssetIndexEntry, ISet <PackageIdentity> >();
foreach (var package in packages)
{
var assetIndex = assetIndexFiles[package];
var assets = await assetIndex.GetAssetsAsync();
packageAssemblyFiles.Add(package, assets);
foreach (var asset in assets)
{
if (!assemblyIndexFiles.ContainsKey(asset))
{
var packageIndex = GetPackageIndexFile(asset);
assemblyIndexFiles.Add(asset, packageIndex);
}
if (!packageAssemblyFilesRev.TryGetValue(asset, out var packageSet))
{
packageSet = new HashSet <PackageIdentity>();
packageAssemblyFilesRev.Add(asset, packageSet);
}
packageSet.Add(package);
}
}
foreach (var package in symbolsPackages)
{
var assetIndex = assetIndexFiles[package];
var assets = await assetIndex.GetSymbolsAssetsAsync();
symbolsAssemblyFiles.Add(package, assets);
foreach (var asset in assets)
{
if (!assemblyIndexFiles.ContainsKey(asset))
{
var packageIndex = GetPackageIndexFile(asset);
assemblyIndexFiles.Add(asset, packageIndex);
}
if (!symbolsAssemblyFilesRev.TryGetValue(asset, out var packageSet))
{
packageSet = new HashSet <PackageIdentity>();
symbolsAssemblyFilesRev.Add(asset, packageSet);
}
packageSet.Add(package);
}
}
// Retrieve all indexes in parallel
await Task.WhenAll(assemblyIndexFiles.Values.Select(e => e.File.FetchAsync(_context.Log, _context.Token)));
// Get all referenced files
expectedFiles.UnionWith(assemblyIndexFiles
.SelectMany(e => new[] { e.Key.Asset, e.Key.PackageIndex })
.Select(e => _context.Source.Get(e)));
// 3. Verify that the assembly -> package index contains the same identities.
foreach (var asset in assemblyIndexFiles.Keys)
{
if (!packageAssemblyFilesRev.TryGetValue(asset, out var toAssembly))
{
toAssembly = new HashSet <PackageIdentity>();
}
var fromAssembly = await assemblyIndexFiles[asset].GetPackagesAsync();
var diff = new PackageDiff(toAssembly, fromAssembly);
if (diff.HasErrors)
{
var sb = new StringBuilder();
sb.AppendLine($"Checking package indexes for {asset.Asset.AbsoluteUri}");
sb.Append(diff.ToString());
messages.Add(new LogMessage(LogLevel.Error, sb.ToString()));
}
else
{
messages.Add(new LogMessage(LogLevel.Verbose, $"Package indexes for {asset.Asset.AbsoluteUri} are valid."));
}
}
foreach (var asset in assemblyIndexFiles.Keys)
{
if (!symbolsAssemblyFilesRev.TryGetValue(asset, out var toAssembly))
{
toAssembly = new HashSet <PackageIdentity>();
}
var fromAssembly = await assemblyIndexFiles[asset].GetSymbolsPackagesAsync();
var diff = new PackageDiff(toAssembly, fromAssembly);
if (diff.HasErrors)
{
var sb = new StringBuilder();
sb.AppendLine($"Checking symbols package indexes for {asset.Asset.AbsoluteUri}");
sb.Append(diff.ToString());
messages.Add(new LogMessage(LogLevel.Error, sb.ToString()));
}
else
{
messages.Add(new LogMessage(LogLevel.Verbose, $"Symbols package indexes for {asset.Asset.AbsoluteUri} are valid."));
}
}
// Check that all expected files exist
var existsTasks = expectedFiles
.OrderBy(e => e.EntityUri.AbsoluteUri, StringComparer.Ordinal)
.Select(e => new KeyValuePair <ISleetFile, Task <bool> >(e, e.Exists(_context.Log, _context.Token)))
.ToList();
foreach (var existsTask in existsTasks)
{
if (await existsTask.Value)
{
messages.Add(new LogMessage(LogLevel.Verbose, $"Found {existsTask.Key.EntityUri.AbsoluteUri}"));
}
else
{
messages.Add(new LogMessage(LogLevel.Error, $"Unable to find {existsTask.Key.EntityUri.AbsoluteUri}"));
}
}
return(messages);
}
public static async Task <int> RunCore(LocalSettings settings, ISleetFileSystem source, ILogger log)
{
var exitCode = 0;
var token = CancellationToken.None;
// Check if already initialized
using (var feedLock = await SourceUtility.VerifyInitAndLock(source, log, token))
{
// Validate source
await UpgradeUtility.UpgradeIfNeeded(source, log, token);
// Get sleet.settings.json
var sourceSettings = new SourceSettings();
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
// Create all services
var catalog = new Catalog(context);
var registrations = new Registrations(context);
var flatContainer = new FlatContainer(context);
var search = new Search(context);
var autoComplete = new AutoComplete(context);
var packageIndex = new PackageIndex(context);
var services = new List <ISleetService>();
services.Add(catalog);
services.Add(registrations);
services.Add(flatContainer);
services.Add(search);
// Verify against the package index
var indexedPackages = await packageIndex.GetPackages();
var allIndexIds = indexedPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
// Verify auto complete
log.LogMinimal($"Validating {autoComplete.Name}");
var autoCompleteIds = await autoComplete.GetPackageIds();
var missingACIds = allIndexIds.Except(autoCompleteIds).ToList();
var extraACIds = autoCompleteIds.Except(allIndexIds).ToList();
if (missingACIds.Count() > 0 || extraACIds.Count() > 0)
{
log.LogError("Missing autocomplete packages: " + string.Join(", ", missingACIds));
log.LogError("Extra autocomplete packages: " + string.Join(", ", extraACIds));
exitCode = 1;
}
else
{
log.LogMinimal("Autocomplete packages valid");
}
// Verify everything else
foreach (var service in services)
{
log.LogMinimal($"Validating {service.Name}");
var allPackagesService = service as IPackagesLookup;
var byIdService = service as IPackageIdLookup;
var servicePackages = new HashSet <PackageIdentity>();
// Use get all if possible
if (allPackagesService != null)
{
servicePackages.UnionWith(await allPackagesService.GetPackages());
}
else if (byIdService != null)
{
foreach (var id in allIndexIds)
{
servicePackages.UnionWith(await byIdService.GetPackagesById(id));
}
}
else
{
log.LogError($"Unable to get packages for {service.Name}");
continue;
}
var diff = new PackageDiff(indexedPackages, servicePackages);
if (diff.HasErrors)
{
log.LogError(diff.ToString());
exitCode = 1;
}
else
{
log.LogMinimal(diff.ToString());
log.LogMinimal($"{service.Name} packages valid");
}
}
if (exitCode != 0)
{
log.LogError($"Feed invalid!");
}
else
{
log.LogMinimal($"Feed valid");
}
}
return(exitCode);
}