/// <summary>
/// Returns the nupkg URI. This will be different for Symbols packages.
/// </summary>
public Uri GetNupkgUri(SleetContext context)
{
if (IsSymbolsPackage)
{
return(context.Source.Get(SymbolsIndexUtility.GetSymbolsNupkgPath(Identity)).EntityUri);
}
return(FlatContainer.GetNupkgPath(context, Identity));
}
/// <summary>
/// Returns the nupkg icon URI..
/// </summary>
public Uri GetIconUri(SleetContext context)
{
Uri result = null;
// Ignore symbol packages, icons from these are not used
if (!IsSymbolsPackage)
{
// Ignore legacy iconURL nuspec properties
if (!string.IsNullOrEmpty(Nuspec.GetIcon()))
{
result = FlatContainer.GetIconPath(context, Identity);
}
}
return(result);
}
/// <summary>
/// Validate packages. This does not lock or verify the version of the feed.
/// </summary>
public static async Task <bool> Validate(LocalSettings settings, ISleetFileSystem source, ILogger log, CancellationToken token)
{
var success = true;
// Get sleet.settings.json
var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
// Create all services
var services = new List <ISleetService>();
var registrations = new Registrations(context);
var flatContainer = new FlatContainer(context);
var search = new Search(context);
var autoComplete = new AutoComplete(context);
var packageIndex = new PackageIndex(context);
if (context.SourceSettings.CatalogEnabled)
{
// Add the catalog only if it is enabled
var catalog = new Catalog(context);
services.Add(catalog);
}
services.Add(registrations);
services.Add(flatContainer);
services.Add(search);
if (context.SourceSettings.SymbolsEnabled)
{
var symbols = new Symbols(context);
services.Add(symbols);
}
// Verify against the package index
var indexedPackages = await packageIndex.GetPackagesAsync();
var allIndexIds = indexedPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
// Get symbols packages from index
var indexedSymbolsPackages = await packageIndex.GetSymbolsPackagesAsync();
var allIndexSymbolsIds = indexedSymbolsPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
// Verify auto complete
log.LogMinimal($"Validating {autoComplete.Name}");
var autoCompleteIds = await autoComplete.GetPackageIds();
var missingACIds = allIndexIds.Except(autoCompleteIds).ToList();
var extraACIds = autoCompleteIds.Except(allIndexIds).ToList();
if (missingACIds.Count() > 0 || extraACIds.Count() > 0)
{
log.LogError("Missing autocomplete packages: " + string.Join(", ", missingACIds));
log.LogError("Extra autocomplete packages: " + string.Join(", ", extraACIds));
success = false;
}
else
{
log.LogMinimal("Autocomplete packages valid");
}
// Verify everything else
foreach (var service in services)
{
log.LogMinimal($"Validating {service.Name}");
var validatableService = service as IValidatableService;
if (validatableService != null)
{
// Run internal validations if the service supports it.
var messages = await validatableService.ValidateAsync();
success &= messages.All(e => e.Level != LogLevel.Error);
foreach (var message in messages)
{
await log.LogAsync(message);
}
}
else
{
var allPackagesService = service as IPackagesLookup;
var byIdService = service as IPackageIdLookup;
var allSymbolsPackagesService = service as ISymbolsPackagesLookup;
var symbolsByIdService = service as ISymbolsPackageIdLookup;
var servicePackages = new HashSet <PackageIdentity>();
var serviceSymbolsPackages = new HashSet <PackageIdentity>();
// Non-Symbols packages
if (allPackagesService != null)
{
// Use get all if possible
servicePackages.UnionWith(await allPackagesService.GetPackagesAsync());
}
else if (byIdService != null)
{
foreach (var id in allIndexIds)
{
servicePackages.UnionWith(await byIdService.GetPackagesByIdAsync(id));
}
}
else
{
log.LogError($"Unable to get packages for {service.Name}");
continue;
}
var diff = new PackageDiff(indexedPackages, servicePackages);
if (diff.HasErrors)
{
log.LogError(diff.ToString());
success = false;
}
else
{
log.LogMinimal(diff.ToString());
log.LogMinimal($"{service.Name} packages valid");
}
// Symbols packages
if (allSymbolsPackagesService != null)
{
// Use get all if possible
serviceSymbolsPackages.UnionWith(await allSymbolsPackagesService.GetSymbolsPackagesAsync());
}
else if (symbolsByIdService != null)
{
foreach (var id in allIndexSymbolsIds)
{
serviceSymbolsPackages.UnionWith(await symbolsByIdService.GetSymbolsPackagesByIdAsync(id));
}
}
else
{
// Symbols are not supported by this service
continue;
}
var symbolsDiff = new PackageDiff(indexedSymbolsPackages, serviceSymbolsPackages);
if (symbolsDiff.HasErrors)
{
log.LogError(symbolsDiff.ToString());
success = false;
}
else
{
log.LogMinimal(symbolsDiff.ToString());
log.LogMinimal($"{service.Name} symbols packages valid");
}
}
}
if (success)
{
log.LogMinimal($"Feed valid");
}
else
{
log.LogError($"Feed invalid!");
}
return(success);
}
/// <summary>
/// Download packages. This method does not lock the feed or verify the client version.
/// </summary>
public static async Task <bool> DownloadPackages(LocalSettings settings, ISleetFileSystem source, string outputPath, bool ignoreErrors, bool skipExisting, ILogger log, CancellationToken token)
{
if (string.IsNullOrEmpty(outputPath))
{
throw new ArgumentException("Missing output path parameter!");
}
var success = true;
// Get sleet.settings.json
var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
log.LogMinimal($"Reading feed {source.BaseURI.AbsoluteUri}");
// Find all packages
var packageIndex = new PackageIndex(context);
var flatContainer = new FlatContainer(context);
var symbols = new Symbols(context);
// Discover all packages
var packages = new List <KeyValuePair <PackageIdentity, ISleetFile> >();
packages.AddRange((await packageIndex.GetPackagesAsync()).Select(e =>
new KeyValuePair <PackageIdentity, ISleetFile>(e, context.Source.Get(flatContainer.GetNupkgPath(e)))));
packages.AddRange((await packageIndex.GetSymbolsPackagesAsync()).Select(e =>
new KeyValuePair <PackageIdentity, ISleetFile>(e, symbols.GetSymbolsNupkgFile(e))));
log.LogMinimal($"Downloading nupkgs to {outputPath}");
// Run downloads
var tasks = packages.Select(e => new Func <Task <bool> >(() => DownloadPackageAsync(outputPath, skipExisting, log, e, token)));
var results = await TaskUtils.RunAsync(tasks, useTaskRun : true, token : CancellationToken.None);
var downloadSuccess = results.All(e => e);
success &= downloadSuccess;
if (packages.Count < 1)
{
log.LogWarning("The feed does not contain any packages.");
}
if (downloadSuccess)
{
if (packages.Count > 0)
{
log.LogMinimal("Successfully downloaded packages.");
}
}
else
{
var message = $"Failed to download all packages!";
if (ignoreErrors)
{
log.LogWarning(message);
}
else
{
log.LogError(message);
}
}
return(success);
}
public static async Task <int> RunCore(LocalSettings settings, ISleetFileSystem source, ILogger log)
{
var exitCode = 0;
var token = CancellationToken.None;
// Check if already initialized
using (var feedLock = await SourceUtility.VerifyInitAndLock(source, log, token))
{
// Validate source
await UpgradeUtility.UpgradeIfNeeded(source, log, token);
// Get sleet.settings.json
var sourceSettings = new SourceSettings();
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
// Create all services
var catalog = new Catalog(context);
var registrations = new Registrations(context);
var flatContainer = new FlatContainer(context);
var search = new Search(context);
var autoComplete = new AutoComplete(context);
var packageIndex = new PackageIndex(context);
var services = new List <ISleetService>();
services.Add(catalog);
services.Add(registrations);
services.Add(flatContainer);
services.Add(search);
// Verify against the package index
var indexedPackages = await packageIndex.GetPackages();
var allIndexIds = indexedPackages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
// Verify auto complete
log.LogMinimal($"Validating {autoComplete.Name}");
var autoCompleteIds = await autoComplete.GetPackageIds();
var missingACIds = allIndexIds.Except(autoCompleteIds).ToList();
var extraACIds = autoCompleteIds.Except(allIndexIds).ToList();
if (missingACIds.Count() > 0 || extraACIds.Count() > 0)
{
log.LogError("Missing autocomplete packages: " + string.Join(", ", missingACIds));
log.LogError("Extra autocomplete packages: " + string.Join(", ", extraACIds));
exitCode = 1;
}
else
{
log.LogMinimal("Autocomplete packages valid");
}
// Verify everything else
foreach (var service in services)
{
log.LogMinimal($"Validating {service.Name}");
var allPackagesService = service as IPackagesLookup;
var byIdService = service as IPackageIdLookup;
var servicePackages = new HashSet <PackageIdentity>();
// Use get all if possible
if (allPackagesService != null)
{
servicePackages.UnionWith(await allPackagesService.GetPackages());
}
else if (byIdService != null)
{
foreach (var id in allIndexIds)
{
servicePackages.UnionWith(await byIdService.GetPackagesById(id));
}
}
else
{
log.LogError($"Unable to get packages for {service.Name}");
continue;
}
var diff = new PackageDiff(indexedPackages, servicePackages);
if (diff.HasErrors)
{
log.LogError(diff.ToString());
exitCode = 1;
}
else
{
log.LogMinimal(diff.ToString());
log.LogMinimal($"{service.Name} packages valid");
}
}
if (exitCode != 0)
{
log.LogError($"Feed invalid!");
}
else
{
log.LogMinimal($"Feed valid");
}
}
return(exitCode);
}
public static async Task <bool?> IsPackageIdenticalOnFeedAsync(string item, PackageIndex packageIndex, ISleetFileSystem source, FlatContainer flatContainer, ILogger log)
{
using (var package = new PackageArchiveReader(item))
{
var id = await package.GetIdentityAsync(CancellationToken.None);
if (await packageIndex.Exists(id))
{
using (var remoteStream = await source.Get(flatContainer.GetNupkgPath(id)).GetStream(log, CancellationToken.None))
using (var remote = new MemoryStream())
{
await remoteStream.CopyToAsync(remote);
var existingBytes = remote.ToArray();
var localBytes = File.ReadAllBytes(item);
return(existingBytes.SequenceEqual(localBytes));
}
}
return(null);
}
}
/// <summary>
/// Download packages. This method does not lock the feed or verify the client version.
/// </summary>
public static async Task <bool> DownloadPackages(LocalSettings settings, ISleetFileSystem source, string outputPath, bool ignoreErrors, ILogger log, CancellationToken token)
{
if (string.IsNullOrEmpty(outputPath))
{
throw new ArgumentException("Missing output path parameter!");
}
var success = true;
// Get sleet.settings.json
var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
log.LogMinimal($"Reading feed {source.BaseURI.AbsoluteUri}");
// Find all packages
var packageIndex = new PackageIndex(context);
var flatContainer = new FlatContainer(context);
var symbols = new Symbols(context);
// Discover all packages
var packages = new List <KeyValuePair <PackageIdentity, ISleetFile> >();
packages.AddRange((await packageIndex.GetPackagesAsync()).Select(e =>
new KeyValuePair <PackageIdentity, ISleetFile>(e, context.Source.Get(flatContainer.GetNupkgPath(e)))));
packages.AddRange((await packageIndex.GetSymbolsPackagesAsync()).Select(e =>
new KeyValuePair <PackageIdentity, ISleetFile>(e, symbols.GetSymbolsNupkgFile(e))));
var tasks = new List <Task <bool> >(MaxThreads);
var downloadSuccess = true;
log.LogMinimal($"Downloading nupkgs to {outputPath}");
foreach (var pair in packages)
{
if (tasks.Count >= MaxThreads)
{
downloadSuccess &= await CompleteTask(tasks);
}
var package = pair.Key;
var nupkgFile = pair.Value;
var fileName = UriUtility.GetFileName(nupkgFile.EntityUri);
// id/id.version.nupkg or id/id.version.symbols.nupkg
var outputNupkgPath = Path.Combine(outputPath,
package.Id.ToLowerInvariant(),
fileName.ToLowerInvariant());
log.LogInformation($"Downloading {outputNupkgPath}");
tasks.Add(nupkgFile.CopyTo(outputNupkgPath, overwrite: true, log: log, token: token));
}
while (tasks.Count > 0)
{
downloadSuccess &= await CompleteTask(tasks);
}
success &= downloadSuccess;
if (packages.Count < 1)
{
log.LogWarning("The feed does not contain any packages.");
}
if (downloadSuccess)
{
if (packages.Count > 0)
{
log.LogMinimal("Successfully downloaded packages.");
}
}
else
{
var message = $"Failed to download all packages!";
if (ignoreErrors)
{
log.LogWarning(message);
}
else
{
log.LogError(message);
}
}
return(success);
}
