public static async Task <bool> RunAsync(LocalSettings settings, ISleetFileSystem source, string packageId, string version, string reason, bool force, ILogger log) { var success = true; var token = CancellationToken.None; var now = DateTimeOffset.UtcNow; log.LogMinimal($"Reading feed {source.BaseURI.AbsoluteUri}"); // Check if already initialized using (var feedLock = await SourceUtility.VerifyInitAndLock(settings, source, "Delete", log, token)) { // Validate source await SourceUtility.ValidateFeedForClient(source, log, token); // Get sleet.settings.json var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token); // Settings context used for all operations var context = new SleetContext() { LocalSettings = settings, SourceSettings = sourceSettings, Log = log, Source = source, Token = token }; var packageIndex = new PackageIndex(context); var existingPackageSets = await packageIndex.GetPackageSetsAsync(); var packages = new HashSet <PackageIdentity>(); if (!string.IsNullOrEmpty(version)) { // Delete a single version of the package var packageVersion = NuGetVersion.Parse(version); packages.Add(new PackageIdentity(packageId, packageVersion)); } else { // Delete all versions of the package packages.UnionWith(await existingPackageSets.Packages.GetPackagesByIdAsync(packageId)); packages.UnionWith(await existingPackageSets.Symbols.GetPackagesByIdAsync(packageId)); } if (string.IsNullOrEmpty(reason)) { reason = string.Empty; } var toRemove = new HashSet <PackageIdentity>(); var toRemoveSymbols = new HashSet <PackageIdentity>(); foreach (var package in packages) { var exists = existingPackageSets.Packages.Exists(package); var symbolsExists = existingPackageSets.Symbols.Exists(package); if (!exists && !symbolsExists) { log.LogInformation($"{package.ToString()} does not exist."); if (force) { // ignore failures continue; } else { throw new InvalidOperationException($"Package does not exists: {package.ToString()}"); } } if (exists) { toRemove.Add(package); } if (symbolsExists) { toRemoveSymbols.Add(package); } var message = $"Removing {package.ToString()}"; if (exists && symbolsExists) { message = $"Removing {package.ToString()} and symbols package for {package.ToString()}"; } else if (symbolsExists) { message = $"Removing symbols package {package.ToString()}"; } await log.LogAsync(LogLevel.Information, message); } // Update feed await log.LogAsync(LogLevel.Information, "Removing packages from feed locally"); // Add/Remove packages var changeContext = SleetOperations.CreateDelete(existingPackageSets, toRemove, toRemoveSymbols); await SleetUtility.ApplyPackageChangesAsync(context, changeContext); // Save all log.LogMinimal($"Committing changes to {source.BaseURI.AbsoluteUri}"); success &= await source.Commit(log, token); } if (success) { log.LogMinimal($"Successfully deleted packages."); } else { log.LogError($"Failed to delete packages."); } return(success); }
/// <summary> /// Download packages. This method does not lock the feed or verify the client version. /// </summary> public static async Task <bool> DownloadPackages(LocalSettings settings, ISleetFileSystem source, string outputPath, bool ignoreErrors, ILogger log, CancellationToken token) { if (string.IsNullOrEmpty(outputPath)) { throw new ArgumentException("Missing output path parameter!"); } var success = true; // Get sleet.settings.json var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token); // Settings context used for all operations var context = new SleetContext() { LocalSettings = settings, SourceSettings = sourceSettings, Log = log, Source = source, Token = token }; log.LogMinimal($"Reading feed {source.BaseURI.AbsoluteUri}"); // Find all packages var packageIndex = new PackageIndex(context); var flatContainer = new FlatContainer(context); var symbols = new Symbols(context); // Discover all packages var packages = new List <KeyValuePair <PackageIdentity, ISleetFile> >(); packages.AddRange((await packageIndex.GetPackagesAsync()).Select(e => new KeyValuePair <PackageIdentity, ISleetFile>(e, context.Source.Get(flatContainer.GetNupkgPath(e))))); packages.AddRange((await packageIndex.GetSymbolsPackagesAsync()).Select(e => new KeyValuePair <PackageIdentity, ISleetFile>(e, symbols.GetSymbolsNupkgFile(e)))); var tasks = new List <Task <bool> >(MaxThreads); var downloadSuccess = true; log.LogMinimal($"Downloading nupkgs to {outputPath}"); foreach (var pair in packages) { if (tasks.Count >= MaxThreads) { downloadSuccess &= await CompleteTask(tasks); } var package = pair.Key; var nupkgFile = pair.Value; var fileName = UriUtility.GetFileName(nupkgFile.EntityUri); // id/id.version.nupkg or id/id.version.symbols.nupkg var outputNupkgPath = Path.Combine(outputPath, package.Id.ToLowerInvariant(), fileName.ToLowerInvariant()); log.LogInformation($"Downloading {outputNupkgPath}"); tasks.Add(nupkgFile.CopyTo(outputNupkgPath, overwrite: true, log: log, token: token)); } while (tasks.Count > 0) { downloadSuccess &= await CompleteTask(tasks); } success &= downloadSuccess; if (packages.Count < 1) { log.LogWarning("The feed does not contain any packages."); } if (downloadSuccess) { if (packages.Count > 0) { log.LogMinimal("Successfully downloaded packages."); } } else { var message = $"Failed to download all packages!"; if (ignoreErrors) { log.LogWarning(message); } else { log.LogError(message); } } return(success); }
/// <summary> /// Parses sleet.json to find the source and constructs it. /// </summary> public static async Task <ISleetFileSystem> CreateFileSystemAsync(LocalSettings settings, LocalCache cache, string source) { ISleetFileSystem result = null; var sources = settings.Json["sources"] as JArray; if (sources == null) { throw new ArgumentException("Invalid config. No sources found."); } foreach (var sourceEntry in sources.Select(e => (JObject)e)) { var sourceName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "name"); if (source.Equals(sourceName, StringComparison.OrdinalIgnoreCase)) { var path = JsonUtility.GetValueCaseInsensitive(sourceEntry, "path"); var baseURIString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "baseURI"); var feedSubPath = JsonUtility.GetValueCaseInsensitive(sourceEntry, "feedSubPath"); var type = JsonUtility.GetValueCaseInsensitive(sourceEntry, "type")?.ToLowerInvariant(); string absolutePath; if (path != null && type == "local") { if (settings.Path == null && !Path.IsPathRooted(NuGetUriUtility.GetLocalPath(path))) { throw new ArgumentException("Cannot use a relative 'path' without a sleet.json file."); } var nonEmptyPath = path == "" ? "." : path; var absoluteSettingsPath = NuGetUriUtility.GetAbsolutePath(Directory.GetCurrentDirectory(), settings.Path); var settingsDir = Path.GetDirectoryName(absoluteSettingsPath); absolutePath = NuGetUriUtility.GetAbsolutePath(settingsDir, nonEmptyPath); } else { absolutePath = path; } var pathUri = absolutePath != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(absolutePath)) : null; var baseUri = baseURIString != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(baseURIString)) : pathUri; if (type == "local") { if (pathUri == null) { throw new ArgumentException("Missing path for account."); } result = new PhysicalFileSystem(cache, pathUri, baseUri); } else if (type == "azure") { var connectionString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "connectionString"); var container = JsonUtility.GetValueCaseInsensitive(sourceEntry, "container"); if (string.IsNullOrEmpty(connectionString)) { throw new ArgumentException("Missing connectionString for azure account."); } if (connectionString.Equals(AzureFileSystem.AzureEmptyConnectionString, StringComparison.OrdinalIgnoreCase)) { throw new ArgumentException("Invalid connectionString for azure account."); } if (string.IsNullOrEmpty(container)) { throw new ArgumentException("Missing container for azure account."); } var azureAccount = CloudStorageAccount.Parse(connectionString); if (pathUri == null) { // Get the default url from the container pathUri = AzureUtility.GetContainerPath(azureAccount, container); } if (baseUri == null) { baseUri = pathUri; } result = new AzureFileSystem(cache, pathUri, baseUri, azureAccount, container, feedSubPath); } #if !SLEETLEGACY else if (type == "s3") { var profileName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "profileName"); var accessKeyId = JsonUtility.GetValueCaseInsensitive(sourceEntry, "accessKeyId"); var secretAccessKey = JsonUtility.GetValueCaseInsensitive(sourceEntry, "secretAccessKey"); var bucketName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "bucketName"); var region = JsonUtility.GetValueCaseInsensitive(sourceEntry, "region"); var serviceURL = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serviceURL"); var compress = JsonUtility.GetBoolCaseInsensitive(sourceEntry, "compress", true); if (string.IsNullOrEmpty(bucketName)) { throw new ArgumentException("Missing bucketName for Amazon S3 account."); } if (string.IsNullOrEmpty(region) && string.IsNullOrEmpty(serviceURL)) { throw new ArgumentException("Either 'region' or 'serviceURL' must be specified for an Amazon S3 account"); } if (!string.IsNullOrEmpty(region) && !string.IsNullOrEmpty(serviceURL)) { throw new ArgumentException("Options 'region' and 'serviceURL' cannot be used together"); } AmazonS3Config config = null; if (serviceURL != null) { config = new AmazonS3Config() { ServiceURL = serviceURL, ProxyCredentials = CredentialCache.DefaultNetworkCredentials }; } else { config = new AmazonS3Config() { RegionEndpoint = RegionEndpoint.GetBySystemName(region), ProxyCredentials = CredentialCache.DefaultNetworkCredentials }; } AmazonS3Client amazonS3Client = null; // Load credentials from the current profile if (!string.IsNullOrWhiteSpace(profileName)) { var credFile = new SharedCredentialsFile(); if (credFile.TryGetProfile(profileName, out var profile)) { amazonS3Client = new AmazonS3Client(profile.GetAWSCredentials(profileSource: null), config); } else { throw new ArgumentException($"The specified AWS profileName {profileName} could not be found. The feed must specify a valid profileName for an AWS credentials file. For help on credential files see: https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file"); } } // Load credentials explicitly with an accessKey and secretKey else if ( !string.IsNullOrWhiteSpace(accessKeyId) && !string.IsNullOrWhiteSpace(secretAccessKey)) { amazonS3Client = new AmazonS3Client(new BasicAWSCredentials(accessKeyId, secretAccessKey), config); } // Load credentials from Environment Variables else if ( !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(EnvironmentVariablesAWSCredentials.ENVIRONMENT_VARIABLE_ACCESSKEY)) && !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(EnvironmentVariablesAWSCredentials.ENVIRONMENT_VARIABLE_SECRETKEY))) { amazonS3Client = new AmazonS3Client(new EnvironmentVariablesAWSCredentials(), config); } // Load credentials from an ECS docker container else if ( !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(ECSTaskCredentials.ContainerCredentialsURIEnvVariable))) { amazonS3Client = new AmazonS3Client(new ECSTaskCredentials(), config); } // Assume IAM role else { using (var client = new AmazonSecurityTokenServiceClient(config.RegionEndpoint)) { try { var identity = await client.GetCallerIdentityAsync(new GetCallerIdentityRequest()); } catch (Exception ex) { throw new ArgumentException( "Failed to determine AWS identity - ensure you have an IAM " + "role set, have set up default credentials or have specified a profile/key pair.", ex); } } amazonS3Client = new AmazonS3Client(config); } if (pathUri == null) { // Find the default path pathUri = AmazonS3Utility.GetBucketPath(bucketName, config.RegionEndpoint.SystemName); } if (baseUri == null) { baseUri = pathUri; } result = new AmazonS3FileSystem( cache, pathUri, baseUri, amazonS3Client, bucketName, feedSubPath, compress); } #endif } } return(result); }
public static async Task <bool> PushPackages(LocalSettings settings, ISleetFileSystem source, List <string> inputs, bool force, bool skipExisting, ILogger log, CancellationToken token) { var exitCode = true; var now = DateTimeOffset.UtcNow; var packages = new List <PackageInput>(); try { // Get packages packages.AddRange(GetPackageInputs(inputs, now, log)); // Get sleet.settings.json await log.LogAsync(LogLevel.Minimal, "Reading feed"); var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token); // Settings context used for all operations var context = new SleetContext() { LocalSettings = settings, SourceSettings = sourceSettings, Log = log, Source = source, Token = token }; // Fetch feed await SleetUtility.FetchFeed(context); await log.LogAsync(LogLevel.Information, "Reading existing package index"); var packageIndex = new PackageIndex(context); foreach (var package in packages) { await PushPackage(package, context, packageIndex, force, skipExisting, log); } // Save all await log.LogAsync(LogLevel.Minimal, $"Committing changes to {source.BaseURI.AbsoluteUri}"); await source.Commit(log, token); } finally { // Close all zip readers foreach (var package in packages) { package.Dispose(); } } if (exitCode) { await log.LogAsync(LogLevel.Minimal, "Successfully pushed packages."); } else { await log.LogAsync(LogLevel.Error, "Failed to push packages."); } return(exitCode); }