public static Uri GetId(Uri rootId, string subId)
{
return(UriUtility.CreateUri($"{rootId.AbsoluteUri}#{subId}"));
}
public static Uri GetPackageUri(Uri sourceRoot, PackageIdentity package)
{
return(UriUtility.CreateUri($"{sourceRoot.AbsoluteUri}registration/{package.Id.ToLowerInvariant()}/{package.Version.ToIdentityString().ToLowerInvariant()}.json"));
}
/// <summary>
/// Get the @id of a json page.
/// </summary>
public static Uri GetIdUri(JObject json)
{
var s = json["@id"].ToObject <string>();
return(UriUtility.CreateUri(s));
}
public static Uri GetIndexUri(Uri sourceRoot, string packageId)
{
return(UriUtility.CreateUri($"{sourceRoot.AbsoluteUri}registration/{packageId.ToLowerInvariant()}/index.json"));
}
public Uri GetPackageUri(PackageIdentity package)
{
return(UriUtility.CreateUri($"{_context.Source.BaseURI}registration/{package.Id.ToLowerInvariant()}/{package.Version.ToIdentityString().ToLowerInvariant()}.json"));
}
public Catalog(SleetContext context)
: this(context, UriUtility.GetPath(context.Source.BaseURI, "catalog/"))
{
}
public static Uri AddTrailingSlash(string path)
{
return(UriUtility.CreateUri(path.TrimEnd(new char[] { '/', '\\' }) + '/'));
}
/// <summary>
/// Parses sleet.json to find the source and constructs it.
/// </summary>
public static ISleetFileSystem CreateFileSystem(LocalSettings settings, LocalCache cache, string source)
{
ISleetFileSystem result = null;
var sources = settings.Json["sources"] as JArray;
if (sources == null)
{
throw new ArgumentException("Invalid config. No sources found.");
}
foreach (var sourceEntry in sources.Select(e => (JObject)e))
{
var sourceName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "name");
if (source.Equals(sourceName, StringComparison.OrdinalIgnoreCase))
{
var path = JsonUtility.GetValueCaseInsensitive(sourceEntry, "path");
var baseURIString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "baseURI");
var feedSubPath = JsonUtility.GetValueCaseInsensitive(sourceEntry, "feedSubPath");
var type = JsonUtility.GetValueCaseInsensitive(sourceEntry, "type")?.ToLowerInvariant();
string absolutePath;
if (path != null && type == "local")
{
if (settings.Path == null && !Path.IsPathRooted(NuGetUriUtility.GetLocalPath(path)))
{
throw new ArgumentException("Cannot use a relative 'path' without a sleet.json file.");
}
var nonEmptyPath = path == "" ? "." : path;
var absoluteSettingsPath = NuGetUriUtility.GetAbsolutePath(Directory.GetCurrentDirectory(), settings.Path);
var settingsDir = Path.GetDirectoryName(absoluteSettingsPath);
absolutePath = NuGetUriUtility.GetAbsolutePath(settingsDir, nonEmptyPath);
}
else
{
absolutePath = path;
}
var pathUri = absolutePath != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(absolutePath)) : null;
var baseUri = baseURIString != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(baseURIString)) : pathUri;
if (type == "local")
{
if (pathUri == null)
{
throw new ArgumentException("Missing path for account.");
}
result = new PhysicalFileSystem(cache, pathUri, baseUri);
}
else if (type == "azure")
{
var connectionString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "connectionString");
var container = JsonUtility.GetValueCaseInsensitive(sourceEntry, "container");
if (string.IsNullOrEmpty(connectionString))
{
throw new ArgumentException("Missing connectionString for azure account.");
}
if (connectionString.Equals(AzureFileSystem.AzureEmptyConnectionString, StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentException("Invalid connectionString for azure account.");
}
if (string.IsNullOrEmpty(container))
{
throw new ArgumentException("Missing container for azure account.");
}
var azureAccount = CloudStorageAccount.Parse(connectionString);
if (pathUri == null)
{
// Get the default url from the container
pathUri = AzureUtility.GetContainerPath(azureAccount, container);
}
if (baseUri == null)
{
baseUri = pathUri;
}
result = new AzureFileSystem(cache, pathUri, baseUri, azureAccount, container, feedSubPath);
}
#if !SLEETLEGACY
else if (type == "s3")
{
var accessKeyId = JsonUtility.GetValueCaseInsensitive(sourceEntry, "accessKeyId");
var secretAccessKey = JsonUtility.GetValueCaseInsensitive(sourceEntry, "secretAccessKey");
var profileName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "profileName");
var bucketName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "bucketName");
var region = JsonUtility.GetValueCaseInsensitive(sourceEntry, "region");
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentException("Missing bucketName for Amazon S3 account.");
}
if (string.IsNullOrEmpty(region))
{
throw new ArgumentException("Missing region for Amazon S3 account.");
}
if (string.IsNullOrEmpty(profileName) && string.IsNullOrEmpty(accessKeyId))
{
throw new ArgumentException("Must provide a profileName or accessKeyId and secretAccessKey for Amazon S3 account.");
}
var regionSystemName = RegionEndpoint.GetBySystemName(region);
var config = new AmazonS3Config()
{
RegionEndpoint = regionSystemName,
ProxyCredentials = CredentialCache.DefaultNetworkCredentials
};
AmazonS3Client amazonS3Client = null;
if (string.IsNullOrEmpty(profileName))
{
// Access key in sleet.json
if (string.IsNullOrEmpty(accessKeyId))
{
throw new ArgumentException("Missing accessKeyId for Amazon S3 account.");
}
if (string.IsNullOrEmpty(secretAccessKey))
{
throw new ArgumentException("Missing secretAccessKey for Amazon S3 account.");
}
amazonS3Client = new AmazonS3Client(accessKeyId, secretAccessKey, config);
}
else
{
// Avoid mismatched configs, this would get confusing for users.
if (!string.IsNullOrEmpty(accessKeyId) || !string.IsNullOrEmpty(secretAccessKey))
{
throw new ArgumentException("accessKeyId/secretAccessKey may not be used with profileName. Either use profileName with a credential file containing the access keys, or set the access keys in sleet.json and remove profileName.");
}
// Credential file
var credFile = new SharedCredentialsFile();
if (credFile.TryGetProfile(profileName, out var profile))
{
amazonS3Client = new AmazonS3Client(profile.GetAWSCredentials(profileSource: null), config);
}
else
{
throw new ArgumentException($"The specified AWS profileName {profileName} could not be found. The feed must specify a valid profileName for an AWS credentials file, or accessKeyId and secretAccessKey must be provided. For help on credential files see: https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file");
}
}
if (pathUri == null)
{
// Find the default path
pathUri = AmazonS3Utility.GetBucketPath(bucketName, regionSystemName.SystemName);
}
if (baseUri == null)
{
baseUri = pathUri;
}
result = new AmazonS3FileSystem(
cache,
pathUri,
baseUri,
amazonS3Client,
bucketName,
feedSubPath);
}
#endif
}
}
return(result);
}
public override async Task <IReadOnlyList <ISleetFile> > GetFiles(ILogger log, CancellationToken token)
{
string prefix = null;
var useFlatBlobListing = true;
var blobListingDetails = BlobListingDetails.All;
int? maxResults = null;
// Return all files except feedlock
var blobs = new List <IListBlobItem>();
BlobResultSegment result = null;
do
{
result = await _container.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails, maxResults, result?.ContinuationToken, options : null, operationContext : null);
blobs.AddRange(result.Results);
}while (result.ContinuationToken != null);
// Skip the feed lock, and limit this to the current sub feed.
return(blobs.Where(e => !e.Uri.AbsoluteUri.EndsWith($"/{AzureFileSystemLock.LockFile}"))
.Where(e => string.IsNullOrEmpty(FeedSubPath) || e.Uri.AbsoluteUri.StartsWith(UriUtility.EnsureTrailingSlash(BaseURI).AbsoluteUri, StringComparison.Ordinal))
.Select(e => Get(e.Uri))
.ToList());
}
/// <summary>
/// Add a package to the catalog.
/// </summary>
public async Task AddPackage(PackageInput packageInput)
{
// Create package details page
var packageDetails = CreatePackageDetails(packageInput);
var packageDetailsFile = _context.Source.Get(UriUtility.CreateUri(packageDetails["@id"].ToString()));
await packageDetailsFile.Write(packageDetails, _context.Log, _context.Token);
// Add catalog page entry
var catalogIndexUri = _context.Source.GetPath("/catalog/index.json");
var catalogIndexFile = _context.Source.Get(catalogIndexUri);
var catalogIndexJson = await catalogIndexFile.GetJson(_context.Log, _context.Token);
catalogIndexJson["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
catalogIndexJson["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
var pages = GetItems(catalogIndexJson);
var currentPageUri = GetCurrentPage(catalogIndexJson);
var currentPageFile = _context.Source.Get(currentPageUri);
var pageCommits = new List <JObject>();
if (await currentPageFile.Exists(_context.Log, _context.Token))
{
var currentPageJson = await currentPageFile.GetJson(_context.Log, _context.Token);
pageCommits = GetItems(currentPageJson);
}
else
{
var newPage = JsonUtility.Create(currentPageFile.EntityUri, "CatalogPage");
newPage["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
newPage["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
newPage["count"] = 0;
newPage = JsonLDTokenComparer.Format(newPage);
var pageArray = (JArray)catalogIndexJson["items"];
pageArray.Add(newPage);
// Update pages
pages = GetItems(catalogIndexJson);
}
// Create commit
var pageCommit = JsonUtility.Create(packageDetailsFile.EntityUri, "nuget:PackageDetails");
pageCommit["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
pageCommit["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
pageCommit["nuget:id"] = packageInput.Identity.Id;
pageCommit["nuget:version"] = packageInput.Identity.Version.ToFullVersionString();
pageCommit["sleet:operation"] = "add";
pageCommits.Add(pageCommit);
// Write catalog page
var pageJson = CreateCatalogPage(catalogIndexUri, currentPageUri, pageCommits);
await currentPageFile.Write(pageJson, _context.Log, _context.Token);
// Update index
var pageEntry = pages.Where(e => e["@id"].ToString() == currentPageFile.EntityUri.AbsoluteUri).Single();
pageEntry["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
pageEntry["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
pageEntry["count"] = pageCommits.Count;
catalogIndexJson["count"] = pages.Count;
catalogIndexJson["nuget:lastCreated"] = DateTimeOffset.UtcNow.GetDateString();
catalogIndexJson = JsonLDTokenComparer.Format(catalogIndexJson);
await catalogIndexFile.Write(catalogIndexJson, _context.Log, _context.Token);
}
public Uri GetPath(string relativePath)
{
return(UriUtility.GetPath(BaseURI, relativePath));
}
/// <summary>
/// Create a PackageDetails page that contains all the package information.
/// </summary>
public JObject CreatePackageDetails(PackageInput packageInput)
{
var now = DateTimeOffset.UtcNow;
var package = packageInput.Package;
var nuspec = XDocument.Load(package.GetNuspec());
var nuspecReader = new NuspecReader(nuspec);
var pageId = Guid.NewGuid().ToString().ToLowerInvariant();
var rootUri = UriUtility.CreateUri($"{_context.Source.BaseURI}catalog/data/{pageId}.json");
packageInput.PackageDetailsUri = rootUri;
var json = JsonUtility.Create(rootUri, new List <string>()
{
"PackageDetails", "catalog:Permalink"
});
json.Add("commitId", _context.CommitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", DateTimeOffset.UtcNow.GetDateString());
json.Add("sleet:operation", "add");
var context = JsonUtility.GetContext("Catalog");
json.Add("@context", context);
json.Add("id", packageInput.Identity.Id);
json.Add("version", packageInput.Identity.Version.ToFullVersionString());
json.Add("created", now.GetDateString());
json.Add("lastEdited", "0001-01-01T00:00:00Z");
var copyProperties = new List <string>()
{
"authors",
"copyright",
"description",
"iconUrl",
"projectUrl",
"licenseUrl",
"language",
"summary",
"owners",
"releaseNotes"
};
foreach (var propertyName in copyProperties)
{
json.Add(CreateProperty(propertyName, propertyName, nuspecReader));
}
json.Add("isPrerelease", packageInput.Identity.Version.IsPrerelease);
// Unused?
json.Add("licenseNames", string.Empty);
json.Add("licenseReportUrl", string.Empty);
// All packages are listed
json.Add("listed", true);
var titleValue = GetEntry(nuspecReader, "title");
if (!string.IsNullOrEmpty(titleValue))
{
json.Add("title", titleValue);
}
using (var stream = File.OpenRead(packageInput.PackagePath))
{
using (var sha512 = SHA512.Create())
{
var packageHash = Convert.ToBase64String(sha512.ComputeHash(stream));
json.Add("packageHash", packageHash);
json.Add("packageHashAlgorithm", "SHA512");
}
json.Add("packageSize", stream.Length);
}
json.Add("published", now.GetDateString());
json.Add("requireLicenseAcceptance", GetEntry(nuspecReader, "requireLicenseAcceptance").Equals("true", StringComparison.OrdinalIgnoreCase));
var minVersion = nuspecReader.GetMinClientVersion();
if (minVersion != null)
{
json.Add("minClientVersion", minVersion.ToIdentityString());
}
// Tags
var tagSet = new HashSet <string>(GetEntry(nuspecReader, "tags").Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase);
tagSet.Remove(string.Empty);
var tagArray = new JArray(tagSet);
json.Add("tags", tagArray);
// Framework assemblies
var fwrGroups = nuspecReader.GetFrameworkReferenceGroups();
var fwrArray = new JArray();
json.Add("frameworkAssemblyGroup", fwrArray);
foreach (var group in fwrGroups.OrderBy(e => e.TargetFramework.GetShortFolderName(), StringComparer.OrdinalIgnoreCase))
{
var groupTFM = group.TargetFramework.GetShortFolderName().ToLowerInvariant();
var groupNode = JsonUtility.Create(rootUri, $"frameworkassemblygroup/{groupTFM}".ToLowerInvariant(), "FrameworkAssemblyGroup");
// Leave the framework property out for the 'any' group
if (!group.TargetFramework.IsAny)
{
groupNode.Add("targetFramework", groupTFM);
}
fwrArray.Add(groupNode);
if (group.Items.Any())
{
var assemblyArray = new JArray();
groupNode.Add("assembly", assemblyArray);
foreach (var fwAssembly in group.Items.Distinct().OrderBy(e => e, StringComparer.OrdinalIgnoreCase))
{
assemblyArray.Add(fwAssembly);
}
}
}
// Dependencies
var dependencyGroups = nuspecReader.GetDependencyGroups();
var depArray = new JArray();
json.Add("dependencyGroups", depArray);
foreach (var group in dependencyGroups.OrderBy(e => e.TargetFramework.GetShortFolderName(), StringComparer.OrdinalIgnoreCase))
{
var groupTFM = group.TargetFramework.GetShortFolderName().ToLowerInvariant();
var groupNode = JsonUtility.Create(rootUri, $"dependencygroup/{groupTFM}".ToLowerInvariant(), "PackageDependencyGroup");
// Leave the framework property out for the 'any' group
if (!group.TargetFramework.IsAny)
{
groupNode.Add("targetFramework", groupTFM);
}
depArray.Add(groupNode);
if (group.Packages.Any())
{
var packageArray = new JArray();
groupNode.Add("dependencies", packageArray);
foreach (var depPackage in group.Packages.Distinct().OrderBy(e => e.Id, StringComparer.OrdinalIgnoreCase))
{
var packageNode = JsonUtility.Create(rootUri, $"dependencygroup/{groupTFM}/{depPackage.Id}".ToLowerInvariant(), "PackageDependency");
packageNode.Add("id", depPackage.Id);
packageNode.Add("range", depPackage.VersionRange.ToNormalizedString());
packageArray.Add(packageNode);
}
}
}
json.Add("packageContent", packageInput.NupkgUri.AbsoluteUri);
// add flatcontainer files
var packageEntriesArray = new JArray();
json.Add("packageEntries", packageEntriesArray);
var packageEntryIndex = 0;
foreach (var entry in packageInput.Zip.Entries.OrderBy(e => e.FullName, StringComparer.OrdinalIgnoreCase))
{
var fileEntry = JsonUtility.Create(rootUri, $"packageEntry/{packageEntryIndex}", "packageEntry");
fileEntry.Add("fullName", entry.FullName);
fileEntry.Add("length", entry.Length);
fileEntry.Add("lastWriteTime", entry.LastWriteTime.GetDateString());
packageEntriesArray.Add(fileEntry);
packageEntryIndex++;
}
json.Add("sleet:toolVersion", Constants.SleetVersion.ToFullVersionString());
return(JsonLDTokenComparer.Format(json));
}
/// <summary>
/// Parses sleet.json to find the source and constructs it.
/// </summary>
public static ISleetFileSystem CreateFileSystem(LocalSettings settings, LocalCache cache, string source)
{
ISleetFileSystem result = null;
var sources = settings.Json["sources"] as JArray;
if (sources == null)
{
throw new ArgumentException("Invalid config. No sources found.");
}
if (sources != null)
{
foreach (var sourceEntry in sources)
{
if (source.Equals(sourceEntry["name"]?.ToObject <string>(), StringComparison.OrdinalIgnoreCase))
{
if (string.IsNullOrEmpty(sourceEntry["path"]?.ToString()))
{
throw new ArgumentException("Missing path for account.");
}
var path = sourceEntry["path"]?.ToObject <string>();
var baseURI = sourceEntry["baseURI"]?.ToObject <string>() ?? path;
var type = sourceEntry["type"]?.ToObject <string>().ToLowerInvariant();
if (type == "local")
{
result = new PhysicalFileSystem(cache, UriUtility.CreateUri(path), UriUtility.CreateUri(baseURI));
}
else if (type == "azure")
{
var connectionString = sourceEntry["connectionString"]?.ToObject <string>();
var container = sourceEntry["container"]?.ToObject <string>();
if (string.IsNullOrEmpty(connectionString))
{
throw new ArgumentException("Missing connectionString for azure account.");
}
if (string.IsNullOrEmpty(container))
{
throw new ArgumentException("Missing container for azure account.");
}
var azureAccount = CloudStorageAccount.Parse(connectionString);
result = new AzureFileSystem(cache, UriUtility.CreateUri(path), UriUtility.CreateUri(baseURI), azureAccount, container);
}
}
}
}
return(result);
}
/// <summary>
/// Parses sleet.json to find the source and constructs it.
/// </summary>
public static ISleetFileSystem CreateFileSystem(LocalSettings settings, LocalCache cache, string source)
{
ISleetFileSystem result = null;
var sources = settings.Json["sources"] as JArray;
if (sources == null)
{
throw new ArgumentException("Invalid config. No sources found.");
}
if (sources != null)
{
foreach (var sourceEntry in sources)
{
if (source.Equals(sourceEntry["name"]?.ToObject <string>(), StringComparison.OrdinalIgnoreCase))
{
if (string.IsNullOrEmpty(sourceEntry["path"]?.ToString()))
{
throw new ArgumentException("Missing path for account.");
}
var path = sourceEntry["path"]?.ToObject <string>();
var baseURI = sourceEntry["baseURI"]?.ToObject <string>() ?? path;
var feedSubPath = sourceEntry["feedSubPath"]?.ToObject <string>();
var type = sourceEntry["type"]?.ToObject <string>().ToLowerInvariant();
if (type == "local")
{
result = new PhysicalFileSystem(cache, UriUtility.CreateUri(path), UriUtility.CreateUri(baseURI));
}
else if (type == "azure")
{
var connectionString = sourceEntry["connectionString"]?.ToObject <string>();
var container = sourceEntry["container"]?.ToObject <string>();
if (string.IsNullOrEmpty(connectionString))
{
throw new ArgumentException("Missing connectionString for azure account.");
}
if (connectionString.Equals(AzureFileSystem.AzureEmptyConnectionString, StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentException("Invalid connectionString for azure account.");
}
if (string.IsNullOrEmpty(container))
{
throw new ArgumentException("Missing container for azure account.");
}
var azureAccount = CloudStorageAccount.Parse(connectionString);
result = new AzureFileSystem(cache, UriUtility.CreateUri(path), UriUtility.CreateUri(baseURI), azureAccount, container, feedSubPath);
}
#if !SLEETLEGACY
else if (type == "s3")
{
string accessKeyId = sourceEntry["accessKeyId"]?.ToObject <string>();
string secretAccessKey = sourceEntry["secretAccessKey"]?.ToObject <string>();
string bucketName = sourceEntry["bucketName"]?.ToObject <string>();
string region = sourceEntry["region"]?.ToObject <string>();
if (string.IsNullOrEmpty(accessKeyId))
{
throw new ArgumentException("Missing accessKeyId for Amazon S3 account.");
}
if (string.IsNullOrEmpty(secretAccessKey))
{
throw new ArgumentException("Missing secretAccessKey for Amazon S3 account.");
}
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentException("Missing bucketName for Amazon S3 account.");
}
if (string.IsNullOrEmpty(region))
{
throw new ArgumentException("Missing region for Amazon S3 account.");
}
var amazonS3Client = new AmazonS3Client(
accessKeyId, secretAccessKey, RegionEndpoint.GetBySystemName(region));
result = new AmazonS3FileSystem(
cache,
UriUtility.CreateUri(path),
UriUtility.CreateUri(baseURI),
amazonS3Client,
bucketName,
feedSubPath);
}
#endif
}
}
}
return(result);
}
protected override async Task CopyToSource(ILogger log, CancellationToken token)
{
var absoluteUri = UriUtility.GetPath(RootPath, key);
if (!File.Exists(LocalCacheFile.FullName))
{
if (await FileExistsAsync(client, bucketName, key, token).ConfigureAwait(false))
{
log.LogVerbose($"Removing {absoluteUri}");
await RemoveFileAsync(client, bucketName, key, token).ConfigureAwait(false);
}
else
{
log.LogVerbose($"Skipping {absoluteUri}");
}
return;
}
log.LogVerbose($"Pushing {absoluteUri}");
using (var cache = LocalCacheFile.OpenRead())
{
Stream writeStream = cache;
string contentType = null, contentEncoding = null;
if (key.EndsWith(".nupkg", StringComparison.Ordinal))
{
contentType = "application/zip";
}
else if (key.EndsWith(".xml", StringComparison.Ordinal) ||
key.EndsWith(".nuspec", StringComparison.Ordinal))
{
contentType = "application/xml";
}
else if (key.EndsWith(".svg", StringComparison.Ordinal))
{
contentType = "image/svg+xml";
if (compress)
{
contentEncoding = "gzip";
writeStream = await Utility.GZipAsync(cache);
}
}
else if (key.EndsWith(".json", StringComparison.Ordinal) ||
await JsonUtility.IsJsonAsync(LocalCacheFile.FullName))
{
contentType = "application/json";
if (compress)
{
contentEncoding = "gzip";
writeStream = await JsonUtility.GZipAndMinifyAsync(cache);
}
}
else if (key.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) ||
key.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase))
{
contentType = "application/octet-stream";
}
else
{
log.LogWarning($"Unknown file type: {absoluteUri}");
}
await UploadFileAsync(client, bucketName, key, contentType, contentEncoding, writeStream, serverSideEncryptionMethod, token)
.ConfigureAwait(false);
writeStream.Dispose();
}
}
/// <summary>
/// Parses sleet.json to find the source and constructs it.
/// </summary>
public static async Task <ISleetFileSystem> CreateFileSystemAsync(LocalSettings settings, LocalCache cache, string source)
{
ISleetFileSystem result = null;
var sources = settings.Json["sources"] as JArray;
if (sources == null)
{
throw new ArgumentException("Invalid config. No sources found.");
}
foreach (var sourceEntry in sources.Select(e => (JObject)e))
{
var sourceName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "name");
if (source.Equals(sourceName, StringComparison.OrdinalIgnoreCase))
{
var path = JsonUtility.GetValueCaseInsensitive(sourceEntry, "path");
var baseURIString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "baseURI");
var feedSubPath = JsonUtility.GetValueCaseInsensitive(sourceEntry, "feedSubPath");
var type = JsonUtility.GetValueCaseInsensitive(sourceEntry, "type")?.ToLowerInvariant();
string absolutePath;
if (path != null && type == "local")
{
if (settings.Path == null && !Path.IsPathRooted(NuGetUriUtility.GetLocalPath(path)))
{
throw new ArgumentException("Cannot use a relative 'path' without a sleet.json file.");
}
var nonEmptyPath = path == "" ? "." : path;
var absoluteSettingsPath = NuGetUriUtility.GetAbsolutePath(Directory.GetCurrentDirectory(), settings.Path);
var settingsDir = Path.GetDirectoryName(absoluteSettingsPath);
absolutePath = NuGetUriUtility.GetAbsolutePath(settingsDir, nonEmptyPath);
}
else
{
absolutePath = path;
}
var pathUri = absolutePath != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(absolutePath)) : null;
var baseUri = baseURIString != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(baseURIString)) : pathUri;
if (type == "local")
{
if (pathUri == null)
{
throw new ArgumentException("Missing path for account.");
}
result = new PhysicalFileSystem(cache, pathUri, baseUri);
}
else if (type == "azure")
{
var connectionString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "connectionString");
var container = JsonUtility.GetValueCaseInsensitive(sourceEntry, "container");
if (string.IsNullOrEmpty(connectionString))
{
throw new ArgumentException("Missing connectionString for azure account.");
}
if (connectionString.Equals(AzureFileSystem.AzureEmptyConnectionString, StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentException("Invalid connectionString for azure account.");
}
if (string.IsNullOrEmpty(container))
{
throw new ArgumentException("Missing container for azure account.");
}
var azureAccount = CloudStorageAccount.Parse(connectionString);
if (pathUri == null)
{
// Get the default url from the container
pathUri = AzureUtility.GetContainerPath(azureAccount, container);
}
if (baseUri == null)
{
baseUri = pathUri;
}
result = new AzureFileSystem(cache, pathUri, baseUri, azureAccount, container, feedSubPath);
}
else if (type == "s3")
{
var profileName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "profileName");
var accessKeyId = JsonUtility.GetValueCaseInsensitive(sourceEntry, "accessKeyId");
var secretAccessKey = JsonUtility.GetValueCaseInsensitive(sourceEntry, "secretAccessKey");
var bucketName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "bucketName");
var region = JsonUtility.GetValueCaseInsensitive(sourceEntry, "region");
var serviceURL = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serviceURL");
var serverSideEncryptionMethod = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serverSideEncryptionMethod") ?? "None";
var compress = JsonUtility.GetBoolCaseInsensitive(sourceEntry, "compress", true);
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentException("Missing bucketName for Amazon S3 account.");
}
if (string.IsNullOrEmpty(region) && string.IsNullOrEmpty(serviceURL))
{
throw new ArgumentException("Either 'region' or 'serviceURL' must be specified for an Amazon S3 account");
}
if (!string.IsNullOrEmpty(region) && !string.IsNullOrEmpty(serviceURL))
{
throw new ArgumentException("Options 'region' and 'serviceURL' cannot be used together");
}
if (serverSideEncryptionMethod != "None" && serverSideEncryptionMethod != "AES256")
{
throw new ArgumentException("Only 'None' or 'AES256' are currently supported for serverSideEncryptionMethod");
}
// Use the SDK value
var serverSideEncryptionMethodValue = ServerSideEncryptionMethod.None;
if (serverSideEncryptionMethod == "AES256")
{
serverSideEncryptionMethodValue = ServerSideEncryptionMethod.AES256;
}
AmazonS3Config config = null;
if (serviceURL != null)
{
config = new AmazonS3Config()
{
ServiceURL = serviceURL,
ProxyCredentials = CredentialCache.DefaultNetworkCredentials
};
}
else
{
config = new AmazonS3Config()
{
RegionEndpoint = RegionEndpoint.GetBySystemName(region),
ProxyCredentials = CredentialCache.DefaultNetworkCredentials
};
}
AmazonS3Client amazonS3Client = null;
// Load credentials from the current profile
if (!string.IsNullOrWhiteSpace(profileName))
{
var credFile = new SharedCredentialsFile();
if (credFile.TryGetProfile(profileName, out var profile))
{
amazonS3Client = new AmazonS3Client(profile.GetAWSCredentials(profileSource: null), config);
}
else
{
throw new ArgumentException($"The specified AWS profileName {profileName} could not be found. The feed must specify a valid profileName for an AWS credentials file. For help on credential files see: https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file");
}
}
// Load credentials explicitly with an accessKey and secretKey
else if (
!string.IsNullOrWhiteSpace(accessKeyId) &&
!string.IsNullOrWhiteSpace(secretAccessKey))
{
amazonS3Client = new AmazonS3Client(new BasicAWSCredentials(accessKeyId, secretAccessKey), config);
}
// Load credentials from Environment Variables
else if (
!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(EnvironmentVariablesAWSCredentials.ENVIRONMENT_VARIABLE_ACCESSKEY)) &&
!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(EnvironmentVariablesAWSCredentials.ENVIRONMENT_VARIABLE_SECRETKEY)))
{
amazonS3Client = new AmazonS3Client(new EnvironmentVariablesAWSCredentials(), config);
}
// Load credentials from an ECS docker container
else if (
!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(ECSTaskCredentials.ContainerCredentialsURIEnvVariable)))
{
amazonS3Client = new AmazonS3Client(new ECSTaskCredentials(), config);
}
// Assume IAM role
else
{
using (var client = new AmazonSecurityTokenServiceClient(config.RegionEndpoint))
{
try
{
var identity = await client.GetCallerIdentityAsync(new GetCallerIdentityRequest());
}
catch (Exception ex)
{
throw new ArgumentException(
"Failed to determine AWS identity - ensure you have an IAM " +
"role set, have set up default credentials or have specified a profile/key pair.", ex);
}
}
amazonS3Client = new AmazonS3Client(config);
}
if (pathUri == null)
{
// Find the default path
pathUri = AmazonS3Utility.GetBucketPath(bucketName, config.RegionEndpoint.SystemName);
}
if (baseUri == null)
{
baseUri = pathUri;
}
result = new AmazonS3FileSystem(
cache,
pathUri,
baseUri,
amazonS3Client,
bucketName,
serverSideEncryptionMethodValue,
feedSubPath,
compress);
}
}
}
return(result);
}
/// <summary>
/// Parses sleet.json to find the source and constructs it.
/// </summary>
public static ISleetFileSystem CreateFileSystem(LocalSettings settings, LocalCache cache, string source)
{
ISleetFileSystem result = null;
var sources = settings.Json["sources"] as JArray;
if (sources == null)
{
throw new ArgumentException("Invalid config. No sources found.");
}
foreach (var sourceEntry in sources.Select(e => (JObject)e))
{
var sourceName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "name");
if (source.Equals(sourceName, StringComparison.OrdinalIgnoreCase))
{
var path = JsonUtility.GetValueCaseInsensitive(sourceEntry, "path");
var baseURIString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "baseURI");
var feedSubPath = JsonUtility.GetValueCaseInsensitive(sourceEntry, "feedSubPath");
var type = JsonUtility.GetValueCaseInsensitive(sourceEntry, "type")?.ToLowerInvariant();
string absolutePath;
if (path != null && type == "local")
{
if (settings.Path == null && !Path.IsPathRooted(NuGetUriUtility.GetLocalPath(path)))
{
throw new ArgumentException("Cannot use a relative 'path' without a settings.json file.");
}
var nonEmptyPath = path == "" ? "." : path;
var settingsDir = Path.GetDirectoryName(NuGetUriUtility.GetLocalPath(settings.Path));
absolutePath = NuGetUriUtility.GetAbsolutePath(settingsDir, nonEmptyPath);
}
else
{
absolutePath = path;
}
var pathUri = absolutePath != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(absolutePath)) : null;
var baseUri = baseURIString != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(baseURIString)) : pathUri;
if (type == "local")
{
if (pathUri == null)
{
throw new ArgumentException("Missing path for account.");
}
result = new PhysicalFileSystem(cache, pathUri, baseUri);
}
else if (type == "azure")
{
var connectionString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "connectionString");
var container = JsonUtility.GetValueCaseInsensitive(sourceEntry, "container");
if (string.IsNullOrEmpty(connectionString))
{
throw new ArgumentException("Missing connectionString for azure account.");
}
if (connectionString.Equals(AzureFileSystem.AzureEmptyConnectionString, StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentException("Invalid connectionString for azure account.");
}
if (string.IsNullOrEmpty(container))
{
throw new ArgumentException("Missing container for azure account.");
}
var azureAccount = CloudStorageAccount.Parse(connectionString);
if (pathUri == null)
{
// Get the default url from the container
pathUri = AzureUtility.GetContainerPath(azureAccount, container);
}
if (baseUri == null)
{
baseUri = pathUri;
}
result = new AzureFileSystem(cache, pathUri, baseUri, azureAccount, container, feedSubPath);
}
#if !SLEETLEGACY
else if (type == "s3")
{
var accessKeyId = JsonUtility.GetValueCaseInsensitive(sourceEntry, "accessKeyId");
var secretAccessKey = JsonUtility.GetValueCaseInsensitive(sourceEntry, "secretAccessKey");
var profileName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "profileName");
var bucketName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "bucketName");
var region = JsonUtility.GetValueCaseInsensitive(sourceEntry, "region");
if (string.IsNullOrEmpty(profileName) && string.IsNullOrEmpty(accessKeyId))
{
throw new ArgumentException("Must provide a profileName or accessKeyId and secretAccessKey for Amazon S3 account.");
}
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentException("Missing bucketName for Amazon S3 account.");
}
if (string.IsNullOrEmpty(region))
{
throw new ArgumentException("Missing region for Amazon S3 account.");
}
var regionSystemName = RegionEndpoint.GetBySystemName(region);
if (!new Amazon.Runtime.CredentialManagement.SharedCredentialsFile().TryGetProfile(profileName, out var profile))
{
throw new ArgumentException($"The specified profile {profileName} could not be found.");
}
if (pathUri == null)
{
// Find the default path
pathUri = AmazonS3Utility.GetBucketPath(bucketName, regionSystemName.SystemName);
}
if (baseUri == null)
{
baseUri = pathUri;
}
var amazonS3Client = string.IsNullOrEmpty(accessKeyId) ?
string.IsNullOrEmpty(profileName) ?
new AmazonS3Client(regionSystemName)
: new AmazonS3Client(profile.GetAWSCredentials(null), regionSystemName)
: new AmazonS3Client(accessKeyId, secretAccessKey, regionSystemName);
result = new AmazonS3FileSystem(
cache,
pathUri,
baseUri,
amazonS3Client,
bucketName,
feedSubPath);
}
#endif
}
}
return(result);
}
public Uri GetIndexUri(PackageIdentity package)
{
return(UriUtility.CreateUri($"{_context.Source.BaseURI}registration/{package.Id.ToLowerInvariant()}/index.json"));
}
/// <summary>
/// Parses sleet.json to find the source and constructs it.
/// </summary>
public static ISleetFileSystem CreateFileSystem(LocalSettings settings, LocalCache cache, string source)
{
ISleetFileSystem result = null;
var sources = settings.Json["sources"] as JArray;
if (sources == null)
{
throw new ArgumentException("Invalid config. No sources found.");
}
if (sources != null)
{
foreach (var sourceEntry in sources.Select(e => (JObject)e))
{
var sourceName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "name");
if (source.Equals(sourceName, StringComparison.OrdinalIgnoreCase))
{
var path = JsonUtility.GetValueCaseInsensitive(sourceEntry, "path");
var baseURIString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "baseURI");
var feedSubPath = JsonUtility.GetValueCaseInsensitive(sourceEntry, "feedSubPath");
var type = JsonUtility.GetValueCaseInsensitive(sourceEntry, "type")?.ToLowerInvariant();
var pathUri = path != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(path)) : null;
var baseUri = baseURIString != null?UriUtility.EnsureTrailingSlash(UriUtility.CreateUri(baseURIString)) : pathUri;
if (type == "local")
{
if (pathUri == null)
{
throw new ArgumentException("Missing path for account.");
}
result = new PhysicalFileSystem(cache, pathUri, baseUri);
}
else if (type == "azure")
{
var connectionString = JsonUtility.GetValueCaseInsensitive(sourceEntry, "connectionString");
var container = JsonUtility.GetValueCaseInsensitive(sourceEntry, "container");
if (string.IsNullOrEmpty(connectionString))
{
throw new ArgumentException("Missing connectionString for azure account.");
}
if (connectionString.Equals(AzureFileSystem.AzureEmptyConnectionString, StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentException("Invalid connectionString for azure account.");
}
if (string.IsNullOrEmpty(container))
{
throw new ArgumentException("Missing container for azure account.");
}
var azureAccount = CloudStorageAccount.Parse(connectionString);
if (pathUri == null)
{
// Get the default url from the container
pathUri = AzureUtility.GetContainerPath(azureAccount, container);
}
if (baseUri == null)
{
baseUri = pathUri;
}
result = new AzureFileSystem(cache, pathUri, baseUri, azureAccount, container, feedSubPath);
}
#if !SLEETLEGACY
else if (type == "s3")
{
var accessKeyId = JsonUtility.GetValueCaseInsensitive(sourceEntry, "accessKeyId");
var secretAccessKey = JsonUtility.GetValueCaseInsensitive(sourceEntry, "secretAccessKey");
var bucketName = JsonUtility.GetValueCaseInsensitive(sourceEntry, "bucketName");
var region = JsonUtility.GetValueCaseInsensitive(sourceEntry, "region");
if (string.IsNullOrEmpty(accessKeyId))
{
throw new ArgumentException("Missing accessKeyId for Amazon S3 account.");
}
if (string.IsNullOrEmpty(secretAccessKey))
{
throw new ArgumentException("Missing secretAccessKey for Amazon S3 account.");
}
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentException("Missing bucketName for Amazon S3 account.");
}
if (string.IsNullOrEmpty(region))
{
throw new ArgumentException("Missing region for Amazon S3 account.");
}
var regionSystemName = RegionEndpoint.GetBySystemName(region);
if (pathUri == null)
{
// Find the default path
pathUri = AmazonS3Utility.GetBucketPath(bucketName, regionSystemName.SystemName);
}
if (baseUri == null)
{
baseUri = pathUri;
}
var amazonS3Client = new AmazonS3Client(
accessKeyId, secretAccessKey, regionSystemName);
result = new AmazonS3FileSystem(
cache,
pathUri,
baseUri,
amazonS3Client,
bucketName,
feedSubPath);
}
#endif
}
}
}
return(result);
}
/// <summary>
/// Download packages. This method does not lock the feed or verify the client version.
/// </summary>
public static async Task <bool> DownloadPackages(LocalSettings settings, ISleetFileSystem source, string outputPath, bool ignoreErrors, ILogger log, CancellationToken token)
{
if (string.IsNullOrEmpty(outputPath))
{
throw new ArgumentException("Missing output path parameter!");
}
var success = true;
// Get sleet.settings.json
var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
log.LogMinimal($"Reading feed {source.BaseURI.AbsoluteUri}");
// Find all packages
var packageIndex = new PackageIndex(context);
var flatContainer = new FlatContainer(context);
var symbols = new Symbols(context);
// Discover all packages
var packages = new List <KeyValuePair <PackageIdentity, ISleetFile> >();
packages.AddRange((await packageIndex.GetPackagesAsync()).Select(e =>
new KeyValuePair <PackageIdentity, ISleetFile>(e, context.Source.Get(flatContainer.GetNupkgPath(e)))));
packages.AddRange((await packageIndex.GetSymbolsPackagesAsync()).Select(e =>
new KeyValuePair <PackageIdentity, ISleetFile>(e, symbols.GetSymbolsNupkgFile(e))));
var tasks = new List <Task <bool> >(MaxThreads);
var downloadSuccess = true;
log.LogMinimal($"Downloading nupkgs to {outputPath}");
foreach (var pair in packages)
{
if (tasks.Count >= MaxThreads)
{
downloadSuccess &= await CompleteTask(tasks);
}
var package = pair.Key;
var nupkgFile = pair.Value;
var fileName = UriUtility.GetFileName(nupkgFile.EntityUri);
// id/id.version.nupkg or id/id.version.symbols.nupkg
var outputNupkgPath = Path.Combine(outputPath,
package.Id.ToLowerInvariant(),
fileName.ToLowerInvariant());
log.LogInformation($"Downloading {outputNupkgPath}");
tasks.Add(nupkgFile.CopyTo(outputNupkgPath, overwrite: true, log: log, token: token));
}
while (tasks.Count > 0)
{
downloadSuccess &= await CompleteTask(tasks);
}
success &= downloadSuccess;
if (packages.Count < 1)
{
log.LogWarning("The feed does not contain any packages.");
}
if (downloadSuccess)
{
if (packages.Count > 0)
{
log.LogMinimal("Successfully downloaded packages.");
}
}
else
{
var message = $"Failed to download all packages!";
if (ignoreErrors)
{
log.LogWarning(message);
}
else
{
log.LogError(message);
}
}
return(success);
}
