public static void RewritingTheNuspecCreatesValidNewElements()
{
var packageStream = CreateTestPackageStream();
// ensure
using (var nupkg = new PackageArchiveReader(packageStream, leaveStreamOpen: true))
{
var nuspec = nupkg.GetNuspecReader();
Assert.Equal(nuspec.GetMetadata().Any(kvp => kvp.Key == PackageMetadataStrings.LicenseUrl), false);
}
// Act
NupkgRewriter.RewriteNupkgManifest(packageStream,
new List<Action<ManifestEdit>>
{
metadata => { metadata.Authors = "Me and You"; },
metadata => { metadata.Tags = "Peas In A Pod"; },
metadata => { metadata.LicenseUrl = "http://myget.org"; },
metadata => { metadata.RequireLicenseAcceptance = true; }
});
// Assert
using (var nupkg = new PackageArchiveReader(packageStream, leaveStreamOpen: false))
{
var nuspec = nupkg.GetNuspecReader();
Assert.Equal("TestPackage", nuspec.GetId());
Assert.Equal(NuGetVersion.Parse("0.0.0.1"), nuspec.GetVersion());
Assert.Equal(nuspec.GetMetadata().Any(kvp => kvp.Key == PackageMetadataStrings.LicenseUrl), true);
Assert.Equal(nuspec.GetMetadata().First(kvp => kvp.Key == PackageMetadataStrings.LicenseUrl).Value, "http://myget.org");
Assert.Equal(nuspec.GetMetadata().First(kvp => kvp.Key == PackageMetadataStrings.RequireLicenseAcceptance).Value, "true");
}
}
public static void CanRewriteTheNuspecInANupkg()
{
var packageStream = CreateTestPackageStream();
// Act
NupkgRewriter.RewriteNupkgManifest(packageStream,
new List<Action<ManifestEdit>>
{
metadata => { metadata.Authors = "Me and You"; },
metadata => { metadata.Tags = "Peas In A Pod"; },
metadata => { metadata.ReleaseNotes = "In perfect harmony"; }
});
// Assert
using (var nupkg = new PackageArchiveReader(packageStream, leaveStreamOpen: false))
{
var nuspec = nupkg.GetNuspecReader();
Assert.Equal("TestPackage", nuspec.GetId());
Assert.Equal(NuGetVersion.Parse("0.0.0.1"), nuspec.GetVersion());
Assert.Equal("Me and You", nuspec.GetMetadata().First(kvp => kvp.Key == PackageMetadataStrings.Authors).Value);
Assert.Equal("Peas In A Pod", nuspec.GetMetadata().First(kvp => kvp.Key == PackageMetadataStrings.Tags).Value);
Assert.Equal("In perfect harmony", nuspec.GetMetadata().First(kvp => kvp.Key == PackageMetadataStrings.ReleaseNotes).Value);
}
}
internal static bool ShouldCuratePackage(
CuratedFeed curatedFeed,
Package galleryPackage,
PackageArchiveReader packageArchiveReader)
{
var nuspec = packageArchiveReader.GetNuspecReader();
return
// Must have min client version of null or <= 2.2
(nuspec.GetMinClientVersion() == null || nuspec.GetMinClientVersion() <= new NuGetVersion(2, 2, 0)) &&
// Must be latest stable
galleryPackage.IsLatestStable &&
// Must support net40
SupportsNet40(galleryPackage) &&
(
// Must have AspNetWebPages tag
ContainsAspNetWebPagesTag(galleryPackage) ||
// OR: Must not contain powershell or T4
DoesNotContainUnsupportedFiles(packageArchiveReader)
) &&
// Dependencies on the gallery must be curated
DependenciesAreCurated(galleryPackage, curatedFeed);
}
public override async Task CurateAsync(Package galleryPackage, PackageArchiveReader nugetPackage, bool commitChanges)
{
// Make sure the target feed exists
CuratedFeed feed = CuratedFeedService.GetFeedByName(CuratedFeedName, includePackages: true);
if (feed != null && galleryPackage.Tags != null)
{
// Break the tags up so we can be sure we don't catch any partial matches (i.e. "foobar" when we're looking for "foo")
string[] tags = galleryPackage.Tags.Split();
// Check if this package should be curated
if (tags.Any(tag => RequiredTags.Contains(tag, StringComparer.OrdinalIgnoreCase)))
{
// It should!
// But now we need to ensure that the package's dependencies are also curated
if (DependenciesAreCurated(galleryPackage, feed))
{
await CuratedFeedService.CreatedCuratedPackageAsync(
feed,
galleryPackage.PackageRegistration,
automaticallyCurated: true,
commitChanges: commitChanges);
}
}
}
}
public async Task ExecuteAsync(Package galleryPackage, PackageArchiveReader nugetPackage, bool commitChanges)
{
foreach (var curator in _curators)
{
await curator.CurateAsync(galleryPackage, nugetPackage, commitChanges: commitChanges);
}
}
public void Execute(Package galleryPackage, PackageArchiveReader nugetPackage, bool commitChanges)
{
foreach (var curator in _curators)
{
curator.Curate(galleryPackage, nugetPackage, commitChanges: commitChanges);
}
}
public async Task<Package> ReflowAsync(string id, string version)
{
var package = _packageService.FindPackageByIdAndVersion(id, version);
if (package == null)
{
return null;
}
EntitiesConfiguration.SuspendExecutionStrategy = true;
using (var transaction = _entitiesContext.GetDatabase().BeginTransaction())
{
// 1) Download package binary to memory
using (var packageStream = await _packageFileService.DownloadPackageFileAsync(package))
{
using (var packageArchive = new PackageArchiveReader(packageStream, leaveStreamOpen: false))
{
// 2) Determine package metadata from binary
var packageStreamMetadata = new PackageStreamMetadata
{
HashAlgorithm = Constants.Sha512HashAlgorithmId,
Hash = CryptographyService.GenerateHash(packageStream.AsSeekableStream()),
Size = packageStream.Length,
};
var packageMetadata = PackageMetadata.FromNuspecReader(packageArchive.GetNuspecReader());
// 3) Clear referenced objects that will be reflowed
ClearSupportedFrameworks(package);
ClearAuthors(package);
ClearDependencies(package);
// 4) Reflow the package
var listed = package.Listed;
package = _packageService.EnrichPackageFromNuGetPackage(
package,
packageArchive,
packageMetadata,
packageStreamMetadata,
package.User);
package.LastEdited = DateTime.UtcNow;
package.Listed = listed;
// 5) Save and profit
await _entitiesContext.SaveChangesAsync();
}
}
// Commit transaction
transaction.Commit();
}
EntitiesConfiguration.SuspendExecutionStrategy = false;
return package;
}
public NupkgNameInfo(string path)
{
using (PackageArchiveReader archiveReader = new PackageArchiveReader(path))
{
PackageIdentity identity = archiveReader.GetIdentity();
Id = identity.Id;
Version = identity.Version.ToString();
Prerelease = identity.Version.Release;
}
SymbolPackage = path.EndsWith(".symbols.nupkg");
}
public void ProcessAssembliesAndContent()
{
Parallel.ForEach(_packageIdentities, packageIdentity =>
{
DirectoryPath installedPath = new DirectoryPath(GetInstalledPath(packageIdentity));
string packageFilePath = GetInstalledPackageFilePath(packageIdentity);
PackageArchiveReader archiveReader = new PackageArchiveReader(packageFilePath, null, null);
AddReferencedAssemblies(installedPath, archiveReader);
IncludeContentDirectories(installedPath, archiveReader);
Trace.Verbose($"Finished processing NuGet package at {installedPath}");
});
}
public static void RewritingTheNuSpecDoesNotMessUpTheNuspecStream()
{
var packageStream = CreateTestPackageStream();
var manifestStreamLengthOriginal = GetManifestStreamLength(packageStream);
var longValue = new String('x', 200);
var shortValue = "y";
// Act 1 - Make the stream bigger
NupkgRewriter.RewriteNupkgManifest(packageStream,
new List<Action<ManifestEdit>>
{
metadata => { metadata.Description = longValue; },
metadata => { metadata.Summary = longValue; }
});
// Assert 1
var manifestStreamLength1 = GetManifestStreamLength(packageStream);
Assert.True(manifestStreamLength1 > manifestStreamLengthOriginal);
using (var nupkg = new PackageArchiveReader(packageStream, leaveStreamOpen: true))
{
var nuspec = nupkg.GetNuspecReader();
Assert.Equal("TestPackage", nuspec.GetId());
Assert.Equal(NuGetVersion.Parse("0.0.0.1"), nuspec.GetVersion());
Assert.Equal(longValue, nuspec.GetMetadata().First(kvp => kvp.Key == "description").Value);
Assert.Equal(longValue, nuspec.GetMetadata().First(kvp => kvp.Key == "summary").Value);
}
// Act 2 - Make the stream smaller
NupkgRewriter.RewriteNupkgManifest(packageStream,
new List<Action<ManifestEdit>>
{
metadata => { metadata.Description = shortValue; },
metadata => { metadata.Summary = shortValue; }
});
// Assert 2
var manifestStreamLength2 = GetManifestStreamLength(packageStream);
Assert.True(manifestStreamLength2 < manifestStreamLength1);
using (var nupkg = new PackageArchiveReader(packageStream, leaveStreamOpen: true))
{
var nuspec = nupkg.GetNuspecReader();
Assert.Equal("TestPackage", nuspec.GetId());
Assert.Equal(NuGetVersion.Parse("0.0.0.1"), nuspec.GetVersion());
Assert.Equal(shortValue, nuspec.GetMetadata().First(kvp => kvp.Key == "description").Value);
Assert.Equal(shortValue, nuspec.GetMetadata().First(kvp => kvp.Key == "summary").Value);
}
}
public void EnsureValid(PackageArchiveReader packageArchiveReader)
{
var packageMetadata = PackageMetadata.FromNuspecReader(packageArchiveReader.GetNuspecReader());
ValidateNuGetPackageMetadata(packageMetadata);
ValidatePackageTitle(packageMetadata);
var supportedFrameworks = GetSupportedFrameworks(packageArchiveReader).Select(fn => fn.ToShortNameOrNull()).ToArray();
if (!supportedFrameworks.AnySafe(sf => sf == null))
{
ValidateSupportedFrameworks(supportedFrameworks);
}
}
// Add all reference items to the assembly list
private void AddReferencedAssemblies(DirectoryPath installedPath, PackageArchiveReader archiveReader)
{
FrameworkSpecificGroup referenceGroup = GetMostCompatibleGroup(_reducer,
_currentFramework, archiveReader.GetReferenceItems().ToList());
if (referenceGroup != null)
{
foreach (FilePath itemPath in referenceGroup.Items
.Select(x => new FilePath(x))
.Where(x => x.FileName.Extension == ".dll" || x.FileName.Extension == ".exe"))
{
FilePath assemblyPath = installedPath.CombineFile(itemPath);
_assemblyLoader.Add(assemblyPath.FullPath);
Trace.Verbose($"Added NuGet reference {assemblyPath} for loading");
}
}
}
private async Task<bool> ExecuteAsync(CancellationToken cancellationToken)
{
using (Stream stream = File.OpenRead(PackageFile))
{
var reader = new PackageArchiveReader(stream, leaveStreamOpen: true);
var packageIdentity = reader.GetIdentity();
reader.Dispose();
stream.Seek(0, SeekOrigin.Begin);
await
InstallFromStream(ThreadingTask.FromResult(stream), packageIdentity, PackagesFolder,
cancellationToken);
}
return true;
}
// Add content directories to the input paths
private void IncludeContentDirectories(DirectoryPath installedPath, PackageArchiveReader archiveReader)
{
FrameworkSpecificGroup contentGroup = GetMostCompatibleGroup(_reducer,
_currentFramework, archiveReader.GetContentItems().ToList());
if (contentGroup != null)
{
// We need to use the directory name from an actual file to make sure we get the casing right
foreach (string contentSegment in contentGroup.Items
.Select(x => new FilePath(x).Segments[0])
.Distinct())
{
DirectoryPath contentPath = installedPath.Combine(contentSegment);
_fileSystem.InputPaths.Insert(0, contentPath);
Trace.Verbose($"Added content path {contentPath} to included paths");
}
}
}
protected static void AssertPackage(string packageFilePath, Action<PackageArchiveReader> packageAssertions)
{
var fullPath = Path.Combine(Environment.CurrentDirectory, packageFilePath);
if (!File.Exists(fullPath))
{
Assert.Fail("Could not find package file: " + fullPath);
}
Trace.WriteLine("Checking package: " + fullPath);
using (var package = new PackageArchiveReader(File.OpenRead(fullPath)))
{
packageAssertions(package);
}
Trace.WriteLine("Success!");
}
public async Task<Package> CreatePackageAsync(PackageArchiveReader nugetPackage, PackageStreamMetadata packageStreamMetadata, User user, bool commitChanges = true)
{
var packageMetadata = PackageMetadata.FromNuspecReader(nugetPackage.GetNuspecReader());
ValidateNuGetPackageMetadata(packageMetadata);
ValidatePackageTitle(packageMetadata);
var packageRegistration = CreateOrGetPackageRegistration(user, packageMetadata);
var package = CreatePackageFromNuGetPackage(packageRegistration, nugetPackage, packageMetadata, packageStreamMetadata, user);
packageRegistration.Packages.Add(package);
await UpdateIsLatestAsync(packageRegistration, false);
if (commitChanges)
{
await _packageRegistrationRepository.CommitChangesAsync();
NotifyIndexingService();
}
return package;
}
public static void CanReadBasicMetadataProperties()
{
var packageStream = CreateTestPackageStream();
var nupkg = new PackageArchiveReader(packageStream, leaveStreamOpen: false);
var nuspec = nupkg.GetNuspecReader();
// Act
var packageMetadata = PackageMetadata.FromNuspecReader(nuspec);
// Assert
Assert.Equal("TestPackage", packageMetadata.Id);
Assert.Equal(NuGetVersion.Parse("0.0.0.1"), packageMetadata.Version);
Assert.Equal("Package A", packageMetadata.Title);
Assert.Equal(2, packageMetadata.Authors.Count);
Assert.Equal("ownera, ownerb", packageMetadata.Owners);
Assert.False(packageMetadata.RequireLicenseAcceptance);
Assert.Equal("package A description.", packageMetadata.Description);
Assert.Equal("en-US", packageMetadata.Language);
Assert.Equal("http://www.nuget.org/", packageMetadata.ProjectUrl.ToString());
Assert.Equal("http://www.nuget.org/", packageMetadata.IconUrl.ToString());
Assert.Equal("http://www.nuget.org/", packageMetadata.LicenseUrl.ToString());
}
public override async Task CurateAsync(
Package galleryPackage,
PackageArchiveReader nugetPackage,
bool commitChanges)
{
var curatedFeed = CuratedFeedService.GetFeedByName("webmatrix", includePackages: true);
if (curatedFeed == null)
{
return;
}
var shouldBeIncluded = ShouldCuratePackage(curatedFeed, galleryPackage, nugetPackage);
if (shouldBeIncluded)
{
await CuratedFeedService.CreatedCuratedPackageAsync(
curatedFeed,
galleryPackage.PackageRegistration,
included: true,
automaticallyCurated: true,
commitChanges: commitChanges);
}
}
public static void RewritingTheNuspecDoesNotAddEmptyMetadataElements()
{
var packageStream = CreateTestPackageStream();
// Act
NupkgRewriter.RewriteNupkgManifest(packageStream,
new List<Action<ManifestEdit>>
{
metadata => { metadata.Authors = "Me and You"; },
metadata => { metadata.Tags = "Peas In A Pod"; },
metadata => { metadata.ReleaseNotes = "In perfect harmony"; }
});
// Assert
using (var nupkg = new PackageArchiveReader(packageStream, leaveStreamOpen: false))
{
var nuspec = nupkg.GetNuspecReader();
var metadataDescendants = nuspec.Xml.Document.Descendants().Where(d => d.Name.LocalName == PackageMetadataStrings.Metadata).Descendants();
foreach (var element in metadataDescendants)
{
Assert.False(string.IsNullOrEmpty(element.Value), $"Nuspec contains a null or emtpy tag <{element.Name.LocalName}>");
}
}
}
public static void RewriteNupkg(PackageFile packageFile, NuGetVersion nuGetVersion = null)
{
if (packageFile == null)
{
throw new ArgumentNullException(nameof(packageFile));
}
var randomId = Guid.NewGuid().ToString("N");
using var packageArchiveReader = new PackageArchiveReader(
packageFile.FilenameAbsolutePath.ReadSharedToStream(), false);
var nuspecXDocument = packageArchiveReader.NuspecReader.Xml;
var packageXElement = nuspecXDocument.Single("package");
var metadataXElement = packageXElement.Single("metadata");
var packageId = packageXElement.Single("id").Value;
var versionXElement = metadataXElement.Single("version");
if (nuGetVersion != null)
{
versionXElement.SetValue(nuGetVersion);
}
else
{
nuGetVersion = NuGetVersion.Parse(versionXElement.Value);
}
var repositoryXElement = metadataXElement.SingleOrDefault("repository");
if (repositoryXElement == null)
{
repositoryXElement = new XElement("repository");
repositoryXElement.SetAttributeValue("url", packageFile.RepositoryUrl);
repositoryXElement.SetAttributeValue("type", "git");
metadataXElement.Add(repositoryXElement);
}
else
{
repositoryXElement.SetAttributeValue("url", packageFile.RepositoryUrl);
repositoryXElement.SetAttributeValue("type", "git");
}
using var nuspecMemoryStream = new MemoryStream();
nuspecXDocument.Save(nuspecMemoryStream);
nuspecMemoryStream.Seek(0, SeekOrigin.Begin);
var packageFileWorkingDirectoryAbsolutePath = Path.GetDirectoryName(packageFile.FilenameAbsolutePath);
var packageFileRewriteWorkingDirectory = Path.Combine(packageFileWorkingDirectoryAbsolutePath,
$"{packageId}.{nuGetVersion}_{randomId}");
using var tmpDirectory = new DisposableDirectory(packageFileRewriteWorkingDirectory);
ZipFile.ExtractToDirectory(packageFile.FilenameAbsolutePath, tmpDirectory);
var nuspecDstFilename = Path.Combine(tmpDirectory, $"{packageId}.nuspec");
File.WriteAllBytes(nuspecDstFilename, nuspecMemoryStream.ToArray());
using var outputStream = new MemoryStream();
var packageBuilder = new PackageBuilder(nuspecMemoryStream, tmpDirectory,
propertyProvider => throw new NotImplementedException());
packageBuilder.Save(outputStream);
packageFile.Filename = $"{packageId}.{nuGetVersion}.nupkg";
packageFile.FilenameAbsolutePath = Path.Combine(packageFileWorkingDirectoryAbsolutePath, Path.ChangeExtension(packageFile.Filename, ".zip"));
packageFile.IsNuspecRewritten = true;
File.WriteAllBytes(packageFile.FilenameAbsolutePath, outputStream.ToArray());
}
private Package CreatePackageFromNuGetPackage(PackageRegistration packageRegistration, PackageArchiveReader nugetPackage, PackageMetadata packageMetadata, PackageStreamMetadata packageStreamMetadata, User user)
{
var package = packageRegistration.Packages.SingleOrDefault(pv => pv.Version == packageMetadata.Version.OriginalVersion);
if (package != null)
{
throw new PackageAlreadyExistsException(
string.Format(ServicesStrings.PackageExistsAndCannotBeModified, packageRegistration.Id, package.Version));
}
package = new Package();
package.PackageRegistration = packageRegistration;
package = EnrichPackageFromNuGetPackage(package, nugetPackage, packageMetadata, packageStreamMetadata, user);
return(package);
}
private void UpdateFromPackage(PackageIndex index, string packagePath, bool filter = false)
{
string id;
NuGetVersion version;
IEnumerable<Version> assemblyVersions;
IEnumerable<string> dllNames;
using (var reader = new PackageArchiveReader(packagePath))
{
var identity = reader.GetIdentity();
id = identity.Id;
version = identity.Version;
if (filter && !ShouldInclude(id))
{
return;
}
var dlls = reader.GetFiles().Where(f => Path.GetExtension(f).Equals(".dll", StringComparison.OrdinalIgnoreCase));
assemblyVersions = dlls.Select(refFile =>
{
using (var refStream = reader.GetStream(refFile))
using (var memStream = new MemoryStream())
{
refStream.CopyTo(memStream);
memStream.Seek(0, SeekOrigin.Begin);
return VersionUtility.GetAssemblyVersion(memStream);
}
}).ToArray();
dllNames = dlls.Select(f => Path.GetFileNameWithoutExtension(f)).Distinct().ToArray();
}
UpdateFromValues(index, id, version, assemblyVersions, dllNames);
}
private async Task<ActionResult> CreatePackageInternal()
{
// Get the user
var user = GetCurrentUser();
using (var packageStream = ReadPackageFromRequest())
{
try
{
using (var archive = new ZipArchive(packageStream, ZipArchiveMode.Read, leaveOpen: true))
{
var reference = DateTime.UtcNow.AddDays(1); // allow "some" clock skew
var entryInTheFuture = archive.Entries.FirstOrDefault(
e => e.LastWriteTime.UtcDateTime > reference);
if (entryInTheFuture != null)
{
return new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(
CultureInfo.CurrentCulture,
Strings.PackageEntryFromTheFuture,
entryInTheFuture.Name));
}
}
using (var packageToPush = new PackageArchiveReader(packageStream, leaveStreamOpen: false))
{
NuspecReader nuspec = null;
try
{
nuspec = packageToPush.GetNuspecReader();
}
catch (Exception ex)
{
return new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(
CultureInfo.CurrentCulture,
Strings.UploadPackage_InvalidNuspec,
ex.Message));
}
if (nuspec.GetMinClientVersion() > Constants.MaxSupportedMinClientVersion)
{
return new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(
CultureInfo.CurrentCulture,
Strings.UploadPackage_MinClientVersionOutOfRange,
nuspec.GetMinClientVersion()));
}
// Ensure that the user can push packages for this partialId.
var packageRegistration = PackageService.FindPackageRegistrationById(nuspec.GetId());
if (packageRegistration != null)
{
if (!packageRegistration.IsOwner(user))
{
return new HttpStatusCodeWithBodyResult(HttpStatusCode.Forbidden,
Strings.ApiKeyNotAuthorized);
}
// Check if a particular Id-Version combination already exists. We eventually need to remove this check.
string normalizedVersion = nuspec.GetVersion().ToNormalizedString();
bool packageExists =
packageRegistration.Packages.Any(
p => String.Equals(
p.NormalizedVersion,
normalizedVersion,
StringComparison.OrdinalIgnoreCase));
if (packageExists)
{
return new HttpStatusCodeWithBodyResult(
HttpStatusCode.Conflict,
String.Format(CultureInfo.CurrentCulture, Strings.PackageExistsAndCannotBeModified,
nuspec.GetId(), nuspec.GetVersion().ToNormalizedStringSafe()));
}
}
var packageStreamMetadata = new PackageStreamMetadata
{
HashAlgorithm = Constants.Sha512HashAlgorithmId,
Hash = CryptographyService.GenerateHash(packageStream.AsSeekableStream()),
Size = packageStream.Length,
};
var package =
await
PackageService.CreatePackageAsync(packageToPush, packageStreamMetadata, user,
commitChanges: false);
await AutoCuratePackage.ExecuteAsync(package, packageToPush, commitChanges: false);
await EntitiesContext.SaveChangesAsync();
using (Stream uploadStream = packageStream)
{
uploadStream.Position = 0;
await PackageFileService.SavePackageFileAsync(package, uploadStream.AsSeekableStream());
IndexingService.UpdatePackage(package);
}
MessageService.SendPackageAddedNotice(package,
Url.Action("DisplayPackage", "Packages", routeValues: new { id = package.PackageRegistration.Id, version = package.Version }, protocol: Request.Url.Scheme),
Url.Action("ReportMyPackage", "Packages", routeValues: new { id = package.PackageRegistration.Id, version = package.Version }, protocol: Request.Url.Scheme),
Url.Action("Account", "Users", routeValues: null, protocol: Request.Url.Scheme));
return new HttpStatusCodeResult(HttpStatusCode.Created);
}
}
catch (InvalidPackageException ex)
{
return BadRequestForExceptionMessage(ex);
}
catch (InvalidDataException ex)
{
return BadRequestForExceptionMessage(ex);
}
catch (EntityException ex)
{
return BadRequestForExceptionMessage(ex);
}
catch (FrameworkException ex)
{
return BadRequestForExceptionMessage(ex);
}
}
}
public override void ExecuteCommand()
{
if (!Directory.Exists(WorkDirectory))
{
Directory.CreateDirectory(WorkDirectory);
}
Log.Info("Getting all package metadata...");
var packages = GetAllPackages();
var totalCount = packages.Count;
var processedCount = 0;
Log.Info(
"Populating frameworks for {0} packages on '{1}',",
totalCount,
ConnectionString);
packages
.AsParallel()
.AsOrdered()
.WithDegreeOfParallelism(10)
.ForAll(package =>
{
// Allocate a processed count number for this package
var thisPackageId = Interlocked.Increment(ref processedCount);
try
{
var reportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + ".json");
var bustedReportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + "_" + package.Hash + ".json");
var report = new PackageFrameworkReport()
{
Id = package.Id,
Version = package.Version,
Key = package.Key,
Hash = package.Hash,
Created = package.Created.Value,
State = PackageReportState.Unresolved
};
if (File.Exists(bustedReportPath))
{
File.Move(bustedReportPath, reportPath);
}
if (File.Exists(reportPath))
{
using (var reader = File.OpenText(reportPath))
{
report = (PackageFrameworkReport)_serializer.Deserialize(reader, typeof(PackageFrameworkReport));
}
ResolveReport(report);
}
else
{
try
{
var downloadPath = DownloadPackage(package);
using (var nugetPackage = new PackageArchiveReader(File.OpenRead(downloadPath)))
{
var supportedFrameworks = GetSupportedFrameworks(nugetPackage);
report.PackageFrameworks = supportedFrameworks.ToArray();
report = PopulateFrameworks(package, report);
}
File.Delete(downloadPath);
// Resolve the report
ResolveReport(report);
}
catch (Exception ex)
{
report.State = PackageReportState.Error;
report.Error = ex.ToString();
}
}
using (var writer = File.CreateText(reportPath))
{
_serializer.Serialize(writer, report);
}
Log.Info("[{2}/{3} {4}%] {6} Package: {0}@{1} (created {5})",
package.Id,
package.Version,
thisPackageId.ToString("000000"),
totalCount.ToString("000000"),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
package.Created.Value,
report.State.ToString().PadRight(_padLength, ' '));
}
catch (Exception ex)
{
Log.Error("[{2}/{3} {4}%] Error For Package: {0}@{1}: {5}",
package.Id,
package.Version,
thisPackageId.ToString("000000"),
totalCount.ToString("000000"),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
ex.ToString());
}
});
}
public abstract void Curate(
Package galleryPackage,
PackageArchiveReader nugetPackage,
bool commitChanges);
public abstract Task CurateAsync(
Package galleryPackage,
PackageArchiveReader nugetPackage,
bool commitChanges);
public virtual Package EnrichPackageFromNuGetPackage(
Package package,
PackageArchiveReader packageArchive,
PackageMetadata packageMetadata,
PackageStreamMetadata packageStreamMetadata,
User user)
{
// Version must always be the exact string from the nuspec, which OriginalVersion will return to us.
// However, we do also store a normalized copy for looking up later.
package.Version = packageMetadata.Version.OriginalVersion;
package.NormalizedVersion = packageMetadata.Version.ToNormalizedString();
package.Description = packageMetadata.Description;
package.ReleaseNotes = packageMetadata.ReleaseNotes;
package.HashAlgorithm = packageStreamMetadata.HashAlgorithm;
package.Hash = packageStreamMetadata.Hash;
package.PackageFileSize = packageStreamMetadata.Size;
package.Language = packageMetadata.Language;
package.Copyright = packageMetadata.Copyright;
package.FlattenedAuthors = packageMetadata.Authors.Flatten();
package.IsPrerelease = packageMetadata.Version.IsPrerelease;
package.Listed = true;
package.RequiresLicenseAcceptance = packageMetadata.RequireLicenseAcceptance;
package.Summary = packageMetadata.Summary;
package.Tags = PackageHelper.ParseTags(packageMetadata.Tags);
package.Title = packageMetadata.Title;
package.User = user;
package.IconUrl = packageMetadata.IconUrl.ToEncodedUrlStringOrNull();
package.LicenseUrl = packageMetadata.LicenseUrl.ToEncodedUrlStringOrNull();
package.ProjectUrl = packageMetadata.ProjectUrl.ToEncodedUrlStringOrNull();
package.MinClientVersion = packageMetadata.MinClientVersion.ToStringOrNull();
#pragma warning disable 618 // TODO: remove Package.Authors completely once production services definitely no longer need it
foreach (var author in packageMetadata.Authors)
{
package.Authors.Add(new PackageAuthor {
Name = author
});
}
#pragma warning restore 618
var supportedFrameworks = GetSupportedFrameworks(packageArchive)
.ToArray();
if (!supportedFrameworks.Any(fx => fx != null && fx.IsAny))
{
var supportedFrameworkNames = supportedFrameworks
.Select(fn => fn.ToShortNameOrNull())
.Where(fn => fn != null)
.ToArray();
ValidateSupportedFrameworks(supportedFrameworkNames);
foreach (var supportedFramework in supportedFrameworkNames)
{
package.SupportedFrameworks.Add(new PackageFramework {
TargetFramework = supportedFramework
});
}
}
package.Dependencies = packageMetadata
.GetDependencyGroups()
.AsPackageDependencyEnumerable()
.ToList();
package.PackageTypes = packageMetadata
.GetPackageTypes()
.AsPackageTypeEnumerable()
.ToList();
package.FlattenedDependencies = package.Dependencies.Flatten();
package.FlattenedPackageTypes = package.PackageTypes.Flatten();
// Identify the SemVerLevelKey using the original package version string and package dependencies
package.SemVerLevelKey = SemVerLevelKey.ForPackage(packageMetadata.Version, package.Dependencies);
return(package);
}
public async Task VerifySignaturesAsync_ExpiredCertificateAndTimestampWithTooLargeRange_Fails()
{
var testServer = await _testFixture.GetSigningTestServerAsync();
var ca = await _testFixture.GetDefaultTrustedCertificateAuthorityAsync();
var accuracy = new BcAccuracy(seconds: new DerInteger(30), millis: null, micros: null);
var serviceOptions = new TimestampServiceOptions()
{
Accuracy = accuracy
};
var timestampService = TimestampService.Create(ca, serviceOptions);
var keyPair = SigningTestUtility.GenerateKeyPair(publicKeyLength: 2048);
var now = DateTimeOffset.UtcNow;
var issueOptions = new IssueCertificateOptions(keyPair.Public)
{
NotAfter = now.AddSeconds(10),
NotBefore = now.AddSeconds(-2),
SubjectName = new X509Name("CN=NuGet Test Expired Certificate")
};
var bcCertificate = ca.IssueCertificate(issueOptions);
using (testServer.RegisterResponder(timestampService))
using (var certificate = new X509Certificate2(bcCertificate.GetEncoded()))
using (var directory = TestDirectory.Create())
{
certificate.PrivateKey = DotNetUtilities.ToRSA(keyPair.Private as RsaPrivateCrtKeyParameters);
var packageContext = new SimpleTestPackageContext();
var signedPackagePath = await SignedArchiveTestUtility.CreateSignedAndTimeStampedPackageAsync(
certificate,
packageContext,
directory,
timestampService.Url);
var waitDuration = (issueOptions.NotAfter - DateTimeOffset.UtcNow).Add(TimeSpan.FromSeconds(1));
// Wait for the certificate to expire. Trust of the signature will require a valid timestamp.
if (waitDuration > TimeSpan.Zero)
{
await Task.Delay(waitDuration);
}
Assert.True(DateTime.UtcNow > issueOptions.NotAfter);
var verifier = new PackageSignatureVerifier(_trustProviders, SignedPackageVerifierSettings.VerifyCommandDefaultPolicy);
using (var packageReader = new PackageArchiveReader(signedPackagePath))
{
var results = await verifier.VerifySignaturesAsync(packageReader, CancellationToken.None);
var result = results.Results.Single();
Assert.False(results.Valid);
Assert.Equal(SignatureVerificationStatus.Untrusted, result.Trust);
Assert.Equal(1, result.Issues.Count(issue => issue.Level == LogLevel.Error));
Assert.Equal(0, result.Issues.Count(issue => issue.Level == LogLevel.Warning));
Assert.Contains(result.Issues, issue =>
issue.Code == NuGetLogCode.NU3011 &&
issue.Level == LogLevel.Error &&
issue.Message == "The primary signature validity period has expired.");
}
}
}
/// <summary>
/// Asynchronously returns a <see cref="DownloadResourceResult" /> for a given package identity
/// and source repository.
/// </summary>
/// <param name="sourceRepository">A source repository.</param>
/// <param name="packageIdentity">A package identity.</param>
/// <param name="downloadContext">A package download context.</param>
/// <param name="globalPackagesFolder">A global packages folder path.</param>
/// <param name="logger">A logger.</param>
/// <param name="token">A cancellation token.</param>
/// <returns>A task that represents the asynchronous operation.
/// The task result (<see cref="Task{TResult}.Result" />) returns a <see cref="DownloadResourceResult" />
/// instance.</returns>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="sourceRepository" />
/// is either <c>null</c> or empty.</exception>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="packageIdentity" />
/// is either <c>null</c> or empty.</exception>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="downloadContext" />
/// is either <c>null</c> or empty.</exception>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="logger" />
/// is either <c>null</c> or empty.</exception>
/// <exception cref="OperationCanceledException">Thrown if <paramref name="token" />
/// is cancelled.</exception>
public static async Task <DownloadResourceResult> GetDownloadResourceResultAsync(
SourceRepository sourceRepository,
PackageIdentity packageIdentity,
PackageDownloadContext downloadContext,
string globalPackagesFolder,
ILogger logger,
CancellationToken token)
{
if (sourceRepository == null)
{
throw new ArgumentNullException(nameof(sourceRepository));
}
if (packageIdentity == null)
{
throw new ArgumentNullException(nameof(packageIdentity));
}
if (downloadContext == null)
{
throw new ArgumentNullException(nameof(downloadContext));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
var downloadResource = await sourceRepository.GetResourceAsync <DownloadResource>(token);
if (downloadResource == null)
{
throw new InvalidOperationException(
string.Format(
CultureInfo.CurrentCulture,
Strings.DownloadResourceNotFound,
sourceRepository.PackageSource.Source));
}
token.ThrowIfCancellationRequested();
DownloadResourceResult result;
try
{
result = await downloadResource.GetDownloadResourceResultAsync(
packageIdentity,
downloadContext,
globalPackagesFolder,
logger,
token);
}
catch (OperationCanceledException)
{
result = new DownloadResourceResult(DownloadResourceResultStatus.Cancelled);
}
if (result == null)
{
throw new FatalProtocolException(string.Format(
CultureInfo.CurrentCulture,
Strings.DownloadStreamNotAvailable,
packageIdentity,
sourceRepository.PackageSource.Source));
}
if (result.Status == DownloadResourceResultStatus.Cancelled)
{
throw new RetriableProtocolException(string.Format(
CultureInfo.CurrentCulture,
Strings.PackageCancelledFromSource,
packageIdentity,
sourceRepository.PackageSource.Source));
}
if (result.Status == DownloadResourceResultStatus.NotFound)
{
throw new FatalProtocolException(string.Format(
CultureInfo.CurrentCulture,
Strings.PackageNotFoundOnSource,
packageIdentity,
sourceRepository.PackageSource.Source));
}
if (result.PackageReader == null)
{
result.PackageStream.Seek(0, SeekOrigin.Begin);
var packageReader = new PackageArchiveReader(result.PackageStream);
result.PackageStream.Seek(0, SeekOrigin.Begin);
result = new DownloadResourceResult(result.PackageStream, packageReader, sourceRepository.PackageSource.Source)
{
SignatureVerified = result.SignatureVerified
};
}
else if (result.Status != DownloadResourceResultStatus.AvailableWithoutStream)
{
// bind the source
result = new DownloadResourceResult(result.PackageStream, result.PackageReader, sourceRepository.PackageSource.Source)
{
SignatureVerified = result.SignatureVerified
};
}
return(result);
}
public async Task ClientPolicies_WithSignerInTrustedSignersListAsync(SigningTestType signature, SignaturePlacement trustedSigner, string validationMode)
{
// Arrange
using (var dir = TestDirectory.Create())
using (var authorCertificate = new X509Certificate2(_trustedAuthorTestCert.Source.Cert))
using (var repoCertificate = new X509Certificate2(_trustedRepoTestCert.Source.Cert))
{
var authorCertificateFingerprintString = SignatureTestUtility.GetFingerprint(authorCertificate, HashAlgorithmName.SHA256);
var repoCertificateFingerprintString = SignatureTestUtility.GetFingerprint(repoCertificate, HashAlgorithmName.SHA256);
var signedPackagePath = await CreateSignedPackageAsync(dir, signature, authorCertificate, repoCertificate);
var trustedSignerString = "";
if (signature == SigningTestType.Author || (signature == SigningTestType.RepositoryCountersigned && trustedSigner == SignaturePlacement.PrimarySignature))
{
trustedSignerString = $@"<author name=""author1""><certificate fingerprint=""{authorCertificateFingerprintString}"" hashAlgorithm=""SHA256"" allowUntrustedRoot=""false"" /></author>";
}
else
{
trustedSignerString = $@"<repository name=""repo1"" serviceIndex=""https://api.v3serviceIndex.test/json""><certificate fingerprint=""{repoCertificateFingerprintString}"" hashAlgorithm=""SHA256"" allowUntrustedRoot=""false"" /></repository>";
}
var config = $@"
<configuration>
<config>
<add key=""signatureValidationMode"" value=""{validationMode}"" />
</config>
<trustedSigners>
{trustedSignerString}
</trustedSigners>
</configuration>";
var nugetConfigPath = "NuGet.Config";
SettingsTestUtils.CreateConfigurationFile(nugetConfigPath, dir, config);
// Act and Assert
var settings = new Settings(dir);
var clientPolicyContext = ClientPolicyContext.GetClientPolicy(settings, NullLogger.Instance);
var trustProviders = new[]
{
new AllowListVerificationProvider(clientPolicyContext.AllowList, requireNonEmptyAllowList: clientPolicyContext.Policy == SignatureValidationMode.Require)
};
var verifier = new PackageSignatureVerifier(trustProviders);
using (var packageReader = new PackageArchiveReader(signedPackagePath))
{
// Act
var result = await verifier.VerifySignaturesAsync(packageReader, clientPolicyContext.VerifierSettings, CancellationToken.None);
var resultsWithWarnings = result.Results.Where(r => r.GetWarningIssues().Any());
var resultsWithErrors = result.Results.Where(r => r.GetErrorIssues().Any());
var totalWarningIssues = resultsWithWarnings.SelectMany(r => r.GetWarningIssues());
var totalErrorIssues = resultsWithErrors.SelectMany(r => r.GetErrorIssues());
// Assert
result.Valid.Should().BeTrue();
totalWarningIssues.Count().Should().Be(0);
totalErrorIssues.Count().Should().Be(0);
}
}
}
/// <summary>
/// 从文件中加载插件并管理(但不初始化)
/// </summary>
/// <param name="fileName">相对于运行目录的文件名</param>
public async Task LoadPackageAsync(string fileName)
{
using PackageArchiveReader reader = new PackageArchiveReader(File.OpenRead(fileName), false);
NuspecReader nuspecReader = await reader.GetNuspecReaderAsync(CancellationToken.None);
string identity = nuspecReader.GetId();
IEnumerable <FrameworkSpecificGroup> groups = await reader.GetLibItemsAsync(CancellationToken.None);
FrameworkSpecificGroup group = groups.Where(x => x.TargetFramework.GetShortFolderName().StartsWith("netstandard")).OrderByDescending(x => x.TargetFramework.GetShortFolderName()).FirstOrDefault();
foreach (string packageFile in group.Items)
{
if (!packageFile.EndsWith(".dll"))
{
continue;
}
string tmpPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
string path = reader.ExtractFile(packageFile, tmpPath, NullLogger.Instance);
//Type type = Load(await File.ReadAllBytesAsync(path));
Type type = Load(path);
if (type != null)// 为null时则不是插件程序集， 但不一定不是依赖程序集
{
PluginContext context = new PluginContext();
PluginMeta meta = new PluginMeta(identity, fileName, Path.Combine(Environment.CurrentDirectory, "plugins", identity));
context.Meta = meta;
if (!Directory.Exists(meta.SpaceDirectory))
{
Directory.CreateDirectory(meta.SpaceDirectory);
FrameworkSpecificGroup item = (await reader.GetContentItemsAsync(CancellationToken.None)).FirstOrDefault();
if (item != null)
{
foreach (string file in item.Items)
{
if (file.EndsWith('/') || file.EndsWith('\\'))
{
continue;
}
ZipArchiveEntry entry = reader.GetEntry(file);
entry.SaveAsFile(Path.Combine(meta.SpaceDirectory, file.Substring(8)), NullLogger.Instance);
}
}
}
string configFile = Path.Combine(meta.SpaceDirectory, "config.json");
if (File.Exists(configFile))
{
context.Configuration = new Lazy <IConfiguration>(() =>
{
ConfigurationBuilder builder = new ConfigurationBuilder();
builder.AddJsonFile(configFile);
return(builder.Build());
});
}
else
{
context.Configuration = new Lazy <IConfiguration>(() => (new ConfigurationBuilder().Build()));
}
string dataFile = Path.Combine(meta.SpaceDirectory, "data.db");
context.Repository = new Lazy <IRepository>(() => new LiteDbRepository(new LiteDatabase(dataFile)));
plugins.Add(type, context);
}
}
}
/// <summary>
/// Get the converted HTML from the package with Readme markdown.
/// </summary>
/// <param name="readmeFileName">The path of Readme markdown.</param>
/// <param name="packageArchiveReader">
/// The <see cref="PackageArchiveReader"/> instance providing the package metadata.
/// </param>
/// <returns>ReadMe converted to HTML.</returns>
public async Task <RenderedReadMeResult> GetReadMeHtmlAsync(string readmeFileName, PackageArchiveReader packageArchiveReader, Encoding encoding)
{
var readmeMd = await GetReadMeMdAsync(readmeFileName, packageArchiveReader, encoding);
var result = new RenderedReadMeResult
{
Content = readmeMd,
ImagesRewritten = false
};
return(string.IsNullOrEmpty(readmeMd) ?
result :
GetReadMeHtml(readmeMd));
}
public static bool HasReadme(this PackageArchiveReader package) => package.GetFiles().Any(ReadmeFileNames.Contains);
public virtual async Task <IEnumerable <Assembly> > LoadAssembliesFromNuGetPackageAsync(string nupkgFile)
{
if (s_LoadedPackages.ContainsKey(nupkgFile))
{
return(s_LoadedPackages[nupkgFile]);
}
using var packageReader = new PackageArchiveReader(nupkgFile);
var identity = await packageReader.GetIdentityAsync(CancellationToken.None);
var versionedIdentity = identity.Id;
if (identity.HasVersion)
{
versionedIdentity += $"-{identity.Version.OriginalVersion}";
}
if (s_LoadedPackages.ContainsKey(versionedIdentity))
{
return(s_LoadedPackages[versionedIdentity]);
}
Logger.LogInformation("Loading NuGet package: " + Path.GetFileName(nupkgFile));
var fullPath = Path.GetFullPath(nupkgFile).ToLower();
if (m_LoadedPackageAssemblies.ContainsKey(fullPath))
{
if (m_LoadedPackageAssemblies[fullPath].All(d => d.Assembly.IsAlive))
{
return(m_LoadedPackageAssemblies[fullPath].Select(d => d.Assembly.Target).Cast <Assembly>());
}
m_LoadedPackageAssemblies.Remove(fullPath);
}
var assemblies = new List <NuGetAssembly>();
var dependencies = await GetDependenciesAsync(identity);
foreach (var dependency in dependencies.Where(d => !d.Id.Equals(identity.Id)))
{
var package = await GetLatestPackageIdentityAsync(dependency.Id);
if (package == null)
{
throw new Exception($"Failed to load assemblies from {nupkgFile}: dependency {dependency.Id} {dependency.VersionRange.OriginalString} is not installed.");
}
var nupkg = GetNugetPackageFile(package);
if (!File.Exists(nupkg))
{
var latestInstalledVersion = await GetLatestPackageIdentityAsync(dependency.Id);
if (latestInstalledVersion == null)
{
Logger.LogWarning("Failed to resolve: " + dependency.Id);
continue;
}
nupkg = GetNugetPackageFile(latestInstalledVersion);
}
await LoadAssembliesFromNuGetPackageAsync(nupkg);
}
var libItems = packageReader.GetLibItems().ToList();
var nearest = m_FrameworkReducer.GetNearest(m_CurrentFramework, libItems.Select(x => x.TargetFramework));
foreach (var file in libItems.Where(x => x.TargetFramework.Equals(nearest)))
{
foreach (var item in file.Items)
{
try
{
if (!item.EndsWith(".dll"))
{
continue;
}
var entry = packageReader.GetEntry(item);
using var stream = entry.Open();
var ms = new MemoryStream();
await stream.CopyToAsync(ms);
try
{
var asm = m_AssemblyLoader(ms.ToArray());
var name = GetVersionIndependentName(asm.FullName, out var extractedVersion);
var parsedVersion = new Version(extractedVersion);
assemblies.Add(new NuGetAssembly
{
Assembly = new WeakReference(asm),
AssemblyName = name,
Version = parsedVersion,
Package = identity
});
}
finally
{
ms.Close();
stream.Close();
}
}
catch (Exception ex)
{
Logger.LogError($"Failed to load assembly at {item} from file {nupkgFile}");
Logger.LogError(ex.ToString());
}
}
}
m_LoadedPackageAssemblies.Add(fullPath, assemblies);
var result = assemblies.Select(d => d.Assembly.Target).Cast <Assembly>().ToList();
s_LoadedPackages.Add(versionedIdentity, result);
s_LoadedPackages.Add(nupkgFile, result);
return(result);
}
private async Task <PackageValidationResult> ValidateSignatureFilePresenceAsync(
PackageRegistration packageRegistration,
PackageArchiveReader nugetPackage,
User owner,
User currentUser)
{
if (await nugetPackage.IsSignedAsync(CancellationToken.None))
{
if (_config.RejectSignedPackagesWithNoRegisteredCertificate &&
!packageRegistration.IsSigningAllowed())
{
var requiredSigner = packageRegistration.RequiredSigners.FirstOrDefault();
var hasRequiredSigner = requiredSigner != null;
if (hasRequiredSigner)
{
if (requiredSigner == currentUser)
{
return(new PackageValidationResult(
PackageValidationResultType.PackageShouldNotBeSignedButCanManageCertificates,
Strings.UploadPackage_PackageIsSignedButMissingCertificate_CurrentUserCanManageCertificates));
}
else
{
return(new PackageValidationResult(
PackageValidationResultType.PackageShouldNotBeSigned,
string.Format(
Strings.UploadPackage_PackageIsSignedButMissingCertificate_RequiredSigner,
requiredSigner.Username)));
}
}
else
{
var isCurrentUserAnOwner = packageRegistration.Owners.Contains(currentUser);
// Technically, if there is no required signer, any one of the owners can register a
// certificate to resolve this issue. However, we favor either the current user or the provided
// owner since these are both accounts the current user can push on behalf of. In other words
// we provide a message that leads the current user to remedying the problem rather than asking
// someone else for help.
if (isCurrentUserAnOwner)
{
return(new PackageValidationResult(
PackageValidationResultType.PackageShouldNotBeSignedButCanManageCertificates,
Strings.UploadPackage_PackageIsSignedButMissingCertificate_CurrentUserCanManageCertificates));
}
else
{
return(new PackageValidationResult(
PackageValidationResultType.PackageShouldNotBeSigned,
string.Format(
Strings.UploadPackage_PackageIsSignedButMissingCertificate_RequiredSigner,
owner.Username)));
}
}
}
}
else
{
if (packageRegistration.IsSigningRequired())
{
return(PackageValidationResult.Invalid(Strings.UploadPackage_PackageIsNotSigned));
}
}
return(null);
}
internal static async Task DeleteFilesAsync(
IMSBuildProjectSystem projectSystem,
ZipArchive zipArchive,
IEnumerable <string> otherPackagesPath,
FrameworkSpecificGroup frameworkSpecificGroup,
IDictionary <FileTransformExtensions, IPackageFileTransformer> fileTransformers,
CancellationToken cancellationToken)
{
var packageTargetFramework = frameworkSpecificGroup.TargetFramework;
IPackageFileTransformer transformer;
var directoryLookup = frameworkSpecificGroup.Items.ToLookup(
p => Path.GetDirectoryName(ResolveTargetPath(projectSystem,
fileTransformers,
fte => fte.UninstallExtension,
GetEffectivePathForContentFile(packageTargetFramework, p),
out transformer)));
// Get all directories that this package may have added
var directories = from grouping in directoryLookup
from directory in FileSystemUtility.GetDirectories(grouping.Key, altDirectorySeparator: false)
orderby directory.Length descending
select directory;
var projectFullPath = projectSystem.ProjectFullPath;
// Remove files from every directory
foreach (var directory in directories)
{
var directoryFiles = directoryLookup.Contains(directory)
? directoryLookup[directory]
: Enumerable.Empty <string>();
if (!Directory.Exists(Path.Combine(projectFullPath, directory)))
{
continue;
}
foreach (var file in directoryFiles)
{
if (IsEmptyFolder(file))
{
continue;
}
// Resolve the path
var path = ResolveTargetPath(projectSystem,
fileTransformers,
fte => fte.UninstallExtension,
GetEffectivePathForContentFile(packageTargetFramework, file),
out transformer);
if (projectSystem.IsSupportedFile(path))
{
// Register the file being uninstalled (used by web site project system).
projectSystem.RegisterProcessedFiles(new[] { path });
if (transformer != null)
{
// TODO: use the framework from packages.config instead of the current framework
// which may have changed during re-targeting
var projectFramework = projectSystem.TargetFramework;
var matchingFiles = new List <InternalZipFileInfo>();
foreach (var otherPackagePath in otherPackagesPath)
{
using (var otherPackageZipReader = new PackageArchiveReader(otherPackagePath))
{
// use the project framework to find the group that would have been installed
var mostCompatibleContentFilesGroup = GetMostCompatibleGroup(
projectFramework,
otherPackageZipReader.GetContentItems());
if (IsValid(mostCompatibleContentFilesGroup))
{
// Should not normalize content files group.
// It should be like a ZipFileEntry with a forward slash.
foreach (var otherPackageItem in mostCompatibleContentFilesGroup.Items)
{
if (GetEffectivePathForContentFile(packageTargetFramework,
otherPackageItem)
.Equals(
GetEffectivePathForContentFile(packageTargetFramework, file),
StringComparison.OrdinalIgnoreCase))
{
matchingFiles.Add(new InternalZipFileInfo(otherPackagePath,
otherPackageItem));
}
}
}
}
}
try
{
var zipArchiveFileEntry = PathUtility.GetEntry(zipArchive, file);
if (zipArchiveFileEntry != null)
{
await transformer.RevertFileAsync(
() => Task.FromResult(zipArchiveFileEntry.Open()),
path, matchingFiles,
projectSystem,
cancellationToken);
}
}
catch (Exception e)
{
projectSystem.NuGetProjectContext.Log(MessageLevel.Warning, e.Message);
}
}
else
{
try
{
var zipArchiveFileEntry = PathUtility.GetEntry(zipArchive, file);
if (zipArchiveFileEntry != null)
{
await DeleteFileSafeAsync(
path,
() => Task.FromResult(zipArchiveFileEntry.Open()),
projectSystem,
cancellationToken);
}
}
catch (Exception e)
{
projectSystem.NuGetProjectContext.Log(MessageLevel.Warning, e.Message);
}
}
}
}
// If the directory is empty then delete it
if (!GetFilesSafe(projectSystem, directory).Any() &&
!GetDirectoriesSafe(projectSystem, directory).Any())
{
DeleteDirectorySafe(projectSystem, directory);
}
}
}
public virtual IEnumerable <NuGetFramework> GetSupportedFrameworks(PackageArchiveReader package)
{
return(package.GetSupportedFrameworks());
}
// Pushes a package to the Http server.
private async Task PushPackageToServer(string source,
string apiKey,
string pathToPackage,
long packageSize,
bool noServiceEndpoint,
TimeSpan requestTimeout,
ILogger logger,
CancellationToken token)
{
var serviceEndpointUrl = GetServiceEndpointUrl(source, string.Empty, noServiceEndpoint);
var useTempApiKey = IsSourceNuGetSymbolServer(source);
if (useTempApiKey)
{
var maxTries = 3;
using (var packageReader = new PackageArchiveReader(pathToPackage))
{
var packageIdentity = packageReader.GetIdentity();
var success = false;
var retry = 0;
while (retry < maxTries && !success)
{
try
{
retry++;
success = true;
// If user push to https://nuget.smbsrc.net/, use temp api key.
var tmpApiKey = await GetSecureApiKey(packageIdentity, apiKey, noServiceEndpoint, requestTimeout, logger, token);
await _httpSource.ProcessResponseAsync(
new HttpSourceRequest(() => CreateRequest(serviceEndpointUrl, pathToPackage, tmpApiKey, logger))
{
RequestTimeout = requestTimeout,
MaxTries = 1
},
response =>
{
response.EnsureSuccessStatusCode();
return(Task.FromResult(0));
},
logger,
token);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
if (retry == maxTries)
{
throw;
}
success = false;
logger.LogInformation(string.Format(
CultureInfo.CurrentCulture,
Strings.Log_RetryingHttp,
HttpMethod.Put,
source)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(e));
}
}
}
}
else
{
await _httpSource.ProcessResponseAsync(
new HttpSourceRequest(() => CreateRequest(serviceEndpointUrl, pathToPackage, apiKey, logger))
{
RequestTimeout = requestTimeout
},
response =>
{
response.EnsureSuccessStatusCode();
return(Task.FromResult(0));
},
logger,
token);
}
}
/// <summary>
/// Asynchronously deletes a package.
/// </summary>
/// <param name="packageIdentity">A package identity.</param>
/// <param name="nuGetProjectContext">A NuGet project context.</param>
/// <param name="token">A cancellation token.</param>
/// <returns>A task that represents the asynchronous operation.
/// The task result (<see cref="Task{TResult}.Result" />) returns a <see cref="bool" />
/// indication successfulness of the operation.</returns>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="packageIdentity" />
/// is <c>null</c>.</exception>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="nuGetProjectContext" />
/// is <c>null</c>.</exception>
public async Task <bool> DeletePackage(PackageIdentity packageIdentity,
INuGetProjectContext nuGetProjectContext,
CancellationToken token)
{
if (packageIdentity == null)
{
throw new ArgumentNullException(nameof(packageIdentity));
}
if (nuGetProjectContext == null)
{
throw new ArgumentNullException(nameof(nuGetProjectContext));
}
var packageFilePath = GetInstalledPackageFilePath(packageIdentity);
if (File.Exists(packageFilePath))
{
var packageDirectoryPath = Path.GetDirectoryName(packageFilePath);
using (var packageReader = new PackageArchiveReader(packageFilePath))
{
var installedSatelliteFilesPair = await PackageHelper.GetInstalledSatelliteFilesAsync(
packageReader,
PackagePathResolver,
GetPackageSaveMode(nuGetProjectContext),
token);
var runtimePackageDirectory = installedSatelliteFilesPair.Item1;
var installedSatelliteFiles = installedSatelliteFilesPair.Item2;
if (!string.IsNullOrEmpty(runtimePackageDirectory))
{
try
{
// Delete all the package files now
FileSystemUtility.DeleteFiles(installedSatelliteFiles, runtimePackageDirectory, nuGetProjectContext);
}
catch (Exception ex)
{
nuGetProjectContext.Log(MessageLevel.Warning, ex.Message);
// Catch all exception with delete so that the package file is always deleted
}
}
// Get all the package files before deleting the package file
var installedPackageFiles = await PackageHelper.GetInstalledPackageFilesAsync(
packageReader,
packageIdentity,
PackagePathResolver,
GetPackageSaveMode(nuGetProjectContext),
token);
try
{
// Delete all the package files now
FileSystemUtility.DeleteFiles(installedPackageFiles, packageDirectoryPath, nuGetProjectContext);
}
catch (Exception ex)
{
nuGetProjectContext.Log(MessageLevel.Warning, ex.Message);
// Catch all exception with delete so that the package file is always deleted
}
}
// Delete the package file
FileSystemUtility.DeleteFile(packageFilePath, nuGetProjectContext);
// Delete the package directory if any
FileSystemUtility.DeleteDirectorySafe(packageDirectoryPath, recursive: true, nuGetProjectContext: nuGetProjectContext);
// If this is the last package delete the package directory
// If this is the last package delete the package directory
if (!FileSystemUtility.GetFiles(Root, string.Empty, "*.*").Any() &&
!FileSystemUtility.GetDirectories(Root, string.Empty).Any())
{
FileSystemUtility.DeleteDirectorySafe(Root, recursive: false, nuGetProjectContext: nuGetProjectContext);
}
}
return(true);
}
private void FilterDependencies(string targetPath, ISet <string> dependencyToRemove)
{
var fileName = Path.GetFileName(targetPath);
Log.LogMessage($"Updating {fileName}");
using (var fileStream = File.Open(targetPath, FileMode.Open))
using (var package = new ZipArchive(fileStream, ZipArchiveMode.Update))
using (var packageReader = new PackageArchiveReader(fileStream, leaveStreamOpen: true))
{
var referencesFrameworkOnlyAssembly = false;
var nuspecFile = packageReader.GetNuspecFile();
using (var stream = package.OpenFile(nuspecFile))
{
var reader = Manifest.ReadFrom(stream, validateSchema: true);
stream.Position = 0;
var packageBuilder = new PackageBuilder(stream, basePath: null);
var updatedGroups = new List <PackageDependencyGroup>();
foreach (var group in packageBuilder.DependencyGroups)
{
var packages = new List <PackageDependency>();
var updatedGroup = new PackageDependencyGroup(group.TargetFramework, packages);
foreach (var dependency in group.Packages)
{
if (dependencyToRemove.Contains(dependency.Id))
{
referencesFrameworkOnlyAssembly = true;
Log.LogMessage($" Remove dependency on '{dependency.Id}'");
continue;
}
packages.Add(dependency);
}
updatedGroups.Add(updatedGroup);
}
if (referencesFrameworkOnlyAssembly)
{
packageBuilder.DependencyGroups.Clear();
packageBuilder.DependencyGroups.AddRange(updatedGroups);
var updatedManifest = Manifest.Create(packageBuilder);
var inMemory = new MemoryStream();
updatedManifest.Save(inMemory);
inMemory.Position = 0;
// Hack the raw nuspec to add the <frameworkReference> dependency
var rawNuspec = XDocument.Load(inMemory, LoadOptions.PreserveWhitespace);
var ns = rawNuspec.Root.GetDefaultNamespace();
var metadata = rawNuspec.Root.Descendants(ns + "metadata").Single();
metadata.Add(
new XElement(ns + "frameworkReferences",
new XElement(ns + "group",
new XAttribute("targetFramework", NuGetFramework.Parse(SharedFrameworkTargetFramework).GetFrameworkString()),
new XElement(ns + "frameworkReference", new XAttribute("name", "Microsoft.AspNetCore.App")))));
stream.Position = 0;
stream.SetLength(0);
rawNuspec.Save(stream);
Log.LogMessage(MessageImportance.High, "Added <frameworkReference> to {0}", fileName);
}
else
{
Log.LogMessage($"No changes made to {fileName}");
}
}
}
}
private async Task <int> Run()
{
if (_output.HasValue() && _update.HasValue())
{
await Error.WriteLineAsync("'--output' and '--update' options must not be used together.");
return(1);
}
var inputPath = Path.Combine(Directory.GetCurrentDirectory(), "Baseline.xml");
var input = XDocument.Load(inputPath);
var sources = _sources.HasValue() ? _sources.Values.Select(s => s.TrimEnd('/')) : _defaultSources;
var packageSources = sources.Select(s => new PackageSource(s));
var providers = Repository.Provider.GetCoreV3(); // Get v2 and v3 API support
var sourceRepositories = packageSources.Select(ps => new SourceRepository(ps, providers));
if (_update.HasValue())
{
var updateResult = await RunUpdateAsync(inputPath, input, sourceRepositories);
if (updateResult != 0)
{
return(updateResult);
}
}
List <(string packageBase, bool feedV3)> packageBases = new List <(string, bool)>();
foreach (var sourceRepository in sourceRepositories)
{
var feedType = await sourceRepository.GetFeedType(CancellationToken.None);
var feedV3 = feedType == FeedType.HttpV3;
var packageBase = sourceRepository.PackageSource + "/package";
if (feedV3)
{
var resources = await sourceRepository.GetResourceAsync <ServiceIndexResourceV3>();
packageBase = resources.GetServiceEntryUri(ServiceTypes.PackageBaseAddress).ToString().TrimEnd('/');
}
packageBases.Add((packageBase, feedV3));
}
var output = _output.HasValue()
? _output.Value()
: Path.Combine(Directory.GetCurrentDirectory(), "Baseline.Designer.props");
var packageCache = Environment.GetEnvironmentVariable("NUGET_PACKAGES") ??
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages");
var tempDir = Path.Combine(Directory.GetCurrentDirectory(), "obj", "tmp");
Directory.CreateDirectory(tempDir);
var baselineVersion = input.Root.Attribute("Version").Value;
// Baseline and .NET Core versions always align in non-preview releases.
// But, NuspecReader reports netcoreapp5.0 instead of net5.0. We use net5.0 in Baseline.Designer.props.
var parsedVersion = Version.Parse(baselineVersion);
var defaultTarget = ((parsedVersion.Major < 5) ? "netcoreapp" : "net") +
$"{parsedVersion.Major}.{parsedVersion.Minor}";
var matchTarget = $"netcoreapp{parsedVersion.Major}.{parsedVersion.Minor}";
var doc = new XDocument(
new XComment(" Auto generated. Do not edit manually, use eng/tools/BaselineGenerator/ to recreate. "),
new XElement("Project",
new XElement("PropertyGroup",
new XElement("MSBuildAllProjects", "$(MSBuildAllProjects);$(MSBuildThisFileFullPath)"),
new XElement("AspNetCoreBaselineVersion", baselineVersion))));
var client = new HttpClient();
foreach (var pkg in input.Root.Descendants("Package"))
{
var id = pkg.Attribute("Id").Value;
var version = pkg.Attribute("Version").Value;
var packageFileName = $"{id}.{version}.nupkg";
var nupkgPath = Path.Combine(packageCache, id.ToLowerInvariant(), version, packageFileName);
if (!File.Exists(nupkgPath))
{
nupkgPath = Path.Combine(tempDir, packageFileName);
}
if (!File.Exists(nupkgPath))
{
foreach ((string packageBase, bool feedV3) in packageBases)
{
var url = feedV3 ?
$"{packageBase}/{id.ToLowerInvariant()}/{version}/{id.ToLowerInvariant()}.{version}.nupkg" :
$"{packageBase}/{id}/{version}";
Console.WriteLine($"Downloading {url}");
try
{
using (var response = await client.GetStreamAsync(url))
{
using (var file = File.Create(nupkgPath))
{
await response.CopyToAsync(file);
}
}
}
catch (HttpRequestException e) when(e.StatusCode == System.Net.HttpStatusCode.NotFound)
{
// If it's not found, continue onto the next one.
continue;
}
}
if (!File.Exists(nupkgPath))
{
throw new Exception($"Could not download package {id} @ {version} using any input feed");
}
}
using (var reader = new PackageArchiveReader(nupkgPath))
{
doc.Root.Add(new XComment($" Package: {id}"));
var propertyGroup = new XElement(
"PropertyGroup",
new XAttribute("Condition", $" '$(PackageId)' == '{id}' "),
new XElement("BaselinePackageVersion", version));
doc.Root.Add(propertyGroup);
foreach (var group in reader.NuspecReader.GetDependencyGroups())
{
// Don't bother generating empty ItemGroup elements.
if (group.Packages.Count() == 0)
{
continue;
}
// Handle changes to $(DefaultNetCoreTargetFramework) even if some projects are held back.
var targetCondition = $"'$(TargetFramework)' == '{group.TargetFramework.GetShortFolderName()}'";
if (string.Equals(
group.TargetFramework.GetShortFolderName(),
matchTarget,
StringComparison.OrdinalIgnoreCase))
{
targetCondition =
$"('$(TargetFramework)' == '$(DefaultNetCoreTargetFramework)' OR '$(TargetFramework)' == '{defaultTarget}')";
}
var itemGroup = new XElement(
"ItemGroup",
new XAttribute("Condition", $" '$(PackageId)' == '{id}' AND {targetCondition} "));
doc.Root.Add(itemGroup);
foreach (var dependency in group.Packages)
{
itemGroup.Add(
new XElement("BaselinePackageReference",
new XAttribute("Include", dependency.Id),
new XAttribute("Version", dependency.VersionRange.ToString())));
}
}
}
}
var settings = new XmlWriterSettings
{
OmitXmlDeclaration = true,
Encoding = Encoding.UTF8,
Indent = true,
};
using (var writer = XmlWriter.Create(output, settings))
{
doc.Save(writer);
}
Console.WriteLine($"Generated file in {output}");
return(0);
}
private Package CreatePackageFromNuGetPackage(PackageRegistration packageRegistration, PackageArchiveReader nugetPackage, PackageMetadata packageMetadata, PackageStreamMetadata packageStreamMetadata, User user)
{
var package = packageRegistration.Packages.SingleOrDefault(pv => pv.Version == packageMetadata.Version.ToString());
if (package != null)
{
throw new EntityException(
"A package with identifier '{0}' and version '{1}' already exists.", packageRegistration.Id, package.Version);
}
package = new Package();
package.PackageRegistration = packageRegistration;
package = EnrichPackageFromNuGetPackage(package, nugetPackage, packageMetadata, packageStreamMetadata, user);
return package;
}
IEnumerable<string> GetFiles (PackageManagementEventArgs e)
{
if (String.IsNullOrEmpty (e.PackageFilePath))
return new string[0];
using (var packageStream = File.OpenRead (e.PackageFilePath)) {
var zipArchive = new ZipArchive (packageStream);
using (var packageReader = new PackageArchiveReader (zipArchive)) {
return packageReader
.GetFiles ()
.Select (file => Path.GetFullPath (Path.Combine (e.InstallPath, file)))
.ToList ();
}
}
}
private static async Task RunAddPackageAsync(PackageOperation packageOperation, CloudBlockBlob packageBlob, ILogger log, IAsyncCollector <PackageDocument> documentAddCollector)
{
var packagesToIndex = new List <PackageDocument>();
log.LogInformation("Downloading package {packageId}@{packageVersionNormalized} for indexing...",
packageOperation.Id, packageOperation.Version);
using (var packageInputStream = await HttpClient.GetStreamAsync(packageOperation.PackageUrl))
using (var packageInputSeekableStream = TemporaryFileStream.Create())
{
await packageInputStream.CopyToAsync(packageInputSeekableStream);
packageInputSeekableStream.Position = 0;
log.LogInformation("Finished downloading package {packageId}@{packageVersionNormalized} for indexing...",
packageOperation.Id, packageOperation.Version);
using (var nugetPackage = new PackageArchiveReader(packageInputSeekableStream))
{
log.LogInformation("Analyzing package {packageId}@{packageVersionNormalized}...",
packageOperation.Id, packageOperation.Version);
// Get some metadata
var nuspecReader = nugetPackage.NuspecReader;
var packageIdentity = nuspecReader.GetIdentity();
var packageSummary = nuspecReader.GetDescription();
if (string.IsNullOrEmpty(packageSummary))
{
packageSummary = nuspecReader.GetSummary();
}
var packageToIndex = new PackageDocument(
packageIdentity.Id,
packageIdentity.Version.ToNormalizedString(),
packageIdentity.Version.OriginalVersion,
nuspecReader.GetTitle(),
packageSummary,
nuspecReader.GetAuthors(),
nuspecReader.GetTags(),
nuspecReader.GetIconUrl(),
nuspecReader.GetLicenseUrl(),
nuspecReader.GetProjectUrl(),
packageOperation.Published,
AuxiliaryNuGetData.GetDownloadCount(packageIdentity.Id),
packageOperation.IsListed,
packageIdentity.Version.IsPrerelease);
var targetFrameworks = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var typeNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var frameworkSpecificGroups = nugetPackage.GetReferenceItems();
foreach (var frameworkSpecificGroup in frameworkSpecificGroups)
{
// Get some metadata
var targetFramework = frameworkSpecificGroup.TargetFramework.GetShortFolderName();
targetFrameworks.Add(targetFramework);
log.LogInformation(
"Collecting information for {packageId}@{packageVersionNormalized} and framework {targetFramework}...",
packageOperation.Id, packageOperation.Version, targetFramework);
// Collect assembly data
foreach (var item in frameworkSpecificGroup.Items)
{
var entry = nugetPackage.GetEntry(item);
var entryName = item;
log.LogInformation(
"Collecting assembly information from {entryName} for {packageId}@{packageVersionNormalized} and framework {targetFramework}...",
entryName, packageOperation.Id, packageOperation.Version, targetFramework);
using (var assemblyStream = entry.Open())
using (var assemblySeekableStream = TemporaryFileStream.Create())
{
await assemblyStream.CopyToAsync(assemblySeekableStream);
assemblySeekableStream.Position = 0;
using (var portableExecutableReader = new PEReader(assemblySeekableStream))
{
var metadataReader = portableExecutableReader.GetMetadataReader();
foreach (var typeDefinition in metadataReader.TypeDefinitions.Select(metadataReader
.GetTypeDefinition))
{
if (!typeDefinition.Attributes.HasFlag(TypeAttributes.Public))
{
continue;
}
var typeNamespace = metadataReader.GetString(typeDefinition.Namespace);
var typeName = metadataReader.GetString(typeDefinition.Name);
if (typeName.StartsWith("<") || typeName.StartsWith("__Static") ||
typeName.Contains("c__DisplayClass"))
{
continue;
}
log.LogDebug(
"{packageId}@{packageVersionNormalized}, framework {targetFramework}, entry {entryName}: adding {namespace}.{type}",
packageOperation.Id, packageOperation.Version, targetFramework, entryName, typeNamespace, typeName);
typeNames.Add($"{typeNamespace}.{typeName}");
}
}
}
log.LogInformation(
"Finished collecting assembly information from {entryName} for {packageId}@{packageVersionNormalized} and framework {targetFramework}.",
entryName, packageOperation.Id, packageOperation.Version, targetFramework);
}
log.LogInformation(
"Finished collecting information for {packageId}@{packageVersionNormalized} and framework {targetFramework}.",
packageOperation.Id, packageOperation.Version, targetFramework);
}
packageToIndex.TargetFrameworks = targetFrameworks.ToHashSet();
packageToIndex.TypeNames = typeNames.ToHashSet();
log.LogInformation("Finished analyzing package {packageId}@{packageVersionNormalized}.",
packageOperation.Id, packageOperation.Version);
// Add to index blob
try
{
log.LogInformation("Start storing index blob for package {packageId}@{packageVersionNormalized}...",
packageOperation.Id, packageOperation.Version);
await packageBlob.DeleteIfExistsAsync();
using (var jsonStream = await packageBlob.OpenWriteAsync())
using (var jsonWriter = new StreamWriter(jsonStream))
{
JsonSerializer.Serialize(jsonWriter, packagesToIndex);
}
log.LogInformation("Finished storing index blob for package {packageId}@{packageVersionNormalized}.",
packageOperation.Id, packageOperation.Version);
}
catch (Exception ex)
{
log.LogError(ex, "Error storing index blob for package {packageId}@{packageVersionNormalized}.",
packageOperation.Id, packageOperation.Version);
}
// Add to index
try
{
log.LogInformation("Start storing index document for package {packageId}@{packageVersionNormalized}...",
packageOperation.Id, packageOperation.Version);
await documentAddCollector.AddAsync(packageToIndex);
log.LogInformation("Finished storing index document for package {packageId}@{packageVersionNormalized}.",
packageOperation.Id, packageOperation.Version);
}
catch (Exception ex)
{
log.LogError(ex, "Error storing index document for package {packageId}@{packageVersionNormalized}.",
packageOperation.Id, packageOperation.Version);
}
}
}
}
public static void RewritingTheNuspecRemovesInvalidElements()
{
var packageStream = CreateTestPackageStream();
// ensure
using (var nupkg = new PackageArchiveReader(packageStream, leaveStreamOpen: true))
{
var nuspec = nupkg.GetNuspecReader();
Assert.Equal(nuspec.GetMetadata().Any(kvp => kvp.Key == PackageMetadataStrings.Title), true);
}
// Act
NupkgRewriter.RewriteNupkgManifest(packageStream,
new List<Action<ManifestEdit>>
{
metadata => { metadata.Title = ""; }
});
// Assert
using (var nupkg = new PackageArchiveReader(packageStream, leaveStreamOpen: false))
{
var nuspec = nupkg.GetNuspecReader();
Assert.Equal(nuspec.GetMetadata().Any(kvp => kvp.Key == PackageMetadataStrings.Title), false);
}
}
public async Task <PackageValidationResult> ValidateBeforeGeneratePackageAsync(PackageArchiveReader nuGetPackage, PackageMetadata packageMetadata)
{
var warnings = new List <string>();
var result = await CheckPackageEntryCountAsync(nuGetPackage, warnings);
if (result != null)
{
return(result);
}
result = await CheckForUnsignedPushAfterAuthorSignedAsync(
nuGetPackage,
warnings);
if (result != null)
{
return(result);
}
result = CheckRepositoryMetadata(packageMetadata, warnings);
if (result != null)
{
return(result);
}
return(PackageValidationResult.AcceptedWithWarnings(warnings));
}
public virtual IEnumerable<NuGetFramework> GetSupportedFrameworks(PackageArchiveReader package)
{
return package.GetSupportedFrameworks();
}
private async Task <ActionResult> CreatePackageInternal()
{
string id = null;
NuGetVersion version = null;
try
{
var policyResult = await SecurityPolicyService.EvaluateUserPoliciesAsync(SecurityPolicyAction.PackagePush, HttpContext);
if (!policyResult.Success)
{
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, policyResult.ErrorMessage));
}
// Get the user
var currentUser = GetCurrentUser();
using (var packageStream = ReadPackageFromRequest())
{
try
{
using (var archive = new ZipArchive(packageStream, ZipArchiveMode.Read, leaveOpen: true))
{
var reference = DateTime.UtcNow.AddDays(1); // allow "some" clock skew
var entryInTheFuture = archive.Entries.FirstOrDefault(
e => e.LastWriteTime.UtcDateTime > reference);
if (entryInTheFuture != null)
{
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(
CultureInfo.CurrentCulture,
Strings.PackageEntryFromTheFuture,
entryInTheFuture.Name)));
}
}
using (var packageToPush = new PackageArchiveReader(packageStream, leaveStreamOpen: false))
{
try
{
await PackageService.EnsureValid(packageToPush);
}
catch (Exception ex)
{
ex.Log();
var message = Strings.FailedToReadUploadFile;
if (ex is InvalidPackageException || ex is InvalidDataException || ex is EntityException)
{
message = ex.Message;
}
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, message));
}
NuspecReader nuspec;
var errors = ManifestValidator.Validate(packageToPush.GetNuspec(), out nuspec).ToArray();
if (errors.Length > 0)
{
var errorsString = string.Join("', '", errors.Select(error => error.ErrorMessage));
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(
CultureInfo.CurrentCulture,
errors.Length > 1 ? Strings.UploadPackage_InvalidNuspecMultiple : Strings.UploadPackage_InvalidNuspec,
errorsString)));
}
if (nuspec.GetMinClientVersion() > Constants.MaxSupportedMinClientVersion)
{
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(
CultureInfo.CurrentCulture,
Strings.UploadPackage_MinClientVersionOutOfRange,
nuspec.GetMinClientVersion())));
}
User owner;
// Ensure that the user can push packages for this partialId.
id = nuspec.GetId();
version = nuspec.GetVersion();
var packageRegistration = PackageService.FindPackageRegistrationById(id);
if (packageRegistration == null)
{
// Check if the current user's scopes allow pushing a new package ID
var apiScopeEvaluationResult = EvaluateApiScope(ActionsRequiringPermissions.UploadNewPackageId, new ActionOnNewPackageContext(id, ReservedNamespaceService), NuGetScopes.PackagePush);
owner = apiScopeEvaluationResult.Owner;
if (!apiScopeEvaluationResult.IsSuccessful())
{
// User cannot push a new package ID as the current user's scopes does not allow it
return(GetHttpResultFromFailedApiScopeEvaluationForPush(apiScopeEvaluationResult, id, version));
}
}
else
{
// Check if the current user's scopes allow pushing a new version of an existing package ID
var apiScopeEvaluationResult = EvaluateApiScope(ActionsRequiringPermissions.UploadNewPackageVersion, packageRegistration, NuGetScopes.PackagePushVersion, NuGetScopes.PackagePush);
owner = apiScopeEvaluationResult.Owner;
if (!apiScopeEvaluationResult.IsSuccessful())
{
// User cannot push a package as the current user's scopes does not allow it
await AuditingService.SaveAuditRecordAsync(
new FailedAuthenticatedOperationAuditRecord(
currentUser.Username,
AuditedAuthenticatedOperationAction.PackagePushAttemptByNonOwner,
attemptedPackage : new AuditedPackageIdentifier(
id, version.ToNormalizedStringSafe())));
return(GetHttpResultFromFailedApiScopeEvaluationForPush(apiScopeEvaluationResult, id, version));
}
if (packageRegistration.IsLocked)
{
return(new HttpStatusCodeWithBodyResult(
HttpStatusCode.Forbidden,
string.Format(CultureInfo.CurrentCulture, Strings.PackageIsLocked, packageRegistration.Id)));
}
var nuspecVersion = nuspec.GetVersion();
var existingPackage = PackageService.FindPackageByIdAndVersionStrict(nuspec.GetId(), nuspecVersion.ToStringSafe());
if (existingPackage != null)
{
if (existingPackage.PackageStatusKey == PackageStatus.FailedValidation)
{
TelemetryService.TrackPackageReupload(existingPackage);
await PackageDeleteService.HardDeletePackagesAsync(
new[] { existingPackage },
currentUser,
Strings.FailedValidationHardDeleteReason,
Strings.AutomatedPackageDeleteSignature,
deleteEmptyPackageRegistration : false);
}
else
{
return(new HttpStatusCodeWithBodyResult(
HttpStatusCode.Conflict,
string.Format(CultureInfo.CurrentCulture, Strings.PackageExistsAndCannotBeModified,
id, nuspecVersion.ToNormalizedStringSafe())));
}
}
}
var packageStreamMetadata = new PackageStreamMetadata
{
HashAlgorithm = CoreConstants.Sha512HashAlgorithmId,
Hash = CryptographyService.GenerateHash(
packageStream.AsSeekableStream(),
CoreConstants.Sha512HashAlgorithmId),
Size = packageStream.Length
};
var package = await PackageUploadService.GeneratePackageAsync(
id,
packageToPush,
packageStreamMetadata,
owner,
currentUser);
await AutoCuratePackage.ExecuteAsync(package, packageToPush, commitChanges : false);
PackageCommitResult commitResult;
using (Stream uploadStream = packageStream)
{
uploadStream.Position = 0;
commitResult = await PackageUploadService.CommitPackageAsync(
package,
uploadStream.AsSeekableStream());
}
switch (commitResult)
{
case PackageCommitResult.Success:
break;
case PackageCommitResult.Conflict:
return(new HttpStatusCodeWithBodyResult(
HttpStatusCode.Conflict,
Strings.UploadPackage_IdVersionConflict));
default:
throw new NotImplementedException($"The package commit result {commitResult} is not supported.");
}
IndexingService.UpdatePackage(package);
// Write an audit record
await AuditingService.SaveAuditRecordAsync(
new PackageAuditRecord(package, AuditedPackageAction.Create, PackageCreatedVia.Api));
if (!(ConfigurationService.Current.AsynchronousPackageValidationEnabled && ConfigurationService.Current.BlockingAsynchronousPackageValidationEnabled))
{
// Notify user of push unless async validation in blocking mode is used
MessageService.SendPackageAddedNotice(package,
Url.Package(package.PackageRegistration.Id, package.NormalizedVersion, relativeUrl: false),
Url.ReportPackage(package.PackageRegistration.Id, package.NormalizedVersion, relativeUrl: false),
Url.AccountSettings(relativeUrl: false));
}
TelemetryService.TrackPackagePushEvent(package, currentUser, User.Identity);
if (package.SemVerLevelKey == SemVerLevelKey.SemVer2)
{
return(new HttpStatusCodeWithServerWarningResult(HttpStatusCode.Created, Strings.WarningSemVer2PackagePushed));
}
return(new HttpStatusCodeResult(HttpStatusCode.Created));
}
}
catch (InvalidPackageException ex)
{
return(BadRequestForExceptionMessage(ex));
}
catch (InvalidDataException ex)
{
return(BadRequestForExceptionMessage(ex));
}
catch (EntityException ex)
{
return(BadRequestForExceptionMessage(ex));
}
catch (FrameworkException ex)
{
return(BadRequestForExceptionMessage(ex));
}
}
}
catch (HttpException ex) when(ex.IsMaxRequestLengthExceeded())
{
// ASP.NET throws HttpException when maxRequestLength limit is exceeded.
return(new HttpStatusCodeWithBodyResult(
HttpStatusCode.RequestEntityTooLarge,
Strings.PackageFileTooLarge));
}
catch (Exception)
{
TelemetryService.TrackPackagePushFailureEvent(id, version);
throw;
}
}
/// <summary>
/// Given the nupkg file as a read-write stream with random access (e.g. MemoryStream),
/// This will replace the .nuspec file with a new .nuspec generated from
/// a) the old .nuspec file
/// b) supplied edits
///
/// This function leaves readWriteStream open.
/// </summary>
public static void RewriteNupkgManifest(Stream readWriteStream, IEnumerable<Action<ManifestEdit>> edits)
{
if (!readWriteStream.CanRead)
{
throw new ArgumentException(Strings.StreamMustBeReadable, nameof(readWriteStream));
}
if (!readWriteStream.CanWrite)
{
throw new ArgumentException(Strings.StreamMustBeWriteable, nameof(readWriteStream));
}
if (!readWriteStream.CanSeek)
{
throw new ArgumentException(Strings.StreamMustBeSeekable, nameof(readWriteStream));
}
using (var packageArchiveReader = new PackageArchiveReader(readWriteStream, leaveStreamOpen: true))
{
var nuspecReader = packageArchiveReader.GetNuspecReader();
// Read <metadata> node from nuspec
var metadataNode = nuspecReader.Xml.Root.Elements()
.FirstOrDefault(e => StringComparer.Ordinal.Equals(e.Name.LocalName, PackageMetadataStrings.Metadata));
if (metadataNode == null)
{
throw new PackagingException($"The package manifest is missing the '{PackageMetadataStrings.Metadata}' node.");
}
// Convert metadata into a ManifestEdit so that we can run it through the editing pipeline
var editableManifestElements = new ManifestEdit
{
Title = ReadFromMetadata(metadataNode, PackageMetadataStrings.Title),
Authors = ReadFromMetadata(metadataNode, PackageMetadataStrings.Authors),
Copyright = ReadFromMetadata(metadataNode, PackageMetadataStrings.Copyright),
Description = ReadFromMetadata(metadataNode, PackageMetadataStrings.Description),
IconUrl = ReadFromMetadata(metadataNode, PackageMetadataStrings.IconUrl),
LicenseUrl = ReadFromMetadata(metadataNode, PackageMetadataStrings.LicenseUrl),
ProjectUrl = ReadFromMetadata(metadataNode, PackageMetadataStrings.ProjectUrl),
ReleaseNotes = ReadFromMetadata(metadataNode, PackageMetadataStrings.ReleaseNotes),
RequireLicenseAcceptance = ReadBoolFromMetadata(metadataNode, PackageMetadataStrings.RequireLicenseAcceptance),
Summary = ReadFromMetadata(metadataNode, PackageMetadataStrings.Summary),
Tags = ReadFromMetadata(metadataNode, PackageMetadataStrings.Tags)
};
var originalManifestElements = (ManifestEdit)editableManifestElements.Clone();
// Perform edits
foreach (var edit in edits)
{
edit.Invoke(editableManifestElements);
}
// Update the <metadata> node
// Modify metadata elements only if they are changed.
// 1. Do not add empty/null elements to metadata
// 2. Remove the empty/null elements from metadata after edit
// Apart from Authors, Description, Id and Version all other elements are optional.
// Defined by spec here: https://github.com/NuGet/NuGet.Client/blob/dev/src/NuGet.Core/NuGet.Packaging/compiler/resources/nuspec.xsd
WriteToMetadata(metadataNode, PackageMetadataStrings.Title, originalManifestElements.Title, editableManifestElements.Title, canBeRemoved: true);
WriteToMetadata(metadataNode, PackageMetadataStrings.Authors, originalManifestElements.Authors, editableManifestElements.Authors);
WriteToMetadata(metadataNode, PackageMetadataStrings.Copyright, originalManifestElements.Copyright, editableManifestElements.Copyright, canBeRemoved: true);
WriteToMetadata(metadataNode, PackageMetadataStrings.Description, originalManifestElements.Description, editableManifestElements.Description);
WriteToMetadata(metadataNode, PackageMetadataStrings.IconUrl, originalManifestElements.IconUrl, editableManifestElements.IconUrl, canBeRemoved: true);
WriteToMetadata(metadataNode, PackageMetadataStrings.LicenseUrl, originalManifestElements.LicenseUrl, editableManifestElements.LicenseUrl, canBeRemoved: true);
WriteToMetadata(metadataNode, PackageMetadataStrings.ProjectUrl, originalManifestElements.ProjectUrl, editableManifestElements.ProjectUrl, canBeRemoved: true);
WriteToMetadata(metadataNode, PackageMetadataStrings.ReleaseNotes, originalManifestElements.ReleaseNotes, editableManifestElements.ReleaseNotes, canBeRemoved: true);
WriteToMetadata(metadataNode, PackageMetadataStrings.RequireLicenseAcceptance,
originalManifestElements.RequireLicenseAcceptance.ToString(CultureInfo.InvariantCulture).ToLowerInvariant(),
editableManifestElements.RequireLicenseAcceptance.ToString(CultureInfo.InvariantCulture).ToLowerInvariant());
WriteToMetadata(metadataNode, PackageMetadataStrings.Summary, originalManifestElements.Summary, editableManifestElements.Summary, canBeRemoved: true);
WriteToMetadata(metadataNode, PackageMetadataStrings.Tags, originalManifestElements.Tags, editableManifestElements.Tags, canBeRemoved: true);
// Update the package stream
using (var newManifestStream = new MemoryStream())
{
nuspecReader.Xml.Save(newManifestStream);
using (var archive = new ZipArchive(readWriteStream, ZipArchiveMode.Update, leaveOpen: true))
{
var manifestEntries = archive.Entries
.Where(entry => entry.FullName.IndexOf("/", StringComparison.OrdinalIgnoreCase) == -1
&& entry.Name.EndsWith(".nuspec", StringComparison.OrdinalIgnoreCase)).ToList();
if (manifestEntries.Count == 0)
{
throw new PackagingException("Nuspec file does not exist in package.");
}
if (manifestEntries.Count > 1)
{
throw new PackagingException("Package contains multiple nuspec files.");
}
var manifestEntry = manifestEntries[0];
using (var manifestOutputStream = manifestEntry.Open())
{
manifestOutputStream.SetLength(0);
newManifestStream.Position = 0;
newManifestStream.CopyTo(manifestOutputStream);
}
}
}
}
}
public IEnumerable <PackagingLogMessage> Validate(PackageArchiveReader builder)
{
return(Validate(LoadXml(builder.GetNuspec()), builder.GetFiles()));
}
private async Task <PackageValidationResult> CheckLicenseMetadataAsync(PackageArchiveReader nuGetPackage, List <IValidationMessage> warnings)
{
LicenseCheckingNuspecReader nuspecReader = null;
using (var nuspec = nuGetPackage.GetNuspec())
{
nuspecReader = new LicenseCheckingNuspecReader(nuspec);
}
var licenseElement = nuspecReader.LicenseElement;
if (licenseElement != null)
{
if (_config.RejectPackagesWithLicense)
{
return(PackageValidationResult.Invalid(Strings.UploadPackage_NotAcceptingPackagesWithLicense));
}
if (licenseElement.Value.Length > MaxAllowedLicenseNodeValueLength)
{
return(PackageValidationResult.Invalid(Strings.UploadPackage_LicenseNodeValueTooLong));
}
if (HasChildElements(licenseElement))
{
return(PackageValidationResult.Invalid(Strings.UploadPackage_LicenseNodeContainsChildren));
}
var typeText = GetLicenseType(licenseElement);
if (!AllowedLicenseTypes.Contains(typeText, StringComparer.OrdinalIgnoreCase))
{
return(PackageValidationResult.Invalid(string.Format(Strings.UploadPackage_UnsupportedLicenseType, typeText)));
}
var versionText = GetLicenseVersion(licenseElement);
if (versionText != null && AllowedLicenseVersion != versionText)
{
return(PackageValidationResult.Invalid(
string.Format(
Strings.UploadPackage_UnsupportedLicenseVersion,
versionText)));
}
}
var licenseUrl = nuspecReader.GetLicenseUrl();
var licenseMetadata = nuspecReader.GetLicenseMetadata();
var licenseDeprecationUrl = GetExpectedLicenseUrl(licenseMetadata);
if (licenseMetadata == null)
{
if (string.IsNullOrWhiteSpace(licenseUrl))
{
if (!_config.AllowLicenselessPackages)
{
return(PackageValidationResult.Invalid(new LicenseUrlDeprecationValidationMessage(Strings.UploadPackage_MissingLicenseInformation)));
}
else
{
warnings.Add(new LicenseUrlDeprecationValidationMessage(Strings.UploadPackage_LicenseShouldBeSpecified));
}
}
if (licenseDeprecationUrl == licenseUrl)
{
return(PackageValidationResult.Invalid(Strings.UploadPackage_DeprecationUrlUsage));
}
if (!string.IsNullOrWhiteSpace(licenseUrl))
{
if (_config.BlockLegacyLicenseUrl)
{
return(PackageValidationResult.Invalid(new LicenseUrlDeprecationValidationMessage(Strings.UploadPackage_LegacyLicenseUrlNotAllowed)));
}
else
{
warnings.Add(new LicenseUrlDeprecationValidationMessage(Strings.UploadPackage_DeprecatingLicenseUrl));
}
}
// we will return here, so the code below would not need to check for licenseMetadata to be non-null over and over.
return(null);
}
if (licenseMetadata.WarningsAndErrors != null && licenseMetadata.WarningsAndErrors.Any())
{
_telemetryService.TrackInvalidLicenseMetadata(licenseMetadata.License);
return(PackageValidationResult.Invalid(
string.Format(
Strings.UploadPackage_InvalidLicenseMetadata,
string.Join(" ", licenseMetadata.WarningsAndErrors))));
}
if (licenseDeprecationUrl != licenseUrl)
{
if (IsMalformedDeprecationUrl(licenseUrl))
{
return(PackageValidationResult.Invalid(new InvalidUrlEncodingForLicenseUrlValidationMessage()));
}
if (licenseMetadata.Type == LicenseType.File)
{
return(PackageValidationResult.Invalid(
new InvalidLicenseUrlValidationMessage(
string.Format(Strings.UploadPackage_DeprecationUrlRequiredForLicenseFiles, licenseDeprecationUrl))));
}
else if (licenseMetadata.Type == LicenseType.Expression)
{
return(PackageValidationResult.Invalid(
new InvalidLicenseUrlValidationMessage(
string.Format(Strings.UploadPackage_DeprecationUrlRequiredForLicenseExpressions, licenseDeprecationUrl))));
}
}
if (licenseMetadata.Type == LicenseType.File)
{
// fix the path separator. Client enforces forward slashes in all file paths when packing
var licenseFilename = FileNameHelper.GetZipEntryPath(licenseMetadata.License);
if (licenseFilename != licenseMetadata.License)
{
var packageIdentity = nuspecReader.GetIdentity();
_trace.Information($"Transformed license file name from `{licenseMetadata.License}` to `{licenseFilename}` for package {packageIdentity.Id} {packageIdentity.Version}");
}
// check if specified file is present in the package
var fileList = new HashSet <string>(nuGetPackage.GetFiles());
if (!fileList.Contains(licenseFilename))
{
return(PackageValidationResult.Invalid(
string.Format(
Strings.UploadPackage_LicenseFileDoesNotExist,
licenseFilename)));
}
// check if specified file has allowed extension
var licenseFileExtension = Path.GetExtension(licenseFilename);
if (!AllowedLicenseFileExtensions.Contains(licenseFileExtension, StringComparer.OrdinalIgnoreCase))
{
return(PackageValidationResult.Invalid(
string.Format(
Strings.UploadPackage_InvalidLicenseFileExtension,
licenseFileExtension,
string.Join(", ", AllowedLicenseFileExtensions.Where(x => x != string.Empty).Select(extension => $"'{extension}'")))));
}
var licenseFileEntry = nuGetPackage.GetEntry(licenseFilename);
if (licenseFileEntry.Length > MaxAllowedLicenseLengthForUploading)
{
return(PackageValidationResult.Invalid(
string.Format(
Strings.UploadPackage_LicenseFileTooLong,
MaxAllowedLicenseLengthForUploading.ToUserFriendlyBytesLabel())));
}
using (var licenseFileStream = nuGetPackage.GetStream(licenseFilename))
{
if (!await IsStreamLengthMatchesReportedAsync(licenseFileStream, licenseFileEntry.Length))
{
return(PackageValidationResult.Invalid(Strings.UploadPackage_CorruptNupkg));
}
}
// zip streams do not support seeking, so we'll have to reopen them
using (var licenseFileStream = nuGetPackage.GetStream(licenseFilename))
{
// check if specified file is a text file
if (!await TextHelper.LooksLikeUtf8TextStreamAsync(licenseFileStream))
{
return(PackageValidationResult.Invalid(Strings.UploadPackage_LicenseMustBePlainText));
}
}
}
if (licenseMetadata.Type == LicenseType.Expression)
{
if (licenseMetadata.LicenseExpression == null)
{
throw new InvalidOperationException($"Unexpected value of {nameof(licenseMetadata.LicenseExpression)} property");
}
var licenseList = GetLicenseList(licenseMetadata.LicenseExpression);
var unapprovedLicenses = licenseList.Where(license => !license.IsOsiApproved && !license.IsFsfLibre).ToList();
if (unapprovedLicenses.Any())
{
_telemetryService.TrackNonFsfOsiLicenseUse(licenseMetadata.License);
return(PackageValidationResult.Invalid(
string.Format(
Strings.UploadPackage_NonFsfOrOsiLicense, string.Join(", ", unapprovedLicenses.Select(l => l.LicenseID)))));
}
}
return(null);
}
private Package CreatePackageFromNuGetPackage(PackageRegistration packageRegistration, PackageArchiveReader nugetPackage, PackageMetadata packageMetadata, PackageStreamMetadata packageStreamMetadata, User user)
{
var package = packageRegistration.Packages.SingleOrDefault(pv => pv.Version == packageMetadata.Version.OriginalVersion);
if (package != null)
{
throw new EntityException(
"A package with identifier '{0}' and version '{1}' already exists.", packageRegistration.Id, package.Version);
}
package = new Package();
package.PackageRegistration = packageRegistration;
package = EnrichPackageFromNuGetPackage(package, nugetPackage, packageMetadata, packageStreamMetadata, user);
return(package);
}
private static bool DoesNotContainUnsupportedFiles(PackageArchiveReader nugetPackage)
{
foreach (var filePath in nugetPackage.GetFiles())
{
var fi = new FileInfo(filePath);
if (fi.Extension == ".ps1" || fi.Extension == ".t4")
{
return false;
}
}
return true;
}
static int Install([NotNull] ISnapInstallerEnvironment environment,
[NotNull] ISnapInstallerEmbeddedResources snapInstallerEmbeddedResources, [NotNull] ISnapInstaller snapInstaller,
[NotNull] ISnapFilesystem snapFilesystem, [NotNull] ISnapPack snapPack, [NotNull] ISnapOs snapOs,
[NotNull] CoreRunLib coreRunLib, [NotNull] ISnapAppReader snapAppReader, [NotNull] ISnapAppWriter snapAppWriter,
[NotNull] INugetService nugetService, [NotNull] ISnapPackageManager snapPackageManager,
[NotNull] ISnapExtractor snapExtractor, [NotNull] ILog diskLogger,
bool headless)
{
if (environment == null)
{
throw new ArgumentNullException(nameof(environment));
}
if (snapInstallerEmbeddedResources == null)
{
throw new ArgumentNullException(nameof(snapInstallerEmbeddedResources));
}
if (snapInstaller == null)
{
throw new ArgumentNullException(nameof(snapInstaller));
}
if (snapFilesystem == null)
{
throw new ArgumentNullException(nameof(snapFilesystem));
}
if (snapPack == null)
{
throw new ArgumentNullException(nameof(snapPack));
}
if (snapOs == null)
{
throw new ArgumentNullException(nameof(snapOs));
}
if (coreRunLib == null)
{
throw new ArgumentNullException(nameof(coreRunLib));
}
if (snapAppReader == null)
{
throw new ArgumentNullException(nameof(snapAppReader));
}
if (snapAppWriter == null)
{
throw new ArgumentNullException(nameof(snapAppWriter));
}
if (nugetService == null)
{
throw new ArgumentNullException(nameof(nugetService));
}
if (snapPackageManager == null)
{
throw new ArgumentNullException(nameof(snapPackageManager));
}
if (snapExtractor == null)
{
throw new ArgumentNullException(nameof(snapExtractor));
}
if (diskLogger == null)
{
throw new ArgumentNullException(nameof(diskLogger));
}
// NB! All filesystem operations has to be readonly until check that verifies
// current user is not elevated to root has run.
var cancellationToken = environment.CancellationToken;
var installerProgressSource = new SnapProgressSource();
var onFirstAnimationRenderedEvent = new ManualResetEventSlim(false);
var exitCode = 1;
// ReSharper disable once ImplicitlyCapturedClosure
async Task InstallInBackgroundAsync(IMainWindowViewModel mainWindowViewModel)
{
if (mainWindowViewModel == null)
{
throw new ArgumentNullException(nameof(mainWindowViewModel));
}
if (mainWindowViewModel.Headless)
{
diskLogger.Info("Headless install.");
onFirstAnimationRenderedEvent.Dispose();
}
else
{
diskLogger.Info("Waiting for main window to become visible.");
onFirstAnimationRenderedEvent.Wait(cancellationToken);
onFirstAnimationRenderedEvent.Dispose();
diskLogger.Info("Main window should now be visible.");
}
var mainWindowLogger = new LogForwarder(LogLevel.Info, diskLogger, (level, func, exception, parameters) =>
{
var message = func?.Invoke();
if (message == null)
{
return;
}
SetStatusText(mainWindowViewModel, message);
});
if (coreRunLib.IsElevated())
{
var rootUserText = snapOs.OsPlatform == OSPlatform.Windows ? "Administrator" : "root";
mainWindowLogger.Error($"Error! Installer cannot run in an elevated user context: {rootUserText}");
goto done;
}
diskLogger.Debug($"{nameof(environment.Io.WorkingDirectory)}: {environment.Io.WorkingDirectory}");
diskLogger.Debug($"{nameof(environment.Io.ThisExeWorkingDirectory)}: {environment.Io.ThisExeWorkingDirectory}");
var snapAppDllAbsolutePath = snapFilesystem.PathCombine(environment.Io.ThisExeWorkingDirectory, SnapConstants.SnapAppDllFilename);
diskLogger.Debug($"{nameof(snapAppDllAbsolutePath)}: {snapAppDllAbsolutePath}.");
if (!snapFilesystem.FileExists(snapAppDllAbsolutePath))
{
mainWindowLogger.Info($"Unable to find: {snapFilesystem.PathGetFileName(snapAppDllAbsolutePath)}");
goto done;
}
SnapApp snapApp;
SnapChannel snapChannel;
try
{
snapApp = environment.Io.ThisExeWorkingDirectory.GetSnapAppFromDirectory(snapFilesystem, snapAppReader);
snapChannel = snapApp.GetCurrentChannelOrThrow();
}
catch (Exception ex)
{
mainWindowLogger.ErrorException($"Error reading {SnapConstants.SnapAppDllFilename}", ex);
goto done;
}
var nupkgAbsolutePath = snapFilesystem.PathCombine(environment.Io.ThisExeWorkingDirectory, "Setup.nupkg");
var nupkgReleasesAbsolutePath = snapFilesystem.PathCombine(environment.Io.ThisExeWorkingDirectory, snapApp.BuildNugetReleasesFilename());
var offlineInstaller = false;
using (var webInstallerDir = new DisposableDirectory(snapOs.SpecialFolders.NugetCacheDirectory, snapFilesystem))
{
ISnapAppChannelReleases snapAppChannelReleases;
SnapRelease snapReleaseToInstall;
// Offline installer
if (snapFilesystem.FileExists(nupkgAbsolutePath))
{
mainWindowLogger.Info("Offline installer is loading releases nupkg");
try
{
var releasesFileStream = snapFilesystem.FileRead(nupkgReleasesAbsolutePath);
using var packageArchiveReader = new PackageArchiveReader(releasesFileStream);
var snapAppsReleases = await snapExtractor.GetSnapAppsReleasesAsync(packageArchiveReader, snapAppReader, cancellationToken);
snapAppChannelReleases = snapAppsReleases.GetReleases(snapApp, snapChannel);
var isGenesis = !snapAppChannelReleases.HasDeltaReleases();
snapReleaseToInstall = snapAppChannelReleases.GetMostRecentRelease().AsFullRelease(isGenesis);
}
catch (Exception e)
{
mainWindowLogger.ErrorException($"Error reading {nupkgAbsolutePath}", e);
goto done;
}
offlineInstaller = true;
}
// Web installer
else if (snapFilesystem.FileExists(snapAppDllAbsolutePath))
{
mainWindowLogger.Info("Web installer is downloading releases nupkg");
try
{
var(snapAppsReleases, packageSource, releasesMemoryStream) =
await snapPackageManager.GetSnapsReleasesAsync(snapApp, mainWindowLogger, cancellationToken);
releasesMemoryStream?.Dispose();
if (snapAppsReleases == null)
{
mainWindowLogger.Error("Failed to download releases nupkg. Try rerunning the installer.");
goto done;
}
var snapAppsReleasesBytes = snapAppWriter.ToSnapAppsReleases(snapAppsReleases);
await snapFilesystem.FileWriteAsync(snapAppsReleasesBytes, nupkgReleasesAbsolutePath, cancellationToken);
snapAppChannelReleases = snapAppsReleases.GetReleases(snapApp, snapApp.GetCurrentChannelOrThrow());
if (!snapAppChannelReleases.Any())
{
mainWindowLogger.Error($"Unable to find any releases in channel: {snapAppChannelReleases.Channel.Name}.");
goto done;
}
var isGenesis = snapAppChannelReleases.Count() == 1;
snapReleaseToInstall = snapAppChannelReleases.GetMostRecentRelease().AsFullRelease(isGenesis);
snapApp.Version = snapReleaseToInstall.Version;
mainWindowLogger.Info($"Current version: {snapApp.Version}. Channel: {snapAppChannelReleases.Channel.Name}.");
// ReSharper disable once MethodSupportsCancellation
await Task.Delay(TimeSpan.FromSeconds(3));
mainWindowLogger.Info("Downloading required assets");
void UpdateProgress(string type, int totalPercentage,
long releasesChecksummed = 0, long releasesToChecksum = 0,
long releasesDownloaded = 0, long releasesToDownload = 0,
long filesRestored = 0, long filesToRestore = 0,
long totalBytesDownloaded = 0, long totalBytesToDownload = 0)
{
void SetProgressText(long current, long total, string defaultText, string pluralText)
{
var outputText = total > 1 ? pluralText : defaultText;
switch (type)
{
case "Download":
if (total > 1)
{
SetStatusText(mainWindowViewModel,
$"{outputText} ({totalPercentage}%): {current} of {total}. " +
$"Downloaded so far: {totalBytesDownloaded.BytesAsHumanReadable()}. " +
$"Total: {totalBytesToDownload.BytesAsHumanReadable()}");
goto incrementProgress;
}
SetStatusText(mainWindowViewModel,
$"{outputText} ({totalPercentage}%): " +
$"Downloaded so far: {totalBytesDownloaded.BytesAsHumanReadable()}. " +
$"Total: {totalBytesToDownload.BytesAsHumanReadable()}");
goto incrementProgress;
default:
if (total > 1)
{
SetStatusText(mainWindowViewModel, $"{outputText} ({totalPercentage}%): {current} of {total}.");
goto incrementProgress;
}
SetStatusText(mainWindowViewModel, $"{outputText}: {totalPercentage}%");
goto incrementProgress;
}
incrementProgress:
installerProgressSource.Raise(totalPercentage);
}
switch (type)
{
case "Checksum":
SetProgressText(releasesChecksummed, releasesToChecksum, "Validating payload", "Validating payloads");
break;
case "Download":
SetProgressText(releasesDownloaded, releasesToDownload, "Downloading payload", "Downloading payloads");
break;
case "Restore":
SetProgressText(filesRestored, filesToRestore, "Restoring file", "Restoring files");
break;
default:
diskLogger.Warn($"Unknown progress type: {type}");
break;
}
}
var snapPackageManagerProgressSource = new SnapPackageManagerProgressSource
{
ChecksumProgress = x => UpdateProgress("Checksum",
x.progressPercentage,
x.releasesChecksummed,
x.releasesToChecksum),
DownloadProgress = x => UpdateProgress("Download",
x.progressPercentage,
releasesDownloaded: x.releasesDownloaded,
releasesToDownload: x.releasesToDownload,
totalBytesDownloaded: x.totalBytesDownloaded,
totalBytesToDownload: x.totalBytesToDownload),
RestoreProgress = x => UpdateProgress("Restore",
x.progressPercentage,
filesRestored: x.filesRestored,
filesToRestore: x.filesToRestore)
};
var restoreSummary = await snapPackageManager.RestoreAsync(webInstallerDir.WorkingDirectory, snapAppChannelReleases,
packageSource, SnapPackageManagerRestoreType.Default, snapPackageManagerProgressSource, diskLogger, cancellationToken);
if (!restoreSummary.Success)
{
mainWindowLogger.Info("Unknown error while restoring assets.");
goto done;
}
mainWindowLogger.Info("Preparing to install payload");
var setupNupkgAbsolutePath = snapOs.Filesystem.PathCombine(webInstallerDir.WorkingDirectory, snapReleaseToInstall.Filename);
if (!snapFilesystem.FileExists(setupNupkgAbsolutePath))
{
mainWindowLogger.Error($"Payload does not exist on disk: {setupNupkgAbsolutePath}.");
goto done;
}
nupkgAbsolutePath = snapFilesystem.PathCombine(environment.Io.ThisExeWorkingDirectory, "Setup.nupkg");
mainWindowLogger.Info("Copying payload to installer directory");
snapOs.Filesystem.FileDeleteIfExists(nupkgAbsolutePath);
await snapOs.Filesystem.FileCopyAsync(setupNupkgAbsolutePath, nupkgAbsolutePath, cancellationToken);
mainWindowLogger.Info("Successfully copied payload");
installerProgressSource.Reset();
}
catch (Exception e)
{
mainWindowLogger.ErrorException("Unknown error while restoring assets", e);
goto done;
}
}
else
{
mainWindowLogger.Error("Unknown error. Could not find offline or web installer payload.");
goto done;
}
diskLogger.Trace($"Offline installer: {offlineInstaller}");
diskLogger.Trace($"{nameof(nupkgAbsolutePath)}: {nupkgAbsolutePath}");
diskLogger.Trace($"{nameof(nupkgReleasesAbsolutePath)}: {nupkgReleasesAbsolutePath}");
if (!snapFilesystem.FileExists(nupkgAbsolutePath))
{
mainWindowLogger.Error($"Unable to find installer payload: {snapFilesystem.PathGetFileName(nupkgAbsolutePath)}");
goto done;
}
mainWindowLogger.Info("Attempting to read payload release details");
if (!snapFilesystem.FileExists(nupkgReleasesAbsolutePath))
{
mainWindowLogger.Error($"Unable to find releases nupkg: {snapFilesystem.PathGetFileName(nupkgReleasesAbsolutePath)}");
goto done;
}
var baseDirectory = snapFilesystem.PathCombine(snapOs.SpecialFolders.LocalApplicationData, snapApp.Id);
mainWindowLogger.Info($"Installing {snapApp.Id}. Channel name: {snapChannel.Name}");
try
{
var snapAppInstalled = await snapInstaller.InstallAsync(nupkgAbsolutePath, baseDirectory,
snapReleaseToInstall, snapChannel, installerProgressSource, mainWindowLogger, cancellationToken, offlineInstaller);
if (snapAppInstalled == null)
{
goto done;
}
if (!offlineInstaller &&
snapAppChannelReleases.HasDeltaReleases())
{
var snapReleasesToCopy = new List <SnapRelease>
{
snapAppChannelReleases.GetGenesisRelease()
};
snapReleasesToCopy.AddRange(snapAppChannelReleases.GetDeltaReleases());
if (snapReleasesToCopy.Any())
{
var totalSnapReleasesToCopyCount = snapReleasesToCopy.Count;
mainWindowLogger.Info($"Copying 1 of {totalSnapReleasesToCopyCount} payloads to application directory.");
var packagesDirectory = snapFilesystem.PathCombine(baseDirectory, "packages");
var snapReleasesCopied = 1;
foreach (var snapRelease in snapReleasesToCopy)
{
var nupkgPackageWebInstallerDirectoryAbsolutePath = snapFilesystem.PathCombine(
webInstallerDir.WorkingDirectory, snapRelease.Filename);
var nupkgPackagePackagesDirectoryAbsolutePath = snapFilesystem.PathCombine(
packagesDirectory, snapRelease.Filename);
await snapFilesystem.FileCopyAsync(
nupkgPackageWebInstallerDirectoryAbsolutePath,
nupkgPackagePackagesDirectoryAbsolutePath, cancellationToken);
mainWindowLogger.Info($"Copied {snapReleasesCopied} of {totalSnapReleasesToCopyCount} payloads to application directory.");
++snapReleasesCopied;
}
mainWindowLogger.Info("Successfully copied all payloads.");
}
snapFilesystem.FileDeleteIfExists(nupkgAbsolutePath);
}
mainWindowLogger.Info($"Successfully installed {snapApp.Id}.");
exitCode = 0;
}
catch (Exception e)
{
mainWindowLogger.ErrorException("Unknown error during install", e);
}
}
done:
// Give user enough time to read final log message.
Thread.Sleep(exitCode == 0 ? 3000 : 10000);
environment.Shutdown();
}
if (headless)
{
try
{
InstallInBackgroundAsync(new ConsoleMainViewModel()).Wait(cancellationToken);
}
catch (OperationCanceledException)
{
// ignore
}
return(exitCode);
}
var avaloniaApp = BuildAvaloniaApp <App>()
.AfterSetup(builder =>
{
MainWindow.Environment = environment;
MainWindow.ViewModel = new AvaloniaMainWindowViewModel(snapInstallerEmbeddedResources,
installerProgressSource, () => onFirstAnimationRenderedEvent.Set(), cancellationToken);
Task.Factory.StartNew(() => InstallInBackgroundAsync(MainWindow.ViewModel), TaskCreationOptions.LongRunning);
});
avaloniaApp.StartWithClassicDesktopLifetime(null);
return(exitCode);
}
public virtual Package EnrichPackageFromNuGetPackage(
Package package,
PackageArchiveReader packageArchive,
PackageMetadata packageMetadata,
PackageStreamMetadata packageStreamMetadata,
User user)
{
// Version must always be the exact string from the nuspec, which ToString will return to us.
// However, we do also store a normalized copy for looking up later.
package.Version = packageMetadata.Version.ToString();
package.NormalizedVersion = packageMetadata.Version.ToNormalizedString();
package.Description = packageMetadata.Description;
package.ReleaseNotes = packageMetadata.ReleaseNotes;
package.HashAlgorithm = packageStreamMetadata.HashAlgorithm;
package.Hash = packageStreamMetadata.Hash;
package.PackageFileSize = packageStreamMetadata.Size;
package.Language = packageMetadata.Language;
package.Copyright = packageMetadata.Copyright;
package.FlattenedAuthors = packageMetadata.Authors.Flatten();
package.IsPrerelease = packageMetadata.Version.IsPrerelease;
package.Listed = true;
package.RequiresLicenseAcceptance = packageMetadata.RequireLicenseAcceptance;
package.Summary = packageMetadata.Summary;
package.Tags = PackageHelper.ParseTags(packageMetadata.Tags);
package.Title = packageMetadata.Title;
package.User = user;
package.IconUrl = packageMetadata.IconUrl.ToEncodedUrlStringOrNull();
package.LicenseUrl = packageMetadata.LicenseUrl.ToEncodedUrlStringOrNull();
package.ProjectUrl = packageMetadata.ProjectUrl.ToEncodedUrlStringOrNull();
package.MinClientVersion = packageMetadata.MinClientVersion.ToStringOrNull();
#pragma warning disable 618 // TODO: remove Package.Authors completely once prodution services definitely no longer need it
foreach (var author in packageMetadata.Authors)
{
package.Authors.Add(new PackageAuthor { Name = author });
}
#pragma warning restore 618
var supportedFrameworks = GetSupportedFrameworks(packageArchive).Select(fn => fn.ToShortNameOrNull()).ToArray();
if (!supportedFrameworks.AnySafe(sf => sf == null))
{
ValidateSupportedFrameworks(supportedFrameworks);
foreach (var supportedFramework in supportedFrameworks)
{
package.SupportedFrameworks.Add(new PackageFramework {TargetFramework = supportedFramework});
}
}
package.Dependencies = packageMetadata
.GetDependencyGroups()
.AsPackageDependencyEnumerable()
.ToList();
package.FlattenedDependencies = package.Dependencies.Flatten();
return package;
}
public InstalledPackage(string fileName)
{
FileName = fileName;
Metadata = new PackageArchiveReader(fileName).NuspecReader;
}
private static void Run(CommandLineApplication cmd, ILogger log)
{
cmd.Description = "Add empty dependency groups or remove dependencies from existing groups.";
var idFilter = cmd.Option(Constants.IdFilterTemplate, Constants.IdFilterDesc, CommandOptionType.SingleValue);
var versionFilter = cmd.Option(Constants.VersionFilterTemplate, Constants.VersionFilterDesc, CommandOptionType.SingleValue);
var excludeSymbolsFilter = cmd.Option(Constants.ExcludeSymbolsTemplate, Constants.ExcludeSymbolsDesc, CommandOptionType.NoValue);
var highestVersionFilter = cmd.Option(Constants.HighestVersionFilterTemplate, Constants.HighestVersionFilterDesc, CommandOptionType.NoValue);
var frameworkOption = cmd.Option("-f|--framework", "Group target frameworks. Use 'any' for the default group.", CommandOptionType.MultipleValue);
var argRoot = cmd.Argument(
"[root]",
Constants.MultiplePackagesRootDesc,
multipleValues: true);
cmd.HelpOption(Constants.HelpOption);
var required = new List <CommandOption>()
{
frameworkOption
};
cmd.OnExecute(() =>
{
try
{
var inputs = argRoot.Values;
if (inputs.Count < 1)
{
inputs.Add(Directory.GetCurrentDirectory());
}
// Validate parameters
foreach (var requiredOption in required)
{
if (!requiredOption.HasValue())
{
throw new ArgumentException($"Missing required parameter --{requiredOption.LongName}.");
}
}
var frameworks = new HashSet <NuGetFramework>();
if (frameworkOption.HasValue())
{
foreach (var option in frameworkOption.Values)
{
var fw = NuGetFramework.Parse(option);
log.LogInformation($"adding empty dependency groups for {fw.GetShortFolderName()}");
frameworks.Add(fw);
}
}
var packages = Util.GetPackagesWithFilter(idFilter, versionFilter, excludeSymbolsFilter, highestVersionFilter, inputs.ToArray());
foreach (var package in packages)
{
log.LogMinimal($"processing {package}");
// Get nuspec file path
string nuspecPath = null;
XDocument nuspecXml = null;
using (var packageReader = new PackageArchiveReader(package))
{
nuspecPath = packageReader.GetNuspecFile();
nuspecXml = XDocument.Load(packageReader.GetNuspec());
}
var metadata = Util.GetMetadataElement(nuspecXml);
var ns = metadata.GetDefaultNamespace().NamespaceName;
var dependenciesNode = metadata.Elements().FirstOrDefault(e => e.Name.LocalName.Equals("dependencies", StringComparison.OrdinalIgnoreCase));
if (dependenciesNode == null)
{
dependenciesNode = new XElement(XName.Get("dependencies", ns));
metadata.Add(dependenciesNode);
}
// Convert non-grouped to group
var rootDeps = dependenciesNode.Elements()
.Where(e => e.Name.LocalName.Equals("dependency", StringComparison.OrdinalIgnoreCase))
.ToArray();
if (rootDeps.Length > 1)
{
var anyGroup = new XElement(XName.Get("group", ns));
dependenciesNode.AddFirst(anyGroup);
foreach (var rootDep in rootDeps)
{
rootDep.Remove();
anyGroup.Add(rootDep);
}
}
// Remove existing groups
foreach (var node in dependenciesNode.Elements()
.Where(e => e.Name.LocalName.Equals("group", StringComparison.OrdinalIgnoreCase))
.ToArray())
{
var groupFramework = NuGetFramework.AnyFramework;
var tfm = node.Attribute(XName.Get("targetFramework"))?.Value;
if (!string.IsNullOrEmpty(tfm))
{
groupFramework = NuGetFramework.Parse(tfm);
}
if (frameworks.Remove(groupFramework))
{
foreach (var child in node.Elements().ToArray())
{
child.Remove();
}
}
}
// Add empty groups for those remaining
foreach (var fw in frameworks)
{
var groupNode = new XElement(XName.Get("group", ns));
if (!fw.IsAny)
{
var version = fw.Version.ToString();
if (version.EndsWith(".0.0"))
{
version = version.Substring(0, version.Length - 4);
}
if (version.EndsWith(".0") &&
version.IndexOf('.') != version.LastIndexOf('.'))
{
version = version.Substring(0, version.Length - 2);
}
groupNode.Add(new XAttribute(XName.Get("targetFramework"), $"{fw.Framework}{version}"));
}
dependenciesNode.Add(groupNode);
}
// Update zip
Util.AddOrReplaceZipEntry(package, nuspecPath, nuspecXml, log);
}
return(0);
}
catch (Exception ex)
{
log.LogError(ex.Message);
log.LogDebug(ex.ToString());
}
return(1);
});
}
/// <summary>
/// This method does not perform the ownership validations for the symbols package. It is the responsibility
/// of the caller to do it. Also, this method does not dispose the <see cref="Stream"/> object, the caller
/// should take care of it.
/// </summary>
/// <param name="symbolPackageStream"><see cref="Stream"/> object for the symbols package.</param>
/// <param name="currentUser">The user performing the uploads.</param>
/// <returns>Awaitable task for <see cref="SymbolPackageValidationResult"/></returns>
public async Task <SymbolPackageValidationResult> ValidateUploadedSymbolsPackage(Stream symbolPackageStream, User currentUser)
{
Package package = null;
// Check if symbol package upload is allowed for this user.
if (!_contentObjectService.SymbolsConfiguration.IsSymbolsUploadEnabledForUser(currentUser))
{
return(SymbolPackageValidationResult.UserNotAllowedToUpload(Strings.SymbolsPackage_UploadNotAllowed));
}
try
{
if (ZipArchiveHelpers.FoundEntryInFuture(symbolPackageStream, out ZipArchiveEntry entryInTheFuture))
{
return(SymbolPackageValidationResult.Invalid(string.Format(
CultureInfo.CurrentCulture,
Strings.PackageEntryFromTheFuture,
entryInTheFuture.Name)));
}
using (var packageToPush = new PackageArchiveReader(symbolPackageStream, leaveStreamOpen: true))
{
var nuspec = packageToPush.GetNuspecReader();
var id = nuspec.GetId();
var version = nuspec.GetVersion();
var normalizedVersion = version.ToNormalizedStringSafe();
// Ensure the corresponding package exists before pushing a snupkg.
package = _packageService.FindPackageByIdAndVersionStrict(id, version.ToStringSafe());
if (package == null || package.PackageStatusKey == PackageStatus.Deleted)
{
return(SymbolPackageValidationResult.MissingPackage(string.Format(
CultureInfo.CurrentCulture,
Strings.SymbolsPackage_PackageIdAndVersionNotFound,
id,
normalizedVersion)));
}
// Do not allow to upload a snupkg to a package which has symbols package pending validations.
if (package.SymbolPackages.Any(sp => sp.StatusKey == PackageStatus.Validating))
{
return(SymbolPackageValidationResult.SymbolsPackageExists(Strings.SymbolsPackage_ConflictValidating));
}
try
{
await _symbolPackageService.EnsureValidAsync(packageToPush);
}
catch (Exception ex)
{
ex.Log();
var message = Strings.SymbolsPackage_FailedToReadPackage;
if (ex is InvalidPackageException || ex is InvalidDataException || ex is EntityException)
{
message = ex.Message;
}
_telemetryService.TrackSymbolPackageFailedGalleryValidationEvent(id, normalizedVersion);
return(SymbolPackageValidationResult.Invalid(message));
}
}
return(SymbolPackageValidationResult.AcceptedForPackage(package));
}
catch (Exception ex) when(ex is InvalidPackageException ||
ex is InvalidDataException ||
ex is EntityException ||
ex is FrameworkException)
{
return(SymbolPackageValidationResult.Invalid(
string.Format(CultureInfo.CurrentCulture, Strings.UploadPackage_InvalidPackage, ex.Message)));
}
}
private async Task <ActionResult> CreatePackageInternal()
{
var policyResult = await SecurityPolicyService.EvaluateAsync(SecurityPolicyAction.PackagePush, HttpContext);
if (!policyResult.Success)
{
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, policyResult.ErrorMessage));
}
// Get the user
var user = GetCurrentUser();
using (var packageStream = ReadPackageFromRequest())
{
try
{
using (var archive = new ZipArchive(packageStream, ZipArchiveMode.Read, leaveOpen: true))
{
var reference = DateTime.UtcNow.AddDays(1); // allow "some" clock skew
var entryInTheFuture = archive.Entries.FirstOrDefault(
e => e.LastWriteTime.UtcDateTime > reference);
if (entryInTheFuture != null)
{
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(
CultureInfo.CurrentCulture,
Strings.PackageEntryFromTheFuture,
entryInTheFuture.Name)));
}
}
using (var packageToPush = new PackageArchiveReader(packageStream, leaveStreamOpen: false))
{
try
{
PackageService.EnsureValid(packageToPush);
}
catch (Exception ex)
{
ex.Log();
var message = Strings.FailedToReadUploadFile;
if (ex is InvalidPackageException || ex is InvalidDataException || ex is EntityException)
{
message = ex.Message;
}
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, message));
}
NuspecReader nuspec;
var errors = ManifestValidator.Validate(packageToPush.GetNuspec(), out nuspec).ToArray();
if (errors.Length > 0)
{
var errorsString = string.Join("', '", errors.Select(error => error.ErrorMessage));
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(
CultureInfo.CurrentCulture,
errors.Length > 1 ? Strings.UploadPackage_InvalidNuspecMultiple : Strings.UploadPackage_InvalidNuspec,
errorsString)));
}
if (nuspec.GetMinClientVersion() > Constants.MaxSupportedMinClientVersion)
{
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(
CultureInfo.CurrentCulture,
Strings.UploadPackage_MinClientVersionOutOfRange,
nuspec.GetMinClientVersion())));
}
// Ensure that the user can push packages for this partialId.
var id = nuspec.GetId();
var packageRegistration = PackageService.FindPackageRegistrationById(id);
IReadOnlyCollection <ReservedNamespace> userOwnedNamespaces = null;
if (packageRegistration == null)
{
// Check if API key allows pushing a new package id
if (!ApiKeyScopeAllows(
subject: id,
requestedActions: NuGetScopes.PackagePush))
{
// User cannot push a new package ID as the API key scope does not allow it
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.Unauthorized, Strings.ApiKeyNotAuthorized));
}
// For a new package id verify that the user is allowed to push to the matching namespaces, if any.
var isPushAllowed = ReservedNamespaceService.IsPushAllowed(id, user, out userOwnedNamespaces);
if (!isPushAllowed)
{
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.Conflict, Strings.UploadPackage_IdNamespaceConflict));
}
}
else
{
// Is the user allowed to push this Id?
if (!packageRegistration.IsOwner(user))
{
// Audit that a non-owner tried to push the package
await AuditingService.SaveAuditRecordAsync(
new FailedAuthenticatedOperationAuditRecord(
user.Username,
AuditedAuthenticatedOperationAction.PackagePushAttemptByNonOwner,
attemptedPackage : new AuditedPackageIdentifier(
id, nuspec.GetVersion().ToNormalizedStringSafe())));
// User cannot push a package to an ID owned by another user.
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.Conflict,
string.Format(CultureInfo.CurrentCulture, Strings.PackageIdNotAvailable,
id)));
}
// Check if API key allows pushing the current package id
if (!ApiKeyScopeAllows(
packageRegistration.Id,
NuGetScopes.PackagePushVersion, NuGetScopes.PackagePush))
{
// User cannot push a package as the API key scope does not allow it
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.Unauthorized, Strings.ApiKeyNotAuthorized));
}
// Check if a particular Id-Version combination already exists. We eventually need to remove this check.
string normalizedVersion = nuspec.GetVersion().ToNormalizedString();
bool packageExists =
packageRegistration.Packages.Any(
p => string.Equals(
p.NormalizedVersion,
normalizedVersion,
StringComparison.OrdinalIgnoreCase));
if (packageExists)
{
return(new HttpStatusCodeWithBodyResult(
HttpStatusCode.Conflict,
string.Format(CultureInfo.CurrentCulture, Strings.PackageExistsAndCannotBeModified,
id, nuspec.GetVersion().ToNormalizedStringSafe())));
}
}
var packageStreamMetadata = new PackageStreamMetadata
{
HashAlgorithm = Constants.Sha512HashAlgorithmId,
Hash = CryptographyService.GenerateHash(packageStream.AsSeekableStream()),
Size = packageStream.Length
};
var package = await PackageUploadService.GeneratePackageAsync(
id,
packageToPush,
packageStreamMetadata,
user,
commitChanges : false);
await AutoCuratePackage.ExecuteAsync(package, packageToPush, commitChanges : false);
using (Stream uploadStream = packageStream)
{
uploadStream.Position = 0;
try
{
await PackageFileService.SavePackageFileAsync(package, uploadStream.AsSeekableStream());
}
catch (InvalidOperationException ex)
{
ex.Log();
return(new HttpStatusCodeWithBodyResult(HttpStatusCode.Conflict, Strings.UploadPackage_IdVersionConflict));
}
}
try
{
await EntitiesContext.SaveChangesAsync();
}
catch
{
// If saving to the DB fails for any reason, we need to delete the package we just saved.
await PackageFileService.DeletePackageFileAsync(nuspec.GetId(), nuspec.GetVersion().ToNormalizedString());
throw;
}
IndexingService.UpdatePackage(package);
// Write an audit record
await AuditingService.SaveAuditRecordAsync(
new PackageAuditRecord(package, AuditedPackageAction.Create, PackageCreatedVia.Api));
// Notify user of push
MessageService.SendPackageAddedNotice(package,
Url.Package(package.PackageRegistration.Id, package.NormalizedVersion, relativeUrl: false),
Url.ReportPackage(package.PackageRegistration.Id, package.NormalizedVersion, relativeUrl: false),
Url.AccountSettings(relativeUrl: false));
TelemetryService.TrackPackagePushEvent(package, user, User.Identity);
if (package.SemVerLevelKey == SemVerLevelKey.SemVer2)
{
return(new HttpStatusCodeWithServerWarningResult(HttpStatusCode.Created, Strings.WarningSemVer2PackagePushed));
}
return(new HttpStatusCodeResult(HttpStatusCode.Created));
}
}
catch (InvalidPackageException ex)
{
return(BadRequestForExceptionMessage(ex));
}
catch (InvalidDataException ex)
{
return(BadRequestForExceptionMessage(ex));
}
catch (EntityException ex)
{
return(BadRequestForExceptionMessage(ex));
}
catch (FrameworkException ex)
{
return(BadRequestForExceptionMessage(ex));
}
}
}
public async Task <PackageIndexingResult> IndexAsync(Stream packageStream, CancellationToken cancellationToken)
{
// Try to extract all the necessary information from the package.
Package package;
Stream nuspecStream;
Stream readmeStream;
Stream iconStream;
try
{
using (var packageReader = new PackageArchiveReader(packageStream, leaveStreamOpen: true))
{
package = packageReader.GetPackageMetadata();
package.Published = _time.UtcNow;
nuspecStream = await packageReader.GetNuspecAsync(cancellationToken);
nuspecStream = await nuspecStream.AsTemporaryFileStreamAsync();
if (package.HasReadme)
{
readmeStream = await packageReader.GetReadmeAsync(cancellationToken);
readmeStream = await readmeStream.AsTemporaryFileStreamAsync();
}
else
{
readmeStream = null;
}
if (package.HasEmbeddedIcon)
{
iconStream = await packageReader.GetIconAsync(cancellationToken);
iconStream = await iconStream.AsTemporaryFileStreamAsync();
}
else
{
iconStream = null;
}
}
}
catch (Exception e)
{
_logger.LogError(e, "Uploaded package is invalid");
return(PackageIndexingResult.InvalidPackage);
}
// The package is well-formed. Ensure this is a new package.
if (await _packages.ExistsAsync(package.Id, package.Version, cancellationToken))
{
if (!_options.Value.AllowPackageOverwrites)
{
return(PackageIndexingResult.PackageAlreadyExists);
}
await _packages.HardDeletePackageAsync(package.Id, package.Version, cancellationToken);
await _storage.DeleteAsync(package.Id, package.Version, cancellationToken);
}
// TODO: Add more package validations
// TODO: Call PackageArchiveReader.ValidatePackageEntriesAsync
_logger.LogInformation(
"Validated package {PackageId} {PackageVersion}, persisting content to storage...",
package.Id,
package.NormalizedVersionString);
try
{
packageStream.Position = 0;
await _storage.SavePackageContentAsync(
package,
packageStream,
nuspecStream,
readmeStream,
iconStream,
cancellationToken);
}
catch (Exception e)
{
// This may happen due to concurrent pushes.
// TODO: Make IPackageStorageService.SavePackageContentAsync return a result enum so this
// can be properly handled.
_logger.LogError(
e,
"Failed to persist package {PackageId} {PackageVersion} content to storage",
package.Id,
package.NormalizedVersionString);
throw;
}
_logger.LogInformation(
"Persisted package {Id} {Version} content to storage, saving metadata to database...",
package.Id,
package.NormalizedVersionString);
var result = await _packages.AddAsync(package, cancellationToken);
if (result == PackageAddResult.PackageAlreadyExists)
{
_logger.LogWarning(
"Package {Id} {Version} metadata already exists in database",
package.Id,
package.NormalizedVersionString);
return(PackageIndexingResult.PackageAlreadyExists);
}
if (result != PackageAddResult.Success)
{
_logger.LogError($"Unknown {nameof(PackageAddResult)} value: {{PackageAddResult}}", result);
throw new InvalidOperationException($"Unknown {nameof(PackageAddResult)} value: {result}");
}
_logger.LogInformation(
"Successfully persisted package {Id} {Version} metadata to database. Indexing in search...",
package.Id,
package.NormalizedVersionString);
await _search.IndexAsync(package, cancellationToken);
_logger.LogInformation(
"Successfully indexed package {Id} {Version} in search",
package.Id,
package.NormalizedVersionString);
return(PackageIndexingResult.Success);
}