internal static bool ShouldCuratePackage(
CuratedFeed curatedFeed,
Package galleryPackage,
PackageReader nugetPackage)
{
var nuspec = nugetPackage.GetNuspecReader();
return
// Must have min client version of null or <= 2.2
(nuspec.GetMinClientVersion() == null || nuspec.GetMinClientVersion() <= new NuGetVersion(2, 2, 0)) &&
// Must be latest stable
galleryPackage.IsLatestStable &&
// Must support net40
SupportsNet40(galleryPackage) &&
(
// Must have AspNetWebPages tag
ContainsAspNetWebPagesTag(galleryPackage) ||
// OR: Must not contain powershell or T4
DoesNotContainUnsupportedFiles(nugetPackage)
) &&
// Dependencies on the gallery must be curated
DependenciesAreCurated(galleryPackage, curatedFeed);
}
public async override Task<bool> InstallPackageAsync(Packaging.Core.PackageIdentity packageIdentity, System.IO.Stream packageStream,
INuGetProjectContext nuGetProjectContext, CancellationToken token)
{
if (!packageStream.CanSeek)
{
throw new ArgumentException(NuGet.ProjectManagement.Strings.PackageStreamShouldBeSeekable);
}
nuGetProjectContext.Log(MessageLevel.Info, Strings.InstallingPackage, packageIdentity);
packageStream.Seek(0, SeekOrigin.Begin);
var zipArchive = new ZipArchive(packageStream);
PackageReader packageReader = new PackageReader(zipArchive);
var packageSupportedFrameworks = packageReader.GetSupportedFrameworks();
var projectFrameworks = _project.GetSupportedFrameworksAsync(token)
.Result
.Select(f => NuGetFramework.Parse(f.FullName));
var args = new Dictionary<string, object>();
args["Frameworks"] = projectFrameworks.Where(
projectFramework =>
IsCompatible(projectFramework, packageSupportedFrameworks)).ToArray();
await _project.InstallPackageAsync(
new NuGetPackageMoniker
{
Id = packageIdentity.Id,
Version = packageIdentity.Version.ToNormalizedString()
},
args,
logger: null,
progress: null,
cancellationToken: token);
return true;
}
private IEnumerable<string> GetPackageAssemblyNames(string path)
{
var compatibilityProvider = new DefaultCompatibilityProvider();
var folderReader = new PackageFolderReader(Path.Combine(path, _fileSystem.PackagesFolder));
var nupkgFiles = folderReader.GetFiles().Where(x => Path.GetExtension(x).ToLowerInvariant() == ".nupkg");
var packagesConfig = XDocument.Parse(File.ReadAllText(Path.Combine(path, _fileSystem.PackagesFile)));
var reader = new PackagesConfigReader(packagesConfig);
var contents = reader.GetPackages();
var result = new List<string>();
foreach (var nupkg in nupkgFiles)
{
var stream = folderReader.GetStream(nupkg);
var packageReader = new PackageReader(stream);
var identity = packageReader.GetIdentity();
var packagesConfigReference = contents.FirstOrDefault(x => x.PackageIdentity.Id == identity.Id && x.PackageIdentity.Version == identity.Version);
if (packagesConfigReference == null)
{
break;
}
var packageContents = packageReader.GetLibItems().Where(x => compatibilityProvider.IsCompatible(x.TargetFramework, packagesConfigReference.TargetFramework)).
SelectMany(x => x.Items.Where(i => Path.GetExtension(i).ToLowerInvariant() == ".dll"));
result.AddRange(packageContents);
}
return result;
}
public void Execute(Package galleryPackage, PackageReader nugetPackage, bool commitChanges)
{
foreach (var curator in _curators)
{
curator.Curate(galleryPackage, nugetPackage, commitChanges: commitChanges);
}
}
public override void Curate(Package galleryPackage, PackageReader nugetPackage, bool commitChanges)
{
// Make sure the target feed exists
CuratedFeed feed = CuratedFeedService.GetFeedByName(CuratedFeedName, includePackages: true);
if (feed != null && galleryPackage.Tags != null)
{
// Break the tags up so we can be sure we don't catch any partial matches (i.e. "foobar" when we're looking for "foo")
string[] tags = galleryPackage.Tags.Split();
// Check if this package should be curated
if (tags.Any(tag => RequiredTags.Contains(tag, StringComparer.OrdinalIgnoreCase)))
{
// It should!
// But now we need to ensure that the package's dependencies are also curated
if (DependenciesAreCurated(galleryPackage, feed))
{
CuratedFeedService.CreatedCuratedPackage(
feed,
galleryPackage.PackageRegistration,
automaticallyCurated: true,
commitChanges: commitChanges);
}
}
}
}
public void EnsureValid(PackageReader packageReader)
{
var packageMetadata = PackageMetadata.FromNuspecReader(packageReader.GetNuspecReader());
ValidateNuGetPackageMetadata(packageMetadata);
var supportedFrameworks = GetSupportedFrameworks(packageReader).Select(fn => fn.ToShortNameOrNull()).ToArray();
if (!supportedFrameworks.AnySafe(sf => sf == null))
{
ValidateSupportedFrameworks(supportedFrameworks);
}
}
public static void RewritingTheNuSpecDoesNotMessUpTheNuspecStream()
{
var packageStream = CreateTestPackageStream();
var manifestStreamLengthOriginal = GetManifestStreamLength(packageStream);
var longValue = new String('x', 200);
var shortValue = "y";
// Act 1 - Make the stream bigger
NupkgRewriter.RewriteNupkgManifest(packageStream,
new List<Action<ManifestEdit>>
{
metadata => { metadata.Description = longValue; },
metadata => { metadata.Summary = longValue; }
});
// Assert 1
var manifestStreamLength1 = GetManifestStreamLength(packageStream);
Assert.True(manifestStreamLength1 > manifestStreamLengthOriginal);
using (var nupkg = new PackageReader(packageStream, leaveStreamOpen: true))
{
var nuspec = nupkg.GetNuspecReader();
Assert.Equal("TestPackage", nuspec.GetId());
Assert.Equal(NuGetVersion.Parse("0.0.0.1"), nuspec.GetVersion());
Assert.Equal(longValue, nuspec.GetMetadata().First(kvp => kvp.Key == "description").Value);
Assert.Equal(longValue, nuspec.GetMetadata().First(kvp => kvp.Key == "summary").Value);
}
// Act 2 - Make the stream smaller
NupkgRewriter.RewriteNupkgManifest(packageStream,
new List<Action<ManifestEdit>>
{
metadata => { metadata.Description = shortValue; },
metadata => { metadata.Summary = shortValue; }
});
// Assert 2
var manifestStreamLength2 = GetManifestStreamLength(packageStream);
Assert.True(manifestStreamLength2 < manifestStreamLength1);
using (var nupkg = new PackageReader(packageStream, leaveStreamOpen: true))
{
var nuspec = nupkg.GetNuspecReader();
Assert.Equal("TestPackage", nuspec.GetId());
Assert.Equal(NuGetVersion.Parse("0.0.0.1"), nuspec.GetVersion());
Assert.Equal(shortValue, nuspec.GetMetadata().First(kvp => kvp.Key == "description").Value);
Assert.Equal(shortValue, nuspec.GetMetadata().First(kvp => kvp.Key == "summary").Value);
}
}
public static void CheckMinClientVersion(Stream packageStream, PackageIdentity packageIdentity)
{
var packageZipArchive = new ZipArchive(packageStream);
var packageReader = new PackageReader(packageZipArchive);
var nuspecReader = new NuspecReader(packageReader.GetNuspec());
var packageMinClientVersion = nuspecReader.GetMinClientVersion();
// validate that the current version of NuGet satisfies the minVersion attribute specified in the .nuspec
if (Constants.NuGetSemanticVersion < packageMinClientVersion)
{
throw new NuGetVersionNotSatisfiedException(
String.Format(CultureInfo.CurrentCulture, Strings.PackageMinVersionNotSatisfied, packageIdentity,
packageMinClientVersion.ToNormalizedString(), Constants.NuGetSemanticVersion.ToNormalizedString()));
}
}
public Package CreatePackage(PackageReader nugetPackage, PackageStreamMetadata packageStreamMetadata, User user, bool commitChanges = true)
{
var packageMetadata = PackageMetadata.FromNuspecReader(nugetPackage.GetNuspecReader());
ValidateNuGetPackageMetadata(packageMetadata);
var packageRegistration = CreateOrGetPackageRegistration(user, packageMetadata);
var package = CreatePackageFromNuGetPackage(packageRegistration, nugetPackage, packageMetadata, packageStreamMetadata, user);
packageRegistration.Packages.Add(package);
UpdateIsLatest(packageRegistration, false);
if (commitChanges)
{
_packageRegistrationRepository.CommitChanges();
NotifyIndexingService();
}
return package;
}
public override void Curate(
Package galleryPackage,
PackageReader nugetPackage,
bool commitChanges)
{
var curatedFeed = CuratedFeedService.GetFeedByName("webmatrix", includePackages: true);
if (curatedFeed == null)
{
return;
}
var shouldBeIncluded = ShouldCuratePackage(curatedFeed, galleryPackage, nugetPackage);
if (shouldBeIncluded)
{
CuratedFeedService.CreatedCuratedPackage(
curatedFeed,
galleryPackage.PackageRegistration,
included: true,
automaticallyCurated: true,
commitChanges: commitChanges);
}
}
public static void CanReadBasicMetadataProperties()
{
var packageStream = CreateTestPackageStream();
var nupkg = new PackageReader(packageStream, leaveStreamOpen: false);
var nuspec = nupkg.GetNuspecReader();
// Act
var packageMetadata = PackageMetadata.FromNuspecReader(nuspec);
// Assert
Assert.Equal("TestPackage", packageMetadata.Id);
Assert.Equal(NuGetVersion.Parse("0.0.0.1"), packageMetadata.Version);
Assert.Equal("Package A", packageMetadata.Title);
Assert.Equal(2, packageMetadata.Authors.Count);
Assert.Equal("ownera, ownerb", packageMetadata.Owners);
Assert.False(packageMetadata.RequireLicenseAcceptance);
Assert.Equal("package A description.", packageMetadata.Description);
Assert.Equal("en-US", packageMetadata.Language);
Assert.Equal("http://www.nuget.org/", packageMetadata.ProjectUrl.ToString());
Assert.Equal("http://www.nuget.org/", packageMetadata.IconUrl.ToString());
Assert.Equal("http://www.nuget.org/", packageMetadata.LicenseUrl.ToString());
}
public static IVsPackageMetadata CreateMetadata(string nupkgPath, PackageIdentity package)
{
IEnumerable<string> authors = Enumerable.Empty<string>();
string description = string.Empty;
string title = package.Id;
string installPath = string.Empty;
try
{
// installPath is the nupkg path
FileInfo file = new FileInfo(nupkgPath);
installPath = file.Directory.FullName;
PackageReader reader = new PackageReader(file.OpenRead());
using (var nuspecStream = reader.GetNuspec())
{
NuspecReader nuspec = new NuspecReader(nuspecStream);
var metadata = nuspec.GetMetadata();
authors = GetNuspecValue(metadata, "authors").Split(',').ToArray();
title = GetNuspecValue(metadata, "title");
description = GetNuspecValue(metadata, "description");
}
}
catch (Exception ex)
{
// ignore errors from reading the extra fields
Debug.Fail(ex.ToString());
}
if (String.IsNullOrEmpty(title))
{
title = package.Id;
}
return new VsPackageMetadata(package, title, authors, description, installPath);
}
public static void CanRewriteTheNuspecInANupkg()
{
var packageStream = CreateTestPackageStream();
// Act
NupkgRewriter.RewriteNupkgManifest(packageStream,
new List<Action<ManifestEdit>>
{
metadata => { metadata.Authors = "Me and You"; },
metadata => { metadata.Tags = "Peas In A Pod"; }
});
// Assert
using (var nupkg = new PackageReader(packageStream, leaveStreamOpen: false))
{
var nuspec = nupkg.GetNuspecReader();
Assert.Equal("TestPackage", nuspec.GetId());
Assert.Equal(NuGetVersion.Parse("0.0.0.1"), nuspec.GetVersion());
Assert.Equal("Me and You", nuspec.GetMetadata().First(kvp => kvp.Key == "authors").Value);
Assert.Equal("Peas In A Pod", nuspec.GetMetadata().First(kvp => kvp.Key == "tags").Value);
}
}
private LockFileTargetLibrary CreateLockFileTargetLibrary(LocalPackageInfo package, RestoreTargetGraph targetGraph, DefaultPackagePathResolver defaultPackagePathResolver, string correctedPackageName)
{
var lockFileLib = new LockFileTargetLibrary();
var framework = targetGraph.Framework;
var runtimeIdentifier = targetGraph.RuntimeIdentifier;
// package.Id is read from nuspec and it might be in wrong casing.
// correctedPackageName should be the package name used by dependency graph and
// it has the correct casing that runtime needs during dependency resolution.
lockFileLib.Name = correctedPackageName ?? package.Id;
lockFileLib.Version = package.Version;
IList<string> files;
var contentItems = new ContentItemCollection();
HashSet<string> referenceFilter = null;
using (var nupkgStream = File.OpenRead(package.ZipPath))
{
var packageReader = new PackageReader(nupkgStream);
files = packageReader.GetFiles().Select(p => p.Replace(Path.DirectorySeparatorChar, '/')).ToList();
contentItems.Load(files);
var dependencySet = packageReader.GetPackageDependencies().GetNearest(framework);
if (dependencySet != null)
{
var set = dependencySet.Packages;
if (set != null)
{
lockFileLib.Dependencies = set.ToList();
}
}
var referenceSet = packageReader.GetReferenceItems().GetNearest(framework);
if (referenceSet != null)
{
referenceFilter = new HashSet<string>(referenceSet.Items, StringComparer.OrdinalIgnoreCase);
}
// TODO: Remove this when we do #596
// ASP.NET Core isn't compatible with generic PCL profiles
if (!string.Equals(framework.Framework, FrameworkConstants.FrameworkIdentifiers.AspNetCore, StringComparison.OrdinalIgnoreCase)
&&
!string.Equals(framework.Framework, FrameworkConstants.FrameworkIdentifiers.DnxCore, StringComparison.OrdinalIgnoreCase))
{
var frameworkAssemblies = packageReader.GetFrameworkItems().GetNearest(framework);
if (frameworkAssemblies != null)
{
foreach (var assemblyReference in frameworkAssemblies.Items)
{
lockFileLib.FrameworkAssemblies.Add(assemblyReference);
}
}
}
}
var nativeCriteria = targetGraph.Conventions.Criteria.ForRuntime(targetGraph.RuntimeIdentifier);
var managedCriteria = targetGraph.Conventions.Criteria.ForFrameworkAndRuntime(framework, targetGraph.RuntimeIdentifier);
var compileGroup = contentItems.FindBestItemGroup(managedCriteria, targetGraph.Conventions.Patterns.CompileAssemblies, targetGraph.Conventions.Patterns.RuntimeAssemblies);
if (compileGroup != null)
{
lockFileLib.CompileTimeAssemblies = compileGroup.Items.Select(t => new LockFileItem(t.Path)).ToList();
}
var runtimeGroup = contentItems.FindBestItemGroup(managedCriteria, targetGraph.Conventions.Patterns.RuntimeAssemblies);
if (runtimeGroup != null)
{
lockFileLib.RuntimeAssemblies = runtimeGroup.Items.Select(p => new LockFileItem(p.Path)).ToList();
}
var resourceGroup = contentItems.FindBestItemGroup(managedCriteria, targetGraph.Conventions.Patterns.ResourceAssemblies);
if (resourceGroup != null)
{
lockFileLib.ResourceAssemblies = resourceGroup.Items.Select(ToResourceLockFileItem).ToList();
}
var nativeGroup = contentItems.FindBestItemGroup(nativeCriteria, targetGraph.Conventions.Patterns.NativeLibraries);
if (nativeGroup != null)
{
lockFileLib.NativeLibraries = nativeGroup.Items.Select(p => new LockFileItem(p.Path)).ToList();
}
// COMPAT: Support lib/contract so older packages can be consumed
var contractPath = "lib/contract/" + package.Id + ".dll";
var hasContract = files.Any(path => path == contractPath);
var hasLib = lockFileLib.RuntimeAssemblies.Any();
if (hasContract
&& hasLib
&& !framework.IsDesktop())
{
lockFileLib.CompileTimeAssemblies.Clear();
lockFileLib.CompileTimeAssemblies.Add(new LockFileItem(contractPath));
}
// Apply filters from the <references> node in the nuspec
if (referenceFilter != null)
{
// Remove anything that starts with "lib/" and is NOT specified in the reference filter.
// runtimes/* is unaffected (it doesn't start with lib/)
lockFileLib.RuntimeAssemblies = lockFileLib.RuntimeAssemblies.Where(p => !p.Path.StartsWith("lib/") || referenceFilter.Contains(p.Path)).ToList();
lockFileLib.CompileTimeAssemblies = lockFileLib.CompileTimeAssemblies.Where(p => !p.Path.StartsWith("lib/") || referenceFilter.Contains(p.Path)).ToList();
}
return lockFileLib;
}
public async override Task<IEnumerable<Packaging.PackageReference>> GetInstalledPackagesAsync(CancellationToken token)
{
var result = new List<Packaging.PackageReference>();
foreach (object item in await _project.GetInstalledPackagesAsync(token))
{
Packaging.Core.PackageIdentity identity = null;
var moniker = item as INuGetPackageMoniker;
if (moniker != null)
{
identity = new Packaging.Core.PackageIdentity(
moniker.Id,
NuGetVersion.Parse(moniker.Version));
}
else
{
// otherwise, item is the file name of the nupkg file
var fileName = item as string;
using (var fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read))
{
var zipArchive = new ZipArchive(fileStream);
var packageReader = new PackageReader(zipArchive);
identity = packageReader.GetIdentity();
}
}
result.Add(new Packaging.PackageReference(
identity,
targetFramework: null));
}
return result;
}
public override void ExecuteCommand()
{
if (!Directory.Exists(WorkDirectory))
{
Directory.CreateDirectory(WorkDirectory);
}
Log.Info("Getting all package metadata...");
var packages = GetAllPackages();
var totalCount = packages.Count;
var processedCount = 0;
Log.Info(
"Populating frameworks for {0} packages on '{1}',",
totalCount,
ConnectionString);
packages
.AsParallel()
.AsOrdered()
.WithDegreeOfParallelism(10)
.ForAll(package =>
{
// Allocate a processed count number for this package
var thisPackageId = Interlocked.Increment(ref processedCount);
try
{
var reportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + ".json");
var bustedReportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + "_" + package.Hash + ".json");
var report = new PackageFrameworkReport()
{
Id = package.Id,
Version = package.Version,
Key = package.Key,
Hash = package.Hash,
Created = package.Created.Value,
State = PackageReportState.Unresolved
};
if (File.Exists(bustedReportPath))
{
File.Move(bustedReportPath, reportPath);
}
if (File.Exists(reportPath))
{
using (var reader = File.OpenText(reportPath))
{
report = (PackageFrameworkReport)_serializer.Deserialize(reader, typeof(PackageFrameworkReport));
}
ResolveReport(report);
}
else
{
try
{
var downloadPath = DownloadPackage(package);
using (var nugetPackage = new PackageReader(File.OpenRead(downloadPath)))
{
var supportedFrameworks = GetSupportedFrameworks(nugetPackage);
report.PackageFrameworks = supportedFrameworks.ToArray();
report = PopulateFrameworks(package, report);
}
File.Delete(downloadPath);
// Resolve the report
ResolveReport(report);
}
catch (Exception ex)
{
report.State = PackageReportState.Error;
report.Error = ex.ToString();
}
}
using (var writer = File.CreateText(reportPath))
{
_serializer.Serialize(writer, report);
}
Log.Info("[{2}/{3} {4}%] {6} Package: {0}@{1} (created {5})",
package.Id,
package.Version,
thisPackageId.ToString("000000"),
totalCount.ToString("000000"),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
package.Created.Value,
report.State.ToString().PadRight(_padLength, ' '));
}
catch (Exception ex)
{
Log.Error("[{2}/{3} {4}%] Error For Package: {0}@{1}: {5}",
package.Id,
package.Version,
thisPackageId.ToString("000000"),
totalCount.ToString("000000"),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
ex.ToString());
}
});
}
private static IEnumerable<string> GetSupportedFrameworks(PackageReader packageReader)
{
return packageReader
.GetSupportedFrameworks()
.Select(fn => fn.ToShortNameOrNull())
.ToArray();
}
internal static void DeleteFiles(IMSBuildNuGetProjectSystem msBuildNuGetProjectSystem,
ZipArchive zipArchive,
IEnumerable<string> otherPackagesPath,
FrameworkSpecificGroup frameworkSpecificGroup,
IDictionary<FileTransformExtensions, IPackageFileTransformer> fileTransformers)
{
var packageTargetFramework = frameworkSpecificGroup.TargetFramework;
IPackageFileTransformer transformer;
var directoryLookup = frameworkSpecificGroup.Items.ToLookup(
p => Path.GetDirectoryName(ResolveTargetPath(msBuildNuGetProjectSystem,
fileTransformers,
fte => fte.UninstallExtension,
GetEffectivePathForContentFile(packageTargetFramework, p),
out transformer)));
// Get all directories that this package may have added
var directories = from grouping in directoryLookup
from directory in FileSystemUtility.GetDirectories(grouping.Key, altDirectorySeparator: false)
orderby directory.Length descending
select directory;
// Remove files from every directory
foreach (var directory in directories)
{
var directoryFiles = directoryLookup.Contains(directory) ? directoryLookup[directory] : Enumerable.Empty<string>();
if (!Directory.Exists(Path.Combine(msBuildNuGetProjectSystem.ProjectFullPath, directory)))
{
continue;
}
try
{
foreach (var file in directoryFiles)
{
if (IsEmptyFolder(file))
{
continue;
}
// Resolve the path
string path = ResolveTargetPath(msBuildNuGetProjectSystem,
fileTransformers,
fte => fte.UninstallExtension,
GetEffectivePathForContentFile(packageTargetFramework, file),
out transformer);
if (msBuildNuGetProjectSystem.IsSupportedFile(path))
{
if (transformer != null)
{
// TODO: use the framework from packages.config instead of the current framework
// which may have changed during re-targeting
NuGetFramework projectFramework = msBuildNuGetProjectSystem.TargetFramework;
List<InternalZipFileInfo> matchingFiles = new List<InternalZipFileInfo>();
foreach(var otherPackagePath in otherPackagesPath)
{
using(var otherPackageStream = File.OpenRead(otherPackagePath))
{
var otherPackageZipArchive = new ZipArchive(otherPackageStream);
var otherPackageZipReader = new PackageReader(otherPackageZipArchive);
// use the project framework to find the group that would have been installed
var mostCompatibleContentFilesGroup = GetMostCompatibleGroup(projectFramework, otherPackageZipReader.GetContentItems(), altDirSeparator: true);
if(mostCompatibleContentFilesGroup != null && IsValid(mostCompatibleContentFilesGroup))
{
foreach(var otherPackageItem in mostCompatibleContentFilesGroup.Items)
{
if(GetEffectivePathForContentFile(packageTargetFramework, otherPackageItem)
.Equals(GetEffectivePathForContentFile(packageTargetFramework, file), StringComparison.OrdinalIgnoreCase))
{
matchingFiles.Add(new InternalZipFileInfo(otherPackagePath, otherPackageItem));
}
}
}
}
}
try
{
var zipArchiveFileEntry = zipArchive.GetEntry(PathUtility.ReplaceDirSeparatorWithAltDirSeparator(file));
if (zipArchiveFileEntry != null)
{
transformer.RevertFile(zipArchiveFileEntry, path, matchingFiles, msBuildNuGetProjectSystem);
}
}
catch (Exception e)
{
msBuildNuGetProjectSystem.NuGetProjectContext.Log(MessageLevel.Warning, e.Message);
}
}
else
{
var zipArchiveFileEntry = zipArchive.GetEntry(PathUtility.ReplaceDirSeparatorWithAltDirSeparator(file));
if (zipArchiveFileEntry != null)
{
DeleteFileSafe(path, zipArchiveFileEntry.Open, msBuildNuGetProjectSystem);
}
}
}
}
// If the directory is empty then delete it
if (!GetFilesSafe(msBuildNuGetProjectSystem, directory).Any() &&
!GetDirectoriesSafe(msBuildNuGetProjectSystem, directory).Any())
{
DeleteDirectorySafe(msBuildNuGetProjectSystem, directory, recursive: false);
}
}
finally
{
}
}
}
private void WriteTargetsAndProps(PackageSpec project, List<RestoreTargetGraph> targetGraphs, NuGetv3LocalRepository repository)
{
// Get the runtime-independent graphs
var tfmGraphs = targetGraphs.Where(g => string.IsNullOrEmpty(g.RuntimeIdentifier)).ToList();
if (tfmGraphs.Count > 1)
{
var name = $"{project.Name}.nuget.targets";
var path = Path.Combine(project.BaseDirectory, name);
_log.LogInformation($"Generating MSBuild file {name}");
GenerateMSBuildErrorFile(path);
return;
}
var graph = tfmGraphs[0];
var pathResolver = new DefaultPackagePathResolver(repository.RepositoryRoot);
var targets = new List<string>();
var props = new List<string>();
foreach (var library in graph.Flattened.Distinct().OrderBy(g => g.Data.Match.Library))
{
var package = repository.FindPackagesById(library.Key.Name).FirstOrDefault(p => p.Version == library.Key.Version);
if (package != null)
{
var criteria = graph.Conventions.Criteria.ForFramework(graph.Framework);
var contentItemCollection = new ContentItemCollection();
using (var nupkgStream = File.OpenRead(package.ZipPath))
{
var reader = new PackageReader(nupkgStream);
contentItemCollection.Load(reader.GetFiles());
}
// Find MSBuild thingies
var buildItems = contentItemCollection.FindBestItemGroup(criteria, graph.Conventions.Patterns.MSBuildFiles);
if (buildItems != null)
{
// We need to additionally filter to items that are named "{packageId}.targets" and "{packageId}.props"
// Filter by file name here and we'll filter by extension when we add things to the lists.
var items = buildItems.Items
.Where(item => Path.GetFileNameWithoutExtension(item.Path).Equals(package.Id, StringComparison.OrdinalIgnoreCase))
.ToList();
targets.AddRange(items
.Select(c => c.Path)
.Where(path => Path.GetExtension(path).Equals(".targets", StringComparison.OrdinalIgnoreCase))
.Select(path => Path.Combine(pathResolver.GetPackageDirectory(package.Id, package.Version), path.Replace('/', Path.DirectorySeparatorChar))));
props.AddRange(items
.Select(c => c.Path)
.Where(path => Path.GetExtension(path).Equals(".props", StringComparison.OrdinalIgnoreCase))
.Select(path => Path.Combine(pathResolver.GetPackageDirectory(package.Id, package.Version), path.Replace('/', Path.DirectorySeparatorChar))));
}
}
}
// Generate the files as needed
var targetsName = $"{project.Name}.nuget.targets";
var propsName = $"{project.Name}.nuget.props";
var targetsPath = Path.Combine(project.BaseDirectory, targetsName);
var propsPath = Path.Combine(project.BaseDirectory, propsName);
if (targets.Any())
{
_log.LogInformation($"Generating MSBuild file {targetsName}");
GenerateImportsFile(repository, targetsPath, targets);
}
else if (File.Exists(targetsPath))
{
File.Delete(targetsPath);
}
if (props.Any())
{
_log.LogInformation($"Generating MSBuild file {propsName}");
GenerateImportsFile(repository, propsPath, props);
}
else if (File.Exists(propsPath))
{
File.Delete(propsPath);
}
}
public static bool GetSatelliteFiles(Stream packageStream, PackageIdentity packageIdentity, PackagePathResolver packagePathResolver,
out string language, out string runtimePackageDirectory, out IEnumerable<ZipArchiveEntry> satelliteFiles)
{
var zipArchive = new ZipArchive(packageStream);
var packageReader = new PackageReader(zipArchive);
var nuspecReader = new NuspecReader(packageReader.GetNuspec());
PackageIdentity runtimePackageIdentity = null;
string packageLanguage = null;
if (IsSatellitePackage(nuspecReader, out runtimePackageIdentity, out packageLanguage))
{
// Now, we know that the package is a satellite package and that the runtime package is 'runtimePackageId'
// Check, if the runtimePackage is installed and get the folder to copy over files
var runtimePackageFilePath = packagePathResolver.GetInstalledPackageFilePath(runtimePackageIdentity);
if (File.Exists(runtimePackageFilePath))
{
runtimePackageDirectory = Path.GetDirectoryName(runtimePackageFilePath);
// Existence of the package file is the validation that the package exists
var libItemGroups = packageReader.GetLibItems();
var satelliteFileEntries = new List<ZipArchiveEntry>();
foreach (var libItemGroup in libItemGroups)
{
var satelliteFilesInGroup = libItemGroup.Items.Where(item => Path.GetDirectoryName(item).Split(Path.DirectorySeparatorChar)
.Contains(packageLanguage, StringComparer.OrdinalIgnoreCase));
foreach (var satelliteFile in satelliteFilesInGroup)
{
var zipArchiveEntry = zipArchive.GetEntry(satelliteFile);
if (zipArchiveEntry != null)
{
satelliteFileEntries.Add(zipArchiveEntry);
}
}
}
if (satelliteFileEntries.Count > 0)
{
language = packageLanguage;
satelliteFiles = satelliteFileEntries;
return true;
}
}
}
language = null;
runtimePackageDirectory = null;
satelliteFiles = null;
return false;
}
public abstract void Curate(
Package galleryPackage,
PackageReader nugetPackage,
bool commitChanges);
/// <summary>
/// Returns artifact info for packages in a packages.config that match the given set of properties.
/// </summary>
/// <param name="configPath">Path to packages.config</param>
/// <param name="properties">Property values to filter on</param>
/// <returns>Artifacts matching the property filters.</returns>
public static IEnumerable<NuGetArtifactInfo> GetArtifactInfo(string configPath, IEnumerable<string> propertyKeys)
{
if (configPath == null)
{
throw new ArgumentNullException("configPath");
}
if (propertyKeys == null)
{
throw new ArgumentNullException("propertyKeys");
}
FileInfo file = new FileInfo(configPath);
if (!file.Exists)
{
throw new FileNotFoundException(configPath);
}
List<NuGetArtifactInfo> results = new List<NuGetArtifactInfo>();
using (FileStream stream = file.OpenRead())
{
ConfigReader configReader = new ConfigReader(stream);
foreach (PackageIdentity package in configReader.GetPackages())
{
// TODO: find the real path
string packageName = package.Id + "." + package.Version.ToString();
FileInfo nupkgPath = new FileInfo(Path.Combine(file.Directory.Parent.FullName, "packages", packageName, packageName + ".nupkg"));
if (!nupkgPath.Exists)
{
throw new FileNotFoundException(nupkgPath.FullName);
}
NuGetPackageId id = new NuGetPackageId(package.Id, package.Version, nupkgPath.Directory.FullName);
ZipFileSystem zip = new ZipFileSystem(nupkgPath.OpenRead());
using (PackageReader packageReader = new PackageReader(zip))
{
ComponentTree tree = null;
// TODO: add a better check for this
if (packageReader.GetPackedManifest() == null)
{
using (LegacyPackageReader legacyReader = new LegacyPackageReader(zip))
{
throw new NotImplementedException();
//var packed = PackedManifestCreator.FromLegacy(legacyReader);
//tree = packed.ComponentTree;
}
}
else
{
tree = packageReader.GetComponentTree();
}
List<NuGetArtifactGroup> groups = new List<NuGetArtifactGroup>();
// TODO: use propertyKeys
// TODO: get full paths
foreach (var path in tree.GetPaths())
{
var props = path.Properties.Select(p => (KeyValueTreeProperty)p).Select(p => new KeyValuePair<string, string>(p.Key, p.Value));
var items = path.Items.Select(i => (NuGetTreeItem)i).Select(i => new NuGetArtifact(i.Type, i.Data.Where(p => p.Key == "path").Select(p => p.Value).Single()));
groups.Add(new NuGetArtifactGroup(props, items));
}
NuGetArtifactInfo info = new NuGetArtifactInfo(id, groups.ToArray());
results.Add(info);
}
}
}
return results;
}
public static NuGetDependencyInfo GetDependencyInfo(FileInfo nupkgPath)
{
if (!nupkgPath.Exists)
{
throw new FileNotFoundException(nupkgPath.FullName);
}
using (var stream =nupkgPath.OpenRead())
{
ZipFileSystem zip = new ZipFileSystem(stream);
using (PackageReader packageReader = new PackageReader(zip))
{
using (var nuspecStream = packageReader.GetNuspec())
{
NuspecReader reader = new NuspecReader(nuspecStream);
NuGetPackageId package = CreateIdentity(reader, nupkgPath.FullName);
List<NuGetDependencyGroup> dependencyGroups = new List<NuGetDependencyGroup>();
foreach (var depGroup in reader.GetDependencyGroups())
{
FrameworkName framework = Utilities.GetFrameworkName(depGroup.TargetFramework);
NuGetDependency[] dependencies = depGroup.Packages.Select(d => new NuGetDependency(d.Id, VersionRange.Parse(d.VersionRange))).ToArray();
dependencyGroups.Add(new NuGetDependencyGroup(framework, dependencies));
}
return new NuGetDependencyInfo(package, dependencyGroups);
}
}
}
}
private LockFileLibrary CreateLockFileLibrary(LocalPackageInfo package, SHA512 sha512, string correctedPackageName)
{
var lockFileLib = new LockFileLibrary();
// package.Id is read from nuspec and it might be in wrong casing.
// correctedPackageName should be the package name used by dependency graph and
// it has the correct casing that runtime needs during dependency resolution.
lockFileLib.Name = correctedPackageName ?? package.Id;
lockFileLib.Version = package.Version;
using (var nupkgStream = File.OpenRead(package.ZipPath))
{
lockFileLib.Sha512 = Convert.ToBase64String(sha512.ComputeHash(nupkgStream));
nupkgStream.Seek(0, SeekOrigin.Begin);
var packageReader = new PackageReader(nupkgStream);
// Get package files, excluding directory entries
lockFileLib.Files = packageReader.GetFiles().Where(x => !x.EndsWith("/")).ToList();
}
return lockFileLib;
}
/// <summary>
/// Given the nupkg file as a read-write stream with random access (e.g. MemoryStream),
/// This will replace the .nuspec file with a new .nuspec generated from
/// a) the old .nuspec file
/// b) supplied edits
///
/// This function leaves readWriteStream open.
/// </summary>
public static void RewriteNupkgManifest(Stream readWriteStream, IEnumerable<Action<ManifestEdit>> edits)
{
if (!readWriteStream.CanRead)
{
throw new ArgumentException(Strings.StreamMustBeReadable, nameof(readWriteStream));
}
if (!readWriteStream.CanWrite)
{
throw new ArgumentException(Strings.StreamMustBeWriteable, nameof(readWriteStream));
}
if (!readWriteStream.CanSeek)
{
throw new ArgumentException(Strings.StreamMustBeSeekable, nameof(readWriteStream));
}
using (var packageReader = new PackageReader(readWriteStream, leaveStreamOpen: true))
{
var nuspecReader = packageReader.GetNuspecReader();
// Read <metadata> node from nuspec
var metadataNode = nuspecReader.Xml.Root.Elements()
.FirstOrDefault(e => StringComparer.Ordinal.Equals(e.Name.LocalName, "metadata"));
if (metadataNode == null)
{
throw new PackagingException("The package manifest is missing the 'metadata' node.");
}
// Convert metadata into a ManifestEdit so that we can run it through the editing pipeline
var editableManifestElements = new ManifestEdit
{
Title = ReadFromMetadata(metadataNode, "title"),
Authors = ReadFromMetadata(metadataNode, "authors"),
Copyright = ReadFromMetadata(metadataNode, "copyright"),
Description = ReadFromMetadata(metadataNode, "description"),
IconUrl = ReadFromMetadata(metadataNode, "iconUrl"),
LicenseUrl = ReadFromMetadata(metadataNode, "licenseUrl"),
ProjectUrl = ReadFromMetadata(metadataNode, "projectUrl"),
ReleaseNotes = ReadFromMetadata(metadataNode, "releasenotes"),
RequireLicenseAcceptance = ReadBoolFromMetadata(metadataNode, "requireLicenseAcceptance"),
Summary = ReadFromMetadata(metadataNode, "summary"),
Tags = ReadFromMetadata(metadataNode, "tags")
};
// Perform edits
foreach (var edit in edits)
{
edit.Invoke(editableManifestElements);
}
// Update the <metadata> node
WriteToMetadata(metadataNode, "title", editableManifestElements.Title);
WriteToMetadata(metadataNode, "authors", editableManifestElements.Authors);
WriteToMetadata(metadataNode, "copyright", editableManifestElements.Copyright);
WriteToMetadata(metadataNode, "description", editableManifestElements.Description);
WriteToMetadata(metadataNode, "iconUrl", editableManifestElements.IconUrl);
WriteToMetadata(metadataNode, "licenseUrl", editableManifestElements.LicenseUrl);
WriteToMetadata(metadataNode, "projectUrl", editableManifestElements.ProjectUrl);
WriteToMetadata(metadataNode, "releasenotes", editableManifestElements.ReleaseNotes);
WriteToMetadata(metadataNode, "requireLicenseAcceptance", editableManifestElements.RequireLicenseAcceptance.ToString(CultureInfo.InvariantCulture).ToLowerInvariant());
WriteToMetadata(metadataNode, "summary", editableManifestElements.Summary);
WriteToMetadata(metadataNode, "tags", editableManifestElements.Tags);
// Update the package stream
using (var newManifestStream = new MemoryStream())
{
nuspecReader.Xml.Save(newManifestStream);
using (var archive = new ZipArchive(readWriteStream, ZipArchiveMode.Update, leaveOpen: true))
{
var manifestEntries = archive.Entries
.Where(entry => entry.FullName.IndexOf("/", StringComparison.OrdinalIgnoreCase) == -1
&& entry.Name.EndsWith(".nuspec", StringComparison.OrdinalIgnoreCase)).ToList();
if (manifestEntries.Count == 0)
{
throw new PackagingException("Nuspec file does not exist in package.");
}
if (manifestEntries.Count > 1)
{
throw new PackagingException("Package contains multiple nuspec files.");
}
var manifestEntry = manifestEntries[0];
using (var manifestOutputStream = manifestEntry.Open())
{
manifestOutputStream.SetLength(0);
newManifestStream.Position = 0;
newManifestStream.CopyTo(manifestOutputStream);
}
}
}
}
}
public virtual void VerifyPackageVersion(string packagePath, string expectedVersion)
{
using (var stream = File.OpenRead(packagePath))
using (var reader = new PackageReader(stream))
{
var identity = reader.GetIdentity();
var actualVersion = identity.Version.ToNormalizedString();
Assert.AreEqual(expectedVersion, actualVersion, "Checking package version");
}
}
public override void ExecuteCommand()
{
Log.Trace("Getting latest packages...");
var packages = GetLatestStablePackages();
Log.Trace("Getting previously curated packages...");
var alreadyCuratedPackageIds = GetAlreadyCuratedPackageIds();
Log.Trace("Calculating minimum difference set...");
var packageIdsToCurate = packages.Keys.Except(alreadyCuratedPackageIds).ToList();
var totalCount = packageIdsToCurate.Count;
var processedCount = 0;
Log.Trace(
"Curating {0} packages for the WebMatrix curated on '{1}',",
totalCount,
ConnectionString);
Parallel.ForEach(packageIdsToCurate, new ParallelOptions { MaxDegreeOfParallelism = 10 }, packageIdToCurate =>
{
var package = packages[packageIdToCurate];
try
{
var downloadPath = DownloadPackage(package);
bool shouldBeIncluded;
using (var nugetPackage = new PackageReader(File.OpenRead(downloadPath)))
{
var nuspecReader = nugetPackage.GetNuspecReader();
var metadata = nuspecReader.GetMetadata()
.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase);
string tags;
shouldBeIncluded = metadata.TryGetValue("tags", out tags) && tags.ToLowerInvariant().Contains("aspnetwebpages");
if (!shouldBeIncluded)
{
shouldBeIncluded = true;
foreach (var file in nugetPackage.GetFiles())
{
var fi = new FileInfo(file);
if (fi.Extension == ".ps1" || fi.Extension == ".t4")
{
shouldBeIncluded = false;
break;
}
}
}
if (shouldBeIncluded)
{
AddPackageToCuratedFeed(package);
}
}
File.Delete(downloadPath);
Interlocked.Increment(ref processedCount);
Log.Info(
"{2} package '{0}.{1}' ({3} of {4}).",
package.Id,
package.Version,
shouldBeIncluded ? "Curated" : "Ignored",
processedCount,
totalCount);
}
catch(Exception ex)
{
Interlocked.Increment(ref processedCount);
Log.Error(
"Error curating package '{0}.{1}' ({2} of {3}): {4}.",
package.Id,
package.Version,
processedCount,
totalCount,
ex.Message);
}
});
}
public static async Task<IEnumerable<string>> ExtractPackageAsync(
Stream packageStream,
PackageIdentity packageIdentity,
PackagePathResolver packagePathResolver,
PackageExtractionContext packageExtractionContext,
PackageSaveModes packageSaveMode,
CancellationToken token)
{
var filesAdded = new List<string>();
if (packageStream == null)
{
throw new ArgumentNullException(nameof(packageStream));
}
if (!packageStream.CanSeek)
{
throw new ArgumentException(Strings.PackageStreamShouldBeSeekable);
}
if (packageIdentity == null)
{
throw new ArgumentNullException(nameof(packageIdentity));
}
if (packagePathResolver == null)
{
throw new ArgumentNullException(nameof(packagePathResolver));
}
// TODO: Need to handle PackageSaveMode
// TODO: Support overwriting files also?
var zipArchive = new ZipArchive(packageStream);
var packageReader = new PackageReader(zipArchive);
var nuspecReader = new NuspecReader(packageReader.GetNuspec());
packageIdentity = nuspecReader.GetIdentity();
var packageDirectoryInfo = Directory.CreateDirectory(packagePathResolver.GetInstallPath(packageIdentity));
var packageDirectory = packageDirectoryInfo.FullName;
filesAdded.AddRange(await PackageHelper.CreatePackageFiles(zipArchive.Entries, packageDirectory, packageSaveMode, token));
var nupkgFilePath = Path.Combine(packageDirectory, packagePathResolver.GetPackageFileName(packageIdentity));
if (packageSaveMode.HasFlag(PackageSaveModes.Nupkg))
{
// During package extraction, nupkg is the last file to be created
// Since all the packages are already created, the package stream is likely positioned at its end
// Reset it
packageStream.Seek(0, SeekOrigin.Begin);
filesAdded.Add(await PackageHelper.CreatePackageFile(nupkgFilePath, packageStream, token));
}
// Now, copy satellite files unless requested to not copy them
if (packageExtractionContext == null
|| packageExtractionContext.CopySatelliteFiles)
{
filesAdded.AddRange(await CopySatelliteFilesAsync(packageIdentity, packagePathResolver, packageSaveMode, token));
}
return filesAdded;
}
private async Task<ActionResult> CreatePackageInternal()
{
// Get the user
var user = GetCurrentUser();
using (var packageStream = ReadPackageFromRequest())
using (var packageToPush = new PackageReader(packageStream, leaveStreamOpen: false))
{
NuspecReader nuspec = null;
try
{
nuspec = packageToPush.GetNuspecReader();
}
catch (Exception ex)
{
return new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(
CultureInfo.CurrentCulture,
Strings.UploadPackage_InvalidNuspec,
ex.Message));
}
if (nuspec.GetMinClientVersion() > Constants.MaxSupportedMinClientVersion)
{
return new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(
CultureInfo.CurrentCulture,
Strings.UploadPackage_MinClientVersionOutOfRange,
nuspec.GetMinClientVersion()));
}
// Ensure that the user can push packages for this partialId.
var packageRegistration = PackageService.FindPackageRegistrationById(nuspec.GetId());
if (packageRegistration != null)
{
if (!packageRegistration.IsOwner(user))
{
return new HttpStatusCodeWithBodyResult(HttpStatusCode.Forbidden, Strings.ApiKeyNotAuthorized);
}
// Check if a particular Id-Version combination already exists. We eventually need to remove this check.
string normalizedVersion = nuspec.GetVersion().ToNormalizedString();
bool packageExists =
packageRegistration.Packages.Any(
p => String.Equals(
p.NormalizedVersion,
normalizedVersion,
StringComparison.OrdinalIgnoreCase));
if (packageExists)
{
return new HttpStatusCodeWithBodyResult(
HttpStatusCode.Conflict,
String.Format(CultureInfo.CurrentCulture, Strings.PackageExistsAndCannotBeModified,
nuspec.GetId(), nuspec.GetVersion().ToNormalizedStringSafe()));
}
}
var packageStreamMetadata = new PackageStreamMetadata
{
HashAlgorithm = Constants.Sha512HashAlgorithmId,
Hash = CryptographyService.GenerateHash(packageStream.AsSeekableStream()),
Size = packageStream.Length,
};
var package = PackageService.CreatePackage(packageToPush, packageStreamMetadata, user, commitChanges: false);
AutoCuratePackage.Execute(package, packageToPush, commitChanges: false);
EntitiesContext.SaveChanges();
using (Stream uploadStream = packageStream)
{
uploadStream.Position = 0;
await PackageFileService.SavePackageFileAsync(package, uploadStream.AsSeekableStream());
IndexingService.UpdatePackage(package);
}
}
return new HttpStatusCodeResult(HttpStatusCode.Created);
}
private static bool DoesNotContainUnsupportedFiles(PackageReader nugetPackage)
{
foreach (var filePath in nugetPackage.GetFiles())
{
var fi = new FileInfo(filePath);
if (fi.Extension == ".ps1" || fi.Extension == ".t4")
{
return false;
}
}
return true;
}
