//================================================================
#region Getting the package from NuGet
private InstalledPackage TryGetPackageFromNuGetCache(PackageRequest request, FileSyncer binFileSyncer)
{
var sw = Stopwatch.StartNew();
// Use cache only if not deploying from source and an exact version is specified:
if (request.Source != null)
{
return(null);
}
var requestVersionsRange = !string.IsNullOrEmpty(request.VersionsRange)
? VersionUtility.ParseVersionSpec(request.VersionsRange)
: new VersionSpec();
// Default NuGet behavior is to download the smallest version in the given range, so only MinVersion is checked here:
if (requestVersionsRange.MinVersion == null || requestVersionsRange.MinVersion.Equals(new SemanticVersion("0.0")))
{
_logger.Trace(() => $"Not looking for {request.ReportIdVersionsRange()} in packages cache because the request does not specify an exact version.");
return(null);
}
// Find the NuGet package:
var nugetRepository = new LocalPackageRepository(Paths.PackagesCacheFolder, enableCaching: false);
IPackage package = nugetRepository.FindPackage(request.Id, requestVersionsRange, allowPrereleaseVersions: true, allowUnlisted: true);
_performanceLogger.Write(sw, () => $"PackageDownloader: {(package == null ? "Did not find" : "Found")} the NuGet package {request.ReportIdVersionsRange()} in cache.");
if (package == null)
{
return(null);
}
// Copy binary files and resources:
string packageSubfolder = nugetRepository.PathResolver.GetPackageDirectory(request.Id, package.Version);
_logger.Trace(() => $"Reading package {request.Id} from cache '{packageSubfolder}'.");
_deployPackagesLogger.Trace(() => $"Reading {request.Id} from cache.");
string targetFolder = Path.Combine(Paths.PackagesCacheFolder, packageSubfolder);
foreach (var file in FilterCompatibleLibFiles(package.GetFiles()))
{
binFileSyncer.AddFile(Path.Combine(targetFolder, file.Path), Paths.PluginsFolder, Path.Combine(Paths.PluginsFolder, file.EffectivePath));
}
binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Plugins"), Paths.PluginsFolder, recursive: false); // Obsolete bin folder; lib should be used instead.
binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Resources"), Paths.ResourcesFolder, SimplifyPackageName(package.Id), recursive: true);
return(new InstalledPackage(package.Id, package.Version.ToString(), GetNuGetPackageDependencies(package), targetFolder, request, Paths.PackagesCacheFolder));
}
/// <summary>
/// Downloads the packages from the provided sources, if not already downloaded.
/// Unpacks the packages, if not already unpacked.
/// </summary>
public List <InstalledPackage> GetPackages()
{
var sw = Stopwatch.StartNew();
var installedPackages = new List <InstalledPackage>();
installedPackages.Add(new InstalledPackage("Rhetos", SystemUtility.GetRhetosVersion(), new List <PackageRequest>(), Paths.RhetosServerRootPath,
new PackageRequest {
Id = "Rhetos", VersionsRange = "", Source = "", RequestedBy = "Rhetos framework"
}, ".", new List <ContentFile> {
}));
var binFileSyncer = new FileSyncer(_logProvider);
binFileSyncer.AddDestinations(Paths.PluginsFolder, Paths.ResourcesFolder); // Even if there are no packages, those folders must be created and emptied.
_filesUtility.SafeCreateDirectory(Paths.PackagesCacheFolder);
var packageRequests = _deploymentConfiguration.PackageRequests;
while (packageRequests.Any())
{
var newDependencies = new List <PackageRequest>();
foreach (var request in packageRequests)
{
if (!CheckAlreadyDownloaded(request, installedPackages))
{
var installedPackage = GetPackage(request, binFileSyncer);
ValidatePackage(installedPackage, request, installedPackages);
installedPackages.Add(installedPackage);
newDependencies.AddRange(installedPackage.Dependencies);
}
}
packageRequests = newDependencies;
}
DeleteObsoletePackages(installedPackages);
SortByDependencies(installedPackages);
binFileSyncer.UpdateDestination();
foreach (var package in installedPackages)
{
_packagesLogger.Trace(() => package.Report());
}
_performanceLogger.Write(sw, "PackageDownloader.GetPackages.");
return(installedPackages);
}
private InstalledPackage GetPackage(PackageRequest request, FileSyncer binFileSyncer)
{
var packageSources = SelectPackageSources(request);
foreach (var source in packageSources)
{
var installedPackage = TryGetPackage(source, request, binFileSyncer);
if (installedPackage != null)
{
return(installedPackage);
}
}
throw new UserException("Cannot download package " + request.ReportIdVersionsRange()
+ ". Looked at " + packageSources.Count() + " sources:"
+ string.Concat(packageSources.Select(source => "\r\n" + source.ProcessedLocation)));
}
private void CopyResourcesFromPackages()
{
var stopwatch = Stopwatch.StartNew();
var _fileSyncer = new FileSyncer(_logProvider);
_fileSyncer.AddDestinations(Paths.ResourcesFolder); // Even if there are no packages, the old folder content must be emptied.
const string ResourcesPathPrefix = @"Resources\";
var resourceFiles = _installedPackages.Packages
.SelectMany(package => package.ContentFiles
.Where(file => file.InPackagePath.StartsWith(ResourcesPathPrefix))
.Where(file => !file.PhysicalPath.StartsWith(Paths.ResourcesFolder)) // Prevent from including the generated output folder as in input.
.Select(file => new
{
Package = package,
Source = file.PhysicalPath,
Target = Path.Combine(SimplifyPackageName(package.Id), file.InPackagePath.Substring(ResourcesPathPrefix.Length))
}))
.ToList();
var similarPackages = resourceFiles.Select(file => file.Package).Distinct()
.GroupBy(package => SimplifyPackageName(package.Id), StringComparer.OrdinalIgnoreCase)
.FirstOrDefault(group => group.Count() > 1);
if (similarPackages != null)
{
throw new UserException($"Incompatible package names, resource files would result in the same target folder '{similarPackages.Key}'."
+ $"\r\nPackage 1: {similarPackages.First().ReportIdVersionRequestSource()}"
+ $"\r\nPackage 2: {similarPackages.Last().ReportIdVersionRequestSource()}");
}
foreach (var file in resourceFiles)
{
_fileSyncer.AddFile(file.Source, Paths.ResourcesFolder, file.Target);
}
_logger.Info($"Copying {resourceFiles.Count} resource files.");
_fileSyncer.UpdateDestination();
_performanceLogger.Write(stopwatch, "Resources generated.");
}
private InstalledPackage TryGetPackageFromNuGet(PackageSource source, PackageRequest request, FileSyncer binFileSyncer)
{
var sw = Stopwatch.StartNew();
// Find the NuGet package:
var nugetRepository = (source.Path != null && IsLocalPath(source.Path))
? new LocalPackageRepository(source.Path, enableCaching: false) // When developer rebuilds a package, the package version does not need to be increased every time.
: PackageRepositoryFactory.Default.CreateRepository(source.ProcessedLocation);
var requestVersionsRange = !string.IsNullOrEmpty(request.VersionsRange)
? VersionUtility.ParseVersionSpec(request.VersionsRange)
: new VersionSpec();
IEnumerable <IPackage> packages = nugetRepository.FindPackages(request.Id, requestVersionsRange, allowPrereleaseVersions: true, allowUnlisted: true).ToList();
if (requestVersionsRange.MinVersion != null && !requestVersionsRange.MinVersion.Equals(new SemanticVersion("0.0")))
{
packages = packages.OrderBy(p => p.Version); // Find the lowest compatible version if the version is specified (default NuGet behavior).
}
else
{
packages = packages.OrderByDescending(p => p.Version);
}
var package = packages.FirstOrDefault();
_performanceLogger.Write(sw, () => $"PackageDownloader: {(package == null ? "Did not find" : "Found")} the NuGet package {request.ReportIdVersionsRange()} at {source.ProcessedLocation}.");
if (package == null)
{
return(null);
}
// Download the NuGet package:
_logger.Trace("Downloading NuGet package " + package.Id + " " + package.Version + " from " + source.ProcessedLocation + ".");
var packageManager = new PackageManager(nugetRepository, Paths.PackagesCacheFolder)
{
Logger = new LoggerForNuget(_logProvider)
};
packageManager.LocalRepository.PackageSaveMode = PackageSaveModes.Nuspec;
packageManager.InstallPackage(package, ignoreDependencies: true, allowPrereleaseVersions: true);
_performanceLogger.Write(sw, () => "PackageDownloader: Installed NuGet package " + request.Id + ".");
string targetFolder = packageManager.PathResolver.GetInstallPath(package);
// Copy binary files and resources:
foreach (var file in FilterCompatibleLibFiles(package.GetFiles()))
{
binFileSyncer.AddFile(Path.Combine(targetFolder, file.Path), Paths.PluginsFolder, Path.Combine(Paths.PluginsFolder, file.EffectivePath));
}
binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Plugins"), Paths.PluginsFolder, recursive: false); // Obsolete bin folder; lib should be used instead.
binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Resources"), Paths.ResourcesFolder, SimplifyPackageName(package.Id), recursive: true);
return(new InstalledPackage(package.Id, package.Version.ToString(), GetNuGetPackageDependencies(package), targetFolder, request, source.ProcessedLocation));
}
//================================================================
#region Getting the package from legacy zip file
private InstalledPackage TryGetPackageFromLegacyZipPackage(PackageSource source, PackageRequest request, FileSyncer binFileSyncer)
{
if (source.Path == null)
{
return(null);
}
Version simpleVersion;
if (!Version.TryParse(request.VersionsRange, out simpleVersion))
{
return(null);
}
string zipPackageName = GetZipPackageName(request);
string zipPackagePath = Path.Combine(source.Path, zipPackageName);
if (!File.Exists(zipPackagePath))
{
_logger.Trace(() => "There is no legacy file " + zipPackageName + " in " + source.ProvidedLocation + ".");
return(null);
}
_logger.Trace(() => "Reading package " + request.Id + " from legacy file " + zipPackageName + ".");
string targetFolder = GetTargetFolder(request.Id, request.VersionsRange);
_filesUtility.EmptyDirectory(targetFolder);
using (var zipFile = ZipFile.Read(zipPackagePath))
foreach (var zipEntry in zipFile)
{
zipEntry.Extract(targetFolder, ExtractExistingFileAction.OverwriteSilently);
}
binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Plugins"), Paths.PluginsFolder, recursive: false);
binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Resources"), Paths.ResourcesFolder, SimplifyPackageName(request.Id), recursive: true);
return(new InstalledPackage(request.Id, request.VersionsRange, new List <PackageRequest> {
}, targetFolder, request, source.ProcessedLocation));
}
private InstalledPackage UseFilesFromUnpackedSourceWithoutMetadata(string sourceFolder, PackageRequest request, FileSyncer binFileSyncer)
{
string sourcePluginsFolder = Path.Combine(sourceFolder, "Plugins");
if (Directory.Exists(sourcePluginsFolder) &&
Directory.EnumerateFiles(sourcePluginsFolder, "*.dll", SearchOption.AllDirectories).FirstOrDefault() != null &&
Directory.EnumerateFiles(sourcePluginsFolder, "*.dll", SearchOption.TopDirectoryOnly).FirstOrDefault() == null)
{
_logger.Error(() => "Package " + request.Id + " source folder contains Plugins that will not be installed to the Rhetos server. Add '" + request.Id + ".nuspec' file to define the which plugin files should be installed.");
}
binFileSyncer.AddFolderContent(Path.Combine(sourceFolder, "Plugins"), Paths.PluginsFolder, recursive: false);
binFileSyncer.AddFolderContent(Path.Combine(sourceFolder, "Resources"), Paths.ResourcesFolder, SimplifyPackageName(request.Id), recursive: true);
string defaultVersion = CreateVersionInRangeOrZero(request);
return(new InstalledPackage(request.Id, defaultVersion, new List <PackageRequest> {
}, sourceFolder, request, sourceFolder));
}
private InstalledPackage UseFilesFromUnpackedSourceWithMetadata(string metadataFile, PackageRequest request, FileSyncer binFileSyncer)
{
var properties = new SimplePropertyProvider
{
{ "Configuration", "Debug" },
};
var packageBuilder = new PackageBuilder(metadataFile, properties, includeEmptyDirectories: false);
var sourceFolder = Path.GetDirectoryName(metadataFile);
var targetFolder = GetTargetFolder(packageBuilder.Id, packageBuilder.Version.ToString());
// Copy binary files and resources:
foreach (PhysicalPackageFile file in FilterCompatibleLibFiles(packageBuilder.Files))
{
binFileSyncer.AddFile(file.SourcePath, Paths.PluginsFolder, Path.Combine(Paths.PluginsFolder, file.EffectivePath));
}
foreach (PhysicalPackageFile file in packageBuilder.Files)
{
if (file.Path.StartsWith(@"Plugins\")) // Obsolete bin folder; lib should be used instead.
{
binFileSyncer.AddFile(file.SourcePath, Paths.PluginsFolder);
}
else if (file.Path.StartsWith(@"Resources\"))
{
binFileSyncer.AddFile(file.SourcePath, Paths.ResourcesFolder, Path.Combine(SimplifyPackageName(packageBuilder.Id), file.Path.Substring(@"Resources\".Length)));
}
}
return(new InstalledPackage(packageBuilder.Id, packageBuilder.Version.ToString(), GetNuGetPackageDependencies(packageBuilder),
sourceFolder, request, sourceFolder));
}
//================================================================
#region Getting the package from unpacked source folder
private InstalledPackage TryGetPackageFromUnpackedSourceFolder(PackageSource source, PackageRequest request, FileSyncer binFileSyncer)
{
if (request.Source == null) // Unpacked source folder must be explicitly set in the package request.
{
return(null);
}
if (source.Path == null || !Directory.Exists(source.Path))
{
return(null);
}
var packageExtensions = new[] { "*.nupkg", GetZipPackageName(request) };
var existingPackageFiles = packageExtensions.SelectMany(ext => Directory.GetFiles(source.Path, ext));
var existingMetadataFiles = Directory.GetFiles(source.Path, "*.nuspec");
if (existingMetadataFiles.Length > 1)
{
// If any nuspec file exactly matches the packages name, use that one.
var standardFileName = existingMetadataFiles.Where(f => string.Equals(Path.GetFileName(f), request.Id + ".nuspec", StringComparison.OrdinalIgnoreCase)).ToArray();
if (standardFileName.Length == 1)
{
existingMetadataFiles = standardFileName;
}
}
var packageSourceSubfolders = new[] { "DslScripts", "DataMigration", "Plugins", "Resources" };
var existingSourceSubfolders = packageSourceSubfolders.Where(subfolder => Directory.Exists(Path.Combine(source.Path, subfolder)));
if (existingPackageFiles.Count() > 0)
{
string ambiguousAlternative = null;
if (existingMetadataFiles.Count() > 0)
{
ambiguousAlternative = $".nuspec file '{Path.GetFileName(existingMetadataFiles.First())}'";
}
else if (existingSourceSubfolders.Count() > 0)
{
ambiguousAlternative = $"source folder '{existingSourceSubfolders.First()}'";
}
if (ambiguousAlternative != null)
{
throw new FrameworkException($"Ambiguous source for package {request.Id}. Source folder '{source.Path}' contains both" +
$" package file '{Path.GetFileName(existingPackageFiles.First())}' and {ambiguousAlternative}.");
}
_logger.Trace(() => $"Package {request.Id} source folder is not considered as unpacked source because" +
$" it contains a package file '{Path.GetFileName(existingPackageFiles.First())}'.");
return(null);
}
else if (existingMetadataFiles.Length > 1)
{
_logger.Info(() => "Package " + request.Id + " source folder '" + source.ProvidedLocation + "' contains multiple .nuspec metadata files.");
return(null);
}
else if (existingMetadataFiles.Length == 1)
{
_logger.Trace(() => "Reading package " + request.Id + " from unpacked source folder with metadata " + Path.GetFileName(existingMetadataFiles.Single()) + ".");
return(UseFilesFromUnpackedSourceWithMetadata(existingMetadataFiles.Single(), request, binFileSyncer));
}
else if (existingSourceSubfolders.Any())
{
_logger.Trace(() => "Reading package " + request.Id + " from unpacked source folder without metadata file.");
return(UseFilesFromUnpackedSourceWithoutMetadata(source.Path, request, binFileSyncer));
}
else
{
return(null);
}
}
private InstalledPackage TryGetPackage(PackageSource source, PackageRequest request, FileSyncer binFileSyncer)
{
return(TryGetPackageFromUnpackedSourceFolder(source, request, binFileSyncer)
?? TryGetPackageFromLegacyZipPackage(source, request, binFileSyncer)
?? TryGetPackageFromNuGet(source, request, binFileSyncer));
}
//================================================================
#region Getting the package from legacy zip file
private InstalledPackage TryGetPackageFromLegacyZipPackage(PackageSource source, PackageRequest request, FileSyncer binFileSyncer)
{
if (source.Path == null)
{
return(null);
}
var zipPackage = GetExpectedZipPackage(request);
if (zipPackage == null)
{
return(null);
}
string zipPackagePath = Path.Combine(source.Path, zipPackage.FileName);
if (!File.Exists(zipPackagePath))
{
_logger.Trace(() => $"There is no legacy zip package at {zipPackagePath}.");
return(null);
}
_logger.Trace(() => $"Reading package {request.Id} from legacy file {zipPackagePath}.");
_logger.Info(() => $"Reading {request.Id} from legacy zip file.");
string targetFolder = GetTargetFolder(request.Id, zipPackage.Version);
_filesUtility.EmptyDirectory(targetFolder);
ZipFile.ExtractToDirectory(zipPackagePath, targetFolder);
binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Plugins"), Paths.PluginsFolder, recursive: false);
return(new InstalledPackage(request.Id, zipPackage.Version, new List <PackageRequest> {
}, targetFolder, request, source.ProcessedLocation));
}
private InstalledPackage UseFilesFromUnpackedSourceWithMetadata(string metadataFile, PackageRequest request, FileSyncer binFileSyncer)
{
var properties = new SimplePropertyProvider
{
{ "Configuration", "Debug" },
};
var packageBuilder = new PackageBuilder(metadataFile, properties, includeEmptyDirectories: false);
string sourceFolder = Path.GetDirectoryName(metadataFile);
// Copy binary files:
foreach (var file in FilterCompatibleLibFiles(packageBuilder.Files).Cast <PhysicalPackageFile>())
{
binFileSyncer.AddFile(file.SourcePath, Paths.PluginsFolder, Path.Combine(Paths.PluginsFolder, file.EffectivePath));
}
foreach (var file in packageBuilder.Files.Cast <PhysicalPackageFile>())
{
if (file.Path.StartsWith(@"Plugins\")) // Obsolete bin folder; lib should be used instead.
{
binFileSyncer.AddFile(file.SourcePath, Paths.PluginsFolder);
}
}
var contentFiles = packageBuilder.Files.Cast <PhysicalPackageFile>()
.Select(file => new ContentFile {
PhysicalPath = file.SourcePath, InPackagePath = file.Path
})
.ToList();
return(new InstalledPackage(packageBuilder.Id, packageBuilder.Version.ToString(), GetNuGetPackageDependencies(packageBuilder),
sourceFolder, request, sourceFolder, contentFiles));
}
//================================================================
#region Getting the package from unpacked source folder
private InstalledPackage TryGetPackageFromUnpackedSourceFolder(PackageSource source, PackageRequest request, FileSyncer binFileSyncer)
{
if (request.Source == null) // Unpacked source folder must be explicitly set in the package request.
{
return(null);
}
if (source.Path == null || !Directory.Exists(source.Path))
{
return(null);
}
foreach (string packedExtension in new[] { "zip", "nupkg" })
{
if (Directory.GetFiles(source.Path, "*." + packedExtension).Length > 0)
{
_logger.Trace(() => "Package " + request.Id + " source folder is not considered as unpacked source because it contains ." + packedExtension + " files.");
return(null);
}
}
var metadataFiles = Directory.GetFiles(source.Path, "*.nuspec");
// Disambiguation by name:
if (metadataFiles.Length > 1)
{
var standardFileName = metadataFiles.Where(f => string.Equals(Path.GetFileName(f), request.Id + ".nuspec", StringComparison.OrdinalIgnoreCase)).ToArray();
if (standardFileName.Length == 1)
{
metadataFiles = standardFileName;
}
}
if (metadataFiles.Length > 1)
{
_logger.Info(() => "Package " + request.Id + " source folder '" + source.ProvidedLocation + "' contains multiple .nuspec metadata files.");
return(null);
}
else if (metadataFiles.Length == 1)
{
_logger.Trace(() => "Reading package " + request.Id + " from unpacked source folder with metadata " + Path.GetFileName(metadataFiles.Single()) + ".");
return(UseFilesFromUnpackedSourceWithMetadata(metadataFiles.Single(), request, binFileSyncer));
}
else
{
var rhetosPackageSubfolders = new[] { "DslScripts", "DataMigration", "Plugins", "Resources" };
if (rhetosPackageSubfolders.Any(subfolder => Directory.Exists(Path.Combine(source.Path, subfolder))))
{
_logger.Trace(() => "Reading package " + request.Id + " from unpacked source folder without metadata file.");
return(UseFilesFromUnpackedSourceWithoutMetadata(source.Path, request, binFileSyncer));
}
else
{
return(null);
}
}
}
//================================================================
#region Getting the package from NuGet
private InstalledPackage TryGetPackageFromNuGet(PackageSource source, PackageRequest request, FileSyncer binFileSyncer)
{
var sw = Stopwatch.StartNew();
var nugetRepository = (source.Path != null && IsLocalPath(source.Path))
? new LocalPackageRepository(source.Path, enableCaching: false) // When developer rebuilds a package, the package version does not need to be increased every time.
: PackageRepositoryFactory.Default.CreateRepository(source.ProcessedLocation);
var requestVersionsRange = !string.IsNullOrEmpty(request.VersionsRange)
? VersionUtility.ParseVersionSpec(request.VersionsRange)
: new VersionSpec();
var package = nugetRepository.FindPackage(request.Id, requestVersionsRange, allowPrereleaseVersions: true, allowUnlisted: true);
_performanceLogger.Write(sw, () => "PackageDownloader find NuGet package " + request.Id + ".");
if (package == null)
{
_logger.Trace("Package " + request.ReportIdVersionsRange() + " not found by NuGet at " + source.ProcessedLocation + ".");
return(null);
}
else
{
_logger.Trace("Downloading NuGet package " + package.Id + " " + package.Version + " from " + source.ProcessedLocation + ".");
}
var packageManager = new PackageManager(nugetRepository, Paths.PackagesFolder)
{
Logger = new LoggerForNuget(_logProvider)
};
packageManager.LocalRepository.PackageSaveMode = PackageSaveModes.Nuspec;
packageManager.InstallPackage(package, ignoreDependencies: true, allowPrereleaseVersions: true);
_performanceLogger.Write(sw, () => "PackageDownloader install NuGet package " + request.Id + ".");
string targetFolder = packageManager.PathResolver.GetInstallPath(package);
binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Plugins"), Paths.PluginsFolder, recursive: false);
binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Resources"), Paths.ResourcesFolder, SimplifyPackageName(package.Id), recursive: true);
return(new InstalledPackage(package.Id, package.Version.ToString(), GetNuGetPackageDependencies(package), targetFolder, request, source.ProcessedLocation, GetNuGetRequiredRhetosVersion(package)));
}