//================================================================ #region Getting the package from NuGet private InstalledPackage TryGetPackageFromNuGetCache(PackageRequest request, FileSyncer binFileSyncer) { var sw = Stopwatch.StartNew(); // Use cache only if not deploying from source and an exact version is specified: if (request.Source != null) { return(null); } var requestVersionsRange = !string.IsNullOrEmpty(request.VersionsRange) ? VersionUtility.ParseVersionSpec(request.VersionsRange) : new VersionSpec(); // Default NuGet behavior is to download the smallest version in the given range, so only MinVersion is checked here: if (requestVersionsRange.MinVersion == null || requestVersionsRange.MinVersion.Equals(new SemanticVersion("0.0"))) { _logger.Trace(() => $"Not looking for {request.ReportIdVersionsRange()} in packages cache because the request does not specify an exact version."); return(null); } // Find the NuGet package: var nugetRepository = new LocalPackageRepository(Paths.PackagesCacheFolder, enableCaching: false); IPackage package = nugetRepository.FindPackage(request.Id, requestVersionsRange, allowPrereleaseVersions: true, allowUnlisted: true); _performanceLogger.Write(sw, () => $"PackageDownloader: {(package == null ? "Did not find" : "Found")} the NuGet package {request.ReportIdVersionsRange()} in cache."); if (package == null) { return(null); } // Copy binary files and resources: string packageSubfolder = nugetRepository.PathResolver.GetPackageDirectory(request.Id, package.Version); _logger.Trace(() => $"Reading package {request.Id} from cache '{packageSubfolder}'."); _deployPackagesLogger.Trace(() => $"Reading {request.Id} from cache."); string targetFolder = Path.Combine(Paths.PackagesCacheFolder, packageSubfolder); foreach (var file in FilterCompatibleLibFiles(package.GetFiles())) { binFileSyncer.AddFile(Path.Combine(targetFolder, file.Path), Paths.PluginsFolder, Path.Combine(Paths.PluginsFolder, file.EffectivePath)); } binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Plugins"), Paths.PluginsFolder, recursive: false); // Obsolete bin folder; lib should be used instead. binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Resources"), Paths.ResourcesFolder, SimplifyPackageName(package.Id), recursive: true); return(new InstalledPackage(package.Id, package.Version.ToString(), GetNuGetPackageDependencies(package), targetFolder, request, Paths.PackagesCacheFolder)); }
/// <summary> /// Downloads the packages from the provided sources, if not already downloaded. /// Unpacks the packages, if not already unpacked. /// </summary> public List <InstalledPackage> GetPackages() { var sw = Stopwatch.StartNew(); var installedPackages = new List <InstalledPackage>(); installedPackages.Add(new InstalledPackage("Rhetos", SystemUtility.GetRhetosVersion(), new List <PackageRequest>(), Paths.RhetosServerRootPath, new PackageRequest { Id = "Rhetos", VersionsRange = "", Source = "", RequestedBy = "Rhetos framework" }, ".", new List <ContentFile> { })); var binFileSyncer = new FileSyncer(_logProvider); binFileSyncer.AddDestinations(Paths.PluginsFolder, Paths.ResourcesFolder); // Even if there are no packages, those folders must be created and emptied. _filesUtility.SafeCreateDirectory(Paths.PackagesCacheFolder); var packageRequests = _deploymentConfiguration.PackageRequests; while (packageRequests.Any()) { var newDependencies = new List <PackageRequest>(); foreach (var request in packageRequests) { if (!CheckAlreadyDownloaded(request, installedPackages)) { var installedPackage = GetPackage(request, binFileSyncer); ValidatePackage(installedPackage, request, installedPackages); installedPackages.Add(installedPackage); newDependencies.AddRange(installedPackage.Dependencies); } } packageRequests = newDependencies; } DeleteObsoletePackages(installedPackages); SortByDependencies(installedPackages); binFileSyncer.UpdateDestination(); foreach (var package in installedPackages) { _packagesLogger.Trace(() => package.Report()); } _performanceLogger.Write(sw, "PackageDownloader.GetPackages."); return(installedPackages); }
private InstalledPackage GetPackage(PackageRequest request, FileSyncer binFileSyncer) { var packageSources = SelectPackageSources(request); foreach (var source in packageSources) { var installedPackage = TryGetPackage(source, request, binFileSyncer); if (installedPackage != null) { return(installedPackage); } } throw new UserException("Cannot download package " + request.ReportIdVersionsRange() + ". Looked at " + packageSources.Count() + " sources:" + string.Concat(packageSources.Select(source => "\r\n" + source.ProcessedLocation))); }
private void CopyResourcesFromPackages() { var stopwatch = Stopwatch.StartNew(); var _fileSyncer = new FileSyncer(_logProvider); _fileSyncer.AddDestinations(Paths.ResourcesFolder); // Even if there are no packages, the old folder content must be emptied. const string ResourcesPathPrefix = @"Resources\"; var resourceFiles = _installedPackages.Packages .SelectMany(package => package.ContentFiles .Where(file => file.InPackagePath.StartsWith(ResourcesPathPrefix)) .Where(file => !file.PhysicalPath.StartsWith(Paths.ResourcesFolder)) // Prevent from including the generated output folder as in input. .Select(file => new { Package = package, Source = file.PhysicalPath, Target = Path.Combine(SimplifyPackageName(package.Id), file.InPackagePath.Substring(ResourcesPathPrefix.Length)) })) .ToList(); var similarPackages = resourceFiles.Select(file => file.Package).Distinct() .GroupBy(package => SimplifyPackageName(package.Id), StringComparer.OrdinalIgnoreCase) .FirstOrDefault(group => group.Count() > 1); if (similarPackages != null) { throw new UserException($"Incompatible package names, resource files would result in the same target folder '{similarPackages.Key}'." + $"\r\nPackage 1: {similarPackages.First().ReportIdVersionRequestSource()}" + $"\r\nPackage 2: {similarPackages.Last().ReportIdVersionRequestSource()}"); } foreach (var file in resourceFiles) { _fileSyncer.AddFile(file.Source, Paths.ResourcesFolder, file.Target); } _logger.Info($"Copying {resourceFiles.Count} resource files."); _fileSyncer.UpdateDestination(); _performanceLogger.Write(stopwatch, "Resources generated."); }
private InstalledPackage TryGetPackageFromNuGet(PackageSource source, PackageRequest request, FileSyncer binFileSyncer) { var sw = Stopwatch.StartNew(); // Find the NuGet package: var nugetRepository = (source.Path != null && IsLocalPath(source.Path)) ? new LocalPackageRepository(source.Path, enableCaching: false) // When developer rebuilds a package, the package version does not need to be increased every time. : PackageRepositoryFactory.Default.CreateRepository(source.ProcessedLocation); var requestVersionsRange = !string.IsNullOrEmpty(request.VersionsRange) ? VersionUtility.ParseVersionSpec(request.VersionsRange) : new VersionSpec(); IEnumerable <IPackage> packages = nugetRepository.FindPackages(request.Id, requestVersionsRange, allowPrereleaseVersions: true, allowUnlisted: true).ToList(); if (requestVersionsRange.MinVersion != null && !requestVersionsRange.MinVersion.Equals(new SemanticVersion("0.0"))) { packages = packages.OrderBy(p => p.Version); // Find the lowest compatible version if the version is specified (default NuGet behavior). } else { packages = packages.OrderByDescending(p => p.Version); } var package = packages.FirstOrDefault(); _performanceLogger.Write(sw, () => $"PackageDownloader: {(package == null ? "Did not find" : "Found")} the NuGet package {request.ReportIdVersionsRange()} at {source.ProcessedLocation}."); if (package == null) { return(null); } // Download the NuGet package: _logger.Trace("Downloading NuGet package " + package.Id + " " + package.Version + " from " + source.ProcessedLocation + "."); var packageManager = new PackageManager(nugetRepository, Paths.PackagesCacheFolder) { Logger = new LoggerForNuget(_logProvider) }; packageManager.LocalRepository.PackageSaveMode = PackageSaveModes.Nuspec; packageManager.InstallPackage(package, ignoreDependencies: true, allowPrereleaseVersions: true); _performanceLogger.Write(sw, () => "PackageDownloader: Installed NuGet package " + request.Id + "."); string targetFolder = packageManager.PathResolver.GetInstallPath(package); // Copy binary files and resources: foreach (var file in FilterCompatibleLibFiles(package.GetFiles())) { binFileSyncer.AddFile(Path.Combine(targetFolder, file.Path), Paths.PluginsFolder, Path.Combine(Paths.PluginsFolder, file.EffectivePath)); } binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Plugins"), Paths.PluginsFolder, recursive: false); // Obsolete bin folder; lib should be used instead. binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Resources"), Paths.ResourcesFolder, SimplifyPackageName(package.Id), recursive: true); return(new InstalledPackage(package.Id, package.Version.ToString(), GetNuGetPackageDependencies(package), targetFolder, request, source.ProcessedLocation)); }
//================================================================ #region Getting the package from legacy zip file private InstalledPackage TryGetPackageFromLegacyZipPackage(PackageSource source, PackageRequest request, FileSyncer binFileSyncer) { if (source.Path == null) { return(null); } Version simpleVersion; if (!Version.TryParse(request.VersionsRange, out simpleVersion)) { return(null); } string zipPackageName = GetZipPackageName(request); string zipPackagePath = Path.Combine(source.Path, zipPackageName); if (!File.Exists(zipPackagePath)) { _logger.Trace(() => "There is no legacy file " + zipPackageName + " in " + source.ProvidedLocation + "."); return(null); } _logger.Trace(() => "Reading package " + request.Id + " from legacy file " + zipPackageName + "."); string targetFolder = GetTargetFolder(request.Id, request.VersionsRange); _filesUtility.EmptyDirectory(targetFolder); using (var zipFile = ZipFile.Read(zipPackagePath)) foreach (var zipEntry in zipFile) { zipEntry.Extract(targetFolder, ExtractExistingFileAction.OverwriteSilently); } binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Plugins"), Paths.PluginsFolder, recursive: false); binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Resources"), Paths.ResourcesFolder, SimplifyPackageName(request.Id), recursive: true); return(new InstalledPackage(request.Id, request.VersionsRange, new List <PackageRequest> { }, targetFolder, request, source.ProcessedLocation)); }
private InstalledPackage UseFilesFromUnpackedSourceWithoutMetadata(string sourceFolder, PackageRequest request, FileSyncer binFileSyncer) { string sourcePluginsFolder = Path.Combine(sourceFolder, "Plugins"); if (Directory.Exists(sourcePluginsFolder) && Directory.EnumerateFiles(sourcePluginsFolder, "*.dll", SearchOption.AllDirectories).FirstOrDefault() != null && Directory.EnumerateFiles(sourcePluginsFolder, "*.dll", SearchOption.TopDirectoryOnly).FirstOrDefault() == null) { _logger.Error(() => "Package " + request.Id + " source folder contains Plugins that will not be installed to the Rhetos server. Add '" + request.Id + ".nuspec' file to define the which plugin files should be installed."); } binFileSyncer.AddFolderContent(Path.Combine(sourceFolder, "Plugins"), Paths.PluginsFolder, recursive: false); binFileSyncer.AddFolderContent(Path.Combine(sourceFolder, "Resources"), Paths.ResourcesFolder, SimplifyPackageName(request.Id), recursive: true); string defaultVersion = CreateVersionInRangeOrZero(request); return(new InstalledPackage(request.Id, defaultVersion, new List <PackageRequest> { }, sourceFolder, request, sourceFolder)); }
private InstalledPackage UseFilesFromUnpackedSourceWithMetadata(string metadataFile, PackageRequest request, FileSyncer binFileSyncer) { var properties = new SimplePropertyProvider { { "Configuration", "Debug" }, }; var packageBuilder = new PackageBuilder(metadataFile, properties, includeEmptyDirectories: false); var sourceFolder = Path.GetDirectoryName(metadataFile); var targetFolder = GetTargetFolder(packageBuilder.Id, packageBuilder.Version.ToString()); // Copy binary files and resources: foreach (PhysicalPackageFile file in FilterCompatibleLibFiles(packageBuilder.Files)) { binFileSyncer.AddFile(file.SourcePath, Paths.PluginsFolder, Path.Combine(Paths.PluginsFolder, file.EffectivePath)); } foreach (PhysicalPackageFile file in packageBuilder.Files) { if (file.Path.StartsWith(@"Plugins\")) // Obsolete bin folder; lib should be used instead. { binFileSyncer.AddFile(file.SourcePath, Paths.PluginsFolder); } else if (file.Path.StartsWith(@"Resources\")) { binFileSyncer.AddFile(file.SourcePath, Paths.ResourcesFolder, Path.Combine(SimplifyPackageName(packageBuilder.Id), file.Path.Substring(@"Resources\".Length))); } } return(new InstalledPackage(packageBuilder.Id, packageBuilder.Version.ToString(), GetNuGetPackageDependencies(packageBuilder), sourceFolder, request, sourceFolder)); }
//================================================================ #region Getting the package from unpacked source folder private InstalledPackage TryGetPackageFromUnpackedSourceFolder(PackageSource source, PackageRequest request, FileSyncer binFileSyncer) { if (request.Source == null) // Unpacked source folder must be explicitly set in the package request. { return(null); } if (source.Path == null || !Directory.Exists(source.Path)) { return(null); } var packageExtensions = new[] { "*.nupkg", GetZipPackageName(request) }; var existingPackageFiles = packageExtensions.SelectMany(ext => Directory.GetFiles(source.Path, ext)); var existingMetadataFiles = Directory.GetFiles(source.Path, "*.nuspec"); if (existingMetadataFiles.Length > 1) { // If any nuspec file exactly matches the packages name, use that one. var standardFileName = existingMetadataFiles.Where(f => string.Equals(Path.GetFileName(f), request.Id + ".nuspec", StringComparison.OrdinalIgnoreCase)).ToArray(); if (standardFileName.Length == 1) { existingMetadataFiles = standardFileName; } } var packageSourceSubfolders = new[] { "DslScripts", "DataMigration", "Plugins", "Resources" }; var existingSourceSubfolders = packageSourceSubfolders.Where(subfolder => Directory.Exists(Path.Combine(source.Path, subfolder))); if (existingPackageFiles.Count() > 0) { string ambiguousAlternative = null; if (existingMetadataFiles.Count() > 0) { ambiguousAlternative = $".nuspec file '{Path.GetFileName(existingMetadataFiles.First())}'"; } else if (existingSourceSubfolders.Count() > 0) { ambiguousAlternative = $"source folder '{existingSourceSubfolders.First()}'"; } if (ambiguousAlternative != null) { throw new FrameworkException($"Ambiguous source for package {request.Id}. Source folder '{source.Path}' contains both" + $" package file '{Path.GetFileName(existingPackageFiles.First())}' and {ambiguousAlternative}."); } _logger.Trace(() => $"Package {request.Id} source folder is not considered as unpacked source because" + $" it contains a package file '{Path.GetFileName(existingPackageFiles.First())}'."); return(null); } else if (existingMetadataFiles.Length > 1) { _logger.Info(() => "Package " + request.Id + " source folder '" + source.ProvidedLocation + "' contains multiple .nuspec metadata files."); return(null); } else if (existingMetadataFiles.Length == 1) { _logger.Trace(() => "Reading package " + request.Id + " from unpacked source folder with metadata " + Path.GetFileName(existingMetadataFiles.Single()) + "."); return(UseFilesFromUnpackedSourceWithMetadata(existingMetadataFiles.Single(), request, binFileSyncer)); } else if (existingSourceSubfolders.Any()) { _logger.Trace(() => "Reading package " + request.Id + " from unpacked source folder without metadata file."); return(UseFilesFromUnpackedSourceWithoutMetadata(source.Path, request, binFileSyncer)); } else { return(null); } }
private InstalledPackage TryGetPackage(PackageSource source, PackageRequest request, FileSyncer binFileSyncer) { return(TryGetPackageFromUnpackedSourceFolder(source, request, binFileSyncer) ?? TryGetPackageFromLegacyZipPackage(source, request, binFileSyncer) ?? TryGetPackageFromNuGet(source, request, binFileSyncer)); }
//================================================================ #region Getting the package from legacy zip file private InstalledPackage TryGetPackageFromLegacyZipPackage(PackageSource source, PackageRequest request, FileSyncer binFileSyncer) { if (source.Path == null) { return(null); } var zipPackage = GetExpectedZipPackage(request); if (zipPackage == null) { return(null); } string zipPackagePath = Path.Combine(source.Path, zipPackage.FileName); if (!File.Exists(zipPackagePath)) { _logger.Trace(() => $"There is no legacy zip package at {zipPackagePath}."); return(null); } _logger.Trace(() => $"Reading package {request.Id} from legacy file {zipPackagePath}."); _logger.Info(() => $"Reading {request.Id} from legacy zip file."); string targetFolder = GetTargetFolder(request.Id, zipPackage.Version); _filesUtility.EmptyDirectory(targetFolder); ZipFile.ExtractToDirectory(zipPackagePath, targetFolder); binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Plugins"), Paths.PluginsFolder, recursive: false); return(new InstalledPackage(request.Id, zipPackage.Version, new List <PackageRequest> { }, targetFolder, request, source.ProcessedLocation)); }
private InstalledPackage UseFilesFromUnpackedSourceWithMetadata(string metadataFile, PackageRequest request, FileSyncer binFileSyncer) { var properties = new SimplePropertyProvider { { "Configuration", "Debug" }, }; var packageBuilder = new PackageBuilder(metadataFile, properties, includeEmptyDirectories: false); string sourceFolder = Path.GetDirectoryName(metadataFile); // Copy binary files: foreach (var file in FilterCompatibleLibFiles(packageBuilder.Files).Cast <PhysicalPackageFile>()) { binFileSyncer.AddFile(file.SourcePath, Paths.PluginsFolder, Path.Combine(Paths.PluginsFolder, file.EffectivePath)); } foreach (var file in packageBuilder.Files.Cast <PhysicalPackageFile>()) { if (file.Path.StartsWith(@"Plugins\")) // Obsolete bin folder; lib should be used instead. { binFileSyncer.AddFile(file.SourcePath, Paths.PluginsFolder); } } var contentFiles = packageBuilder.Files.Cast <PhysicalPackageFile>() .Select(file => new ContentFile { PhysicalPath = file.SourcePath, InPackagePath = file.Path }) .ToList(); return(new InstalledPackage(packageBuilder.Id, packageBuilder.Version.ToString(), GetNuGetPackageDependencies(packageBuilder), sourceFolder, request, sourceFolder, contentFiles)); }
//================================================================ #region Getting the package from unpacked source folder private InstalledPackage TryGetPackageFromUnpackedSourceFolder(PackageSource source, PackageRequest request, FileSyncer binFileSyncer) { if (request.Source == null) // Unpacked source folder must be explicitly set in the package request. { return(null); } if (source.Path == null || !Directory.Exists(source.Path)) { return(null); } foreach (string packedExtension in new[] { "zip", "nupkg" }) { if (Directory.GetFiles(source.Path, "*." + packedExtension).Length > 0) { _logger.Trace(() => "Package " + request.Id + " source folder is not considered as unpacked source because it contains ." + packedExtension + " files."); return(null); } } var metadataFiles = Directory.GetFiles(source.Path, "*.nuspec"); // Disambiguation by name: if (metadataFiles.Length > 1) { var standardFileName = metadataFiles.Where(f => string.Equals(Path.GetFileName(f), request.Id + ".nuspec", StringComparison.OrdinalIgnoreCase)).ToArray(); if (standardFileName.Length == 1) { metadataFiles = standardFileName; } } if (metadataFiles.Length > 1) { _logger.Info(() => "Package " + request.Id + " source folder '" + source.ProvidedLocation + "' contains multiple .nuspec metadata files."); return(null); } else if (metadataFiles.Length == 1) { _logger.Trace(() => "Reading package " + request.Id + " from unpacked source folder with metadata " + Path.GetFileName(metadataFiles.Single()) + "."); return(UseFilesFromUnpackedSourceWithMetadata(metadataFiles.Single(), request, binFileSyncer)); } else { var rhetosPackageSubfolders = new[] { "DslScripts", "DataMigration", "Plugins", "Resources" }; if (rhetosPackageSubfolders.Any(subfolder => Directory.Exists(Path.Combine(source.Path, subfolder)))) { _logger.Trace(() => "Reading package " + request.Id + " from unpacked source folder without metadata file."); return(UseFilesFromUnpackedSourceWithoutMetadata(source.Path, request, binFileSyncer)); } else { return(null); } } }
//================================================================ #region Getting the package from NuGet private InstalledPackage TryGetPackageFromNuGet(PackageSource source, PackageRequest request, FileSyncer binFileSyncer) { var sw = Stopwatch.StartNew(); var nugetRepository = (source.Path != null && IsLocalPath(source.Path)) ? new LocalPackageRepository(source.Path, enableCaching: false) // When developer rebuilds a package, the package version does not need to be increased every time. : PackageRepositoryFactory.Default.CreateRepository(source.ProcessedLocation); var requestVersionsRange = !string.IsNullOrEmpty(request.VersionsRange) ? VersionUtility.ParseVersionSpec(request.VersionsRange) : new VersionSpec(); var package = nugetRepository.FindPackage(request.Id, requestVersionsRange, allowPrereleaseVersions: true, allowUnlisted: true); _performanceLogger.Write(sw, () => "PackageDownloader find NuGet package " + request.Id + "."); if (package == null) { _logger.Trace("Package " + request.ReportIdVersionsRange() + " not found by NuGet at " + source.ProcessedLocation + "."); return(null); } else { _logger.Trace("Downloading NuGet package " + package.Id + " " + package.Version + " from " + source.ProcessedLocation + "."); } var packageManager = new PackageManager(nugetRepository, Paths.PackagesFolder) { Logger = new LoggerForNuget(_logProvider) }; packageManager.LocalRepository.PackageSaveMode = PackageSaveModes.Nuspec; packageManager.InstallPackage(package, ignoreDependencies: true, allowPrereleaseVersions: true); _performanceLogger.Write(sw, () => "PackageDownloader install NuGet package " + request.Id + "."); string targetFolder = packageManager.PathResolver.GetInstallPath(package); binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Plugins"), Paths.PluginsFolder, recursive: false); binFileSyncer.AddFolderContent(Path.Combine(targetFolder, "Resources"), Paths.ResourcesFolder, SimplifyPackageName(package.Id), recursive: true); return(new InstalledPackage(package.Id, package.Version.ToString(), GetNuGetPackageDependencies(package), targetFolder, request, source.ProcessedLocation, GetNuGetRequiredRhetosVersion(package))); }