public PackageMetadata DiscoverPackageMetadata(string discoveryRoot) { var packageMetadata = new PackageMetadata(); foreach (var heuristic in _discoveryHeuristics) { heuristic.DiscoverMetadataProperties(packageMetadata, discoveryRoot); } return packageMetadata; }
public virtual Package EnrichPackageFromNuGetPackage( Package package, PackageArchiveReader packageArchive, PackageMetadata packageMetadata, PackageStreamMetadata packageStreamMetadata, User user) { // Version must always be the exact string from the nuspec, which ToString will return to us. // However, we do also store a normalized copy for looking up later. package.Version = packageMetadata.Version.ToString(); package.NormalizedVersion = packageMetadata.Version.ToNormalizedString(); package.Description = packageMetadata.Description; package.ReleaseNotes = packageMetadata.ReleaseNotes; package.HashAlgorithm = packageStreamMetadata.HashAlgorithm; package.Hash = packageStreamMetadata.Hash; package.PackageFileSize = packageStreamMetadata.Size; package.Language = packageMetadata.Language; package.Copyright = packageMetadata.Copyright; package.FlattenedAuthors = packageMetadata.Authors.Flatten(); package.IsPrerelease = packageMetadata.Version.IsPrerelease; package.Listed = true; package.RequiresLicenseAcceptance = packageMetadata.RequireLicenseAcceptance; package.Summary = packageMetadata.Summary; package.Tags = PackageHelper.ParseTags(packageMetadata.Tags); package.Title = packageMetadata.Title; package.User = user; package.IconUrl = packageMetadata.IconUrl.ToEncodedUrlStringOrNull(); package.LicenseUrl = packageMetadata.LicenseUrl.ToEncodedUrlStringOrNull(); package.ProjectUrl = packageMetadata.ProjectUrl.ToEncodedUrlStringOrNull(); package.MinClientVersion = packageMetadata.MinClientVersion.ToStringOrNull(); #pragma warning disable 618 // TODO: remove Package.Authors completely once production services definitely no longer need it foreach (var author in packageMetadata.Authors) { package.Authors.Add(new PackageAuthor { Name = author }); } #pragma warning restore 618 var supportedFrameworks = GetSupportedFrameworks(packageArchive) .ToArray(); if (!supportedFrameworks.Any(fx => fx != null && fx.IsAny)) { var supportedFrameworkNames = supportedFrameworks .Select(fn => fn.ToShortNameOrNull()) .Where(fn => fn != null) .ToArray(); ValidateSupportedFrameworks(supportedFrameworkNames); foreach (var supportedFramework in supportedFrameworkNames) { package.SupportedFrameworks.Add(new PackageFramework { TargetFramework = supportedFramework }); } } package.Dependencies = packageMetadata .GetDependencyGroups() .AsPackageDependencyEnumerable() .ToList(); package.PackageTypes = packageMetadata .GetPackageTypes() .AsPackageTypeEnumerable() .ToList(); package.FlattenedDependencies = package.Dependencies.Flatten(); package.FlattenedPackageTypes = package.PackageTypes.Flatten(); return(package); }
public static void ValidateNuGetPackageMetadata(PackageMetadata packageMetadata) { // TODO: Change this to use DataAnnotations if (packageMetadata.Id.Length > NuGet.Services.Entities.Constants.MaxPackageIdLength) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "ID", NuGet.Services.Entities.Constants.MaxPackageIdLength); } if (packageMetadata.Authors != null && packageMetadata.Authors.Flatten().Length > 4000) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "Authors", "4000"); } if (packageMetadata.Copyright != null && packageMetadata.Copyright.Length > 4000) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "Copyright", "4000"); } if (packageMetadata.Description == null) { throw new EntityException(ServicesStrings.NuGetPackagePropertyMissing, "Description"); } else if (packageMetadata.Description != null && packageMetadata.Description.Length > 4000) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "Description", "4000"); } if (packageMetadata.IconUrl != null && packageMetadata.IconUrl.AbsoluteUri.Length > 4000) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "IconUrl", "4000"); } if (packageMetadata.LicenseUrl != null && packageMetadata.LicenseUrl.AbsoluteUri.Length > 4000) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "LicenseUrl", "4000"); } if (packageMetadata.ProjectUrl != null && packageMetadata.ProjectUrl.AbsoluteUri.Length > 4000) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "ProjectUrl", "4000"); } if (packageMetadata.Summary != null && packageMetadata.Summary.Length > 4000) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "Summary", "4000"); } if (packageMetadata.ReleaseNotes != null && packageMetadata.ReleaseNotes.Length > 35000) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "ReleaseNotes", "35000"); } if (packageMetadata.Tags != null && packageMetadata.Tags.Length > 4000) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "Tags", "4000"); } if (packageMetadata.Title != null && packageMetadata.Title.Length > 256) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "Title", "256"); } if (packageMetadata.Version != null && packageMetadata.Version.ToFullString().Length > 64) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "Version", "64"); } if (packageMetadata.Language != null && packageMetadata.Language.Length > 20) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "Language", "20"); } if (packageMetadata.Id.Length + (packageMetadata.Version?.ToFullString().Length ?? 0) > 160) { throw new EntityException(ServicesStrings.NuGetPackageIdVersionCombinedTooLong, "160"); } // Validate dependencies if (packageMetadata.GetDependencyGroups() != null) { var packageDependencies = packageMetadata.GetDependencyGroups().ToList(); foreach (var dependency in packageDependencies.SelectMany(s => s.Packages)) { // NuGet.Core compatibility - dependency package id can not be > 128 characters if (dependency.Id != null && dependency.Id.Length > NuGet.Services.Entities.Constants.MaxPackageIdLength) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "Dependency.Id", NuGet.Services.Entities.Constants.MaxPackageIdLength); } // NuGet.Core compatibility - dependency versionspec can not be > 256 characters if (dependency.VersionRange != null && dependency.VersionRange.ToString().Length > 256) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "Dependency.VersionSpec", "256"); } } // NuGet.Core compatibility - flattened dependencies should be <= Int16.MaxValue if (packageDependencies.Flatten().Length > Int16.MaxValue) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "Dependencies", Int16.MaxValue); } } // Validate repository metadata if (packageMetadata.RepositoryType != null && packageMetadata.RepositoryType.Length > 100) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "RepositoryType", "100"); } if (packageMetadata.RepositoryUrl != null && packageMetadata.RepositoryUrl.AbsoluteUri.Length > 4000) { throw new EntityException(ServicesStrings.NuGetPackagePropertyTooLong, "RepositoryUrl", "4000"); } }
private static void ValidateSymbolPackage(PackageArchiveReader symbolPackage, PackageMetadata metadata) { PackageHelper.ValidateNuGetPackageMetadata(metadata); // Validate nuspec manifest. var errors = ManifestValidator.Validate(symbolPackage.GetNuspec(), out var nuspec).ToArray(); if (errors.Length > 0) { var errorsString = string.Join("', '", errors.Select(error => error.ErrorMessage)); throw new InvalidDataException(string.Format( CultureInfo.CurrentCulture, errors.Length > 1 ? Strings.UploadPackage_InvalidNuspecMultiple : Strings.UploadPackage_InvalidNuspec, errorsString)); } // Validate that the PII is not embedded in nuspec var invalidItems = new List <string>(); if (metadata.Authors != null && (metadata.Authors.Count > 1 || !string.IsNullOrWhiteSpace(metadata.Authors.FirstOrDefault()))) { invalidItems.Add("Authors"); } if (metadata.Owners != null && metadata.Owners.Any()) { invalidItems.Add("Owners"); } if (invalidItems.Any()) { throw new InvalidDataException(string.Format(Strings.SymbolsPackage_InvalidDataInNuspec, string.Join(",", invalidItems.ToArray()))); } if (!CheckForAllowedFiles(symbolPackage)) { throw new InvalidDataException(string.Format(Strings.SymbolsPackage_InvalidFiles, PDBExtension)); } }
protected override string GetTargetPath(RunningDeployment deployment, PackageMetadata metadata) { return(Environment.CurrentDirectory); }
public async Task Collect(SqlConnection connection, Uri serviceDiscoveryUri, DateTime?lastCreateTime, string fileName) { using (var context = new EntitiesContext(connection, readOnly: true)) using (var cursor = new FileCursor(CursorFileName)) using (var logger = new Logger(ErrorsFileName)) { context.SetCommandTimeout(300); // large query var startTime = await cursor.Read(); logger.Log($"Starting metadata collection - Cursor time: {startTime:u}"); var repository = new EntityRepository <Package>(context); var packages = repository.GetAll().Include(p => p.PackageRegistration); if (QueryIncludes != null) { packages = packages.Include(QueryIncludes); } packages = packages .Where(p => p.Created <lastCreateTime && p.Created> startTime) .Where(p => p.PackageStatusKey == PackageStatus.Available) .OrderBy(p => p.Created); if (LimitTo > 0) { packages = packages.Take(LimitTo); } var flatContainerUri = await GetFlatContainerUri(serviceDiscoveryUri); using (var csv = CreateCsvWriter(fileName)) using (var http = new HttpClient()) { // We want these downloads ignored by stats pipelines - this user agent is automatically skipped. // See https://github.com/NuGet/NuGet.Jobs/blob/262da48ed05d0366613bbf1c54f47879aad96dcd/src/Stats.ImportAzureCdnStatistics/StatisticsParser.cs#L41 http.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights) Backfill Job: NuGet.Gallery GalleryTools"); var counter = 0; var lastCreatedDate = default(DateTime?); foreach (var package in packages) { var id = package.PackageRegistration.Id; var version = package.NormalizedVersion; var idLowered = id.ToLowerInvariant(); var versionLowered = version.ToLowerInvariant(); try { var metadata = default(TMetadata); var nuspecUri = $"{flatContainerUri}/{idLowered}/{versionLowered}/{idLowered}.nuspec"; using (var nuspecStream = await http.GetStreamAsync(nuspecUri)) { var document = LoadDocument(nuspecStream); var nuspecReader = new NuspecReader(document); if (SourceType == MetadataSourceType.NuspecOnly) { metadata = ReadMetadata(nuspecReader); } else if (SourceType == MetadataSourceType.Nupkg) { var nupkgUri = $"{flatContainerUri}/{idLowered}/{versionLowered}/{idLowered}.{versionLowered}.nupkg"; metadata = await FetchMetadataAsync(http, nupkgUri, nuspecReader, id, version, logger); } } if (ShouldWriteMetadata(metadata)) { var record = new PackageMetadata(id, version, metadata, package.Created); csv.WriteRecord(record); await csv.NextRecordAsync(); logger.LogPackage(id, version, $"Metadata saved"); } } catch (Exception e) { await logger.LogPackageError(id, version, e); } counter++; if (!lastCreatedDate.HasValue || lastCreatedDate < package.Created) { lastCreatedDate = package.Created; } if (counter >= CollectBatchSize) { logger.Log($"Writing {package.Created:u} to cursor..."); await cursor.Write(package.Created); counter = 0; } } if (counter > 0 && lastCreatedDate.HasValue) { await cursor.Write(lastCreatedDate.Value); } } } }
private void EnsureMainAssemblyLoadedInAppDomain(IPackagePathResolver pathResolver, IPackage package, PackageMetadata packageInfo, Guid correlationId) { var mainAssemblyFileName = $"{PackageMainAssemblySubFolderPath}{packageInfo.PackageKey}{DefaultAssemblyExtensionLowerCase}"; var mainAssemblyPackageFile = package .GetLibFiles() .FirstOrDefault(libFile => libFile.Path.Equals(mainAssemblyFileName, StringComparison.InvariantCultureIgnoreCase)); if (mainAssemblyPackageFile == null) { this.LogPackageOperation(packageInfo.PackageKey, package.Version, $"No assembly {mainAssemblyFileName} found in {package.GetFullName()}", correlationId); return; } var mainAssemblyFullPath = Path.Combine(pathResolver.GetInstallPath(package), mainAssemblyPackageFile.Path); this.LogPackageOperation(packageInfo.PackageKey, package.Version, $"AppDomain.LoadAssembly:'{mainAssemblyFullPath}'", correlationId); AppDomain.CurrentDomain.Load(AssemblyName.GetAssemblyName(mainAssemblyFullPath)); }
public async Task <Package> ReflowAsync(string id, string version) { var package = _packageService.FindPackageByIdAndVersionStrict(id, version); if (package == null) { return(null); } EntitiesConfiguration.SuspendExecutionStrategy = true; using (var transaction = _entitiesContext.GetDatabase().BeginTransaction()) { // 1) Download package binary to memory using (var packageStream = await _packageFileService.DownloadPackageFileAsync(package)) { using (var packageArchive = new PackageArchiveReader(packageStream, leaveStreamOpen: false)) { // 2) Determine package metadata from binary var packageStreamMetadata = new PackageStreamMetadata { HashAlgorithm = CoreConstants.Sha512HashAlgorithmId, Hash = CryptographyService.GenerateHash( packageStream.AsSeekableStream(), CoreConstants.Sha512HashAlgorithmId), Size = packageStream.Length, }; var packageMetadata = PackageMetadata.FromNuspecReader( packageArchive.GetNuspecReader(), strict: false); // 3) Clear referenced objects that will be reflowed ClearSupportedFrameworks(package); ClearAuthors(package); ClearDependencies(package); // 4) Reflow the package var listed = package.Listed; package = _packageService.EnrichPackageFromNuGetPackage( package, packageArchive, packageMetadata, packageStreamMetadata, package.User); package.LastEdited = DateTime.UtcNow; package.Listed = listed; // 5) Update IsLatest so that reflow can correct concurrent updates (see Gallery #2514) await _packageService.UpdateIsLatestAsync(package.PackageRegistration, commitChanges : false); // 6) Emit telemetry. _telemetryService.TrackPackageReflow(package); // 7) Save and profit await _entitiesContext.SaveChangesAsync(); } } // Commit transaction transaction.Commit(); } EntitiesConfiguration.SuspendExecutionStrategy = false; return(package); }
public void DiscoverMetadataProperties(PackageMetadata discoveredMetadata, string discoveryRoot) { Executed = true; }
private PackageMetadata GetPackageMetadata(Mock <TestPackageReader> mockPackage) { return(PackageMetadata.FromNuspecReader(mockPackage.Object.GetNuspecReader(), strict: true)); }
public async Task <WebLibrary> DeployAsync(IPackage package, RevisionSource source, int publisherId) { #region Preconditions if (package == null) { throw new ArgumentNullException(nameof(package)); } #endregion var metadata = GetMetadata(package); var version = metadata.Version; // var existing = await registry.GetAsync(registry.Lookup(metadata.Name), metadata.Version); // if (existing != null) throw new PublishingConflict(existing); var mainPath = metadata.Main; var bowerFile = package.Find("bower.json"); if (bowerFile != null) { try { var bowerFileStream = await bowerFile.OpenAsync().ConfigureAwait(false); var bower = PackageMetadata.Parse(bowerFileStream); if (bower.Main != null) { mainPath = bower.Main; } } catch { } } if (mainPath == null) { throw new Exception("A main property found in package.json or bower.json."); } var mainFile = package.Find(mainPath); if (mainFile == null) { throw new Exception($"The main file '{mainPath}' was not found"); } var mainText = await mainFile.ReadAllTextAsync().ConfigureAwait(false); if (mainText.Length == 0) { throw new Exception($"{mainPath} is empty"); } var mainBlobStream = new MemoryStream(Encoding.UTF8.GetBytes(mainText)); var mainName = mainPath.Split('/').Last(); if (!mainName.EndsWith(".js")) { throw new Exception($"Must end with js. was {mainName}"); } var mainBlob = new Blob( key: $"libs/{metadata.Name}/{version}/{mainName}", stream: mainBlobStream, properties: new BlobProperties { ContentType = "application/javascript" } ); var mainHash = Hash.ComputeSHA256(mainBlobStream); // Push to CDN await bucket.PutAsync(mainBlob).ConfigureAwait(false); if (metadata.Files != null) { // TODO: Copy over everything from files[] (excluding main) foreach (var fileName in metadata.Files) { var fn = fileName; if (fn.StartsWith("./")) { fn = fileName.Substring(2); } if (fn == mainPath) { continue; } var asset = package.Find(fn); var format = asset.Key.Split('.').Last(); var n = asset.Key.Split('/').Last(); var ms = new MemoryStream(); using (var data = await asset.OpenAsync().ConfigureAwait(false)) { data.CopyTo(ms); ms.Position = 0; } var blob = new Blob( key: $"libs/{metadata.Name}/{version}/{n}", stream: ms, properties: new BlobProperties { ContentType = GetMime(format) } ); await bucket.PutAsync(blob).ConfigureAwait(false); } } var release = new WebLibrary(metadata.Name, version) { MainName = mainName, MainSha256 = mainHash, Source = source.ToString() }; return(release); }
private static string ClipboardText(PackageMetadata p) => $"<PackageReference Include=\"{p.Id}\" Version=\"{p.Version}\" />";
/// <summary> /// Resolve package patterns to package metadata. /// This method also adds package dependencies. /// </summary> public IEnumerable <PackageMetadata> ResolvePackages( VersionMetadata metadata, CachePlatform cachePlatform, IEnumerable <string> packages, IList <string> notFound = null ) { var packageMetadata = metadata.GetPackages(cachePlatform); var metas = new List <PackageMetadata>(); foreach (var pattern in packages) { var id = pattern; bool fuzzy = false, addDependencies = true; while (id.StartsWith("~") || id.StartsWith("=")) { if (id.StartsWith("~")) { fuzzy = true; id = id.Substring(1); } else if (id.StartsWith("=")) { addDependencies = false; id = id.Substring(1); } } PackageMetadata resolved = default; if (fuzzy) { // Contains lookup foreach (var package in packageMetadata) { if (package.name.IndexOf(id, StringComparison.OrdinalIgnoreCase) >= 0) { if (resolved.name == null) { Logger.LogDebug($"Fuzzy lookup '{pattern}' matched package '{resolved.name}'"); resolved = package; } else { throw new Exception($"Fuzzy package match '{pattern}' is ambiguous between '{package.name}' and '{resolved.name}'"); } } } } else { // Exact lookup resolved = metadata.GetPackage(cachePlatform, id); } if (resolved.name != null) { AddPackageWithDependencies(packageMetadata, metas, resolved, addDependencies); } else if (notFound != null) { notFound.Add(id); } } return(metas); }
private static void ValidateNuGetPackageMetadata(PackageMetadata packageMetadata) { // TODO: Change this to use DataAnnotations if (packageMetadata.Id.Length > CoreConstants.MaxPackageIdLength) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Id", CoreConstants.MaxPackageIdLength); } if (packageMetadata.Authors != null && packageMetadata.Authors.Flatten().Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Authors", "4000"); } if (packageMetadata.Copyright != null && packageMetadata.Copyright.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Copyright", "4000"); } if (packageMetadata.Description == null) { throw new EntityException(Strings.NuGetPackagePropertyMissing, "Description"); } else if (packageMetadata.Description != null && packageMetadata.Description.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Description", "4000"); } if (packageMetadata.IconUrl != null && packageMetadata.IconUrl.AbsoluteUri.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "IconUrl", "4000"); } if (packageMetadata.LicenseUrl != null && packageMetadata.LicenseUrl.AbsoluteUri.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "LicenseUrl", "4000"); } if (packageMetadata.ProjectUrl != null && packageMetadata.ProjectUrl.AbsoluteUri.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "ProjectUrl", "4000"); } if (packageMetadata.Summary != null && packageMetadata.Summary.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Summary", "4000"); } // Disabled because PowershellGet package combines all of the exported functions into a single long string. // if (packageMetadata.Tags != null && packageMetadata.Tags.Length > 4000) // { // throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Tags", "4000"); // } if (packageMetadata.Title != null && packageMetadata.Title.Length > 256) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Title", "256"); } if (packageMetadata.Version != null && packageMetadata.Version.ToFullString().Length > 64) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Version", "64"); } if (packageMetadata.Language != null && packageMetadata.Language.Length > 20) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Language", "20"); } // Validate dependencies if (packageMetadata.GetDependencyGroups() != null) { var packageDependencies = packageMetadata.GetDependencyGroups().ToList(); foreach (var dependency in packageDependencies.SelectMany(s => s.Packages)) { // NuGet.Core compatibility - dependency package id can not be > 128 characters if (dependency.Id != null && dependency.Id.Length > CoreConstants.MaxPackageIdLength) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Dependency.Id", CoreConstants.MaxPackageIdLength); } // NuGet.Core compatibility - dependency versionspec can not be > 256 characters if (dependency.VersionRange != null && dependency.VersionRange.ToString().Length > 256) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Dependency.VersionSpec", "256"); } } // NuGet.Core compatibility - flattened dependencies should be < Int16.MaxValue if (packageDependencies.Flatten().Length > Int16.MaxValue) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Dependencies", Int16.MaxValue); } } }
private PackageRegistration CreateOrGetPackageRegistration(User owner, User currentUser, PackageMetadata packageMetadata, bool isVerified) { var packageRegistration = FindPackageRegistrationById(packageMetadata.Id); if (packageRegistration == null) { packageRegistration = new PackageRegistration { Id = packageMetadata.Id, IsVerified = isVerified }; packageRegistration.Owners.Add(owner); _packageRegistrationRepository.InsertOnCommit(packageRegistration); } return(packageRegistration); }
public static void InstallMod(CraftitudeProfile profile, PackageMetadata metadata, string jarFile) { File.Copy(jarFile, GenerateModPath(profile, metadata), true); }
protected override string GetTargetPath(RunningDeployment deployment, PackageMetadata metadata) { var root = GetInitialExtractionDirectory(deployment.Variables); return(EnsureTargetPathIsEmpty(Path.Combine(root, metadata.Id, metadata.Version))); }
protected abstract string GetTargetPath(RunningDeployment deployment, PackageMetadata metadata);
private static PackageUploadService CreateService( Mock <IPackageService> packageService = null, Mock <IReservedNamespaceService> reservedNamespaceService = null, Mock <IValidationService> validationService = null, Mock <IPackageVulnerabilitiesManagementService> vulnerabilityService = null) { packageService = packageService ?? new Mock <IPackageService>(); packageService.Setup(x => x.FindPackageRegistrationById(It.IsAny <string>())).Returns((PackageRegistration)null); packageService.Setup(x => x .CreatePackageAsync(It.IsAny <PackageArchiveReader>(), It.IsAny <PackageStreamMetadata>(), It.IsAny <User>(), It.IsAny <User>(), It.IsAny <bool>())) .Returns((PackageArchiveReader packageArchiveReader, PackageStreamMetadata packageStreamMetadata, User owner, User currentUser, bool isVerified) => { var packageMetadata = PackageMetadata.FromNuspecReader( packageArchiveReader.GetNuspecReader(), strict: true); var newPackage = new Package(); newPackage.PackageRegistration = new PackageRegistration { Id = packageMetadata.Id, IsVerified = isVerified }; newPackage.Version = packageMetadata.Version.ToString(); newPackage.SemVerLevelKey = SemVerLevelKey.ForPackage(packageMetadata.Version, packageMetadata.GetDependencyGroups().AsPackageDependencyEnumerable()); return(Task.FromResult(newPackage)); }); if (reservedNamespaceService == null) { reservedNamespaceService = new Mock <IReservedNamespaceService>(); reservedNamespaceService .Setup(r => r.GetReservedNamespacesForId(It.IsAny <string>())) .Returns(new ReservedNamespace[0]); } if (vulnerabilityService == null) { vulnerabilityService = new Mock <IPackageVulnerabilitiesManagementService>(); } validationService = validationService ?? new Mock <IValidationService>(); var diagnosticsService = new Mock <IDiagnosticsService>(); diagnosticsService .Setup(ds => ds.GetSource(It.IsAny <string>())) .Returns(Mock.Of <IDiagnosticsSource>()); var metadataValidationService = new Mock <IPackageMetadataValidationService>(); var packageUploadService = new Mock <PackageUploadService>( packageService.Object, new Mock <IPackageFileService>().Object, new Mock <IEntitiesContext>().Object, reservedNamespaceService.Object, validationService.Object, Mock.Of <ICoreLicenseFileService>(), diagnosticsService.Object, vulnerabilityService.Object, metadataValidationService.Object); return(packageUploadService.Object); }
private Package CreatePackageFromNuGetPackage(PackageRegistration packageRegistration, PackageArchiveReader nugetPackage, PackageMetadata packageMetadata, PackageStreamMetadata packageStreamMetadata, User user) { var package = packageRegistration.Packages.SingleOrDefault(pv => pv.Version == packageMetadata.Version.OriginalVersion); if (package != null) { throw new PackageAlreadyExistsException( string.Format(ServicesStrings.PackageExistsAndCannotBeModified, packageRegistration.Id, package.Version)); } package = new Package(); package.PackageRegistration = packageRegistration; package = EnrichPackageFromNuGetPackage(package, nugetPackage, packageMetadata, packageStreamMetadata, user); return(package); }
/// <summary> /// Returns the directory where the package will be installed. /// Also ensures the directory exists, and that there is free-space on the disk. /// </summary> public static string GetApplicationDirectory(PackageMetadata package, VariableDictionary variables, ICalamariFileSystem fileSystem) { return(EnsureTargetPathExistsAndIsEmpty( Path.Combine(GetEnvironmentApplicationDirectory(fileSystem, variables), package.PackageId, package.Version), fileSystem)); }
private Package CreatePackageFromNuGetPackage(PackageRegistration packageRegistration, PackageArchiveReader nugetPackage, PackageMetadata packageMetadata, PackageStreamMetadata packageStreamMetadata, User user) { var package = packageRegistration.Packages.SingleOrDefault(pv => pv.Version == packageMetadata.Version.ToString()); if (package != null) { throw new EntityException( "A package with identifier '{0}' and version '{1}' already exists.", packageRegistration.Id, package.Version); } package = new Package { // Version must always be the exact string from the nuspec, which ToString will return to us. // However, we do also store a normalized copy for looking up later. Version = packageMetadata.Version.ToString(), NormalizedVersion = packageMetadata.Version.ToNormalizedString(), Description = packageMetadata.Description, ReleaseNotes = packageMetadata.ReleaseNotes, HashAlgorithm = packageStreamMetadata.HashAlgorithm, Hash = packageStreamMetadata.Hash, PackageFileSize = packageStreamMetadata.Size, Language = packageMetadata.Language, Copyright = packageMetadata.Copyright, FlattenedAuthors = packageMetadata.Authors.Flatten(), IsPrerelease = packageMetadata.Version.IsPrerelease, Listed = true, PackageRegistration = packageRegistration, RequiresLicenseAcceptance = packageMetadata.RequireLicenseAcceptance, Summary = packageMetadata.Summary, Tags = PackageHelper.ParseTags(packageMetadata.Tags), Title = packageMetadata.Title, User = user, }; package.IconUrl = packageMetadata.IconUrl.ToEncodedUrlStringOrNull(); package.LicenseUrl = packageMetadata.LicenseUrl.ToEncodedUrlStringOrNull(); package.ProjectUrl = packageMetadata.ProjectUrl.ToEncodedUrlStringOrNull(); package.MinClientVersion = packageMetadata.MinClientVersion.ToStringOrNull(); #pragma warning disable 618 // TODO: remove Package.Authors completely once prodution services definitely no longer need it foreach (var author in packageMetadata.Authors) { package.Authors.Add(new PackageAuthor { Name = author }); } #pragma warning restore 618 var supportedFrameworks = GetSupportedFrameworks(nugetPackage).Select(fn => fn.ToShortNameOrNull()).ToArray(); if (!supportedFrameworks.AnySafe(sf => sf == null)) { ValidateSupportedFrameworks(supportedFrameworks); foreach (var supportedFramework in supportedFrameworks) { package.SupportedFrameworks.Add(new PackageFramework { TargetFramework = supportedFramework }); } } package.Dependencies = packageMetadata .GetDependencyGroups() .AsPackageDependencyEnumerable() .ToList(); package.FlattenedDependencies = package.Dependencies.Flatten(); return(package); }
public AnyAuthenticatedModule() { this.RequiresAuthentication(); this.RequiresAnyClaim( k => k.Type == AccessLevel.Admin.ToString(), k => k.Type == AccessLevel.Advocate.ToString(), k => k.Type == AccessLevel.Developer.ToString(), k => k.Type == AccessLevel.Envoy.ToString(), k => k.Type == AccessLevel.Player.ToString(), k => k.Type == AccessLevel.Sentinel.ToString()); Get("/api/character/backup", async(_) => { CharacterBackupRequestModel request = this.BindAndValidate <CharacterBackupRequestModel>(); if (!ModelValidationResult.IsValid) { return(Negotiate.WithModel(ModelValidationResult).WithStatusCode(HttpStatusCode.BadRequest)); } PackageMetadata metadata = new PackageMetadata { CharacterId = request.CharacterId, AccountId = uint.Parse(Context.CurrentUser.FindFirst("AccountId").Value), PackageType = PackageType.Backup }; Gate.RunGatedAction(() => { metadata = TransferManager.CreatePackage(metadata).Result; }); if (metadata == null) { return(new CharacterBackupResponseModel { Success = false, CharacterId = request.CharacterId }.AsJsonWebResponse()); } if (!File.Exists(metadata.FilePath)) { return(new CharacterBackupResponseModel { Success = false, CharacterId = request.CharacterId }.AsJsonWebResponse()); } CharacterBackupResponseModel resp = new CharacterBackupResponseModel { SnapshotPackage = File.ReadAllBytes(metadata.FilePath), Success = true, CharacterId = request.CharacterId }; File.Delete(metadata.FilePath); return(resp.AsJsonWebResponse()); }); Get("/api/character/migrationBegin", async(_) => { CharacterMigrationBeginRequestModel request = this.BindAndValidate <CharacterMigrationBeginRequestModel>(); if (!ModelValidationResult.IsValid) { return(Negotiate.WithModel(ModelValidationResult).WithStatusCode(HttpStatusCode.BadRequest)); } PackageMetadata metadata = new PackageMetadata { CharacterId = request.CharacterId, AccountId = uint.Parse(Context.CurrentUser.FindFirst("AccountId").Value), PackageType = PackageType.Migrate }; Gate.RunGatedAction(() => { metadata = TransferManager.CreatePackage(metadata).Result; }); if (metadata == null) { return(new CharacterMigrationBeginResponseModel() { CharacterId = request.CharacterId, Success = false }.AsJsonWebResponse()); } if (!File.Exists(metadata.FilePath)) { return(new CharacterMigrationBeginResponseModel() { CharacterId = request.CharacterId, Success = false }.AsJsonWebResponse()); } CharacterMigrationBeginResponseModel resp = new CharacterMigrationBeginResponseModel { BaseURL = $"https://{WebAPIConfigManager.Config.ExternalIPAddressOrDNSName}:{WebAPIConfigManager.Config.ExternalPort}", Cookie = metadata.Cookie, Success = true, CharacterId = request.CharacterId }; return(resp.AsJsonWebResponse()); }); Get("/api/character/migrationCancel", async(_) => { CharacterMigrationCancelRequestModel request = this.BindAndValidate <CharacterMigrationCancelRequestModel>(); if (!ModelValidationResult.IsValid) { return(Negotiate.WithModel(ModelValidationResult).WithStatusCode(HttpStatusCode.BadRequest)); } PackageMetadata metadata = new PackageMetadata { Cookie = request.Cookie, AccountId = uint.Parse(Context.CurrentUser.FindFirst("AccountId").Value) }; MigrateCloseResult result = null; Gate.RunGatedAction(() => { result = TransferManager.CloseMigration(metadata, MigrationCloseType.Cancel); }); CharacterMigrationCancelResponseModel resp = new CharacterMigrationCancelResponseModel() { Cookie = request.Cookie, Success = result.Success }; return(resp.AsJsonWebResponse()); }); Get("/api/character/migrationComplete", async(_) => { CharacterMigrationCompleteRequestModel request = this.BindAndValidate <CharacterMigrationCompleteRequestModel>(); if (!ModelValidationResult.IsValid) { return(Negotiate.WithModel(ModelValidationResult).WithStatusCode(HttpStatusCode.BadRequest)); } PackageMetadata metadata = new PackageMetadata { NewCharacterName = request.NewCharacterName.Trim(), Cookie = request.Cookie, AccountId = uint.Parse(Context.CurrentUser.FindFirst("AccountId").Value), ImportUrl = new Uri(request.BaseURL), PackageType = PackageType.Migrate }; ImportAndMigrateResult result = null; Gate.RunGatedAction(() => { result = TransferManager.ImportAndMigrate(metadata); }); return(new CharacterMigrationCompleteResponseModel() { Cookie = request.Cookie, Success = result.Success, CharacterName = result.NewCharacterName, FailureReason = result.Success ? null : result.FailReason.ToString(), CharacterId = result.NewCharacterId }.AsJsonWebResponse()); }); Post("/api/character/import", async(_) => { CharacterImportRequestModel request = this.BindAndValidate <CharacterImportRequestModel>(); if (!ModelValidationResult.IsValid) { return(Negotiate.WithModel(ModelValidationResult).WithStatusCode(HttpStatusCode.BadRequest)); } PackageMetadata metadata = new PackageMetadata { NewCharacterName = request.NewCharacterName.Trim(), AccountId = uint.Parse(Context.CurrentUser.FindFirst("AccountId").Value), PackageType = PackageType.Backup }; ImportAndMigrateResult result = null; byte[] fileData = null; try { fileData = Convert.FromBase64String(request.SnapshotPackageBase64); } catch { return(new CharacterImportResponseModel() { Success = false, FailureReason = "SnapshotPackageBase64 is not valid Base64 encoded data." }.AsJsonWebResponse()); } Gate.RunGatedAction(() => { result = TransferManager.ImportAndMigrate(metadata, fileData); }); return(new CharacterImportResponseModel() { Success = result.Success, CharacterName = result.NewCharacterName, CharacterId = result.NewCharacterId, FailureReason = result.Success ? null : result.FailReason.ToString() }.AsJsonWebResponse()); }); }
static string GetNameOfPackage(PackageMetadata metadata) { return(metadata.Id + "." + metadata.Version); }
public async Task Collect(string connectionString, Uri serviceDiscoveryUri, DateTime?lastCreateTime, string fileName) { using (var context = new EntitiesContext(connectionString, readOnly: true)) using (var cursor = new FileCursor(CursorFileName)) using (var logger = new Logger(ErrorsFileName)) { var startTime = await cursor.Read(); logger.Log($"Starting metadata collection - Cursor time: {startTime:u}"); var repository = new EntityRepository <Package>(context); var packages = repository.GetAll() .Include(p => p.PackageRegistration) .Where(p => p.Created <lastCreateTime && p.Created> startTime) .Where(p => p.PackageStatusKey == PackageStatus.Available || p.PackageStatusKey == PackageStatus.Validating) .OrderBy(p => p.Created); var flatContainerUri = await GetFlatContainerUri(serviceDiscoveryUri); using (var csv = CreateCsvWriter(fileName)) using (var http = new HttpClient()) { var counter = 0; var lastCreatedDate = default(DateTime?); foreach (var package in packages) { var id = package.PackageRegistration.Id; var version = package.NormalizedVersion; var nuspecUri = $"{flatContainerUri}/{id.ToLowerInvariant()}/{version.ToLowerInvariant()}/{id.ToLowerInvariant()}.nuspec"; try { var metadata = await FetchMetadata(http, nuspecUri); if (ShouldWriteMetadata(metadata)) { var record = new PackageMetadata(id, version, metadata, package.Created); csv.WriteRecord(record); await csv.NextRecordAsync(); logger.LogPackage(id, version, "Metadata saved."); } } catch (Exception e) { await logger.LogPackageError(id, version, e); } counter++; if (!lastCreatedDate.HasValue || lastCreatedDate < package.Created) { lastCreatedDate = package.Created; } if (counter >= CollectBatchSize) { logger.Log($"Writing {package.Created:u} to cursor..."); await cursor.Write(package.Created); counter = 0; } } if (counter > 0 && lastCreatedDate.HasValue) { await cursor.Write(lastCreatedDate.Value); } } } }
private PackageRegistration CreateOrGetPackageRegistration(User owner, User currentUser, PackageMetadata packageMetadata, bool isVerified) { var packageRegistration = FindPackageRegistrationById(packageMetadata.Id); if (packageRegistration == null) { if (_packageNamingConflictValidator.IdConflictsWithExistingPackageTitle(packageMetadata.Id)) { throw new EntityException(Strings.NewRegistrationIdMatchesExistingPackageTitle, packageMetadata.Id); } packageRegistration = new PackageRegistration { Id = packageMetadata.Id, IsVerified = isVerified }; packageRegistration.Owners.Add(owner); _packageRegistrationRepository.InsertOnCommit(packageRegistration); } return(packageRegistration); }
public async Task <PackageValidationResult> ValidateBeforeGeneratePackageAsync( PackageArchiveReader nuGetPackage, PackageMetadata packageMetadata, User currentUser) { var warnings = new List <IValidationMessage>(); var result = await CheckPackageEntryCountAsync(nuGetPackage, warnings); if (result != null) { return(result); } result = CheckPackageDuplicatedEntries(nuGetPackage); if (result != null) { return(result); } var nuspecFileEntry = nuGetPackage.GetEntry(nuGetPackage.GetNuspecFile()); using (var nuspecFileStream = await nuGetPackage.GetNuspecAsync(CancellationToken.None)) { if (!await IsStreamLengthMatchesReportedAsync(nuspecFileStream, nuspecFileEntry.Length)) { return(PackageValidationResult.Invalid(Strings.UploadPackage_CorruptNupkg)); } } result = await CheckForUnsignedPushAfterAuthorSignedAsync( nuGetPackage, warnings); if (result != null) { return(result); } result = CheckRepositoryMetadata(packageMetadata, warnings); if (result != null) { return(result); } result = await CheckLicenseMetadataAsync(nuGetPackage, warnings, currentUser); if (result != null) { _telemetryService.TrackLicenseValidationFailure(); return(result); } result = await CheckIconMetadataAsync(nuGetPackage, warnings, currentUser); if (result != null) { //_telemetryService.TrackIconValidationFailure(); return(result); } return(PackageValidationResult.AcceptedWithWarnings(warnings)); }
private Package CreatePackageFromNuGetPackage(PackageRegistration packageRegistration, PackageArchiveReader nugetPackage, PackageMetadata packageMetadata, PackageStreamMetadata packageStreamMetadata, User user) { var package = packageRegistration.Packages.SingleOrDefault(pv => pv.Version == packageMetadata.Version.ToString()); if (package != null) { throw new EntityException( "A package with identifier '{0}' and version '{1}' already exists.", packageRegistration.Id, package.Version); } package = new Package(); package.PackageRegistration = packageRegistration; package = EnrichPackageFromNuGetPackage(package, nugetPackage, packageMetadata, packageStreamMetadata, user); return(package); }
/// <summary> /// Resolve package patterns to package metadata. /// This method also adds package dependencies. /// </summary> public IEnumerable <PackageMetadata> ResolvePackages( VersionMetadata metadata, CachePlatform cachePlatform, IEnumerable <string> packages, IList <string> notFound = null, bool addDependencies = true ) { var platform = metadata.GetPlatform(cachePlatform); var metas = new List <PackageMetadata>(); foreach (var id in packages) { PackageMetadata resolved = default; if (id.StartsWith('~')) { // Contains lookup var str = id.Substring(1); foreach (var package in platform.packages) { if (package.name.Contains(str, StringComparison.OrdinalIgnoreCase)) { if (resolved.name == null) { Logger.LogDebug($"Fuzzy lookup '{id}' matched package '{resolved.name}'"); resolved = package; } else { throw new Exception($"Fuzzy package match '{id}' is ambiguous between '{package.name}' and '{resolved.name}'"); } } } } else { // Exact lookup resolved = platform.GetPackage(id); } if (resolved.name != null) { if (addDependencies) { foreach (var package in platform.packages) { if (package.sync == resolved.name) { Logger.LogInformation($"Adding '{package.name}' which '{resolved.name}' is synced with"); metas.Add(package); } } } metas.Add(resolved); } else if (notFound != null) { notFound.Add(id); } } return(metas); }
private static void ValidateNuGetPackageMetadata(PackageMetadata packageMetadata) { // TODO: Change this to use DataAnnotations if (packageMetadata.Id.Length > CoreConstants.MaxPackageIdLength) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Id", CoreConstants.MaxPackageIdLength); } if (packageMetadata.Version.IsPrerelease) { var release = packageMetadata.Version.Release; if (release.Contains(".")) { throw new EntityException(Strings.NuGetPackageReleaseVersionWithDot, "Version"); } long temp; if (long.TryParse(release, out temp)) { throw new EntityException(Strings.NuGetPackageReleaseVersionContainsOnlyNumerics, "Version"); } } if (packageMetadata.Authors != null && packageMetadata.Authors.Flatten().Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Authors", "4000"); } if (packageMetadata.Copyright != null && packageMetadata.Copyright.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Copyright", "4000"); } if (packageMetadata.Description != null && packageMetadata.Description.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Description", "4000"); } if (packageMetadata.IconUrl != null && packageMetadata.IconUrl.AbsoluteUri.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "IconUrl", "4000"); } if (packageMetadata.LicenseUrl != null && packageMetadata.LicenseUrl.AbsoluteUri.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "LicenseUrl", "4000"); } if (packageMetadata.ProjectUrl != null && packageMetadata.ProjectUrl.AbsoluteUri.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "ProjectUrl", "4000"); } if (packageMetadata.Summary != null && packageMetadata.Summary.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Summary", "4000"); } if (packageMetadata.Tags != null && packageMetadata.Tags.Length > 4000) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Tags", "4000"); } if (packageMetadata.Title != null && packageMetadata.Title.Length > 256) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Title", "256"); } if (packageMetadata.Version != null && packageMetadata.Version.ToString().Length > 64) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Version", "64"); } if (packageMetadata.Language != null && packageMetadata.Language.Length > 20) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Language", "20"); } // Validate dependencies if (packageMetadata.GetDependencyGroups() != null) { var packageDependencies = packageMetadata.GetDependencyGroups().ToList(); foreach (var dependency in packageDependencies.SelectMany(s => s.Packages)) { // NuGet.Core compatibility - dependency package id can not be > 128 characters if (dependency.Id != null && dependency.Id.Length > CoreConstants.MaxPackageIdLength) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Dependency.Id", CoreConstants.MaxPackageIdLength); } // NuGet.Core compatibility - dependency versionspec can not be > 256 characters if (dependency.VersionRange != null && dependency.VersionRange.ToString().Length > 256) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Dependency.VersionSpec", "256"); } } // NuGet.Core compatibility - flattened dependencies should be < Int16.MaxValue if (packageDependencies.Flatten().Length > Int16.MaxValue) { throw new EntityException(Strings.NuGetPackagePropertyTooLong, "Dependencies", Int16.MaxValue); } } }
public void CancelEdit() { PackageMetadata.ResetErrors(); IsInEditMetadataMode = false; }
public static string GenerateModPath(CraftitudeProfile profile, PackageMetadata metadata) { return Path.Combine(profile.Directory.CreateSubdirectory("mods").FullName + Path.DirectorySeparatorChar, CoerceValidFileName((metadata.Id + "-" + metadata.Version + ".jar").Replace(" ", ""))); }
public async Task <PackageValidationResult> ValidateBeforeGeneratePackageAsync(PackageArchiveReader nuGetPackage, PackageMetadata packageMetadata) { var warnings = new List <IValidationMessage>(); var result = await CheckPackageEntryCountAsync(nuGetPackage, warnings); if (result != null) { return(result); } result = await CheckForUnsignedPushAfterAuthorSignedAsync( nuGetPackage, warnings); if (result != null) { return(result); } result = CheckRepositoryMetadata(packageMetadata, warnings); if (result != null) { return(result); } result = await CheckLicenseMetadataAsync(nuGetPackage, warnings); if (result != null) { _telemetryService.TrackLicenseValidationFailure(); return(result); } return(PackageValidationResult.AcceptedWithWarnings(warnings)); }
public static void UninstallMod(CraftitudeProfile profile, PackageMetadata metadata) { File.Delete(GenerateModPath(profile, metadata)); }
private async Task<bool> UpdateNeeded(CancellationToken token) { var agentServer = HostContext.GetService<IAgentServer>(); var packages = await agentServer.GetPackagesAsync(_packageType, _platform, 1, token); if (packages == null || packages.Count == 0) { Trace.Info($"There is no package for {_packageType} and {_platform}."); return false; } _latestPackage = packages.FirstOrDefault(); Trace.Info($"Latest version of '{_latestPackage.Type}' package availiable in server is {_latestPackage.Version}"); PackageVersion serverVersion = new PackageVersion(_latestPackage.Version); Trace.Info($"Current running agent version is {Constants.Agent.Version}"); PackageVersion agentVersion = new PackageVersion(Constants.Agent.Version); return serverVersion.CompareTo(agentVersion) > 0; }