public Task AddMvcPackageAsync(string directory, NpmPackageInfo npmPackage, string version = null, bool skipGulpCommand = false) { var packageJsonFilePath = Path.Combine(directory, "package.json"); if (!File.Exists(packageJsonFilePath) || File.ReadAllText(packageJsonFilePath).Contains($"\"{npmPackage.Name}\"")) { return(Task.CompletedTask); } Logger.LogInformation($"Installing '{npmPackage.Name}' package to the project '{packageJsonFilePath}'..."); if (version == null) { version = DetectAbpVersionOrNull(Path.Combine(directory, "package.json")); } var versionPostfix = version != null ? $"@{version}" : string.Empty; using (DirectoryHelper.ChangeCurrentDirectory(directory)) { Logger.LogInformation("yarn add " + npmPackage.Name + versionPostfix); CmdHelper.RunCmd("yarn add " + npmPackage.Name + versionPostfix); if (skipGulpCommand) { return(Task.CompletedTask); } Logger.LogInformation("gulp"); CmdHelper.RunCmd("gulp"); } return(Task.CompletedTask); }
/// <summary> /// Converts a NuGet package to a Unity package. /// </summary> private async Task ConvertNuGetPackageToUnity(PackageIdentity identity, NpmPackageInfo npmPackageInfo, NpmPackageVersion npmPackageVersion, IPackageSearchMetadata packageMeta) { var unityPackageFileName = GetUnityPackageFileName(identity, npmPackageVersion); var unityPackageFilePath = Path.Combine(RootUnityPackageFolder, unityPackageFileName); Logger.LogInformation($"Converting NuGet package {identity} to Unity `{unityPackageFileName}`"); var downloadResource = await _sourceRepository.GetResourceAsync <DownloadResource>(CancellationToken.None); var downloadResult = await downloadResource.GetDownloadResourceResultAsync( identity, new PackageDownloadContext(_sourceCacheContext), SettingsUtility.GetGlobalPackagesFolder(_settings), Logger, CancellationToken.None); var packageReader = downloadResult.PackageReader; // Update Repository metadata if necessary var repoMeta = packageReader.NuspecReader.GetRepositoryMetadata(); if (repoMeta != null && repoMeta.Url != null && repoMeta.Commit != null && repoMeta.Type != null) { npmPackageVersion.Repository = new NpmSourceRepository() { Revision = repoMeta.Commit, Type = repoMeta.Type, Url = repoMeta.Url, }; } else { npmPackageVersion.Repository = null; } try { var memStream = new MemoryStream(); using (var outStream = File.Create(unityPackageFilePath)) using (var gzoStream = new GZipOutputStream(outStream)) using (var tarArchive = new TarOutputStream(gzoStream, Encoding.UTF8)) { // Select the netstandard version that is the closest or equal to netstandard2.0 var versions = (await packageReader.GetLibItemsAsync(CancellationToken.None)).ToList(); var item = versions.Where(x => x.TargetFramework.Framework == ".NETStandard" && x.TargetFramework.Version <= Version200).OrderByDescending(x => x.TargetFramework.Version) .FirstOrDefault(); if (item == null) { throw new InvalidOperationException("The package does not contain a netstandard2.0 compatible assembly"); } foreach (var file in item.Items) { var fileInUnityPackage = Path.GetFileName(file); var meta = UnityMeta.GetMetaForExtension(GetStableGuid(identity, fileInUnityPackage), Path.GetExtension(fileInUnityPackage)); if (meta == null) { continue; } memStream.Position = 0; memStream.SetLength(0); var stream = packageReader.GetStream(file); stream.CopyTo(memStream); var buffer = memStream.ToArray(); // write content WriteBufferToTar(tarArchive, fileInUnityPackage, buffer); // write meta file WriteTextFileToTar(tarArchive, fileInUnityPackage + ".meta", meta); } // Write the package,json var unityPackage = CreateUnityPackage(npmPackageInfo, npmPackageVersion); var unityPackageAsJson = unityPackage.ToJson(); const string packageJsonFileName = "package.json"; WriteTextFileToTar(tarArchive, packageJsonFileName, unityPackageAsJson); WriteTextFileToTar(tarArchive, $"{packageJsonFileName}.meta", UnityMeta.GetMetaForExtension(GetStableGuid(identity, packageJsonFileName), ".json")); // Write the license to the package if any string license = null; string licenseUrlText = null; var licenseUrl = packageMeta.LicenseMetadata?.LicenseUrl.ToString() ?? packageMeta.LicenseUrl?.ToString(); if (!string.IsNullOrEmpty(licenseUrl)) { try { // Try to fetch the license from an URL using (var httpClient = new HttpClient()) { licenseUrlText = await httpClient.GetStringAsync(licenseUrl); } // If the license text is HTML, try to remove all text if (licenseUrlText != null) { licenseUrlText = licenseUrlText.Trim(); if (licenseUrlText.StartsWith("<")) { try { licenseUrlText = NUglify.Uglify.HtmlToText(licenseUrlText, HtmlToTextOptions.KeepStructure).Code ?? licenseUrlText; } catch { // ignored } } } } catch { // ignored } } if (!string.IsNullOrEmpty(packageMeta.LicenseMetadata?.License)) { license = packageMeta.LicenseMetadata.License; } // If the license fetched from the URL is bigger, use that one to put into the file if (licenseUrlText != null && (license == null || licenseUrlText.Length > license.Length)) { license = licenseUrlText; } if (!string.IsNullOrEmpty(license)) { const string licenseMdFile = "License.md"; WriteTextFileToTar(tarArchive, licenseMdFile, license); WriteTextFileToTar(tarArchive, $"{licenseMdFile}.meta", UnityMeta.GetMetaForExtension(GetStableGuid(identity, licenseMdFile), ".md")); } } using (var stream = File.OpenRead(unityPackageFilePath)) { var sha1 = Sha1sum(stream); WriteUnityPackageSha1(identity, npmPackageVersion, sha1); npmPackageVersion.Distribution.Shasum = sha1; } } catch (Exception ex) { try { File.Delete(unityPackageFilePath); } catch { // ignored } LogError($"Error while processing package `{identity}`. Reason: {ex}"); } }
/// <summary> /// Converts a NuGet package to Unity package if not already /// </summary> private async Task ConvertNuGetToUnityPackageIfDoesNotExist(PackageIdentity identity, NpmPackageInfo npmPackageInfo, NpmPackageVersion npmPackageVersion, IPackageSearchMetadata packageMeta) { if (!IsUnityPackageValid(identity, npmPackageVersion) || !IsUnityPackageSha1Valid(identity, npmPackageVersion)) { await ConvertNuGetPackageToUnity(identity, npmPackageInfo, npmPackageVersion, packageMeta); } else { npmPackageVersion.Distribution.Shasum = ReadUnityPackageSha1(identity, npmPackageVersion); } }
/// <summary> /// For each package in our registry.json, query NuGet, extract package metadata, and convert them to unity packages. /// </summary> private async Task BuildInternal() { var packageMetadataResource = _sourceRepository.GetResource <PackageMetadataResource>(); foreach (var packageDesc in _registry) { var packageName = packageDesc.Key; var packageEntry = packageDesc.Value; // A package entry is ignored but allowed in the registry (case of Microsoft.CSharp) if (packageEntry.Ignored) { continue; } var packageMetaIt = await packageMetadataResource.GetMetadataAsync(packageName, false, false, _sourceCacheContext, Logger, CancellationToken.None); var packageMetas = packageMetaIt.ToList(); foreach (var packageMeta in packageMetas) { var packageIdentity = packageMeta.Identity; var packageId = packageIdentity.Id.ToLowerInvariant(); var npmPackageId = $"{UnityScope}.{packageId}"; if (!packageEntry.Version.Satisfies(packageMeta.Identity.Version)) { continue; } PackageDependencyGroup netstd20Dependency = null; foreach (var dependencySet in packageMeta.DependencySets) { if (dependencySet.TargetFramework == NuGetFrameworkNetStandard20) { netstd20Dependency = dependencySet; break; } } if (netstd20Dependency == null) { Logger.LogWarning($"The package `{packageIdentity}` doesn't support `netstandard2.0`"); continue; } if (!_npmPackageRegistry.Packages.TryGetValue(npmPackageId, out var npmPackage)) { npmPackage = new NpmPackage(); _npmPackageRegistry.Packages.Add(npmPackageId, npmPackage); } // One NpmPackage (for package request) var packageInfoList = packageEntry.Listed ? _npmPackageRegistry.ListedPackageInfos : _npmPackageRegistry.UnlistedPackageInfos; if (!packageInfoList.Packages.TryGetValue(npmPackageId, out var npmPackageInfo)) { npmPackageInfo = new NpmPackageInfo(); packageInfoList.Packages.Add(npmPackageId, npmPackageInfo); } // Update latest version var currentVersion = packageIdentity.Version; var update = !npmPackage.DistTags.TryGetValue("latest", out var latestVersion) || (currentVersion > NuGetVersion.Parse(latestVersion)); string npmCurrentVersion = $"{currentVersion.Major}.{currentVersion.Minor}.{currentVersion.Patch}"; if (currentVersion.Revision != 0) { npmCurrentVersion += $"-{currentVersion.Revision}"; } if (update) { npmPackage.DistTags["latest"] = npmCurrentVersion; npmPackageInfo.Versions.Clear(); npmPackageInfo.Versions[npmCurrentVersion] = "latest"; npmPackage.Id = npmPackageId; npmPackage.License = packageMeta.LicenseMetadata?.License ?? packageMeta.LicenseUrl?.ToString(); npmPackage.Name = npmPackageId; npmPackageInfo.Name = npmPackageId; npmPackage.Description = packageMeta.Description; npmPackageInfo.Description = packageMeta.Description; npmPackageInfo.Author = packageMeta.Authors; if (packageMeta.Owners != null) { npmPackageInfo.Maintainers.Clear(); npmPackageInfo.Maintainers.AddRange(SplitCommaSeparatedString(packageMeta.Owners)); } if (packageMeta.Tags != null) { npmPackageInfo.Keywords.Clear(); npmPackageInfo.Keywords.Add("nuget"); npmPackageInfo.Keywords.AddRange(SplitCommaSeparatedString(packageMeta.Tags)); } } var npmVersion = new NpmPackageVersion { Id = $"{npmPackageId}@{npmCurrentVersion}", Version = npmCurrentVersion, Name = npmPackageId, Description = packageMeta.Description, Author = AuthorNameUnityGroup, DisplayName = $"{packageMeta.Title} ({npmPackageInfo.Author})" }; npmVersion.Distribution.Tarball = new Uri($"{RootHttpUrl}/{npmPackage.Id}/-/{GetUnityPackageFileName(packageIdentity, npmVersion)}"); npmVersion.Unity = MinimumUnityVersion; npmPackage.Versions[npmVersion.Version] = npmVersion; bool hasDependencyErrors = false; foreach (var deps in netstd20Dependency.Packages) { var depsId = deps.Id.ToLowerInvariant(); if (!_registry.TryGetValue(deps.Id, out var packageEntryDep)) { LogError($"The package `{packageIdentity}` has a dependency on `{deps.Id}` which is not in the registry. You must add this dependency to the registry.json file."); hasDependencyErrors = true; } else if (packageEntryDep.Ignored) { // A package that is ignored is not declared as an explicit dependency continue; } else if (!deps.VersionRange.IsSubSetOrEqualTo(packageEntryDep.Version)) { LogError($"The version range `{deps.VersionRange}` for the dependency `{deps.Id}` for the package `{packageIdentity}` doesn't match the range allowed from the registry.json: `{packageEntryDep.Version}`"); hasDependencyErrors = true; } // Otherwise add the package as a dependency npmVersion.Dependencies.Add($"{UnityScope}.{depsId}", deps.VersionRange.MinVersion.ToString()); } // If we don't have any dependencies error, generate the package if (!hasDependencyErrors) { await ConvertNuGetToUnityPackageIfDoesNotExist(packageIdentity, npmPackageInfo, npmVersion, packageMeta); npmPackage.Time[npmCurrentVersion] = packageMeta.Published?.UtcDateTime ?? GetUnityPackageFileInfo(packageIdentity, npmVersion).CreationTimeUtc; // Copy repository info if necessary if (update) { npmPackage.Repository = npmVersion.Repository?.Clone(); } } } } }
/// <summary> /// Converts a NuGet package to a Unity package. /// </summary> private async Task ConvertNuGetPackageToUnity(PackageIdentity identity, NpmPackageInfo npmPackageInfo, NpmPackageVersion npmPackageVersion, IPackageSearchMetadata packageMeta, RegistryEntry packageEntry) { var unityPackageFileName = GetUnityPackageFileName(identity, npmPackageVersion); var unityPackageFilePath = Path.Combine(_rootPersistentFolder, unityPackageFileName); _logger.LogInformation($"Converting NuGet package {identity} to Unity `{unityPackageFileName}`"); var downloadResource = await _sourceRepository.GetResourceAsync <DownloadResource>(CancellationToken.None); var downloadResult = await downloadResource.GetDownloadResourceResultAsync( identity, new PackageDownloadContext(_sourceCacheContext), SettingsUtility.GetGlobalPackagesFolder(_settings), _logger, CancellationToken.None); var packageReader = downloadResult.PackageReader; // Update Repository metadata if necessary var repoMeta = packageReader.NuspecReader.GetRepositoryMetadata(); if (repoMeta != null && repoMeta.Url != null && repoMeta.Commit != null && repoMeta.Type != null) { npmPackageVersion.Repository = new NpmSourceRepository() { Revision = repoMeta.Commit, Type = repoMeta.Type, Url = repoMeta.Url, }; } else { npmPackageVersion.Repository = null; } try { var memStream = new MemoryStream(); using (var outStream = File.Create(unityPackageFilePath)) using (var gzoStream = new GZipOutputStream(outStream)) using (var tarArchive = new TarOutputStream(gzoStream, Encoding.UTF8)) { // Select the framework version that is the closest or equal to the latest configured framework version var versions = (await packageReader.GetLibItemsAsync(CancellationToken.None)).ToList(); var collectedItems = new Dictionary <FrameworkSpecificGroup, HashSet <RegistryTargetFramework> >(); foreach (var targetFramework in _targetFrameworks) { var item = versions.Where(x => x.TargetFramework.Framework == targetFramework.Framework.Framework && x.TargetFramework.Version <= targetFramework.Framework.Version).OrderByDescending(x => x.TargetFramework.Version) .FirstOrDefault(); if (item == null) { continue; } if (!collectedItems.TryGetValue(item, out var frameworksPerGroup)) { frameworksPerGroup = new HashSet <RegistryTargetFramework>(); collectedItems.Add(item, frameworksPerGroup); } frameworksPerGroup.Add(targetFramework); } if (!packageEntry.Analyzer && collectedItems.Count == 0) { throw new InvalidOperationException($"The package does not contain a compatible .NET assembly {string.Join(",", _targetFrameworks.Select(x => x.Name))}"); } var isPackageNetStandard21Assembly = DotNetHelper.IsNetStandard21Assembly(identity.Id); var hasMultiNetStandard = collectedItems.Count > 1; var hasOnlyNetStandard21 = collectedItems.Count == 1 && collectedItems.Values.First().All(x => x.Name == "netstandard2.1"); if (isPackageNetStandard21Assembly) { _logger.LogInformation($"Package {identity.Id} is a system package for netstandard2.1 and will be only used for netstandard 2.0"); } if (packageEntry.Analyzer) { var packageFiles = (await packageReader.GetPackageFilesAsync(PackageSaveMode.Files, CancellationToken.None)).ToList(); var analyzerFiles = packageFiles.Where(p => p.StartsWith("analyzers/dotnet/cs")).ToArray(); if (analyzerFiles.Length == 0) { analyzerFiles = packageFiles.Where(p => p.StartsWith("analyzers")).ToArray(); } var createdDirectoryList = new List <string>(); foreach (var analyzerFile in analyzerFiles) { var folderPrefix = $"{Path.GetDirectoryName(analyzerFile).Replace($"analyzers{Path.DirectorySeparatorChar}", string.Empty)}{Path.DirectorySeparatorChar}"; // Write folder meta if (!string.IsNullOrEmpty(folderPrefix)) { var directoryNameBuilder = new StringBuilder(); foreach (var directoryName in folderPrefix.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries)) { directoryNameBuilder.Append(directoryName); directoryNameBuilder.Append(Path.DirectorySeparatorChar); var processedDirectoryName = directoryNameBuilder.ToString()[0..^ 1];
/// <summary> /// Converts a NuGet package to Unity package if not already /// </summary> private async Task ConvertNuGetToUnityPackageIfDoesNotExist(PackageIdentity identity, NpmPackageInfo npmPackageInfo, NpmPackageVersion npmPackageVersion, IPackageSearchMetadata packageMeta, bool forceUpdate, RegistryEntry packageEntry) { // If we need to force the update, we delete the previous package+sha1 files if (forceUpdate) { DeleteUnityPackage(identity, npmPackageVersion); } if (!IsUnityPackageValid(identity, npmPackageVersion) || !IsUnityPackageSha1Valid(identity, npmPackageVersion)) { await ConvertNuGetPackageToUnity(identity, npmPackageInfo, npmPackageVersion, packageMeta, packageEntry); } else { npmPackageVersion.Distribution.Shasum = ReadUnityPackageSha1(identity, npmPackageVersion); } }
/// <summary> /// For each package in our registry.json, query NuGet, extract package metadata, and convert them to unity packages. /// </summary> private async Task BuildInternal() { var packageMetadataResource = _sourceRepository.GetResource <PackageMetadataResource>(); var versionPath = Path.Combine(_rootPersistentFolder, "version.txt"); var forceUpdate = !File.Exists(versionPath) || await File.ReadAllTextAsync(versionPath) != CurrentRegistryVersion; if (forceUpdate) { _logger.LogInformation($"Registry version changed to {CurrentRegistryVersion} - Regenerating all packages"); } var regexFilter = Filter != null ? new Regex(Filter, RegexOptions.IgnoreCase) : null; if (Filter != null) { _logger.LogInformation($"Filtering with regex: {Filter}"); } var onProgress = OnProgress; var progressCount = 0; foreach (var packageDesc in _registry) { var packageName = packageDesc.Key; var packageEntry = packageDesc.Value; // Log progress count onProgress?.Invoke(++progressCount, _registry.Count); // A package entry is ignored but allowed in the registry (case of Microsoft.CSharp) if (packageEntry.Ignored || (regexFilter != null && !regexFilter.IsMatch(packageName))) { continue; } var packageMetaIt = await packageMetadataResource.GetMetadataAsync(packageName, false, false, _sourceCacheContext, _logger, CancellationToken.None); var packageMetas = packageMetaIt.ToList(); foreach (var packageMeta in packageMetas) { var packageIdentity = packageMeta.Identity; var packageId = packageIdentity.Id.ToLowerInvariant(); var npmPackageId = $"{_unityScope}.{packageId}"; if (!packageEntry.Version.Satisfies(packageMeta.Identity.Version)) { continue; } var resolvedDependencyGroups = packageMeta.DependencySets.Where(dependencySet => dependencySet.TargetFramework.IsAny || _targetFrameworks.Any(targetFramework => dependencySet.TargetFramework == targetFramework.Framework)).ToList(); if (!packageEntry.Analyzer && resolvedDependencyGroups.Count == 0) { _logger.LogWarning($"The package `{packageIdentity}` doesn't support `{string.Join(",", _targetFrameworks.Select(x => x.Name))}`"); continue; } if (!_npmPackageRegistry.Packages.TryGetValue(npmPackageId, out var npmPackage)) { npmPackage = new NpmPackage(); _npmPackageRegistry.Packages.Add(npmPackageId, npmPackage); } // One NpmPackage (for package request) var packageInfoList = packageEntry.Listed ? _npmPackageRegistry.ListedPackageInfos : _npmPackageRegistry.UnlistedPackageInfos; if (!packageInfoList.Packages.TryGetValue(npmPackageId, out var npmPackageInfo)) { npmPackageInfo = new NpmPackageInfo(); packageInfoList.Packages.Add(npmPackageId, npmPackageInfo); } // Update latest version var currentVersion = packageIdentity.Version; var update = !npmPackage.DistTags.TryGetValue("latest", out var latestVersion) || (currentVersion > NuGetVersion.Parse(latestVersion)) || forceUpdate; string npmCurrentVersion = GetNpmVersion(currentVersion); if (update) { npmPackage.DistTags["latest"] = npmCurrentVersion; npmPackageInfo.Versions.Clear(); npmPackageInfo.Versions[npmCurrentVersion] = "latest"; npmPackage.Id = npmPackageId; npmPackage.License = packageMeta.LicenseMetadata?.License ?? packageMeta.LicenseUrl?.ToString(); npmPackage.Name = npmPackageId; npmPackageInfo.Name = npmPackageId; npmPackage.Description = packageMeta.Description; npmPackageInfo.Description = packageMeta.Description; npmPackageInfo.Author = packageMeta.Authors; if (packageMeta.Owners != null) { npmPackageInfo.Maintainers.Clear(); npmPackageInfo.Maintainers.AddRange(SplitCommaSeparatedString(packageMeta.Owners)); } if (packageMeta.Tags != null) { npmPackageInfo.Keywords.Clear(); npmPackageInfo.Keywords.Add("nuget"); npmPackageInfo.Keywords.AddRange(SplitCommaSeparatedString(packageMeta.Tags)); } } var npmVersion = new NpmPackageVersion { Id = $"{npmPackageId}@{npmCurrentVersion}", Version = npmCurrentVersion, Name = npmPackageId, Description = packageMeta.Description, Author = npmPackageInfo.Author, DisplayName = packageMeta.Title + _packageNameNuGetPostFix }; npmVersion.Distribution.Tarball = new Uri(_rootHttpUri, $"{npmPackage.Id}/-/{GetUnityPackageFileName(packageIdentity, npmVersion)}"); npmVersion.Unity = _minimumUnityVersion; npmPackage.Versions[npmVersion.Version] = npmVersion; bool hasDependencyErrors = false; foreach (var resolvedDependencyGroup in resolvedDependencyGroups) { foreach (var deps in resolvedDependencyGroup.Packages) { if (!_registry.TryGetValue(deps.Id, out var packageEntryDep)) { LogError($"The package `{packageIdentity}` has a dependency on `{deps.Id}` which is not in the registry. You must add this dependency to the registry.json file."); hasDependencyErrors = true; } else if (packageEntryDep.Ignored) { // A package that is ignored is not declared as an explicit dependency continue; } else if (!deps.VersionRange.IsSubSetOrEqualTo(packageEntryDep.Version)) { LogError($"The version range `{deps.VersionRange}` for the dependency `{deps.Id}` for the package `{packageIdentity}` doesn't match the range allowed from the registry.json: `{packageEntryDep.Version}`"); hasDependencyErrors = true; continue; } // Otherwise add the package as a dependency var depsId = deps.Id.ToLowerInvariant(); var key = $"{_unityScope}.{depsId}"; if (!npmVersion.Dependencies.ContainsKey(key)) { npmVersion.Dependencies.Add(key, GetNpmVersion(deps.VersionRange.MinVersion)); } } } // If we don't have any dependencies error, generate the package if (!hasDependencyErrors) { await ConvertNuGetToUnityPackageIfDoesNotExist(packageIdentity, npmPackageInfo, npmVersion, packageMeta, forceUpdate, packageEntry); npmPackage.Time[npmCurrentVersion] = packageMeta.Published?.UtcDateTime ?? GetUnityPackageFileInfo(packageIdentity, npmVersion).CreationTimeUtc; // Copy repository info if necessary if (update) { npmPackage.Repository = npmVersion.Repository?.Clone(); } } } } if (forceUpdate) { await File.WriteAllTextAsync(versionPath, CurrentRegistryVersion); } }
/// <summary> /// Converts a NuGet package to Unity package if not already /// </summary> private async Task ConvertNuGetToUnityPackageIfDoesNotExist(PackageIdentity identity, NpmPackageInfo npmPackageInfo, NpmPackageVersion npmPackageVersion, IPackageSearchMetadata packageMeta, NuGetFramework targetFramework) { if (!IsUnityPackageExists(identity)) { await ConvertNuGetPackageToUnity(identity, npmPackageInfo, npmPackageVersion, packageMeta, targetFramework); } else { npmPackageVersion.Distribution.Shasum = ReadUnityPackageSha1(identity); } }