private HttpWebRequest CreateWebRequest(string url, PackageDeploymentData deployInfo = null, SecureString apiKey = null) { var request = WebRequest.CreateHttp(url); request.KeepAlive = false; request.AllowReadStreamBuffering = false; request.AllowWriteStreamBuffering = false; request.Timeout = Timeout.Infinite; request.UserAgent = $"{SDK.ProductName}/{SDK.ProductVersion} InedoCore/{typeof(ProGetClient).Assembly.GetName().Version}"; if (apiKey != null) { request.Headers.Add("X-ApiKey", AH.Unprotect(apiKey)); } if (!string.IsNullOrWhiteSpace(this.UserName)) { request.Headers.Add("Authorization", "Basic " + Convert.ToBase64String(InedoLib.UTF8Encoding.GetBytes(this.UserName + ":" + this.Password))); } if (deployInfo != null) { request.Headers.Add(PackageDeploymentData.Headers.Application, deployInfo.Application ?? string.Empty); request.Headers.Add(PackageDeploymentData.Headers.Description, deployInfo.Description ?? string.Empty); request.Headers.Add(PackageDeploymentData.Headers.Url, deployInfo.Url ?? string.Empty); request.Headers.Add(PackageDeploymentData.Headers.Target, deployInfo.Target ?? string.Empty); } return(request); }
private HttpWebRequest CreateRequest(string relativePath, PackageDeploymentData deployInfo = null) { string url = this.FeedUrl + relativePath; this.Log.LogDebug("Creating request: " + url); var asm = typeof(Operation).Assembly; var request = WebRequest.CreateHttp(url); request.UserAgent = $"{asm.GetCustomAttribute<AssemblyProductAttribute>()?.Product} {asm.GetName().Version} ({Environment.OSVersion})"; request.AutomaticDecompression = DecompressionMethods.GZip; if (!string.IsNullOrEmpty(this.UserName) && !string.IsNullOrEmpty(this.Password)) { this.Log.LogDebug($"Using Basic Authentication; user name '{this.UserName}'."); request.Headers.Add(HttpRequestHeader.Authorization, "Basic " + Convert.ToBase64String(InedoLib.UTF8Encoding.GetBytes(this.UserName + ":" + this.Password))); } else { this.Log.LogDebug($"Using integrated authentication; user account '{Environment.UserName}', domain '{Environment.UserDomainName}'."); request.UseDefaultCredentials = true; request.PreAuthenticate = true; } if (deployInfo != null) { request.Headers.Add(PackageDeploymentData.Headers.Application, deployInfo.Application); request.Headers.Add(PackageDeploymentData.Headers.Description, deployInfo.Description); request.Headers.Add(PackageDeploymentData.Headers.Url, deployInfo.Url); request.Headers.Add(PackageDeploymentData.Headers.Target, deployInfo.Target); } return(request); }
private HttpClient CreateClient(PackageDeploymentData deployInfo = null) { HttpClient client; if (!string.IsNullOrWhiteSpace(this.UserName)) { this.Log.LogDebug($"Making request as {this.UserName}..."); client = new HttpClient(new HttpClientHandler { Credentials = new NetworkCredential(this.UserName, this.Password ?? string.Empty) }); } else { client = new HttpClient(); } client.DefaultRequestHeaders.UserAgent.Clear(); client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue(typeof(Operation).Assembly.GetCustomAttribute <AssemblyProductAttribute>().Product, typeof(Operation).Assembly.GetName().Version.ToString())); client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("InedoCore", typeof(ProGetClient).Assembly.GetName().Version.ToString())); if (deployInfo != null) { client.DefaultRequestHeaders.Add(PackageDeploymentData.Headers.Application, deployInfo.Application ?? string.Empty); client.DefaultRequestHeaders.Add(PackageDeploymentData.Headers.Description, deployInfo.Description ?? string.Empty); client.DefaultRequestHeaders.Add(PackageDeploymentData.Headers.Url, deployInfo.Url ?? string.Empty); client.DefaultRequestHeaders.Add(PackageDeploymentData.Headers.Target, deployInfo.Target ?? string.Empty); } return(client); }
public static async Task DeployAsync(IOperationExecutionContext context, IProGetPackageInstallTemplate template, ILogSink log, string installationReason, bool recordDeployment, Action <OperationProgress> setProgress = null) { var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>().ConfigureAwait(false); var client = new ProGetClient(template.FeedUrl, template.FeedName, template.UserName, template.Password, log, context.CancellationToken); try { var packageId = PackageName.Parse(template.PackageName); log.LogInformation($"Connecting to {template.FeedUrl} to get metadata for {template.PackageName}..."); var packageInfo = await client.GetPackageInfoAsync(packageId).ConfigureAwait(false); string version; if (string.Equals(template.PackageVersion, "latest-stable", StringComparison.OrdinalIgnoreCase)) { var stableVersions = packageInfo.versions .Select(v => UniversalPackageVersion.TryParse(v)) .Where(v => string.IsNullOrEmpty(v?.Prerelease)); if (!stableVersions.Any()) { log.LogError($"Package {template.PackageName} does not have any stable versions."); return; } version = stableVersions.Max().ToString(); log.LogInformation($"Latest stable version of {template.PackageName} is {version}."); } else if (!string.IsNullOrEmpty(template.PackageVersion) && !string.Equals(template.PackageVersion, "latest", StringComparison.OrdinalIgnoreCase)) { if (!packageInfo.versions.Contains(template.PackageVersion, StringComparer.OrdinalIgnoreCase)) { log.LogError($"Package {template.PackageName} does not have a version {template.PackageVersion}."); return; } version = template.PackageVersion; } else { version = packageInfo.latestVersion; log.LogInformation($"Latest version of {template.PackageName} is {version}."); } var deployInfo = recordDeployment ? PackageDeploymentData.Create(context, log, $"Deployed by {installationReason} operation. See the URL for more info.") : null; var targetRootPath = context.ResolvePath(template.TargetDirectory); log.LogDebug("Target path: " + targetRootPath); log.LogInformation("Downloading package..."); using (var content = await client.DownloadPackageContentAsync(packageId, version, deployInfo, (position, length) => setProgress?.Invoke(new OperationProgress(length == 0 ? null : (int?)(100 * position / length), "downloading package"))).ConfigureAwait(false)) { var tempDirectoryName = fileOps.CombinePath(await fileOps.GetBaseWorkingDirectoryAsync().ConfigureAwait(false), Guid.NewGuid().ToString("N")); // ensure directory exists on server await fileOps.CreateDirectoryAsync(tempDirectoryName); var tempZipFileName = tempDirectoryName + ".zip"; try { setProgress?.Invoke(new OperationProgress(0, "copying package to agent")); using (var remote = await fileOps.OpenFileAsync(tempZipFileName, FileMode.CreateNew, FileAccess.Write).ConfigureAwait(false)) { await content.CopyToAsync(remote, 81920, context.CancellationToken, position => setProgress?.Invoke(new OperationProgress((int)(100 * position / content.Length), "copying package to agent"))).ConfigureAwait(false); } setProgress?.Invoke(new OperationProgress("extracting package to temporary directory")); await fileOps.ExtractZipFileAsync(tempZipFileName, tempDirectoryName, IO.FileCreationOptions.Overwrite).ConfigureAwait(false); var expectedFiles = new HashSet <string>(StringComparer.OrdinalIgnoreCase); var expectedDirectories = new HashSet <string>(StringComparer.OrdinalIgnoreCase); content.Position = 0; using (var zip = new ZipArchive(content, ZipArchiveMode.Read, true)) { foreach (var entry in zip.Entries) { // TODO: use AH.ReadZip when it is available in Otter. var fullName = entry.FullName.Replace('\\', '/'); if (!fullName.StartsWith("package/", StringComparison.OrdinalIgnoreCase) || fullName.Length <= "package/".Length) { continue; } if (entry.IsDirectory()) { expectedDirectories.Add(fullName.Substring("package/".Length).Trim('/')); } else { expectedFiles.Add(fullName.Substring("package/".Length)); var parts = fullName.Substring("package/".Length).Split('/'); for (int i = 1; i < parts.Length; i++) { // Add directories that are not explicitly in the zip file. expectedDirectories.Add(string.Join("/", parts.Take(i))); } } } } var jobExec = await context.Agent.TryGetServiceAsync <IRemoteJobExecuter>().ConfigureAwait(false); if (jobExec != null) { var job = new PackageDeploymentJob { DeleteExtra = template.DeleteExtra, TargetRootPath = targetRootPath, TempDirectoryName = tempDirectoryName, ExpectedDirectories = expectedDirectories.ToArray(), ExpectedFiles = expectedFiles.ToArray() }; job.MessageLogged += (s, e) => log.Log(e.Level, e.Message); job.ProgressChanged += (s, e) => setProgress?.Invoke(e); setProgress?.Invoke(new OperationProgress("starting remote job on agent")); await jobExec.ExecuteJobAsync(job, context.CancellationToken).ConfigureAwait(false); } else { setProgress?.Invoke(new OperationProgress("ensuring target directory exists")); await fileOps.CreateDirectoryAsync(targetRootPath).ConfigureAwait(false); int index = 0; if (template.DeleteExtra) { setProgress?.Invoke(new OperationProgress("checking existing files")); var remoteFileList = await fileOps.GetFileSystemInfosAsync(targetRootPath, MaskingContext.IncludeAll).ConfigureAwait(false); foreach (var file in remoteFileList) { index++; setProgress?.Invoke(new OperationProgress(100 * index / remoteFileList.Count, "checking existing files")); var relativeName = file.FullName.Substring(targetRootPath.Length).Replace('\\', '/').Trim('/'); if (file is SlimDirectoryInfo) { if (!expectedDirectories.Contains(relativeName)) { log.LogDebug("Deleting extra directory: " + relativeName); await fileOps.DeleteDirectoryAsync(file.FullName).ConfigureAwait(false); } } else { if (!expectedFiles.Contains(relativeName)) { log.LogDebug($"Deleting extra file: " + relativeName); await fileOps.DeleteFileAsync(file.FullName).ConfigureAwait(false); } } } } index = 0; foreach (var relativeName in expectedDirectories) { index++; setProgress?.Invoke(new OperationProgress(100 * index / expectedDirectories.Count, "ensuring target subdirectories exist")); await fileOps.CreateDirectoryAsync(fileOps.CombinePath(targetRootPath, relativeName)).ConfigureAwait(false); } index = 0; foreach (var relativeName in expectedFiles) { var sourcePath = fileOps.CombinePath(tempDirectoryName, "package", relativeName); var targetPath = fileOps.CombinePath(targetRootPath, relativeName); index++; setProgress?.Invoke(new OperationProgress(100 * index / expectedFiles.Count, "moving files to target directory")); await fileOps.MoveFileAsync(sourcePath, targetPath, true).ConfigureAwait(false); } } setProgress?.Invoke(new OperationProgress("cleaning temporary files")); } finally { await Task.WhenAll( fileOps.DeleteFileAsync(tempZipFileName), fileOps.DeleteDirectoryAsync(tempDirectoryName) ).ConfigureAwait(false); } } setProgress?.Invoke(new OperationProgress("recording package installation in machine registry")); using (var registry = await PackageRegistry.GetRegistryAsync(context.Agent, false).ConfigureAwait(false)) { var package = new RegisteredPackage { Group = packageId.Group, Name = packageId.Name, Version = version, InstallPath = targetRootPath, FeedUrl = template.FeedUrl, InstallationDate = DateTimeOffset.Now.ToString("o"), InstallationReason = installationReason, InstalledUsing = $"{SDK.ProductName}/{SDK.ProductVersion} (InedoCore/{Extension.Version})" }; try { using (var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10))) using (context.CancellationToken.Register(() => cancellationTokenSource.Cancel())) { await registry.LockAsync(cancellationTokenSource.Token).ConfigureAwait(false); await registry.RegisterPackageAsync(package, context.CancellationToken).ConfigureAwait(false); // doesn't need to be in a finally because dispose will unlock if necessary, but prefer doing it asynchronously await registry.UnlockAsync().ConfigureAwait(false); } } catch (TaskCanceledException) { log.LogWarning("Registering the package in the machine package registry timed out."); } } } catch (ProGetException ex) { log.LogError(ex.FullMessage); return; } setProgress?.Invoke(null); log.LogInformation("Package deployed!"); }
public async Task <ZipArchive> DownloadPackageAsync(PackageName id, string version, PackageDeploymentData deployInfo) { var stream = await this.DownloadPackageContentAsync(id, version, deployInfo).ConfigureAwait(false); return(new ZipArchive(stream, ZipArchiveMode.Read)); }
public async Task <Stream> DownloadPackageContentAsync(PackageName id, string version, PackageDeploymentData deployInfo, Action <long, long> progressUpdate = null) { if (string.IsNullOrWhiteSpace(id?.Name)) { throw new ArgumentNullException(nameof(id)); } if (string.IsNullOrWhiteSpace(version)) { throw new ArgumentNullException(nameof(version)); } var url = Uri.EscapeDataString(id.Name) + "/" + Uri.EscapeDataString(version); if (!string.IsNullOrEmpty(id.Group)) { url = id.Group + "/" + url; } using (var client = this.CreateClient(deployInfo)) using (var response = await client.GetAsync(this.FeedUrl + "download/" + url, HttpCompletionOption.ResponseHeadersRead, this.CancellationToken).ConfigureAwait(false)) { await HandleError(response).ConfigureAwait(false); using (var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) { var tempStream = TemporaryStream.Create(response.Content.Headers.ContentLength ?? 0L); await responseStream.CopyToAsync(tempStream, 81920, this.CancellationToken, position => { progressUpdate?.Invoke(position, response.Content.Headers.ContentLength ?? 0L); }).ConfigureAwait(false); tempStream.Position = 0; return(tempStream); } } }
public override async Task ConfigureAsync(IOperationExecutionContext context) { var fileOps = context.Agent.GetService <IFileOperationsExecuter>(); var client = new ProGetClient(this.Template.FeedUrl, this.Template.FeedName, this.Template.UserName, this.Template.Password, this); try { var packageId = PackageName.Parse(this.Template.PackageName); this.LogInformation($"Connecting to {this.Template.FeedUrl} to get metadata for {this.Template.PackageName}..."); var packageInfo = await client.GetPackageInfoAsync(packageId).ConfigureAwait(false); string version; if (!string.IsNullOrEmpty(this.Template.PackageVersion) && !string.Equals(this.Template.PackageVersion, "latest", StringComparison.OrdinalIgnoreCase)) { if (!packageInfo.versions.Contains(this.Template.PackageVersion, StringComparer.OrdinalIgnoreCase)) { this.LogError($"Package {this.Template.PackageName} does not have a version {this.Template.PackageVersion}."); return; } version = this.Template.PackageVersion; } else { version = packageInfo.latestVersion; this.LogInformation($"Latest version of {this.Template.PackageName} is {version}."); } var deployInfo = PackageDeploymentData.Create(context, this, "Deployed by Ensure-Package operation, see URL for more info."); this.LogInformation("Downloading package..."); using (var zip = await client.DownloadPackageAsync(packageId, version, deployInfo).ConfigureAwait(false)) { var dirsCreated = new HashSet <string>(StringComparer.OrdinalIgnoreCase); await fileOps.CreateDirectoryAsync(this.Template.TargetDirectory).ConfigureAwait(false); dirsCreated.Add(this.Template.TargetDirectory); foreach (var entry in zip.Entries) { if (!entry.FullName.StartsWith("package/", StringComparison.OrdinalIgnoreCase) || entry.Length <= "package/".Length) { continue; } var relativeName = entry.FullName.Substring("package/".Length); var targetPath = fileOps.CombinePath(this.Template.TargetDirectory, relativeName); if (relativeName.EndsWith("/")) { if (dirsCreated.Add(targetPath)) { await fileOps.CreateDirectoryAsync(targetPath).ConfigureAwait(false); } } else { var dir = PathEx.GetDirectoryName(targetPath); if (dirsCreated.Add(dir)) { await fileOps.CreateDirectoryAsync(dir); } using (var targetStream = await fileOps.OpenFileAsync(targetPath, FileMode.Create, FileAccess.Write).ConfigureAwait(false)) using (var sourceStream = entry.Open()) { await sourceStream.CopyToAsync(targetStream).ConfigureAwait(false); } await fileOps.SetLastWriteTimeAsync(targetPath, entry.LastWriteTime.DateTime).ConfigureAwait(false); } } } } catch (ProGetException ex) { this.LogError(ex.FullMessage); return; } this.LogInformation("Package deployed!"); }
public override async Task ExecuteAsync(IOperationExecutionContext context) { var fileOps = context.Agent.GetService <IFileOperationsExecuter>(); var client = new ProGetClient(this.Server, this.FeedName, this.UserName, this.Password, this); try { var packageId = ProGet.PackageName.Parse(this.PackageName); this.LogInformation($"Connecting to {this.Server} to get metadata for {this.PackageName}..."); var packageInfo = await client.GetPackageInfoAsync(packageId).ConfigureAwait(false); var version = new ProGetPackageVersionSpecifier(this.PackageVersion).GetBestMatch(packageInfo.versions); if (version == null) { this.LogError($"Package {this.PackageName} does not have a version {this.PackageVersion}."); return; } this.LogInformation($"Resolved package version is {version}."); var deployInfo = PackageDeploymentData.Create(context, this, "Deployed by Get-Package operation, see URL for more info."); this.LogInformation("Downloading package..."); using (var zip = await client.DownloadPackageAsync(packageId, version, deployInfo).ConfigureAwait(false)) { var dirsCreated = new HashSet <string>(StringComparer.OrdinalIgnoreCase); this.LogDebug("Creating directory: " + this.TargetDirectory); await fileOps.CreateDirectoryAsync(this.TargetDirectory).ConfigureAwait(false); dirsCreated.Add(this.TargetDirectory); foreach (var entry in zip.Entries) { if (!entry.FullName.StartsWith("package/", StringComparison.OrdinalIgnoreCase) || entry.Length <= "package/".Length) { continue; } var relativeName = entry.FullName.Substring("package/".Length); var targetPath = fileOps.CombinePath(this.TargetDirectory, relativeName); if (relativeName.EndsWith("/")) { if (dirsCreated.Add(targetPath)) { await fileOps.CreateDirectoryAsync(targetPath).ConfigureAwait(false); } } else { var dir = PathEx.GetDirectoryName(targetPath); if (dirsCreated.Add(dir)) { await fileOps.CreateDirectoryAsync(dir).ConfigureAwait(false); } using (var targetStream = await fileOps.OpenFileAsync(targetPath, FileMode.Create, FileAccess.Write).ConfigureAwait(false)) using (var sourceStream = entry.Open()) { await sourceStream.CopyToAsync(targetStream).ConfigureAwait(false); } // timestamp in zip file is stored as UTC by convention await fileOps.SetLastWriteTimeAsync(targetPath, entry.LastWriteTime.DateTime).ConfigureAwait(false); } } } } catch (ProGetException ex) { this.LogError(ex.FullMessage); return; } this.LogInformation("Package deployed!"); }
public async Task <ZipArchive> DownloadPackageAsync(PackageName id, string version, PackageDeploymentData deployInfo) { if (string.IsNullOrWhiteSpace(id?.Name)) { throw new ArgumentNullException(nameof(id)); } if (string.IsNullOrWhiteSpace(version)) { throw new ArgumentNullException(nameof(version)); } var url = Uri.EscapeDataString(id.Name) + "/" + Uri.EscapeDataString(version); if (!string.IsNullOrEmpty(id.Group)) { url = id.Group + "/" + url; } using (var client = this.CreateClient(deployInfo)) { using (var response = await client.GetAsync(this.FeedUrl + "download/" + url).ConfigureAwait(false)) { await HandleError(response).ConfigureAwait(false); using (var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) { var tempStream = TemporaryStream.Create(response.Content.Headers.ContentLength ?? 0L); await responseStream.CopyToAsync(tempStream).ConfigureAwait(false); tempStream.Position = 0; return(new ZipArchive(tempStream, ZipArchiveMode.Read)); } } } }