private HttpWebRequest CreateWebRequest(string url, PackageDeploymentData deployInfo = null, SecureString apiKey = null)
{
var request = WebRequest.CreateHttp(url);
request.KeepAlive = false;
request.AllowReadStreamBuffering = false;
request.AllowWriteStreamBuffering = false;
request.Timeout = Timeout.Infinite;
request.UserAgent = $"{SDK.ProductName}/{SDK.ProductVersion} InedoCore/{typeof(ProGetClient).Assembly.GetName().Version}";
if (apiKey != null)
{
request.Headers.Add("X-ApiKey", AH.Unprotect(apiKey));
}
if (!string.IsNullOrWhiteSpace(this.UserName))
{
request.Headers.Add("Authorization", "Basic " + Convert.ToBase64String(InedoLib.UTF8Encoding.GetBytes(this.UserName + ":" + this.Password)));
}
if (deployInfo != null)
{
request.Headers.Add(PackageDeploymentData.Headers.Application, deployInfo.Application ?? string.Empty);
request.Headers.Add(PackageDeploymentData.Headers.Description, deployInfo.Description ?? string.Empty);
request.Headers.Add(PackageDeploymentData.Headers.Url, deployInfo.Url ?? string.Empty);
request.Headers.Add(PackageDeploymentData.Headers.Target, deployInfo.Target ?? string.Empty);
}
return(request);
}
private HttpWebRequest CreateRequest(string relativePath, PackageDeploymentData deployInfo = null)
{
string url = this.FeedUrl + relativePath;
this.Log.LogDebug("Creating request: " + url);
var asm = typeof(Operation).Assembly;
var request = WebRequest.CreateHttp(url);
request.UserAgent = $"{asm.GetCustomAttribute<AssemblyProductAttribute>()?.Product} {asm.GetName().Version} ({Environment.OSVersion})";
request.AutomaticDecompression = DecompressionMethods.GZip;
if (!string.IsNullOrEmpty(this.UserName) && !string.IsNullOrEmpty(this.Password))
{
this.Log.LogDebug($"Using Basic Authentication; user name '{this.UserName}'.");
request.Headers.Add(HttpRequestHeader.Authorization, "Basic " + Convert.ToBase64String(InedoLib.UTF8Encoding.GetBytes(this.UserName + ":" + this.Password)));
}
else
{
this.Log.LogDebug($"Using integrated authentication; user account '{Environment.UserName}', domain '{Environment.UserDomainName}'.");
request.UseDefaultCredentials = true;
request.PreAuthenticate = true;
}
if (deployInfo != null)
{
request.Headers.Add(PackageDeploymentData.Headers.Application, deployInfo.Application);
request.Headers.Add(PackageDeploymentData.Headers.Description, deployInfo.Description);
request.Headers.Add(PackageDeploymentData.Headers.Url, deployInfo.Url);
request.Headers.Add(PackageDeploymentData.Headers.Target, deployInfo.Target);
}
return(request);
}
private HttpClient CreateClient(PackageDeploymentData deployInfo = null)
{
HttpClient client;
if (!string.IsNullOrWhiteSpace(this.UserName))
{
this.Log.LogDebug($"Making request as {this.UserName}...");
client = new HttpClient(new HttpClientHandler {
Credentials = new NetworkCredential(this.UserName, this.Password ?? string.Empty)
});
}
else
{
client = new HttpClient();
}
client.DefaultRequestHeaders.UserAgent.Clear();
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue(typeof(Operation).Assembly.GetCustomAttribute <AssemblyProductAttribute>().Product, typeof(Operation).Assembly.GetName().Version.ToString()));
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("InedoCore", typeof(ProGetClient).Assembly.GetName().Version.ToString()));
if (deployInfo != null)
{
client.DefaultRequestHeaders.Add(PackageDeploymentData.Headers.Application, deployInfo.Application ?? string.Empty);
client.DefaultRequestHeaders.Add(PackageDeploymentData.Headers.Description, deployInfo.Description ?? string.Empty);
client.DefaultRequestHeaders.Add(PackageDeploymentData.Headers.Url, deployInfo.Url ?? string.Empty);
client.DefaultRequestHeaders.Add(PackageDeploymentData.Headers.Target, deployInfo.Target ?? string.Empty);
}
return(client);
}
public static async Task DeployAsync(IOperationExecutionContext context, IProGetPackageInstallTemplate template, ILogSink log, string installationReason, bool recordDeployment, Action <OperationProgress> setProgress = null)
{
var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>().ConfigureAwait(false);
var client = new ProGetClient(template.FeedUrl, template.FeedName, template.UserName, template.Password, log, context.CancellationToken);
try
{
var packageId = PackageName.Parse(template.PackageName);
log.LogInformation($"Connecting to {template.FeedUrl} to get metadata for {template.PackageName}...");
var packageInfo = await client.GetPackageInfoAsync(packageId).ConfigureAwait(false);
string version;
if (string.Equals(template.PackageVersion, "latest-stable", StringComparison.OrdinalIgnoreCase))
{
var stableVersions = packageInfo.versions
.Select(v => UniversalPackageVersion.TryParse(v))
.Where(v => string.IsNullOrEmpty(v?.Prerelease));
if (!stableVersions.Any())
{
log.LogError($"Package {template.PackageName} does not have any stable versions.");
return;
}
version = stableVersions.Max().ToString();
log.LogInformation($"Latest stable version of {template.PackageName} is {version}.");
}
else if (!string.IsNullOrEmpty(template.PackageVersion) && !string.Equals(template.PackageVersion, "latest", StringComparison.OrdinalIgnoreCase))
{
if (!packageInfo.versions.Contains(template.PackageVersion, StringComparer.OrdinalIgnoreCase))
{
log.LogError($"Package {template.PackageName} does not have a version {template.PackageVersion}.");
return;
}
version = template.PackageVersion;
}
else
{
version = packageInfo.latestVersion;
log.LogInformation($"Latest version of {template.PackageName} is {version}.");
}
var deployInfo = recordDeployment ? PackageDeploymentData.Create(context, log, $"Deployed by {installationReason} operation. See the URL for more info.") : null;
var targetRootPath = context.ResolvePath(template.TargetDirectory);
log.LogDebug("Target path: " + targetRootPath);
log.LogInformation("Downloading package...");
using (var content = await client.DownloadPackageContentAsync(packageId, version, deployInfo, (position, length) => setProgress?.Invoke(new OperationProgress(length == 0 ? null : (int?)(100 * position / length), "downloading package"))).ConfigureAwait(false))
{
var tempDirectoryName = fileOps.CombinePath(await fileOps.GetBaseWorkingDirectoryAsync().ConfigureAwait(false), Guid.NewGuid().ToString("N"));
// ensure directory exists on server
await fileOps.CreateDirectoryAsync(tempDirectoryName);
var tempZipFileName = tempDirectoryName + ".zip";
try
{
setProgress?.Invoke(new OperationProgress(0, "copying package to agent"));
using (var remote = await fileOps.OpenFileAsync(tempZipFileName, FileMode.CreateNew, FileAccess.Write).ConfigureAwait(false))
{
await content.CopyToAsync(remote, 81920, context.CancellationToken, position => setProgress?.Invoke(new OperationProgress((int)(100 * position / content.Length), "copying package to agent"))).ConfigureAwait(false);
}
setProgress?.Invoke(new OperationProgress("extracting package to temporary directory"));
await fileOps.ExtractZipFileAsync(tempZipFileName, tempDirectoryName, IO.FileCreationOptions.Overwrite).ConfigureAwait(false);
var expectedFiles = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var expectedDirectories = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
content.Position = 0;
using (var zip = new ZipArchive(content, ZipArchiveMode.Read, true))
{
foreach (var entry in zip.Entries)
{
// TODO: use AH.ReadZip when it is available in Otter.
var fullName = entry.FullName.Replace('\\', '/');
if (!fullName.StartsWith("package/", StringComparison.OrdinalIgnoreCase) || fullName.Length <= "package/".Length)
{
continue;
}
if (entry.IsDirectory())
{
expectedDirectories.Add(fullName.Substring("package/".Length).Trim('/'));
}
else
{
expectedFiles.Add(fullName.Substring("package/".Length));
var parts = fullName.Substring("package/".Length).Split('/');
for (int i = 1; i < parts.Length; i++)
{
// Add directories that are not explicitly in the zip file.
expectedDirectories.Add(string.Join("/", parts.Take(i)));
}
}
}
}
var jobExec = await context.Agent.TryGetServiceAsync <IRemoteJobExecuter>().ConfigureAwait(false);
if (jobExec != null)
{
var job = new PackageDeploymentJob
{
DeleteExtra = template.DeleteExtra,
TargetRootPath = targetRootPath,
TempDirectoryName = tempDirectoryName,
ExpectedDirectories = expectedDirectories.ToArray(),
ExpectedFiles = expectedFiles.ToArray()
};
job.MessageLogged += (s, e) => log.Log(e.Level, e.Message);
job.ProgressChanged += (s, e) => setProgress?.Invoke(e);
setProgress?.Invoke(new OperationProgress("starting remote job on agent"));
await jobExec.ExecuteJobAsync(job, context.CancellationToken).ConfigureAwait(false);
}
else
{
setProgress?.Invoke(new OperationProgress("ensuring target directory exists"));
await fileOps.CreateDirectoryAsync(targetRootPath).ConfigureAwait(false);
int index = 0;
if (template.DeleteExtra)
{
setProgress?.Invoke(new OperationProgress("checking existing files"));
var remoteFileList = await fileOps.GetFileSystemInfosAsync(targetRootPath, MaskingContext.IncludeAll).ConfigureAwait(false);
foreach (var file in remoteFileList)
{
index++;
setProgress?.Invoke(new OperationProgress(100 * index / remoteFileList.Count, "checking existing files"));
var relativeName = file.FullName.Substring(targetRootPath.Length).Replace('\\', '/').Trim('/');
if (file is SlimDirectoryInfo)
{
if (!expectedDirectories.Contains(relativeName))
{
log.LogDebug("Deleting extra directory: " + relativeName);
await fileOps.DeleteDirectoryAsync(file.FullName).ConfigureAwait(false);
}
}
else
{
if (!expectedFiles.Contains(relativeName))
{
log.LogDebug($"Deleting extra file: " + relativeName);
await fileOps.DeleteFileAsync(file.FullName).ConfigureAwait(false);
}
}
}
}
index = 0;
foreach (var relativeName in expectedDirectories)
{
index++;
setProgress?.Invoke(new OperationProgress(100 * index / expectedDirectories.Count, "ensuring target subdirectories exist"));
await fileOps.CreateDirectoryAsync(fileOps.CombinePath(targetRootPath, relativeName)).ConfigureAwait(false);
}
index = 0;
foreach (var relativeName in expectedFiles)
{
var sourcePath = fileOps.CombinePath(tempDirectoryName, "package", relativeName);
var targetPath = fileOps.CombinePath(targetRootPath, relativeName);
index++;
setProgress?.Invoke(new OperationProgress(100 * index / expectedFiles.Count, "moving files to target directory"));
await fileOps.MoveFileAsync(sourcePath, targetPath, true).ConfigureAwait(false);
}
}
setProgress?.Invoke(new OperationProgress("cleaning temporary files"));
}
finally
{
await Task.WhenAll(
fileOps.DeleteFileAsync(tempZipFileName),
fileOps.DeleteDirectoryAsync(tempDirectoryName)
).ConfigureAwait(false);
}
}
setProgress?.Invoke(new OperationProgress("recording package installation in machine registry"));
using (var registry = await PackageRegistry.GetRegistryAsync(context.Agent, false).ConfigureAwait(false))
{
var package = new RegisteredPackage
{
Group = packageId.Group,
Name = packageId.Name,
Version = version,
InstallPath = targetRootPath,
FeedUrl = template.FeedUrl,
InstallationDate = DateTimeOffset.Now.ToString("o"),
InstallationReason = installationReason,
InstalledUsing = $"{SDK.ProductName}/{SDK.ProductVersion} (InedoCore/{Extension.Version})"
};
try
{
using (var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
using (context.CancellationToken.Register(() => cancellationTokenSource.Cancel()))
{
await registry.LockAsync(cancellationTokenSource.Token).ConfigureAwait(false);
await registry.RegisterPackageAsync(package, context.CancellationToken).ConfigureAwait(false);
// doesn't need to be in a finally because dispose will unlock if necessary, but prefer doing it asynchronously
await registry.UnlockAsync().ConfigureAwait(false);
}
}
catch (TaskCanceledException)
{
log.LogWarning("Registering the package in the machine package registry timed out.");
}
}
}
catch (ProGetException ex)
{
log.LogError(ex.FullMessage);
return;
}
setProgress?.Invoke(null);
log.LogInformation("Package deployed!");
}
public async Task <ZipArchive> DownloadPackageAsync(PackageName id, string version, PackageDeploymentData deployInfo)
{
var stream = await this.DownloadPackageContentAsync(id, version, deployInfo).ConfigureAwait(false);
return(new ZipArchive(stream, ZipArchiveMode.Read));
}
public async Task <Stream> DownloadPackageContentAsync(PackageName id, string version, PackageDeploymentData deployInfo, Action <long, long> progressUpdate = null)
{
if (string.IsNullOrWhiteSpace(id?.Name))
{
throw new ArgumentNullException(nameof(id));
}
if (string.IsNullOrWhiteSpace(version))
{
throw new ArgumentNullException(nameof(version));
}
var url = Uri.EscapeDataString(id.Name) + "/" + Uri.EscapeDataString(version);
if (!string.IsNullOrEmpty(id.Group))
{
url = id.Group + "/" + url;
}
using (var client = this.CreateClient(deployInfo))
using (var response = await client.GetAsync(this.FeedUrl + "download/" + url, HttpCompletionOption.ResponseHeadersRead, this.CancellationToken).ConfigureAwait(false))
{
await HandleError(response).ConfigureAwait(false);
using (var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
{
var tempStream = TemporaryStream.Create(response.Content.Headers.ContentLength ?? 0L);
await responseStream.CopyToAsync(tempStream, 81920, this.CancellationToken, position =>
{
progressUpdate?.Invoke(position, response.Content.Headers.ContentLength ?? 0L);
}).ConfigureAwait(false);
tempStream.Position = 0;
return(tempStream);
}
}
}
public override async Task ConfigureAsync(IOperationExecutionContext context)
{
var fileOps = context.Agent.GetService <IFileOperationsExecuter>();
var client = new ProGetClient(this.Template.FeedUrl, this.Template.FeedName, this.Template.UserName, this.Template.Password, this);
try
{
var packageId = PackageName.Parse(this.Template.PackageName);
this.LogInformation($"Connecting to {this.Template.FeedUrl} to get metadata for {this.Template.PackageName}...");
var packageInfo = await client.GetPackageInfoAsync(packageId).ConfigureAwait(false);
string version;
if (!string.IsNullOrEmpty(this.Template.PackageVersion) && !string.Equals(this.Template.PackageVersion, "latest", StringComparison.OrdinalIgnoreCase))
{
if (!packageInfo.versions.Contains(this.Template.PackageVersion, StringComparer.OrdinalIgnoreCase))
{
this.LogError($"Package {this.Template.PackageName} does not have a version {this.Template.PackageVersion}.");
return;
}
version = this.Template.PackageVersion;
}
else
{
version = packageInfo.latestVersion;
this.LogInformation($"Latest version of {this.Template.PackageName} is {version}.");
}
var deployInfo = PackageDeploymentData.Create(context, this, "Deployed by Ensure-Package operation, see URL for more info.");
this.LogInformation("Downloading package...");
using (var zip = await client.DownloadPackageAsync(packageId, version, deployInfo).ConfigureAwait(false))
{
var dirsCreated = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
await fileOps.CreateDirectoryAsync(this.Template.TargetDirectory).ConfigureAwait(false);
dirsCreated.Add(this.Template.TargetDirectory);
foreach (var entry in zip.Entries)
{
if (!entry.FullName.StartsWith("package/", StringComparison.OrdinalIgnoreCase) || entry.Length <= "package/".Length)
{
continue;
}
var relativeName = entry.FullName.Substring("package/".Length);
var targetPath = fileOps.CombinePath(this.Template.TargetDirectory, relativeName);
if (relativeName.EndsWith("/"))
{
if (dirsCreated.Add(targetPath))
{
await fileOps.CreateDirectoryAsync(targetPath).ConfigureAwait(false);
}
}
else
{
var dir = PathEx.GetDirectoryName(targetPath);
if (dirsCreated.Add(dir))
{
await fileOps.CreateDirectoryAsync(dir);
}
using (var targetStream = await fileOps.OpenFileAsync(targetPath, FileMode.Create, FileAccess.Write).ConfigureAwait(false))
using (var sourceStream = entry.Open())
{
await sourceStream.CopyToAsync(targetStream).ConfigureAwait(false);
}
await fileOps.SetLastWriteTimeAsync(targetPath, entry.LastWriteTime.DateTime).ConfigureAwait(false);
}
}
}
}
catch (ProGetException ex)
{
this.LogError(ex.FullMessage);
return;
}
this.LogInformation("Package deployed!");
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var fileOps = context.Agent.GetService <IFileOperationsExecuter>();
var client = new ProGetClient(this.Server, this.FeedName, this.UserName, this.Password, this);
try
{
var packageId = ProGet.PackageName.Parse(this.PackageName);
this.LogInformation($"Connecting to {this.Server} to get metadata for {this.PackageName}...");
var packageInfo = await client.GetPackageInfoAsync(packageId).ConfigureAwait(false);
var version = new ProGetPackageVersionSpecifier(this.PackageVersion).GetBestMatch(packageInfo.versions);
if (version == null)
{
this.LogError($"Package {this.PackageName} does not have a version {this.PackageVersion}.");
return;
}
this.LogInformation($"Resolved package version is {version}.");
var deployInfo = PackageDeploymentData.Create(context, this, "Deployed by Get-Package operation, see URL for more info.");
this.LogInformation("Downloading package...");
using (var zip = await client.DownloadPackageAsync(packageId, version, deployInfo).ConfigureAwait(false))
{
var dirsCreated = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
this.LogDebug("Creating directory: " + this.TargetDirectory);
await fileOps.CreateDirectoryAsync(this.TargetDirectory).ConfigureAwait(false);
dirsCreated.Add(this.TargetDirectory);
foreach (var entry in zip.Entries)
{
if (!entry.FullName.StartsWith("package/", StringComparison.OrdinalIgnoreCase) || entry.Length <= "package/".Length)
{
continue;
}
var relativeName = entry.FullName.Substring("package/".Length);
var targetPath = fileOps.CombinePath(this.TargetDirectory, relativeName);
if (relativeName.EndsWith("/"))
{
if (dirsCreated.Add(targetPath))
{
await fileOps.CreateDirectoryAsync(targetPath).ConfigureAwait(false);
}
}
else
{
var dir = PathEx.GetDirectoryName(targetPath);
if (dirsCreated.Add(dir))
{
await fileOps.CreateDirectoryAsync(dir).ConfigureAwait(false);
}
using (var targetStream = await fileOps.OpenFileAsync(targetPath, FileMode.Create, FileAccess.Write).ConfigureAwait(false))
using (var sourceStream = entry.Open())
{
await sourceStream.CopyToAsync(targetStream).ConfigureAwait(false);
}
// timestamp in zip file is stored as UTC by convention
await fileOps.SetLastWriteTimeAsync(targetPath, entry.LastWriteTime.DateTime).ConfigureAwait(false);
}
}
}
}
catch (ProGetException ex)
{
this.LogError(ex.FullMessage);
return;
}
this.LogInformation("Package deployed!");
}
public async Task <ZipArchive> DownloadPackageAsync(PackageName id, string version, PackageDeploymentData deployInfo)
{
if (string.IsNullOrWhiteSpace(id?.Name))
{
throw new ArgumentNullException(nameof(id));
}
if (string.IsNullOrWhiteSpace(version))
{
throw new ArgumentNullException(nameof(version));
}
var url = Uri.EscapeDataString(id.Name) + "/" + Uri.EscapeDataString(version);
if (!string.IsNullOrEmpty(id.Group))
{
url = id.Group + "/" + url;
}
using (var client = this.CreateClient(deployInfo))
{
using (var response = await client.GetAsync(this.FeedUrl + "download/" + url).ConfigureAwait(false))
{
await HandleError(response).ConfigureAwait(false);
using (var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
{
var tempStream = TemporaryStream.Create(response.Content.Headers.ContentLength ?? 0L);
await responseStream.CopyToAsync(tempStream).ConfigureAwait(false);
tempStream.Position = 0;
return(new ZipArchive(tempStream, ZipArchiveMode.Read));
}
}
}
}