public async Task <IReadOnlyCollection <PackageVersion> > GetPackageVersionsAsync(
[NotNull] string packageId,
bool useCache = true,
ILogger logger = null,
bool includePreReleased = false,
string nugetPackageSource = null,
string nugetConfigFile = null,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(packageId))
{
throw new ArgumentException("Value cannot be null or whitespace.", nameof(packageId));
}
if (packageId.Equals(Constants.NotAvailable))
{
return(ImmutableArray <PackageVersion> .Empty);
}
string NormalizeKey(string key)
{
return(key.Replace(":", "_")
.Replace("/", "")
.Replace(".", "")
.Replace(Path.DirectorySeparatorChar.ToString(), "_"));
}
string cacheKey = AllPackagesCacheKey;
if (!string.IsNullOrWhiteSpace(nugetConfigFile))
{
string configCachePart = $"{PackagesCacheKeyBaseUrn}:{NormalizeKey(nugetConfigFile)}";
cacheKey = !string.IsNullOrWhiteSpace(nugetPackageSource) ? $"{configCachePart}:{NormalizeKey(nugetPackageSource)}" : configCachePart;
}
else if (!string.IsNullOrWhiteSpace(nugetPackageSource))
{
cacheKey = $"{PackagesCacheKeyBaseUrn}:{NormalizeKey(nugetPackageSource)}";
}
cacheKey += $":{packageId}";
_logger.Verbose("Using package cache key {Key}", cacheKey);
if (useCache)
{
if (_memoryCache.TryGetValue(cacheKey, out IReadOnlyCollection <PackageVersion> packages))
{
if (packages.Count > 0)
{
_logger.Debug("Returning packages from cache with key {Key} for package id {PackageId}",
cacheKey,
packageId);
return(packages);
}
}
}
var nuGetDownloadClient = new NuGetDownloadClient();
HttpClient httpClient = _httpClientFactory.CreateClient("nuget");
NuGetDownloadResult nuGetDownloadResult;
using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
{
nuGetDownloadResult = await nuGetDownloadClient.DownloadNuGetAsync(NuGetDownloadSettings.Default,
_logger,
httpClient,
cts.Token);
}
if (!nuGetDownloadResult.Succeeded)
{
if (nuGetDownloadResult.Exception != null)
{
_logger.Error(nuGetDownloadResult.Exception,
"Could not download NuGet.exe: {Result}",
nuGetDownloadResult.Result);
}
else
{
_logger.Error("Could not download NuGet.exe: {Result}", nuGetDownloadResult.Result);
}
}
string nugetExe = nuGetDownloadResult.NuGetExePath;
if (string.IsNullOrWhiteSpace(nugetExe))
{
throw new DeployerAppException("The nuget.exe path is not set");
}
if (!File.Exists(nugetExe))
{
throw new DeployerAppException($"The nuget.exe path '{nugetExe}' does not exist");
}
string packageSourceAppSettingsKey = ConfigurationConstants.NuGetPackageSourceName;
string packageSource = nugetPackageSource.WithDefault(_keyValueConfiguration[packageSourceAppSettingsKey]);
var args = new List <string> {
"list", packageId
};
if (includePreReleased)
{
args.Add("-PreRelease");
}
if (!string.IsNullOrWhiteSpace(packageSource))
{
logger?.Debug("Using package source '{PackageSource}' for package {Package}", packageSource, packageId);
args.Add("-source");
args.Add(packageSource);
}
else
{
logger?.Debug(
"There is no package source defined i app settings, key '{PackageSourceAppSettingsKey}', using all sources",
packageSourceAppSettingsKey);
}
args.Add("-AllVersions");
args.Add("-NonInteractive");
args.Add("-Verbosity");
args.Add("normal");
string configFile =
nugetConfigFile.WithDefault(_keyValueConfiguration[ConfigurationConstants.NugetConfigFile]);
if (configFile.HasValue() && File.Exists(configFile))
{
_logger.Debug("Using NuGet config file {NuGetConfigFile} for package {Package}", configFile, packageId);
args.Add("-ConfigFile");
args.Add(configFile);
}
var builder = new List <string>();
var errorBuild = new List <string>();
logger?.Debug("Running NuGet from package service to find packages with timeout {Seconds} seconds",
_deploymentConfiguration.ListTimeOutInSeconds);
ExitCode exitCode;
using (var cancellationTokenSource =
new CancellationTokenSource(TimeSpan.FromSeconds(_deploymentConfiguration.ListTimeOutInSeconds)))
{
using (CancellationTokenSource linked =
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, cancellationTokenSource.Token))
{
exitCode = await ProcessRunner.ExecuteProcessAsync(nugetExe,
args,
(message, category) =>
{
builder.Add(message);
_logger.Debug("{Category} {Message}", category, message);
},
(message, category) =>
{
errorBuild.Add(message);
_logger.Error("{Category} {Message}", category, message);
},
(message, category) => _logger.Debug("{Category} {ProcessToolMessage}", category, message),
(message, category) => _logger.Verbose("{Category} {ProcessToolMessage}", category, message),
cancellationToken : linked.Token);
}
}
string standardOut = string.Join(Environment.NewLine, builder);
string standardErrorOut = string.Join(Environment.NewLine, errorBuild);
if (!exitCode.IsSuccess)
{
var sources = new List <string>();
var sourcesError = new List <string>();
var sourcesArgs = new List <string> {
"sources"
};
if (configFile.HasValue() && File.Exists(configFile))
{
sourcesArgs.Add("-ConfigFile");
sourcesArgs.Add(configFile);
}
sourcesArgs.Add("-NonInteractive");
if (_logger.IsEnabled(LogEventLevel.Debug) || _logger.IsEnabled(LogEventLevel.Verbose))
{
sourcesArgs.Add("-Verbosity");
sourcesArgs.Add("detailed");
}
await ProcessRunner.ExecuteProcessAsync(nugetExe,
sourcesArgs,
(message, _) => sources.Add(message),
(message, _) => sourcesError.Add(message),
(message, category) => _logger.Information("{Category} {ProcessToolMessage}", category, message),
(message, category) => _logger.Verbose("{Category} {ProcessToolMessage}", category, message),
cancellationToken : cancellationToken);
string sourcesOut = string.Join(Environment.NewLine, sources);
string sourcesErrorOut = string.Join(Environment.NewLine, sourcesError);
_logger.Error(
"Exit code {Code} when running NuGet list packages; standard out '{StandardOut}', standard error '{StandardErrorOut}', exe path '{NugetExe}', arguments '{Arguments}', nuget sources '{SourcesOut}', sources error '{SourcesErrorOut}'",
exitCode.Code,
standardOut,
standardErrorOut,
nugetExe,
string.Join(" ", args),
sourcesOut,
sourcesErrorOut);
return(Array.Empty <PackageVersion>());
}
var ignoredOutputStatements = new List <string> {
"Using credentials", "No packages found"
};
List <string> included =
builder.Where(line => !ignoredOutputStatements.Any(ignored =>
line.IndexOf(ignored, StringComparison.InvariantCultureIgnoreCase) >= 0))
.ToList();
List <PackageVersion> items = included.Select(
package =>
{
string[] parts = package.Split(' ');
string currentPackageId = parts[0];
try
{
string version = parts.Last();
if (!SemanticVersion.TryParse(version, out SemanticVersion semanticVersion))
{
_logger.Debug(
"Found package version {Version} for package {Package}, skipping because it could not be parsed as semantic version",
version,
currentPackageId);
return(null);
}
if (!packageId.Equals(currentPackageId, StringComparison.OrdinalIgnoreCase))
{
_logger.Debug(
"Found package {Package}, skipping because it does match requested package {RequestedPackage}",
currentPackageId,
packageId);
return(null);
}
return(new PackageVersion(packageId, semanticVersion));
}
catch (Exception ex) when(!ex.IsFatal())
{
_logger.Warning(ex, "Error parsing package '{Package}'", package);
return(null);
}
})
.Where(packageVersion => packageVersion != null)
.OrderBy(packageVersion => packageVersion.PackageId)
.ThenByDescending(packageVersion => packageVersion.Version)
.ToList();
var addedPackages = new List <string>();
foreach (PackageVersion packageVersion in items)
{
addedPackages.Add(packageVersion.ToString());
}
if (_logger.IsEnabled(LogEventLevel.Verbose))
{
_logger.Verbose("Added {Count} packages to in-memory cache with cache key {CacheKey} {PackageVersions}",
addedPackages.Count,
cacheKey,
addedPackages);
}
else if (addedPackages.Count > 0 && addedPackages.Count < 20)
{
_logger.Information(
"Added {Count} packages to in-memory cache with cache key {CacheKey} {PackageVersions}",
addedPackages.Count,
cacheKey,
addedPackages);
}
else if (addedPackages.Any())
{
_logger.Information("Added {Count} packages to in-memory cache with cache key {CacheKey}",
addedPackages.Count,
cacheKey);
}
if (addedPackages.Any())
{
_memoryCache.Set(cacheKey, items);
}
else
{
_logger.Debug("Added no packages to in-memory cache for cache key {CacheKey}", cacheKey);
}
return(items);
}
public async Task <ExitCode> ExecuteAsync(
DeploymentTask deploymentTask,
ILogger logger,
CancellationToken cancellationToken = default)
{
string jobId = "MDep_" + Guid.NewGuid();
logger.Information("Starting job {JobId}", jobId);
DeploymentTarget deploymentTarget =
await GetDeploymentTarget(deploymentTask.DeploymentTargetId, cancellationToken);
Environment.SetEnvironmentVariable(ConfigurationConstants.AllowPreReleaseEnabled,
"true"); // TODO try to remove
SetLogging();
string targetDirectoryPath = GetTargetDirectoryPath(deploymentTarget, jobId, deploymentTask);
string targetEnvironmentConfigName =
deploymentTarget.EnvironmentConfiguration;
var arguments = new List <string>();
logger.Information("Using manifest file for job {JobId}", jobId);
string publishSettingsFile = deploymentTarget.PublishSettingFile;
string deploymentTargetParametersFile = deploymentTarget.ParameterFile;
var tempManifestFile = new FileInfo(Path.Combine(Path.GetTempPath(), $"{jobId}.manifest"));
deploymentTask.TempFiles.Add(tempManifestFile);
ImmutableDictionary <string, string[]> parameterDictionary;
if (!string.IsNullOrWhiteSpace(deploymentTargetParametersFile) &&
!Path.IsPathRooted(deploymentTargetParametersFile))
{
throw new DeployerAppException(
$"The deployment target {deploymentTarget} parameter file '{deploymentTargetParametersFile}' is not a rooted path");
}
if (!string.IsNullOrWhiteSpace(deploymentTargetParametersFile) &&
File.Exists(deploymentTargetParametersFile))
{
string parametersJson = File.ReadAllText(deploymentTargetParametersFile, Encoding.UTF8);
parameterDictionary = JsonConvert
.DeserializeObject <Dictionary <string, string[]> >(parametersJson).ToImmutableDictionary();
logger.Information("Using WebDeploy parameters from file {DeploymentTargetParametersFile}",
deploymentTargetParametersFile);
}
else
{
logger.Information("No WebDeploy parameters file exists ('{DeploymentTargetParametersFile}')",
deploymentTargetParametersFile);
parameterDictionary = deploymentTarget.Parameters;
}
ImmutableDictionary <string, string[]> parameters = parameterDictionary;
if (deploymentTarget.PublishSettingsXml.HasValue())
{
string tempFileName = Path.GetTempFileName();
await File.WriteAllTextAsync(tempFileName,
deploymentTarget.PublishSettingsXml,
Encoding.UTF8,
cancellationToken);
deploymentTask.TempFiles.Add(new FileInfo(tempFileName));
publishSettingsFile = tempFileName;
}
if (!File.Exists(publishSettingsFile))
{
const string secretKeyPrefix = "publish-settings";
string id = deploymentTarget.Id;
const string usernameKey = secretKeyPrefix + ":username";
const string passwordKey = secretKeyPrefix + ":password";
const string publishUrlKey = secretKeyPrefix + ":publish-url";
const string msdeploySiteKey = secretKeyPrefix + ":msdeploySite";
string username = _credentialReadService.GetSecret(id, usernameKey);
string password = _credentialReadService.GetSecret(id, passwordKey);
string publishUrl = _credentialReadService.GetSecret(id, publishUrlKey);
string msdeploySite = _credentialReadService.GetSecret(id, msdeploySiteKey);
if (StringUtils.AllHaveValues(username, password, publishUrl, msdeploySite))
{
FileInfo fileInfo = CreateTempPublishFile(deploymentTarget,
username,
password,
publishUrl);
deploymentTask.TempFiles.Add(fileInfo);
publishSettingsFile = fileInfo.FullName;
}
else
{
Log.Warning("Could not get secrets for deployment target id {DeploymentTargetId}", id);
}
}
var definitions = new
{
definitions = new object[]
{
new
{
deploymentTask.PackageId,
targetDirectoryPath,
isPreRelease = deploymentTask.SemanticVersion.IsPrerelease,
environmentConfig = targetEnvironmentConfigName,
publishSettingsFile,
parameters,
deploymentTarget.NuGetConfigFile,
deploymentTarget.NuGetPackageSource,
semanticVersion = deploymentTask.SemanticVersion.ToNormalizedString(),
iisSiteName = deploymentTarget.IisSiteName,
webConfigTransform = deploymentTarget.WebConfigTransform
}
}
};
string json = JsonConvert.SerializeObject(definitions, Formatting.Indented);
logger.Information("Using definitions JSON: {Json}", json);
logger.Information("Using temp manifest file '{ManifestFile}'", tempManifestFile.FullName);
await File.WriteAllTextAsync(tempManifestFile.FullName, json, Encoding.UTF8, cancellationToken);
arguments.Add($"\"{tempManifestFile.FullName}\"");
//TODO propagate properties by direct command or default
Environment.SetEnvironmentVariable("urn:milou-deployer:tools:nuget:exe-path",
_keyValueConfiguration["urn:milou-deployer:tools:nuget:exe-path"]);
arguments.Add(Bootstrapper.Common.Constants.AllowPreRelease);
arguments.Add(Milou.Deployer.Core.LoggingConstants.PlainOutputFormatEnabled);
string[] deployerArgs = arguments.ToArray();
logger.Verbose("Running Milou Deployer bootstrapper");
HttpClient httpClient = _clientFactory.CreateClient("Bootstrapper");
using (Bootstrapper.Common.App deployerApp =
await Bootstrapper.Common.App.CreateAsync(deployerArgs,
logger,
httpClient,
false))
{
NuGetPackageInstallResult result =
await deployerApp.ExecuteAsync(deployerArgs.ToImmutableArray(), cancellationToken);
if (result.PackageDirectory is null || result.SemanticVersion is null)
{
logger.Warning("Milou.Deployer failed");
return(ExitCode.Failure);
}
}
ClearTemporaryDirectoriesAndFiles(deploymentTask);
return(ExitCode.Success);
}