private async Task <int> InvokeMSBuildAsync(IRemoteOperationExecutionContext context, string arguments, string workingDirectory)
{
var msbuildLoggerPath = Path.Combine(
Path.GetDirectoryName(typeof(BuildMSBuildProjectOperation).Assembly.Location),
"BmBuildLogger.dll"
);
var allArgs = $"\"/logger:{msbuildLoggerPath}\" /noconsolelogger " + arguments;
var msBuildPath = await this.GetMSBuildToolsPath(context).ConfigureAwait(false);
if (msBuildPath == null)
{
return(-1);
}
msBuildPath = Path.Combine(msBuildPath, "msbuild.exe");
var startInfo = new RemoteProcessStartInfo
{
FileName = msBuildPath,
Arguments = allArgs,
WorkingDirectory = workingDirectory
};
this.LogDebug("Process: " + startInfo.FileName);
this.LogDebug("Arguments: " + startInfo.Arguments);
this.LogDebug("Working directory: " + startInfo.WorkingDirectory);
return(await this.ExecuteCommandLineAsync(context, startInfo).ConfigureAwait(false));
}
private async Task <SvnClientExecutionResult> ExecuteCommandLineAsync(SvnArgumentBuilder args)
{
args.Append("--non-interactive");
args.Append("--trust-server-cert");
if (!string.IsNullOrEmpty(this.userName))
{
args.Append("--username");
args.AppendQuoted(this.userName);
}
if (this.password != null)
{
args.Append("--password");
args.AppendSensitive(AH.Unprotect(this.password));
}
var startInfo = new RemoteProcessStartInfo
{
FileName = this.svnExePath,
Arguments = args.ToString()
};
this.log.LogDebug("Working directory: " + startInfo.WorkingDirectory);
this.log.LogDebug("Executing: " + startInfo.FileName + " " + args.ToSensitiveString());
var execOps = this.execOps.Value;
using (var process = execOps.CreateProcess(startInfo))
{
var outputLines = new List <string>();
var errorLines = new List <string>();
process.OutputDataReceived += (s, e) => { if (e?.Data != null)
{
outputLines.Add(e.Data);
}
};
process.ErrorDataReceived += (s, e) => { if (e?.Data != null)
{
errorLines.Add(e.Data);
}
};
process.Start();
await process.WaitAsync(this.cancellationToken).ConfigureAwait(false);
return(new SvnClientExecutionResult(process.ExitCode ?? -1, outputLines, errorLines));
}
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
this.serverName = context.ServerName;
var execOps = await context.Agent.GetServiceAsync <IRemoteProcessExecuter>();
var makeStartInfo = new RemoteProcessStartInfo
{
FileName = "dfhack-make",
Arguments = this.Target.EscapeLinuxArg(),
WorkingDirectory = context.WorkingDirectory
};
if (this.UseNinja)
{
makeStartInfo.EnvironmentVariables["DFHACK_USE_NINJA"] = "1";
}
var cidfile = await this.LogAndWrapCommandAsync(context, makeStartInfo);
using (var make = execOps.CreateProcess(makeStartInfo))
{
this.progress = new OperationProgress(0);
make.OutputDataReceived += this.OutputDataReceived;
make.ErrorDataReceived += this.ErrorDataReceived;
make.Start();
using (ManageContainerIDFile(context, cidfile))
{
await make.WaitAsync(context.CancellationToken);
}
var processName = this.UseNinja ? "ninja" : "make";
if (make.ExitCode == 0)
{
this.LogDebug($"{processName} exited with code 0 (success)");
}
else if (make.ExitCode.HasValue)
{
this.LogError($"{processName} exited with code {make.ExitCode} (failure)");
}
else
{
this.LogError($"{processName} exited with unknown code");
}
}
}
private async Task <string> FindVsTestConsoleWithVsWhereAsync(IOperationExecutionContext context)
{
var remoteMethodEx = await context.Agent.GetServiceAsync <IRemoteMethodExecuter>().ConfigureAwait(false);
string vsWherePath = await remoteMethodEx.InvokeFuncAsync(RemoteGetVsWherePath).ConfigureAwait(false);
string outputFile = await remoteMethodEx.InvokeFuncAsync(Path.GetTempFileName).ConfigureAwait(false);
// vswhere.exe documentation: https://github.com/Microsoft/vswhere/wiki
// component IDs documented here: https://docs.microsoft.com/en-us/visualstudio/install/workload-and-component-ids
var startInfo = new RemoteProcessStartInfo
{
FileName = vsWherePath,
WorkingDirectory = Path.GetDirectoryName(vsWherePath),
Arguments = @"-products * -nologo -format xml -utf8 -latest -sort -requiresAny -requires Microsoft.VisualStudio.PackageGroup.TestTools.Core Microsoft.VisualStudio.Component.TestTools.BuildTools -find **\vstest.console.exe",
OutputFileName = outputFile
};
this.LogDebug("Process: " + startInfo.FileName);
this.LogDebug("Arguments: " + startInfo.Arguments);
this.LogDebug("Working directory: " + startInfo.WorkingDirectory);
await this.ExecuteCommandLineAsync(context, startInfo).ConfigureAwait(false);
var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>().ConfigureAwait(false);
XDocument xdoc;
using (var file = await fileOps.OpenFileAsync(outputFile, FileMode.Open, FileAccess.Read).ConfigureAwait(false))
{
xdoc = XDocument.Load(file);
}
var files = from f in xdoc.Root.Descendants("file")
let file = f.Value
select file;
var filePath = files.FirstOrDefault();
if (string.IsNullOrWhiteSpace(filePath))
{
return(null);
}
return(filePath);
}
public static async Task <string> WrapInBuildEnvAsync(this RemoteProcessStartInfo info, IOperationExecutionContext context, string imageName, bool shareCache, bool allowNetwork = false, bool forceASLR = true, params string[] additionalPaths)
{
var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>();
var executionBaseDir = await context.GetExecutionBaseDirAsync();
var volumes = string.Join(" ",
new[] { executionBaseDir }.Concat(additionalPaths)
.Select(s => fileOps.CombinePath(info.WorkingDirectory, s))
.Select(s => "-v " + $"{s}:{s.Replace($"_E{context.ExecutionId}", "_E0")}".EscapeLinuxArg())
.Concat(new[]
{
"-v " + string.Format("{0}:{0}:ro", (await DFHackCacheVariableFunction.GetAsync(context))).EscapeLinuxArg(),
shareCache ? "-v \"/home/buildmaster/.ccache:/home/buildmaster/.ccache\"" : $"-v \"/home/buildmaster/.ccache:/home/buildmaster/.ccache:ro\" -e CCACHE_READONLY=1 -e CCACHE_TEMPDIR=\"/home/buildmaster/.ccache_temp\" -v {fileOps.CombinePath(executionBaseDir, ".ccache").EscapeLinuxArg()}\":/home/buildmaster/.ccache_temp\""
})
);
var network = allowNetwork ? string.Empty : "--network none";
var security = string.Empty;
if (!forceASLR)
{
// Store seccomp.json outside of the execution directory root so the image cannot modify it.
var seccompPath = fileOps.CombinePath(executionBaseDir, "..", "docker-seccomp.json");
using (var output = await fileOps.OpenFileAsync(seccompPath, FileMode.Create, FileAccess.Write))
using (var input = typeof(Utils).Assembly.GetManifestResourceStream("Inedo.Extensions.DFHack.docker-seccomp.json"))
{
await input.CopyToAsync(output);
}
security = "--security-opt seccomp=" + seccompPath.EscapeLinuxArg();
}
var env = string.Join(" ", info.EnvironmentVariables
.Concat(new[] { new KeyValuePair <string, string>("CCACHE_BASEDIR", executionBaseDir.Replace($"_E{context.ExecutionId}", "_E0")) })
.Select(kv => $"-e {kv.Key.EscapeLinuxArg()}={kv.Value.EscapeLinuxArg()}"));
info.EnvironmentVariables.Clear();
var cidfile = fileOps.CombinePath(executionBaseDir, "cid-" + Guid.NewGuid().ToString("N"));
info.Arguments = $"run --rm {env} {volumes} {network} {security} --cidfile {cidfile.EscapeLinuxArg()} -w {info.WorkingDirectory.Replace($"_E{context.ExecutionId}", "_E0").EscapeLinuxArg()} -e CCACHE_DIR=\"/home/buildmaster/.ccache\" -u $(id -u):$(id -g) {imageName.EscapeLinuxArg()} {info.FileName.EscapeLinuxArg()} {info.Arguments}";
info.FileName = "/usr/bin/docker";
return(cidfile);
}
private async Task <ProcessResults> ExecuteCommandLineAsync(GitArgumentsBuilder args, string workingDirectory, bool throwOnFailure = true)
{
var startInfo = new RemoteProcessStartInfo
{
FileName = this.gitExePath,
Arguments = args.ToString(),
WorkingDirectory = workingDirectory ?? await this.fileOps.GetBaseWorkingDirectoryAsync()
};
if (this.repository.HasLocalRepository)
{
this.log.LogDebug("Ensuring local repository path exists...");
await this.fileOps.CreateDirectoryAsync(startInfo.WorkingDirectory).ConfigureAwait(false);
}
this.log.LogDebug("Working directory: " + startInfo.WorkingDirectory);
this.log.LogDebug("Executing: " + startInfo.FileName + " " + args.ToSensitiveString());
using (var process = this.processExecuter.CreateProcess(startInfo))
{
var outputLines = new List <string>();
var errorLines = new List <string>();
process.OutputDataReceived += (s, e) => { if (e?.Data != null)
{
outputLines.Add(e.Data);
}
};
process.ErrorDataReceived += (s, e) => { if (e?.Data != null)
{
errorLines.Add(e.Data);
}
};
process.Start();
await process.WaitAsync(this.cancellationToken).ConfigureAwait(false);
if (throwOnFailure && process.ExitCode != 0)
{
throw new ExecutionFailureException($"git returned error code {process.ExitCode}\n{string.Join("\n", errorLines)}");
}
return(new ProcessResults(process.ExitCode ?? -1, outputLines, errorLines));
}
}
private async Task <string> FindMSBuildPathUsingVSWhereAsync(IRemoteOperationExecutionContext context)
{
this.LogDebug("$MSBuildToolsPath variable is not set. Attempting to find the path to the latest version using vswhere.exe...");
string vsWherePath = PathEx.Combine(
Path.GetDirectoryName(typeof(BuildMSBuildProjectOperation).Assembly.Location),
"vswhere.exe"
);
string outputFile = Path.GetTempFileName();
// vswhere.exe documentation: https://github.com/Microsoft/vswhere/wiki
// component IDs documented here: https://docs.microsoft.com/en-us/visualstudio/install/workload-and-component-ids
var startInfo = new RemoteProcessStartInfo
{
FileName = vsWherePath,
WorkingDirectory = Path.GetDirectoryName(vsWherePath),
Arguments = @"-products * -nologo -format xml -utf8 -latest -sort -requires Microsoft.Component.MSBuild -find **\MSBuild.exe",
OutputFileName = outputFile
};
this.LogDebug("Process: " + startInfo.FileName);
this.LogDebug("Arguments: " + startInfo.Arguments);
this.LogDebug("Working directory: " + startInfo.WorkingDirectory);
await this.ExecuteCommandLineAsync(context, startInfo).ConfigureAwait(false);
var xdoc = XDocument.Load(outputFile);
var files = from f in xdoc.Root.Descendants("file")
let file = f.Value
// prefer 32-bit MSBuild
orderby file.IndexOf("amd64", StringComparison.OrdinalIgnoreCase) > -1 ? 1 : 0
select file;
var filePath = files.FirstOrDefault();
if (string.IsNullOrWhiteSpace(filePath))
{
return(null);
}
return(Path.GetDirectoryName(filePath));
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var startInfo = new RemoteProcessStartInfo
{
FileName = this.PythonExePath,
Arguments = "-m pip install --progress-bar=off --no-color",
WorkingDirectory = context.WorkingDirectory
};
if (this.InstallFromRequirements)
{
startInfo.Arguments += " -r requirements.txt";
}
startInfo.Arguments += AH.ConcatNE(" ", this.AdditionalArguments);
await this.WrapInVirtualEnv(context, startInfo);
await this.ExecuteCommandLineAsync(context, startInfo);
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var startInfo = new RemoteProcessStartInfo
{
FileName = this.PythonExePath,
Arguments = "-m pip list --local --not-required --format=json",
WorkingDirectory = context.WorkingDirectory
};
startInfo.Arguments += AH.ConcatNE(" ", this.AdditionalArguments);
await this.WrapInVirtualEnv(context, startInfo);
var procExec = await context.Agent.GetServiceAsync <IRemoteProcessExecuter>();
var output = new StringBuilder();
using (var process = procExec.CreateProcess(startInfo))
{
process.OutputDataReceived += (s, e) => output.Append(e.Data);
process.ErrorDataReceived += (s, e) => this.LogWarning(e.Data);
await process.WaitAsync(context.CancellationToken);
if (process.ExitCode != 0)
{
this.LogError($"Process exited with code {process.ExitCode}");
return;
}
}
var installedPackages = JsonConvert.DeserializeAnonymousType(output.ToString(), new[] { new { name = string.Empty, version = string.Empty } });
this.LogInformation("Installed packages:");
foreach (var package in installedPackages)
{
this.LogInformation($"{package.name} v{package.version}");
}
}
private async Task <ProcessResults> ExecuteCommandLineAsync(GitArgumentsBuilder args, string workingDirectory)
{
var startInfo = new RemoteProcessStartInfo
{
FileName = this.gitExePath,
Arguments = args.ToString(),
WorkingDirectory = workingDirectory
};
this.log.LogDebug("Ensuring local repository path exists...");
await this.fileOps.CreateDirectoryAsync(this.repository.LocalRepositoryPath).ConfigureAwait(false);
this.log.LogDebug("Working directory: " + startInfo.WorkingDirectory);
this.log.LogDebug("Executing: " + startInfo.FileName + " " + args.ToSensitiveString());
using (var process = this.processExecuter.CreateProcess(startInfo))
{
var outputLines = new List <string>();
var errorLines = new List <string>();
process.OutputDataReceived += (s, e) => { if (e?.Data != null)
{
outputLines.Add(e.Data);
}
};
process.ErrorDataReceived += (s, e) => { if (e?.Data != null)
{
errorLines.Add(e.Data);
}
};
process.Start();
await process.WaitAsync(this.cancellationToken).ConfigureAwait(false);
return(new ProcessResults(process.ExitCode ?? -1, outputLines, errorLines));
}
}
private async Task RunTestsAsync(IOperationExecutionContext context)
{
var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>();
var scriptsDir = fileOps.CombinePath(await fileOps.GetBaseWorkingDirectoryAsync(), "scripts");
await fileOps.CreateDirectoryAsync(scriptsDir);
var runnerFileName = fileOps.CombinePath(scriptsDir, $"BuildMasterTestRunner_{Guid.NewGuid():N}.py");
using (var fileStream = await fileOps.OpenFileAsync(runnerFileName, FileMode.Create, FileAccess.Write))
using (var stream = typeof(PyUnitOperation).Assembly.GetManifestResourceStream("Inedo.Extensions.Python.BuildMasterTestRunner.py"))
{
await stream.CopyToAsync(fileStream);
}
try
{
var startInfo = new RemoteProcessStartInfo
{
FileName = this.PythonExePath,
Arguments = $"{runnerFileName} {this.Arguments}{(this.Verbose ? " -v" : string.Empty)}{(this.FailFast ? " -f" : string.Empty)}{(this.RecordOutput ? " -b" : string.Empty)}",
WorkingDirectory = context.WorkingDirectory
};
await this.WrapInVirtualEnv(context, startInfo);
var exit = await this.ExecuteCommandLineAsync(context, startInfo);
if (exit != 0)
{
this.LogError($"Exited with code {exit}");
}
}
finally
{
try { await fileOps.DeleteFileAsync(runnerFileName); } catch { }
}
}
internal static async Task WrapInVirtualEnv(ILogSink logger, IOperationExecutionContext context, RemoteProcessStartInfo startInfo, string pythonExePath, string virtualEnv)
{
if (string.IsNullOrEmpty(virtualEnv))
{
return;
}
var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>();
if (!await fileOps.DirectoryExistsAsync(virtualEnv))
{
var procExec = await context.Agent.GetServiceAsync <IRemoteProcessExecuter>();
logger.LogDebug($"Virtual environment in {virtualEnv} is not present. Attempting venv (Python 3.3+)...");
var success = false;
using (var process = procExec.CreateProcess(new RemoteProcessStartInfo
{
FileName = pythonExePath,
WorkingDirectory = context.WorkingDirectory,
Arguments = "-m venv -- " + virtualEnv,
}))
{
process.OutputDataReceived += (s, e) => logger.LogDebug("(venv) " + e.Data);
process.ErrorDataReceived += (s, e) => logger.LogDebug("(venv) " + e.Data);
await process.WaitAsync(context.CancellationToken);
success = process.ExitCode == 0;
}
if (!success)
{
logger.LogDebug("Attempting virtualenv (any Python version, but requires separate installation)...");
using var process = procExec.CreateProcess(
new RemoteProcessStartInfo
{
FileName = "virtualenv",
WorkingDirectory = context.WorkingDirectory,
Arguments = "-- " + virtualEnv,
EnvironmentVariables =
{
["VIRTUALENV_PYTHON"] = pythonExePath
}
}
);
process.OutputDataReceived += (s, e) => logger.LogDebug("(virtualenv) " + e.Data);
process.ErrorDataReceived += (s, e) => logger.LogDebug("(virtualenv) " + e.Data);
await process.WaitAsync(context.CancellationToken);
success = process.ExitCode == 0;
}
if (!success)
{
throw new ExecutionFailureException("Could not create a virtual environment. See debug logs from this operation for more information.");
}
}
startInfo.FileName = (await fileOps.GetFileInfoAsync(fileOps.CombinePath(virtualEnv, "bin", "python"))).FullName;
}
protected Task WrapInVirtualEnv(IOperationExecutionContext context, RemoteProcessStartInfo startInfo) => WrapInVirtualEnv(this, context, startInfo, this.PythonExePath, this.VirtualEnv);
protected virtual async Task RunDockerComposeAsync(IOperationExecutionContext context, IEnumerable <string> args)
{
var fileOps = await context.Agent.TryGetServiceAsync <ILinuxFileOperationsExecuter>() ?? await context.Agent.GetServiceAsync <IFileOperationsExecuter>();
var procExec = await context.Agent.GetServiceAsync <IRemoteProcessExecuter>();
var workingDirectory = context.ResolvePath(string.IsNullOrWhiteSpace(this.WorkingDirectory) ? context.WorkingDirectory : this.WorkingDirectory);
this.LogDebug($"Working directory: {workingDirectory}");
await fileOps.CreateDirectoryAsync(workingDirectory);
var isYamlPathSpecified = this.ComposeFileYaml.EndsWith(".yml", StringComparison.OrdinalIgnoreCase);
string composeFileName = null;
if (isYamlPathSpecified)
{
composeFileName = context.ResolvePath(this.ComposeFileYaml, workingDirectory);
}
else
{
await fileOps.CreateDirectoryAsync(fileOps.CombinePath(workingDirectory, "scripts"));
composeFileName = fileOps.CombinePath(workingDirectory, "scripts", Guid.NewGuid().ToString("N") + ".yml");
}
var startInfo = new RemoteProcessStartInfo
{
FileName = "docker-compose",
WorkingDirectory = workingDirectory
};
startInfo.AppendArgs(procExec, new[]
{
"--file",
composeFileName,
"--project-name",
this.ProjectName,
this.Verbose ? "--verbose" : null,
"--no-ansi",
this.Command
}
.Concat(this.AddArgs ?? new string[0])
.Concat(args ?? new string[0])
.Where(arg => arg != null));
try
{
if (!isYamlPathSpecified)
{
await fileOps.WriteAllTextAsync(composeFileName, this.ComposeFileYaml);
}
this.LogDebug($"Running command: {startInfo.FileName} {startInfo.Arguments}");
int?exitCode;
using (var process = procExec.CreateProcess(startInfo))
{
process.OutputDataReceived += (s, e) => this.LogProcessOutput(e.Data);
process.ErrorDataReceived += (s, e) => this.LogProcessError(e.Data);
process.Start();
await process.WaitAsync(context.CancellationToken);
exitCode = process.ExitCode;
}
if (exitCode == 0)
{
this.LogInformation("Process exit code indicates success.");
return;
}
this.LogError($"Process exit code indicates failure. ({AH.CoalesceString(exitCode, "(unknown)")})");
}
finally
{
if (!isYamlPathSpecified)
{
await fileOps.DeleteFileAsync(composeFileName);
}
}
// Command failed. Try to give a better error message if docker-compose isn't even installed.
var verifyInstalledStartInfo = new RemoteProcessStartInfo
{
FileName = fileOps is ILinuxFileOperationsExecuter ? "/usr/bin/which" : "System32\\where.exe",
Arguments = procExec.EscapeArg(startInfo.FileName),
WorkingDirectory = workingDirectory
};
if (fileOps is ILinuxFileOperationsExecuter)
{
verifyInstalledStartInfo.Arguments = "-- " + verifyInstalledStartInfo.Arguments;
}
else
{
verifyInstalledStartInfo.FileName = fileOps.CombinePath(await procExec.GetEnvironmentVariableValueAsync("SystemRoot"), verifyInstalledStartInfo.FileName);
}
using (var process = procExec.CreateProcess(verifyInstalledStartInfo))
{
// Don't care about output.
process.Start();
await process.WaitAsync(context.CancellationToken);
// 0 = file exists, anything other than 0 or 1 = error trying to run which/where.exe
if (process.ExitCode == 1)
{
this.LogWarning("Is docker-compose installed and in the PATH?");
}
}
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
await this.LoginAsync(context, this.ContainerSource);
try
{
var procExec = await context.Agent.GetServiceAsync <IRemoteProcessExecuter>();
var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>();
await fileOps.CreateDirectoryAsync(context.WorkingDirectory);
var sourcePath = context.ResolvePath(this.SourceDirectory);
await fileOps.CreateDirectoryAsync(sourcePath);
var dockerfilePath = fileOps.CombinePath(sourcePath, "Dockerfile");
if (!string.IsNullOrWhiteSpace(this.DockerfileTemplate))
{
var item = SDK.GetRaftItem(RaftItemType.TextTemplate, this.DockerfileTemplate, context);
if (item == null)
{
this.LogError($"Text template \"{this.DockerfileTemplate}\" not found.");
return;
}
var dockerfileText = await context.ApplyTextTemplateAsync(item.Content, this.TemplateArguments != null?new Dictionary <string, RuntimeValue>(this.TemplateArguments) : null);
await fileOps.WriteAllTextAsync(dockerfilePath, dockerfileText, InedoLib.UTF8Encoding);
}
var containerSource = (ContainerSource)SecureResource.Create(this.ContainerSource, (IResourceResolutionContext)context);
var containerId = new ContainerId(this.ContainerSource, containerSource?.RegistryPrefix, this.RepositoryName, this.Tag);
var escapeArg = GetEscapeArg(context);
var args = $"build --force-rm --progress=plain --tag={escapeArg(containerId.FullName)} {this.AdditionalArguments} {escapeArg(sourcePath)}";
this.LogDebug("Executing docker " + args);
var startInfo = new RemoteProcessStartInfo
{
FileName = this.DockerExePath,
Arguments = args,
WorkingDirectory = sourcePath
};
using (var process = procExec.CreateProcess(startInfo))
{
process.OutputDataReceived += (s, e) => this.LogInformation(e.Data);
process.ErrorDataReceived += (s, e) => this.LogBuildError(context, e.Data);
process.Start();
await process.WaitAsync(context.CancellationToken);
if (process.ExitCode != 0)
{
this.LogError($"exit code: {process.ExitCode ?? -1}");
return;
}
}
var digest = await this.ExecuteGetDigest(context, containerId.FullName);
containerId = containerId.WithDigest(digest);
if (!string.IsNullOrEmpty(this.ContainerSource))
{
await this.PushAsync(context, containerId);
if (this.RemoveAfterPush)
{
this.LogDebug("Removing local image after successful push...");
await this.RemoveAsync(context, containerId);
}
}
if (this.AttachToBuild)
{
await this.AttachToBuildAsync(context, containerId);
}
}
finally
{
await this.LogoutAsync(context, this.ContainerSource);
}
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var execOps = await context.Agent.GetServiceAsync <IRemoteProcessExecuter>();
var fileOps = await context.Agent.GetServiceAsync <ILinuxFileOperationsExecuter>();
var bits = AH.Switch <BuildArchitecture, int>(this.Architecture)
.Case(BuildArchitecture.i386, 32)
.Case(BuildArchitecture.x86_64, 64)
.End();
await fileOps.CreateDirectoryAsync(context.WorkingDirectory);
var cmakeStartInfo = new RemoteProcessStartInfo
{
FileName = "dfhack-configure",
Arguments = $"{this.OperatingSystem.ToString().ToLowerInvariant()} {bits} {this.BuildType} {this.SourcePath.EscapeLinuxArg()} -DCMAKE_INSTALL_PREFIX={this.InstallPrefix.EscapeLinuxArg()} -DBUILD_SUPPORTED={(this.IncludeSupported ? 1 : 0)} -DBUILD_DEVEL=0 -DBUILD_DEV_PLUGINS=0 -DBUILD_DOCS={(this.IncludeDocumentation ? 1 : 0)} -DBUILD_STONESENSE={(this.IncludeStonesense ? 1 : 0)}{string.Join("", this.AdditionalArgs.Select(a => " " + a.EscapeLinuxArg()))}",
WorkingDirectory = context.WorkingDirectory
};
if (this.UseNinja)
{
cmakeStartInfo.EnvironmentVariables["DFHACK_USE_NINJA"] = "1";
}
var cidfile = await cmakeStartInfo.WrapInBuildEnvAsync(context, this.BuildEnv + ":" + this.ImageTag, this.TrustedBuild);
this.LogDebug($"Running in directory: {cmakeStartInfo.WorkingDirectory}");
this.LogDebug($"Executing command: {cmakeStartInfo.FileName} {cmakeStartInfo.Arguments}");
using (var cmake = execOps.CreateProcess(cmakeStartInfo))
{
cmake.OutputDataReceived += (s, e) =>
{
this.LogInformation(e.Data);
};
bool isError = false;
cmake.ErrorDataReceived += (s, e) =>
{
var line = this.RemoveLogRubbish(e);
if (line == null)
{
return;
}
if (line.StartsWith("* "))
{
this.LogDebug(line);
isError = false;
return;
}
if (e.Data.Contains("Error"))
{
isError = true;
}
else if (string.IsNullOrWhiteSpace(line))
{
isError = false;
}
if (isError)
{
this.LogError(line);
}
else
{
this.LogWarning(line);
}
};
cmake.Start();
using (ManageContainerIDFile(context, cidfile))
{
await cmake.WaitAsync(context.CancellationToken);
}
if (cmake.ExitCode == 0)
{
this.LogDebug("cmake exited with code 0 (success)");
}
else if (cmake.ExitCode.HasValue)
{
this.LogError($"cmake exited with code {cmake.ExitCode} (failure)");
}
else
{
this.LogError("cmake exited with unknown code");
}
}
}
internal static async Task RunKubeCtlAsync(this ILogSink log, IOperationExecutionContext context, IEnumerable <string> args, Action <string> logOutput = null, Action <string> logError = null, Func <int?, bool> handleExit = null, CancellationToken?cancellationToken = null)
{
var fileOps = await context.Agent.TryGetServiceAsync <ILinuxFileOperationsExecuter>() ?? await context.Agent.GetServiceAsync <IFileOperationsExecuter>();
var procExec = await context.Agent.GetServiceAsync <IRemoteProcessExecuter>();
var escapeArg = fileOps is ILinuxFileOperationsExecuter ? (Func <string, string>)Utils.EscapeLinuxArg : Utils.EscapeWindowsArg;
var startInfo = new RemoteProcessStartInfo
{
FileName = "kubectl",
Arguments = string.Join(" ", (args ?? new string[0]).Where(arg => arg != null).Select(escapeArg)),
WorkingDirectory = context.WorkingDirectory
};
log.LogDebug($"Working directory: {startInfo.WorkingDirectory}");
await fileOps.CreateDirectoryAsync(startInfo.WorkingDirectory);
log.LogDebug($"Running command: {escapeArg(startInfo.FileName)} {startInfo.Arguments}");
int?exitCode;
using (var process = procExec.CreateProcess(startInfo))
{
process.OutputDataReceived += (s, e) => (logOutput ?? log.LogInformation)(e.Data);
process.ErrorDataReceived += (s, e) => (logError ?? log.LogError)(e.Data);
process.Start();
await process.WaitAsync(cancellationToken ?? context.CancellationToken);
exitCode = process.ExitCode;
}
if (handleExit?.Invoke(exitCode) == true)
{
return;
}
if (exitCode == 0)
{
log.LogInformation("Process exit code indicates success.");
return;
}
log.LogError($"Process exit code indicates failure. ({AH.CoalesceString(exitCode, "(unknown)")})");
// Command failed. Try to give a better error message if kubectl isn't even installed.
var verifyInstalledStartInfo = new RemoteProcessStartInfo
{
FileName = fileOps is ILinuxFileOperationsExecuter ? "/usr/bin/which" : "System32\\where.exe",
Arguments = escapeArg(startInfo.FileName),
WorkingDirectory = context.WorkingDirectory
};
if (fileOps is ILinuxFileOperationsExecuter)
{
verifyInstalledStartInfo.Arguments = "-- " + verifyInstalledStartInfo.Arguments;
}
else
{
verifyInstalledStartInfo.FileName = PathEx.Combine(await procExec.GetEnvironmentVariableValueAsync("SystemRoot"), verifyInstalledStartInfo.FileName);
}
using (var process = procExec.CreateProcess(verifyInstalledStartInfo))
{
// Don't care about output.
process.Start();
await process.WaitAsync(cancellationToken ?? context.CancellationToken);
// 0 = file exists, anything other than 0 or 1 = error trying to run which/where.exe
if (process.ExitCode == 1)
{
log.LogWarning("Is kubectl installed and in the PATH?");
}
}
}
public IRemoteProcess CreateProcess(RemoteProcessStartInfo startInfo) => new LocalProcess(startInfo);
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var execOps = await context.Agent.GetServiceAsync <IRemoteProcessExecuter>();
var bits = AH.Switch <BuildArchitecture, int>(this.Architecture)
.Case(BuildArchitecture.i386, 32)
.Case(BuildArchitecture.x86_64, 64)
.End();
var testStartInfo = new RemoteProcessStartInfo
{
FileName = "dfhack-test",
Arguments = $"{this.OperatingSystem.ToString().ToLowerInvariant()} {bits}",
WorkingDirectory = context.WorkingDirectory
};
var cidfile = await this.LogAndWrapCommandAsync(context, testStartInfo, false, false);
var testLines = new List <string>();
var recorder = await context.TryGetServiceAsync <IUnitTestRecorder>();
Task recorderTask = InedoLib.NullTask;
using (var test = execOps.CreateProcess(testStartInfo))
{
var lastTime = DateTimeOffset.Now;
test.OutputDataReceived += (s, e) =>
{
string text = e.Data;
if (this.ImageTag == "msvc")
{
text = e.Data.TrimEnd();
if (text == @"wine: cannot find L""C:\\windows\\Microsoft.NET\\Framework\\v4.0.30319\\mscorsvw.exe""")
{
return;
}
}
testLines.Add(text);
if (text.StartsWith("Running ") && text.EndsWith(" tests"))
{
testLines.Clear();
lastTime = DateTimeOffset.Now;
}
if (text.StartsWith("ERROR: "))
{
this.LogError(text.Substring("ERROR: ".Length));
return;
}
if (text.StartsWith("WARN: "))
{
this.LogWarning(text.Substring("WARN: ".Length));
return;
}
if (text.StartsWith("warning: "))
{
this.LogWarning(text.Substring("warning: ".Length));
return;
}
if (text.StartsWith("test passed: "))
{
var testName = text.Substring("test passed: ".Length);
recordUnitTest(testName, UnitTestStatus.Passed, ref lastTime);
}
this.LogInformation(text);
};
test.ErrorDataReceived += (s, e) =>
{
string text = e.Data;
if (this.ImageTag == "msvc")
{
text = e.Data.TrimEnd();
if (text == @"wine: cannot find L""C:\\windows\\Microsoft.NET\\Framework\\v4.0.30319\\mscorsvw.exe""")
{
return;
}
}
testLines.Add(text);
if (text.StartsWith("Plugin ") && text.Contains(" is missing required globals: "))
{
this.LogWarning(text);
return;
}
if (text.StartsWith("test failed: "))
{
var testName = text.Substring("test failed: ".Length);
recordUnitTest(testName, UnitTestStatus.Failed, ref lastTime);
}
if (text.StartsWith("test errored: "))
{
var testName = string.Join(":", text.Substring("test errored: ".Length).Split(new[] { ':' }, 3).Take(2));
recordUnitTest(testName, UnitTestStatus.Failed, ref lastTime);
}
this.LogError(text);
};
test.Start();
using (ManageContainerIDFile(context, cidfile))
{
await test.WaitAsync(context.CancellationToken);
}
await recorderTask;
if (test.ExitCode != 0)
{
this.LogError("Tests failed!");
}
void recordUnitTest(string testName, UnitTestStatus testStatus, ref DateTimeOffset now)
{
var splitName = testName.Split(new[] { ':' }, 2);
var groupName = splitName[0];
if (splitName.Length > 1)
{
testName = splitName[1];
}
var testResult = string.Join("\n", testLines);
testLines.Clear();
var prevRecorderTask = recorderTask;
var endTime = DateTimeOffset.Now;
var startTime = now;
now = endTime;
recorderTask = Task.Run(async() =>
{
await prevRecorderTask;
await recorder.RecordUnitTestAsync(groupName, testName, testStatus, testResult, startTime, endTime);
});
}
}
}
protected async Task <string> LogAndWrapCommandAsync(IOperationExecutionContext context, RemoteProcessStartInfo info, bool allowNetwork = true, bool forceASLR = true)
{
this.LogDebug($"Running in directory: {info.WorkingDirectory}");
this.LogDebug($"Executing command: {info.FileName} {info.Arguments}");
var cidfile = await info.WrapInBuildEnvAsync(context, this.BuildEnv + ":" + this.ImageTag, this.TrustedBuild, allowNetwork, forceASLR);
this.LogDebug($"Full build-env command line: {info.FileName} {info.Arguments}");
return(cidfile);
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
var sourceDirectory = context.ResolvePath(this.SourceDirectory);
this.LogDebug($"Signing files in {sourceDirectory}...");
var fileOps = context.Agent.GetService <IFileOperationsExecuter>();
var matches = fileOps.GetFileSystemInfos(sourceDirectory, new MaskingContext(this.Includes, this.Excludes))
.OfType <SlimFileInfo>()
.ToList();
if (matches.Count == 0)
{
this.LogWarning("No files found which match the specified criteria.");
return;
}
var args = new StringBuilder("sign /sm");
args.Append($" /n \"{this.SubjectName}\"");
if (!string.IsNullOrEmpty(this.TimestampServer))
{
args.Append($" /t \"{this.TimestampServer}\"");
}
if (!string.IsNullOrEmpty(this.ContentDescription))
{
args.Append($" /d \"{this.ContentDescription}\"");
}
if (!string.IsNullOrEmpty(this.ContentUrl))
{
args.Append($" /du \"{this.ContentUrl}\"");
}
var signToolPath = this.GetSignToolPath(context.Agent);
if (signToolPath != null)
{
if (!fileOps.FileExists(signToolPath))
{
this.LogError("Cannot find signtool.exe at: " + signToolPath);
return;
}
foreach (var match in matches)
{
var startInfo = new RemoteProcessStartInfo
{
FileName = signToolPath,
Arguments = args + " \"" + match.FullName + "\"",
WorkingDirectory = sourceDirectory
};
this.LogInformation($"Signing {match.FullName}...");
int exitCode = await this.ExecuteCommandLineAsync(context, startInfo);
if (exitCode != 0)
{
this.LogError("Signtool.exe returned exit code " + exitCode);
}
}
}
}
IRemoteProcess IRemoteProcessExecuter.CreateProcess(RemoteProcessStartInfo startInfo) => new LocalProcess(startInfo);
public override async Task ConfigureAsync(IOperationExecutionContext context)
{
var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>();
var execOps = await context.Agent.GetServiceAsync <IRemoteProcessExecuter>();
var startInfo = new RemoteProcessStartInfo();
var collected = await this.CollectAsync(context);
if (collected.Version == "not-installed")
{
this.LogDebug("Installing Chocolatey...");
startInfo.FileName = fileOps.CombinePath(await execOps.GetEnvironmentVariableValueAsync("SystemRoot"), @"System32\WindowsPowerShell\v1.0\powershell.exe");
startInfo.Arguments = @"-NoProfile -InputFormat None -ExecutionPolicy AllSigned -Command ""iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))""";
bool specificVersion = !string.IsNullOrEmpty(this.Template.Version) && !string.Equals(this.Template.Version, "latest", StringComparison.OrdinalIgnoreCase);
if (!string.IsNullOrEmpty(this.Template.Source))
{
if (!specificVersion)
{
var client = PackageRepositoryFactory.Default.CreateRepository(this.Template.Source);
var package = client.FindPackage("chocolatey", (SemanticVersion)null, false, false);
this.Template.Version = package.Version.ToOriginalString();
}
startInfo.EnvironmentVariables["chocolateyDownloadUrl"] = PathEx.Combine(this.Template.Source, "package", "chocolatey", this.Template.Version);
}
else if (specificVersion)
{
startInfo.EnvironmentVariables["chocolateyVersion"] = this.Template.Version;
}
if (context.Simulation)
{
return;
}
}
else if (this.Compare(collected).AreEqual)
{
this.LogDebug("No action needed.");
return;
}
else
{
this.LogDebug("Upgrading Chocolatey...");
var buffer = new StringBuilder(200);
buffer.Append("upgrade --yes --fail-on-unfound ");
if (context.Simulation)
{
buffer.Append("--what-if ");
}
if (!string.IsNullOrEmpty(this.Template.Version) && !string.Equals(this.Template.Version, "latest", StringComparison.OrdinalIgnoreCase))
{
buffer.Append("--version \"");
buffer.Append(this.Template.Version);
buffer.Append("\" ");
buffer.Append("--allow-downgrade ");
}
if (!string.IsNullOrEmpty(this.Template.Source))
{
buffer.Append("--source \"");
buffer.Append(this.Template.Source);
buffer.Append("\" ");
}
buffer.Append("chocolatey");
}
await this.ExecuteCommandLineAsync(context, startInfo);
}
