ProcessStartData CreateTailLogsProcessStartData(SshTarget target, uint remotePid)
{
// If the game process exits, give the tail process a chance to shut down gracefully.
ProcessManager.ProcessStopHandler stopHandler = (process, reason) =>
{
// Did the game process, i.e. the process with pid |remotePid|, exit?
if (reason != ExitReason.ProcessExited &&
reason != ExitReason.DebuggerTerminated)
{
Trace.WriteLine("Game process did not exit, won't wait for tail process exit");
return;
}
// Give it a second to finish.
bool exited = process.WaitForExit(TimeSpan.FromSeconds(1));
// Emit a log message to help tracking down issues, just in case.
Trace.WriteLine($"Tail process {(exited ? "exited" : "did not exit")} gracefully");
};
var startInfo = ProcessStartInfoBuilder.BuildForSsh(
$"tail --pid={remotePid} -n +0 -F -q /var/game/stdout /var/game/stderr",
new List <string>(), target);
return(new ProcessStartData("output tail", startInfo, monitorExit: false,
outputToConsole: true, stopHandler: stopHandler));
}
/// <summary>
/// Check that the specified remote process's binary has a valid build id. Log messages and
/// record metrics to indicate the result of the checks.
/// </summary>
/// <param name="pid">Process ID of the remote process that we will check</param>
/// <param name="target">The machine that should have a valid remote binary</param>
public async Task CheckRemoteBinaryOnAttachAsync(uint pid, SshTarget target,
IAction action)
{
var remoteTargetPath = string.Format(PID_EXE_PATH_TEMPLATE, pid);
try
{
var dataRecorder =
new DataRecorder(action, DebugPreflightCheckData.Types.CheckType.AttachOnly);
BuildId remoteBuildId;
try
{
remoteBuildId = await binaryFileUtil.ReadBuildIdAsync(remoteTargetPath,
target);
}
catch (BinaryFileUtilException e) when(dataRecorder.RemoteBuildIdError(e))
{
Debug.Fail("Exception should never be caught");
throw;
}
// Log the remote Build ID for debugging purposes.
dataRecorder.ValidRemoveBuildId();
Trace.WriteLine("Remote build ID: " + remoteBuildId.ToString());
}
catch (BinaryFileUtilException e)
{
Trace.WriteLine($"Failed to read build ID for '{remoteTargetPath}' " +
$"on '{target.GetString()}': " + e.ToString());
throw new PreflightBinaryCheckerException(
ErrorStrings.FailedToCheckRemoteBuildIdWithExplanation(e.Message), e);
}
}
public void AttachToCoreWithEmptyTargetSucceeds()
{
int calls = 0;
SshTarget expectedTarget = null;
var attachReason = enum_ATTACH_REASON.ATTACH_REASON_LAUNCH;
IDebugSessionLauncher debugSessionLauncher =
CreateConfiguredDebugSessionLauncher(expectedTarget, x => {
calls++;
var attachedProgram = Substitute.For <ILldbAttachedProgram>();
return(Task.FromResult(attachedProgram));
});
IDebugSessionLauncherFactory debugSessionLauncherFactory =
CreateDebugSessionLauncherFactory(debugSessionLauncher);
IGgpDebugEngine debugEngine = CreateGgpDebugEngine(debugSessionLauncherFactory);
var debugPort = Substitute.For <IDebugPort2>();
debugEngine.LaunchSuspended("", debugPort, _exePath, null, null, null, null,
enum_LAUNCH_FLAGS.LAUNCH_DEBUG, 0, 0, 0, null,
out IDebugProcess2 _);
var rgpPrograms = new[] { Substitute.For <IDebugProgram2>() };
var rgpProgramNodes = new[] { Substitute.For <IDebugProgramNode2>() };
int result =
debugEngine.Attach(rgpPrograms, rgpProgramNodes, _celtPrograms, null, attachReason);
Assert.Multiple(() => {
debugPort.Received().GetProcess(Arg.Any <AD_PROCESS_ID>(), out _);
Assert.That(result, Is.EqualTo(VSConstants.S_OK));
Assert.That(calls, Is.EqualTo(1));
});
}
public void StartPreGame(LaunchOption launchOption, bool rgpEnabled, bool renderdocEnabled,
SshTarget target, out GrpcConnection grpcConnection,
out ITransportSession transportSession)
{
lock (thisLock)
{
grpcConnection = null;
this.transportSession = transportSession = transportSessionFactory.Create();
if (transportSession == null)
{
Trace.WriteLine("Unable to start the debug transport, invalid session.");
throw new YetiDebugTransportException(ErrorStrings.FailedToStartTransport);
}
if (!LaunchPreGameProcesses(launchOption, rgpEnabled, renderdocEnabled, target))
{
Stop(ExitReason.Unknown);
throw new YetiDebugTransportException(
"Failed to launch all needed pre-game processes");
}
Trace.WriteLine("Started debug transport. Session ID: " +
transportSession.GetSessionId());
// The grpcConnection is created during the launch of one of the processes.
grpcConnection = this.grpcConnection;
}
}
async Task DeployToTargetAsync(DataRecorder record, ICancelable task, SshTarget target,
string localPath, string remotePath, bool force = false)
{
Stopwatch stopwatch = Stopwatch.StartNew();
try
{
BinarySignatureCheck.Types.Result signatureCheck = force
? BinarySignatureCheck.Types.Result.AlwaysCopy
: BinarySignatureCheck.Types.Result.YesCopy;
record.SetCopyAttempted(true);
record.BinarySize(FileUtil.GetFileSize(localPath, _fileSystem));
record.SignatureCheckResult(signatureCheck);
record.DeploymentMode();
await _remoteFile.SyncAsync(target, localPath, remotePath, task, force);
record.CopyBinary(stopwatch.ElapsedMilliseconds, DataRecorder.NoError);
}
catch (ProcessException exception)
{
record.CopyBinary(stopwatch.ElapsedMilliseconds, exception);
throw new DeployException(
ErrorStrings.FailedToDeployExecutable(exception.Message),
exception);
}
}
public void StartPreGameLaunchAborted()
{
SshTarget sshTarget = new SshTarget(_targetString);
var mockProcess = Substitute.For <IProcess>();
ProcessStartInfo startInfo = null;
mockManagedProcessFactory
.Create(Arg.Is <ProcessStartInfo>(x => Path.GetFileName(x.FileName) ==
YetiConstants.DebuggerGrpcServerExecutable))
.Returns(mockProcess).AndDoes(x => { startInfo = x.Arg <ProcessStartInfo>(); });
yetiDebugTransport.StartPreGame(LaunchOption.LaunchGame, false, false, sshTarget, out _,
out _);
int exitCode = 123;
mockProcess.StartInfo.Returns(startInfo);
mockProcess.ExitCode.Returns(exitCode);
mockProcess.OnExit += Raise.EventWith(mockProcess, new EventArgs());
Assert.IsInstanceOf(typeof(ProcessExecutionException), abortError);
var processError = abortError as ProcessExecutionException;
Assert.AreEqual(exitCode, ((ProcessExecutionException)abortError).ExitCode);
}
/// <summary>
/// Launches all needed processes that can be launched before the game.
/// </summary>
/// <param name="launchOption">How the game will be launched</param>
/// <param name="rgpEnabled">Whether RPG is enabled</param>
/// <param name="renderdocEnabled">Whether Renderdoc is enabled</param>
/// <param name="target">Remote instance</param>
/// <returns>
/// True if all processes launched successfully and false otherwise and we should abort.
/// </returns>
bool LaunchPreGameProcesses(LaunchOption launchOption, bool rgpEnabled,
bool renderdocEnabled, SshTarget target)
{
var processes = new List <ProcessStartData>();
if (launchOption == LaunchOption.LaunchGame ||
launchOption == LaunchOption.AttachToGame)
{
processes.Add(CreatePortForwardingProcessStartData(target));
processes.Add(CreateLldbServerProcessStartData(target));
}
if (launchOption == LaunchOption.LaunchGame)
{
if (renderdocEnabled)
{
processes.Add(CreateRenderDocPortForwardingProcessStartData(target));
}
if (rgpEnabled)
{
processes.Add(CreateRgpPortForwardingProcessStartData(target));
}
}
processes.Add(CreateDebuggerGrpcServerProcessStartData());
return(LaunchProcesses(processes, "pre-game"));
}
ProcessStartData CreateLldbServerProcessStartData(SshTarget target)
{
string lldbServerCommand = string.Format(
"{0} platform --listen 127.0.0.1:{1} --min-gdbserver-port={2} " +
"--max-gdbserver-port={3}",
Path.Combine(YetiConstants.LldbServerLinuxPath,
YetiConstants.LldbServerLinuxExecutable),
transportSession.GetRemoteDebuggerPort(),
transportSession.GetReservedLocalAndRemotePort(),
transportSession.GetReservedLocalAndRemotePort() + 1);
List <string> lldbServerEnvironment = new List <string>();
if (yetiVSIService.DebuggerOptions[DebuggerOption.SERVER_LOGGING] ==
DebuggerOptionState.ENABLED)
{
string channels = "lldb default:posix default:gdb-remote default";
// gdb-server.log
lldbServerEnvironment.Add(
"LLDB_DEBUGSERVER_LOG_FILE=/usr/local/cloudcast/log/gdb-server.log");
lldbServerEnvironment.Add("LLDB_SERVER_LOG_CHANNELS=\\\"" + channels + "\\\"");
// lldb-server.log
lldbServerCommand += " --log-file=/usr/local/cloudcast/log/lldb-server.log " +
"--log-channels=\\\"" + channels + "\\\"";
}
var startInfo = ProcessStartInfoBuilder.BuildForSsh(
lldbServerCommand, lldbServerEnvironment, target);
return(new ProcessStartData("lldb server", startInfo));
}
// Queries the provided port (gamelet) for a list of running cores. On an error, a
// TransportException will be thrown with the error code.
public async Task <List <CoreListEntry> > GetCoreListAsync(SshTarget sshTarget)
{
// TODO: Find a more robust method of listing files on the gamelet.
ProcessStartInfo processStartInfo = ProcessStartInfoBuilder.BuildForSsh(
COMMAND, new List <string>(), sshTarget);
return(await GetCoreListFromProcessStartInfoAsync(processStartInfo));
}
public void StartPostGameLaunchNoCaptureOutput()
{
SshTarget sshTarget = new SshTarget(_targetString);
optionPageGrid.CaptureGameOutput.Returns(false);
yetiDebugTransport.StartPostGame(LaunchOption.AttachToGame, sshTarget, _remotePid);
Assert.IsNull(abortError);
}
public async Task <List <ProcessListEntry> > GetBySshAsync(SshTarget target)
{
using (var process = remoteProcessFactory.Create(
ProcessStartInfoBuilder.BuildForSsh(COMMAND, new List <string>(), target)))
{
return(await GetByProcessAsync(process));
}
}
public void StartPreGameAttachToCoreLocal()
{
SshTarget sshTargetNull = null;
yetiDebugTransport.StartPreGame(LaunchOption.AttachToCore, false, false, sshTargetNull,
out _, out _);
ExpectLocalProcessWithName(YetiConstants.DebuggerGrpcServerExecutable);
Assert.IsNull(abortError);
}
public void StartPostGameLaunchCaptureOutput()
{
SshTarget sshTarget = new SshTarget(_targetString);
optionPageGrid.CaptureGameOutput.Returns(true);
yetiDebugTransport.StartPostGame(LaunchOption.AttachToGame, sshTarget, _remotePid);
ExpectRemoteProcessWithArg($"tail --pid={_remotePid}", 1);
Assert.IsNull(abortError);
}
/// <summary>
/// Parses an elf binary or symbol file and returns the build ID encoded
/// in the .note.gnu.build-id section of the file.
/// </summary>
/// <param name="filepath">The local or remote absolute file path.</param>
/// <param name="target">Optional parameter specifying the remote gamelet.</param>
/// <returns>A non-empty build id.</returns>
/// <exception cref="BinaryFileUtilException">
/// Thrown when an error is encountered reading or parsing the build id.
/// InnerException contains more details.
/// </exception>
public async Task <BuildId> ReadBuildIdAsync(string filepath, SshTarget target = null)
{
try
{
var outputLines = await ReadSectionFromFileAsync(".note.gnu.build-id", filepath,
target);
var hexString = ParseHexDump(outputLines);
var result = ParseBuildIdOutput(hexString);
if (result == BuildId.Empty)
{
throw new InvalidBuildIdException(
ErrorStrings.FailedToReadBuildId(filepath,
ErrorStrings.EmptyBuildId));
}
return(result);
}
catch (ProcessExecutionException e)
{
LogObjdumpOutput(e);
// objdump returned an error code, possibly because the file being parsed is not
// actually an elf file. With an SSH target, exit code 255 means SSH failed before
// it had a chance to execute the remote command.
if (target != null && e.ExitCode < 255)
{
// The remote command failed, so we need to fix the exception message.
// TODO: ManagedProcess should report the remote filename.
throw new BinaryFileUtilException(
ErrorStrings.FailedToReadBuildId(
filepath, ErrorStrings.ProcessExitedWithErrorCode(
YetiConstants.ObjDumpLinuxExecutable, e.ExitCode)),
e);
}
else
{
throw new BinaryFileUtilException(
ErrorStrings.FailedToReadBuildId(filepath, e.Message), e);
}
}
catch (ProcessException e)
{
// objdump failed to launch, possibly because the SDK was not found. With an SSH
// target, this indicates that SSH failed to launch. In either case, the specific
// filepath was never accessed, so it is not part of the error.
throw new BinaryFileUtilException(
ErrorStrings.FailedToReadBuildId(e.Message), e);
}
catch (FormatException e)
{
// Indicates the build ID section is malformed.
throw new InvalidBuildIdException(
ErrorStrings.FailedToReadBuildId(
filepath, ErrorStrings.MalformedBuildID),
e);
}
}
public async Task GetAsync(SshTarget target, string file, string destination,
ICancelable task)
{
await ScpAsync(ProcessStartInfoBuilder.BuildForScpGet(file, target, destination),
ProcessManager.CreateForCancelableTask(task));
// Notify client if operation was cancelled.
task.ThrowIfCancellationRequested();
}
public async Task RunWithSuccessAsync(SshTarget target, string command)
{
var startInfo =
ProcessStartInfoBuilder.BuildForSsh(command, new List <string>(), target);
using (var process = remoteProcessFactory.Create(startInfo))
{
await process.RunToExitWithSuccessAsync();
}
}
public void SetUp()
{
var testGamelet = new Gamelet {
Id = TEST_GAMELET_ID, IpAddr = TEST_IP
};
sshTarget = new SshTarget(testGamelet);
managedProcessFactory = Substitute.For <ManagedProcess.Factory>();
coreListRequestFactory = new CoreListRequest.Factory(managedProcessFactory);
}
public void StartPreGameLaunchRenderDoc()
{
SshTarget sshTarget = new SshTarget(_targetString);
yetiDebugTransport.StartPreGame(LaunchOption.LaunchGame, false, true, sshTarget, out _,
out _);
ExpectRemoteProcessWithArg("lldb-server", 1);
ExpectRemoteProcessWithArg("-L", 2);
ExpectLocalProcessWithName(YetiConstants.DebuggerGrpcServerExecutable);
Assert.IsNull(abortError);
}
public async Task ReadBuildId_RemoteAsync(string buildIdStr, string[] outputLines)
{
mockRemoteProcess.When(x => x.RunToExitAsync()).Do(x =>
{
OutputTestData(mockRemoteProcess, outputLines);
});
SshTarget target = new SshTarget(fakeGameletIp + ":" + fakeGameletPort);
var buildId = await elfFileUtil.ReadBuildIdAsync(fakeRemoteFilename, target);
Assert.AreEqual(new BuildId(buildIdStr), buildId);
}
public void StartPostGame(LaunchOption launchOption, SshTarget target, uint remotePid)
{
lock (thisLock)
{
if (!LaunchPostGameProcesses(launchOption, target, remotePid))
{
Stop(ExitReason.Unknown);
throw new YetiDebugTransportException(
"Failed to launch all needed post-game processes");
}
}
}
public void ReadBuildId_Remote_ProcessException()
{
mockRemoteProcess.RunToExitAsync().Returns(
Task.FromException <int>(new ProcessException("test")));
SshTarget target = new SshTarget(fakeGameletIp + ":" + fakeGameletPort);
var ex = Assert.ThrowsAsync <BinaryFileUtilException>(
() => elfFileUtil.ReadBuildIdAsync(fakeRemoteFilename, target));
Assert.IsInstanceOf <ProcessException>(ex.InnerException);
// Note: message doesn't need to include remote filename.
}
public void StartPreGameNoSession()
{
SshTarget sshTargetNull = null;
mockMemoryMappedFileFactory.CreateNew(Arg.Any <string>(), Arg.Any <long>())
.Returns((IMemoryMappedFile)null);
Assert.Throws <YetiDebugTransportException>(
() => yetiDebugTransport.StartPreGame(LaunchOption.AttachToCore, false, false,
sshTargetNull, out _, out _));
// Early errors don't cause aborts.
Assert.IsNull(abortError);
}
/// <summary>
/// Launches all needed processes that can be launched after the game.
/// </summary>
/// <param name="launchOption">How the game will be launched</param>
/// <param name="target">Remote instance</param>
/// <param name="remotePid">Id of the remote process</param>
/// <returns>
/// True if all processes launched successfully and false otherwise and we should abort.
/// </returns>
bool LaunchPostGameProcesses(LaunchOption launchOption, SshTarget target, uint remotePid)
{
var processes = new List <ProcessStartData>();
if ((launchOption == LaunchOption.LaunchGame ||
launchOption == LaunchOption.AttachToGame) &&
yetiVSIService.Options.CaptureGameOutput)
{
processes.Add(CreateTailLogsProcessStartData(target, remotePid));
}
return(LaunchProcesses(processes, "post-game"));
}
ProcessStartData CreateRgpPortForwardingProcessStartData(SshTarget target)
{
var ports = new List <ProcessStartInfoBuilder.PortForwardEntry>()
{
new ProcessStartInfoBuilder.PortForwardEntry
{
LocalPort = WorkstationPorts.RGP_LOCAL,
RemotePort = WorkstationPorts.RGP_REMOTE,
}
};
var startInfo = ProcessStartInfoBuilder.BuildForSshPortForward(ports, target);
return(new ProcessStartData("rgp port forwarding", startInfo));
}
public void TailKilledIfOtherExitReason()
{
IProcess tailProcess = Substitute.For <IProcess>();
SshTarget sshTarget = new SshTarget(_targetString);
optionPageGrid.CaptureGameOutput.Returns(true);
mockManagedProcessFactory.Create(Arg.Any <ProcessStartInfo>(), Arg.Any <int>())
.Returns(tailProcess);
yetiDebugTransport.StartPostGame(LaunchOption.AttachToGame, sshTarget, _remotePid);
yetiDebugTransport.Stop(ExitReason.Unknown);
tailProcess.DidNotReceiveWithAnyArgs().WaitForExit(Arg.Any <TimeSpan>());
tailProcess.Received().Kill();
}
public async Task DeployLldbServerAsync(SshTarget target, Metrics.IAction action)
{
DataRecorder record = new DataRecorder(action, DataRecorder.File.LLDB_SERVER);
record.SetCopyAttempted(true);
string localLldbServerPath = GetLldbServerPath();
string remotePath = Path.Combine(YetiConstants.LldbServerLinuxPath,
YetiConstants.LldbServerLinuxExecutable);
await DeployToTargetAsync(record, new NothingToCancel(), target, localLldbServerPath,
YetiConstants.LldbServerLinuxPath);
await SetRemoteExecutableBitAsync(target, remotePath, record);
}
public void StartPreGameAttachNoGrpcPipeLeak()
{
// Verify that things happen in this order:
// 1) Other processes are started (if any)
// 2) PipeCallInvoker is created (and the pipes long with it)
// 3) DebuggerGrpcServer is created
// 4) Client pipe handles are released
// 5) Other processes are started (if any)
// Otherwise, other processes might get a copy of the pipes and cause the VSI to
// freeze, see (internal).
GrpcState state = GrpcState.Initial;
mockGrpcCallInvokerFactory.When(x => x.Create()).Do(x =>
{
// 2)
Assert.That(state, Is.EqualTo(GrpcState.Initial));
state = GrpcState.PipeCallInvokerCreated;
});
mockGrpcCallInvoker.When(x => x.DisposeLocalCopyOfClientPipeHandles()).Do(x =>
{
// 4)
Assert.That(state, Is.EqualTo(GrpcState.DebuggerGrpcServerCreated));
state = GrpcState.ClientPipeHandlesReleased;
});
mockManagedProcessFactory.Create(Arg.Do <ProcessStartInfo>(x =>
{
if (Path.GetFileName(x.FileName) == YetiConstants.DebuggerGrpcServerExecutable)
{
// 3)
Assert.That(state, Is.EqualTo(GrpcState.PipeCallInvokerCreated));
state = GrpcState.DebuggerGrpcServerCreated;
}
else
{
// 1) or 5)
Assert.IsTrue(state == GrpcState.Initial ||
state == GrpcState.ClientPipeHandlesReleased);
}
}));
SshTarget sshTarget = new SshTarget(_targetString);
yetiDebugTransport.StartPreGame(LaunchOption.AttachToGame, true, true, sshTarget, out _,
out _);
Assert.That(state, Is.EqualTo(GrpcState.ClientPipeHandlesReleased));
}
public void TailKilledIfGracefulExitFailsAndGameExits()
{
IProcess tailProcess = Substitute.For <IProcess>();
SshTarget sshTarget = new SshTarget(_targetString);
optionPageGrid.CaptureGameOutput.Returns(true);
mockManagedProcessFactory.Create(Arg.Any <ProcessStartInfo>(), Arg.Any <int>())
.Returns(tailProcess);
yetiDebugTransport.StartPostGame(LaunchOption.AttachToGame, sshTarget, _remotePid);
tailProcess.WaitForExit(_tailExitTimeout).Returns(false);
yetiDebugTransport.Stop(ExitReason.ProcessExited);
tailProcess.Received().WaitForExit(_tailExitTimeout);
tailProcess.Received().Kill();
}
ProcessStartInfo BuildForGgpSync(SshTarget target, string localPath, string remotePath,
bool force)
{
string tunnelSetup = $"--port {target.Port} --ip {target.IpAddress} --compress";
string quotedLocalPath = ProcessUtil.QuoteArgument(localPath);
string quotedRemotePath = ProcessUtil.QuoteArgument(remotePath);
string copyWholeFiles = force ? "--whole-file --checksum" : "";
return(new ProcessStartInfo
{
FileName = Path.Combine(SDKUtil.GetSDKToolsPath(), "ggp_rsync.exe"),
Arguments =
$"{tunnelSetup} {copyWholeFiles} {quotedLocalPath} {quotedRemotePath}",
});
}
/// <summary>
/// Get IDebugSessionLauncher instance that will execute `func` on LaunchAsync call
/// (with specified targetIpAddress and targetIpPort (all other arguments ignored)).
/// </summary>
/// <param name="expectedTarget">Gamelet target used to check that LaunchAsync was called
/// with expected values for IpAddress and Port of the Gamelet.</param>
/// <param name="func">Function being called when LaunchAsync with specified arguments
/// is called from the code.</param>
IDebugSessionLauncher CreateConfiguredDebugSessionLauncher(
SshTarget expectedTarget, Func <CallInfo, Task <ILldbAttachedProgram> > func = null)
{
var debugSessionLauncher = Substitute.For <IDebugSessionLauncher>();
debugSessionLauncher.LaunchAsync(Arg.Any <ICancelableTask>(), Arg.Any <IDebugProcess2>(),
Arg.Any <Guid>(), Arg.Any <uint?>(),
Arg.Any <YetiVSI.DebuggerOptions.DebuggerOptions>(),
Arg.Any <HashSet <string> >(), Arg.Any <GrpcConnection>(),
Arg.Any <int>(), Arg.Is(expectedTarget?.IpAddress),
Arg.Is(expectedTarget?.Port ?? 0),
Arg.Any <IDebugEventCallback2>()).Returns(func);
return(debugSessionLauncher);
}
