private static void WithReadableStream(string path, Func <AsyncFileStream, Task> action) { #if NET_FRAMEWORK using (var io = new IOCompletionManager()) #else IIOCompletionManager io = null; #endif { using ( IAsyncFile file = AsyncFileFactory.CreateOrOpen( path, FileDesiredAccess.GenericRead, FileShare.Read | FileShare.Delete, FileMode.Open, FileFlagsAndAttributes.None, io)) { using (AsyncFileStream stream = file.CreateReadableStream()) { XAssert.IsTrue(stream.CanRead); XAssert.IsTrue(stream.CanSeek); XAssert.IsFalse(stream.CanWrite); action(stream).GetAwaiter().GetResult(); } } } }
public ProcessTreeContext(Guid payloadGuid, SafeHandle reportPipe, ArraySegment <byte> payload, string dllNameX64, string dllNameX86, LoggingContext loggingContext) { // We cannot create this object in a wow64 process Contract.Assume( !ProcessUtilities.IsWow64Process(), "ProcessTreeContext:ctor - Cannot run injection server in a wow64 32 bit process"); SafeFileHandle childHandle = null; m_loggingContext = loggingContext; // This object will be the server for the tree. CreateSourceFile the pipe server. try { SafeFileHandle injectorHandle; // Create a pipe for the requests Pipes.CreateInheritablePipe(Pipes.PipeInheritance.InheritWrite, Pipes.PipeFlags.ReadSideAsync, out injectorHandle, out childHandle); // Create the injector. This will duplicate the handles. Injector = ProcessUtilities.CreateProcessInjector(payloadGuid, childHandle, reportPipe, dllNameX86, dllNameX64, payload); // Create the request reader. We don't start listening until requested var injectionRequestFile = AsyncFileFactory.CreateAsyncFile( injectorHandle, FileDesiredAccess.GenericRead, ownsHandle: true, kind: FileKind.Pipe); m_injectionRequestReader = new AsyncPipeReader(injectionRequestFile, InjectCallback, Encoding.Unicode, BufferSize); } catch (Exception exception) { if (Injector != null) { Injector.Dispose(); Injector = null; } if (m_injectionRequestReader != null) { m_injectionRequestReader.Dispose(); m_injectionRequestReader = null; } throw new BuildXLException("Process Tree Context injector could not be created", exception); } finally { // Release memory. Since the child handle is duplicated, it can be released if (childHandle != null && !childHandle.IsInvalid) { childHandle.Dispose(); } } }
public async Task ReadEmptyFile() { string path = GetFullPath("file"); using (File.Create(path)) { } #if NET_FRAMEWORK using (var io = new IOCompletionManager()) #else IIOCompletionManager io = null; #endif { using ( IAsyncFile file = AsyncFileFactory.CreateOrOpen( path, FileDesiredAccess.GenericRead, FileShare.Read | FileShare.Delete, FileMode.Open, FileFlagsAndAttributes.None, io)) { XAssert.IsTrue(file.CanRead); XAssert.IsFalse(file.CanWrite); var buffer = new byte[10]; FileAsyncIOResult result = await file.ReadAsync(buffer, buffer.Length, 0); XAssert.AreEqual(FileAsyncIOStatus.Failed, result.Status); XAssert.IsTrue(result.ErrorIndicatesEndOfFile); result = await file.ReadAsync(buffer, buffer.Length, 16); XAssert.AreEqual(FileAsyncIOStatus.Failed, result.Status); XAssert.IsTrue(result.ErrorIndicatesEndOfFile); } } }
public void Start() { Contract.Assume(!m_processStarted); Encoding reportEncoding = Encoding.Unicode; SafeFileHandle childHandle = null; DetouredProcess detouredProcess = m_detouredProcess; using (m_reportReaderSemaphore.AcquireSemaphore()) { SafeFileHandle reportHandle; try { Pipes.CreateInheritablePipe( Pipes.PipeInheritance.InheritWrite, Pipes.PipeFlags.ReadSideAsync, readHandle: out reportHandle, writeHandle: out childHandle); var setup = new FileAccessSetup { ReportPath = "#" + childHandle.DangerousGetHandle().ToInt64(), DllNameX64 = s_binaryPaths.DllNameX64, DllNameX86 = s_binaryPaths.DllNameX86, }; bool debugFlagsMatch = true; ArraySegment <byte> manifestBytes = new ArraySegment <byte>(); if (m_fileAccessManifest != null) { manifestBytes = m_fileAccessManifest.GetPayloadBytes(setup, FileAccessManifestStream, m_timeoutMins, ref debugFlagsMatch); } if (!debugFlagsMatch) { throw new BuildXLException("Mismatching build type for BuildXL and DetoursServices.dll."); } m_standardInputTcs = TaskSourceSlim.Create <bool>(); detouredProcess.Start( s_payloadGuid, manifestBytes, childHandle, s_binaryPaths.DllNameX64, s_binaryPaths.DllNameX86); // At this point, we believe calling 'kill' will result in an eventual callback for job teardown. // This knowledge is significant for ensuring correct cleanup if we did vs. did not start a process; // if started, we expect teardown to happen eventually and clean everything up. m_processStarted = true; ProcessId = detouredProcess.GetProcessId(); m_processIdListener?.Invoke(ProcessId); } finally { // release memory m_fileAccessManifest = null; // Note that in the success path, childHandle should already be closed (by Start). if (childHandle != null && !childHandle.IsInvalid) { childHandle.Dispose(); } } var reportFile = AsyncFileFactory.CreateAsyncFile( reportHandle, FileDesiredAccess.GenericRead, ownsHandle: true, kind: FileKind.Pipe); StreamDataReceived reportLineReceivedCallback = m_reports == null ? (StreamDataReceived)null : ReportLineReceived; m_reportReader = new AsyncPipeReader(reportFile, reportLineReceivedCallback, reportEncoding, m_bufferSize); m_reportReader.BeginReadLine(); } // don't wait, we want feeding in of standard input to happen asynchronously Analysis.IgnoreResult(FeedStandardInputAsync(detouredProcess, m_standardInputReader, m_standardInputTcs)); }
public async Task ReadFileRandomAccess() { const int NumberOfReads = 16; const int NumberOfWordsPerRead = 64 * 1024; const int NumberOfBytesPerRead = NumberOfWordsPerRead * 4; const int NumberOfWords = NumberOfWordsPerRead * NumberOfReads; const int TotalSize = NumberOfWords * 4; string path = GetFullPath("file"); using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Delete)) { using (var writer = new BinaryWriter(fs, Encoding.UTF8, leaveOpen: true)) { for (int i = 0; i < NumberOfWords; i++) { writer.Write((int)i); } } } #if NET_FRAMEWORK using (var io = new IOCompletionManager()) #else IIOCompletionManager io = null; #endif { using ( IAsyncFile file = AsyncFileFactory.CreateOrOpen( path, FileDesiredAccess.GenericRead, FileShare.Read | FileShare.Delete, FileMode.Open, FileFlagsAndAttributes.None, io)) { XAssert.IsTrue(file.CanRead); XAssert.IsFalse(file.CanWrite); var readBuffer = new byte[TotalSize]; var readTasks = new Task[NumberOfReads]; for (int i = 0; i < readTasks.Length; i++) { int offset = NumberOfBytesPerRead * i; readTasks[i] = Task.Run( async() => { byte[] localBuffer = new byte[NumberOfBytesPerRead]; int readSoFar = 0; while (readSoFar < NumberOfBytesPerRead) { FileAsyncIOResult result = await file.ReadAsync(localBuffer, bytesToRead: NumberOfBytesPerRead - readSoFar, fileOffset: offset + readSoFar); XAssert.AreEqual(FileAsyncIOStatus.Succeeded, result.Status); XAssert.IsTrue(result.BytesTransferred > 0); XAssert.IsTrue(readSoFar + result.BytesTransferred <= NumberOfBytesPerRead); Buffer.BlockCopy(localBuffer, 0, readBuffer, offset + readSoFar, result.BytesTransferred); readSoFar += result.BytesTransferred; } Contract.Assert(readSoFar == NumberOfBytesPerRead); }); } for (int i = 0; i < readTasks.Length; i++) { await readTasks[i]; } using (var reader = new BinaryReader(new MemoryStream(readBuffer, writable: false), Encoding.UTF8, leaveOpen: false)) { for (int i = 0; i < NumberOfWords; i++) { XAssert.AreEqual(i, reader.ReadInt32()); } } } } }
public void Start( Guid payloadGuid, ArraySegment <byte> payloadData, SafeFileHandle inheritableReportHandle, string dllNameX64, string dllNameX86) { using (m_syncSemaphore.AcquireSemaphore()) { if (m_starting || m_disposed) { throw new InvalidOperationException("Cannot invoke start process more than once or after this instance has been Disposed."); } m_starting = true; // The process creation flags // We use CREATE_DEFAULT_ERROR_MODE to ensure that the hard error mode of the child process (i.e., GetErrorMode) // is deterministic. Inheriting error mode is the default, but there may be some concurrent operation that temporarily // changes it (process global). The CLR has been observed to do so. // We use CREATE_NO_WINDOW in case BuildXL is attached to a console windows to prevent child processes from messing up // the console window. If BuildXL itself is started without a console window the flag is not set to prevent creating // extra conhost.exe processes. int creationFlags = ((s_consoleWindow == IntPtr.Zero && !this.m_disableConHostSharing) ? 0 : Native.Processes.ProcessUtilities.CREATE_NO_WINDOW) | Native.Processes.ProcessUtilities.CREATE_DEFAULT_ERROR_MODE; SafeFileHandle standardInputWritePipeHandle = null; SafeFileHandle standardOutputReadPipeHandle = null; SafeFileHandle standardErrorReadPipeHandle = null; try { // set up the environment block parameter var environmentHandle = default(GCHandle); var payloadHandle = default(GCHandle); SafeFileHandle hStdInput = null; SafeFileHandle hStdOutput = null; SafeFileHandle hStdError = null; SafeThreadHandle threadHandle = null; try { IntPtr environmentPtr = IntPtr.Zero; if (m_unicodeEnvironmentBlock != null) { creationFlags |= Native.Processes.ProcessUtilities.CREATE_UNICODE_ENVIRONMENT; environmentHandle = GCHandle.Alloc(m_unicodeEnvironmentBlock, GCHandleType.Pinned); environmentPtr = environmentHandle.AddrOfPinnedObject(); } Pipes.CreateInheritablePipe( Pipes.PipeInheritance.InheritRead, Pipes.PipeFlags.WriteSideAsync, readHandle: out hStdInput, writeHandle: out standardInputWritePipeHandle); Pipes.CreateInheritablePipe( Pipes.PipeInheritance.InheritWrite, Pipes.PipeFlags.ReadSideAsync, readHandle: out standardOutputReadPipeHandle, writeHandle: out hStdOutput); Pipes.CreateInheritablePipe( Pipes.PipeInheritance.InheritWrite, Pipes.PipeFlags.ReadSideAsync, readHandle: out standardErrorReadPipeHandle, writeHandle: out hStdError); // We want a per-process job primarily. If nested job support is not available, then we make sure to not have a BuildXL-level job. if (JobObject.OSSupportsNestedJobs) { JobObject.SetTerminateOnCloseOnCurrentProcessJob(); } // Initialize the injector m_processInjector = new ProcessTreeContext(payloadGuid, inheritableReportHandle, payloadData, dllNameX64, dllNameX86, m_loggingContext); // If path remapping is enabled then we wrap the job object in a container, so the filter drivers get // configured (and they get cleaned up when the container is disposed) if (m_containerConfiguration.IsIsolationEnabled) { m_job = new Container( name: null, containerConfiguration: m_containerConfiguration, loggingContext: m_loggingContext); } else { m_job = new JobObject(null); } // We want the effects of SEM_NOGPFAULTERRORBOX on all children (but can't set that with CreateProcess). // That's not set otherwise (even if set in this process) due to CREATE_DEFAULT_ERROR_MODE above. m_job.SetLimitInformation(terminateOnClose: true, failCriticalErrors: false); m_processInjector.Listen(); if (m_containerConfiguration.IsIsolationEnabled) { // After calling SetLimitInformation, start up the container if present // This will throw if the container is not set up properly m_job.StartContainerIfPresent(); } // The call to the CreateDetouredProcess below will add a newly created process to the job. System.Diagnostics.Stopwatch m_startUpTimeWatch = System.Diagnostics.Stopwatch.StartNew(); var detouredProcessCreationStatus = Native.Processes.ProcessUtilities.CreateDetouredProcess( m_commandLine, creationFlags, environmentPtr, m_workingDirectory, hStdInput, hStdOutput, hStdError, m_job, m_processInjector.Injector, m_containerConfiguration.IsIsolationEnabled, out m_processHandle, out threadHandle, out m_processId, out int errorCode); m_startUpTimeWatch.Stop(); m_startUpTime = m_startUpTimeWatch.ElapsedMilliseconds; if (detouredProcessCreationStatus != CreateDetouredProcessStatus.Succeeded) { // TODO: Indicating user vs. internal errors (and particular phase failures e.g. adding to job object or injecting detours) // is good progress on the transparency into these failures. But consider making this indication visible beyond this // function without throwing exceptions; consider returning a structured value or logging events. string message; if (detouredProcessCreationStatus.IsDetoursSpecific()) { message = string.Format( CultureInfo.InvariantCulture, "Internal error during process creation: {0:G}", detouredProcessCreationStatus); } else if (detouredProcessCreationStatus == CreateDetouredProcessStatus.ProcessCreationFailed) { message = "Process creation failed"; } else { message = string.Format( CultureInfo.InvariantCulture, "Process creation failed: {0:G}", detouredProcessCreationStatus); } throw new BuildXLException( message, new NativeWin32Exception(errorCode)); } // TODO: We should establish good post-conditions for CreateDetouredProcess. As a temporary measure, it would be nice // to determine if we are sometimes getting invalid process handles with retVal == true. So for now we differentiate // that possible case with a unique error string. if (m_processHandle.IsInvalid) { throw new BuildXLException("Unable to start or detour a process (process handle invalid)", new NativeWin32Exception(errorCode)); } } finally { if (environmentHandle.IsAllocated) { environmentHandle.Free(); } if (payloadHandle.IsAllocated) { payloadHandle.Free(); } if (hStdInput != null && !hStdInput.IsInvalid) { hStdInput.Dispose(); } if (hStdOutput != null && !hStdOutput.IsInvalid) { hStdOutput.Dispose(); } if (hStdError != null && !hStdError.IsInvalid) { hStdError.Dispose(); } if (inheritableReportHandle != null && !inheritableReportHandle.IsInvalid) { inheritableReportHandle.Dispose(); } if (threadHandle != null && !threadHandle.IsInvalid) { threadHandle.Dispose(); } } var standardInputStream = new FileStream(standardInputWritePipeHandle, FileAccess.Write, m_bufferSize, isAsync: true); m_standardInputWriter = new StreamWriter(standardInputStream, m_standardInputEncoding, m_bufferSize) { AutoFlush = true }; var standardOutputFile = AsyncFileFactory.CreateAsyncFile( standardOutputReadPipeHandle, FileDesiredAccess.GenericRead, ownsHandle: true, kind: FileKind.Pipe); m_outputReader = new AsyncPipeReader(standardOutputFile, m_outputDataReceived, m_standardOutputEncoding, m_bufferSize); m_outputReader.BeginReadLine(); var standardErrorFile = AsyncFileFactory.CreateAsyncFile( standardErrorReadPipeHandle, FileDesiredAccess.GenericRead, ownsHandle: true, kind: FileKind.Pipe); m_errorReader = new AsyncPipeReader(standardErrorFile, m_errorDataReceived, m_standardErrorEncoding, m_bufferSize); m_errorReader.BeginReadLine(); Contract.Assert(!m_processHandle.IsInvalid); m_processWaitHandle = new SafeWaitHandleFromSafeHandle(m_processHandle); m_waiting = true; TimeSpan timeout = m_timeout ?? Timeout.InfiniteTimeSpan; m_registeredWaitHandle = ThreadPool.RegisterWaitForSingleObject( m_processWaitHandle, CompletionCallback, null, timeout, true); m_started = true; } catch (Exception) { // Dispose pipe handles in case they are not assigned to streams. if (m_standardInputWriter == null) { standardInputWritePipeHandle?.Dispose(); } if (m_outputReader == null) { standardOutputReadPipeHandle?.Dispose(); } if (m_errorReader == null) { standardErrorReadPipeHandle?.Dispose(); } throw; } } }
public void Start() { Contract.Assume(!m_processStarted); Encoding reportEncoding = Encoding.Unicode; SafeFileHandle childHandle = null; DetouredProcess detouredProcess = m_detouredProcess; bool useNonDefaultPipeReader = PipeReaderFactory.GetKind() != PipeReaderFactory.Kind.Default; using (m_reportReaderSemaphore.AcquireSemaphore()) { NamedPipeServerStream pipeStream = null; SafeFileHandle reportHandle = null; try { if (useNonDefaultPipeReader) { pipeStream = Pipes.CreateNamedPipeServerStream( PipeDirection.In, PipeOptions.Asynchronous, PipeOptions.None, out childHandle); } else { Pipes.CreateInheritablePipe( Pipes.PipeInheritance.InheritWrite, Pipes.PipeFlags.ReadSideAsync, readHandle: out reportHandle, writeHandle: out childHandle); } var setup = new FileAccessSetup { ReportPath = "#" + childHandle.DangerousGetHandle().ToInt64(), DllNameX64 = s_binaryPaths.DllNameX64, DllNameX86 = s_binaryPaths.DllNameX86, }; bool debugFlagsMatch = true; ArraySegment <byte> manifestBytes = new ArraySegment <byte>(); if (m_fileAccessManifest != null) { manifestBytes = m_fileAccessManifest.GetPayloadBytes(m_loggingContext, setup, FileAccessManifestStream, m_timeoutMins, ref debugFlagsMatch); } if (!debugFlagsMatch) { throw new BuildXLException("Mismatching build type for BuildXL and DetoursServices.dll."); } m_standardInputTcs = TaskSourceSlim.Create <bool>(); detouredProcess.Start( s_payloadGuid, manifestBytes, childHandle, s_binaryPaths.DllNameX64, s_binaryPaths.DllNameX86); // At this point, we believe calling 'kill' will result in an eventual callback for job teardown. // This knowledge is significant for ensuring correct cleanup if we did vs. did not start a process; // if started, we expect teardown to happen eventually and clean everything up. m_processStarted = true; ProcessId = detouredProcess.GetProcessId(); } catch (AccessViolationException) { int ramPercent = 0, availableRamMb = 0, availablePageFileMb = 0, totalPageFileMb = 0; MEMORYSTATUSEX memoryStatusEx = new MEMORYSTATUSEX(); if (GlobalMemoryStatusEx(memoryStatusEx)) { ramPercent = (int)memoryStatusEx.dwMemoryLoad; availableRamMb = new FileSize(memoryStatusEx.ullAvailPhys).MB; availablePageFileMb = new FileSize(memoryStatusEx.ullAvailPageFile).MB; totalPageFileMb = new FileSize(memoryStatusEx.ullTotalPageFile).MB; } string memUsage = $"RamPercent: {ramPercent}, AvailableRamMb: {availableRamMb}, AvailablePageFileMb: {availablePageFileMb}, TotalPageFileMb: {totalPageFileMb}"; Native.Tracing.Logger.Log.DetouredProcessAccessViolationException(m_loggingContext, (m_reports?.PipDescription ?? "") + " - " + memUsage); throw; } finally { // release memory m_fileAccessManifest = null; // Note that in the success path, childHandle should already be closed (by Start). if (childHandle != null && !childHandle.IsInvalid) { childHandle.Dispose(); } } StreamDataReceived reportLineReceivedCallback = m_reports == null ? null : ReportLineReceived; if (useNonDefaultPipeReader) { m_reportReader = PipeReaderFactory.CreateNonDefaultPipeReader( pipeStream, message => reportLineReceivedCallback(message), reportEncoding, m_bufferSize); } else { var reportFile = AsyncFileFactory.CreateAsyncFile( reportHandle, FileDesiredAccess.GenericRead, ownsHandle: true, kind: FileKind.Pipe); m_reportReader = new AsyncPipeReader( reportFile, reportLineReceivedCallback, reportEncoding, m_bufferSize, numOfRetriesOnCancel: m_numRetriesPipeReadOnCancel, debugPipeReporter: new AsyncPipeReader.DebugReporter(errorMsg => DebugPipeConnection($"ReportReader: {errorMsg}"))); } m_reportReader.BeginReadLine(); } // don't wait, we want feeding in of standard input to happen asynchronously Analysis.IgnoreResult(FeedStandardInputAsync(detouredProcess, m_standardInputReader, m_standardInputTcs)); }
public ProcessTreeContext( Guid payloadGuid, SafeHandle reportPipe, ArraySegment <byte> payload, string dllNameX64, string dllNameX86, int numRetriesPipeReadOnCancel, Action <string> debugPipeReporter, LoggingContext loggingContext) { // We cannot create this object in a wow64 process Contract.Assume( !ProcessUtilities.IsWow64Process(), "ProcessTreeContext:ctor - Cannot run injection server in a wow64 32 bit process"); SafeFileHandle childHandle = null; m_loggingContext = loggingContext; NamedPipeServerStream serverStream = null; bool useNonDefaultPipeReader = PipeReaderFactory.GetKind() != PipeReaderFactory.Kind.Default; // This object will be the server for the tree. CreateSourceFile the pipe server. try { SafeFileHandle injectorHandle = null; if (useNonDefaultPipeReader) { serverStream = Pipes.CreateNamedPipeServerStream( PipeDirection.In, PipeOptions.Asynchronous, PipeOptions.None, out childHandle); } else { // Create a pipe for the requests Pipes.CreateInheritablePipe(Pipes.PipeInheritance.InheritWrite, Pipes.PipeFlags.ReadSideAsync, out injectorHandle, out childHandle); } // Create the injector. This will duplicate the handles. Injector = ProcessUtilities.CreateProcessInjector(payloadGuid, childHandle, reportPipe, dllNameX86, dllNameX64, payload); if (useNonDefaultPipeReader) { m_injectionRequestReader = PipeReaderFactory.CreateNonDefaultPipeReader( serverStream, InjectCallback, Encoding.Unicode, BufferSize); } else { // Create the request reader. We don't start listening until requested var injectionRequestFile = AsyncFileFactory.CreateAsyncFile( injectorHandle, FileDesiredAccess.GenericRead, ownsHandle: true, kind: FileKind.Pipe); m_injectionRequestReader = new AsyncPipeReader( injectionRequestFile, InjectCallback, Encoding.Unicode, BufferSize, numOfRetriesOnCancel: numRetriesPipeReadOnCancel, debugPipeReporter: new AsyncPipeReader.DebugReporter(debugMsg => debugPipeReporter?.Invoke($"InjectionRequestReader: {debugMsg}"))); } } catch (Exception exception) { if (Injector != null) { Injector.Dispose(); Injector = null; } if (m_injectionRequestReader != null) { m_injectionRequestReader.Dispose(); m_injectionRequestReader = null; } throw new BuildXLException("Process Tree Context injector could not be created", exception); } finally { // Release memory. Since the child handle is duplicated, it can be released if (childHandle != null && !childHandle.IsInvalid) { childHandle.Dispose(); } } }