public static ReportedFileAccess Create( ReportedFileOperation operation, ReportedProcess process, RequestedAccess requestedAccess, FileAccessStatus status, bool explicitlyReported, uint error, Usn usn, DesiredAccess desiredAccess, ShareMode shareMode, CreationDisposition creationDisposition, FlagsAndAttributes flagsAndAttributes, AbsolutePath path, string enumeratePattern = null) { Contract.Requires(process != null); return(new ReportedFileAccess( operation, process, requestedAccess, status, explicitlyReported, error, usn, desiredAccess, shareMode, creationDisposition, flagsAndAttributes, path, null, enumeratePattern)); }
private Dictionary <ReportedProcess, int> CreateAndSerializeProcessMap(BuildXLWriter writer) { var processMap = new Dictionary <ReportedProcess, int>(); PopulateProcesses(Processes); PopulateProcesses(SurvivingChildProcesses); PopulateProcesses(FileAccesses?.Select(f => f.Process)); PopulateProcesses(ExplicitlyReportedFileAccesses?.Select(f => f.Process)); PopulateProcesses(AllUnexpectedFileAccesses?.Select(f => f.Process)); ReportedProcess[] processes = new ReportedProcess[processMap.Count]; foreach (var process in processMap) { processes[process.Value] = process.Key; } writer.WriteReadOnlyList(processes, (w, v) => v.Serialize(w)); return(processMap); void PopulateProcesses(IEnumerable <ReportedProcess> processesToPopulate) { if (processesToPopulate != null) { foreach (var process in processesToPopulate) { if (!processMap.ContainsKey(process)) { processMap.Add(process, processMap.Count); } } } } }
public ReportedFileAccess( ReportedFileOperation operation, ReportedProcess process, RequestedAccess requestedAccess, FileAccessStatus status, bool explicitlyReported, uint error, Usn usn, DesiredAccess desiredAccess, ShareMode shareMode, CreationDisposition creationDisposition, FlagsAndAttributes flagsAndAttributes, AbsolutePath manifestPath, string path, string enumeratePatttern) { Contract.Requires(process != null); Operation = operation; Process = process; RequestedAccess = requestedAccess; Status = status; ExplicitlyReported = explicitlyReported; Error = error; Usn = usn; DesiredAccess = desiredAccess; ShareMode = shareMode; CreationDisposition = creationDisposition; FlagsAndAttributes = flagsAndAttributes; ManifestPath = manifestPath; Path = path; EnumeratePattern = enumeratePatttern; }
public static ReportedFileAccess Create( ReportedFileOperation operation, ReportedProcess process, RequestedAccess requestedAccess, FileAccessStatus status, bool explicitlyReported, uint error, Usn usn, DesiredAccess desiredAccess, ShareMode shareMode, CreationDisposition creationDisposition, FlagsAndAttributes flagsAndAttributes, FlagsAndAttributes openedFileOrDirectoryAttribute, PathTable pathTable, string path, string enumeratePattern = null) { Contract.Requires(process != null); Contract.Requires(pathTable != null); Contract.Requires(path != null); AbsolutePath absolutePath; if (AbsolutePath.TryGet(pathTable, (StringSegment)path, out absolutePath)) { return(new ReportedFileAccess( operation, process, requestedAccess, status, explicitlyReported, error, usn, desiredAccess, shareMode, creationDisposition, flagsAndAttributes, openedFileOrDirectoryAttribute, absolutePath, null, enumeratePattern)); } return(new ReportedFileAccess( operation, process, requestedAccess, status, explicitlyReported, error, usn, desiredAccess, shareMode, creationDisposition, flagsAndAttributes, openedFileOrDirectoryAttribute, AbsolutePath.Invalid, path, enumeratePattern)); }
/// <nodoc /> public static ReportedProcess Deserialize(BuildXLReader reader) { var reportedProcess = new ReportedProcess( processId: reader.ReadUInt32(), path: reader.ReadString(), args: reader.ReadString()) { CreationTime = reader.ReadDateTime(), ExitTime = reader.ReadDateTime(), KernelTime = reader.ReadTimeSpan(), UserTime = reader.ReadTimeSpan(), IOCounters = IOCounters.Deserialize(reader), ExitCode = reader.ReadUInt32(), ParentProcessId = reader.ReadUInt32() }; return(reportedProcess); }
private bool FileAccessReportLineReceived <T>(ref T data, FileAccessReportProvider <T> parser, bool isAnAugmentedFileAccess, out string errorMessage) { Contract.Assume(!IsFrozen, "FileAccessReportLineReceived: !IsFrozen"); if (!parser( ref data, out var processId, out var operation, out var requestedAccess, out var status, out var explicitlyReported, out var error, out var usn, out var desiredAccess, out var shareMode, out var creationDisposition, out var flagsAndAttributes, out var manifestPath, out var path, out var enumeratePattern, out var processArgs, out errorMessage)) { return(false); } // Special case seen with vstest.console.exe if (string.IsNullOrEmpty(path)) { return(true); } // If there is a listener registered and notifications allowed, notify over the interface. if (m_detoursEventListener != null && (m_detoursEventListener.GetMessageHandlingFlags() & MessageHandlingFlags.FileAccessNotify) != 0) { m_detoursEventListener.HandleFileAccess( PipSemiStableHash, PipDescription, operation, requestedAccess, status, explicitlyReported, processId, error, desiredAccess, shareMode, creationDisposition, flagsAndAttributes, path, processArgs); } // If there is a listener registered that disables the collection of data in the collections, just exit. if (m_detoursEventListener != null && (m_detoursEventListener.GetMessageHandlingFlags() & MessageHandlingFlags.FileAccessCollect) == 0) { return(true); } if (m_manifest.DirectoryTranslator != null) { path = m_manifest.DirectoryTranslator.Translate(path); } // If we are getting a message for ChangedReadWriteToReadAccess operation, // just log it as a warning and return if (operation == ReportedFileOperation.ChangedReadWriteToReadAccess) { Tracing.Logger.Log.ReadWriteFileAccessConvertedToReadMessage(m_loggingContext, PipSemiStableHash, PipDescription, processId, path); HasReadWriteToReadFileAccessRequest = true; return(true); } // A process id is only unique during the lifetime of the process (so there may be duplicates reported), // but the ID is at least consistent with other tracing tools including procmon. // For the purposes of event correlation, m_activeProcesses keeps track which process id maps to which process a the current time. // We also record all processes in a list. Because process create and exit messages can arrive out of order on macOS when multiple queues // are used, we have to keep track of the reported exits using the m_processesExits dictionary. ReportedProcess process; if (operation == ReportedFileOperation.Process) { process = new ReportedProcess(processId, path, processArgs); m_activeProcesses[processId] = process; Processes.Add(process); } else { m_activeProcesses.TryGetValue(processId, out process); if (operation == ReportedFileOperation.ProcessExit) { AddLookupEntryForProcessExit(processId, process); if (process != null) { m_activeProcesses.TryRemove(processId, out _); path = process.Path; } else { // no process to remove; return(true); } } } // This assertion doesn't have to hold when using /sandboxKind:macOsKext because some messages may come out of order Contract.Assert(OperatingSystemHelper.IsUnixOS || process != null, "Should see a process creation before its accesses (malformed report)"); // If no active ReportedProcess is found (e.g., because it already completed but we are still processing its access reports), // try to see the latest exiting process with the same process id. Otherwise, it's ok to just create an unnamed one since ReportedProcess is used for descriptive purposes only if (process == null && (!m_processesExits.TryGetValue(processId, out process) || process == null)) { process = new ReportedProcess(processId, string.Empty, string.Empty); } // For exact matches (i.e., not a scope rule), the manifest path is the same as the full path. // In that case we don't want to keep carrying around the giant string. if (AbsolutePath.TryGet(m_pathTable, path, out AbsolutePath finalPath) && finalPath == manifestPath) { path = null; } var pathAsAbsolutePath = finalPath; if (!pathAsAbsolutePath.IsValid) { AbsolutePath.TryCreate(m_pathTable, path, out pathAsAbsolutePath); } if (pathAsAbsolutePath.IsValid && m_sharedOpaqueOutputLogger != null && (requestedAccess & RequestedAccess.Write) != 0) { m_sharedOpaqueOutputLogger.RecordFileWrite(m_pathTable, pathAsAbsolutePath); } Contract.Assume(manifestPath.IsValid || !string.IsNullOrEmpty(path)); if (path != null) { if (m_pathCache.TryGetValue(path, out var cachedPath)) { path = cachedPath; } else { m_pathCache[path] = path; } } if (operation == ReportedFileOperation.FirstAllowWriteCheckInProcess) { // This operation represents that a given path was checked for write access for the first time // within the scope of a process. The status of the operation represents whether that access should // have been allowed/denied, based on the existence of the file. // However, we need to determine whether to deny the access based on the first time the path was // checked for writes across the whole process tree. This means checking the first time this operation // is reported for a given path, and ignore subsequent reports. // Races are ignored: a race means two child processes are racing to create or delete the same file // - something that is not a good build behavior anyway - and the outcome will be that we will // non-deterministically deny the access // We store the path as an absolute path in order to guarantee canonicalization: e.g. prefixes like \\?\ // are not canonicalized in detours if (path != null && pathAsAbsolutePath.IsValid && !m_overrideAllowedWritePaths.ContainsKey(pathAsAbsolutePath)) { // We should override write allowed accesses for this path if the status of the special operation was 'denied' m_overrideAllowedWritePaths[pathAsAbsolutePath] = (status == FileAccessStatus.Denied); } return(true); } // If this is an augmented file access, the method was not based on policy, but a trusted tool reported the access FileAccessStatusMethod method = isAnAugmentedFileAccess? FileAccessStatusMethod.TrustedTool : FileAccessStatusMethod.PolicyBased; // If we are processing an allowed write, but this should be overridden based on file existence, // we change the status here // Observe that if the access is coming from a trusted tool, that trumps file existence and we don't deny the access if (method != FileAccessStatusMethod.TrustedTool && path != null && (requestedAccess & RequestedAccess.Write) != 0 && status == FileAccessStatus.Allowed && m_overrideAllowedWritePaths.Count > 0 && // Avoid creating the absolute path if the override allowed writes flag is off pathAsAbsolutePath.IsValid && m_overrideAllowedWritePaths.TryGetValue(pathAsAbsolutePath, out bool shouldOverrideAllowedAccess) && shouldOverrideAllowedAccess) { status = FileAccessStatus.Denied; method = FileAccessStatusMethod.FileExistenceBased; } var reportedAccess = new ReportedFileAccess( operation, process, requestedAccess, status, explicitlyReported, error, usn, desiredAccess, shareMode, creationDisposition, flagsAndAttributes, manifestPath, path, enumeratePattern, method); HandleReportedAccess(reportedAccess); return(true); }
private void AddLookupEntryForProcessExit(uint processId, ReportedProcess reportedProcess) { m_processesExits[processId] = reportedProcess; }
/// <nodoc /> public static ReportedFileAccess Deserialize( BuildXLReader reader, IReadOnlyList <ReportedProcess> processes, Func <BuildXLReader, AbsolutePath> readPath) { return(new ReportedFileAccess( operation: (ReportedFileOperation)reader.ReadByte(), process: processes != null ? processes[reader.ReadInt32Compact()] : ReportedProcess.Deserialize(reader), requestedAccess: (RequestedAccess)reader.ReadInt32Compact(), status: (FileAccessStatus)reader.ReadInt32Compact(), explicitlyReported: reader.ReadBoolean(), error: reader.ReadUInt32(), // In general if process is executed externally, e.g., in VM, the obtained USN cannot be translated to the host. // However, for our low-privilege build, we are going to map the host volumes to the VM, and thus the USN // can still be used. usn: new Usn(reader.ReadUInt64()), desiredAccess: (DesiredAccess)reader.ReadUInt32(), shareMode: (ShareMode)reader.ReadUInt32(), creationDisposition: (CreationDisposition)reader.ReadUInt32(), flagsAndAttributes: (FlagsAndAttributes)reader.ReadUInt32(), manifestPath: readPath != null ? readPath(reader) : reader.ReadAbsolutePath(), path: reader.ReadNullableString(), enumeratePatttern: reader.ReadNullableString(), fileAccessStatusMethod: (FileAccessStatusMethod)reader.ReadByte())); }
/// <summary> /// Deserializes an instance of <see cref="SandboxedProcessResult"/>. /// </summary> /// <param name="reader"></param> /// <returns></returns> public static SandboxedProcessResult Deserialize(BuildXLReader reader) { int exitCode = reader.ReadInt32(); bool killed = reader.ReadBoolean(); bool timedOut = reader.ReadBoolean(); bool hasDetoursInjectionFailures = reader.ReadBoolean(); IReadOnlyList <ReportedProcess> allReportedProcesses = reader.ReadReadOnlyList(r => ReportedProcess.Deserialize(r)); IReadOnlyList <ReportedProcess> survivingChildProcesses = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => allReportedProcesses[r2.ReadInt32()])); ProcessTimes primaryProcessTimes = reader.ReadNullable(r => ProcessTimes.Deserialize(r)); JobObject.AccountingInformation? jobAccountingInformation = reader.ReadNullableStruct(r => JobObject.AccountingInformation.Deserialize(r)); SandboxedProcessOutput standardOutput = reader.ReadNullable(r => SandboxedProcessOutput.Deserialize(r)); SandboxedProcessOutput standardError = reader.ReadNullable(r => SandboxedProcessOutput.Deserialize(r)); IReadOnlyList <ReportedFileAccess> fileAccesses = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => ReportedFileAccess.Deserialize(r2, allReportedProcesses, readPath: null))); IReadOnlyList <ReportedFileAccess> explicitlyReportedFileAccesses = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => ReportedFileAccess.Deserialize(r2, allReportedProcesses, readPath: null))); IReadOnlyList <ReportedFileAccess> allUnexpectedFileAccesses = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => ReportedFileAccess.Deserialize(r2, allReportedProcesses, readPath: null))); IReadOnlyList <ReportedProcess> processes = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => allReportedProcesses[r2.ReadInt32()])); IReadOnlyList <ProcessDetouringStatusData> detouringStatuses = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => ProcessDetouringStatusData.Deserialize(r2))); string dumpFileDirectory = reader.ReadNullableString(); string dumpCreationExceptionMessage = reader.ReadNullableString(); string standardInputExceptionMessage = reader.ReadNullableString(); int numberOfPRocessLaunchRetries = reader.ReadInt32(); bool hasReadWriteToReadFileAccessRequest = reader.ReadBoolean(); string messageProcessingFailureMessage = reader.ReadNullableString(); long processStartTime = reader.ReadInt64(); int warningCount = reader.ReadInt32(); long detoursMaxHeapSize = reader.ReadInt64(); int lastMessageCount = reader.ReadInt32(); bool messageCountSemaphoreCreated = reader.ReadBoolean(); return(new SandboxedProcessResult() { ExitCode = exitCode, Killed = killed, TimedOut = timedOut, HasDetoursInjectionFailures = hasDetoursInjectionFailures, SurvivingChildProcesses = survivingChildProcesses, PrimaryProcessTimes = primaryProcessTimes, JobAccountingInformation = jobAccountingInformation, StandardOutput = standardOutput, StandardError = standardError, FileAccesses = fileAccesses != null ? new HashSet <ReportedFileAccess>(fileAccesses) : null, ExplicitlyReportedFileAccesses = explicitlyReportedFileAccesses != null ? new HashSet <ReportedFileAccess>(explicitlyReportedFileAccesses) : null, AllUnexpectedFileAccesses = allUnexpectedFileAccesses != null ? new HashSet <ReportedFileAccess>(allUnexpectedFileAccesses) : null, Processes = processes, DetouringStatuses = detouringStatuses, DumpFileDirectory = dumpFileDirectory, DumpCreationException = dumpCreationExceptionMessage != null ? new Exception(dumpCreationExceptionMessage) : null, StandardInputException = standardInputExceptionMessage != null ? new Exception(standardInputExceptionMessage) : null, NumberOfProcessLaunchRetries = numberOfPRocessLaunchRetries, HasReadWriteToReadFileAccessRequest = hasReadWriteToReadFileAccessRequest, MessageProcessingFailure = messageProcessingFailureMessage != null ? new Failure <string>(messageProcessingFailureMessage) : null, ProcessStartTime = processStartTime, WarningCount = warningCount, DetoursMaxHeapSize = detoursMaxHeapSize, LastMessageCount = lastMessageCount, MessageCountSemaphoreCreated = messageCountSemaphoreCreated }); }
private bool FileAccessReportLineReceived(string data, out string errorMessage) { Contract.Assume(!IsFrozen, "FileAccessReportLineReceived: !IsFrozen"); if (!FileAccessReportLine.TryParse( data, out var processId, out var operation, out var requestedAccess, out var status, out var explicitlyReported, out var error, out var usn, out var desiredAccess, out var shareMode, out var creationDisposition, out var flagsAndAttributes, out var manifestPath, out var path, out var enumeratePattern, out var processArgs, out errorMessage)) { return false; } // If there is a listener registered and notifications allowed, notify over the interface. if (m_detoursEventListener != null && (m_detoursEventListener.GetMessageHandlingFlags() & MessageHandlingFlags.FileAccessNotify) != 0) { m_detoursEventListener.HandleFileAccess( PipSemiStableHash, PipDescription, operation, requestedAccess, status, explicitlyReported, processId, error, desiredAccess, shareMode, creationDisposition, flagsAndAttributes, path == null ? manifestPath.ToString(m_pathTable) : path, processArgs); } // If there is a listener registered that disables the collection of data in the collections, just exit. if (m_detoursEventListener != null && (m_detoursEventListener.GetMessageHandlingFlags() & MessageHandlingFlags.FileAccessCollect) == 0) { return true; } // Special case seen with vstest.console.exe if (path.Length == 0) { return true; } if (m_manifest.DirectoryTranslator != null) { path = m_manifest.DirectoryTranslator.Translate(path); } // If we are getting a message for ChangedReadWriteToReadAccess operation, // just log it as a warning and return if (operation == ReportedFileOperation.ChangedReadWriteToReadAccess) { Tracing.Logger.Log.ReadWriteFileAccessConvertedToReadMessage(m_loggingContext, PipSemiStableHash, PipDescription, processId, path); HasReadWriteToReadFileAccessRequest = true; return true; } // A process id is only unique during the lifetime of the process (so there may be duplicates reported), // but the ID is at least consistent with other tracing tools including procmon. // For the purposes of event correlation, m_activeProcesses keeps track which process id maps to which process a the current time. // We also record all processes in a list. Because process create and exit messages can arrive out of order on macOS when multiple queues // are used, we have to keep track of the reported exits using the m_processesExits dictionary. ReportedProcess process; if (operation == ReportedFileOperation.Process) { process = new ReportedProcess(processId, path, processArgs); m_activeProcesses[processId] = process; Processes.Add(process); } else { m_activeProcesses.TryGetValue(processId, out process); if (operation == ReportedFileOperation.ProcessExit) { AddLookupEntryForProcessExit(processId); if (process != null) { m_activeProcesses.Remove(processId); path = process.Path; } else { // no process to remove; return true; } } } // This assertion doesn't have to hold when using /sandboxKind:macOsKext because some messages may come out of order Contract.Assert(OperatingSystemHelper.IsUnixOS || process != null, "Should see a process creation before its accesses (malformed report)"); // If no active ReportedProcess is found (e.g., because it already completed but we are still processing its access reports), // it's ok to just create a new one since ReportedProcess is used for descriptive purposes only if (process == null) { process = new ReportedProcess(processId, string.Empty, string.Empty); } // For exact matches (i.e., not a scope rule), the manifest path is the same as the full path. // In that case we don't want to keep carrying around the giant string. if (AbsolutePath.TryGet(m_pathTable, path, out AbsolutePath finalPath) && finalPath == manifestPath) { path = null; } Contract.Assume(manifestPath.IsValid || !string.IsNullOrEmpty(path)); if (path != null) { if (m_pathCache.TryGetValue(path, out var cachedPath)) { path = cachedPath; } else { m_pathCache[path] = path; } } var reportedAccess = new ReportedFileAccess( operation, process, requestedAccess, status, explicitlyReported, error, usn, desiredAccess, shareMode, creationDisposition, flagsAndAttributes, manifestPath, path, enumeratePattern); HandleReportedAccess(reportedAccess); return true; }